diff --git a/.claude/skills/article-v2/SKILL.md b/.claude/skills/article-v2/SKILL.md new file mode 100644 index 0000000..58636ee --- /dev/null +++ b/.claude/skills/article-v2/SKILL.md @@ -0,0 +1,237 @@ +--- +name: article-v2 +description: Generate a data-backed article for visionanalysis.org with the v2 article engine (article-pipeline/v2). Supports five article types (vs, hardware-guide, runtime-guide, license-guide, finding). Use when the user asks to write, generate, or regenerate an article, a model comparison, a hardware buying guide, a runtime conversion guide, a license guide, or a findings post. Supersedes the model-comparison-article skill. +--- + +# The v2 article engine + +You are writing for engineers who deploy detection models. The engine's core +rule: **the site renders the data; you write only the interpretation.** Your +prose is under 20% of the page. Every number you type must come from a claims +file that a script computed. If you follow the workflow, nothing you produce +can be wrong or slop. + +Never use an em dash anywhere (prose, titles, deks, code comments). Use a +colon, comma, or period. + +## Commands + +Python: use `python3` on POSIX shells. On this Windows box python is not on +PATH; use `C:\Users\Usuario\AppData\Local\Programs\Python\Python312\python.exe`. +Run everything from the repo root (`vision-analysis/`). + +``` +python3 article-pipeline/v2/claims.py list # coverage: what can be written +python3 article-pipeline/v2/claims.py plan [--limit N] [--json] # ranked, de-duplicated worklist of every viable article +python3 article-pipeline/v2/claims.py family-vs [--hardware H] [--runtime R] +python3 article-pipeline/v2/claims.py vs [--hardware H] [--runtime R] +python3 article-pipeline/v2/claims.py hardware-guide [--runtime R] +python3 article-pipeline/v2/claims.py runtime-guide +python3 article-pipeline/v2/claims.py license-guide [--hardware H] [--runtime R] +python3 article-pipeline/v2/claims.py finding scan [--hardware H] +python3 article-pipeline/v2/validate.py website/src/content/articles/.json +``` + +Defaults: hardware `nvidia_geforce_rtx_5070_ti`, runtime `pytorch_fp32`. +Hardware ids with data: `nvidia_geforce_rtx_5070_ti`, `jetson_orin`, `rpi5`, +`a100`, `rpi5_hailo8`. Runtime ids: `pytorch_fp32`, `onnx_fp32`, +`tensorrt_fp32`, `tensorrt_fp16`, `ncnn_fp32`, `hailo_int8`. +Note: non-permissive-license models (YOLO-NAS) only have data on +`jetson_orin`, `rpi5`, `a100`, `rpi5_hailo8`, so license-guide needs +`--hardware jetson_orin` (or another of those). + +## Workflow (follow exactly) + +1. **Check coverage.** `claims.py list`. If the requested combination lacks + data, say so and stop. Never substitute a different combination silently. + +2. **Compile claims.** Run the claims command for the article type. It writes + `article-pipeline/v2/output/.claims.json` and prints a gate verdict. + - **GATE FAILED (exit 2): do not write the article. Ever.** Report the gate + reason to the user and stop. A failed gate means the data has no story + (models are different size classes, too few significant deltas, etc.). + - GATE PASSED: read the claims file. It contains `facts` (all usable + numbers), `claims` (pre-phrased candidate statements, each with + `passed: true/false`), and `allowed_numbers`. + +3. **Write the content JSON** at `website/src/content/articles/.json`. + The `slug` must equal the claims file's slug (finding articles may choose + their own descriptive slug). Schema and block reference below. + - Use ONLY claims with `passed: true`. Rephrase them; do not copy the + statement text verbatim (it is written for you, not for readers). + - Every number in your prose must appear in `allowed_numbers` (the + validator enforces this; rounding to fewer decimals is allowed). + - Claims with `strength: "strong"` are the story. Lead with them. + - Claims with kind `caveat` are MANDATORY: surface them in the verdict or + Setup prose (e.g. mismatched input sizes). + +4. **Validate.** `validate.py website/src/content/articles/.json`. + Fix every reported error and rerun until it prints OK. Do not publish a + file that fails validation. Do not weaken the validator. + +5. **Build check.** From `website/`: `npm run build` (node lives at + `C:\Program Files\nodejs` on this box if not on PATH). The article appears + at `/articles/` and enters the sitemap automatically; there is no + registration step. Fix any build error you introduced. + +6. **Report.** Tell the user the slug, the gate summary (n claims passed), + validation result, and whether the build passed. Commit only if asked. + +## Content JSON schema + +```json +{ + "slug": "dfine-s-vs-yolov9s", + "type": "vs", + "title": "D-FINE-S vs YOLOv9-S: measured on the same protocol", + "dek": "One to two sentences with a real number. This is the meta description: 50-165 chars.", + "date": "2026-07-03", + "author": "Vision Analysis", + "readingMinutes": 5, + "tags": ["comparison", "dfine", "yolov9"], + "status": "published", + "claimsFile": "article-pipeline/v2/output/dfine-s-vs-yolov9s.claims.json", + "blocks": [ ... ] +} +``` + +- `date`: today's date, ISO. +- `title`: 15-70 chars, must contain the search phrase for the type (see SEO + patterns below). No hardware in vs titles. +- `status`: `"published"` unless the user asked for a draft. +- `readingMinutes`: total prose words / 200, rounded up, min 3. + +## Block reference + +Text blocks (LLM-written, validated): + +| kind | purpose | +|---|---| +| `{"kind": "verdict", "text": ...}` | 2-4 sentences answering the reader's question with numbers. First block of every article. | +| `{"kind": "prose", "text": ...}` | One paragraph. Minimal markdown: `[text](href)`, `` `code` ``, `**bold**`. No headings inside. | +| `{"kind": "heading", "text": ...}` | h2 section heading. | +| `{"kind": "callout", "text": ...}` | Short caveat or honesty note. | + +Data blocks (rendered live from the verified dataset; they self-update when +new benchmark runs land, so prefer them over prose for anything numeric): + +| kind | renders | +|---|---| +| `{"kind": "comparison-table", "models": ["a","b"], "hardware": H, "runtime": R}` | side-by-side metric table with winner highlighting, license row included | +| `{"kind": "ranking-table", "hardware": H, "runtime": R, "metric": "mAP_50_95"\|"throughputFps", "limit": 10, "licenseFilter": "permissive"?, "maxParamsM": 20?}` | leaderboard slice | +| `{"kind": "speedup-table", "hardware": H, "baseline": RT, "target": RT, "limit": 15?}` | per-model conversion speedups + mAP delta | +| `{"kind": "family-frontier", "familyA": famId, "familyB": famId, "hardware": H, "runtime": R}` | both families' variant ladders interleaved by params (the family-vs signature block) | +| `{"kind": "chart", "src": "/embed/scatter?highlight=a,b", "caption": ...}` | live scatter iframe | +| `{"kind": "kv", "title": ..., "items": [{"k": ..., "v": ..., "href"?: ...}]}` | license/provenance key-value block | +| `{"kind": "code", "language": "python", "text": ...}` | LibreYOLO snippet | +| `{"kind": "methodology"}` | fixed protocol paragraph with links. REQUIRED in every article. | + +All blocks take an optional `caption`. + +## Required structure per type + +Every article: `verdict` first, at least one data block, one `methodology` +block, prose budget max 900 words total (target 350-600). + +**family-vs** (slug `-vs-`, family ids alphabetical) is the HEADLINE +type: it compares two architecture families across their whole size range, which +is the higher-traffic head-term query ("YOLOv9 vs RT-DETR", "D-FINE vs DEIM"). +Prefer it over vs. Structure: +verdict (the matched-compute accuracy edge AND the speed edge in one breath, plus +the crossover if `family_crossover` passed: name which family wins at the small +end vs the large end), lede prose (how many variants each family fields, both on +one protocol), `family-frontier` block (familyA/familyB/hardware/runtime from +claims meta: this is the signature visual, both ladders interleaved by params), +chart using `facts.chart_highlight` as the highlight list, heading "Accuracy at +matched compute" + prose (use `aggregate.mean_mAP_delta_pts`, `a_wins`/`b_wins`, +flagship and efficient-end claims; the comparison is nearest-params, say so), +heading "Speed" + prose (`aggregate.mean_fps_delta_pct`), heading "Where the +frontier crosses" + prose ONLY if `family_crossover` passed, kv "Licensing" +(per-family permissiveness from `facts.license`; surface `family_license_edge` +if it passed), heading "Which family to pick" + prose (small-model use, large- +model use, license-driven, runtime-driven), methodology. No per-model code block +(too many variants); instead one sentence pointing to the family's model pages. +Never claim a single overall winner when the crossover claim passed. + +**vs** (slug `-vs-`, ids alphabetical) is now the LONG-TAIL supporting +type, for a specific high-intent variant pair only. Structure: +verdict, lede prose, comparison-table, chart (`/embed/scatter?highlight=a,b`), +heading "Accuracy" + prose, heading "Speed" + prose, cross-runtime or +cross-hardware observation if the claims contain `ranking_flip` or +`ranking_stable`, kv "License and provenance" (license, release date, repo, +LibreYOLO weights from `facts`), heading "When to pick which" + prose (one +sentence per model), methodology, optional code block. LibreYOLO class names: +read `libreyolo/docs/nomenclature.md` conventions; weights are +`Libre.pt` (e.g. `LibreYOLO9s.pt`); only include a code block +for models with `inLibreYOLO: true` in facts. + +**hardware-guide** (slug `best-object-detection-`): +verdict (name the winner + the budget picks), lede, ranking-table by +mAP_50_95 (limit 15), heading + prose on the frontier, ranking-table by +throughputFps (limit 10), budget-pick prose from `budget_picks` claims, +permissive-license prose + ranking-table with `licenseFilter` if the +`best_permissive` claim passed, methodology. + +**runtime-guide** (slug from claims, e.g. `tensorrt-fp16-vs-pytorch-fp32-jetson-orin`): +verdict (median speedup + accuracy cost), lede, speedup-table (limit 15), +prose on family spread, prose on accuracy losers (or the no-loss claim), +prose on regressions if any (this is the interesting part), methodology. + +**license-guide** (slug `permissive-license-detection-models-`): +verdict, lede on why license matters at deploy time (no legal advice; say +"not legal advice" in a callout), one prose + comparison-table per matchup +from the `alt_*` claims, ranking-table with `licenseFilter: "permissive"`, +methodology. + +**finding** (slug: descriptive, your choice, e.g. +`tensorrt-fp16-makes-these-models-slower`): +run `finding scan`, pick ONE coherent group of findings (same kind), write a +short post: verdict, what we observed (prose + the relevant data block), +which models/hardware, what we did NOT determine (no root-cause speculation), +methodology. Findings posts are the only type where curiosity is the angle; +still no speculation about why architectures behave as they do. + +## SEO title patterns + +- family-vs: `"{FamA} vs {FamB}: which detector family wins"` or `"{FamA} vs {FamB}: full benchmark comparison"` +- vs: `"{A} vs {B}: measured on the same protocol"` or `"{A} vs {B}: benchmark"` +- hardware-guide: `"Best object detection models for {hardware} ({year})"` +- runtime-guide: `"{Target} vs {Baseline} on {hardware}: N models measured"` +- license-guide: `"Commercially safe object detection models for {hardware}"` +- finding: state the finding plainly, numbers allowed. + +Deks must contain at least one concrete number from the claims file. + +## Voice (non-negotiable) + +- Short sentences. One idea per sentence. Active voice. +- No superlatives, no hedging, no "really/quite/very". +- No em dashes, no en dashes. +- No architectural speculation: never explain WHY a model is fast or accurate. + Backbone/neck names stay out of prose entirely. +- Never mention paper-reported mAP or compare measured numbers to any paper. +- No "Conclusion" or "Bottom line" sections: the verdict block owns that. +- No AI-slop phrasing: "dive into", "landscape", "it's worth noting", + "in the realm of", "seamless", "cutting-edge". +- The reader knows what mAP is. State once that mAP is shown in percent form + and move on (the methodology block covers protocol). + +## Batch generation + +Use `claims.py plan` to get a ranked, de-duplicated worklist. It scores every +viable article by demand (popular families) plus differentiation (cross-family, +ranking flips, license edges, small-object gaps) and penalizes the Nth article +with the same family+size shape, so near-duplicates sink. Take the top N from +the plan, not an arbitrary enumeration. + +Do NOT try to write all ~680 viable articles. The gate proves each has a real +delta, but it cannot see search demand or judge whether a page is differentiated +from its neighbors. A pile of near-identical vs pages triggers site-wide +Helpful-Content demotion (the pSEO failure mode in SEO-ROADMAP.md). Generate in +waves: a differentiated core first, measure in Search Console, expand where real +queries land. Guides (hardware, runtime, license) are each structurally unique, +so prefer them; vs pages need demand behind the pair. + +Process one article at a time through the full workflow. If any gate fails, skip +it and list it in the final report with the reason. Never lower a threshold in +claims.py to force a gate to pass. diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml new file mode 100644 index 0000000..2035cf7 --- /dev/null +++ b/.github/workflows/website.yml @@ -0,0 +1,60 @@ +name: Website + +on: + pull_request: + paths: + - "website/**" + - "article-pipeline/v2/**" + - "generated/verified-results.v1.json" + - "submissions/**" + - "scripts/**" + - ".github/workflows/website.yml" + push: + branches: + - main + paths: + - "website/**" + - "article-pipeline/v2/**" + - "generated/verified-results.v1.json" + - "submissions/**" + - "scripts/**" + - ".github/workflows/website.yml" + +jobs: + validate: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "24" + cache: npm + cache-dependency-path: website/package-lock.json + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install website dependencies + run: npm ci --prefix website + + - name: Check benchmark coordinate compatibility + run: npm test + + - name: Validate article content + shell: bash + run: | + for article in website/src/content/articles/*.json; do + python3 article-pipeline/v2/validate.py "$article" + done + + - name: Lint website + run: npm run lint --prefix website + + - name: Build website + run: npm run build diff --git a/.gitignore b/.gitignore index 55af558..edeae41 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ __pycache__/ article-pipeline/output/* !article-pipeline/output/.gitkeep .vercel + +# build/SDK logs and agent worktrees +*.log +.claude/worktrees/ diff --git a/article-pipeline/v2/README.md b/article-pipeline/v2/README.md new file mode 100644 index 0000000..0510b75 --- /dev/null +++ b/article-pipeline/v2/README.md @@ -0,0 +1,72 @@ +# Article engine v2 + +Generates data-backed articles for visionanalysis.org without the slop +failure mode of v1. The inversion vs v1: **the site renders the data, the LLM +writes only the interpretation**, and an article only exists when the data +clears a significance gate. + +## Why v1 was retired + +v1 (`article-pipeline/fact-sheet` + the `model-comparison-article` skill) had +the LLM write a full 16-section HTML article from a fact sheet. Every article +was the same skeleton with paraphrased connective prose (slop), it was keyed +to arbitrary model pairs (most of which have no story), the output HTML was +orphaned (no route served it), and its defaults went stale (`rtx5080`). + +## Architecture + +``` +claims.py -> output/.claims.json -> LLM writes content JSON -> validate.py +(mechanistic) facts + gated claims + website/src/content/ rejects unsourced + allowed_numbers articles/.json numbers + style +``` + +- **claims.py** reads `generated/verified-results.v1.json` + metadata and + emits, per article type, every number the article may cite plus pre-phrased + candidate claims, each marked passed/failed against significance thresholds. + A gate (min passed claims) decides whether the article should exist at all. + Exit 2 = do not write. +- **The writer** (Opus subagent via `.claude/skills/article-v2/SKILL.md`) + produces a content JSON: metadata + ordered blocks. Text blocks are the only + prose. Data blocks (`comparison-table`, `ranking-table`, `speedup-table`, + `chart`) are declarative dataset slices. +- **validate.py** enforces: schema, gate passed, every prose number present in + `allowed_numbers`, style rules (no em dashes, no superlatives, no paper-mAP + mentions, no AI-slop phrases), prose budget (max 900 words, data blocks must + dominate), mandatory caveats surfaced. +- **The site** (`website/src/components/articles/ArticleRenderer.tsx`) renders + data blocks live from the verified dataset at build time, so every table in + every published article updates when new benchmark runs land. Dropping a + validated JSON into `website/src/content/articles/` publishes it: the + `/articles` index, the article route, and `sitemap.xml` all pick it up + automatically (`website/src/lib/articles.ts` reads the folder). + +## Article types + +| type | claims command | story source | +|---|---|---| +| vs | `claims.py vs a b` | pair deltas, ranking flips, license edge | +| hardware-guide | `claims.py hardware-guide hw` | leaderboard, pareto frontier, latency-budget picks | +| runtime-guide | `claims.py runtime-guide hw base target` | speedup distribution, accuracy cost, regressions | +| license-guide | `claims.py license-guide --hardware hw` | permissive alternatives to non-permissive models | +| finding | `claims.py finding scan` | anomalies: jitter, FP16 cliffs, cross-hardware flips | + +`claims.py list` prints coverage (which hardware/runtime combos have enough +models). + +## Anti-slop invariants + +1. No claim passes the gate, no article. Pairs in different size classes + (params ratio > 2.5x) can never pass. +2. Prose is capped and validated; numbers must trace to the claims file. +3. Everything numeric on the page is rendered from the dataset, not written. +4. Freshness is free: rebuild = every article's tables update; sitemap + `lastModified` comes from the article date. + +## Publishing checklist (automated by the skill) + +1. `claims.py ...` gate passes +2. write `website/src/content/articles/.json` +3. `validate.py` passes +4. `cd website && npm run build` passes +5. deploy from repo root: `npx vercel --prod` diff --git a/article-pipeline/v2/claims.py b/article-pipeline/v2/claims.py new file mode 100644 index 0000000..41715fc --- /dev/null +++ b/article-pipeline/v2/claims.py @@ -0,0 +1,1226 @@ +#!/usr/bin/env python3 +"""Claims compiler for the v2 article engine. + +Reads the canonical verified dataset (generated/verified-results.v1.json) plus +metadata, computes every number an article of a given type is allowed to cite, +and emits pre-phrased CANDIDATE CLAIMS with significance gates. The writer +(an LLM subagent) picks and phrases from passed claims only; it never invents +a number. The validator (validate.py) enforces this. + +Article types: + vs [--hardware H] [--runtime R] + hardware-guide [--runtime R] + runtime-guide + license-guide [--hardware H] [--runtime R] + finding scan [--hardware H] + list (coverage report: what can be written) + +Output: JSON written to article-pipeline/v2/output/.claims.json and a +one-line gate summary on stderr. Exit code 0 if the gate passed, 2 if not +(DO NOT write an article when the gate fails). +""" +import argparse +import json +import sys +from collections import defaultdict +from datetime import datetime, timezone +from functools import lru_cache +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +GENERATED = ROOT / "generated" / "verified-results.v1.json" +METADATA = ROOT / "website" / "src" / "data" / "metadata" +OUTPUT = Path(__file__).resolve().parent / "output" + +DEFAULT_HARDWARE = "nvidia_geforce_rtx_5070_ti" +DEFAULT_RUNTIME = "pytorch_fp32" + +PERMISSIVE = {"Apache-2.0", "MIT", "BSD-3-Clause"} + +# Significance thresholds (relative %, unless noted) +TIE_PCT = 2.0 # below this a metric axis is a tie +SPEED_PCT = 5.0 # fps delta worth claiming +MEMORY_PCT = 10.0 # VRAM delta worth claiming +EFFICIENCY_PCT = 15.0 # mAP-per-GFLOP delta worth claiming +SMALL_OBJ_PCT = 10.0 # mAP_small relative delta worth claiming +ACC_PTS = 0.5 # mAP@50-95 percentage-point delta worth claiming +PAIR_PARAMS_RATIO = 2.5 # above this a vs pair is not meaningful +JITTER_PCT = 12.0 # fps_mean vs fps_p50 gap that counts as jitter +TRT_ACC_DROP_PTS = 0.5 # accuracy loss under conversion worth flagging + + +# ---------------------------------------------------------------- loading + +def _load(p): + return json.loads(p.read_text(encoding="utf-8")) + + +def load_dataset(): + data = _load(GENERATED) + models_meta = {m["id"]: m for m in _load(METADATA / "models.json")["models"]} + hardware_meta = {h["id"]: h for h in _load(METADATA / "hardware.json")["hardware"]} + runtimes_meta = {r["id"]: r for r in _load(METADATA / "runtimes.json")["runtimes"]} + + index = {} # (model_id, hardware_id, runtime_id) -> latest bench + for bench in data.get("results", []): + model_id = (bench.get("model") or {}).get("id") + hw_id = (bench.get("hardware") or {}).get("id") + runtime = bench.get("runtime") or {} + rt_id = f"{runtime.get('format')}_{runtime.get('precision')}" + if not model_id or not hw_id: + continue + key = (model_id, hw_id, rt_id) + created = bench.get("created_at") or "" + prev = index.get(key) + if prev is None or created > (prev.get("created_at") or ""): + index[key] = bench + return index, models_meta, hardware_meta, runtimes_meta + + +# ---------------------------------------------------------------- helpers + +def pct(a, b): + if a is None or b is None or b == 0: + return None + return round((a - b) / b * 100, 2) + + +def get_map(bench): + return bench["accuracy"]["mAP_50_95"] + + +def get_fps(bench): + return bench["throughput"]["fps_mean"] + + +def timing(bench, key): + t = bench.get("timing") or {} + total = t.get("total_ms") + if isinstance(total, dict): + if key == "ms_per_image": + return total.get("mean") + return total.get(key) + return t.get(key) + + +def params_m(bench, meta): + v = (bench.get("model_stats") or {}).get("params_millions") + if v: + return v + m = meta.get(bench["model"]["id"]) if isinstance(meta, dict) else None + return (m or {}).get("specs", {}).get("paramsM") + + +def flops_g(bench, meta): + v = (bench.get("model_stats") or {}).get("gflops") + if v: + return v + m = meta.get(bench["model"]["id"]) if isinstance(meta, dict) else None + return (m or {}).get("specs", {}).get("flopsG") + + +def rt_display(rt_id, runtimes_meta): + r = runtimes_meta.get(rt_id) + return r["displayName"] if r else rt_id + + +def hw_display(hw_id, hardware_meta): + h = hardware_meta.get(hw_id) + return h["displayName"] if h else hw_id + + +def model_display(model_id, models_meta): + m = models_meta.get(model_id) + return m["displayName"] if m else model_id + + +def is_permissive(model_id, models_meta): + return (models_meta.get(model_id) or {}).get("license") in PERMISSIVE + + +def map_pts(x): + """decimal mAP -> percent points, 1 decimal""" + return round(x * 100, 1) + + +def core_slice(bench): + return { + "mAP_50_95": bench["accuracy"]["mAP_50_95"], + "mAP_50": bench["accuracy"]["mAP_50"], + "mAP_small": bench["accuracy"].get("mAP_small"), + "mAP_medium": bench["accuracy"].get("mAP_medium"), + "mAP_large": bench["accuracy"].get("mAP_large"), + "fps_mean": bench["throughput"]["fps_mean"], + "fps_p50": bench["throughput"].get("fps_p50"), + "ms_per_image": timing(bench, "ms_per_image"), + "inference_ms": timing(bench, "inference_ms"), + "preprocess_ms": timing(bench, "preprocess_ms"), + "postprocess_ms": timing(bench, "postprocess_ms"), + "p95_ms": timing(bench, "p95"), + "peak_vram_mb": (bench.get("memory") or {}).get("peak_vram_mb"), + "peak_ram_mb": (bench.get("memory") or {}).get("peak_ram_mb"), + "params_millions": (bench.get("model_stats") or {}).get("params_millions"), + "gflops": (bench.get("model_stats") or {}).get("gflops"), + "input_size": bench["model"].get("input_size"), + "config": bench.get("config"), + "eval": bench.get("eval"), + "benchmark_date": (bench.get("metadata") or {}).get("benchmark_date"), + } + + +def claim(cid, kind, statement, values, passed, strength="normal"): + return { + "id": cid, + "kind": kind, + "statement": statement, + "values": values, + "passed": bool(passed), + "strength": strength, + } + + +def collect_numbers(obj, out): + """Recursively collect every numeric leaf; for decimal mAP-like values also + add the x100 percent form so prose can quote either form.""" + if isinstance(obj, bool): + return + if isinstance(obj, (int, float)): + if obj is not None: + out.add(round(float(obj), 4)) + if 0 < obj <= 1.0: + out.add(round(float(obj) * 100, 4)) + return + if isinstance(obj, dict): + for v in obj.values(): + collect_numbers(v, out) + elif isinstance(obj, list): + for v in obj: + collect_numbers(v, out) + + +ALWAYS_ALLOWED = { + 0.001, 0.6, 300, 640, 500, 5000, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 50, 75, 95, 100, +} + + +def finalize(meta_block, facts, claims, gate_min, slug, extra_allowed=None): + passed = [c for c in claims if c["passed"]] + gate = { + "passed": len(passed) >= gate_min, + "passed_claims": len(passed), + "required": gate_min, + "reason": None if len(passed) >= gate_min else + f"only {len(passed)} claims cleared significance thresholds; " + f"{gate_min} required. Do not write this article.", + } + allowed = set() + collect_numbers(facts, allowed) + for c in claims: + collect_numbers(c["values"], allowed) + allowed |= ALWAYS_ALLOWED + if extra_allowed: + allowed |= set(extra_allowed) + doc = { + "engine": "article-pipeline-v2", + "generated_at": datetime.now(timezone.utc).isoformat(), + "slug": slug, + "meta": meta_block, + "facts": facts, + "claims": claims, + "gate": gate, + "allowed_numbers": sorted(allowed), + } + return doc + + +# ---------------------------------------------------------------- vs + +def build_vs(index, models_meta, hardware_meta, runtimes_meta, a_id, b_id, hw, rt): + a_id, b_id = sorted([a_id, b_id]) # canonical order, matches /vs slug rule + a = index.get((a_id, hw, rt)) + b = index.get((b_id, hw, rt)) + if a is None or b is None: + missing = [m for m, x in [(a_id, a), (b_id, b)] if x is None] + sys.exit(f"ERROR: no verified result for {missing} on {hw}/{rt}. " + f"Run 'claims.py list' to see coverage.") + + a_meta, b_meta = models_meta.get(a_id, {}), models_meta.get(b_id, {}) + a_name, b_name = model_display(a_id, models_meta), model_display(b_id, models_meta) + hw_name = hw_display(hw, hardware_meta) + rt_name = rt_display(rt, runtimes_meta) + + a_map, b_map = get_map(a), get_map(b) + a_fps, b_fps = get_fps(a), get_fps(b) + a_params, b_params = params_m(a, models_meta), params_m(b, models_meta) + a_flops, b_flops = flops_g(a, models_meta), flops_g(b, models_meta) + + facts = { + "a": {"id": a_id, "displayName": a_name, + "license": a_meta.get("license"), "releaseDate": a_meta.get("releaseDate"), + "inLibreYOLO": a_meta.get("inLibreYOLO"), "bench": core_slice(a)}, + "b": {"id": b_id, "displayName": b_name, + "license": b_meta.get("license"), "releaseDate": b_meta.get("releaseDate"), + "inLibreYOLO": b_meta.get("inLibreYOLO"), "bench": core_slice(b)}, + "primary": {"hardware": hw, "hardware_display": hw_name, + "runtime": rt, "runtime_display": rt_name}, + "deltas_pct": { + "mAP_50_95": pct(a_map, b_map), + "fps": pct(a_fps, b_fps), + "params": pct(a_params, b_params), + "flops": pct(a_flops, b_flops), + "peak_vram": pct(core_slice(a)["peak_vram_mb"], core_slice(b)["peak_vram_mb"]), + }, + "map_pts": {"a": map_pts(a_map), "b": map_pts(b_map), + "delta": round(map_pts(a_map) - map_pts(b_map), 1)}, + } + + claims = [] + + # pair meaningfulness + ratio = max(a_params, b_params) / min(a_params, b_params) if a_params and b_params else None + pair_ok = ratio is not None and ratio <= PAIR_PARAMS_RATIO + facts["pair"] = {"params_ratio": round(ratio, 2) if ratio else None, "meaningful": pair_ok} + + # accuracy + d_pts = facts["map_pts"]["delta"] + if abs(d_pts) >= ACC_PTS: + w, l = (a_name, b_name) if d_pts > 0 else (b_name, a_name) + claims.append(claim( + "accuracy_winner", "accuracy", + f"{w} measures {abs(d_pts)} mAP points higher than {l} on {hw_name} " + f"({rt_name}): {facts['map_pts']['a']} vs {facts['map_pts']['b']} mAP@50-95.", + {"delta_pts": abs(d_pts), "a_map": facts["map_pts"]["a"], "b_map": facts["map_pts"]["b"]}, + True, "strong")) + else: + claims.append(claim( + "accuracy_tie", "accuracy", + f"Accuracy is a tie: {facts['map_pts']['a']} vs {facts['map_pts']['b']} " + f"mAP@50-95 ({abs(d_pts)} points apart, below the {ACC_PTS}-point threshold).", + {"a_map": facts["map_pts"]["a"], "b_map": facts["map_pts"]["b"]}, + True)) + + # speed + d_fps = facts["deltas_pct"]["fps"] + if d_fps is not None and abs(d_fps) >= SPEED_PCT: + w = a_name if d_fps > 0 else b_name + claims.append(claim( + "speed_winner", "speed", + f"{w} is {abs(d_fps)}% faster: {round(a_fps, 1)} vs {round(b_fps, 1)} FPS " + f"({round(timing(a, 'ms_per_image'), 2)} vs {round(timing(b, 'ms_per_image'), 2)} ms/image).", + {"delta_pct": abs(d_fps), "a_fps": round(a_fps, 1), "b_fps": round(b_fps, 1)}, + True, "strong")) + + # efficiency + a_eff = round(a_map / a_flops, 5) if a_flops else None + b_eff = round(b_map / b_flops, 5) if b_flops else None + facts["efficiency"] = {"a_mAP_per_GFLOP": a_eff, "b_mAP_per_GFLOP": b_eff} + if a_eff and b_eff: + d_eff = pct(a_eff, b_eff) + if abs(d_eff) >= EFFICIENCY_PCT: + w = a_name if d_eff > 0 else b_name + claims.append(claim( + "efficiency_winner", "efficiency", + f"{w} extracts {abs(d_eff)}% more mAP per GFLOP " + f"({a_eff} vs {b_eff}).", + {"a": a_eff, "b": b_eff, "delta_pct": abs(d_eff)}, True)) + + # memory + d_vram = facts["deltas_pct"]["peak_vram"] + if d_vram is not None and abs(d_vram) >= MEMORY_PCT: + w = b_name if d_vram > 0 else a_name # lower is better + claims.append(claim( + "memory_winner", "memory", + f"{w} uses less peak VRAM: " + f"{round(core_slice(a)['peak_vram_mb'])} vs {round(core_slice(b)['peak_vram_mb'])} MB.", + {"a_vram": round(core_slice(a)["peak_vram_mb"]), + "b_vram": round(core_slice(b)["peak_vram_mb"])}, True)) + + # small objects + a_s, b_s = a["accuracy"].get("mAP_small"), b["accuracy"].get("mAP_small") + if a_s and b_s: + d_small = pct(a_s, b_s) + if abs(d_small) >= SMALL_OBJ_PCT: + w = a_name if d_small > 0 else b_name + claims.append(claim( + "small_object_gap", "accuracy", + f"{w} leads on small objects: {map_pts(a_s)} vs {map_pts(b_s)} mAP_small " + f"({abs(d_small)}% relative gap).", + {"a": map_pts(a_s), "b": map_pts(b_s), "delta_pct": abs(d_small)}, True)) + + # license edge + a_perm, b_perm = is_permissive(a_id, models_meta), is_permissive(b_id, models_meta) + if a_perm != b_perm: + perm_name = a_name if a_perm else b_name + other = b_name if a_perm else a_name + perm_lic = a_meta.get("license") if a_perm else b_meta.get("license") + other_lic = b_meta.get("license") if a_perm else a_meta.get("license") + claims.append(claim( + "license_edge", "license", + f"{perm_name} ships under {perm_lic}; {other} is {other_lic}. For commercial " + f"embedding, {perm_name} is the safe default.", + {"permissive": perm_lic, "other": other_lic}, True, "strong")) + + # cross-runtime and cross-hardware: does the accuracy/speed verdict survive? + cross_runtime, cross_hardware = [], [] + primary_speed_winner = a_id if (d_fps or 0) > 0 else b_id + flips = [] + all_rts = sorted({k[2] for k in index if k[1] == hw}) + for rt2 in all_rts: + a2, b2 = index.get((a_id, hw, rt2)), index.get((b_id, hw, rt2)) + if a2 and b2: + cross_runtime.append({ + "runtime": rt2, "display": rt_display(rt2, runtimes_meta), + "a": {"mAP": map_pts(get_map(a2)), "fps": round(get_fps(a2), 1)}, + "b": {"mAP": map_pts(get_map(b2)), "fps": round(get_fps(b2), 1)}, + }) + w2 = a_id if get_fps(a2) > get_fps(b2) else b_id + if rt2 != rt and abs(pct(get_fps(a2), get_fps(b2)) or 0) >= SPEED_PCT and w2 != primary_speed_winner: + flips.append(("runtime", rt_display(rt2, runtimes_meta))) + all_hws = sorted({k[1] for k in index}) + for hw2 in all_hws: + a2, b2 = index.get((a_id, hw2, rt)), index.get((b_id, hw2, rt)) + if a2 and b2: + cross_hardware.append({ + "hardware": hw2, "display": hw_display(hw2, hardware_meta), + "a": {"mAP": map_pts(get_map(a2)), "fps": round(get_fps(a2), 1)}, + "b": {"mAP": map_pts(get_map(b2)), "fps": round(get_fps(b2), 1)}, + }) + w2 = a_id if get_fps(a2) > get_fps(b2) else b_id + if hw2 != hw and abs(pct(get_fps(a2), get_fps(b2)) or 0) >= SPEED_PCT and w2 != primary_speed_winner: + flips.append(("hardware", hw_display(hw2, hardware_meta))) + facts["cross_runtime"] = cross_runtime + facts["cross_hardware"] = cross_hardware + + if flips: + where = ", ".join(f"{k}: {v}" for k, v in flips) + claims.append(claim( + "ranking_flip", "robustness", + f"The speed verdict does NOT survive everywhere. It flips on {where}. " + f"State this; it is the most useful fact in the article.", + {"flips": [f"{k}:{v}" for k, v in flips]}, True, "strong")) + elif len(cross_hardware) + len(cross_runtime) > 2: + claims.append(claim( + "ranking_stable", "robustness", + f"The verdict holds across {len(cross_runtime)} runtimes and " + f"{len(cross_hardware)} hardware platforms with verified data.", + {"n_runtimes": len(cross_runtime), "n_hardware": len(cross_hardware)}, True)) + + # input size caveat (mandatory if mismatched) + a_in, b_in = a["model"].get("input_size"), b["model"].get("input_size") + facts["input_sizes"] = {"a": a_in, "b": b_in, "match": a_in == b_in} + if a_in != b_in: + claims.append(claim( + "input_size_caveat", "caveat", + f"MANDATORY CAVEAT: {a_name} runs at {a_in} px and {b_name} at {b_in} px " + f"(each model at its author default). Say this in Setup and the verdict.", + {"a": a_in, "b": b_in}, True, "strong")) + + if not pair_ok: + claims.append(claim( + "pair_not_meaningful", "caveat", + f"Params ratio is {facts['pair']['params_ratio']}x (> {PAIR_PARAMS_RATIO}x). " + f"These models are different size classes; the comparison is not meaningful.", + {"ratio": facts["pair"]["params_ratio"]}, True)) + + slug = f"{a_id}-vs-{b_id}" + meta_block = {"type": "vs", "a": a_id, "b": b_id, "hardware": hw, "runtime": rt} + gate_min = 2 if pair_ok else 999 # unmeaningful pairs never pass + return finalize(meta_block, facts, claims, gate_min, slug) + + +# ---------------------------------------------------------------- family-vs + +@lru_cache(maxsize=1) +def _families_meta(): + data = _load(METADATA / "families.json")["families"] + return {f["id"]: f for f in data} + + +def _family_display(fam_id): + f = _families_meta().get(fam_id) + return f["displayName"] if f else fam_id + + +def _family_all_permissive(fam_id, models_meta): + variants = [m for m in models_meta.values() if m.get("family") == fam_id] + lics = [m.get("license") for m in variants] + return bool(lics) and all(l in PERMISSIVE for l in lics) + + +def _family_frontier(fam_id, hw, rt, index, models_meta): + """Every variant of a family with data on hw/rt, sorted by params ascending.""" + rows = [] + for m in models_meta.values(): + if m.get("family") != fam_id: + continue + bench = index.get((m["id"], hw, rt)) + if not bench: + continue + p = params_m(bench, models_meta) + if not p: + continue + rows.append({ + "id": m["id"], "displayName": m["displayName"], "params": round(p, 2), + "mAP": map_pts(get_map(bench)), "fps": round(get_fps(bench), 1), + "license": m.get("license"), + }) + return sorted(rows, key=lambda r: r["params"]) + + +def build_family_vs(index, models_meta, hardware_meta, runtimes_meta, + fam_a, fam_b, hw=DEFAULT_HARDWARE, rt=DEFAULT_RUNTIME): + fam_a, fam_b = sorted([fam_a, fam_b]) # canonical slug order + a_front = _family_frontier(fam_a, hw, rt, index, models_meta) + b_front = _family_frontier(fam_b, hw, rt, index, models_meta) + if len(a_front) < 2 or len(b_front) < 2: + sys.exit(f"ERROR: need >=2 variants each with data on {hw}/{rt}; " + f"{fam_a}={len(a_front)} {fam_b}={len(b_front)}.") + + a_name, b_name = _family_display(fam_a), _family_display(fam_b) + hw_name, rt_name = hw_display(hw, hardware_meta), rt_display(rt, runtimes_meta) + + # match each A variant to nearest-params B variant (matched-compute view) + matched = [] + for a in a_front: + b = min(b_front, key=lambda r: abs(r["params"] - a["params"])) + ratio = max(a["params"], b["params"]) / min(a["params"], b["params"]) + if ratio > 1.6: # no fair compute match + continue + matched.append({ + "a": a, "b": b, + "mAP_delta_pts": round(a["mAP"] - b["mAP"], 1), + "fps_delta_pct": pct(a["fps"], b["fps"]), + }) + # dedup on (a_id,b_id) + seen, uniq = set(), [] + for mp in matched: + key = (mp["a"]["id"], mp["b"]["id"]) + if key not in seen: + seen.add(key) + uniq.append(mp) + matched = uniq + if len(matched) < 2: + sys.exit(f"ERROR: fewer than 2 matched-compute pairs between {fam_a} and {fam_b} " + f"on {hw}/{rt}. Frontiers do not overlap in params.") + + deltas = [mp["mAP_delta_pts"] for mp in matched] + mean_map_delta = round(sum(deltas) / len(deltas), 1) + a_wins = sum(1 for d in deltas if d > 0) + b_wins = sum(1 for d in deltas if d < 0) + fps_deltas = [mp["fps_delta_pct"] for mp in matched if mp["fps_delta_pct"] is not None] + mean_fps_delta = round(sum(fps_deltas) / len(fps_deltas), 1) if fps_deltas else None + + flagship = {"a": a_front[-1], "b": b_front[-1]} + smallest = {"a": a_front[0], "b": b_front[0]} + + # crossover: which family leads on accuracy in the low-param vs high-param half + low = [mp for mp in matched if mp["a"]["params"] <= (matched[0]["a"]["params"] + matched[-1]["a"]["params"]) / 2] + high = [mp for mp in matched if mp not in low] + def _lead(group): + if not group: + return None + s = sum(mp["mAP_delta_pts"] for mp in group) / len(group) + return fam_a if s > 0 else (fam_b if s < 0 else "tie") + low_lead, high_lead = _lead(low), _lead(high) + + a_perm = _family_all_permissive(fam_a, models_meta) + b_perm = _family_all_permissive(fam_b, models_meta) + + facts = { + "families": {"a": fam_a, "b": fam_b, "a_display": a_name, "b_display": b_name}, + "primary": {"hardware": hw, "hardware_display": hw_name, + "runtime": rt, "runtime_display": rt_name}, + "a_frontier": a_front, "b_frontier": b_front, + "matched_pairs": matched, + "aggregate": { + "n_pairs": len(matched), "mean_mAP_delta_pts": mean_map_delta, + "a_wins": a_wins, "b_wins": b_wins, "mean_fps_delta_pct": mean_fps_delta, + }, + "flagship": flagship, "smallest": smallest, + "crossover": {"low_param_leader": low_lead, "high_param_leader": high_lead}, + "license": {"a_all_permissive": a_perm, "b_all_permissive": b_perm}, + "chart_highlight": ",".join([r["id"] for r in a_front] + [r["id"] for r in b_front]), + } + + claims = [] + + # aggregate accuracy edge at matched compute + if abs(mean_map_delta) >= 0.5: + lead = a_name if mean_map_delta > 0 else b_name + claims.append(claim( + "family_accuracy_edge", "accuracy", + f"At matched compute, {lead} averages {abs(mean_map_delta)} mAP points higher across " + f"{len(matched)} paired variants on {hw_name}. {a_name} wins {a_wins}, {b_name} wins {b_wins}.", + {"mean_delta_pts": abs(mean_map_delta), "a_wins": a_wins, "b_wins": b_wins}, True, "strong")) + else: + claims.append(claim( + "family_accuracy_tie", "accuracy", + f"At matched compute the two families are even: mean gap {abs(mean_map_delta)} mAP points " + f"across {len(matched)} pairs ({a_name} wins {a_wins}, {b_name} wins {b_wins}).", + {"mean_delta_pts": abs(mean_map_delta)}, True)) + + # aggregate speed edge + if mean_fps_delta is not None and abs(mean_fps_delta) >= SPEED_PCT: + lead = a_name if mean_fps_delta > 0 else b_name + claims.append(claim( + "family_speed_edge", "speed", + f"At matched compute, {lead} is {abs(mean_fps_delta)}% faster on average on {hw_name} " + f"({rt_name}).", + {"mean_delta_pct": abs(mean_fps_delta)}, True, "strong")) + + # crossover + if low_lead and high_lead and low_lead != high_lead and "tie" not in (low_lead, high_lead): + claims.append(claim( + "family_crossover", "robustness", + f"The frontier crosses: {_family_display(low_lead)} leads at the small-model end, " + f"{_family_display(high_lead)} leads at the large-model end. There is no single winner; " + f"the choice depends on your size class.", + {"low": low_lead, "high": high_lead}, True, "strong")) + + # flagship end + fa, fb = flagship["a"], flagship["b"] + claims.append(claim( + "flagship", "accuracy", + f"At the top end, {a_name}'s largest measured variant ({fa['displayName']}) reaches " + f"{fa['mAP']} mAP; {b_name}'s ({fb['displayName']}) reaches {fb['mAP']} mAP.", + {"a": fa, "b": fb}, abs(fa["mAP"] - fb["mAP"]) >= 0.5)) + + # efficient end + sa, sb = smallest["a"], smallest["b"] + claims.append(claim( + "efficient_end", "efficiency", + f"At the small end, {a_name}'s {sa['displayName']} ({sa['params']}M, {sa['mAP']} mAP, {sa['fps']} FPS) " + f"faces {b_name}'s {sb['displayName']} ({sb['params']}M, {sb['mAP']} mAP, {sb['fps']} FPS).", + {"a": sa, "b": sb}, True)) + + # license edge at family level + if a_perm != b_perm: + perm_fam = a_name if a_perm else b_name + other_fam = b_name if a_perm else a_name + claims.append(claim( + "family_license_edge", "license", + f"Every measured {perm_fam} variant ships under a permissive license; {other_fam} does not. " + f"For commercial embedding, {perm_fam} is the safe default.", + {"permissive_family": perm_fam}, True, "strong")) + + # coverage + claims.append(claim( + "coverage", "ranking", + f"{a_name} fields {len(a_front)} measured variants, {b_name} fields {len(b_front)}, " + f"all on the same protocol on {hw_name}.", + {"a_n": len(a_front), "b_n": len(b_front)}, True)) + + slug = f"{fam_a}-vs-{fam_b}" + meta_block = {"type": "family-vs", "a": fam_a, "b": fam_b, "hardware": hw, "runtime": rt} + return finalize(meta_block, facts, claims, 2, slug) + + +# ---------------------------------------------------------------- hardware-guide + +def build_hardware_guide(index, models_meta, hardware_meta, runtimes_meta, hw, rt): + rows = [] + for (m, h, r), bench in index.items(): + if h == hw and r == rt: + rows.append((m, bench)) + if len(rows) < 8: + sys.exit(f"ERROR: only {len(rows)} models on {hw}/{rt}; need at least 8 for a guide.") + + hw_name = hw_display(hw, hardware_meta) + rt_name = rt_display(rt, runtimes_meta) + + entries = [] + for m, bench in rows: + entries.append({ + "model": m, "displayName": model_display(m, models_meta), + "family": (models_meta.get(m) or {}).get("family"), + "license": (models_meta.get(m) or {}).get("license"), + "mAP": map_pts(get_map(bench)), "fps": round(get_fps(bench), 1), + "ms": round(timing(bench, "ms_per_image") or 0, 2), + "params_millions": (bench.get("model_stats") or {}).get("params_millions"), + }) + by_map = sorted(entries, key=lambda e: -e["mAP"]) + by_fps = sorted(entries, key=lambda e: -e["fps"]) + + # pareto frontier on (fps, mAP) + pareto = [] + best_map = -1 + for e in by_fps: + if e["mAP"] > best_map: + pareto.append(e["model"]) + best_map = e["mAP"] + + # latency budget picks + budgets = [10, 33, 100] if "rpi5" not in hw else [50, 100, 500] + budget_picks = {} + for b in budgets: + ok = [e for e in entries if e["ms"] and e["ms"] <= b] + if ok: + budget_picks[str(b)] = max(ok, key=lambda e: e["mAP"]) + + permissive = [e for e in entries if e["license"] in PERMISSIVE] + best_perm = max(permissive, key=lambda e: e["mAP"]) if permissive else None + overall = by_map[0] + + facts = { + "hardware": hw, "hardware_display": hw_name, + "runtime": rt, "runtime_display": rt_name, + "n_models": len(entries), + "leaderboard": by_map, + "pareto_models": pareto, + "budget_picks": budget_picks, + "best_overall": overall, + "best_permissive": best_perm, + "fastest": by_fps[0], + } + + claims = [ + claim("best_overall", "ranking", + f"{overall['displayName']} leads on {hw_name} at {overall['mAP']} mAP@50-95 " + f"({overall['fps']} FPS, {rt_name}).", + overall, True, "strong"), + claim("fastest", "ranking", + f"{by_fps[0]['displayName']} is the fastest measured: {by_fps[0]['fps']} FPS " + f"at {by_fps[0]['mAP']} mAP.", + by_fps[0], True, "strong"), + claim("pareto", "ranking", + f"The measured accuracy-speed frontier on {hw_name} is: " + + ", ".join(model_display(m, models_meta) for m in pareto) + ".", + {"pareto": pareto}, len(pareto) >= 3), + ] + if best_perm and best_perm["model"] != overall["model"]: + gap = round(overall["mAP"] - best_perm["mAP"], 1) + claims.append(claim( + "best_permissive", "license", + f"Best permissive-license pick: {best_perm['displayName']} " + f"({best_perm['license']}), {gap} mAP points behind the overall leader.", + {"pick": best_perm, "gap_pts": gap}, True, "strong")) + for b, e in budget_picks.items(): + claims.append(claim( + f"budget_{b}ms", "budget", + f"Under {b} ms/image on {hw_name}: {e['displayName']} is the most accurate " + f"({e['mAP']} mAP at {e['ms']} ms).", + {"budget_ms": int(b), "pick": e}, True)) + + slug = f"best-object-detection-{hw.replace('_', '-')}" + meta_block = {"type": "hardware-guide", "hardware": hw, "runtime": rt} + return finalize(meta_block, facts, claims, 3, slug) + + +# ---------------------------------------------------------------- runtime-guide + +def build_runtime_guide(index, models_meta, hardware_meta, runtimes_meta, hw, base_rt, tgt_rt): + pairs = [] + for (m, h, r), bench in index.items(): + if h == hw and r == base_rt: + tgt = index.get((m, hw, tgt_rt)) + if tgt: + pairs.append((m, bench, tgt)) + if len(pairs) < 8: + sys.exit(f"ERROR: only {len(pairs)} models have both {base_rt} and {tgt_rt} on {hw}.") + + hw_name = hw_display(hw, hardware_meta) + base_name = rt_display(base_rt, runtimes_meta) + tgt_name = rt_display(tgt_rt, runtimes_meta) + + rows = [] + for m, b0, b1 in pairs: + speedup = round(get_fps(b1) / get_fps(b0), 2) if get_fps(b0) else None + acc_delta = round(map_pts(get_map(b1)) - map_pts(get_map(b0)), 1) + rows.append({ + "model": m, "displayName": model_display(m, models_meta), + "family": (models_meta.get(m) or {}).get("family"), + "baseline_fps": round(get_fps(b0), 1), "target_fps": round(get_fps(b1), 1), + "speedup_x": speedup, "mAP_delta_pts": acc_delta, + "baseline_mAP": map_pts(get_map(b0)), "target_mAP": map_pts(get_map(b1)), + }) + rows.sort(key=lambda r: -(r["speedup_x"] or 0)) + speedups = sorted(r["speedup_x"] for r in rows if r["speedup_x"]) + median = speedups[len(speedups) // 2] + fam_speed = defaultdict(list) + for r in rows: + fam_speed[r["family"]].append(r["speedup_x"]) + fam_median = {f: sorted(v)[len(v) // 2] for f, v in fam_speed.items()} + best_fam = max(fam_median.items(), key=lambda kv: kv[1]) + worst_fam = min(fam_median.items(), key=lambda kv: kv[1]) + acc_losers = [r for r in rows if r["mAP_delta_pts"] <= -TRT_ACC_DROP_PTS] + regressions = [r for r in rows if r["speedup_x"] and r["speedup_x"] < 1.0] + + facts = { + "hardware": hw, "hardware_display": hw_name, + "baseline": base_rt, "baseline_display": base_name, + "target": tgt_rt, "target_display": tgt_name, + "n_models": len(rows), "rows": rows, + "speedup": {"median_x": median, "min_x": speedups[0], "max_x": speedups[-1]}, + "family_median_speedup": fam_median, + "accuracy_losers": acc_losers, "speed_regressions": regressions, + } + + claims = [ + claim("median_speedup", "speed", + f"Across {len(rows)} models on {hw_name}, {tgt_name} gives a median " + f"{median}x over {base_name} (range {speedups[0]}x to {speedups[-1]}x).", + facts["speedup"], True, "strong"), + claim("top_gainer", "speed", + f"Biggest gain: {rows[0]['displayName']} at {rows[0]['speedup_x']}x " + f"({rows[0]['baseline_fps']} to {rows[0]['target_fps']} FPS).", + rows[0], True), + claim("family_spread", "speed", + f"Family medians differ: {best_fam[0]} gains {best_fam[1]}x, " + f"{worst_fam[0]} only {worst_fam[1]}x. Conversion gain is architecture-dependent.", + {"best": best_fam, "worst": worst_fam}, + abs(best_fam[1] - worst_fam[1]) >= 0.3), + claim("accuracy_cost", "accuracy", + (f"{len(acc_losers)} of {len(rows)} models lose {TRT_ACC_DROP_PTS}+ mAP points " + f"under {tgt_name}: " + ", ".join(f"{r['displayName']} ({r['mAP_delta_pts']})" for r in acc_losers[:5]) + ) if acc_losers else + f"No model loses more than {TRT_ACC_DROP_PTS} mAP points converting to {tgt_name}.", + {"losers": acc_losers[:5]}, True, "strong"), + ] + if regressions: + claims.append(claim( + "speed_regression", "anomaly", + f"{len(regressions)} models get SLOWER under {tgt_name}: " + + ", ".join(f"{r['displayName']} ({r['speedup_x']}x)" for r in regressions[:5]), + {"regressions": regressions[:5]}, True, "strong")) + + slug = f"{tgt_rt.replace('_', '-')}-vs-{base_rt.replace('_', '-')}-{hw.replace('_', '-')}" + meta_block = {"type": "runtime-guide", "hardware": hw, + "baseline": base_rt, "target": tgt_rt} + return finalize(meta_block, facts, claims, 3, slug) + + +# ---------------------------------------------------------------- license-guide + +def build_license_guide(index, models_meta, hardware_meta, runtimes_meta, hw, rt): + entries = {} + for (m, h, r), bench in index.items(): + if h == hw and r == rt: + entries[m] = bench + perm = {m: b for m, b in entries.items() if is_permissive(m, models_meta)} + nonperm = {m: b for m, b in entries.items() if not is_permissive(m, models_meta) + and (models_meta.get(m) or {}).get("license")} + if len(perm) < 5 or len(nonperm) < 2: + sys.exit(f"ERROR: need >=5 permissive and >=2 non-permissive models with data on {hw}/{rt} " + f"(have {len(perm)}/{len(nonperm)}).") + + hw_name = hw_display(hw, hardware_meta) + rt_name = rt_display(rt, runtimes_meta) + + matchups, claims = [], [] + for m, bench in sorted(nonperm.items(), key=lambda kv: -get_map(kv[1])): + m_map, m_fps = map_pts(get_map(bench)), round(get_fps(bench), 1) + # nearest permissive within accuracy, prefer faster + cands = [(p, pb) for p, pb in perm.items() + if map_pts(get_map(pb)) >= m_map - 1.0] + alt = max(cands, key=lambda kv: get_fps(kv[1]))[0] if cands else None + entry = { + "model": m, "displayName": model_display(m, models_meta), + "license": (models_meta.get(m) or {}).get("license"), + "mAP": m_map, "fps": m_fps, + "alternative": None, + } + if alt: + ab = perm[alt] + entry["alternative"] = { + "model": alt, "displayName": model_display(alt, models_meta), + "license": (models_meta.get(alt) or {}).get("license"), + "mAP": map_pts(get_map(ab)), "fps": round(get_fps(ab), 1), + "mAP_gap_pts": round(map_pts(get_map(ab)) - m_map, 1), + } + claims.append(claim( + f"alt_{m}", "license", + f"{entry['alternative']['displayName']} ({entry['alternative']['license']}) matches " + f"{entry['displayName']} ({entry['license']}) within " + f"{abs(entry['alternative']['mAP_gap_pts'])} mAP points on {hw_name}: " + f"{entry['alternative']['mAP']} vs {m_map} mAP, " + f"{entry['alternative']['fps']} vs {m_fps} FPS.", + entry, True, "strong")) + matchups.append(entry) + + facts = { + "hardware": hw, "hardware_display": hw_name, + "runtime": rt, "runtime_display": rt_name, + "n_permissive": len(perm), "n_nonpermissive": len(nonperm), + "matchups": matchups, + "permissive_leaderboard": sorted( + [{"model": m, "displayName": model_display(m, models_meta), + "license": (models_meta.get(m) or {}).get("license"), + "mAP": map_pts(get_map(b)), "fps": round(get_fps(b), 1)} + for m, b in perm.items()], key=lambda e: -e["mAP"]), + } + slug = f"permissive-license-detection-models-{hw.replace('_', '-')}" + meta_block = {"type": "license-guide", "hardware": hw, "runtime": rt} + return finalize(meta_block, facts, claims, 2, slug) + + +# ---------------------------------------------------------------- finding scan + +def build_finding_scan(index, models_meta, hardware_meta, runtimes_meta, hw_filter): + findings = [] + + # 1. jitter outliers: fps_mean much higher than fps_p50 + for (m, h, r), bench in index.items(): + if hw_filter and h != hw_filter: + continue + mean, p50 = get_fps(bench), bench["throughput"].get("fps_p50") + if mean and p50: + gap = pct(mean, p50) + if gap and abs(gap) >= JITTER_PCT: + findings.append({ + "kind": "jitter", "model": m, "hardware": h, "runtime": r, + "fps_mean": round(mean, 1), "fps_p50": round(p50, 1), + "gap_pct": gap, + "note": f"{model_display(m, models_meta)} on {hw_display(h, hardware_meta)}/{r}: " + f"fps_mean {round(mean,1)} vs fps_p50 {round(p50,1)} ({gap}% gap).", + }) + + # 2. tensorrt regressions (fp16 slower than fp32, or accuracy cliff) + for (m, h, r), bench in index.items(): + if r != "tensorrt_fp16" or (hw_filter and h != hw_filter): + continue + fp32 = index.get((m, h, "tensorrt_fp32")) + if fp32: + if get_fps(bench) < get_fps(fp32): + findings.append({ + "kind": "trt_fp16_slower", "model": m, "hardware": h, + "fp16_fps": round(get_fps(bench), 1), "fp32_fps": round(get_fps(fp32), 1), + "note": f"{model_display(m, models_meta)} on {hw_display(h, hardware_meta)}: " + f"TensorRT FP16 ({round(get_fps(bench),1)} FPS) is slower than FP32 " + f"({round(get_fps(fp32),1)} FPS).", + }) + acc_drop = map_pts(get_map(fp32)) - map_pts(get_map(bench)) + if acc_drop >= TRT_ACC_DROP_PTS: + findings.append({ + "kind": "fp16_accuracy_cliff", "model": m, "hardware": h, + "drop_pts": round(acc_drop, 1), + "fp32_mAP": map_pts(get_map(fp32)), "fp16_mAP": map_pts(get_map(bench)), + "note": f"{model_display(m, models_meta)} loses {round(acc_drop,1)} mAP points " + f"going FP32 to FP16 on {hw_display(h, hardware_meta)}.", + }) + + # 3. cross-hardware ranking flips among near-peers (same runtime) + hws = sorted({k[1] for k in index}) + models = sorted({k[0] for k in index}) + rt = DEFAULT_RUNTIME + for i, m1 in enumerate(models): + for m2 in models[i + 1:]: + p1, p2 = params_m_safe(index, models_meta, m1), params_m_safe(index, models_meta, m2) + if not p1 or not p2 or max(p1, p2) / min(p1, p2) > PAIR_PARAMS_RATIO: + continue + winners = {} + for h in hws: + b1, b2 = index.get((m1, h, rt)), index.get((m2, h, rt)) + if b1 and b2 and abs(pct(get_fps(b1), get_fps(b2)) or 0) >= SPEED_PCT: + winners[h] = m1 if get_fps(b1) > get_fps(b2) else m2 + if len(set(winners.values())) > 1: + findings.append({ + "kind": "ranking_flip", "models": [m1, m2], "runtime": rt, + "winners_by_hardware": winners, + "note": f"Speed winner between {model_display(m1, models_meta)} and " + f"{model_display(m2, models_meta)} flips across hardware: " + + ", ".join(f"{hw_display(h, hardware_meta)}: {model_display(w, models_meta)}" + for h, w in winners.items()), + }) + + facts = {"n_findings": len(findings), "findings": findings} + claims = [claim(f"finding_{i}", f["kind"], f["note"], f, True) + for i, f in enumerate(findings)] + slug = "finding-scan" + meta_block = {"type": "finding", "hardware": hw_filter} + return finalize(meta_block, facts, claims, 1, slug) + + +def params_m_safe(index, models_meta, m): + for (mm, h, r), bench in index.items(): + if mm == m: + return params_m(bench, models_meta) + return None + + +# ---------------------------------------------------------------- list + +def coverage_report(index, models_meta, hardware_meta, runtimes_meta): + by_hw_rt = defaultdict(set) + for (m, h, r) in index: + by_hw_rt[(h, r)].add(m) + print("Coverage (models with verified data):") + for (h, r), ms in sorted(by_hw_rt.items(), key=lambda kv: -len(kv[1])): + print(f" {h:32s} {r:16s} {len(ms):3d} models") + print("\nViable vs pairs (same hw/rt, params ratio <= " + f"{PAIR_PARAMS_RATIO}, cross-family): run vs with any two model ids above.") + print("Model ids:", ", ".join(sorted({k[0] for k in index}))) + + +# ---------------------------------------------------------------- plan + +# Demand proxy: families and sizes people actually search for and deploy. This +# is the ONE piece of editorial judgment in the engine. Not in the allowlist +# does not mean "never write", it means "do not auto-prioritize at scale". +POPULAR_FAMILIES = { + "yolov9": 3, "yolox": 3, "rtdetr": 3, "rfdetr": 3, "dfine": 2, + "deim": 2, "rtdetrv2": 2, "yolonas": 2, "deimv2": 1, "picodet": 1, + "rtdetrv4": 1, "ec": 1, +} +SIZE_ORDER = ["n", "nano", "atto", "femto", "pico", "t", "tiny", "s", "m", + "c", "l", "r18", "r34", "r50", "r50m", "r101", "x"] + + +def _size_token(model_id, models_meta): + return (models_meta.get(model_id) or {}).get("variant", "") + + +def _size_rank(model_id, models_meta): + tok = _size_token(model_id, models_meta) + return SIZE_ORDER.index(tok) if tok in SIZE_ORDER else 99 + + +def _vs_significance(a, b, models_meta): + """Cheap significance count + which axes, mirroring build_vs gates.""" + axes = [] + am, bm = get_map(a), get_map(b) + if abs(map_pts(am) - map_pts(bm)) >= ACC_PTS: + axes.append("accuracy") + d_fps = pct(get_fps(a), get_fps(b)) + if d_fps is not None and abs(d_fps) >= SPEED_PCT: + axes.append("speed") + afl, bfl = flops_g(a, models_meta), flops_g(b, models_meta) + if afl and bfl and abs(pct(am / afl, bm / bfl) or 0) >= EFFICIENCY_PCT: + axes.append("efficiency") + va = (a.get("memory") or {}).get("peak_vram_mb") + vb = (b.get("memory") or {}).get("peak_vram_mb") + if va and vb and abs(pct(va, vb) or 0) >= MEMORY_PCT: + axes.append("memory") + as_, bs_ = a["accuracy"].get("mAP_small"), b["accuracy"].get("mAP_small") + if as_ and bs_ and abs(pct(as_, bs_) or 0) >= SMALL_OBJ_PCT: + axes.append("small_object") + la = (models_meta.get(a["model"]["id"]) or {}).get("license") in PERMISSIVE + lb = (models_meta.get(b["model"]["id"]) or {}).get("license") in PERMISSIVE + if la != lb: + axes.append("license") + return axes + + +def _has_ranking_flip(a_id, b_id, index): + """Speed winner differs across hardware on the default runtime.""" + winners = set() + for h in {k[1] for k in index}: + a2, b2 = index.get((a_id, h, DEFAULT_RUNTIME)), index.get((b_id, h, DEFAULT_RUNTIME)) + if a2 and b2 and abs(pct(get_fps(a2), get_fps(b2)) or 0) >= SPEED_PCT: + winners.add(a_id if get_fps(a2) > get_fps(b2) else b_id) + return len(winners) > 1 + + +def build_plan(index, models_meta, hardware_meta, runtimes_meta, limit, vs_slice): + """Enumerate every viable article, score, de-duplicate, rank.""" + hw, rt = vs_slice + items = [] + + # --- vs pairs on the chosen primary slice --- + models = sorted({k[0] for k in index if k[1] == hw and k[2] == rt}) + seen_shape = defaultdict(int) # (family-pair, size-class-pair) -> count + vs_rows = [] + for a_id, b_id in __import__("itertools").combinations(models, 2): + a, b = index[(a_id, hw, rt)], index[(b_id, hw, rt)] + ap, bp = params_m(a, models_meta), params_m(b, models_meta) + if not ap or not bp or max(ap, bp) / min(ap, bp) > PAIR_PARAMS_RATIO: + continue + axes = _vs_significance(a, b, models_meta) + if len(axes) < 2: + continue + fa = (models_meta.get(a_id) or {}).get("family") + fb = (models_meta.get(b_id) or {}).get("family") + cross = fa != fb + flip = _has_ranking_flip(a_id, b_id, index) + # score: demand (popular families) + differentiation (cross-family, + # ranking flip, license edge, small-object story) + delta richness + score = 0.0 + score += POPULAR_FAMILIES.get(fa, 0) + POPULAR_FAMILIES.get(fb, 0) + score += 3 if cross else -1 # same-family adjacent is lower value + score += 4 if flip else 0 + score += 2 if "license" in axes else 0 + score += 1.5 if "small_object" in axes else 0 + score += 0.5 * len(axes) + # de-dup: penalize the Nth article with the same family+size shape + shape = (frozenset([fa, fb]), + frozenset([_size_token(a_id, models_meta), _size_token(b_id, models_meta)])) + dup_n = seen_shape[shape] + score -= 3 * dup_n + seen_shape[shape] += 1 + vs_rows.append({ + "type": "vs", "slug": f"{a_id}-vs-{b_id}", + "cmd": f"vs {a_id} {b_id} --hardware {hw} --runtime {rt}", + "score": round(score, 2), "axes": axes, "cross_family": cross, + "ranking_flip": flip, + }) + vs_rows.sort(key=lambda r: -r["score"]) + items += vs_rows + + # --- hardware guides (every hw x meaningful runtime with >=8 models) --- + hw_rt = defaultdict(set) + for (m, h, r) in index: + hw_rt[(h, r)].add(m) + for (h, r), ms in hw_rt.items(): + if len(ms) >= 8: + items.append({ + "type": "hardware-guide", + "slug": f"best-object-detection-{h.replace('_', '-')}", + "cmd": f"hardware-guide {h} --runtime {r}", + "score": round(6 + min(len(ms), 20) * 0.1, 2), + "n_models": len(ms), "runtime": r, + }) + + # --- runtime guides (hw with a baseline+target pair, >=8 overlap) --- + for h in {k[1] for k in index}: + rts = sorted({k[2] for k in index if k[1] == h}) + for base in rts: + for tgt in rts: + if base == tgt: + continue + overlap = sum(1 for m in {k[0] for k in index} + if index.get((m, h, base)) and index.get((m, h, tgt))) + if overlap >= 8 and ("fp16" in tgt or tgt.startswith("tensorrt") + or tgt.startswith("onnx") or "int8" in tgt): + items.append({ + "type": "runtime-guide", + "slug": f"{tgt.replace('_', '-')}-vs-{base.replace('_', '-')}-{h.replace('_', '-')}", + "cmd": f"runtime-guide {h} {base} {tgt}", + "score": round(5 + overlap * 0.05, 2), "overlap": overlap, + }) + + # --- license guides (hw with >=2 non-permissive + >=5 permissive) --- + for (h, r), ms in hw_rt.items(): + perm = sum(1 for m in ms if (models_meta.get(m) or {}).get("license") in PERMISSIVE) + nonperm = sum(1 for m in ms + if (models_meta.get(m) or {}).get("license") + and (models_meta.get(m) or {}).get("license") not in PERMISSIVE) + if perm >= 5 and nonperm >= 2: + items.append({ + "type": "license-guide", + "slug": f"permissive-license-detection-models-{h.replace('_', '-')}", + "cmd": f"license-guide --hardware {h} --runtime {r}", + "score": round(5.5 + nonperm * 0.2, 2), "nonpermissive": nonperm, + }) + + # dedupe guide slugs (same slug from multiple runtimes -> keep best score) + best = {} + for it in items: + key = it["slug"] + if key not in best or it["score"] > best[key]["score"]: + best[key] = it + ranked = sorted(best.values(), key=lambda r: -r["score"]) + if limit: + ranked = ranked[:limit] + return ranked + + +def plan_report(index, models_meta, hardware_meta, runtimes_meta, limit, vs_slice, as_json): + ranked = build_plan(index, models_meta, hardware_meta, runtimes_meta, limit, vs_slice) + if as_json: + print(json.dumps(ranked, indent=2)) + return + by_type = defaultdict(int) + for r in ranked: + by_type[r["type"]] += 1 + print(f"PLAN: {len(ranked)} articles (primary vs slice {vs_slice[0]}/{vs_slice[1]})") + for t, n in sorted(by_type.items(), key=lambda kv: -kv[1]): + print(f" {t:16s} {n}") + print("\nrank score type slug") + for i, r in enumerate(ranked, 1): + tags = [] + if r.get("ranking_flip"): + tags.append("FLIP") + if r.get("cross_family") is False: + tags.append("same-fam") + tag = (" [" + ",".join(tags) + "]") if tags else "" + print(f"{i:4d} {r['score']:5.1f} {r['type']:16s} {r['slug']}{tag}") + + +# ---------------------------------------------------------------- main + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + + p_vs = sub.add_parser("vs") + p_vs.add_argument("a"); p_vs.add_argument("b") + p_vs.add_argument("--hardware", default=DEFAULT_HARDWARE) + p_vs.add_argument("--runtime", default=DEFAULT_RUNTIME) + + p_fv = sub.add_parser("family-vs") + p_fv.add_argument("a"); p_fv.add_argument("b") + p_fv.add_argument("--hardware", default=DEFAULT_HARDWARE) + p_fv.add_argument("--runtime", default=DEFAULT_RUNTIME) + + p_hg = sub.add_parser("hardware-guide") + p_hg.add_argument("hardware") + p_hg.add_argument("--runtime", default=DEFAULT_RUNTIME) + + p_rg = sub.add_parser("runtime-guide") + p_rg.add_argument("hardware"); p_rg.add_argument("baseline"); p_rg.add_argument("target") + + p_lg = sub.add_parser("license-guide") + p_lg.add_argument("--hardware", default=DEFAULT_HARDWARE) + p_lg.add_argument("--runtime", default=DEFAULT_RUNTIME) + + p_f = sub.add_parser("finding") + p_f.add_argument("action", choices=["scan"]) + p_f.add_argument("--hardware", default=None) + + sub.add_parser("list") + + p_plan = sub.add_parser("plan") + p_plan.add_argument("--limit", type=int, default=0, help="top N articles (0 = all)") + p_plan.add_argument("--hardware", default=DEFAULT_HARDWARE, help="primary hw for vs pairs") + p_plan.add_argument("--runtime", default=DEFAULT_RUNTIME, help="primary runtime for vs pairs") + p_plan.add_argument("--json", action="store_true", help="emit ranked worklist as JSON") + + args = ap.parse_args() + index, models_meta, hardware_meta, runtimes_meta = load_dataset() + + if args.cmd == "list": + coverage_report(index, models_meta, hardware_meta, runtimes_meta) + return + + if args.cmd == "plan": + plan_report(index, models_meta, hardware_meta, runtimes_meta, + args.limit, (args.hardware, args.runtime), args.json) + return + + if args.cmd == "vs": + doc = build_vs(index, models_meta, hardware_meta, runtimes_meta, + args.a, args.b, args.hardware, args.runtime) + elif args.cmd == "family-vs": + doc = build_family_vs(index, models_meta, hardware_meta, runtimes_meta, + args.a, args.b, args.hardware, args.runtime) + elif args.cmd == "hardware-guide": + doc = build_hardware_guide(index, models_meta, hardware_meta, runtimes_meta, + args.hardware, args.runtime) + elif args.cmd == "runtime-guide": + doc = build_runtime_guide(index, models_meta, hardware_meta, runtimes_meta, + args.hardware, args.baseline, args.target) + elif args.cmd == "license-guide": + doc = build_license_guide(index, models_meta, hardware_meta, runtimes_meta, + args.hardware, args.runtime) + elif args.cmd == "finding": + doc = build_finding_scan(index, models_meta, hardware_meta, runtimes_meta, + args.hardware) + + OUTPUT.mkdir(exist_ok=True) + out_path = OUTPUT / f"{doc['slug']}.claims.json" + out_path.write_text(json.dumps(doc, indent=2), encoding="utf-8") + + gate = doc["gate"] + n_passed = gate["passed_claims"] + print(json.dumps(doc, indent=2)) + print(f"\nWROTE {out_path}", file=sys.stderr) + if gate["passed"]: + print(f"GATE PASSED: {n_passed} claims cleared (need {gate['required']}). " + f"OK to write '{doc['slug']}'.", file=sys.stderr) + else: + print(f"GATE FAILED: {gate['reason']}", file=sys.stderr) + sys.exit(2) + + +if __name__ == "__main__": + main() diff --git a/article-pipeline/v2/output/_vs_wave.json b/article-pipeline/v2/output/_vs_wave.json new file mode 100644 index 0000000..9459734 --- /dev/null +++ b/article-pipeline/v2/output/_vs_wave.json @@ -0,0 +1,32 @@ +[ + "rtdetr-r34-vs-yolov9c", + "rtdetr-r34-vs-yolox-l", + "rfdetr-n-vs-rtdetr-r18", + "rfdetr-n-vs-yolov9c", + "rfdetr-n-vs-yolox-l", + "rfdetr-s-vs-yolov9c", + "deim-l-vs-rfdetr-l", + "deim-l-vs-rtdetr-r101", + "deim-l-vs-rtdetr-r50", + "deim-m-vs-rtdetr-r34", + "deim-n-vs-yolov9t", + "deim-s-vs-rtdetr-r18", + "deim-s-vs-yolov9s", + "dfine-l-vs-rfdetr-l", + "dfine-l-vs-rtdetr-r101", + "dfine-l-vs-rtdetr-r50", + "dfine-m-vs-rtdetr-r34", + "dfine-s-vs-rtdetr-r18", + "rfdetr-n-vs-yolox-m", + "rfdetr-s-vs-yolox-l", + "deim-l-vs-rtdetr-r50m", + "deim-l-vs-rtdetr-x", + "deim-l-vs-yolox-l", + "dfine-l-vs-rtdetr-r50m", + "dfine-l-vs-yolox-l", + "dfine-m-vs-rtdetr-r50m", + "dfine-n-vs-yolov9t", + "rfdetr-n-vs-rtdetrv2-r18", + "deim-l-vs-rtdetrv2-r101", + "deim-l-vs-rtdetrv2-r50" +] \ No newline at end of file diff --git a/article-pipeline/v2/output/best-object-detection-jetson-orin.claims.json b/article-pipeline/v2/output/best-object-detection-jetson-orin.claims.json new file mode 100644 index 0000000..4038045 --- /dev/null +++ b/article-pipeline/v2/output/best-object-detection-jetson-orin.claims.json @@ -0,0 +1,939 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T21:56:12.338781+00:00", + "slug": "best-object-detection-jetson-orin", + "meta": { + "type": "hardware-guide", + "hardware": "jetson_orin", + "runtime": "tensorrt_fp16" + }, + "facts": { + "hardware": "jetson_orin", + "hardware_display": "NVIDIA Jetson Orin Nano Super 8GB", + "runtime": "tensorrt_fp16", + "runtime_display": "TensorRT FP16", + "n_models": 55, + "leaderboard": [ + { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.4, + "fps": 14.2, + "ms": 70.45, + "params_millions": 0.0 + }, + { + "model": "ec-x", + "displayName": "ECDet-X", + "family": "ec", + "license": "Apache-2.0", + "mAP": 61.0, + "fps": 5.7, + "ms": 174.81, + "params_millions": 0.0 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 60.7, + "fps": 5.4, + "ms": 184.54, + "params_millions": 0.0 + }, + { + "model": "dfine-l", + "displayName": "D-FINE-L", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 60.0, + "fps": 18.6, + "ms": 53.64, + "params_millions": 0.0 + }, + { + "model": "rtdetrv4-x", + "displayName": "RT-DETRv4-X", + "family": "rtdetrv4", + "license": "Apache-2.0", + "mAP": 60.0, + "fps": 13.8, + "ms": 72.69, + "params_millions": 0.0 + }, + { + "model": "ec-l", + "displayName": "ECDet-L", + "family": "ec", + "license": "Apache-2.0", + "mAP": 60.0, + "fps": 6.2, + "ms": 161.52, + "params_millions": 0.0 + }, + { + "model": "deim-x", + "displayName": "DEIM-X", + "family": "deim", + "license": "Apache-2.0", + "mAP": 59.6, + "fps": 14.1, + "ms": 70.93, + "params_millions": 0.0 + }, + { + "model": "rfdetr-l", + "displayName": "RF-DETR-L", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 58.7, + "fps": 8.4, + "ms": 119.33, + "params_millions": 128.0 + }, + { + "model": "deimv2-l", + "displayName": "DEIMv2-L", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 58.5, + "fps": 6.2, + "ms": 160.81, + "params_millions": 0.0 + }, + { + "model": "ec-m", + "displayName": "ECDet-M", + "family": "ec", + "license": "Apache-2.0", + "mAP": 58.3, + "fps": 8.3, + "ms": 119.7, + "params_millions": 0.0 + }, + { + "model": "deim-l", + "displayName": "DEIM-L", + "family": "deim", + "license": "Apache-2.0", + "mAP": 57.9, + "fps": 18.5, + "ms": 53.96, + "params_millions": 0.0 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 57.9, + "fps": 23.7, + "ms": 42.26, + "params_millions": 0.0 + }, + { + "model": "rtdetr-x", + "displayName": "RT-DETR-X", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 57.9, + "fps": 17.0, + "ms": 58.9, + "params_millions": 0.0 + }, + { + "model": "rtdetrv4-l", + "displayName": "RT-DETRv4-L", + "family": "rtdetrv4", + "license": "Apache-2.0", + "mAP": 57.8, + "fps": 18.1, + "ms": 55.15, + "params_millions": 0.0 + }, + { + "model": "rfdetr-m", + "displayName": "RF-DETR-M", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 57.4, + "fps": 12.5, + "ms": 79.75, + "params_millions": 0.0 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 57.1, + "fps": 24.9, + "ms": 40.11, + "params_millions": 25.5 + }, + { + "model": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 56.8, + "fps": 17.0, + "ms": 58.76, + "params_millions": 0.0 + }, + { + "model": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 56.8, + "fps": 17.3, + "ms": 57.69, + "params_millions": 0.0 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "family": "rtdetrv4", + "license": "Apache-2.0", + "mAP": 56.6, + "fps": 23.1, + "ms": 43.29, + "params_millions": 0.0 + }, + { + "model": "yolox-x", + "displayName": "YOLOX-X", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 56.2, + "fps": 14.6, + "ms": 68.7, + "params_millions": 99.07 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "license": "MIT", + "mAP": 56.1, + "fps": 25.8, + "ms": 38.8, + "params_millions": 20.12 + }, + { + "model": "deimv2-m", + "displayName": "DEIMv2-M", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 56.0, + "fps": 7.9, + "ms": 126.52, + "params_millions": 0.0 + }, + { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 55.8, + "fps": 24.1, + "ms": 41.5, + "params_millions": 0.0 + }, + { + "model": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 55.7, + "fps": 23.6, + "ms": 42.42, + "params_millions": 0.0 + }, + { + "model": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 55.7, + "fps": 24.2, + "ms": 41.28, + "params_millions": 0.0 + }, + { + "model": "deim-m", + "displayName": "DEIM-M", + "family": "deim", + "license": "Apache-2.0", + "mAP": 55.5, + "fps": 23.5, + "ms": 42.55, + "params_millions": 0.0 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 55.5, + "fps": 21.6, + "ms": 46.34, + "params_millions": 54.21 + }, + { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 55.2, + "fps": 15.5, + "ms": 64.36, + "params_millions": 0.0 + }, + { + "model": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 54.8, + "fps": 29.7, + "ms": 33.62, + "params_millions": 0.0 + }, + { + "model": "ec-s", + "displayName": "ECDet-S", + "family": "ec", + "license": "Apache-2.0", + "mAP": 54.3, + "fps": 10.6, + "ms": 94.78, + "params_millions": 0.0 + }, + { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 53.8, + "fps": 29.1, + "ms": 34.35, + "params_millions": 0.0 + }, + { + "model": "dfine-s", + "displayName": "D-FINE-S", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 53.4, + "fps": 30.2, + "ms": 33.15, + "params_millions": 0.0 + }, + { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 53.2, + "fps": 32.7, + "ms": 30.59, + "params_millions": 0.0 + }, + { + "model": "deimv2-s", + "displayName": "DEIMv2-S", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 53.0, + "fps": 10.5, + "ms": 95.51, + "params_millions": 0.0 + }, + { + "model": "rtdetrv4-s", + "displayName": "RT-DETRv4-S", + "family": "rtdetrv4", + "license": "Apache-2.0", + "mAP": 52.9, + "fps": 29.6, + "ms": 33.74, + "params_millions": 0.0 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 52.3, + "fps": 31.9, + "ms": 31.36, + "params_millions": 0.0 + }, + { + "model": "deim-s", + "displayName": "DEIM-S", + "family": "deim", + "license": "Apache-2.0", + "mAP": 52.0, + "fps": 29.9, + "ms": 33.44, + "params_millions": 0.0 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 51.8, + "fps": 28.0, + "ms": 35.74, + "params_millions": 25.33 + }, + { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 51.3, + "fps": 25.4, + "ms": 39.36, + "params_millions": 0.0 + }, + { + "model": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 50.8, + "fps": 39.9, + "ms": 25.06, + "params_millions": 0.0 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "license": "MIT", + "mAP": 50.4, + "fps": 33.2, + "ms": 30.15, + "params_millions": 7.23 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 49.7, + "fps": 38.8, + "ms": 25.75, + "params_millions": 0.0 + }, + { + "model": "deim-n", + "displayName": "DEIM-N", + "family": "deim", + "license": "Apache-2.0", + "mAP": 46.8, + "fps": 41.8, + "ms": 23.91, + "params_millions": 0.0 + }, + { + "model": "deimv2-n", + "displayName": "DEIMv2-N", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 46.6, + "fps": 41.2, + "ms": 24.25, + "params_millions": 0.0 + }, + { + "model": "dfine-n", + "displayName": "D-FINE-N", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 45.8, + "fps": 42.1, + "ms": 23.77, + "params_millions": 0.0 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 44.2, + "fps": 34.9, + "ms": 28.64, + "params_millions": 8.97 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "license": "Apache-2.0", + "mAP": 44.0, + "fps": 19.1, + "ms": 52.34, + "params_millions": 0.0 + }, + { + "model": "deimv2-pico", + "displayName": "DEIMv2-Pico", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 42.3, + "fps": 44.5, + "ms": 22.5, + "params_millions": 0.0 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "license": "MIT", + "mAP": 41.8, + "fps": 34.3, + "ms": 29.15, + "params_millions": 2.04 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "license": "Apache-2.0", + "mAP": 37.9, + "fps": 32.4, + "ms": 30.84, + "params_millions": 0.0 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 35.5, + "fps": 51.6, + "ms": 19.39, + "params_millions": 5.06 + }, + { + "model": "deimv2-femto", + "displayName": "DEIMv2-Femto", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 34.5, + "fps": 65.7, + "ms": 15.23, + "params_millions": 0.0 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "license": "Apache-2.0", + "mAP": 30.4, + "fps": 39.6, + "ms": 25.26, + "params_millions": 0.0 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 28.8, + "fps": 51.0, + "ms": 19.62, + "params_millions": 0.91 + }, + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 27.5, + "fps": 89.1, + "ms": 11.23, + "params_millions": 0.0 + } + ], + "pareto_models": [ + "deimv2-atto", + "deimv2-femto", + "yolox-tiny", + "deimv2-pico", + "dfine-n", + "deim-n", + "rtdetrv2-r18", + "rtdetrv2-r34", + "dfine-s", + "rtdetrv2-r50m", + "yolov9m", + "yolov9c", + "dfine-m", + "dfine-l", + "dfine-x" + ], + "budget_picks": { + "33": { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 53.2, + "fps": 32.7, + "ms": 30.59, + "params_millions": 0.0 + }, + "100": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.4, + "fps": 14.2, + "ms": 70.45, + "params_millions": 0.0 + } + }, + "best_overall": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.4, + "fps": 14.2, + "ms": 70.45, + "params_millions": 0.0 + }, + "best_permissive": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.4, + "fps": 14.2, + "ms": 70.45, + "params_millions": 0.0 + }, + "fastest": { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 27.5, + "fps": 89.1, + "ms": 11.23, + "params_millions": 0.0 + } + }, + "claims": [ + { + "id": "best_overall", + "kind": "ranking", + "statement": "D-FINE-X leads on NVIDIA Jetson Orin Nano Super 8GB at 61.4 mAP@50-95 (14.2 FPS, TensorRT FP16).", + "values": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.4, + "fps": 14.2, + "ms": 70.45, + "params_millions": 0.0 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "fastest", + "kind": "ranking", + "statement": "DEIMv2-Atto is the fastest measured: 89.1 FPS at 27.5 mAP.", + "values": { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 27.5, + "fps": 89.1, + "ms": 11.23, + "params_millions": 0.0 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "pareto", + "kind": "ranking", + "statement": "The measured accuracy-speed frontier on NVIDIA Jetson Orin Nano Super 8GB is: DEIMv2-Atto, DEIMv2-Femto, YOLOX-Tiny, DEIMv2-Pico, D-FINE-N, DEIM-N, RT-DETRv2-R18, RT-DETRv2-R34, D-FINE-S, RT-DETRv2-R50m, YOLOv9-M, YOLOv9-C, D-FINE-M, D-FINE-L, D-FINE-X.", + "values": { + "pareto": [ + "deimv2-atto", + "deimv2-femto", + "yolox-tiny", + "deimv2-pico", + "dfine-n", + "deim-n", + "rtdetrv2-r18", + "rtdetrv2-r34", + "dfine-s", + "rtdetrv2-r50m", + "yolov9m", + "yolov9c", + "dfine-m", + "dfine-l", + "dfine-x" + ] + }, + "passed": true, + "strength": "normal" + }, + { + "id": "budget_33ms", + "kind": "budget", + "statement": "Under 33 ms/image on NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34 is the most accurate (53.2 mAP at 30.59 ms).", + "values": { + "budget_ms": 33, + "pick": { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 53.2, + "fps": 32.7, + "ms": 30.59, + "params_millions": 0.0 + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "budget_100ms", + "kind": "budget", + "statement": "Under 100 ms/image on NVIDIA Jetson Orin Nano Super 8GB: D-FINE-X is the most accurate (61.4 mAP at 70.45 ms).", + "values": { + "budget_ms": 100, + "pick": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.4, + "fps": 14.2, + "ms": 70.45, + "params_millions": 0.0 + } + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 3, + "reason": null + }, + "allowed_numbers": [ + 0.0, + 0.001, + 0.6, + 0.91, + 1, + 2, + 2.04, + 3, + 4, + 5, + 5.06, + 5.4, + 5.7, + 6, + 6.2, + 7, + 7.23, + 7.9, + 8, + 8.3, + 8.4, + 8.97, + 9, + 10, + 10.5, + 10.6, + 11, + 11.23, + 12, + 12.5, + 13.8, + 14.1, + 14.2, + 14.6, + 15.23, + 15.5, + 17.0, + 17.3, + 18.1, + 18.5, + 18.6, + 19.1, + 19.39, + 19.62, + 20.12, + 21.6, + 22.5, + 23.1, + 23.5, + 23.6, + 23.7, + 23.77, + 23.91, + 24.1, + 24.2, + 24.25, + 24.9, + 25.06, + 25.26, + 25.33, + 25.4, + 25.5, + 25.75, + 25.8, + 27.5, + 28.0, + 28.64, + 28.8, + 29.1, + 29.15, + 29.6, + 29.7, + 29.9, + 30.15, + 30.2, + 30.4, + 30.59, + 30.84, + 31.36, + 31.9, + 32.4, + 32.7, + 33.0, + 33.15, + 33.2, + 33.44, + 33.62, + 33.74, + 34.3, + 34.35, + 34.5, + 34.9, + 35.5, + 35.74, + 37.9, + 38.8, + 39.36, + 39.6, + 39.9, + 40.11, + 41.2, + 41.28, + 41.5, + 41.8, + 42.1, + 42.26, + 42.3, + 42.42, + 42.55, + 43.29, + 44.0, + 44.2, + 44.5, + 45.8, + 46.34, + 46.6, + 46.8, + 49.7, + 50, + 50.4, + 50.8, + 51.0, + 51.3, + 51.6, + 51.8, + 52.0, + 52.3, + 52.34, + 52.9, + 53.0, + 53.2, + 53.4, + 53.64, + 53.8, + 53.96, + 54.21, + 54.3, + 54.8, + 55.0, + 55.15, + 55.2, + 55.5, + 55.7, + 55.8, + 56.0, + 56.1, + 56.2, + 56.6, + 56.8, + 57.1, + 57.4, + 57.69, + 57.8, + 57.9, + 58.3, + 58.5, + 58.7, + 58.76, + 58.9, + 59.6, + 60.0, + 60.7, + 61.0, + 61.4, + 64.36, + 65.7, + 68.7, + 70.45, + 70.93, + 72.69, + 75, + 79.75, + 89.1, + 91.0, + 94.78, + 95, + 95.51, + 99.07, + 100.0, + 119.33, + 119.7, + 126.52, + 128.0, + 160.81, + 161.52, + 174.81, + 184.54, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/best-object-detection-nvidia-geforce-rtx-5070-ti.claims.json b/article-pipeline/v2/output/best-object-detection-nvidia-geforce-rtx-5070-ti.claims.json new file mode 100644 index 0000000..9f1574d --- /dev/null +++ b/article-pipeline/v2/output/best-object-detection-nvidia-geforce-rtx-5070-ti.claims.json @@ -0,0 +1,903 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T21:56:12.485771+00:00", + "slug": "best-object-detection-nvidia-geforce-rtx-5070-ti", + "meta": { + "type": "hardware-guide", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "tensorrt_fp16" + }, + "facts": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "tensorrt_fp16", + "runtime_display": "TensorRT FP16", + "n_models": 51, + "leaderboard": [ + { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.5, + "fps": 73.0, + "ms": 13.7, + "params_millions": 0.0 + }, + { + "model": "ec-x", + "displayName": "ECDet-X", + "family": "ec", + "license": "Apache-2.0", + "mAP": 61.0, + "fps": 41.3, + "ms": 24.22, + "params_millions": 0.0 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 60.8, + "fps": 37.9, + "ms": 26.4, + "params_millions": 0.0 + }, + { + "model": "dfine-l", + "displayName": "D-FINE-L", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 60.0, + "fps": 69.3, + "ms": 14.42, + "params_millions": 0.0 + }, + { + "model": "ec-l", + "displayName": "ECDet-L", + "family": "ec", + "license": "Apache-2.0", + "mAP": 60.0, + "fps": 41.7, + "ms": 24.0, + "params_millions": 0.0 + }, + { + "model": "rtdetrv4-x", + "displayName": "RT-DETRv4-X", + "family": "rtdetrv4", + "license": "Apache-2.0", + "mAP": 59.9, + "fps": 71.0, + "ms": 14.07, + "params_millions": 0.0 + }, + { + "model": "deim-x", + "displayName": "DEIM-X", + "family": "deim", + "license": "Apache-2.0", + "mAP": 59.5, + "fps": 70.7, + "ms": 14.15, + "params_millions": 0.0 + }, + { + "model": "rfdetr-l", + "displayName": "RF-DETR-L", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 58.6, + "fps": 47.6, + "ms": 21.01, + "params_millions": 128.0 + }, + { + "model": "deimv2-l", + "displayName": "DEIMv2-L", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 58.5, + "fps": 38.7, + "ms": 25.86, + "params_millions": 0.0 + }, + { + "model": "ec-m", + "displayName": "ECDet-M", + "family": "ec", + "license": "Apache-2.0", + "mAP": 58.2, + "fps": 48.9, + "ms": 20.46, + "params_millions": 0.0 + }, + { + "model": "rtdetr-x", + "displayName": "RT-DETR-X", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 57.9, + "fps": 82.3, + "ms": 12.15, + "params_millions": 0.0 + }, + { + "model": "rtdetrv4-l", + "displayName": "RT-DETRv4-L", + "family": "rtdetrv4", + "license": "Apache-2.0", + "mAP": 57.7, + "fps": 78.0, + "ms": 12.83, + "params_millions": 0.0 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 57.6, + "fps": 74.5, + "ms": 13.42, + "params_millions": 0.0 + }, + { + "model": "rfdetr-m", + "displayName": "RF-DETR-M", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 57.4, + "fps": 62.9, + "ms": 15.91, + "params_millions": 0.0 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 57.2, + "fps": 74.7, + "ms": 13.39, + "params_millions": 25.5 + }, + { + "model": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 56.8, + "fps": 87.4, + "ms": 11.44, + "params_millions": 0.0 + }, + { + "model": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 56.7, + "fps": 88.0, + "ms": 11.36, + "params_millions": 0.0 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "family": "rtdetrv4", + "license": "Apache-2.0", + "mAP": 56.4, + "fps": 86.7, + "ms": 11.54, + "params_millions": 0.0 + }, + { + "model": "yolox-x", + "displayName": "YOLOX-X", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 56.3, + "fps": 72.1, + "ms": 13.88, + "params_millions": 99.07 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "license": "MIT", + "mAP": 56.1, + "fps": 72.3, + "ms": 13.84, + "params_millions": 20.12 + }, + { + "model": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 55.8, + "fps": 97.4, + "ms": 10.27, + "params_millions": 0.0 + }, + { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 55.8, + "fps": 122.0, + "ms": 8.2, + "params_millions": 0.0 + }, + { + "model": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 55.7, + "fps": 103.6, + "ms": 9.65, + "params_millions": 0.0 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 55.4, + "fps": 79.2, + "ms": 12.62, + "params_millions": 54.21 + }, + { + "model": "deim-m", + "displayName": "DEIM-M", + "family": "deim", + "license": "Apache-2.0", + "mAP": 55.4, + "fps": 83.3, + "ms": 12.0, + "params_millions": 0.0 + }, + { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 55.3, + "fps": 71.6, + "ms": 13.96, + "params_millions": 0.0 + }, + { + "model": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 54.7, + "fps": 113.0, + "ms": 8.85, + "params_millions": 0.0 + }, + { + "model": "ec-s", + "displayName": "ECDet-S", + "family": "ec", + "license": "Apache-2.0", + "mAP": 54.4, + "fps": 43.5, + "ms": 23.02, + "params_millions": 0.0 + }, + { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 53.9, + "fps": 110.1, + "ms": 9.08, + "params_millions": 0.0 + }, + { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 53.2, + "fps": 132.9, + "ms": 7.52, + "params_millions": 0.0 + }, + { + "model": "dfine-s", + "displayName": "D-FINE-S", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 53.2, + "fps": 86.5, + "ms": 11.55, + "params_millions": 0.0 + }, + { + "model": "deimv2-s", + "displayName": "DEIMv2-S", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 53.0, + "fps": 50.4, + "ms": 19.84, + "params_millions": 0.0 + }, + { + "model": "rtdetrv4-s", + "displayName": "RT-DETRv4-S", + "family": "rtdetrv4", + "license": "Apache-2.0", + "mAP": 52.8, + "fps": 92.3, + "ms": 10.84, + "params_millions": 0.0 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 52.3, + "fps": 108.4, + "ms": 9.22, + "params_millions": 0.0 + }, + { + "model": "deim-s", + "displayName": "DEIM-S", + "family": "deim", + "license": "Apache-2.0", + "mAP": 51.9, + "fps": 82.4, + "ms": 12.14, + "params_millions": 0.0 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 51.6, + "fps": 83.0, + "ms": 12.04, + "params_millions": 25.33 + }, + { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 51.4, + "fps": 58.3, + "ms": 17.14, + "params_millions": 0.0 + }, + { + "model": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 50.8, + "fps": 117.8, + "ms": 8.49, + "params_millions": 0.0 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "license": "MIT", + "mAP": 50.5, + "fps": 63.8, + "ms": 15.66, + "params_millions": 7.23 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 49.8, + "fps": 113.4, + "ms": 8.82, + "params_millions": 0.0 + }, + { + "model": "deimv2-n", + "displayName": "DEIMv2-N", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 46.7, + "fps": 78.0, + "ms": 12.82, + "params_millions": 0.0 + }, + { + "model": "deim-n", + "displayName": "DEIM-N", + "family": "deim", + "license": "Apache-2.0", + "mAP": 46.6, + "fps": 81.3, + "ms": 12.3, + "params_millions": 0.0 + }, + { + "model": "dfine-n", + "displayName": "D-FINE-N", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 45.8, + "fps": 79.7, + "ms": 12.54, + "params_millions": 0.0 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "license": "Apache-2.0", + "mAP": 44.0, + "fps": 38.0, + "ms": 26.32, + "params_millions": 0.0 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 43.4, + "fps": 70.9, + "ms": 14.11, + "params_millions": 8.97 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "license": "MIT", + "mAP": 41.8, + "fps": 59.0, + "ms": 16.96, + "params_millions": 2.04 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "license": "Apache-2.0", + "mAP": 37.3, + "fps": 65.7, + "ms": 15.21, + "params_millions": 0.0 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 35.4, + "fps": 97.7, + "ms": 10.23, + "params_millions": 5.06 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "license": "Apache-2.0", + "mAP": 30.4, + "fps": 78.6, + "ms": 12.73, + "params_millions": 0.0 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 28.7, + "fps": 100.0, + "ms": 9.99, + "params_millions": 0.91 + }, + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 25.8, + "fps": 132.1, + "ms": 7.57, + "params_millions": 0.0 + } + ], + "pareto_models": [ + "rtdetrv2-r34", + "rtdetr-l", + "rtdetr-r101", + "rtdetrv2-r101", + "rtdetr-x", + "dfine-x" + ], + "budget_picks": { + "10": { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 55.8, + "fps": 122.0, + "ms": 8.2, + "params_millions": 0.0 + }, + "33": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.5, + "fps": 73.0, + "ms": 13.7, + "params_millions": 0.0 + }, + "100": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.5, + "fps": 73.0, + "ms": 13.7, + "params_millions": 0.0 + } + }, + "best_overall": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.5, + "fps": 73.0, + "ms": 13.7, + "params_millions": 0.0 + }, + "best_permissive": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.5, + "fps": 73.0, + "ms": 13.7, + "params_millions": 0.0 + }, + "fastest": { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 53.2, + "fps": 132.9, + "ms": 7.52, + "params_millions": 0.0 + } + }, + "claims": [ + { + "id": "best_overall", + "kind": "ranking", + "statement": "D-FINE-X leads on NVIDIA RTX 5070 Ti at 61.5 mAP@50-95 (73.0 FPS, TensorRT FP16).", + "values": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.5, + "fps": 73.0, + "ms": 13.7, + "params_millions": 0.0 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "fastest", + "kind": "ranking", + "statement": "RT-DETRv2-R34 is the fastest measured: 132.9 FPS at 53.2 mAP.", + "values": { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 53.2, + "fps": 132.9, + "ms": 7.52, + "params_millions": 0.0 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "pareto", + "kind": "ranking", + "statement": "The measured accuracy-speed frontier on NVIDIA RTX 5070 Ti is: RT-DETRv2-R34, RT-DETR-L, RT-DETR-R101, RT-DETRv2-R101, RT-DETR-X, D-FINE-X.", + "values": { + "pareto": [ + "rtdetrv2-r34", + "rtdetr-l", + "rtdetr-r101", + "rtdetrv2-r101", + "rtdetr-x", + "dfine-x" + ] + }, + "passed": true, + "strength": "normal" + }, + { + "id": "budget_10ms", + "kind": "budget", + "statement": "Under 10 ms/image on NVIDIA RTX 5070 Ti: RT-DETR-L is the most accurate (55.8 mAP at 8.2 ms).", + "values": { + "budget_ms": 10, + "pick": { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 55.8, + "fps": 122.0, + "ms": 8.2, + "params_millions": 0.0 + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "budget_33ms", + "kind": "budget", + "statement": "Under 33 ms/image on NVIDIA RTX 5070 Ti: D-FINE-X is the most accurate (61.5 mAP at 13.7 ms).", + "values": { + "budget_ms": 33, + "pick": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.5, + "fps": 73.0, + "ms": 13.7, + "params_millions": 0.0 + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "budget_100ms", + "kind": "budget", + "statement": "Under 100 ms/image on NVIDIA RTX 5070 Ti: D-FINE-X is the most accurate (61.5 mAP at 13.7 ms).", + "values": { + "budget_ms": 100, + "pick": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.5, + "fps": 73.0, + "ms": 13.7, + "params_millions": 0.0 + } + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 3, + "reason": null + }, + "allowed_numbers": [ + 0.0, + 0.001, + 0.6, + 0.91, + 1, + 2, + 2.04, + 3, + 4, + 5, + 5.06, + 6, + 7, + 7.23, + 7.52, + 7.57, + 8, + 8.2, + 8.49, + 8.82, + 8.85, + 8.97, + 9, + 9.08, + 9.22, + 9.65, + 9.99, + 10.0, + 10.23, + 10.27, + 10.84, + 11, + 11.36, + 11.44, + 11.54, + 11.55, + 12.0, + 12.04, + 12.14, + 12.15, + 12.3, + 12.54, + 12.62, + 12.73, + 12.82, + 12.83, + 13.39, + 13.42, + 13.7, + 13.84, + 13.88, + 13.96, + 14.07, + 14.11, + 14.15, + 14.42, + 15.21, + 15.66, + 15.91, + 16.96, + 17.14, + 19.84, + 20.12, + 20.46, + 21.01, + 23.02, + 24.0, + 24.22, + 25.33, + 25.5, + 25.8, + 25.86, + 26.32, + 26.4, + 28.7, + 30.4, + 33.0, + 35.4, + 37.3, + 37.9, + 38.0, + 38.7, + 41.3, + 41.7, + 41.8, + 43.4, + 43.5, + 44.0, + 45.8, + 46.6, + 46.7, + 47.6, + 48.9, + 49.8, + 50, + 50.4, + 50.5, + 50.8, + 51.0, + 51.4, + 51.6, + 51.9, + 52.3, + 52.8, + 53.0, + 53.2, + 53.9, + 54.21, + 54.4, + 54.7, + 55.3, + 55.4, + 55.7, + 55.8, + 56.1, + 56.3, + 56.4, + 56.7, + 56.8, + 57.2, + 57.4, + 57.6, + 57.7, + 57.9, + 58.2, + 58.3, + 58.5, + 58.6, + 59.0, + 59.5, + 59.9, + 60.0, + 60.8, + 61.0, + 61.5, + 62.9, + 63.8, + 65.7, + 69.3, + 70.7, + 70.9, + 71.0, + 71.6, + 72.1, + 72.3, + 73.0, + 74.5, + 74.7, + 75, + 78.0, + 78.6, + 79.2, + 79.7, + 81.3, + 82.3, + 82.4, + 83.0, + 83.3, + 86.5, + 86.7, + 87.4, + 88.0, + 91.0, + 92.3, + 95, + 97.4, + 97.7, + 99.07, + 100.0, + 103.6, + 108.4, + 110.1, + 113.0, + 113.4, + 117.8, + 122.0, + 128.0, + 132.1, + 132.9, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/best-object-detection-rpi5-hailo8.claims.json b/article-pipeline/v2/output/best-object-detection-rpi5-hailo8.claims.json new file mode 100644 index 0000000..bc75e9b --- /dev/null +++ b/article-pipeline/v2/output/best-object-detection-rpi5-hailo8.claims.json @@ -0,0 +1,370 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:28.188785+00:00", + "slug": "best-object-detection-rpi5-hailo8", + "meta": { + "type": "hardware-guide", + "hardware": "rpi5_hailo8", + "runtime": "hailo_int8" + }, + "facts": { + "hardware": "rpi5_hailo8", + "hardware_display": "Raspberry Pi 5 + Hailo-8", + "runtime": "hailo_int8", + "runtime_display": "HailoRT INT8", + "n_models": 10, + "leaderboard": [ + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 54.8, + "fps": 21.2, + "ms": 47.08, + "params_millions": 25.5 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "license": "MIT", + "mAP": 53.3, + "fps": 30.1, + "ms": 33.23, + "params_millions": 20.12 + }, + { + "model": "yolonas-l", + "displayName": "YOLO-NAS-L", + "family": "yolonas", + "license": "non-permissive", + "mAP": 52.4, + "fps": 15.4, + "ms": 64.89, + "params_millions": 66.97 + }, + { + "model": "yolonas-m", + "displayName": "YOLO-NAS-M", + "family": "yolonas", + "license": "non-permissive", + "mAP": 47.4, + "fps": 21.7, + "ms": 46.05, + "params_millions": 51.1 + }, + { + "model": "yolonas-s", + "displayName": "YOLO-NAS-S", + "family": "yolonas", + "license": "non-permissive", + "mAP": 44.5, + "fps": 39.7, + "ms": 25.2, + "params_millions": 19.02 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 43.7, + "fps": 34.8, + "ms": 28.76, + "params_millions": 25.33 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 41.1, + "fps": 95.8, + "ms": 10.44, + "params_millions": 8.97 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 33.5, + "fps": 195.6, + "ms": 5.11, + "params_millions": 5.06 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "license": "MIT", + "mAP": 32.0, + "fps": 36.3, + "ms": 27.52, + "params_millions": 7.23 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "license": "MIT", + "mAP": 25.7, + "fps": 76.5, + "ms": 13.08, + "params_millions": 2.04 + } + ], + "pareto_models": [ + "yolox-tiny", + "yolox-s", + "yolonas-s", + "yolov9m", + "yolov9c" + ], + "budget_picks": { + "50": { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 54.8, + "fps": 21.2, + "ms": 47.08, + "params_millions": 25.5 + }, + "100": { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 54.8, + "fps": 21.2, + "ms": 47.08, + "params_millions": 25.5 + }, + "500": { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 54.8, + "fps": 21.2, + "ms": 47.08, + "params_millions": 25.5 + } + }, + "best_overall": { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 54.8, + "fps": 21.2, + "ms": 47.08, + "params_millions": 25.5 + }, + "best_permissive": { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 54.8, + "fps": 21.2, + "ms": 47.08, + "params_millions": 25.5 + }, + "fastest": { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 33.5, + "fps": 195.6, + "ms": 5.11, + "params_millions": 5.06 + } + }, + "claims": [ + { + "id": "best_overall", + "kind": "ranking", + "statement": "YOLOv9-C leads on Raspberry Pi 5 + Hailo-8 at 54.8 mAP@50-95 (21.2 FPS, HailoRT INT8).", + "values": { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 54.8, + "fps": 21.2, + "ms": 47.08, + "params_millions": 25.5 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "fastest", + "kind": "ranking", + "statement": "YOLOX-Tiny is the fastest measured: 195.6 FPS at 33.5 mAP.", + "values": { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 33.5, + "fps": 195.6, + "ms": 5.11, + "params_millions": 5.06 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "pareto", + "kind": "ranking", + "statement": "The measured accuracy-speed frontier on Raspberry Pi 5 + Hailo-8 is: YOLOX-Tiny, YOLOX-S, YOLO-NAS-S, YOLOv9-M, YOLOv9-C.", + "values": { + "pareto": [ + "yolox-tiny", + "yolox-s", + "yolonas-s", + "yolov9m", + "yolov9c" + ] + }, + "passed": true, + "strength": "normal" + }, + { + "id": "budget_50ms", + "kind": "budget", + "statement": "Under 50 ms/image on Raspberry Pi 5 + Hailo-8: YOLOv9-C is the most accurate (54.8 mAP at 47.08 ms).", + "values": { + "budget_ms": 50, + "pick": { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 54.8, + "fps": 21.2, + "ms": 47.08, + "params_millions": 25.5 + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "budget_100ms", + "kind": "budget", + "statement": "Under 100 ms/image on Raspberry Pi 5 + Hailo-8: YOLOv9-C is the most accurate (54.8 mAP at 47.08 ms).", + "values": { + "budget_ms": 100, + "pick": { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 54.8, + "fps": 21.2, + "ms": 47.08, + "params_millions": 25.5 + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "budget_500ms", + "kind": "budget", + "statement": "Under 500 ms/image on Raspberry Pi 5 + Hailo-8: YOLOv9-C is the most accurate (54.8 mAP at 47.08 ms).", + "values": { + "budget_ms": 500, + "pick": { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 54.8, + "fps": 21.2, + "ms": 47.08, + "params_millions": 25.5 + } + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 3, + "reason": null + }, + "allowed_numbers": [ + 0.001, + 0.6, + 1, + 2, + 2.04, + 3, + 4, + 5, + 5.06, + 5.11, + 6, + 7, + 7.23, + 8, + 8.97, + 9, + 10.0, + 10.44, + 11, + 12, + 13.08, + 15.4, + 19.02, + 20.12, + 21.2, + 21.7, + 25.2, + 25.33, + 25.5, + 25.7, + 27.52, + 28.76, + 30.1, + 32.0, + 33.23, + 33.5, + 34.8, + 36.3, + 39.7, + 41.1, + 43.7, + 44.5, + 46.05, + 47.08, + 47.4, + 50.0, + 51.1, + 52.4, + 53.3, + 54.8, + 64.89, + 66.97, + 75, + 76.5, + 95, + 95.8, + 100.0, + 195.6, + 300, + 500.0, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/best-object-detection-rpi5.claims.json b/article-pipeline/v2/output/best-object-detection-rpi5.claims.json new file mode 100644 index 0000000..0a2c2df --- /dev/null +++ b/article-pipeline/v2/output/best-object-detection-rpi5.claims.json @@ -0,0 +1,982 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T21:56:12.168774+00:00", + "slug": "best-object-detection-rpi5", + "meta": { + "type": "hardware-guide", + "hardware": "rpi5", + "runtime": "onnx_fp32" + }, + "facts": { + "hardware": "rpi5", + "hardware_display": "Raspberry Pi 5", + "runtime": "onnx_fp32", + "runtime_display": "ONNX Runtime FP32", + "n_models": 55, + "leaderboard": [ + { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.4, + "fps": 0.4, + "ms": 2638.75, + "params_millions": 61.71 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 61.3, + "fps": 0.3, + "ms": 3453.03, + "params_millions": 50.27 + }, + { + "model": "ec-x", + "displayName": "ECDet-X", + "family": "ec", + "license": "Apache-2.0", + "mAP": 61.1, + "fps": 0.3, + "ms": 3331.42, + "params_millions": 48.99 + }, + { + "model": "ec-l", + "displayName": "ECDet-L", + "family": "ec", + "license": "Apache-2.0", + "mAP": 60.1, + "fps": 0.4, + "ms": 2681.8, + "params_millions": 32.69 + }, + { + "model": "dfine-l", + "displayName": "D-FINE-L", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 60.0, + "fps": 0.7, + "ms": 1402.91, + "params_millions": 30.79 + }, + { + "model": "rtdetrv4-x", + "displayName": "RT-DETRv4-X", + "family": "rtdetrv4", + "license": "Apache-2.0", + "mAP": 60.0, + "fps": 0.4, + "ms": 2685.68, + "params_millions": 62.6 + }, + { + "model": "deim-x", + "displayName": "DEIM-X", + "family": "deim", + "license": "Apache-2.0", + "mAP": 59.6, + "fps": 0.4, + "ms": 2635.56, + "params_millions": 61.71 + }, + { + "model": "rfdetr-l", + "displayName": "RF-DETR-L", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 58.6, + "fps": 0.5, + "ms": 1924.5, + "params_millions": 30.33 + }, + { + "model": "deimv2-l", + "displayName": "DEIMv2-L", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 58.6, + "fps": 0.4, + "ms": 2678.98, + "params_millions": 32.19 + }, + { + "model": "ec-m", + "displayName": "ECDet-M", + "family": "ec", + "license": "Apache-2.0", + "mAP": 58.4, + "fps": 0.6, + "ms": 1708.19, + "params_millions": 19.23 + }, + { + "model": "rtdetr-x", + "displayName": "RT-DETR-X", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 57.9, + "fps": 0.4, + "ms": 2801.48, + "params_millions": 67.33 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 57.8, + "fps": 1.0, + "ms": 992.43, + "params_millions": 19.26 + }, + { + "model": "deim-l", + "displayName": "DEIM-L", + "family": "deim", + "license": "Apache-2.0", + "mAP": 57.8, + "fps": 0.7, + "ms": 1400.24, + "params_millions": 30.79 + }, + { + "model": "rtdetrv4-l", + "displayName": "RT-DETRv4-L", + "family": "rtdetrv4", + "license": "Apache-2.0", + "mAP": 57.8, + "fps": 0.7, + "ms": 1427.4, + "params_millions": 31.19 + }, + { + "model": "rfdetr-m", + "displayName": "RF-DETR-M", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 57.4, + "fps": 0.9, + "ms": 1142.18, + "params_millions": 30.08 + }, + { + "model": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 56.7, + "fps": 0.3, + "ms": 3019.38, + "params_millions": 76.52 + }, + { + "model": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 56.7, + "fps": 0.3, + "ms": 3042.68, + "params_millions": 76.56 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "family": "rtdetrv4", + "license": "Apache-2.0", + "mAP": 56.5, + "fps": 1.0, + "ms": 1006.49, + "params_millions": 19.52 + }, + { + "model": "deimv2-m", + "displayName": "DEIMv2-M", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 56.0, + "fps": 0.6, + "ms": 1699.67, + "params_millions": 18.13 + }, + { + "model": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 55.9, + "fps": 0.6, + "ms": 1772.89, + "params_millions": 42.81 + }, + { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 55.8, + "fps": 0.7, + "ms": 1460.99, + "params_millions": 32.84 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "license": "MIT", + "mAP": 55.7, + "fps": 0.7, + "ms": 1398.3, + "params_millions": 25.47 + }, + { + "model": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 55.7, + "fps": 0.6, + "ms": 1785.3, + "params_millions": 42.85 + }, + { + "model": "deim-m", + "displayName": "DEIM-M", + "family": "deim", + "license": "Apache-2.0", + "mAP": 55.4, + "fps": 1.0, + "ms": 990.65, + "params_millions": 19.26 + }, + { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 55.1, + "fps": 1.2, + "ms": 850.1, + "params_millions": 28.51 + }, + { + "model": "yolox-x", + "displayName": "YOLOX-X", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 54.9, + "fps": 0.3, + "ms": 3200.47, + "params_millions": 99.02 + }, + { + "model": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 54.8, + "fps": 0.8, + "ms": 1326.47, + "params_millions": 33.17 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 54.3, + "fps": 0.5, + "ms": 1882.4, + "params_millions": 54.17 + }, + { + "model": "ec-s", + "displayName": "ECDet-S", + "family": "ec", + "license": "Apache-2.0", + "mAP": 54.3, + "fps": 0.9, + "ms": 1154.94, + "params_millions": 9.84 + }, + { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 53.8, + "fps": 0.8, + "ms": 1329.01, + "params_millions": 33.13 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "license": "MIT", + "mAP": 53.6, + "fps": 1.0, + "ms": 1039.93, + "params_millions": 20.1 + }, + { + "model": "dfine-s", + "displayName": "D-FINE-S", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 53.4, + "fps": 1.8, + "ms": 559.27, + "params_millions": 10.25 + }, + { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 53.2, + "fps": 0.9, + "ms": 1122.15, + "params_millions": 31.38 + }, + { + "model": "deimv2-s", + "displayName": "DEIMv2-S", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 53.0, + "fps": 0.9, + "ms": 1162.77, + "params_millions": 9.73 + }, + { + "model": "rtdetrv4-s", + "displayName": "RT-DETRv4-S", + "family": "rtdetrv4", + "license": "Apache-2.0", + "mAP": 52.8, + "fps": 1.8, + "ms": 562.51, + "params_millions": 10.29 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 52.2, + "fps": 0.9, + "ms": 1118.68, + "params_millions": 31.34 + }, + { + "model": "deim-s", + "displayName": "DEIM-S", + "family": "deim", + "license": "Apache-2.0", + "mAP": 52.1, + "fps": 1.8, + "ms": 559.18, + "params_millions": 10.25 + }, + { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 51.4, + "fps": 2.4, + "ms": 409.24, + "params_millions": 26.86 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 50.8, + "fps": 1.1, + "ms": 922.05, + "params_millions": 25.3 + }, + { + "model": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "family": "rtdetrv2", + "license": "Apache-2.0", + "mAP": 50.8, + "fps": 1.3, + "ms": 791.23, + "params_millions": 20.15 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "family": "rtdetr", + "license": "Apache-2.0", + "mAP": 49.8, + "fps": 1.3, + "ms": 786.55, + "params_millions": 20.11 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "license": "MIT", + "mAP": 48.5, + "fps": 2.4, + "ms": 425.08, + "params_millions": 7.21 + }, + { + "model": "deim-n", + "displayName": "DEIM-N", + "family": "deim", + "license": "Apache-2.0", + "mAP": 46.8, + "fps": 4.3, + "ms": 229.71, + "params_millions": 3.75 + }, + { + "model": "deimv2-n", + "displayName": "DEIMv2-N", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 46.7, + "fps": 4.4, + "ms": 226.26, + "params_millions": 3.56 + }, + { + "model": "dfine-n", + "displayName": "D-FINE-N", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 45.8, + "fps": 4.3, + "ms": 229.68, + "params_millions": 3.75 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 43.1, + "fps": 2.5, + "ms": 393.52, + "params_millions": 8.96 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "license": "Apache-2.0", + "mAP": 42.7, + "fps": 1.7, + "ms": 589.47, + "params_millions": 3.29 + }, + { + "model": "deimv2-pico", + "displayName": "DEIMv2-Pico", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 42.2, + "fps": 6.2, + "ms": 161.53, + "params_millions": 1.51 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "license": "MIT", + "mAP": 40.7, + "fps": 5.9, + "ms": 169.97, + "params_millions": 2.03 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "license": "Apache-2.0", + "mAP": 36.5, + "fps": 6.2, + "ms": 160.15, + "params_millions": 2.14 + }, + { + "model": "deimv2-femto", + "displayName": "DEIMv2-Femto", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 34.5, + "fps": 16.7, + "ms": 59.79, + "params_millions": 0.96 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 34.3, + "fps": 9.4, + "ms": 106.69, + "params_millions": 5.05 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "license": "Apache-2.0", + "mAP": 28.1, + "fps": 15.0, + "ms": 66.77, + "params_millions": 0.98 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 27.7, + "fps": 21.3, + "ms": 46.98, + "params_millions": 0.9 + }, + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 27.5, + "fps": 30.3, + "ms": 33.02, + "params_millions": 0.49 + } + ], + "pareto_models": [ + "deimv2-atto", + "yolox-nano", + "deimv2-femto", + "picodet-m", + "deimv2-pico", + "deimv2-n", + "deim-n", + "yolov9s", + "rfdetr-n", + "dfine-s", + "rfdetr-s", + "dfine-m", + "dfine-l", + "ec-l", + "dfine-x" + ], + "budget_picks": { + "50": { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 27.7, + "fps": 21.3, + "ms": 46.98, + "params_millions": 0.9 + }, + "100": { + "model": "deimv2-femto", + "displayName": "DEIMv2-Femto", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 34.5, + "fps": 16.7, + "ms": 59.79, + "params_millions": 0.96 + }, + "500": { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 51.4, + "fps": 2.4, + "ms": 409.24, + "params_millions": 26.86 + } + }, + "best_overall": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.4, + "fps": 0.4, + "ms": 2638.75, + "params_millions": 61.71 + }, + "best_permissive": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.4, + "fps": 0.4, + "ms": 2638.75, + "params_millions": 61.71 + }, + "fastest": { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 27.5, + "fps": 30.3, + "ms": 33.02, + "params_millions": 0.49 + } + }, + "claims": [ + { + "id": "best_overall", + "kind": "ranking", + "statement": "D-FINE-X leads on Raspberry Pi 5 at 61.4 mAP@50-95 (0.4 FPS, ONNX Runtime FP32).", + "values": { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "license": "Apache-2.0", + "mAP": 61.4, + "fps": 0.4, + "ms": 2638.75, + "params_millions": 61.71 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "fastest", + "kind": "ranking", + "statement": "DEIMv2-Atto is the fastest measured: 30.3 FPS at 27.5 mAP.", + "values": { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 27.5, + "fps": 30.3, + "ms": 33.02, + "params_millions": 0.49 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "pareto", + "kind": "ranking", + "statement": "The measured accuracy-speed frontier on Raspberry Pi 5 is: DEIMv2-Atto, YOLOX-Nano, DEIMv2-Femto, PicoDet-M, DEIMv2-Pico, DEIMv2-N, DEIM-N, YOLOv9-S, RF-DETR-N, D-FINE-S, RF-DETR-S, D-FINE-M, D-FINE-L, ECDet-L, D-FINE-X.", + "values": { + "pareto": [ + "deimv2-atto", + "yolox-nano", + "deimv2-femto", + "picodet-m", + "deimv2-pico", + "deimv2-n", + "deim-n", + "yolov9s", + "rfdetr-n", + "dfine-s", + "rfdetr-s", + "dfine-m", + "dfine-l", + "ec-l", + "dfine-x" + ] + }, + "passed": true, + "strength": "normal" + }, + { + "id": "budget_50ms", + "kind": "budget", + "statement": "Under 50 ms/image on Raspberry Pi 5: YOLOX-Nano is the most accurate (27.7 mAP at 46.98 ms).", + "values": { + "budget_ms": 50, + "pick": { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "license": "Apache-2.0", + "mAP": 27.7, + "fps": 21.3, + "ms": 46.98, + "params_millions": 0.9 + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "budget_100ms", + "kind": "budget", + "statement": "Under 100 ms/image on Raspberry Pi 5: DEIMv2-Femto is the most accurate (34.5 mAP at 59.79 ms).", + "values": { + "budget_ms": 100, + "pick": { + "model": "deimv2-femto", + "displayName": "DEIMv2-Femto", + "family": "deimv2", + "license": "Apache-2.0", + "mAP": 34.5, + "fps": 16.7, + "ms": 59.79, + "params_millions": 0.96 + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "budget_500ms", + "kind": "budget", + "statement": "Under 500 ms/image on Raspberry Pi 5: RF-DETR-N is the most accurate (51.4 mAP at 409.24 ms).", + "values": { + "budget_ms": 500, + "pick": { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "family": "rfdetr", + "license": "Apache-2.0", + "mAP": 51.4, + "fps": 2.4, + "ms": 409.24, + "params_millions": 26.86 + } + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 3, + "reason": null + }, + "allowed_numbers": [ + 0.001, + 0.3, + 0.4, + 0.49, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 0.96, + 0.98, + 1.0, + 1.1, + 1.2, + 1.3, + 1.51, + 1.7, + 1.8, + 2, + 2.03, + 2.14, + 2.4, + 2.5, + 3, + 3.29, + 3.56, + 3.75, + 4, + 4.3, + 4.4, + 5, + 5.05, + 5.9, + 6, + 6.2, + 7, + 7.21, + 8, + 8.96, + 9, + 9.4, + 9.73, + 9.84, + 10, + 10.25, + 10.29, + 11, + 12, + 15.0, + 16.7, + 18.13, + 19.23, + 19.26, + 19.52, + 20.1, + 20.11, + 20.15, + 21.3, + 25.3, + 25.47, + 26.86, + 27.5, + 27.7, + 28.1, + 28.51, + 30.0, + 30.08, + 30.3, + 30.33, + 30.79, + 31.19, + 31.34, + 31.38, + 32.19, + 32.69, + 32.84, + 33.02, + 33.13, + 33.17, + 34.3, + 34.5, + 36.5, + 40.0, + 40.7, + 42.2, + 42.7, + 42.81, + 42.85, + 43.1, + 45.8, + 46.7, + 46.8, + 46.98, + 48.5, + 48.99, + 49.0, + 49.8, + 50.0, + 50.27, + 50.8, + 51.4, + 52.1, + 52.2, + 52.8, + 53.0, + 53.2, + 53.4, + 53.6, + 53.8, + 54.17, + 54.3, + 54.8, + 54.9, + 55.0, + 55.1, + 55.4, + 55.7, + 55.8, + 55.9, + 56.0, + 56.5, + 56.7, + 57.4, + 57.8, + 57.9, + 58.4, + 58.6, + 59.6, + 59.79, + 60.0, + 60.1, + 61.1, + 61.3, + 61.4, + 61.71, + 62.6, + 66.77, + 67.33, + 70.0, + 75, + 76.52, + 76.56, + 80.0, + 90.0, + 95, + 96.0, + 98.0, + 99.02, + 100.0, + 106.69, + 160.15, + 161.53, + 169.97, + 226.26, + 229.68, + 229.71, + 300, + 393.52, + 409.24, + 425.08, + 500.0, + 559.18, + 559.27, + 562.51, + 589.47, + 640, + 786.55, + 791.23, + 850.1, + 922.05, + 990.65, + 992.43, + 1006.49, + 1039.93, + 1118.68, + 1122.15, + 1142.18, + 1154.94, + 1162.77, + 1326.47, + 1329.01, + 1398.3, + 1400.24, + 1402.91, + 1427.4, + 1460.99, + 1699.67, + 1708.19, + 1772.89, + 1785.3, + 1882.4, + 1924.5, + 2635.56, + 2638.75, + 2678.98, + 2681.8, + 2685.68, + 2801.48, + 3019.38, + 3042.68, + 3200.47, + 3331.42, + 3453.03, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-l-vs-rfdetr-l.claims.json b/article-pipeline/v2/output/deim-l-vs-rfdetr-l.claims.json new file mode 100644 index 0000000..4ff5949 --- /dev/null +++ b/article-pipeline/v2/output/deim-l-vs-rfdetr-l.claims.json @@ -0,0 +1,389 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:42.525938+00:00", + "slug": "deim-l-vs-rfdetr-l", + "meta": { + "type": "vs", + "a": "deim-l", + "b": "rfdetr-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-l", + "displayName": "DEIM-L", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5782621609188436, + "mAP_50": 0.7554692546710929, + "mAP_small": 0.45153510221580306, + "mAP_medium": 0.6352409300103963, + "mAP_large": 0.7456268178594337, + "fps_mean": 18.6, + "fps_p50": 19.9, + "ms_per_image": 53.759, + "inference_ms": 46.269, + "preprocess_ms": 6.228, + "postprocess_ms": 1.262, + "p95_ms": 69.775, + "peak_vram_mb": 245.2, + "peak_ram_mb": 0.0, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:37:04.298988Z" + } + }, + "b": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "license": "Apache-2.0", + "releaseDate": "2026-02-23", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5855194790573519, + "mAP_50": 0.7602544384315711, + "mAP_small": 0.40866936129290937, + "mAP_medium": 0.6398630646189177, + "mAP_large": 0.7508947139639672, + "fps_mean": 25.08, + "fps_p50": 26.03, + "ms_per_image": 39.865, + "inference_ms": 21.946, + "preprocess_ms": 15.635, + "postprocess_ms": 2.284, + "p95_ms": 45.271, + "peak_vram_mb": 222.7, + "peak_ram_mb": 0.0, + "params_millions": 33.93, + "gflops": 340.0, + "input_size": 704, + "config": { + "batch_size": 1, + "input_size": 704, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-17T17:15:09.275538Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -1.24, + "fps": -25.84, + "params": -7.93, + "flops": -73.24, + "peak_vram": 10.1 + }, + "map_pts": { + "a": 57.8, + "b": 58.6, + "delta": -0.8 + }, + "pair": { + "params_ratio": 1.09, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00635, + "b_mAP_per_GFLOP": 0.00172 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 57.8, + "fps": 50.5 + }, + "b": { + "mAP": 58.6, + "fps": 43.8 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 58.6, + "fps": 25.1 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 57.8, + "fps": 75.7 + }, + "b": { + "mAP": 58.6, + "fps": 45.1 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 57.8, + "fps": 4.8 + }, + "b": { + "mAP": 58.6, + "fps": 4.0 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 58.6, + "fps": 25.1 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 57.8, + "fps": 0.4 + }, + "b": { + "mAP": 58.6, + "fps": 0.5 + } + } + ], + "input_sizes": { + "a": 640, + "b": 704, + "match": false + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "RF-DETR-L measures 0.8 mAP points higher than DEIM-L on NVIDIA RTX 5070 Ti (PyTorch FP32): 57.8 vs 58.6 mAP@50-95.", + "values": { + "delta_pts": 0.8, + "a_map": 57.8, + "b_map": 58.6 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RF-DETR-L is 25.84% faster: 18.6 vs 25.1 FPS (53.76 vs 39.87 ms/image).", + "values": { + "delta_pct": 25.84, + "a_fps": 18.6, + "b_fps": 25.1 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "DEIM-L extracts 269.19% more mAP per GFLOP (0.00635 vs 0.00172).", + "values": { + "a": 0.00635, + "b": 0.00172, + "delta_pct": 269.19 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "RF-DETR-L uses less peak VRAM: 245 vs 223 MB.", + "values": { + "a_vram": 245, + "b_vram": 223 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "DEIM-L leads on small objects: 45.2 vs 40.9 mAP_small (10.49% relative gap).", + "values": { + "a": 45.2, + "b": 40.9, + "delta_pct": 10.49 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: ONNX Runtime FP32, runtime: TensorRT FP32, hardware: NVIDIA Jetson Orin Nano Super 8GB. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:ONNX Runtime FP32", + "runtime:TensorRT FP32", + "hardware:NVIDIA Jetson Orin Nano Super 8GB" + ] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "input_size_caveat", + "kind": "caveat", + "statement": "MANDATORY CAVEAT: DEIM-L runs at 640 px and RF-DETR-L at 704 px (each model at its author default). Say this in Setup and the verdict.", + "values": { + "a": 640, + "b": 704 + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 7, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -73.24, + -25.84, + -7.93, + -1.24, + -0.8, + 0.0, + 0.001, + 0.0017, + 0.0063, + 0.1, + 0.172, + 0.4, + 0.4087, + 0.4515, + 0.5, + 0.5783, + 0.5855, + 0.6, + 0.635, + 0.6352, + 0.6399, + 0.7456, + 0.7509, + 0.7555, + 0.7603, + 0.8, + 1.0, + 1.09, + 1.262, + 2, + 2.284, + 3, + 4.0, + 4.8, + 5, + 6, + 6.228, + 7, + 8, + 9, + 10, + 10.1, + 10.49, + 11, + 12, + 15.635, + 18.6, + 19.9, + 21.946, + 25.08, + 25.1, + 25.84, + 26.03, + 31.24, + 33.93, + 39.865, + 40.0, + 40.8669, + 40.9, + 43.8, + 45.1, + 45.1535, + 45.2, + 45.271, + 46.269, + 50.0, + 50.5, + 53.759, + 57.8, + 57.8262, + 58.5519, + 58.6, + 60.0, + 63.5241, + 63.9863, + 69.775, + 74.5627, + 75, + 75.0895, + 75.5469, + 75.7, + 76.0254, + 80.0, + 95, + 100.0, + 222.7, + 223.0, + 245.0, + 245.2, + 269.19, + 300.0, + 340.0, + 500.0, + 640.0, + 704.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-l-vs-rtdetr-r101.claims.json b/article-pipeline/v2/output/deim-l-vs-rtdetr-r101.claims.json new file mode 100644 index 0000000..d6d2039 --- /dev/null +++ b/article-pipeline/v2/output/deim-l-vs-rtdetr-r101.claims.json @@ -0,0 +1,359 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:42.604637+00:00", + "slug": "deim-l-vs-rtdetr-r101", + "meta": { + "type": "vs", + "a": "deim-l", + "b": "rtdetr-r101", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-l", + "displayName": "DEIM-L", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5782621609188436, + "mAP_50": 0.7554692546710929, + "mAP_small": 0.45153510221580306, + "mAP_medium": 0.6352409300103963, + "mAP_large": 0.7456268178594337, + "fps_mean": 18.6, + "fps_p50": 19.9, + "ms_per_image": 53.759, + "inference_ms": 46.269, + "preprocess_ms": 6.228, + "postprocess_ms": 1.262, + "p95_ms": 69.775, + "peak_vram_mb": 245.2, + "peak_ram_mb": 0.0, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:37:04.298988Z" + } + }, + "b": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5676679551135575, + "mAP_50": 0.7399443292361064, + "mAP_small": 0.39201568552303717, + "mAP_medium": 0.6204984188822439, + "mAP_large": 0.7176352255205386, + "fps_mean": 23.57, + "fps_p50": 24.34, + "ms_per_image": 42.434, + "inference_ms": 37.87, + "preprocess_ms": 3.486, + "postprocess_ms": 1.077, + "p95_ms": 50.804, + "peak_vram_mb": 455.5, + "peak_ram_mb": 0.0, + "params_millions": 76.56, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:28:05.081801Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 1.87, + "fps": -21.09, + "params": -59.2, + "flops": -64.86, + "peak_vram": -46.17 + }, + "map_pts": { + "a": 57.8, + "b": 56.8, + "delta": 1.0 + }, + "pair": { + "params_ratio": 2.45, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00635, + "b_mAP_per_GFLOP": 0.00219 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 57.8, + "fps": 50.5 + }, + "b": { + "mAP": 56.8, + "fps": 47.5 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 56.8, + "fps": 23.6 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 57.8, + "fps": 75.7 + }, + "b": { + "mAP": 56.8, + "fps": 64.4 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 57.8, + "fps": 4.8 + }, + "b": { + "mAP": 56.8, + "fps": 2.8 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 56.8, + "fps": 23.6 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "DEIM-L measures 1.0 mAP points higher than RT-DETR-R101 on NVIDIA RTX 5070 Ti (PyTorch FP32): 57.8 vs 56.8 mAP@50-95.", + "values": { + "delta_pts": 1.0, + "a_map": 57.8, + "b_map": 56.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R101 is 21.09% faster: 18.6 vs 23.6 FPS (53.76 vs 42.43 ms/image).", + "values": { + "delta_pct": 21.09, + "a_fps": 18.6, + "b_fps": 23.6 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "DEIM-L extracts 189.95% more mAP per GFLOP (0.00635 vs 0.00219).", + "values": { + "a": 0.00635, + "b": 0.00219, + "delta_pct": 189.95 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "DEIM-L uses less peak VRAM: 245 vs 456 MB.", + "values": { + "a_vram": 245, + "b_vram": 456 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "DEIM-L leads on small objects: 45.2 vs 39.2 mAP_small (15.18% relative gap).", + "values": { + "a": 45.2, + "b": 39.2, + "delta_pct": 15.18 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: ONNX Runtime FP32, runtime: TensorRT FP32, hardware: NVIDIA Jetson Orin Nano Super 8GB. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:ONNX Runtime FP32", + "runtime:TensorRT FP32", + "hardware:NVIDIA Jetson Orin Nano Super 8GB" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -64.86, + -59.2, + -46.17, + -21.09, + 0.0, + 0.001, + 0.0022, + 0.0063, + 0.1, + 0.219, + 0.392, + 0.4515, + 0.5677, + 0.5783, + 0.6, + 0.6205, + 0.635, + 0.6352, + 0.7176, + 0.7399, + 0.7456, + 0.7555, + 1.0, + 1.077, + 1.262, + 1.87, + 2, + 2.45, + 2.8, + 3, + 3.486, + 4, + 4.8, + 5, + 6, + 6.228, + 7, + 8, + 9, + 10, + 11, + 12, + 15.18, + 18.6, + 19.9, + 21.09, + 23.57, + 23.6, + 24.34, + 31.24, + 37.87, + 39.2, + 39.2016, + 42.434, + 45.1535, + 45.2, + 46.269, + 47.5, + 50, + 50.5, + 50.804, + 53.759, + 56.7668, + 56.8, + 57.8, + 57.8262, + 60.0, + 62.0498, + 63.5241, + 64.4, + 69.775, + 71.7635, + 73.9944, + 74.5627, + 75, + 75.5469, + 75.7, + 76.56, + 95, + 100.0, + 189.95, + 245.0, + 245.2, + 300.0, + 455.5, + 456.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-l-vs-rtdetr-r50.claims.json b/article-pipeline/v2/output/deim-l-vs-rtdetr-r50.claims.json new file mode 100644 index 0000000..60008d8 --- /dev/null +++ b/article-pipeline/v2/output/deim-l-vs-rtdetr-r50.claims.json @@ -0,0 +1,376 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:42.678567+00:00", + "slug": "deim-l-vs-rtdetr-r50", + "meta": { + "type": "vs", + "a": "deim-l", + "b": "rtdetr-r50", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-l", + "displayName": "DEIM-L", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5782621609188436, + "mAP_50": 0.7554692546710929, + "mAP_small": 0.45153510221580306, + "mAP_medium": 0.6352409300103963, + "mAP_large": 0.7456268178594337, + "fps_mean": 18.6, + "fps_p50": 19.9, + "ms_per_image": 53.759, + "inference_ms": 46.269, + "preprocess_ms": 6.228, + "postprocess_ms": 1.262, + "p95_ms": 69.775, + "peak_vram_mb": 245.2, + "peak_ram_mb": 0.0, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:37:04.298988Z" + } + }, + "b": { + "id": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5587851459971446, + "mAP_50": 0.730485227930135, + "mAP_small": 0.3826303200265574, + "mAP_medium": 0.6127541876937886, + "mAP_large": 0.7053415491294674, + "fps_mean": 29.76, + "fps_p50": 30.87, + "ms_per_image": 33.598, + "inference_ms": 29.052, + "preprocess_ms": 3.448, + "postprocess_ms": 1.099, + "p95_ms": 39.857, + "peak_vram_mb": 326.4, + "peak_ram_mb": 0.0, + "params_millions": 42.89, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:24:24.814896Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 3.49, + "fps": -37.5, + "params": -27.16, + "flops": -33.09, + "peak_vram": -24.88 + }, + "map_pts": { + "a": 57.8, + "b": 55.9, + "delta": 1.9 + }, + "pair": { + "params_ratio": 1.37, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00635, + "b_mAP_per_GFLOP": 0.00411 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 57.8, + "fps": 50.5 + }, + "b": { + "mAP": 55.9, + "fps": 62.8 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 55.9, + "fps": 29.8 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 57.8, + "fps": 75.7 + }, + "b": { + "mAP": 55.8, + "fps": 87.4 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 57.8, + "fps": 4.8 + }, + "b": { + "mAP": 55.9, + "fps": 4.3 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 55.9, + "fps": 29.8 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 57.8, + "fps": 0.4 + }, + "b": { + "mAP": 55.9, + "fps": 0.3 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "DEIM-L measures 1.9 mAP points higher than RT-DETR-R50 on NVIDIA RTX 5070 Ti (PyTorch FP32): 57.8 vs 55.9 mAP@50-95.", + "values": { + "delta_pts": 1.9, + "a_map": 57.8, + "b_map": 55.9 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R50 is 37.5% faster: 18.6 vs 29.8 FPS (53.76 vs 33.6 ms/image).", + "values": { + "delta_pct": 37.5, + "a_fps": 18.6, + "b_fps": 29.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "DEIM-L extracts 54.5% more mAP per GFLOP (0.00635 vs 0.00411).", + "values": { + "a": 0.00635, + "b": 0.00411, + "delta_pct": 54.5 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "DEIM-L uses less peak VRAM: 245 vs 326 MB.", + "values": { + "a_vram": 245, + "b_vram": 326 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "DEIM-L leads on small objects: 45.2 vs 38.3 mAP_small (18.01% relative gap).", + "values": { + "a": 45.2, + "b": 38.3, + "delta_pct": 18.01 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: NVIDIA Jetson Orin Nano Super 8GB, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:NVIDIA Jetson Orin Nano Super 8GB", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -37.5, + -33.09, + -27.16, + -24.88, + 0.0, + 0.001, + 0.0041, + 0.0063, + 0.1, + 0.3, + 0.3826, + 0.4, + 0.411, + 0.4515, + 0.5588, + 0.5783, + 0.6, + 0.6128, + 0.635, + 0.6352, + 0.7053, + 0.7305, + 0.7456, + 0.7555, + 1.0, + 1.099, + 1.262, + 1.37, + 1.9, + 2, + 3, + 3.448, + 3.49, + 4, + 4.3, + 4.8, + 5, + 6, + 6.228, + 7, + 8, + 9, + 10, + 11, + 12, + 18.01, + 18.6, + 19.9, + 29.052, + 29.76, + 29.8, + 30.0, + 30.87, + 31.24, + 33.598, + 37.5, + 38.263, + 38.3, + 39.857, + 40.0, + 42.89, + 45.1535, + 45.2, + 46.269, + 50, + 50.5, + 53.759, + 54.5, + 55.8, + 55.8785, + 55.9, + 57.8, + 57.8262, + 60.0, + 61.2754, + 62.8, + 63.5241, + 69.775, + 70.5342, + 73.0485, + 74.5627, + 75, + 75.5469, + 75.7, + 87.4, + 95, + 100.0, + 245.0, + 245.2, + 300.0, + 326.0, + 326.4, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-l-vs-rtdetr-r50m.claims.json b/article-pipeline/v2/output/deim-l-vs-rtdetr-r50m.claims.json new file mode 100644 index 0000000..864edb6 --- /dev/null +++ b/article-pipeline/v2/output/deim-l-vs-rtdetr-r50m.claims.json @@ -0,0 +1,355 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.637814+00:00", + "slug": "deim-l-vs-rtdetr-r50m", + "meta": { + "type": "vs", + "a": "deim-l", + "b": "rtdetr-r50m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-l", + "displayName": "DEIM-L", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5782621609188436, + "mAP_50": 0.7554692546710929, + "mAP_small": 0.45153510221580306, + "mAP_medium": 0.6352409300103963, + "mAP_large": 0.7456268178594337, + "fps_mean": 18.6, + "fps_p50": 19.9, + "ms_per_image": 53.759, + "inference_ms": 46.269, + "preprocess_ms": 6.228, + "postprocess_ms": 1.262, + "p95_ms": 69.775, + "peak_vram_mb": 245.2, + "peak_ram_mb": 0.0, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:37:04.298988Z" + } + }, + "b": { + "id": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "license": "Apache-2.0", + "releaseDate": "2026-04-22", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5381740345956989, + "mAP_50": 0.7122612376832924, + "mAP_small": 0.37622664093206376, + "mAP_medium": 0.5945722582993921, + "mAP_large": 0.6898610320313556, + "fps_mean": 35.26, + "fps_p50": 36.57, + "ms_per_image": 28.362, + "inference_ms": 23.897, + "preprocess_ms": 3.41, + "postprocess_ms": 1.054, + "p95_ms": 34.582, + "peak_vram_mb": 301.7, + "peak_ram_mb": 0.0, + "params_millions": 36.59, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:24:50.077554Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 7.45, + "fps": -47.25, + "params": -14.62, + "flops": null, + "peak_vram": -18.73 + }, + "map_pts": { + "a": 57.8, + "b": 53.8, + "delta": 4.0 + }, + "pair": { + "params_ratio": 1.17, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00635, + "b_mAP_per_GFLOP": null + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 57.8, + "fps": 50.5 + }, + "b": { + "mAP": 53.8, + "fps": 76.4 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 53.8, + "fps": 35.3 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 57.8, + "fps": 75.7 + }, + "b": { + "mAP": 53.9, + "fps": 102.8 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 57.8, + "fps": 4.8 + }, + "b": { + "mAP": 53.8, + "fps": 5.0 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 53.8, + "fps": 35.3 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 57.8, + "fps": 0.4 + }, + "b": { + "mAP": 53.8, + "fps": 0.4 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "DEIM-L measures 4.0 mAP points higher than RT-DETR-R50m on NVIDIA RTX 5070 Ti (PyTorch FP32): 57.8 vs 53.8 mAP@50-95.", + "values": { + "delta_pts": 4.0, + "a_map": 57.8, + "b_map": 53.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R50m is 47.25% faster: 18.6 vs 35.3 FPS (53.76 vs 28.36 ms/image).", + "values": { + "delta_pct": 47.25, + "a_fps": 18.6, + "b_fps": 35.3 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "DEIM-L uses less peak VRAM: 245 vs 302 MB.", + "values": { + "a_vram": 245, + "b_vram": 302 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "DEIM-L leads on small objects: 45.2 vs 37.6 mAP_small (20.02% relative gap).", + "values": { + "a": 45.2, + "b": 37.6, + "delta_pct": 20.02 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -47.25, + -18.73, + -14.62, + 0.0, + 0.001, + 0.0063, + 0.1, + 0.3762, + 0.4, + 0.4515, + 0.5382, + 0.5783, + 0.5946, + 0.6, + 0.635, + 0.6352, + 0.6899, + 0.7123, + 0.7456, + 0.7555, + 1.0, + 1.054, + 1.17, + 1.262, + 2, + 3, + 3.41, + 4.0, + 4.8, + 5.0, + 6, + 6.228, + 7, + 7.45, + 8, + 9, + 10, + 11, + 12, + 18.6, + 19.9, + 20.02, + 23.897, + 28.362, + 31.24, + 34.582, + 35.26, + 35.3, + 36.57, + 36.59, + 37.6, + 37.6227, + 40.0, + 45.1535, + 45.2, + 46.269, + 47.25, + 50, + 50.5, + 53.759, + 53.8, + 53.8174, + 53.9, + 57.8, + 57.8262, + 59.4572, + 60.0, + 63.5241, + 68.9861, + 69.775, + 71.2261, + 74.5627, + 75, + 75.5469, + 75.7, + 76.4, + 95, + 100.0, + 102.8, + 245.0, + 245.2, + 300.0, + 301.7, + 302.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-l-vs-rtdetr-x.claims.json b/article-pipeline/v2/output/deim-l-vs-rtdetr-x.claims.json new file mode 100644 index 0000000..1286fad --- /dev/null +++ b/article-pipeline/v2/output/deim-l-vs-rtdetr-x.claims.json @@ -0,0 +1,358 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.729481+00:00", + "slug": "deim-l-vs-rtdetr-x", + "meta": { + "type": "vs", + "a": "deim-l", + "b": "rtdetr-x", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-l", + "displayName": "DEIM-L", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5782621609188436, + "mAP_50": 0.7554692546710929, + "mAP_small": 0.45153510221580306, + "mAP_medium": 0.6352409300103963, + "mAP_large": 0.7456268178594337, + "fps_mean": 18.6, + "fps_p50": 19.9, + "ms_per_image": 53.759, + "inference_ms": 46.269, + "preprocess_ms": 6.228, + "postprocess_ms": 1.262, + "p95_ms": 69.775, + "peak_vram_mb": 245.2, + "peak_ram_mb": 0.0, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:37:04.298988Z" + } + }, + "b": { + "id": "rtdetr-x", + "displayName": "RT-DETR-X", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5793982773908788, + "mAP_50": 0.760282028028533, + "mAP_small": 0.3976041463870124, + "mAP_medium": 0.6439424553072328, + "mAP_large": 0.7254662911881059, + "fps_mean": 21.76, + "fps_p50": 22.25, + "ms_per_image": 45.952, + "inference_ms": 41.266, + "preprocess_ms": 3.58, + "postprocess_ms": 1.106, + "p95_ms": 52.564, + "peak_vram_mb": 400.8, + "peak_ram_mb": 5.1, + "params_millions": 67.37, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:29:12.501490Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -0.2, + "fps": -14.52, + "params": -53.63, + "flops": -61.11, + "peak_vram": -38.82 + }, + "map_pts": { + "a": 57.8, + "b": 57.9, + "delta": -0.1 + }, + "pair": { + "params_ratio": 2.16, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00635, + "b_mAP_per_GFLOP": 0.00248 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 57.8, + "fps": 50.5 + }, + "b": { + "mAP": 57.9, + "fps": 48.4 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 57.9, + "fps": 21.8 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 57.8, + "fps": 75.7 + }, + "b": { + "mAP": 57.9, + "fps": 68.5 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 57.8, + "fps": 4.8 + }, + "b": { + "mAP": 57.9, + "fps": 3.0 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 57.9, + "fps": 21.8 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_tie", + "kind": "accuracy", + "statement": "Accuracy is a tie: 57.8 vs 57.9 mAP@50-95 (0.1 points apart, below the 0.5-point threshold).", + "values": { + "a_map": 57.8, + "b_map": 57.9 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-X is 14.52% faster: 18.6 vs 21.8 FPS (53.76 vs 45.95 ms/image).", + "values": { + "delta_pct": 14.52, + "a_fps": 18.6, + "b_fps": 21.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "DEIM-L extracts 156.05% more mAP per GFLOP (0.00635 vs 0.00248).", + "values": { + "a": 0.00635, + "b": 0.00248, + "delta_pct": 156.05 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "DEIM-L uses less peak VRAM: 245 vs 401 MB.", + "values": { + "a_vram": 245, + "b_vram": 401 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "DEIM-L leads on small objects: 45.2 vs 39.8 mAP_small (13.56% relative gap).", + "values": { + "a": 45.2, + "b": 39.8, + "delta_pct": 13.56 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: TensorRT FP32, hardware: NVIDIA Jetson Orin Nano Super 8GB. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:TensorRT FP32", + "hardware:NVIDIA Jetson Orin Nano Super 8GB" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -61.11, + -53.63, + -38.82, + -14.52, + -0.2, + -0.1, + 0.0, + 0.001, + 0.0025, + 0.0063, + 0.1, + 0.248, + 0.3976, + 0.4515, + 0.5783, + 0.5794, + 0.6, + 0.635, + 0.6352, + 0.6439, + 0.7255, + 0.7456, + 0.7555, + 0.7603, + 1.0, + 1.106, + 1.262, + 2, + 2.16, + 3.0, + 3.58, + 4, + 4.8, + 5, + 5.1, + 6, + 6.228, + 7, + 8, + 9, + 10, + 11, + 12, + 13.56, + 14.52, + 18.6, + 19.9, + 21.76, + 21.8, + 22.25, + 31.24, + 39.7604, + 39.8, + 41.266, + 45.1535, + 45.2, + 45.952, + 46.269, + 48.4, + 50, + 50.5, + 52.564, + 53.759, + 57.8, + 57.8262, + 57.9, + 57.9398, + 60.0, + 63.5241, + 64.3942, + 67.37, + 68.5, + 69.775, + 72.5466, + 74.5627, + 75, + 75.5469, + 75.7, + 76.0282, + 95, + 100.0, + 156.05, + 245.0, + 245.2, + 300.0, + 400.8, + 401.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-l-vs-rtdetrv2-r101.claims.json b/article-pipeline/v2/output/deim-l-vs-rtdetrv2-r101.claims.json new file mode 100644 index 0000000..2aed59c --- /dev/null +++ b/article-pipeline/v2/output/deim-l-vs-rtdetrv2-r101.claims.json @@ -0,0 +1,358 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:44.312336+00:00", + "slug": "deim-l-vs-rtdetrv2-r101", + "meta": { + "type": "vs", + "a": "deim-l", + "b": "rtdetrv2-r101", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-l", + "displayName": "DEIM-L", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5782621609188436, + "mAP_50": 0.7554692546710929, + "mAP_small": 0.45153510221580306, + "mAP_medium": 0.6352409300103963, + "mAP_large": 0.7456268178594337, + "fps_mean": 18.6, + "fps_p50": 19.9, + "ms_per_image": 53.759, + "inference_ms": 46.269, + "preprocess_ms": 6.228, + "postprocess_ms": 1.262, + "p95_ms": 69.775, + "peak_vram_mb": 245.2, + "peak_ram_mb": 0.0, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:37:04.298988Z" + } + }, + "b": { + "id": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "license": "Apache-2.0", + "releaseDate": "2024-05-01", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5676679344811996, + "mAP_50": 0.7399443292361064, + "mAP_small": 0.39201544668877547, + "mAP_medium": 0.6204984188822439, + "mAP_large": 0.7176352255205386, + "fps_mean": 25.67, + "fps_p50": 25.81, + "ms_per_image": 38.956, + "inference_ms": 34.323, + "preprocess_ms": 3.602, + "postprocess_ms": 1.032, + "p95_ms": 45.662, + "peak_vram_mb": 455.7, + "peak_ram_mb": 0.0, + "params_millions": 76.56, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:31:17.900165Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 1.87, + "fps": -27.54, + "params": -59.2, + "flops": -64.86, + "peak_vram": -46.19 + }, + "map_pts": { + "a": 57.8, + "b": 56.8, + "delta": 1.0 + }, + "pair": { + "params_ratio": 2.45, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00635, + "b_mAP_per_GFLOP": 0.00219 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 57.8, + "fps": 50.5 + }, + "b": { + "mAP": 56.8, + "fps": 49.3 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 56.8, + "fps": 25.7 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 57.8, + "fps": 75.7 + }, + "b": { + "mAP": 56.8, + "fps": 64.8 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 57.8, + "fps": 4.8 + }, + "b": { + "mAP": 56.8, + "fps": 2.8 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 56.8, + "fps": 25.7 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "DEIM-L measures 1.0 mAP points higher than RT-DETRv2-R101 on NVIDIA RTX 5070 Ti (PyTorch FP32): 57.8 vs 56.8 mAP@50-95.", + "values": { + "delta_pts": 1.0, + "a_map": 57.8, + "b_map": 56.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETRv2-R101 is 27.54% faster: 18.6 vs 25.7 FPS (53.76 vs 38.96 ms/image).", + "values": { + "delta_pct": 27.54, + "a_fps": 18.6, + "b_fps": 25.7 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "DEIM-L extracts 189.95% more mAP per GFLOP (0.00635 vs 0.00219).", + "values": { + "a": 0.00635, + "b": 0.00219, + "delta_pct": 189.95 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "DEIM-L uses less peak VRAM: 245 vs 456 MB.", + "values": { + "a_vram": 245, + "b_vram": 456 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "DEIM-L leads on small objects: 45.2 vs 39.2 mAP_small (15.18% relative gap).", + "values": { + "a": 45.2, + "b": 39.2, + "delta_pct": 15.18 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: TensorRT FP32, hardware: NVIDIA Jetson Orin Nano Super 8GB. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:TensorRT FP32", + "hardware:NVIDIA Jetson Orin Nano Super 8GB" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -64.86, + -59.2, + -46.19, + -27.54, + 0.0, + 0.001, + 0.0022, + 0.0063, + 0.1, + 0.219, + 0.392, + 0.4515, + 0.5677, + 0.5783, + 0.6, + 0.6205, + 0.635, + 0.6352, + 0.7176, + 0.7399, + 0.7456, + 0.7555, + 1.0, + 1.032, + 1.262, + 1.87, + 2, + 2.45, + 2.8, + 3, + 3.602, + 4, + 4.8, + 5, + 6, + 6.228, + 7, + 8, + 9, + 10, + 11, + 12, + 15.18, + 18.6, + 19.9, + 25.67, + 25.7, + 25.81, + 27.54, + 31.24, + 34.323, + 38.956, + 39.2, + 39.2015, + 45.1535, + 45.2, + 45.662, + 46.269, + 49.3, + 50, + 50.5, + 53.759, + 56.7668, + 56.8, + 57.8, + 57.8262, + 60.0, + 62.0498, + 63.5241, + 64.8, + 69.775, + 71.7635, + 73.9944, + 74.5627, + 75, + 75.5469, + 75.7, + 76.56, + 95, + 100.0, + 189.95, + 245.0, + 245.2, + 300.0, + 455.7, + 456.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-l-vs-rtdetrv2-r50.claims.json b/article-pipeline/v2/output/deim-l-vs-rtdetrv2-r50.claims.json new file mode 100644 index 0000000..b1041ac --- /dev/null +++ b/article-pipeline/v2/output/deim-l-vs-rtdetrv2-r50.claims.json @@ -0,0 +1,375 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:44.394744+00:00", + "slug": "deim-l-vs-rtdetrv2-r50", + "meta": { + "type": "vs", + "a": "deim-l", + "b": "rtdetrv2-r50", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-l", + "displayName": "DEIM-L", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5782621609188436, + "mAP_50": 0.7554692546710929, + "mAP_small": 0.45153510221580306, + "mAP_medium": 0.6352409300103963, + "mAP_large": 0.7456268178594337, + "fps_mean": 18.6, + "fps_p50": 19.9, + "ms_per_image": 53.759, + "inference_ms": 46.269, + "preprocess_ms": 6.228, + "postprocess_ms": 1.262, + "p95_ms": 69.775, + "peak_vram_mb": 245.2, + "peak_ram_mb": 0.0, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:37:04.298988Z" + } + }, + "b": { + "id": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "license": "Apache-2.0", + "releaseDate": "2024-05-01", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5571064919084201, + "mAP_50": 0.7284079773165765, + "mAP_small": 0.38918755047843295, + "mAP_medium": 0.6039164248992706, + "mAP_large": 0.7190851732735052, + "fps_mean": 32.11, + "fps_p50": 33.22, + "ms_per_image": 31.147, + "inference_ms": 26.733, + "preprocess_ms": 3.372, + "postprocess_ms": 1.042, + "p95_ms": 35.569, + "peak_vram_mb": 326.6, + "peak_ram_mb": 0.0, + "params_millions": 42.89, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:30:22.449507Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 3.8, + "fps": -42.07, + "params": -27.16, + "flops": -33.09, + "peak_vram": -24.92 + }, + "map_pts": { + "a": 57.8, + "b": 55.7, + "delta": 2.1 + }, + "pair": { + "params_ratio": 1.37, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00635, + "b_mAP_per_GFLOP": 0.0041 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 57.8, + "fps": 50.5 + }, + "b": { + "mAP": 55.7, + "fps": 66.0 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 55.7, + "fps": 32.1 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 57.8, + "fps": 75.7 + }, + "b": { + "mAP": 55.7, + "fps": 88.5 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 57.8, + "fps": 4.8 + }, + "b": { + "mAP": 55.7, + "fps": 4.3 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 55.7, + "fps": 32.1 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 57.8, + "fps": 0.4 + }, + "b": { + "mAP": 55.7, + "fps": 0.3 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "DEIM-L measures 2.1 mAP points higher than RT-DETRv2-R50 on NVIDIA RTX 5070 Ti (PyTorch FP32): 57.8 vs 55.7 mAP@50-95.", + "values": { + "delta_pts": 2.1, + "a_map": 57.8, + "b_map": 55.7 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETRv2-R50 is 42.07% faster: 18.6 vs 32.1 FPS (53.76 vs 31.15 ms/image).", + "values": { + "delta_pct": 42.07, + "a_fps": 18.6, + "b_fps": 32.1 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "DEIM-L extracts 54.88% more mAP per GFLOP (0.00635 vs 0.0041).", + "values": { + "a": 0.00635, + "b": 0.0041, + "delta_pct": 54.88 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "DEIM-L uses less peak VRAM: 245 vs 327 MB.", + "values": { + "a_vram": 245, + "b_vram": 327 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "DEIM-L leads on small objects: 45.2 vs 38.9 mAP_small (16.02% relative gap).", + "values": { + "a": 45.2, + "b": 38.9, + "delta_pct": 16.02 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: NVIDIA Jetson Orin Nano Super 8GB, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:NVIDIA Jetson Orin Nano Super 8GB", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -42.07, + -33.09, + -27.16, + -24.92, + 0.0, + 0.001, + 0.0041, + 0.0063, + 0.1, + 0.3, + 0.3892, + 0.4, + 0.41, + 0.4515, + 0.5571, + 0.5783, + 0.6, + 0.6039, + 0.635, + 0.6352, + 0.7191, + 0.7284, + 0.7456, + 0.7555, + 1.0, + 1.042, + 1.262, + 1.37, + 2, + 2.1, + 3, + 3.372, + 3.8, + 4, + 4.3, + 4.8, + 5, + 6, + 6.228, + 7, + 8, + 9, + 10, + 11, + 12, + 16.02, + 18.6, + 19.9, + 26.733, + 30.0, + 31.147, + 31.24, + 32.1, + 32.11, + 33.22, + 35.569, + 38.9, + 38.9188, + 40.0, + 42.07, + 42.89, + 45.1535, + 45.2, + 46.269, + 50, + 50.5, + 53.759, + 54.88, + 55.7, + 55.7106, + 57.8, + 57.8262, + 60.0, + 60.3916, + 63.5241, + 66.0, + 69.775, + 71.9085, + 72.8408, + 74.5627, + 75, + 75.5469, + 75.7, + 88.5, + 95, + 100.0, + 245.0, + 245.2, + 300.0, + 326.6, + 327.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-l-vs-yolox-l.claims.json b/article-pipeline/v2/output/deim-l-vs-yolox-l.claims.json new file mode 100644 index 0000000..7a934a5 --- /dev/null +++ b/article-pipeline/v2/output/deim-l-vs-yolox-l.claims.json @@ -0,0 +1,361 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.808155+00:00", + "slug": "deim-l-vs-yolox-l", + "meta": { + "type": "vs", + "a": "deim-l", + "b": "yolox-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-l", + "displayName": "DEIM-L", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5782621609188436, + "mAP_50": 0.7554692546710929, + "mAP_small": 0.45153510221580306, + "mAP_medium": 0.6352409300103963, + "mAP_large": 0.7456268178594337, + "fps_mean": 18.6, + "fps_p50": 19.9, + "ms_per_image": 53.759, + "inference_ms": 46.269, + "preprocess_ms": 6.228, + "postprocess_ms": 1.262, + "p95_ms": 69.775, + "peak_vram_mb": 245.2, + "peak_ram_mb": 0.0, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:37:04.298988Z" + } + }, + "b": { + "id": "yolox-l", + "displayName": "YOLOX-L", + "license": "Apache-2.0", + "releaseDate": "2021-07-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5542460212991265, + "mAP_50": 0.7302189854897115, + "mAP_small": 0.39036786287091835, + "mAP_medium": 0.5988447680065735, + "mAP_large": 0.685177367489107, + "fps_mean": 42.84, + "fps_p50": 42.78, + "ms_per_image": 23.341, + "inference_ms": 13.911, + "preprocess_ms": 6.863, + "postprocess_ms": 2.567, + "p95_ms": 27.837, + "peak_vram_mb": 307.0, + "peak_ram_mb": 0.0, + "params_millions": 54.21, + "gflops": 78.01, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:20:38.605152Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 4.33, + "fps": -56.58, + "params": -42.37, + "flops": 16.65, + "peak_vram": -20.13 + }, + "map_pts": { + "a": 57.8, + "b": 55.4, + "delta": 2.4 + }, + "pair": { + "params_ratio": 1.74, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00635, + "b_mAP_per_GFLOP": 0.0071 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 57.8, + "fps": 50.5 + }, + "b": { + "mAP": 55.4, + "fps": 50.6 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 55.4, + "fps": 42.8 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 57.8, + "fps": 75.7 + }, + "b": { + "mAP": 55.4, + "fps": 61.8 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 57.8, + "fps": 4.8 + }, + "b": { + "mAP": 55.4, + "fps": 5.8 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 57.8, + "fps": 18.6 + }, + "b": { + "mAP": 55.4, + "fps": 42.8 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 57.8, + "fps": 0.4 + }, + "b": { + "mAP": 53.9, + "fps": 0.4 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "DEIM-L measures 2.4 mAP points higher than YOLOX-L on NVIDIA RTX 5070 Ti (PyTorch FP32): 57.8 vs 55.4 mAP@50-95.", + "values": { + "delta_pts": 2.4, + "a_map": 57.8, + "b_map": 55.4 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "YOLOX-L is 56.58% faster: 18.6 vs 42.8 FPS (53.76 vs 23.34 ms/image).", + "values": { + "delta_pct": 56.58, + "a_fps": 18.6, + "b_fps": 42.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "DEIM-L uses less peak VRAM: 245 vs 307 MB.", + "values": { + "a_vram": 245, + "b_vram": 307 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "DEIM-L leads on small objects: 45.2 vs 39.0 mAP_small (15.67% relative gap).", + "values": { + "a": 45.2, + "b": 39.0, + "delta_pct": 15.67 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: TensorRT FP32, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:TensorRT FP32", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -56.58, + -42.37, + -20.13, + 0.0, + 0.001, + 0.0063, + 0.0071, + 0.1, + 0.3904, + 0.4, + 0.4515, + 0.5542, + 0.5783, + 0.5988, + 0.6, + 0.635, + 0.6352, + 0.6852, + 0.71, + 0.7302, + 0.7456, + 0.7555, + 1.0, + 1.262, + 1.74, + 2, + 2.4, + 2.567, + 3, + 4, + 4.33, + 4.8, + 5, + 5.8, + 6, + 6.228, + 6.863, + 7, + 8, + 9, + 10, + 11, + 12, + 13.911, + 15.67, + 16.65, + 18.6, + 19.9, + 23.341, + 27.837, + 31.24, + 39.0, + 39.0368, + 40.0, + 42.78, + 42.8, + 42.84, + 45.1535, + 45.2, + 46.269, + 50, + 50.5, + 50.6, + 53.759, + 53.9, + 54.21, + 55.4, + 55.4246, + 56.58, + 57.8, + 57.8262, + 59.8845, + 60.0, + 61.8, + 63.5241, + 68.5177, + 69.775, + 73.0219, + 74.5627, + 75, + 75.5469, + 75.7, + 78.01, + 95, + 100.0, + 245.0, + 245.2, + 300.0, + 307.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-m-vs-rtdetr-r34.claims.json b/article-pipeline/v2/output/deim-m-vs-rtdetr-r34.claims.json new file mode 100644 index 0000000..2e79eec --- /dev/null +++ b/article-pipeline/v2/output/deim-m-vs-rtdetr-r34.claims.json @@ -0,0 +1,386 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:42.767416+00:00", + "slug": "deim-m-vs-rtdetr-r34", + "meta": { + "type": "vs", + "a": "deim-m", + "b": "rtdetr-r34", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-m", + "displayName": "DEIM-M", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5548884434266282, + "mAP_50": 0.720638307865753, + "mAP_small": 0.39057370155679993, + "mAP_medium": 0.6055416329671153, + "mAP_large": 0.7175060793963718, + "fps_mean": 28.35, + "fps_p50": 28.5, + "ms_per_image": 35.274, + "inference_ms": 28.019, + "preprocess_ms": 6.134, + "postprocess_ms": 1.121, + "p95_ms": 39.735, + "peak_vram_mb": 188.8, + "peak_ram_mb": 4.7, + "params_millions": 19.59, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:34:55.136391Z" + } + }, + "b": { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5222774052742704, + "mAP_50": 0.6960054462347887, + "mAP_small": 0.34944822231901834, + "mAP_medium": 0.5588794736724594, + "mAP_large": 0.6678097562687866, + "fps_mean": 40.16, + "fps_p50": 41.3, + "ms_per_image": 24.903, + "inference_ms": 20.296, + "preprocess_ms": 3.515, + "postprocess_ms": 1.092, + "p95_ms": 29.392, + "peak_vram_mb": 212.9, + "peak_ram_mb": 0.0, + "params_millions": 31.44, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:23:56.394255Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 6.24, + "fps": -29.41, + "params": -37.69, + "flops": -37.36, + "peak_vram": -11.32 + }, + "map_pts": { + "a": 55.5, + "b": 52.2, + "delta": 3.3 + }, + "pair": { + "params_ratio": 1.6, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00973, + "b_mAP_per_GFLOP": 0.00574 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 55.4, + "fps": 57.6 + }, + "b": { + "mAP": 52.2, + "fps": 77.7 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 55.5, + "fps": 28.4 + }, + "b": { + "mAP": 52.2, + "fps": 40.2 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 55.4, + "fps": 83.3 + }, + "b": { + "mAP": 52.3, + "fps": 108.4 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 55.5, + "fps": 85.4 + }, + "b": { + "mAP": 52.2, + "fps": 98.1 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 55.4, + "fps": 7.0 + }, + "b": { + "mAP": 52.2, + "fps": 8.0 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 55.5, + "fps": 28.4 + }, + "b": { + "mAP": 52.2, + "fps": 40.2 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 55.4, + "fps": 0.6 + }, + "b": { + "mAP": 52.2, + "fps": 0.6 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "DEIM-M measures 3.3 mAP points higher than RT-DETR-R34 on NVIDIA RTX 5070 Ti (PyTorch FP32): 55.5 vs 52.2 mAP@50-95.", + "values": { + "delta_pts": 3.3, + "a_map": 55.5, + "b_map": 52.2 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R34 is 29.41% faster: 28.4 vs 40.2 FPS (35.27 vs 24.9 ms/image).", + "values": { + "delta_pct": 29.41, + "a_fps": 28.4, + "b_fps": 40.2 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "DEIM-M extracts 69.51% more mAP per GFLOP (0.00973 vs 0.00574).", + "values": { + "a": 0.00973, + "b": 0.00574, + "delta_pct": 69.51 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "DEIM-M uses less peak VRAM: 189 vs 213 MB.", + "values": { + "a_vram": 189, + "b_vram": 213 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "DEIM-M leads on small objects: 39.1 vs 34.9 mAP_small (11.77% relative gap).", + "values": { + "a": 39.1, + "b": 34.9, + "delta_pct": 11.77 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -37.69, + -37.36, + -29.41, + -11.32, + 0.0, + 0.001, + 0.0057, + 0.0097, + 0.1, + 0.3494, + 0.3906, + 0.5223, + 0.5549, + 0.5589, + 0.574, + 0.6, + 0.6055, + 0.6678, + 0.696, + 0.7175, + 0.7206, + 0.973, + 1.0, + 1.092, + 1.121, + 1.6, + 2, + 3, + 3.3, + 3.515, + 4, + 4.7, + 5, + 6, + 6.134, + 6.24, + 7.0, + 8.0, + 9, + 10, + 11, + 11.77, + 12, + 19.59, + 20.296, + 24.903, + 28.019, + 28.35, + 28.4, + 28.5, + 29.392, + 29.41, + 31.44, + 34.9, + 34.9448, + 35.274, + 39.0574, + 39.1, + 39.735, + 40.16, + 40.2, + 41.3, + 50, + 52.2, + 52.2277, + 52.3, + 55.4, + 55.4888, + 55.5, + 55.8879, + 57.6, + 60.0, + 60.5542, + 66.781, + 69.51, + 69.6005, + 71.7506, + 72.0638, + 75, + 77.7, + 83.3, + 85.4, + 95, + 98.1, + 100.0, + 108.4, + 188.8, + 189.0, + 212.9, + 213.0, + 300.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-n-vs-yolov9t.claims.json b/article-pipeline/v2/output/deim-n-vs-yolov9t.claims.json new file mode 100644 index 0000000..8a26d25 --- /dev/null +++ b/article-pipeline/v2/output/deim-n-vs-yolov9t.claims.json @@ -0,0 +1,388 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:42.848775+00:00", + "slug": "deim-n-vs-yolov9t", + "meta": { + "type": "vs", + "a": "deim-n", + "b": "yolov9t", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-n", + "displayName": "DEIM-N", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.4679166163127116, + "mAP_50": 0.6384759841148699, + "mAP_small": 0.2822402977108775, + "mAP_medium": 0.5241954344200933, + "mAP_large": 0.6491737523512869, + "fps_mean": 37.0, + "fps_p50": 37.45, + "ms_per_image": 27.03, + "inference_ms": 19.959, + "preprocess_ms": 5.969, + "postprocess_ms": 1.103, + "p95_ms": 30.511, + "peak_vram_mb": 70.2, + "peak_ram_mb": 3.1, + "params_millions": 3.78, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:34:02.298521Z" + } + }, + "b": { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "license": "MIT", + "releaseDate": "2024-02-21", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.4177885668656335, + "mAP_50": 0.5661372018894871, + "mAP_small": 0.2225871925542081, + "mAP_medium": 0.46300230914348306, + "mAP_large": 0.57077934037184, + "fps_mean": 31.94, + "fps_p50": 33.42, + "ms_per_image": 31.307, + "inference_ms": 25.151, + "preprocess_ms": 3.907, + "postprocess_ms": 2.249, + "p95_ms": 37.685, + "peak_vram_mb": 48.0, + "peak_ram_mb": 340.2, + "params_millions": 2.04, + "gflops": 3.98, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:18:14.276169Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 12.0, + "fps": 15.84, + "params": 85.29, + "flops": 75.88, + "peak_vram": 46.25 + }, + "map_pts": { + "a": 46.8, + "b": 41.8, + "delta": 5.0 + }, + "pair": { + "params_ratio": 1.85, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.06685, + "b_mAP_per_GFLOP": 0.10497 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 46.8, + "fps": 75.9 + }, + "b": { + "mAP": 41.8, + "fps": 52.1 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 46.8, + "fps": 37.0 + }, + "b": { + "mAP": 41.8, + "fps": 31.9 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 46.6, + "fps": 81.3 + }, + "b": { + "mAP": 41.8, + "fps": 59.0 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 46.8, + "fps": 67.5 + }, + "b": { + "mAP": 41.8, + "fps": 72.0 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 46.8, + "fps": 11.3 + }, + "b": { + "mAP": 41.8, + "fps": 9.9 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 46.8, + "fps": 37.0 + }, + "b": { + "mAP": 41.8, + "fps": 31.9 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 46.8, + "fps": 2.3 + }, + "b": { + "mAP": 41.4, + "fps": 2.9 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "DEIM-N measures 5.0 mAP points higher than YOLOv9-T on NVIDIA RTX 5070 Ti (PyTorch FP32): 46.8 vs 41.8 mAP@50-95.", + "values": { + "delta_pts": 5.0, + "a_map": 46.8, + "b_map": 41.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "DEIM-N is 15.84% faster: 37.0 vs 31.9 FPS (27.03 vs 31.31 ms/image).", + "values": { + "delta_pct": 15.84, + "a_fps": 37.0, + "b_fps": 31.9 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "YOLOv9-T extracts 36.32% more mAP per GFLOP (0.06685 vs 0.10497).", + "values": { + "a": 0.06685, + "b": 0.10497, + "delta_pct": 36.32 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "YOLOv9-T uses less peak VRAM: 70 vs 48 MB.", + "values": { + "a_vram": 70, + "b_vram": 48 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "DEIM-N leads on small objects: 28.2 vs 22.3 mAP_small (26.8% relative gap).", + "values": { + "a": 28.2, + "b": 22.3, + "delta_pct": 26.8 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: TensorRT FP32, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:TensorRT FP32", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + 0.0, + 0.001, + 0.0669, + 0.1, + 0.105, + 0.2226, + 0.2822, + 0.4178, + 0.463, + 0.4679, + 0.5242, + 0.5661, + 0.5708, + 0.6, + 0.6385, + 0.6492, + 1.0, + 1.103, + 1.85, + 2, + 2.04, + 2.249, + 2.3, + 2.9, + 3, + 3.1, + 3.78, + 3.907, + 3.98, + 4, + 5.0, + 5.969, + 6, + 6.685, + 7, + 8, + 9, + 9.9, + 10, + 10.497, + 11, + 11.3, + 12.0, + 15.84, + 19.959, + 22.2587, + 22.3, + 25.151, + 26.8, + 27.03, + 28.2, + 28.224, + 30.511, + 31.307, + 31.9, + 31.94, + 33.42, + 36.32, + 37.0, + 37.45, + 37.685, + 41.4, + 41.7789, + 41.8, + 46.25, + 46.3002, + 46.6, + 46.7917, + 46.8, + 48.0, + 50, + 52.1, + 52.4195, + 56.6137, + 57.0779, + 59.0, + 60.0, + 63.8476, + 64.9174, + 67.5, + 70.0, + 70.2, + 72.0, + 75, + 75.88, + 75.9, + 81.3, + 85.29, + 95, + 100.0, + 300.0, + 340.2, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-s-vs-deimv2-s.claims.json b/article-pipeline/v2/output/deim-s-vs-deimv2-s.claims.json new file mode 100644 index 0000000..62a258f --- /dev/null +++ b/article-pipeline/v2/output/deim-s-vs-deimv2-s.claims.json @@ -0,0 +1,362 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T21:56:13.803574+00:00", + "slug": "deim-s-vs-deimv2-s", + "meta": { + "type": "vs", + "a": "deim-s", + "b": "deimv2-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-s", + "displayName": "DEIM-S", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5209629773845224, + "mAP_50": 0.6832814321073832, + "mAP_small": 0.3601841065325898, + "mAP_medium": 0.563738694771644, + "mAP_large": 0.6699383734649564, + "fps_mean": 33.35, + "fps_p50": 34.25, + "ms_per_image": 29.985, + "inference_ms": 22.909, + "preprocess_ms": 5.964, + "postprocess_ms": 1.112, + "p95_ms": 33.963, + "peak_vram_mb": 141.6, + "peak_ram_mb": 3.7, + "params_millions": 10.32, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:34:26.555726Z" + } + }, + "b": { + "id": "deimv2-s", + "displayName": "DEIMv2-S", + "license": "Apache-2.0", + "releaseDate": "2025-09-25", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5299704599483922, + "mAP_50": 0.6974881834026754, + "mAP_small": 0.34964631645738836, + "mAP_medium": 0.5805276638457348, + "mAP_large": 0.7300115678011079, + "fps_mean": 24.26, + "fps_p50": 24.89, + "ms_per_image": 41.226, + "inference_ms": 28.326, + "preprocess_ms": 11.744, + "postprocess_ms": 1.156, + "p95_ms": 46.991, + "peak_vram_mb": 103.7, + "peak_ram_mb": 0.0, + "params_millions": 9.78, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:42:59.953562Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -1.7, + "fps": 37.47, + "params": 5.52, + "flops": -2.42, + "peak_vram": 36.55 + }, + "map_pts": { + "a": 52.1, + "b": 53.0, + "delta": -0.9 + }, + "pair": { + "params_ratio": 1.06, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.02084, + "b_mAP_per_GFLOP": 0.02069 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 52.1, + "fps": 67.5 + }, + "b": { + "mAP": 53.0, + "fps": 39.6 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 52.1, + "fps": 33.4 + }, + "b": { + "mAP": 53.0, + "fps": 24.3 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 51.9, + "fps": 82.4 + }, + "b": { + "mAP": 53.0, + "fps": 50.4 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 52.1, + "fps": 97.6 + }, + "b": { + "mAP": 53.0, + "fps": 52.5 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 52.1, + "fps": 10.2 + }, + "b": { + "mAP": 53.0, + "fps": 5.7 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 52.1, + "fps": 33.4 + }, + "b": { + "mAP": 53.0, + "fps": 24.3 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 52.1, + "fps": 1.1 + }, + "b": { + "mAP": 53.0, + "fps": 0.8 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "DEIMv2-S measures 0.9 mAP points higher than DEIM-S on NVIDIA RTX 5070 Ti (PyTorch FP32): 52.1 vs 53.0 mAP@50-95.", + "values": { + "delta_pts": 0.9, + "a_map": 52.1, + "b_map": 53.0 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "DEIM-S is 37.47% faster: 33.4 vs 24.3 FPS (29.98 vs 41.23 ms/image).", + "values": { + "delta_pct": 37.47, + "a_fps": 33.4, + "b_fps": 24.3 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "DEIMv2-S uses less peak VRAM: 142 vs 104 MB.", + "values": { + "a_vram": 142, + "b_vram": 104 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_stable", + "kind": "robustness", + "statement": "The verdict holds across 4 runtimes and 3 hardware platforms with verified data.", + "values": { + "n_runtimes": 4, + "n_hardware": 3 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 4, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -2.42, + -1.7, + -0.9, + 0.0, + 0.001, + 0.0207, + 0.0208, + 0.1, + 0.3496, + 0.3602, + 0.521, + 0.53, + 0.5637, + 0.5805, + 0.6, + 0.6699, + 0.6833, + 0.6975, + 0.73, + 0.8, + 0.9, + 1.0, + 1.06, + 1.1, + 1.112, + 1.156, + 2, + 2.069, + 2.084, + 3.0, + 3.7, + 4.0, + 5, + 5.52, + 5.7, + 5.964, + 6, + 7, + 8, + 9, + 9.78, + 10, + 10.2, + 10.32, + 11, + 11.744, + 12, + 22.909, + 24.26, + 24.3, + 24.89, + 28.326, + 29.985, + 33.35, + 33.4, + 33.963, + 34.25, + 34.9646, + 36.0184, + 36.55, + 37.47, + 39.6, + 41.226, + 46.991, + 50, + 50.4, + 51.9, + 52.0963, + 52.1, + 52.5, + 52.997, + 53.0, + 56.3739, + 58.0528, + 60.0, + 66.9938, + 67.5, + 68.3281, + 69.7488, + 73.0012, + 75, + 80.0, + 82.4, + 90.0, + 95, + 97.6, + 100.0, + 103.7, + 104.0, + 141.6, + 142.0, + 300.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-s-vs-dfine-s.claims.json b/article-pipeline/v2/output/deim-s-vs-dfine-s.claims.json new file mode 100644 index 0000000..689991f --- /dev/null +++ b/article-pipeline/v2/output/deim-s-vs-dfine-s.claims.json @@ -0,0 +1,331 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T21:56:13.351692+00:00", + "slug": "deim-s-vs-dfine-s", + "meta": { + "type": "vs", + "a": "deim-s", + "b": "dfine-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-s", + "displayName": "DEIM-S", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5209629773845224, + "mAP_50": 0.6832814321073832, + "mAP_small": 0.3601841065325898, + "mAP_medium": 0.563738694771644, + "mAP_large": 0.6699383734649564, + "fps_mean": 33.35, + "fps_p50": 34.25, + "ms_per_image": 29.985, + "inference_ms": 22.909, + "preprocess_ms": 5.964, + "postprocess_ms": 1.112, + "p95_ms": 33.963, + "peak_vram_mb": 141.6, + "peak_ram_mb": 3.7, + "params_millions": 10.32, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:34:26.555726Z" + } + }, + "b": { + "id": "dfine-s", + "displayName": "D-FINE-S", + "license": "Apache-2.0", + "releaseDate": "2024-10-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5339360463706191, + "mAP_50": 0.6979289246485497, + "mAP_small": 0.37902381194039897, + "mAP_medium": 0.5785906686133171, + "mAP_large": 0.6701666032622975, + "fps_mean": 34.17, + "fps_p50": 34.82, + "ms_per_image": 29.267, + "inference_ms": 22.32, + "preprocess_ms": 5.837, + "postprocess_ms": 1.111, + "p95_ms": 34.014, + "peak_vram_mb": 141.6, + "peak_ram_mb": 0.7, + "params_millions": 10.32, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:45:41.588316Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -2.43, + "fps": -2.4, + "params": 0.0, + "flops": 0.0, + "peak_vram": 0.0 + }, + "map_pts": { + "a": 52.1, + "b": 53.4, + "delta": -1.3 + }, + "pair": { + "params_ratio": 1.0, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.02084, + "b_mAP_per_GFLOP": 0.02136 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 52.1, + "fps": 67.5 + }, + "b": { + "mAP": 53.4, + "fps": 70.9 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 52.1, + "fps": 33.4 + }, + "b": { + "mAP": 53.4, + "fps": 34.2 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 51.9, + "fps": 82.4 + }, + "b": { + "mAP": 53.2, + "fps": 86.5 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 52.1, + "fps": 97.6 + }, + "b": { + "mAP": 53.4, + "fps": 98.7 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 52.1, + "fps": 10.2 + }, + "b": { + "mAP": 53.4, + "fps": 10.3 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 52.1, + "fps": 33.4 + }, + "b": { + "mAP": 53.4, + "fps": 34.2 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 52.1, + "fps": 1.1 + }, + "b": { + "mAP": 53.4, + "fps": 1.1 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "D-FINE-S measures 1.3 mAP points higher than DEIM-S on NVIDIA RTX 5070 Ti (PyTorch FP32): 52.1 vs 53.4 mAP@50-95.", + "values": { + "delta_pts": 1.3, + "a_map": 52.1, + "b_map": 53.4 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "ranking_stable", + "kind": "robustness", + "statement": "The verdict holds across 4 runtimes and 3 hardware platforms with verified data.", + "values": { + "n_runtimes": 4, + "n_hardware": 3 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 2, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -2.43, + -2.4, + -1.3, + 0.0, + 0.001, + 0.0208, + 0.0214, + 0.1, + 0.3602, + 0.379, + 0.521, + 0.5339, + 0.5637, + 0.5786, + 0.6, + 0.6699, + 0.6702, + 0.6833, + 0.6979, + 0.7, + 1.0, + 1.1, + 1.111, + 1.112, + 1.3, + 2, + 2.084, + 2.136, + 3.0, + 3.7, + 4.0, + 5, + 5.837, + 5.964, + 6, + 7, + 8, + 9, + 10, + 10.2, + 10.3, + 10.32, + 11, + 12, + 22.32, + 22.909, + 29.267, + 29.985, + 33.35, + 33.4, + 33.963, + 34.014, + 34.17, + 34.2, + 34.25, + 34.82, + 36.0184, + 37.9024, + 50, + 51.9, + 52.0963, + 52.1, + 53.2, + 53.3936, + 53.4, + 56.3739, + 57.8591, + 60.0, + 66.9938, + 67.0167, + 67.5, + 68.3281, + 69.7929, + 70.0, + 70.9, + 75, + 82.4, + 86.5, + 95, + 97.6, + 98.7, + 100.0, + 141.6, + 300.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-s-vs-rtdetr-r18.claims.json b/article-pipeline/v2/output/deim-s-vs-rtdetr-r18.claims.json new file mode 100644 index 0000000..42e9d0b --- /dev/null +++ b/article-pipeline/v2/output/deim-s-vs-rtdetr-r18.claims.json @@ -0,0 +1,390 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:42.917152+00:00", + "slug": "deim-s-vs-rtdetr-r18", + "meta": { + "type": "vs", + "a": "deim-s", + "b": "rtdetr-r18", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-s", + "displayName": "DEIM-S", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5209629773845224, + "mAP_50": 0.6832814321073832, + "mAP_small": 0.3601841065325898, + "mAP_medium": 0.563738694771644, + "mAP_large": 0.6699383734649564, + "fps_mean": 33.35, + "fps_p50": 34.25, + "ms_per_image": 29.985, + "inference_ms": 22.909, + "preprocess_ms": 5.964, + "postprocess_ms": 1.112, + "p95_ms": 33.963, + "peak_vram_mb": 141.6, + "peak_ram_mb": 3.7, + "params_millions": 10.32, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:34:26.555726Z" + } + }, + "b": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.49787927521150827, + "mAP_50": 0.663704009633566, + "mAP_small": 0.32710231823159713, + "mAP_medium": 0.5484637312981646, + "mAP_large": 0.6407169138607522, + "fps_mean": 47.22, + "fps_p50": 48.89, + "ms_per_image": 21.179, + "inference_ms": 16.645, + "preprocess_ms": 3.452, + "postprocess_ms": 1.082, + "p95_ms": 25.931, + "peak_vram_mb": 168.9, + "peak_ram_mb": 2.0, + "params_millions": 20.18, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:23:32.849123Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 4.64, + "fps": -29.37, + "params": -48.86, + "flops": -58.33, + "peak_vram": -16.16 + }, + "map_pts": { + "a": 52.1, + "b": 49.8, + "delta": 2.3 + }, + "pair": { + "params_ratio": 1.96, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.02084, + "b_mAP_per_GFLOP": 0.0083 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 52.1, + "fps": 67.5 + }, + "b": { + "mAP": 49.8, + "fps": 92.5 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 52.1, + "fps": 33.4 + }, + "b": { + "mAP": 49.8, + "fps": 47.2 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 51.9, + "fps": 82.4 + }, + "b": { + "mAP": 49.8, + "fps": 113.4 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 52.1, + "fps": 97.6 + }, + "b": { + "mAP": 49.8, + "fps": 120.6 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 52.1, + "fps": 10.2 + }, + "b": { + "mAP": 49.8, + "fps": 10.2 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 52.1, + "fps": 33.4 + }, + "b": { + "mAP": 49.8, + "fps": 47.2 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 52.1, + "fps": 1.1 + }, + "b": { + "mAP": 49.7, + "fps": 0.8 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "DEIM-S measures 2.3 mAP points higher than RT-DETR-R18 on NVIDIA RTX 5070 Ti (PyTorch FP32): 52.1 vs 49.8 mAP@50-95.", + "values": { + "delta_pts": 2.3, + "a_map": 52.1, + "b_map": 49.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R18 is 29.37% faster: 33.4 vs 47.2 FPS (29.98 vs 21.18 ms/image).", + "values": { + "delta_pct": 29.37, + "a_fps": 33.4, + "b_fps": 47.2 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "DEIM-S extracts 151.08% more mAP per GFLOP (0.02084 vs 0.0083).", + "values": { + "a": 0.02084, + "b": 0.0083, + "delta_pct": 151.08 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "DEIM-S uses less peak VRAM: 142 vs 169 MB.", + "values": { + "a_vram": 142, + "b_vram": 169 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "DEIM-S leads on small objects: 36.0 vs 32.7 mAP_small (10.11% relative gap).", + "values": { + "a": 36.0, + "b": 32.7, + "delta_pct": 10.11 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -58.33, + -48.86, + -29.37, + -16.16, + 0.0, + 0.001, + 0.0083, + 0.0208, + 0.1, + 0.3271, + 0.3602, + 0.4979, + 0.521, + 0.5485, + 0.5637, + 0.6, + 0.6407, + 0.6637, + 0.6699, + 0.6833, + 0.8, + 0.83, + 1.0, + 1.082, + 1.1, + 1.112, + 1.96, + 2.0, + 2.084, + 2.3, + 3, + 3.452, + 3.7, + 4, + 4.64, + 5, + 5.964, + 6, + 7, + 8, + 9, + 10, + 10.11, + 10.2, + 10.32, + 11, + 12, + 16.645, + 20.18, + 21.179, + 22.909, + 25.931, + 29.37, + 29.985, + 32.7, + 32.7102, + 33.35, + 33.4, + 33.963, + 34.25, + 36.0, + 36.0184, + 47.2, + 47.22, + 48.89, + 49.7, + 49.7879, + 49.8, + 50, + 51.9, + 52.0963, + 52.1, + 54.8464, + 56.3739, + 60.0, + 64.0717, + 66.3704, + 66.9938, + 67.5, + 68.3281, + 75, + 80.0, + 82.4, + 92.5, + 95, + 97.6, + 100.0, + 113.4, + 120.6, + 141.6, + 142.0, + 151.08, + 168.9, + 169.0, + 300.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-s-vs-yolov9s.claims.json b/article-pipeline/v2/output/deim-s-vs-yolov9s.claims.json new file mode 100644 index 0000000..7b4a0ee --- /dev/null +++ b/article-pipeline/v2/output/deim-s-vs-yolov9s.claims.json @@ -0,0 +1,392 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.008651+00:00", + "slug": "deim-s-vs-yolov9s", + "meta": { + "type": "vs", + "a": "deim-s", + "b": "yolov9s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "deim-s", + "displayName": "DEIM-S", + "license": "Apache-2.0", + "releaseDate": "2024-12-05", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5209629773845224, + "mAP_50": 0.6832814321073832, + "mAP_small": 0.3601841065325898, + "mAP_medium": 0.563738694771644, + "mAP_large": 0.6699383734649564, + "fps_mean": 33.35, + "fps_p50": 34.25, + "ms_per_image": 29.985, + "inference_ms": 22.909, + "preprocess_ms": 5.964, + "postprocess_ms": 1.112, + "p95_ms": 33.963, + "peak_vram_mb": 141.6, + "peak_ram_mb": 3.7, + "params_millions": 10.32, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:34:26.555726Z" + } + }, + "b": { + "id": "yolov9s", + "displayName": "YOLOv9-S", + "license": "MIT", + "releaseDate": "2024-02-21", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5044737676946623, + "mAP_50": 0.6719744329452191, + "mAP_small": 0.307408881744591, + "mAP_medium": 0.5643791782271478, + "mAP_large": 0.653006070420912, + "fps_mean": 30.76, + "fps_p50": 31.79, + "ms_per_image": 32.507, + "inference_ms": 26.448, + "preprocess_ms": 3.813, + "postprocess_ms": 2.246, + "p95_ms": 39.868, + "peak_vram_mb": 91.5, + "peak_ram_mb": 20.4, + "params_millions": 7.23, + "gflops": 13.52, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:18:36.206342Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 3.27, + "fps": 8.42, + "params": 42.74, + "flops": 84.91, + "peak_vram": 54.75 + }, + "map_pts": { + "a": 52.1, + "b": 50.4, + "delta": 1.7 + }, + "pair": { + "params_ratio": 1.43, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.02084, + "b_mAP_per_GFLOP": 0.03731 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 52.1, + "fps": 67.5 + }, + "b": { + "mAP": 50.4, + "fps": 54.5 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 52.1, + "fps": 33.4 + }, + "b": { + "mAP": 50.4, + "fps": 30.8 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 51.9, + "fps": 82.4 + }, + "b": { + "mAP": 50.5, + "fps": 63.8 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 52.1, + "fps": 97.6 + }, + "b": { + "mAP": 50.4, + "fps": 75.1 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 52.1, + "fps": 10.2 + }, + "b": { + "mAP": 50.4, + "fps": 9.9 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 52.1, + "fps": 33.4 + }, + "b": { + "mAP": 50.4, + "fps": 30.8 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 52.1, + "fps": 1.1 + }, + "b": { + "mAP": 49.4, + "fps": 1.4 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "DEIM-S measures 1.7 mAP points higher than YOLOv9-S on NVIDIA RTX 5070 Ti (PyTorch FP32): 52.1 vs 50.4 mAP@50-95.", + "values": { + "delta_pts": 1.7, + "a_map": 52.1, + "b_map": 50.4 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "DEIM-S is 8.42% faster: 33.4 vs 30.8 FPS (29.98 vs 32.51 ms/image).", + "values": { + "delta_pct": 8.42, + "a_fps": 33.4, + "b_fps": 30.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "YOLOv9-S extracts 44.14% more mAP per GFLOP (0.02084 vs 0.03731).", + "values": { + "a": 0.02084, + "b": 0.03731, + "delta_pct": 44.14 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "YOLOv9-S uses less peak VRAM: 142 vs 92 MB.", + "values": { + "a_vram": 142, + "b_vram": 92 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "DEIM-S leads on small objects: 36.0 vs 30.7 mAP_small (17.17% relative gap).", + "values": { + "a": 36.0, + "b": 30.7, + "delta_pct": 17.17 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + 0.0, + 0.001, + 0.0208, + 0.0373, + 0.1, + 0.3074, + 0.3602, + 0.5045, + 0.521, + 0.5637, + 0.5644, + 0.6, + 0.653, + 0.6699, + 0.672, + 0.6833, + 1.0, + 1.1, + 1.112, + 1.4, + 1.43, + 1.7, + 2, + 2.084, + 2.246, + 3, + 3.27, + 3.7, + 3.731, + 3.813, + 4, + 5, + 5.964, + 6, + 7, + 7.23, + 8, + 8.42, + 9, + 9.9, + 10, + 10.2, + 10.32, + 11, + 12, + 13.52, + 17.17, + 20.4, + 22.909, + 26.448, + 29.985, + 30.7, + 30.7409, + 30.76, + 30.8, + 31.79, + 32.507, + 33.35, + 33.4, + 33.963, + 34.25, + 36.0, + 36.0184, + 39.868, + 42.74, + 44.14, + 49.4, + 50, + 50.4, + 50.4474, + 50.5, + 51.9, + 52.0963, + 52.1, + 54.5, + 54.75, + 56.3739, + 56.4379, + 60.0, + 63.8, + 65.3006, + 66.9938, + 67.1974, + 67.5, + 68.3281, + 75, + 75.1, + 82.4, + 84.91, + 91.5, + 92.0, + 95, + 97.6, + 100.0, + 141.6, + 142.0, + 300.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-vs-deimv2.claims.json b/article-pipeline/v2/output/deim-vs-deimv2.claims.json new file mode 100644 index 0000000..fcd21ba --- /dev/null +++ b/article-pipeline/v2/output/deim-vs-deimv2.claims.json @@ -0,0 +1,457 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:18.672405+00:00", + "slug": "deim-vs-deimv2", + "meta": { + "type": "family-vs", + "a": "deim", + "b": "deimv2", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "deim", + "b": "deimv2", + "a_display": "DEIM", + "b_display": "DEIMv2" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "deim-n", + "displayName": "DEIM-N", + "params": 3.78, + "mAP": 46.8, + "fps": 37.0, + "license": "Apache-2.0" + }, + { + "id": "deim-s", + "displayName": "DEIM-S", + "params": 10.32, + "mAP": 52.1, + "fps": 33.4, + "license": "Apache-2.0" + }, + { + "id": "deim-m", + "displayName": "DEIM-M", + "params": 19.59, + "mAP": 55.5, + "fps": 28.4, + "license": "Apache-2.0" + }, + { + "id": "deim-l", + "displayName": "DEIM-L", + "params": 31.24, + "mAP": 57.8, + "fps": 18.6, + "license": "Apache-2.0" + }, + { + "id": "deim-x", + "displayName": "DEIM-X", + "params": 62.62, + "mAP": 59.6, + "fps": 15.0, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "params": 0.51, + "mAP": 27.5, + "fps": 40.4, + "license": "Apache-2.0" + }, + { + "id": "deimv2-femto", + "displayName": "DEIMv2-Femto", + "params": 0.98, + "mAP": 34.5, + "fps": 45.3, + "license": "Apache-2.0" + }, + { + "id": "deimv2-pico", + "displayName": "DEIMv2-Pico", + "params": 1.54, + "mAP": 42.3, + "fps": 40.5, + "license": "Apache-2.0" + }, + { + "id": "deimv2-n", + "displayName": "DEIMv2-N", + "params": 3.6, + "mAP": 46.7, + "fps": 35.1, + "license": "Apache-2.0" + }, + { + "id": "deimv2-s", + "displayName": "DEIMv2-S", + "params": 9.78, + "mAP": 53.0, + "fps": 24.3, + "license": "Apache-2.0" + }, + { + "id": "deimv2-m", + "displayName": "DEIMv2-M", + "params": 18.36, + "mAP": 56.0, + "fps": 22.1, + "license": "Apache-2.0" + }, + { + "id": "deimv2-l", + "displayName": "DEIMv2-L", + "params": 32.55, + "mAP": 58.6, + "fps": 19.7, + "license": "Apache-2.0" + }, + { + "id": "deimv2-x", + "displayName": "DEIMv2-X", + "params": 51.21, + "mAP": 61.3, + "fps": 17.4, + "license": "Apache-2.0" + } + ], + "matched_pairs": [ + { + "a": { + "id": "deim-n", + "displayName": "DEIM-N", + "params": 3.78, + "mAP": 46.8, + "fps": 37.0, + "license": "Apache-2.0" + }, + "b": { + "id": "deimv2-n", + "displayName": "DEIMv2-N", + "params": 3.6, + "mAP": 46.7, + "fps": 35.1, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 0.1, + "fps_delta_pct": 5.41 + }, + { + "a": { + "id": "deim-s", + "displayName": "DEIM-S", + "params": 10.32, + "mAP": 52.1, + "fps": 33.4, + "license": "Apache-2.0" + }, + "b": { + "id": "deimv2-s", + "displayName": "DEIMv2-S", + "params": 9.78, + "mAP": 53.0, + "fps": 24.3, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -0.9, + "fps_delta_pct": 37.45 + }, + { + "a": { + "id": "deim-m", + "displayName": "DEIM-M", + "params": 19.59, + "mAP": 55.5, + "fps": 28.4, + "license": "Apache-2.0" + }, + "b": { + "id": "deimv2-m", + "displayName": "DEIMv2-M", + "params": 18.36, + "mAP": 56.0, + "fps": 22.1, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -0.5, + "fps_delta_pct": 28.51 + }, + { + "a": { + "id": "deim-l", + "displayName": "DEIM-L", + "params": 31.24, + "mAP": 57.8, + "fps": 18.6, + "license": "Apache-2.0" + }, + "b": { + "id": "deimv2-l", + "displayName": "DEIMv2-L", + "params": 32.55, + "mAP": 58.6, + "fps": 19.7, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -0.8, + "fps_delta_pct": -5.58 + }, + { + "a": { + "id": "deim-x", + "displayName": "DEIM-X", + "params": 62.62, + "mAP": 59.6, + "fps": 15.0, + "license": "Apache-2.0" + }, + "b": { + "id": "deimv2-x", + "displayName": "DEIMv2-X", + "params": 51.21, + "mAP": 61.3, + "fps": 17.4, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -1.7, + "fps_delta_pct": -13.79 + } + ], + "aggregate": { + "n_pairs": 5, + "mean_mAP_delta_pts": -0.8, + "a_wins": 1, + "b_wins": 4, + "mean_fps_delta_pct": 10.4 + }, + "flagship": { + "a": { + "id": "deim-x", + "displayName": "DEIM-X", + "params": 62.62, + "mAP": 59.6, + "fps": 15.0, + "license": "Apache-2.0" + }, + "b": { + "id": "deimv2-x", + "displayName": "DEIMv2-X", + "params": 51.21, + "mAP": 61.3, + "fps": 17.4, + "license": "Apache-2.0" + } + }, + "smallest": { + "a": { + "id": "deim-n", + "displayName": "DEIM-N", + "params": 3.78, + "mAP": 46.8, + "fps": 37.0, + "license": "Apache-2.0" + }, + "b": { + "id": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "params": 0.51, + "mAP": 27.5, + "fps": 40.4, + "license": "Apache-2.0" + } + }, + "crossover": { + "low_param_leader": "deimv2", + "high_param_leader": "deimv2" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "deim-n,deim-s,deim-m,deim-l,deim-x,deimv2-atto,deimv2-femto,deimv2-pico,deimv2-n,deimv2-s,deimv2-m,deimv2-l,deimv2-x" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, DEIMv2 averages 0.8 mAP points higher across 5 paired variants on NVIDIA RTX 5070 Ti. DEIM wins 1, DEIMv2 wins 4.", + "values": { + "mean_delta_pts": 0.8, + "a_wins": 1, + "b_wins": 4 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, DEIM is 10.4% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 10.4 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, DEIM's largest measured variant (DEIM-X) reaches 59.6 mAP; DEIMv2's (DEIMv2-X) reaches 61.3 mAP.", + "values": { + "a": { + "id": "deim-x", + "displayName": "DEIM-X", + "params": 62.62, + "mAP": 59.6, + "fps": 15.0, + "license": "Apache-2.0" + }, + "b": { + "id": "deimv2-x", + "displayName": "DEIMv2-X", + "params": 51.21, + "mAP": 61.3, + "fps": 17.4, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, DEIM's DEIM-N (3.78M, 46.8 mAP, 37.0 FPS) faces DEIMv2's DEIMv2-Atto (0.51M, 27.5 mAP, 40.4 FPS).", + "values": { + "a": { + "id": "deim-n", + "displayName": "DEIM-N", + "params": 3.78, + "mAP": 46.8, + "fps": 37.0, + "license": "Apache-2.0" + }, + "b": { + "id": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "params": 0.51, + "mAP": 27.5, + "fps": 40.4, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "DEIM fields 5 measured variants, DEIMv2 fields 8, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 5, + "b_n": 8 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -13.79, + -5.58, + -1.7, + -0.9, + -0.8, + -0.5, + 0.001, + 0.1, + 0.51, + 0.6, + 0.8, + 0.98, + 1.0, + 1.54, + 2, + 3, + 3.6, + 3.78, + 4.0, + 5.0, + 5.41, + 6, + 7, + 8.0, + 9, + 9.78, + 10.0, + 10.32, + 10.4, + 11, + 12, + 15.0, + 17.4, + 18.36, + 18.6, + 19.59, + 19.7, + 22.1, + 24.3, + 27.5, + 28.4, + 28.51, + 31.24, + 32.55, + 33.4, + 34.5, + 35.1, + 37.0, + 37.45, + 40.4, + 40.5, + 42.3, + 45.3, + 46.7, + 46.8, + 50, + 51.0, + 51.21, + 52.1, + 53.0, + 55.5, + 56.0, + 57.8, + 58.6, + 59.6, + 61.3, + 62.62, + 75, + 80.0, + 95, + 98.0, + 100.0, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-vs-dfine.claims.json b/article-pipeline/v2/output/deim-vs-dfine.claims.json new file mode 100644 index 0000000..8b0808a --- /dev/null +++ b/article-pipeline/v2/output/deim-vs-dfine.claims.json @@ -0,0 +1,404 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:18.513679+00:00", + "slug": "deim-vs-dfine", + "meta": { + "type": "family-vs", + "a": "deim", + "b": "dfine", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "deim", + "b": "dfine", + "a_display": "DEIM", + "b_display": "D-FINE" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "deim-n", + "displayName": "DEIM-N", + "params": 3.78, + "mAP": 46.8, + "fps": 37.0, + "license": "Apache-2.0" + }, + { + "id": "deim-s", + "displayName": "DEIM-S", + "params": 10.32, + "mAP": 52.1, + "fps": 33.4, + "license": "Apache-2.0" + }, + { + "id": "deim-m", + "displayName": "DEIM-M", + "params": 19.59, + "mAP": 55.5, + "fps": 28.4, + "license": "Apache-2.0" + }, + { + "id": "deim-l", + "displayName": "DEIM-L", + "params": 31.24, + "mAP": 57.8, + "fps": 18.6, + "license": "Apache-2.0" + }, + { + "id": "deim-x", + "displayName": "DEIM-X", + "params": 62.62, + "mAP": 59.6, + "fps": 15.0, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + { + "id": "dfine-s", + "displayName": "D-FINE-S", + "params": 10.32, + "mAP": 53.4, + "fps": 34.2, + "license": "Apache-2.0" + }, + { + "id": "dfine-m", + "displayName": "D-FINE-M", + "params": 19.59, + "mAP": 57.8, + "fps": 28.8, + "license": "Apache-2.0" + }, + { + "id": "dfine-l", + "displayName": "D-FINE-L", + "params": 31.24, + "mAP": 60.0, + "fps": 21.9, + "license": "Apache-2.0" + }, + { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + } + ], + "matched_pairs": [ + { + "a": { + "id": "deim-n", + "displayName": "DEIM-N", + "params": 3.78, + "mAP": 46.8, + "fps": 37.0, + "license": "Apache-2.0" + }, + "b": { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 1.0, + "fps_delta_pct": 13.85 + }, + { + "a": { + "id": "deim-s", + "displayName": "DEIM-S", + "params": 10.32, + "mAP": 52.1, + "fps": 33.4, + "license": "Apache-2.0" + }, + "b": { + "id": "dfine-s", + "displayName": "D-FINE-S", + "params": 10.32, + "mAP": 53.4, + "fps": 34.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -1.3, + "fps_delta_pct": -2.34 + }, + { + "a": { + "id": "deim-m", + "displayName": "DEIM-M", + "params": 19.59, + "mAP": 55.5, + "fps": 28.4, + "license": "Apache-2.0" + }, + "b": { + "id": "dfine-m", + "displayName": "D-FINE-M", + "params": 19.59, + "mAP": 57.8, + "fps": 28.8, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -2.3, + "fps_delta_pct": -1.39 + }, + { + "a": { + "id": "deim-l", + "displayName": "DEIM-L", + "params": 31.24, + "mAP": 57.8, + "fps": 18.6, + "license": "Apache-2.0" + }, + "b": { + "id": "dfine-l", + "displayName": "D-FINE-L", + "params": 31.24, + "mAP": 60.0, + "fps": 21.9, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -2.2, + "fps_delta_pct": -15.07 + }, + { + "a": { + "id": "deim-x", + "displayName": "DEIM-X", + "params": 62.62, + "mAP": 59.6, + "fps": 15.0, + "license": "Apache-2.0" + }, + "b": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -1.8, + "fps_delta_pct": -17.13 + } + ], + "aggregate": { + "n_pairs": 5, + "mean_mAP_delta_pts": -1.3, + "a_wins": 1, + "b_wins": 4, + "mean_fps_delta_pct": -4.4 + }, + "flagship": { + "a": { + "id": "deim-x", + "displayName": "DEIM-X", + "params": 62.62, + "mAP": 59.6, + "fps": 15.0, + "license": "Apache-2.0" + }, + "b": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + } + }, + "smallest": { + "a": { + "id": "deim-n", + "displayName": "DEIM-N", + "params": 3.78, + "mAP": 46.8, + "fps": 37.0, + "license": "Apache-2.0" + }, + "b": { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + } + }, + "crossover": { + "low_param_leader": "dfine", + "high_param_leader": "dfine" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "deim-n,deim-s,deim-m,deim-l,deim-x,dfine-n,dfine-s,dfine-m,dfine-l,dfine-x" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, D-FINE averages 1.3 mAP points higher across 5 paired variants on NVIDIA RTX 5070 Ti. DEIM wins 1, D-FINE wins 4.", + "values": { + "mean_delta_pts": 1.3, + "a_wins": 1, + "b_wins": 4 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, DEIM's largest measured variant (DEIM-X) reaches 59.6 mAP; D-FINE's (D-FINE-X) reaches 61.4 mAP.", + "values": { + "a": { + "id": "deim-x", + "displayName": "DEIM-X", + "params": 62.62, + "mAP": 59.6, + "fps": 15.0, + "license": "Apache-2.0" + }, + "b": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, DEIM's DEIM-N (3.78M, 46.8 mAP, 37.0 FPS) faces D-FINE's D-FINE-N (3.78M, 45.8 mAP, 32.5 FPS).", + "values": { + "a": { + "id": "deim-n", + "displayName": "DEIM-N", + "params": 3.78, + "mAP": 46.8, + "fps": 37.0, + "license": "Apache-2.0" + }, + "b": { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "DEIM fields 5 measured variants, D-FINE fields 5, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 5, + "b_n": 5 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 4, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -17.13, + -15.07, + -4.4, + -2.34, + -2.3, + -2.2, + -1.8, + -1.39, + -1.3, + 0.001, + 0.6, + 1.0, + 1.3, + 2, + 3, + 3.78, + 4.0, + 5.0, + 6, + 7, + 8, + 9, + 10, + 10.32, + 11, + 12, + 13.85, + 15.0, + 18.1, + 18.6, + 19.59, + 21.9, + 28.4, + 28.8, + 31.24, + 32.5, + 33.4, + 34.2, + 37.0, + 45.8, + 46.8, + 50, + 52.1, + 53.4, + 55.5, + 57.8, + 59.6, + 60.0, + 61.4, + 62.62, + 75, + 95, + 100.0, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/deim-vs-rtdetr.claims.json b/article-pipeline/v2/output/deim-vs-rtdetr.claims.json new file mode 100644 index 0000000..c1b0497 --- /dev/null +++ b/article-pipeline/v2/output/deim-vs-rtdetr.claims.json @@ -0,0 +1,401 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:19.642154+00:00", + "slug": "deim-vs-rtdetr", + "meta": { + "type": "family-vs", + "a": "deim", + "b": "rtdetr", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "deim", + "b": "rtdetr", + "a_display": "DEIM", + "b_display": "RT-DETR" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "deim-n", + "displayName": "DEIM-N", + "params": 3.78, + "mAP": 46.8, + "fps": 37.0, + "license": "Apache-2.0" + }, + { + "id": "deim-s", + "displayName": "DEIM-S", + "params": 10.32, + "mAP": 52.1, + "fps": 33.4, + "license": "Apache-2.0" + }, + { + "id": "deim-m", + "displayName": "DEIM-M", + "params": 19.59, + "mAP": 55.5, + "fps": 28.4, + "license": "Apache-2.0" + }, + { + "id": "deim-l", + "displayName": "DEIM-L", + "params": 31.24, + "mAP": 57.8, + "fps": 18.6, + "license": "Apache-2.0" + }, + { + "id": "deim-x", + "displayName": "DEIM-X", + "params": 62.62, + "mAP": 59.6, + "fps": 15.0, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-l", + "displayName": "RT-DETR-L", + "params": 32.93, + "mAP": 55.8, + "fps": 27.7, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "params": 36.59, + "mAP": 53.8, + "fps": 35.3, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "params": 42.89, + "mAP": 55.9, + "fps": 29.8, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-x", + "displayName": "RT-DETR-X", + "params": 67.37, + "mAP": 57.9, + "fps": 21.8, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + } + ], + "matched_pairs": [ + { + "a": { + "id": "deim-m", + "displayName": "DEIM-M", + "params": 19.59, + "mAP": 55.5, + "fps": 28.4, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 5.7, + "fps_delta_pct": -39.83 + }, + { + "a": { + "id": "deim-l", + "displayName": "DEIM-L", + "params": 31.24, + "mAP": 57.8, + "fps": 18.6, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 5.6, + "fps_delta_pct": -53.73 + }, + { + "a": { + "id": "deim-x", + "displayName": "DEIM-X", + "params": 62.62, + "mAP": 59.6, + "fps": 15.0, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-x", + "displayName": "RT-DETR-X", + "params": 67.37, + "mAP": 57.9, + "fps": 21.8, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 1.7, + "fps_delta_pct": -31.19 + } + ], + "aggregate": { + "n_pairs": 3, + "mean_mAP_delta_pts": 4.3, + "a_wins": 3, + "b_wins": 0, + "mean_fps_delta_pct": -41.6 + }, + "flagship": { + "a": { + "id": "deim-x", + "displayName": "DEIM-X", + "params": 62.62, + "mAP": 59.6, + "fps": 15.0, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + } + }, + "smallest": { + "a": { + "id": "deim-n", + "displayName": "DEIM-N", + "params": 3.78, + "mAP": 46.8, + "fps": 37.0, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + } + }, + "crossover": { + "low_param_leader": "deim", + "high_param_leader": "deim" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "deim-n,deim-s,deim-m,deim-l,deim-x,rtdetr-r18,rtdetr-r34,rtdetr-l,rtdetr-r50m,rtdetr-r50,rtdetr-x,rtdetr-r101" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, DEIM averages 4.3 mAP points higher across 3 paired variants on NVIDIA RTX 5070 Ti. DEIM wins 3, RT-DETR wins 0.", + "values": { + "mean_delta_pts": 4.3, + "a_wins": 3, + "b_wins": 0 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, RT-DETR is 41.6% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 41.6 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, DEIM's largest measured variant (DEIM-X) reaches 59.6 mAP; RT-DETR's (RT-DETR-R101) reaches 56.8 mAP.", + "values": { + "a": { + "id": "deim-x", + "displayName": "DEIM-X", + "params": 62.62, + "mAP": 59.6, + "fps": 15.0, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, DEIM's DEIM-N (3.78M, 46.8 mAP, 37.0 FPS) faces RT-DETR's RT-DETR-R18 (20.18M, 49.8 mAP, 47.2 FPS).", + "values": { + "a": { + "id": "deim-n", + "displayName": "DEIM-N", + "params": 3.78, + "mAP": 46.8, + "fps": 37.0, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "DEIM fields 5 measured variants, RT-DETR fields 7, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 5, + "b_n": 7 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -53.73, + -41.6, + -39.83, + -31.19, + 0.0, + 0.001, + 0.6, + 1, + 1.7, + 2, + 3.0, + 3.78, + 4, + 4.3, + 5.0, + 5.6, + 5.7, + 6, + 7.0, + 8, + 9, + 10, + 10.32, + 11, + 12, + 15.0, + 18.6, + 19.59, + 20.18, + 21.8, + 23.6, + 27.7, + 28.4, + 29.8, + 31.24, + 31.44, + 32.93, + 33.4, + 35.3, + 36.59, + 37.0, + 40.2, + 41.6, + 42.89, + 46.8, + 47.2, + 49.8, + 50, + 52.1, + 52.2, + 53.8, + 55.5, + 55.8, + 55.9, + 56.8, + 57.8, + 57.9, + 59.6, + 62.62, + 67.37, + 75, + 76.56, + 95, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-l-vs-rfdetr-l.claims.json b/article-pipeline/v2/output/dfine-l-vs-rfdetr-l.claims.json new file mode 100644 index 0000000..67b667b --- /dev/null +++ b/article-pipeline/v2/output/dfine-l-vs-rfdetr-l.claims.json @@ -0,0 +1,402 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.090391+00:00", + "slug": "dfine-l-vs-rfdetr-l", + "meta": { + "type": "vs", + "a": "dfine-l", + "b": "rfdetr-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "dfine-l", + "displayName": "D-FINE-L", + "license": "Apache-2.0", + "releaseDate": "2024-10-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5996245426151996, + "mAP_50": 0.7707410623602181, + "mAP_small": 0.45942561116012665, + "mAP_medium": 0.651596755649647, + "mAP_large": 0.7515868250873818, + "fps_mean": 21.85, + "fps_p50": 21.75, + "ms_per_image": 45.771, + "inference_ms": 38.652, + "preprocess_ms": 6.029, + "postprocess_ms": 1.09, + "p95_ms": 50.628, + "peak_vram_mb": 245.2, + "peak_ram_mb": 3.7, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:46:44.881379Z" + } + }, + "b": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "license": "Apache-2.0", + "releaseDate": "2026-02-23", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5855194790573519, + "mAP_50": 0.7602544384315711, + "mAP_small": 0.40866936129290937, + "mAP_medium": 0.6398630646189177, + "mAP_large": 0.7508947139639672, + "fps_mean": 25.08, + "fps_p50": 26.03, + "ms_per_image": 39.865, + "inference_ms": 21.946, + "preprocess_ms": 15.635, + "postprocess_ms": 2.284, + "p95_ms": 45.271, + "peak_vram_mb": 222.7, + "peak_ram_mb": 0.0, + "params_millions": 33.93, + "gflops": 340.0, + "input_size": 704, + "config": { + "batch_size": 1, + "input_size": 704, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-17T17:15:09.275538Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 2.41, + "fps": -12.88, + "params": -7.93, + "flops": -73.24, + "peak_vram": 10.1 + }, + "map_pts": { + "a": 60.0, + "b": 58.6, + "delta": 1.4 + }, + "pair": { + "params_ratio": 1.09, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00659, + "b_mAP_per_GFLOP": 0.00172 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 60.0, + "fps": 50.9 + }, + "b": { + "mAP": 58.6, + "fps": 43.8 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 60.0, + "fps": 21.9 + }, + "b": { + "mAP": 58.6, + "fps": 25.1 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 60.0, + "fps": 69.3 + }, + "b": { + "mAP": 58.6, + "fps": 47.6 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 60.0, + "fps": 72.3 + }, + "b": { + "mAP": 58.6, + "fps": 45.1 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 60.0, + "fps": 4.7 + }, + "b": { + "mAP": 58.6, + "fps": 4.0 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 60.0, + "fps": 21.9 + }, + "b": { + "mAP": 58.6, + "fps": 25.1 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 60.0, + "fps": 0.4 + }, + "b": { + "mAP": 58.6, + "fps": 0.5 + } + } + ], + "input_sizes": { + "a": 640, + "b": 704, + "match": false + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "D-FINE-L measures 1.4 mAP points higher than RF-DETR-L on NVIDIA RTX 5070 Ti (PyTorch FP32): 60.0 vs 58.6 mAP@50-95.", + "values": { + "delta_pts": 1.4, + "a_map": 60.0, + "b_map": 58.6 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RF-DETR-L is 12.88% faster: 21.9 vs 25.1 FPS (45.77 vs 39.87 ms/image).", + "values": { + "delta_pct": 12.88, + "a_fps": 21.9, + "b_fps": 25.1 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "D-FINE-L extracts 283.14% more mAP per GFLOP (0.00659 vs 0.00172).", + "values": { + "a": 0.00659, + "b": 0.00172, + "delta_pct": 283.14 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "RF-DETR-L uses less peak VRAM: 245 vs 223 MB.", + "values": { + "a_vram": 245, + "b_vram": 223 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "D-FINE-L leads on small objects: 45.9 vs 40.9 mAP_small (12.42% relative gap).", + "values": { + "a": 45.9, + "b": 40.9, + "delta_pct": 12.42 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: ONNX Runtime FP32, runtime: TensorRT FP16, runtime: TensorRT FP32, hardware: NVIDIA Jetson Orin Nano Super 8GB. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:ONNX Runtime FP32", + "runtime:TensorRT FP16", + "runtime:TensorRT FP32", + "hardware:NVIDIA Jetson Orin Nano Super 8GB" + ] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "input_size_caveat", + "kind": "caveat", + "statement": "MANDATORY CAVEAT: D-FINE-L runs at 640 px and RF-DETR-L at 704 px (each model at its author default). Say this in Setup and the verdict.", + "values": { + "a": 640, + "b": 704 + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 7, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -73.24, + -12.88, + -7.93, + 0.0, + 0.001, + 0.0017, + 0.0066, + 0.1, + 0.172, + 0.4, + 0.4087, + 0.4594, + 0.5, + 0.5855, + 0.5996, + 0.6, + 0.6399, + 0.6516, + 0.659, + 0.7509, + 0.7516, + 0.7603, + 0.7707, + 1.0, + 1.09, + 1.4, + 2, + 2.284, + 2.41, + 3, + 3.7, + 4.0, + 4.7, + 5, + 6, + 6.029, + 7, + 8, + 9, + 10, + 10.1, + 11, + 12, + 12.42, + 12.88, + 15.635, + 21.75, + 21.85, + 21.9, + 21.946, + 25.08, + 25.1, + 26.03, + 31.24, + 33.93, + 38.652, + 39.865, + 40.0, + 40.8669, + 40.9, + 43.8, + 45.1, + 45.271, + 45.771, + 45.9, + 45.9426, + 47.6, + 50.0, + 50.628, + 50.9, + 58.5519, + 58.6, + 59.9625, + 60.0, + 63.9863, + 65.1597, + 69.3, + 72.3, + 75, + 75.0895, + 75.1587, + 76.0254, + 77.0741, + 95, + 100.0, + 222.7, + 223.0, + 245.0, + 245.2, + 283.14, + 300.0, + 340.0, + 500.0, + 640.0, + 704.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-l-vs-rtdetr-r101.claims.json b/article-pipeline/v2/output/dfine-l-vs-rtdetr-r101.claims.json new file mode 100644 index 0000000..50a26c2 --- /dev/null +++ b/article-pipeline/v2/output/dfine-l-vs-rtdetr-r101.claims.json @@ -0,0 +1,392 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.158716+00:00", + "slug": "dfine-l-vs-rtdetr-r101", + "meta": { + "type": "vs", + "a": "dfine-l", + "b": "rtdetr-r101", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "dfine-l", + "displayName": "D-FINE-L", + "license": "Apache-2.0", + "releaseDate": "2024-10-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5996245426151996, + "mAP_50": 0.7707410623602181, + "mAP_small": 0.45942561116012665, + "mAP_medium": 0.651596755649647, + "mAP_large": 0.7515868250873818, + "fps_mean": 21.85, + "fps_p50": 21.75, + "ms_per_image": 45.771, + "inference_ms": 38.652, + "preprocess_ms": 6.029, + "postprocess_ms": 1.09, + "p95_ms": 50.628, + "peak_vram_mb": 245.2, + "peak_ram_mb": 3.7, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:46:44.881379Z" + } + }, + "b": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5676679551135575, + "mAP_50": 0.7399443292361064, + "mAP_small": 0.39201568552303717, + "mAP_medium": 0.6204984188822439, + "mAP_large": 0.7176352255205386, + "fps_mean": 23.57, + "fps_p50": 24.34, + "ms_per_image": 42.434, + "inference_ms": 37.87, + "preprocess_ms": 3.486, + "postprocess_ms": 1.077, + "p95_ms": 50.804, + "peak_vram_mb": 455.5, + "peak_ram_mb": 0.0, + "params_millions": 76.56, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:28:05.081801Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 5.63, + "fps": -7.3, + "params": -59.2, + "flops": -64.86, + "peak_vram": -46.17 + }, + "map_pts": { + "a": 60.0, + "b": 56.8, + "delta": 3.2 + }, + "pair": { + "params_ratio": 2.45, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00659, + "b_mAP_per_GFLOP": 0.00219 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 60.0, + "fps": 50.9 + }, + "b": { + "mAP": 56.8, + "fps": 47.5 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 60.0, + "fps": 21.9 + }, + "b": { + "mAP": 56.8, + "fps": 23.6 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 60.0, + "fps": 69.3 + }, + "b": { + "mAP": 56.7, + "fps": 88.0 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 60.0, + "fps": 72.3 + }, + "b": { + "mAP": 56.8, + "fps": 64.4 + } + } + ], + "cross_hardware": [ + { + "hardware": "a100", + "display": "NVIDIA A100", + "a": { + "mAP": 57.2, + "fps": 17.1 + }, + "b": { + "mAP": 53.9, + "fps": 18.6 + } + }, + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 60.0, + "fps": 4.7 + }, + "b": { + "mAP": 56.8, + "fps": 2.8 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 60.0, + "fps": 21.9 + }, + "b": { + "mAP": 56.8, + "fps": 23.6 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "D-FINE-L measures 3.2 mAP points higher than RT-DETR-R101 on NVIDIA RTX 5070 Ti (PyTorch FP32): 60.0 vs 56.8 mAP@50-95.", + "values": { + "delta_pts": 3.2, + "a_map": 60.0, + "b_map": 56.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R101 is 7.3% faster: 21.9 vs 23.6 FPS (45.77 vs 42.43 ms/image).", + "values": { + "delta_pct": 7.3, + "a_fps": 21.9, + "b_fps": 23.6 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "D-FINE-L extracts 200.91% more mAP per GFLOP (0.00659 vs 0.00219).", + "values": { + "a": 0.00659, + "b": 0.00219, + "delta_pct": 200.91 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "D-FINE-L uses less peak VRAM: 245 vs 456 MB.", + "values": { + "a_vram": 245, + "b_vram": 456 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "D-FINE-L leads on small objects: 45.9 vs 39.2 mAP_small (17.2% relative gap).", + "values": { + "a": 45.9, + "b": 39.2, + "delta_pct": 17.2 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: ONNX Runtime FP32, runtime: TensorRT FP32, hardware: NVIDIA Jetson Orin Nano Super 8GB. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:ONNX Runtime FP32", + "runtime:TensorRT FP32", + "hardware:NVIDIA Jetson Orin Nano Super 8GB" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -64.86, + -59.2, + -46.17, + -7.3, + 0.0, + 0.001, + 0.0022, + 0.0066, + 0.1, + 0.219, + 0.392, + 0.4594, + 0.5677, + 0.5996, + 0.6, + 0.6205, + 0.6516, + 0.659, + 0.7176, + 0.7399, + 0.7516, + 0.7707, + 1.0, + 1.077, + 1.09, + 2, + 2.45, + 2.8, + 3, + 3.2, + 3.486, + 3.7, + 4, + 4.7, + 5, + 5.63, + 6, + 6.029, + 7, + 7.3, + 8, + 9, + 10, + 11, + 12, + 17.1, + 17.2, + 18.6, + 21.75, + 21.85, + 21.9, + 23.57, + 23.6, + 24.34, + 31.24, + 37.87, + 38.652, + 39.2, + 39.2016, + 42.434, + 45.771, + 45.9, + 45.9426, + 47.5, + 50, + 50.628, + 50.804, + 50.9, + 53.9, + 56.7, + 56.7668, + 56.8, + 57.2, + 59.9625, + 60.0, + 62.0498, + 64.4, + 65.1597, + 69.3, + 71.7635, + 72.3, + 73.9944, + 75, + 75.1587, + 76.56, + 77.0741, + 88.0, + 95, + 100.0, + 200.91, + 245.0, + 245.2, + 300.0, + 455.5, + 456.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-l-vs-rtdetr-r50.claims.json b/article-pipeline/v2/output/dfine-l-vs-rtdetr-r50.claims.json new file mode 100644 index 0000000..6b3cea4 --- /dev/null +++ b/article-pipeline/v2/output/dfine-l-vs-rtdetr-r50.claims.json @@ -0,0 +1,407 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.248665+00:00", + "slug": "dfine-l-vs-rtdetr-r50", + "meta": { + "type": "vs", + "a": "dfine-l", + "b": "rtdetr-r50", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "dfine-l", + "displayName": "D-FINE-L", + "license": "Apache-2.0", + "releaseDate": "2024-10-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5996245426151996, + "mAP_50": 0.7707410623602181, + "mAP_small": 0.45942561116012665, + "mAP_medium": 0.651596755649647, + "mAP_large": 0.7515868250873818, + "fps_mean": 21.85, + "fps_p50": 21.75, + "ms_per_image": 45.771, + "inference_ms": 38.652, + "preprocess_ms": 6.029, + "postprocess_ms": 1.09, + "p95_ms": 50.628, + "peak_vram_mb": 245.2, + "peak_ram_mb": 3.7, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:46:44.881379Z" + } + }, + "b": { + "id": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5587851459971446, + "mAP_50": 0.730485227930135, + "mAP_small": 0.3826303200265574, + "mAP_medium": 0.6127541876937886, + "mAP_large": 0.7053415491294674, + "fps_mean": 29.76, + "fps_p50": 30.87, + "ms_per_image": 33.598, + "inference_ms": 29.052, + "preprocess_ms": 3.448, + "postprocess_ms": 1.099, + "p95_ms": 39.857, + "peak_vram_mb": 326.4, + "peak_ram_mb": 0.0, + "params_millions": 42.89, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:24:24.814896Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 7.31, + "fps": -26.58, + "params": -27.16, + "flops": -33.09, + "peak_vram": -24.88 + }, + "map_pts": { + "a": 60.0, + "b": 55.9, + "delta": 4.1 + }, + "pair": { + "params_ratio": 1.37, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00659, + "b_mAP_per_GFLOP": 0.00411 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 60.0, + "fps": 50.9 + }, + "b": { + "mAP": 55.9, + "fps": 62.8 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 60.0, + "fps": 21.9 + }, + "b": { + "mAP": 55.9, + "fps": 29.8 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 60.0, + "fps": 69.3 + }, + "b": { + "mAP": 55.8, + "fps": 97.4 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 60.0, + "fps": 72.3 + }, + "b": { + "mAP": 55.8, + "fps": 87.4 + } + } + ], + "cross_hardware": [ + { + "hardware": "a100", + "display": "NVIDIA A100", + "a": { + "mAP": 57.2, + "fps": 17.1 + }, + "b": { + "mAP": 52.7, + "fps": 22.7 + } + }, + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 60.0, + "fps": 4.7 + }, + "b": { + "mAP": 55.9, + "fps": 4.3 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 60.0, + "fps": 21.9 + }, + "b": { + "mAP": 55.9, + "fps": 29.8 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 60.0, + "fps": 0.4 + }, + "b": { + "mAP": 55.9, + "fps": 0.3 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "D-FINE-L measures 4.1 mAP points higher than RT-DETR-R50 on NVIDIA RTX 5070 Ti (PyTorch FP32): 60.0 vs 55.9 mAP@50-95.", + "values": { + "delta_pts": 4.1, + "a_map": 60.0, + "b_map": 55.9 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R50 is 26.58% faster: 21.9 vs 29.8 FPS (45.77 vs 33.6 ms/image).", + "values": { + "delta_pct": 26.58, + "a_fps": 21.9, + "b_fps": 29.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "D-FINE-L extracts 60.34% more mAP per GFLOP (0.00659 vs 0.00411).", + "values": { + "a": 0.00659, + "b": 0.00411, + "delta_pct": 60.34 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "D-FINE-L uses less peak VRAM: 245 vs 326 MB.", + "values": { + "a_vram": 245, + "b_vram": 326 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "D-FINE-L leads on small objects: 45.9 vs 38.3 mAP_small (20.07% relative gap).", + "values": { + "a": 45.9, + "b": 38.3, + "delta_pct": 20.07 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: NVIDIA Jetson Orin Nano Super 8GB, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:NVIDIA Jetson Orin Nano Super 8GB", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -33.09, + -27.16, + -26.58, + -24.88, + 0.0, + 0.001, + 0.0041, + 0.0066, + 0.1, + 0.3, + 0.3826, + 0.4, + 0.411, + 0.4594, + 0.5588, + 0.5996, + 0.6, + 0.6128, + 0.6516, + 0.659, + 0.7053, + 0.7305, + 0.7516, + 0.7707, + 1.0, + 1.09, + 1.099, + 1.37, + 2, + 3, + 3.448, + 3.7, + 4, + 4.1, + 4.3, + 4.7, + 5, + 6, + 6.029, + 7, + 7.31, + 8, + 9, + 10, + 11, + 12, + 17.1, + 20.07, + 21.75, + 21.85, + 21.9, + 22.7, + 26.58, + 29.052, + 29.76, + 29.8, + 30.0, + 30.87, + 31.24, + 33.598, + 38.263, + 38.3, + 38.652, + 39.857, + 40.0, + 42.89, + 45.771, + 45.9, + 45.9426, + 50, + 50.628, + 50.9, + 52.7, + 55.8, + 55.8785, + 55.9, + 57.2, + 59.9625, + 60.0, + 60.34, + 61.2754, + 62.8, + 65.1597, + 69.3, + 70.5342, + 72.3, + 73.0485, + 75, + 75.1587, + 77.0741, + 87.4, + 95, + 97.4, + 100.0, + 245.0, + 245.2, + 300.0, + 326.0, + 326.4, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-l-vs-rtdetr-r50m.claims.json b/article-pipeline/v2/output/dfine-l-vs-rtdetr-r50m.claims.json new file mode 100644 index 0000000..f95b1cf --- /dev/null +++ b/article-pipeline/v2/output/dfine-l-vs-rtdetr-r50m.claims.json @@ -0,0 +1,387 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.886020+00:00", + "slug": "dfine-l-vs-rtdetr-r50m", + "meta": { + "type": "vs", + "a": "dfine-l", + "b": "rtdetr-r50m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "dfine-l", + "displayName": "D-FINE-L", + "license": "Apache-2.0", + "releaseDate": "2024-10-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5996245426151996, + "mAP_50": 0.7707410623602181, + "mAP_small": 0.45942561116012665, + "mAP_medium": 0.651596755649647, + "mAP_large": 0.7515868250873818, + "fps_mean": 21.85, + "fps_p50": 21.75, + "ms_per_image": 45.771, + "inference_ms": 38.652, + "preprocess_ms": 6.029, + "postprocess_ms": 1.09, + "p95_ms": 50.628, + "peak_vram_mb": 245.2, + "peak_ram_mb": 3.7, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:46:44.881379Z" + } + }, + "b": { + "id": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "license": "Apache-2.0", + "releaseDate": "2026-04-22", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5381740345956989, + "mAP_50": 0.7122612376832924, + "mAP_small": 0.37622664093206376, + "mAP_medium": 0.5945722582993921, + "mAP_large": 0.6898610320313556, + "fps_mean": 35.26, + "fps_p50": 36.57, + "ms_per_image": 28.362, + "inference_ms": 23.897, + "preprocess_ms": 3.41, + "postprocess_ms": 1.054, + "p95_ms": 34.582, + "peak_vram_mb": 301.7, + "peak_ram_mb": 0.0, + "params_millions": 36.59, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:24:50.077554Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 11.42, + "fps": -38.03, + "params": -14.62, + "flops": null, + "peak_vram": -18.73 + }, + "map_pts": { + "a": 60.0, + "b": 53.8, + "delta": 6.2 + }, + "pair": { + "params_ratio": 1.17, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00659, + "b_mAP_per_GFLOP": null + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 60.0, + "fps": 50.9 + }, + "b": { + "mAP": 53.8, + "fps": 76.4 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 60.0, + "fps": 21.9 + }, + "b": { + "mAP": 53.8, + "fps": 35.3 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 60.0, + "fps": 69.3 + }, + "b": { + "mAP": 53.9, + "fps": 110.1 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 60.0, + "fps": 72.3 + }, + "b": { + "mAP": 53.9, + "fps": 102.8 + } + } + ], + "cross_hardware": [ + { + "hardware": "a100", + "display": "NVIDIA A100", + "a": { + "mAP": 57.2, + "fps": 17.1 + }, + "b": { + "mAP": 50.8, + "fps": 25.1 + } + }, + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 60.0, + "fps": 4.7 + }, + "b": { + "mAP": 53.8, + "fps": 5.0 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 60.0, + "fps": 21.9 + }, + "b": { + "mAP": 53.8, + "fps": 35.3 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 60.0, + "fps": 0.4 + }, + "b": { + "mAP": 53.8, + "fps": 0.4 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "D-FINE-L measures 6.2 mAP points higher than RT-DETR-R50m on NVIDIA RTX 5070 Ti (PyTorch FP32): 60.0 vs 53.8 mAP@50-95.", + "values": { + "delta_pts": 6.2, + "a_map": 60.0, + "b_map": 53.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R50m is 38.03% faster: 21.9 vs 35.3 FPS (45.77 vs 28.36 ms/image).", + "values": { + "delta_pct": 38.03, + "a_fps": 21.9, + "b_fps": 35.3 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "D-FINE-L uses less peak VRAM: 245 vs 302 MB.", + "values": { + "a_vram": 245, + "b_vram": 302 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "D-FINE-L leads on small objects: 45.9 vs 37.6 mAP_small (22.11% relative gap).", + "values": { + "a": 45.9, + "b": 37.6, + "delta_pct": 22.11 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -38.03, + -18.73, + -14.62, + 0.0, + 0.001, + 0.0066, + 0.1, + 0.3762, + 0.4, + 0.4594, + 0.5382, + 0.5946, + 0.5996, + 0.6, + 0.6516, + 0.659, + 0.6899, + 0.7123, + 0.7516, + 0.7707, + 1.0, + 1.054, + 1.09, + 1.17, + 2, + 3, + 3.41, + 3.7, + 4, + 4.7, + 5.0, + 6, + 6.029, + 6.2, + 7, + 8, + 9, + 10, + 11, + 11.42, + 12, + 17.1, + 21.75, + 21.85, + 21.9, + 22.11, + 23.897, + 25.1, + 28.362, + 31.24, + 34.582, + 35.26, + 35.3, + 36.57, + 36.59, + 37.6, + 37.6227, + 38.03, + 38.652, + 40.0, + 45.771, + 45.9, + 45.9426, + 50, + 50.628, + 50.8, + 50.9, + 53.8, + 53.8174, + 53.9, + 57.2, + 59.4572, + 59.9625, + 60.0, + 65.1597, + 68.9861, + 69.3, + 71.2261, + 72.3, + 75, + 75.1587, + 76.4, + 77.0741, + 95, + 100.0, + 102.8, + 110.1, + 245.0, + 245.2, + 300.0, + 301.7, + 302.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-l-vs-yolox-l.claims.json b/article-pipeline/v2/output/dfine-l-vs-yolox-l.claims.json new file mode 100644 index 0000000..dd0ac16 --- /dev/null +++ b/article-pipeline/v2/output/dfine-l-vs-yolox-l.claims.json @@ -0,0 +1,376 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.977504+00:00", + "slug": "dfine-l-vs-yolox-l", + "meta": { + "type": "vs", + "a": "dfine-l", + "b": "yolox-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "dfine-l", + "displayName": "D-FINE-L", + "license": "Apache-2.0", + "releaseDate": "2024-10-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5996245426151996, + "mAP_50": 0.7707410623602181, + "mAP_small": 0.45942561116012665, + "mAP_medium": 0.651596755649647, + "mAP_large": 0.7515868250873818, + "fps_mean": 21.85, + "fps_p50": 21.75, + "ms_per_image": 45.771, + "inference_ms": 38.652, + "preprocess_ms": 6.029, + "postprocess_ms": 1.09, + "p95_ms": 50.628, + "peak_vram_mb": 245.2, + "peak_ram_mb": 3.7, + "params_millions": 31.24, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:46:44.881379Z" + } + }, + "b": { + "id": "yolox-l", + "displayName": "YOLOX-L", + "license": "Apache-2.0", + "releaseDate": "2021-07-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5542460212991265, + "mAP_50": 0.7302189854897115, + "mAP_small": 0.39036786287091835, + "mAP_medium": 0.5988447680065735, + "mAP_large": 0.685177367489107, + "fps_mean": 42.84, + "fps_p50": 42.78, + "ms_per_image": 23.341, + "inference_ms": 13.911, + "preprocess_ms": 6.863, + "postprocess_ms": 2.567, + "p95_ms": 27.837, + "peak_vram_mb": 307.0, + "peak_ram_mb": 0.0, + "params_millions": 54.21, + "gflops": 78.01, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:20:38.605152Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 8.19, + "fps": -49.0, + "params": -42.37, + "flops": 16.65, + "peak_vram": -20.13 + }, + "map_pts": { + "a": 60.0, + "b": 55.4, + "delta": 4.6 + }, + "pair": { + "params_ratio": 1.74, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00659, + "b_mAP_per_GFLOP": 0.0071 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 60.0, + "fps": 50.9 + }, + "b": { + "mAP": 55.4, + "fps": 50.6 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 60.0, + "fps": 21.9 + }, + "b": { + "mAP": 55.4, + "fps": 42.8 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 60.0, + "fps": 69.3 + }, + "b": { + "mAP": 55.4, + "fps": 79.2 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 60.0, + "fps": 72.3 + }, + "b": { + "mAP": 55.4, + "fps": 61.8 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 60.0, + "fps": 4.7 + }, + "b": { + "mAP": 55.4, + "fps": 5.8 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 60.0, + "fps": 21.9 + }, + "b": { + "mAP": 55.4, + "fps": 42.8 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 60.0, + "fps": 0.4 + }, + "b": { + "mAP": 53.9, + "fps": 0.4 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "D-FINE-L measures 4.6 mAP points higher than YOLOX-L on NVIDIA RTX 5070 Ti (PyTorch FP32): 60.0 vs 55.4 mAP@50-95.", + "values": { + "delta_pts": 4.6, + "a_map": 60.0, + "b_map": 55.4 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "YOLOX-L is 49.0% faster: 21.9 vs 42.8 FPS (45.77 vs 23.34 ms/image).", + "values": { + "delta_pct": 49.0, + "a_fps": 21.9, + "b_fps": 42.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "D-FINE-L uses less peak VRAM: 245 vs 307 MB.", + "values": { + "a_vram": 245, + "b_vram": 307 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "D-FINE-L leads on small objects: 45.9 vs 39.0 mAP_small (17.69% relative gap).", + "values": { + "a": 45.9, + "b": 39.0, + "delta_pct": 17.69 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: TensorRT FP32, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:TensorRT FP32", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -49.0, + -42.37, + -20.13, + 0.0, + 0.001, + 0.0066, + 0.0071, + 0.1, + 0.3904, + 0.4, + 0.4594, + 0.5542, + 0.5988, + 0.5996, + 0.6, + 0.6516, + 0.659, + 0.6852, + 0.71, + 0.7302, + 0.7516, + 0.7707, + 1.0, + 1.09, + 1.74, + 2, + 2.567, + 3, + 3.7, + 4, + 4.6, + 4.7, + 5, + 5.8, + 6, + 6.029, + 6.863, + 7, + 8, + 8.19, + 9, + 10, + 11, + 12, + 13.911, + 16.65, + 17.69, + 21.75, + 21.85, + 21.9, + 23.341, + 27.837, + 31.24, + 38.652, + 39.0, + 39.0368, + 40.0, + 42.78, + 42.8, + 42.84, + 45.771, + 45.9, + 45.9426, + 49.0, + 50, + 50.6, + 50.628, + 50.9, + 53.9, + 54.21, + 55.4, + 55.4246, + 59.8845, + 59.9625, + 60.0, + 61.8, + 65.1597, + 68.5177, + 69.3, + 72.3, + 73.0219, + 75, + 75.1587, + 77.0741, + 78.01, + 79.2, + 95, + 100.0, + 245.0, + 245.2, + 300.0, + 307.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-m-vs-rtdetr-r34.claims.json b/article-pipeline/v2/output/dfine-m-vs-rtdetr-r34.claims.json new file mode 100644 index 0000000..1b71656 --- /dev/null +++ b/article-pipeline/v2/output/dfine-m-vs-rtdetr-r34.claims.json @@ -0,0 +1,401 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.328294+00:00", + "slug": "dfine-m-vs-rtdetr-r34", + "meta": { + "type": "vs", + "a": "dfine-m", + "b": "rtdetr-r34", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "dfine-m", + "displayName": "D-FINE-M", + "license": "Apache-2.0", + "releaseDate": "2024-10-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5782253623733835, + "mAP_50": 0.7467018799159137, + "mAP_small": 0.429814826506027, + "mAP_medium": 0.6141089399545165, + "mAP_large": 0.729734668604178, + "fps_mean": 28.75, + "fps_p50": 29.07, + "ms_per_image": 34.788, + "inference_ms": 27.652, + "preprocess_ms": 6.021, + "postprocess_ms": 1.114, + "p95_ms": 39.663, + "peak_vram_mb": 188.8, + "peak_ram_mb": 0.0, + "params_millions": 19.59, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:46:11.122560Z" + } + }, + "b": { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5222774052742704, + "mAP_50": 0.6960054462347887, + "mAP_small": 0.34944822231901834, + "mAP_medium": 0.5588794736724594, + "mAP_large": 0.6678097562687866, + "fps_mean": 40.16, + "fps_p50": 41.3, + "ms_per_image": 24.903, + "inference_ms": 20.296, + "preprocess_ms": 3.515, + "postprocess_ms": 1.092, + "p95_ms": 29.392, + "peak_vram_mb": 212.9, + "peak_ram_mb": 0.0, + "params_millions": 31.44, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:23:56.394255Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 10.71, + "fps": -28.41, + "params": -37.69, + "flops": -37.36, + "peak_vram": -11.32 + }, + "map_pts": { + "a": 57.8, + "b": 52.2, + "delta": 5.6 + }, + "pair": { + "params_ratio": 1.6, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.01014, + "b_mAP_per_GFLOP": 0.00574 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 57.8, + "fps": 59.8 + }, + "b": { + "mAP": 52.2, + "fps": 77.7 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 57.8, + "fps": 28.8 + }, + "b": { + "mAP": 52.2, + "fps": 40.2 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 57.6, + "fps": 74.5 + }, + "b": { + "mAP": 52.3, + "fps": 108.4 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 57.8, + "fps": 85.3 + }, + "b": { + "mAP": 52.2, + "fps": 98.1 + } + } + ], + "cross_hardware": [ + { + "hardware": "a100", + "display": "NVIDIA A100", + "a": { + "mAP": 55.1, + "fps": 21.7 + }, + "b": { + "mAP": 48.2, + "fps": 27.8 + } + }, + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 57.8, + "fps": 7.0 + }, + "b": { + "mAP": 52.2, + "fps": 8.0 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 57.8, + "fps": 28.8 + }, + "b": { + "mAP": 52.2, + "fps": 40.2 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 57.8, + "fps": 0.6 + }, + "b": { + "mAP": 52.2, + "fps": 0.6 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "D-FINE-M measures 5.6 mAP points higher than RT-DETR-R34 on NVIDIA RTX 5070 Ti (PyTorch FP32): 57.8 vs 52.2 mAP@50-95.", + "values": { + "delta_pts": 5.6, + "a_map": 57.8, + "b_map": 52.2 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R34 is 28.41% faster: 28.8 vs 40.2 FPS (34.79 vs 24.9 ms/image).", + "values": { + "delta_pct": 28.41, + "a_fps": 28.8, + "b_fps": 40.2 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "D-FINE-M extracts 76.66% more mAP per GFLOP (0.01014 vs 0.00574).", + "values": { + "a": 0.01014, + "b": 0.00574, + "delta_pct": 76.66 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "D-FINE-M uses less peak VRAM: 189 vs 213 MB.", + "values": { + "a_vram": 189, + "b_vram": 213 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "D-FINE-M leads on small objects: 43.0 vs 34.9 mAP_small (23.0% relative gap).", + "values": { + "a": 43.0, + "b": 34.9, + "delta_pct": 23.0 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -37.69, + -37.36, + -28.41, + -11.32, + 0.0, + 0.001, + 0.0057, + 0.0101, + 0.1, + 0.3494, + 0.4298, + 0.5223, + 0.5589, + 0.574, + 0.5782, + 0.6, + 0.6141, + 0.6678, + 0.696, + 0.7297, + 0.7467, + 1.0, + 1.014, + 1.092, + 1.114, + 1.6, + 2, + 3, + 3.515, + 4, + 5, + 5.6, + 6, + 6.021, + 7.0, + 8.0, + 9, + 10, + 10.71, + 11, + 12, + 19.59, + 20.296, + 21.7, + 23.0, + 24.903, + 27.652, + 27.8, + 28.41, + 28.75, + 28.8, + 29.07, + 29.392, + 31.44, + 34.788, + 34.9, + 34.9448, + 39.663, + 40.16, + 40.2, + 41.3, + 42.9815, + 43.0, + 48.2, + 50, + 52.2, + 52.2277, + 52.3, + 55.1, + 55.8879, + 57.6, + 57.8, + 57.8225, + 59.8, + 60.0, + 61.4109, + 66.781, + 69.6005, + 72.9735, + 74.5, + 74.6702, + 75, + 76.66, + 77.7, + 85.3, + 95, + 98.1, + 100.0, + 108.4, + 188.8, + 189.0, + 212.9, + 213.0, + 300.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-m-vs-rtdetr-r50m.claims.json b/article-pipeline/v2/output/dfine-m-vs-rtdetr-r50m.claims.json new file mode 100644 index 0000000..c297eac --- /dev/null +++ b/article-pipeline/v2/output/dfine-m-vs-rtdetr-r50m.claims.json @@ -0,0 +1,387 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:44.047823+00:00", + "slug": "dfine-m-vs-rtdetr-r50m", + "meta": { + "type": "vs", + "a": "dfine-m", + "b": "rtdetr-r50m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "dfine-m", + "displayName": "D-FINE-M", + "license": "Apache-2.0", + "releaseDate": "2024-10-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5782253623733835, + "mAP_50": 0.7467018799159137, + "mAP_small": 0.429814826506027, + "mAP_medium": 0.6141089399545165, + "mAP_large": 0.729734668604178, + "fps_mean": 28.75, + "fps_p50": 29.07, + "ms_per_image": 34.788, + "inference_ms": 27.652, + "preprocess_ms": 6.021, + "postprocess_ms": 1.114, + "p95_ms": 39.663, + "peak_vram_mb": 188.8, + "peak_ram_mb": 0.0, + "params_millions": 19.59, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:46:11.122560Z" + } + }, + "b": { + "id": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "license": "Apache-2.0", + "releaseDate": "2026-04-22", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5381740345956989, + "mAP_50": 0.7122612376832924, + "mAP_small": 0.37622664093206376, + "mAP_medium": 0.5945722582993921, + "mAP_large": 0.6898610320313556, + "fps_mean": 35.26, + "fps_p50": 36.57, + "ms_per_image": 28.362, + "inference_ms": 23.897, + "preprocess_ms": 3.41, + "postprocess_ms": 1.054, + "p95_ms": 34.582, + "peak_vram_mb": 301.7, + "peak_ram_mb": 0.0, + "params_millions": 36.59, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:24:50.077554Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 7.44, + "fps": -18.46, + "params": -46.46, + "flops": null, + "peak_vram": -37.42 + }, + "map_pts": { + "a": 57.8, + "b": 53.8, + "delta": 4.0 + }, + "pair": { + "params_ratio": 1.87, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.01014, + "b_mAP_per_GFLOP": null + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 57.8, + "fps": 59.8 + }, + "b": { + "mAP": 53.8, + "fps": 76.4 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 57.8, + "fps": 28.8 + }, + "b": { + "mAP": 53.8, + "fps": 35.3 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 57.6, + "fps": 74.5 + }, + "b": { + "mAP": 53.9, + "fps": 110.1 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 57.8, + "fps": 85.3 + }, + "b": { + "mAP": 53.9, + "fps": 102.8 + } + } + ], + "cross_hardware": [ + { + "hardware": "a100", + "display": "NVIDIA A100", + "a": { + "mAP": 55.1, + "fps": 21.7 + }, + "b": { + "mAP": 50.8, + "fps": 25.1 + } + }, + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 57.8, + "fps": 7.0 + }, + "b": { + "mAP": 53.8, + "fps": 5.0 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 57.8, + "fps": 28.8 + }, + "b": { + "mAP": 53.8, + "fps": 35.3 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 57.8, + "fps": 0.6 + }, + "b": { + "mAP": 53.8, + "fps": 0.4 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "D-FINE-M measures 4.0 mAP points higher than RT-DETR-R50m on NVIDIA RTX 5070 Ti (PyTorch FP32): 57.8 vs 53.8 mAP@50-95.", + "values": { + "delta_pts": 4.0, + "a_map": 57.8, + "b_map": 53.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R50m is 18.46% faster: 28.8 vs 35.3 FPS (34.79 vs 28.36 ms/image).", + "values": { + "delta_pct": 18.46, + "a_fps": 28.8, + "b_fps": 35.3 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "D-FINE-M uses less peak VRAM: 189 vs 302 MB.", + "values": { + "a_vram": 189, + "b_vram": 302 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "D-FINE-M leads on small objects: 43.0 vs 37.6 mAP_small (14.24% relative gap).", + "values": { + "a": 43.0, + "b": 37.6, + "delta_pct": 14.24 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: NVIDIA Jetson Orin Nano Super 8GB, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:NVIDIA Jetson Orin Nano Super 8GB", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -46.46, + -37.42, + -18.46, + 0.0, + 0.001, + 0.0101, + 0.1, + 0.3762, + 0.4, + 0.4298, + 0.5382, + 0.5782, + 0.5946, + 0.6, + 0.6141, + 0.6899, + 0.7123, + 0.7297, + 0.7467, + 1.0, + 1.014, + 1.054, + 1.114, + 1.87, + 2, + 3, + 3.41, + 4.0, + 5.0, + 6, + 6.021, + 7.0, + 7.44, + 8, + 9, + 10, + 11, + 12, + 14.24, + 18.46, + 19.59, + 21.7, + 23.897, + 25.1, + 27.652, + 28.362, + 28.75, + 28.8, + 29.07, + 34.582, + 34.788, + 35.26, + 35.3, + 36.57, + 36.59, + 37.6, + 37.6227, + 39.663, + 40.0, + 42.9815, + 43.0, + 50, + 50.8, + 53.8, + 53.8174, + 53.9, + 55.1, + 57.6, + 57.8, + 57.8225, + 59.4572, + 59.8, + 60.0, + 61.4109, + 68.9861, + 71.2261, + 72.9735, + 74.5, + 74.6702, + 75, + 76.4, + 85.3, + 95, + 100.0, + 102.8, + 110.1, + 188.8, + 189.0, + 300.0, + 301.7, + 302.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-n-vs-yolov9t.claims.json b/article-pipeline/v2/output/dfine-n-vs-yolov9t.claims.json new file mode 100644 index 0000000..740cbd8 --- /dev/null +++ b/article-pipeline/v2/output/dfine-n-vs-yolov9t.claims.json @@ -0,0 +1,376 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:44.136260+00:00", + "slug": "dfine-n-vs-yolov9t", + "meta": { + "type": "vs", + "a": "dfine-n", + "b": "yolov9t", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "dfine-n", + "displayName": "D-FINE-N", + "license": "Apache-2.0", + "releaseDate": "2024-10-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.457506099610084, + "mAP_50": 0.6273955560653228, + "mAP_small": 0.25794608999739005, + "mAP_medium": 0.5147011969399619, + "mAP_large": 0.6162508784225, + "fps_mean": 32.54, + "fps_p50": 33.05, + "ms_per_image": 30.734, + "inference_ms": 22.192, + "preprocess_ms": 7.261, + "postprocess_ms": 1.281, + "p95_ms": 36.882, + "peak_vram_mb": 70.2, + "peak_ram_mb": 6.5, + "params_millions": 3.78, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:45:17.563790Z" + } + }, + "b": { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "license": "MIT", + "releaseDate": "2024-02-21", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.4177885668656335, + "mAP_50": 0.5661372018894871, + "mAP_small": 0.2225871925542081, + "mAP_medium": 0.46300230914348306, + "mAP_large": 0.57077934037184, + "fps_mean": 31.94, + "fps_p50": 33.42, + "ms_per_image": 31.307, + "inference_ms": 25.151, + "preprocess_ms": 3.907, + "postprocess_ms": 2.249, + "p95_ms": 37.685, + "peak_vram_mb": 48.0, + "peak_ram_mb": 340.2, + "params_millions": 2.04, + "gflops": 3.98, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:18:14.276169Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 9.51, + "fps": 1.88, + "params": 85.29, + "flops": 75.88, + "peak_vram": 46.25 + }, + "map_pts": { + "a": 45.8, + "b": 41.8, + "delta": 4.0 + }, + "pair": { + "params_ratio": 1.85, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.06536, + "b_mAP_per_GFLOP": 0.10497 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 45.8, + "fps": 78.9 + }, + "b": { + "mAP": 41.8, + "fps": 52.1 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 45.8, + "fps": 32.5 + }, + "b": { + "mAP": 41.8, + "fps": 31.9 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 45.8, + "fps": 79.7 + }, + "b": { + "mAP": 41.8, + "fps": 59.0 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 45.8, + "fps": 98.7 + }, + "b": { + "mAP": 41.8, + "fps": 72.0 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 45.8, + "fps": 11.7 + }, + "b": { + "mAP": 41.8, + "fps": 9.9 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 45.8, + "fps": 32.5 + }, + "b": { + "mAP": 41.8, + "fps": 31.9 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 45.8, + "fps": 2.3 + }, + "b": { + "mAP": 41.4, + "fps": 2.9 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "D-FINE-N measures 4.0 mAP points higher than YOLOv9-T on NVIDIA RTX 5070 Ti (PyTorch FP32): 45.8 vs 41.8 mAP@50-95.", + "values": { + "delta_pts": 4.0, + "a_map": 45.8, + "b_map": 41.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "YOLOv9-T extracts 37.73% more mAP per GFLOP (0.06536 vs 0.10497).", + "values": { + "a": 0.06536, + "b": 0.10497, + "delta_pct": 37.73 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "YOLOv9-T uses less peak VRAM: 70 vs 48 MB.", + "values": { + "a_vram": 70, + "b_vram": 48 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "D-FINE-N leads on small objects: 25.8 vs 22.3 mAP_small (15.89% relative gap).", + "values": { + "a": 25.8, + "b": 22.3, + "delta_pct": 15.89 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + 0.0, + 0.001, + 0.0654, + 0.1, + 0.105, + 0.2226, + 0.2579, + 0.4178, + 0.4575, + 0.463, + 0.5147, + 0.5661, + 0.5708, + 0.6, + 0.6163, + 0.6274, + 1.0, + 1.281, + 1.85, + 1.88, + 2, + 2.04, + 2.249, + 2.3, + 2.9, + 3, + 3.78, + 3.907, + 3.98, + 4.0, + 5, + 6, + 6.5, + 6.536, + 7, + 7.261, + 8, + 9, + 9.51, + 9.9, + 10, + 10.497, + 11, + 11.7, + 12, + 15.89, + 22.192, + 22.2587, + 22.3, + 25.151, + 25.7946, + 25.8, + 30.734, + 31.307, + 31.9, + 31.94, + 32.5, + 32.54, + 33.05, + 33.42, + 36.882, + 37.685, + 37.73, + 41.4, + 41.7789, + 41.8, + 45.7506, + 45.8, + 46.25, + 46.3002, + 48.0, + 50, + 51.4701, + 52.1, + 56.6137, + 57.0779, + 59.0, + 60.0, + 61.6251, + 62.7396, + 70.0, + 70.2, + 72.0, + 75, + 75.88, + 78.9, + 79.7, + 85.29, + 95, + 98.7, + 100.0, + 300.0, + 340.2, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-s-vs-rtdetr-r18.claims.json b/article-pipeline/v2/output/dfine-s-vs-rtdetr-r18.claims.json new file mode 100644 index 0000000..ce3015e --- /dev/null +++ b/article-pipeline/v2/output/dfine-s-vs-rtdetr-r18.claims.json @@ -0,0 +1,408 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.398061+00:00", + "slug": "dfine-s-vs-rtdetr-r18", + "meta": { + "type": "vs", + "a": "dfine-s", + "b": "rtdetr-r18", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "dfine-s", + "displayName": "D-FINE-S", + "license": "Apache-2.0", + "releaseDate": "2024-10-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5339360463706191, + "mAP_50": 0.6979289246485497, + "mAP_small": 0.37902381194039897, + "mAP_medium": 0.5785906686133171, + "mAP_large": 0.6701666032622975, + "fps_mean": 34.17, + "fps_p50": 34.82, + "ms_per_image": 29.267, + "inference_ms": 22.32, + "preprocess_ms": 5.837, + "postprocess_ms": 1.111, + "p95_ms": 34.014, + "peak_vram_mb": 141.6, + "peak_ram_mb": 0.7, + "params_millions": 10.32, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:45:41.588316Z" + } + }, + "b": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.49787927521150827, + "mAP_50": 0.663704009633566, + "mAP_small": 0.32710231823159713, + "mAP_medium": 0.5484637312981646, + "mAP_large": 0.6407169138607522, + "fps_mean": 47.22, + "fps_p50": 48.89, + "ms_per_image": 21.179, + "inference_ms": 16.645, + "preprocess_ms": 3.452, + "postprocess_ms": 1.082, + "p95_ms": 25.931, + "peak_vram_mb": 168.9, + "peak_ram_mb": 2.0, + "params_millions": 20.18, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:23:32.849123Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 7.24, + "fps": -27.64, + "params": -48.86, + "flops": -58.33, + "peak_vram": -16.16 + }, + "map_pts": { + "a": 53.4, + "b": 49.8, + "delta": 3.6 + }, + "pair": { + "params_ratio": 1.96, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.02136, + "b_mAP_per_GFLOP": 0.0083 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 53.4, + "fps": 70.9 + }, + "b": { + "mAP": 49.8, + "fps": 92.5 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 53.4, + "fps": 34.2 + }, + "b": { + "mAP": 49.8, + "fps": 47.2 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 53.2, + "fps": 86.5 + }, + "b": { + "mAP": 49.8, + "fps": 113.4 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 53.4, + "fps": 98.7 + }, + "b": { + "mAP": 49.8, + "fps": 120.6 + } + } + ], + "cross_hardware": [ + { + "hardware": "a100", + "display": "NVIDIA A100", + "a": { + "mAP": 50.7, + "fps": 24.0 + }, + "b": { + "mAP": 45.6, + "fps": 27.4 + } + }, + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 53.4, + "fps": 10.3 + }, + "b": { + "mAP": 49.8, + "fps": 10.2 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 53.4, + "fps": 34.2 + }, + "b": { + "mAP": 49.8, + "fps": 47.2 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 53.4, + "fps": 1.1 + }, + "b": { + "mAP": 49.7, + "fps": 0.8 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "D-FINE-S measures 3.6 mAP points higher than RT-DETR-R18 on NVIDIA RTX 5070 Ti (PyTorch FP32): 53.4 vs 49.8 mAP@50-95.", + "values": { + "delta_pts": 3.6, + "a_map": 53.4, + "b_map": 49.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R18 is 27.64% faster: 34.2 vs 47.2 FPS (29.27 vs 21.18 ms/image).", + "values": { + "delta_pct": 27.64, + "a_fps": 34.2, + "b_fps": 47.2 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "D-FINE-S extracts 157.35% more mAP per GFLOP (0.02136 vs 0.0083).", + "values": { + "a": 0.02136, + "b": 0.0083, + "delta_pct": 157.35 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "D-FINE-S uses less peak VRAM: 142 vs 169 MB.", + "values": { + "a_vram": 142, + "b_vram": 169 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "D-FINE-S leads on small objects: 37.9 vs 32.7 mAP_small (15.87% relative gap).", + "values": { + "a": 37.9, + "b": 32.7, + "delta_pct": 15.87 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -58.33, + -48.86, + -27.64, + -16.16, + 0.0, + 0.001, + 0.0083, + 0.0214, + 0.1, + 0.3271, + 0.379, + 0.4979, + 0.5339, + 0.5485, + 0.5786, + 0.6, + 0.6407, + 0.6637, + 0.6702, + 0.6979, + 0.7, + 0.8, + 0.83, + 1.0, + 1.082, + 1.1, + 1.111, + 1.96, + 2.0, + 2.136, + 3, + 3.452, + 3.6, + 4, + 5, + 5.837, + 6, + 7, + 7.24, + 8, + 9, + 10, + 10.2, + 10.3, + 10.32, + 11, + 12, + 15.87, + 16.645, + 20.18, + 21.179, + 22.32, + 24.0, + 25.931, + 27.4, + 27.64, + 29.267, + 32.7, + 32.7102, + 34.014, + 34.17, + 34.2, + 34.82, + 37.9, + 37.9024, + 45.6, + 47.2, + 47.22, + 48.89, + 49.7, + 49.7879, + 49.8, + 50, + 50.7, + 53.2, + 53.3936, + 53.4, + 54.8464, + 57.8591, + 60.0, + 64.0717, + 66.3704, + 67.0167, + 69.7929, + 70.0, + 70.9, + 75, + 80.0, + 86.5, + 92.5, + 95, + 98.7, + 100.0, + 113.4, + 120.6, + 141.6, + 142.0, + 157.35, + 168.9, + 169.0, + 300.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-s-vs-yolov9s.claims.json b/article-pipeline/v2/output/dfine-s-vs-yolov9s.claims.json new file mode 100644 index 0000000..d5cf7e3 --- /dev/null +++ b/article-pipeline/v2/output/dfine-s-vs-yolov9s.claims.json @@ -0,0 +1,392 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T21:24:12.412409+00:00", + "slug": "dfine-s-vs-yolov9s", + "meta": { + "type": "vs", + "a": "dfine-s", + "b": "yolov9s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "dfine-s", + "displayName": "D-FINE-S", + "license": "Apache-2.0", + "releaseDate": "2024-10-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5339360463706191, + "mAP_50": 0.6979289246485497, + "mAP_small": 0.37902381194039897, + "mAP_medium": 0.5785906686133171, + "mAP_large": 0.6701666032622975, + "fps_mean": 34.17, + "fps_p50": 34.82, + "ms_per_image": 29.267, + "inference_ms": 22.32, + "preprocess_ms": 5.837, + "postprocess_ms": 1.111, + "p95_ms": 34.014, + "peak_vram_mb": 141.6, + "peak_ram_mb": 0.7, + "params_millions": 10.32, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:45:41.588316Z" + } + }, + "b": { + "id": "yolov9s", + "displayName": "YOLOv9-S", + "license": "MIT", + "releaseDate": "2024-02-21", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5044737676946623, + "mAP_50": 0.6719744329452191, + "mAP_small": 0.307408881744591, + "mAP_medium": 0.5643791782271478, + "mAP_large": 0.653006070420912, + "fps_mean": 30.76, + "fps_p50": 31.79, + "ms_per_image": 32.507, + "inference_ms": 26.448, + "preprocess_ms": 3.813, + "postprocess_ms": 2.246, + "p95_ms": 39.868, + "peak_vram_mb": 91.5, + "peak_ram_mb": 20.4, + "params_millions": 7.23, + "gflops": 13.52, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:18:36.206342Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 5.84, + "fps": 11.09, + "params": 42.74, + "flops": 84.91, + "peak_vram": 54.75 + }, + "map_pts": { + "a": 53.4, + "b": 50.4, + "delta": 3.0 + }, + "pair": { + "params_ratio": 1.43, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.02136, + "b_mAP_per_GFLOP": 0.03731 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 53.4, + "fps": 70.9 + }, + "b": { + "mAP": 50.4, + "fps": 54.5 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 53.4, + "fps": 34.2 + }, + "b": { + "mAP": 50.4, + "fps": 30.8 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 53.2, + "fps": 86.5 + }, + "b": { + "mAP": 50.5, + "fps": 63.8 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 53.4, + "fps": 98.7 + }, + "b": { + "mAP": 50.4, + "fps": 75.1 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 53.4, + "fps": 10.3 + }, + "b": { + "mAP": 50.4, + "fps": 9.9 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 53.4, + "fps": 34.2 + }, + "b": { + "mAP": 50.4, + "fps": 30.8 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 53.4, + "fps": 1.1 + }, + "b": { + "mAP": 49.4, + "fps": 1.4 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "D-FINE-S measures 3.0 mAP points higher than YOLOv9-S on NVIDIA RTX 5070 Ti (PyTorch FP32): 53.4 vs 50.4 mAP@50-95.", + "values": { + "delta_pts": 3.0, + "a_map": 53.4, + "b_map": 50.4 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "D-FINE-S is 11.09% faster: 34.2 vs 30.8 FPS (29.27 vs 32.51 ms/image).", + "values": { + "delta_pct": 11.09, + "a_fps": 34.2, + "b_fps": 30.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "YOLOv9-S extracts 42.75% more mAP per GFLOP (0.02136 vs 0.03731).", + "values": { + "a": 0.02136, + "b": 0.03731, + "delta_pct": 42.75 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "YOLOv9-S uses less peak VRAM: 142 vs 92 MB.", + "values": { + "a_vram": 142, + "b_vram": 92 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "D-FINE-S leads on small objects: 37.9 vs 30.7 mAP_small (23.3% relative gap).", + "values": { + "a": 37.9, + "b": 30.7, + "delta_pct": 23.3 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + 0.0, + 0.001, + 0.0214, + 0.0373, + 0.1, + 0.3074, + 0.379, + 0.5045, + 0.5339, + 0.5644, + 0.5786, + 0.6, + 0.653, + 0.6702, + 0.672, + 0.6979, + 0.7, + 1.0, + 1.1, + 1.111, + 1.4, + 1.43, + 2, + 2.136, + 2.246, + 3.0, + 3.731, + 3.813, + 4, + 5, + 5.837, + 5.84, + 6, + 7, + 7.23, + 8, + 9, + 9.9, + 10, + 10.3, + 10.32, + 11, + 11.09, + 12, + 13.52, + 20.4, + 22.32, + 23.3, + 26.448, + 29.267, + 30.7, + 30.7409, + 30.76, + 30.8, + 31.79, + 32.507, + 34.014, + 34.17, + 34.2, + 34.82, + 37.9, + 37.9024, + 39.868, + 42.74, + 42.75, + 49.4, + 50, + 50.4, + 50.4474, + 50.5, + 53.2, + 53.3936, + 53.4, + 54.5, + 54.75, + 56.4379, + 57.8591, + 60.0, + 63.8, + 65.3006, + 67.0167, + 67.1974, + 69.7929, + 70.0, + 70.9, + 75, + 75.1, + 84.91, + 86.5, + 91.5, + 92.0, + 95, + 98.7, + 100.0, + 141.6, + 142.0, + 300.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-vs-rfdetr.claims.json b/article-pipeline/v2/output/dfine-vs-rfdetr.claims.json new file mode 100644 index 0000000..510dc2a --- /dev/null +++ b/article-pipeline/v2/output/dfine-vs-rfdetr.claims.json @@ -0,0 +1,345 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:18.973810+00:00", + "slug": "dfine-vs-rfdetr", + "meta": { + "type": "family-vs", + "a": "dfine", + "b": "rfdetr", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "dfine", + "b": "rfdetr", + "a_display": "D-FINE", + "b_display": "RF-DETR" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + { + "id": "dfine-s", + "displayName": "D-FINE-S", + "params": 10.32, + "mAP": 53.4, + "fps": 34.2, + "license": "Apache-2.0" + }, + { + "id": "dfine-m", + "displayName": "D-FINE-M", + "params": 19.59, + "mAP": 57.8, + "fps": 28.8, + "license": "Apache-2.0" + }, + { + "id": "dfine-l", + "displayName": "D-FINE-L", + "params": 31.24, + "mAP": 60.0, + "fps": 21.9, + "license": "Apache-2.0" + }, + { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + { + "id": "rfdetr-s", + "displayName": "RF-DETR-S", + "params": 32.11, + "mAP": 55.1, + "fps": 35.8, + "license": "Apache-2.0" + }, + { + "id": "rfdetr-m", + "displayName": "RF-DETR-M", + "params": 33.69, + "mAP": 57.3, + "fps": 30.3, + "license": "Apache-2.0" + }, + { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + } + ], + "matched_pairs": [ + { + "a": { + "id": "dfine-m", + "displayName": "D-FINE-M", + "params": 19.59, + "mAP": 57.8, + "fps": 28.8, + "license": "Apache-2.0" + }, + "b": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 6.4, + "fps_delta_pct": -27.09 + }, + { + "a": { + "id": "dfine-l", + "displayName": "D-FINE-L", + "params": 31.24, + "mAP": 60.0, + "fps": 21.9, + "license": "Apache-2.0" + }, + "b": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 8.6, + "fps_delta_pct": -44.56 + } + ], + "aggregate": { + "n_pairs": 2, + "mean_mAP_delta_pts": 7.5, + "a_wins": 2, + "b_wins": 0, + "mean_fps_delta_pct": -35.8 + }, + "flagship": { + "a": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + }, + "b": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + } + }, + "smallest": { + "a": { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + "b": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + } + }, + "crossover": { + "low_param_leader": "dfine", + "high_param_leader": "dfine" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "dfine-n,dfine-s,dfine-m,dfine-l,dfine-x,rfdetr-n,rfdetr-s,rfdetr-m,rfdetr-l" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, D-FINE averages 7.5 mAP points higher across 2 paired variants on NVIDIA RTX 5070 Ti. D-FINE wins 2, RF-DETR wins 0.", + "values": { + "mean_delta_pts": 7.5, + "a_wins": 2, + "b_wins": 0 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, RF-DETR is 35.8% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 35.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, D-FINE's largest measured variant (D-FINE-X) reaches 61.4 mAP; RF-DETR's (RF-DETR-L) reaches 58.6 mAP.", + "values": { + "a": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + }, + "b": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, D-FINE's D-FINE-N (3.78M, 45.8 mAP, 32.5 FPS) faces RF-DETR's RF-DETR-N (30.47M, 51.4 mAP, 39.5 FPS).", + "values": { + "a": { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + "b": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "D-FINE fields 5 measured variants, RF-DETR fields 4, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 5, + "b_n": 4 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -44.56, + -35.8, + -27.09, + 0.0, + 0.001, + 0.6, + 1, + 2.0, + 3, + 3.78, + 4.0, + 5.0, + 6, + 6.4, + 7, + 7.5, + 8, + 8.6, + 9, + 10, + 10.32, + 11, + 12, + 18.1, + 19.59, + 21.9, + 25.1, + 28.8, + 30.3, + 30.47, + 31.24, + 32.11, + 32.5, + 33.69, + 33.93, + 34.2, + 35.8, + 39.5, + 45.8, + 50, + 51.4, + 53.4, + 55.1, + 57.3, + 57.8, + 58.6, + 60.0, + 61.4, + 62.62, + 75, + 95, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-vs-rtdetr.claims.json b/article-pipeline/v2/output/dfine-vs-rtdetr.claims.json new file mode 100644 index 0000000..c316f58 --- /dev/null +++ b/article-pipeline/v2/output/dfine-vs-rtdetr.claims.json @@ -0,0 +1,400 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:18.353372+00:00", + "slug": "dfine-vs-rtdetr", + "meta": { + "type": "family-vs", + "a": "dfine", + "b": "rtdetr", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "dfine", + "b": "rtdetr", + "a_display": "D-FINE", + "b_display": "RT-DETR" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + { + "id": "dfine-s", + "displayName": "D-FINE-S", + "params": 10.32, + "mAP": 53.4, + "fps": 34.2, + "license": "Apache-2.0" + }, + { + "id": "dfine-m", + "displayName": "D-FINE-M", + "params": 19.59, + "mAP": 57.8, + "fps": 28.8, + "license": "Apache-2.0" + }, + { + "id": "dfine-l", + "displayName": "D-FINE-L", + "params": 31.24, + "mAP": 60.0, + "fps": 21.9, + "license": "Apache-2.0" + }, + { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-l", + "displayName": "RT-DETR-L", + "params": 32.93, + "mAP": 55.8, + "fps": 27.7, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "params": 36.59, + "mAP": 53.8, + "fps": 35.3, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "params": 42.89, + "mAP": 55.9, + "fps": 29.8, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-x", + "displayName": "RT-DETR-X", + "params": 67.37, + "mAP": 57.9, + "fps": 21.8, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + } + ], + "matched_pairs": [ + { + "a": { + "id": "dfine-m", + "displayName": "D-FINE-M", + "params": 19.59, + "mAP": 57.8, + "fps": 28.8, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 8.0, + "fps_delta_pct": -38.98 + }, + { + "a": { + "id": "dfine-l", + "displayName": "D-FINE-L", + "params": 31.24, + "mAP": 60.0, + "fps": 21.9, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 7.8, + "fps_delta_pct": -45.52 + }, + { + "a": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-x", + "displayName": "RT-DETR-X", + "params": 67.37, + "mAP": 57.9, + "fps": 21.8, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 3.5, + "fps_delta_pct": -16.97 + } + ], + "aggregate": { + "n_pairs": 3, + "mean_mAP_delta_pts": 6.4, + "a_wins": 3, + "b_wins": 0, + "mean_fps_delta_pct": -33.8 + }, + "flagship": { + "a": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + } + }, + "smallest": { + "a": { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + } + }, + "crossover": { + "low_param_leader": "dfine", + "high_param_leader": "dfine" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "dfine-n,dfine-s,dfine-m,dfine-l,dfine-x,rtdetr-r18,rtdetr-r34,rtdetr-l,rtdetr-r50m,rtdetr-r50,rtdetr-x,rtdetr-r101" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, D-FINE averages 6.4 mAP points higher across 3 paired variants on NVIDIA RTX 5070 Ti. D-FINE wins 3, RT-DETR wins 0.", + "values": { + "mean_delta_pts": 6.4, + "a_wins": 3, + "b_wins": 0 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, RT-DETR is 33.8% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 33.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, D-FINE's largest measured variant (D-FINE-X) reaches 61.4 mAP; RT-DETR's (RT-DETR-R101) reaches 56.8 mAP.", + "values": { + "a": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, D-FINE's D-FINE-N (3.78M, 45.8 mAP, 32.5 FPS) faces RT-DETR's RT-DETR-R18 (20.18M, 49.8 mAP, 47.2 FPS).", + "values": { + "a": { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "D-FINE fields 5 measured variants, RT-DETR fields 7, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 5, + "b_n": 7 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -45.52, + -38.98, + -33.8, + -16.97, + 0.0, + 0.001, + 0.6, + 1, + 2, + 3.0, + 3.5, + 3.78, + 4, + 5.0, + 6, + 6.4, + 7.0, + 7.8, + 8.0, + 9, + 10, + 10.32, + 11, + 12, + 18.1, + 19.59, + 20.18, + 21.8, + 21.9, + 23.6, + 27.7, + 28.8, + 29.8, + 31.24, + 31.44, + 32.5, + 32.93, + 33.8, + 34.2, + 35.3, + 36.59, + 40.2, + 42.89, + 45.8, + 47.2, + 49.8, + 50, + 52.2, + 53.4, + 53.8, + 55.8, + 55.9, + 56.8, + 57.8, + 57.9, + 60.0, + 61.4, + 62.62, + 67.37, + 75, + 76.56, + 95, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-vs-rtdetrv2.claims.json b/article-pipeline/v2/output/dfine-vs-rtdetrv2.claims.json new file mode 100644 index 0000000..059028e --- /dev/null +++ b/article-pipeline/v2/output/dfine-vs-rtdetrv2.claims.json @@ -0,0 +1,379 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:19.981437+00:00", + "slug": "dfine-vs-rtdetrv2", + "meta": { + "type": "family-vs", + "a": "dfine", + "b": "rtdetrv2", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "dfine", + "b": "rtdetrv2", + "a_display": "D-FINE", + "b_display": "RT-DETRv2" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + { + "id": "dfine-s", + "displayName": "D-FINE-S", + "params": 10.32, + "mAP": 53.4, + "fps": 34.2, + "license": "Apache-2.0" + }, + { + "id": "dfine-m", + "displayName": "D-FINE-M", + "params": 19.59, + "mAP": 57.8, + "fps": 28.8, + "license": "Apache-2.0" + }, + { + "id": "dfine-l", + "displayName": "D-FINE-L", + "params": 31.24, + "mAP": 60.0, + "fps": 21.9, + "license": "Apache-2.0" + }, + { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "params": 20.18, + "mAP": 50.7, + "fps": 51.1, + "license": "Apache-2.0" + }, + { + "id": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "params": 31.44, + "mAP": 53.2, + "fps": 40.4, + "license": "Apache-2.0" + }, + { + "id": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "params": 36.59, + "mAP": 54.8, + "fps": 39.9, + "license": "Apache-2.0" + }, + { + "id": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "params": 42.89, + "mAP": 55.7, + "fps": 32.1, + "license": "Apache-2.0" + }, + { + "id": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 25.7, + "license": "Apache-2.0" + } + ], + "matched_pairs": [ + { + "a": { + "id": "dfine-m", + "displayName": "D-FINE-M", + "params": 19.59, + "mAP": 57.8, + "fps": 28.8, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "params": 20.18, + "mAP": 50.7, + "fps": 51.1, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 7.1, + "fps_delta_pct": -43.64 + }, + { + "a": { + "id": "dfine-l", + "displayName": "D-FINE-L", + "params": 31.24, + "mAP": 60.0, + "fps": 21.9, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "params": 31.44, + "mAP": 53.2, + "fps": 40.4, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 6.8, + "fps_delta_pct": -45.79 + }, + { + "a": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 25.7, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 4.6, + "fps_delta_pct": -29.57 + } + ], + "aggregate": { + "n_pairs": 3, + "mean_mAP_delta_pts": 6.2, + "a_wins": 3, + "b_wins": 0, + "mean_fps_delta_pct": -39.7 + }, + "flagship": { + "a": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 25.7, + "license": "Apache-2.0" + } + }, + "smallest": { + "a": { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "params": 20.18, + "mAP": 50.7, + "fps": 51.1, + "license": "Apache-2.0" + } + }, + "crossover": { + "low_param_leader": "dfine", + "high_param_leader": "dfine" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "dfine-n,dfine-s,dfine-m,dfine-l,dfine-x,rtdetrv2-r18,rtdetrv2-r34,rtdetrv2-r50m,rtdetrv2-r50,rtdetrv2-r101" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, D-FINE averages 6.2 mAP points higher across 3 paired variants on NVIDIA RTX 5070 Ti. D-FINE wins 3, RT-DETRv2 wins 0.", + "values": { + "mean_delta_pts": 6.2, + "a_wins": 3, + "b_wins": 0 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, RT-DETRv2 is 39.7% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 39.7 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, D-FINE's largest measured variant (D-FINE-X) reaches 61.4 mAP; RT-DETRv2's (RT-DETRv2-R101) reaches 56.8 mAP.", + "values": { + "a": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 25.7, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, D-FINE's D-FINE-N (3.78M, 45.8 mAP, 32.5 FPS) faces RT-DETRv2's RT-DETRv2-R18 (20.18M, 50.7 mAP, 51.1 FPS).", + "values": { + "a": { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "params": 20.18, + "mAP": 50.7, + "fps": 51.1, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "D-FINE fields 5 measured variants, RT-DETRv2 fields 5, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 5, + "b_n": 5 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -45.79, + -43.64, + -39.7, + -29.57, + 0.0, + 0.001, + 0.6, + 1, + 2, + 3.0, + 3.78, + 4, + 4.6, + 5.0, + 6, + 6.2, + 6.8, + 7, + 7.1, + 8, + 9, + 10, + 10.32, + 11, + 12, + 18.1, + 19.59, + 20.18, + 21.9, + 25.7, + 28.8, + 31.24, + 31.44, + 32.1, + 32.5, + 34.2, + 36.59, + 39.7, + 39.9, + 40.4, + 42.89, + 45.8, + 50, + 50.7, + 51.1, + 53.2, + 53.4, + 54.8, + 55.7, + 56.8, + 57.8, + 60.0, + 61.4, + 62.62, + 75, + 76.56, + 95, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/dfine-vs-yolov9.claims.json b/article-pipeline/v2/output/dfine-vs-yolov9.claims.json new file mode 100644 index 0000000..e8af030 --- /dev/null +++ b/article-pipeline/v2/output/dfine-vs-yolov9.claims.json @@ -0,0 +1,367 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:19.158659+00:00", + "slug": "dfine-vs-yolov9", + "meta": { + "type": "family-vs", + "a": "dfine", + "b": "yolov9", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "dfine", + "b": "yolov9", + "a_display": "D-FINE", + "b_display": "YOLOv9" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + { + "id": "dfine-s", + "displayName": "D-FINE-S", + "params": 10.32, + "mAP": 53.4, + "fps": 34.2, + "license": "Apache-2.0" + }, + { + "id": "dfine-m", + "displayName": "D-FINE-M", + "params": 19.59, + "mAP": 57.8, + "fps": 28.8, + "license": "Apache-2.0" + }, + { + "id": "dfine-l", + "displayName": "D-FINE-L", + "params": 31.24, + "mAP": 60.0, + "fps": 21.9, + "license": "Apache-2.0" + }, + { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "params": 2.04, + "mAP": 41.8, + "fps": 31.9, + "license": "MIT" + }, + { + "id": "yolov9s", + "displayName": "YOLOv9-S", + "params": 7.23, + "mAP": 50.4, + "fps": 30.8, + "license": "MIT" + }, + { + "id": "yolov9m", + "displayName": "YOLOv9-M", + "params": 20.12, + "mAP": 56.1, + "fps": 37.1, + "license": "MIT" + }, + { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + } + ], + "matched_pairs": [ + { + "a": { + "id": "dfine-s", + "displayName": "D-FINE-S", + "params": 10.32, + "mAP": 53.4, + "fps": 34.2, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9s", + "displayName": "YOLOv9-S", + "params": 7.23, + "mAP": 50.4, + "fps": 30.8, + "license": "MIT" + }, + "mAP_delta_pts": 3.0, + "fps_delta_pct": 11.04 + }, + { + "a": { + "id": "dfine-m", + "displayName": "D-FINE-M", + "params": 19.59, + "mAP": 57.8, + "fps": 28.8, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9m", + "displayName": "YOLOv9-M", + "params": 20.12, + "mAP": 56.1, + "fps": 37.1, + "license": "MIT" + }, + "mAP_delta_pts": 1.7, + "fps_delta_pct": -22.37 + }, + { + "a": { + "id": "dfine-l", + "displayName": "D-FINE-L", + "params": 31.24, + "mAP": 60.0, + "fps": 21.9, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + }, + "mAP_delta_pts": 2.9, + "fps_delta_pct": -48.23 + } + ], + "aggregate": { + "n_pairs": 3, + "mean_mAP_delta_pts": 2.5, + "a_wins": 3, + "b_wins": 0, + "mean_fps_delta_pct": -19.9 + }, + "flagship": { + "a": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + } + }, + "smallest": { + "a": { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "params": 2.04, + "mAP": 41.8, + "fps": 31.9, + "license": "MIT" + } + }, + "crossover": { + "low_param_leader": "dfine", + "high_param_leader": "dfine" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "dfine-n,dfine-s,dfine-m,dfine-l,dfine-x,yolov9t,yolov9s,yolov9m,yolov9c" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, D-FINE averages 2.5 mAP points higher across 3 paired variants on NVIDIA RTX 5070 Ti. D-FINE wins 3, YOLOv9 wins 0.", + "values": { + "mean_delta_pts": 2.5, + "a_wins": 3, + "b_wins": 0 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, YOLOv9 is 19.9% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 19.9 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, D-FINE's largest measured variant (D-FINE-X) reaches 61.4 mAP; YOLOv9's (YOLOv9-C) reaches 57.1 mAP.", + "values": { + "a": { + "id": "dfine-x", + "displayName": "D-FINE-X", + "params": 62.62, + "mAP": 61.4, + "fps": 18.1, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, D-FINE's D-FINE-N (3.78M, 45.8 mAP, 32.5 FPS) faces YOLOv9's YOLOv9-T (2.04M, 41.8 mAP, 31.9 FPS).", + "values": { + "a": { + "id": "dfine-n", + "displayName": "D-FINE-N", + "params": 3.78, + "mAP": 45.8, + "fps": 32.5, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "params": 2.04, + "mAP": 41.8, + "fps": 31.9, + "license": "MIT" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "D-FINE fields 5 measured variants, YOLOv9 fields 4, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 5, + "b_n": 4 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -48.23, + -22.37, + -19.9, + 0.0, + 0.001, + 0.6, + 1, + 1.7, + 2, + 2.04, + 2.5, + 2.9, + 3.0, + 3.78, + 4.0, + 5.0, + 6, + 7, + 7.23, + 8, + 9, + 10, + 10.32, + 11, + 11.04, + 12, + 18.1, + 19.59, + 19.9, + 20.12, + 21.9, + 25.5, + 28.8, + 30.8, + 31.24, + 31.9, + 32.5, + 34.2, + 37.1, + 41.8, + 42.3, + 45.8, + 50, + 50.4, + 53.4, + 56.1, + 57.1, + 57.8, + 60.0, + 61.4, + 62.62, + 75, + 95, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/finding-scan.claims.json b/article-pipeline/v2/output/finding-scan.claims.json new file mode 100644 index 0000000..25b4bd6 --- /dev/null +++ b/article-pipeline/v2/output/finding-scan.claims.json @@ -0,0 +1,6717 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T21:56:13.177635+00:00", + "slug": "finding-scan", + "meta": { + "type": "finding", + "hardware": null + }, + "facts": { + "n_findings": 202, + "findings": [ + { + "kind": "jitter", + "model": "rfdetr-n", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "onnx_fp32", + "fps_mean": 65.7, + "fps_p50": 94.0, + "gap_pct": -30.08, + "note": "RF-DETR-N on NVIDIA RTX 5070 Ti/onnx_fp32: fps_mean 65.7 vs fps_p50 94.0 (-30.08% gap)." + }, + { + "kind": "jitter", + "model": "rfdetr-n", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "tensorrt_fp16", + "fps_mean": 58.3, + "fps_p50": 99.3, + "gap_pct": -41.23, + "note": "RF-DETR-N on NVIDIA RTX 5070 Ti/tensorrt_fp16: fps_mean 58.3 vs fps_p50 99.3 (-41.23% gap)." + }, + { + "kind": "jitter", + "model": "rfdetr-n", + "hardware": "jetson_orin", + "runtime": "tensorrt_fp32", + "fps_mean": 23.7, + "fps_p50": 27.2, + "gap_pct": -12.76, + "note": "RF-DETR-N on NVIDIA Jetson Orin Nano Super 8GB/tensorrt_fp32: fps_mean 23.7 vs fps_p50 27.2 (-12.76% gap)." + }, + { + "kind": "jitter", + "model": "rfdetr-n", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "tensorrt_fp32", + "fps_mean": 67.6, + "fps_p50": 108.1, + "gap_pct": -37.46, + "note": "RF-DETR-N on NVIDIA RTX 5070 Ti/tensorrt_fp32: fps_mean 67.6 vs fps_p50 108.1 (-37.46% gap)." + }, + { + "kind": "jitter", + "model": "rfdetr-n", + "hardware": "jetson_orin", + "runtime": "tensorrt_fp16", + "fps_mean": 25.4, + "fps_p50": 29.5, + "gap_pct": -13.81, + "note": "RF-DETR-N on NVIDIA Jetson Orin Nano Super 8GB/tensorrt_fp16: fps_mean 25.4 vs fps_p50 29.5 (-13.81% gap)." + }, + { + "kind": "trt_fp16_slower", + "model": "yolov9t", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 59.0, + "fp32_fps": 72.0, + "note": "YOLOv9-T on NVIDIA RTX 5070 Ti: TensorRT FP16 (59.0 FPS) is slower than FP32 (72.0 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "yolov9s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 63.8, + "fp32_fps": 75.1, + "note": "YOLOv9-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (63.8 FPS) is slower than FP32 (75.1 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "yolov9m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 72.3, + "fp32_fps": 77.5, + "note": "YOLOv9-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (72.3 FPS) is slower than FP32 (77.5 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "yolov9c", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 74.7, + "fp32_fps": 76.7, + "note": "YOLOv9-C on NVIDIA RTX 5070 Ti: TensorRT FP16 (74.7 FPS) is slower than FP32 (76.7 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "rfdetr-n", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 58.3, + "fp32_fps": 67.6, + "note": "RF-DETR-N on NVIDIA RTX 5070 Ti: TensorRT FP16 (58.3 FPS) is slower than FP32 (67.6 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "rfdetr-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 71.6, + "fp32_fps": 77.5, + "note": "RF-DETR-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (71.6 FPS) is slower than FP32 (77.5 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "rfdetr-m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 62.9, + "fp32_fps": 63.7, + "note": "RF-DETR-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (62.9 FPS) is slower than FP32 (63.7 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "rtdetr-r18", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 113.4, + "fp32_fps": 120.6, + "note": "RT-DETR-R18 on NVIDIA RTX 5070 Ti: TensorRT FP16 (113.4 FPS) is slower than FP32 (120.6 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "rtdetrv2-r18", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 117.8, + "fp32_fps": 125.5, + "note": "RT-DETRv2-R18 on NVIDIA RTX 5070 Ti: TensorRT FP16 (117.8 FPS) is slower than FP32 (125.5 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "dfine-n", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 79.7, + "fp32_fps": 98.7, + "note": "D-FINE-N on NVIDIA RTX 5070 Ti: TensorRT FP16 (79.7 FPS) is slower than FP32 (98.7 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "yolox-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 70.9, + "fp32_fps": 74.5, + "note": "YOLOX-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (70.9 FPS) is slower than FP32 (74.5 FPS)." + }, + { + "kind": "fp16_accuracy_cliff", + "model": "yolox-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "drop_pts": 0.9, + "fp32_mAP": 44.3, + "fp16_mAP": 43.4, + "note": "YOLOX-S loses 0.9 mAP points going FP32 to FP16 on NVIDIA RTX 5070 Ti." + }, + { + "kind": "fp16_accuracy_cliff", + "model": "deimv2-x", + "hardware": "jetson_orin", + "drop_pts": 0.6, + "fp32_mAP": 61.3, + "fp16_mAP": 60.7, + "note": "DEIMv2-X loses 0.6 mAP points going FP32 to FP16 on NVIDIA Jetson Orin Nano Super 8GB." + }, + { + "kind": "trt_fp16_slower", + "model": "rtdetrv4-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 92.3, + "fp32_fps": 97.1, + "note": "RT-DETRv4-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (92.3 FPS) is slower than FP32 (97.1 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "deim-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 82.4, + "fp32_fps": 97.6, + "note": "DEIM-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (82.4 FPS) is slower than FP32 (97.6 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "deim-m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 83.3, + "fp32_fps": 85.4, + "note": "DEIM-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (83.3 FPS) is slower than FP32 (85.4 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "deimv2-atto", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 132.1, + "fp32_fps": 137.9, + "note": "DEIMv2-Atto on NVIDIA RTX 5070 Ti: TensorRT FP16 (132.1 FPS) is slower than FP32 (137.9 FPS)." + }, + { + "kind": "fp16_accuracy_cliff", + "model": "deimv2-atto", + "hardware": "nvidia_geforce_rtx_5070_ti", + "drop_pts": 1.7, + "fp32_mAP": 27.5, + "fp16_mAP": 25.8, + "note": "DEIMv2-Atto loses 1.7 mAP points going FP32 to FP16 on NVIDIA RTX 5070 Ti." + }, + { + "kind": "trt_fp16_slower", + "model": "deimv2-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 50.4, + "fp32_fps": 52.5, + "note": "DEIMv2-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (50.4 FPS) is slower than FP32 (52.5 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "deimv2-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 38.7, + "fp32_fps": 43.2, + "note": "DEIMv2-L on NVIDIA RTX 5070 Ti: TensorRT FP16 (38.7 FPS) is slower than FP32 (43.2 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "deimv2-x", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 37.9, + "fp32_fps": 38.1, + "note": "DEIMv2-X on NVIDIA RTX 5070 Ti: TensorRT FP16 (37.9 FPS) is slower than FP32 (38.1 FPS)." + }, + { + "kind": "fp16_accuracy_cliff", + "model": "deimv2-x", + "hardware": "nvidia_geforce_rtx_5070_ti", + "drop_pts": 0.5, + "fp32_mAP": 61.3, + "fp16_mAP": 60.8, + "note": "DEIMv2-X loses 0.5 mAP points going FP32 to FP16 on NVIDIA RTX 5070 Ti." + }, + { + "kind": "trt_fp16_slower", + "model": "dfine-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 86.5, + "fp32_fps": 98.7, + "note": "D-FINE-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (86.5 FPS) is slower than FP32 (98.7 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "dfine-m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 74.5, + "fp32_fps": 85.3, + "note": "D-FINE-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (74.5 FPS) is slower than FP32 (85.3 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "dfine-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 69.3, + "fp32_fps": 72.3, + "note": "D-FINE-L on NVIDIA RTX 5070 Ti: TensorRT FP16 (69.3 FPS) is slower than FP32 (72.3 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "ec-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 43.5, + "fp32_fps": 63.4, + "note": "ECDet-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (43.5 FPS) is slower than FP32 (63.4 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "ec-m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 48.9, + "fp32_fps": 59.2, + "note": "ECDet-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (48.9 FPS) is slower than FP32 (59.2 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "ec-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 41.7, + "fp32_fps": 49.0, + "note": "ECDet-L on NVIDIA RTX 5070 Ti: TensorRT FP16 (41.7 FPS) is slower than FP32 (49.0 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "ec-x", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 41.3, + "fp32_fps": 45.6, + "note": "ECDet-X on NVIDIA RTX 5070 Ti: TensorRT FP16 (41.3 FPS) is slower than FP32 (45.6 FPS)." + }, + { + "kind": "trt_fp16_slower", + "model": "deimv2-m", + "hardware": "jetson_orin", + "fp16_fps": 7.9, + "fp32_fps": 8.0, + "note": "DEIMv2-M on NVIDIA Jetson Orin Nano Super 8GB: TensorRT FP16 (7.9 FPS) is slower than FP32 (8.0 FPS)." + }, + { + "kind": "ranking_flip", + "models": [ + "deim-l", + "deimv2-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "deimv2-l", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and DEIMv2-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: DEIMv2-L, Raspberry Pi 5: DEIM-L" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-l", + "ec-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "ec-l", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and ECDet-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: ECDet-L, Raspberry Pi 5: DEIM-L" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-l", + "ec-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "ec-x", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and ECDet-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: ECDet-X, Raspberry Pi 5: DEIM-L" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rfdetr-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "rfdetr-l", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between DEIM-L and RF-DETR-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RF-DETR-L, Raspberry Pi 5: RF-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetr-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r101" + }, + "note": "Speed winner between DEIM-L and RT-DETR-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETR-R101" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: DEIM-L" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: DEIM-L" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetr-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-x" + }, + "note": "Speed winner between DEIM-L and RT-DETR-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETR-X" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetrv2-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r101" + }, + "note": "Speed winner between DEIM-L and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R101" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: DEIM-L" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: DEIM-L" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-l", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-l", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: DEIM-L" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-m", + "deimv2-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "nvidia_geforce_rtx_5070_ti": "deim-m", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIM-M and DEIMv2-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: DEIM-M, Raspberry Pi 5: DEIMv2-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-m", + "ec-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "rpi5": "ec-s" + }, + "note": "Speed winner between DEIM-M and ECDet-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, Raspberry Pi 5: ECDet-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-m", + "rfdetr-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "nvidia_geforce_rtx_5070_ti": "rfdetr-m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between DEIM-M and RF-DETR-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: RF-DETR-M, Raspberry Pi 5: RF-DETR-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-m", + "rtdetr-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r34", + "rpi5": "deim-m" + }, + "note": "Speed winner between DEIM-M and RT-DETR-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: RT-DETR-R34, Raspberry Pi 5: DEIM-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-m", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "deim-m" + }, + "note": "Speed winner between DEIM-M and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: DEIM-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-m", + "rtdetrv2-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r34", + "rpi5": "deim-m" + }, + "note": "Speed winner between DEIM-M and RT-DETRv2-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: RT-DETRv2-R34, Raspberry Pi 5: DEIM-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-m", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "deim-m" + }, + "note": "Speed winner between DEIM-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: DEIM-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-m", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "deim-m" + }, + "note": "Speed winner between DEIM-M and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: DEIM-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-m", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "deim-m" + }, + "note": "Speed winner between DEIM-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: DEIM-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-n", + "picodet-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "picodet-l", + "rpi5": "deim-n" + }, + "note": "Speed winner between DEIM-N and PicoDet-L flips across hardware: NVIDIA RTX 5070 Ti: PicoDet-L, Raspberry Pi 5: DEIM-N" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-n", + "yolov9t" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-n", + "nvidia_geforce_rtx_5070_ti": "deim-n", + "rpi5": "yolov9t" + }, + "note": "Speed winner between DEIM-N and YOLOv9-T flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-N, NVIDIA RTX 5070 Ti: DEIM-N, Raspberry Pi 5: YOLOv9-T" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-n", + "yolox-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-s", + "nvidia_geforce_rtx_5070_ti": "yolox-s", + "rpi5": "deim-n" + }, + "note": "Speed winner between DEIM-N and YOLOX-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-S, NVIDIA RTX 5070 Ti: YOLOX-S, Raspberry Pi 5: DEIM-N" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-s", + "rtdetr-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetr-r18", + "rpi5": "deim-s" + }, + "note": "Speed winner between DEIM-S and RT-DETR-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: DEIM-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-s", + "rtdetrv2-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r18", + "rpi5": "deim-s" + }, + "note": "Speed winner between DEIM-S and RT-DETRv2-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: DEIM-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-s", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-s", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "deim-s" + }, + "note": "Speed winner between DEIM-S and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-S, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: DEIM-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-s", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-s", + "nvidia_geforce_rtx_5070_ti": "yolov9m", + "rpi5": "deim-s" + }, + "note": "Speed winner between DEIM-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-S, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: DEIM-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-s", + "yolov9s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "deim-s", + "rpi5": "yolov9s" + }, + "note": "Speed winner between DEIM-S and YOLOv9-S flips across hardware: NVIDIA RTX 5070 Ti: DEIM-S, Raspberry Pi 5: YOLOv9-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-s", + "yolox-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-s", + "nvidia_geforce_rtx_5070_ti": "yolox-m", + "rpi5": "deim-s" + }, + "note": "Speed winner between DEIM-S and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-S, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: DEIM-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deim-x", + "deimv2-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-x", + "nvidia_geforce_rtx_5070_ti": "deimv2-x" + }, + "note": "Speed winner between DEIM-X and DEIMv2-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-X, NVIDIA RTX 5070 Ti: DEIMv2-X" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-atto", + "deimv2-femto" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "deimv2-femto", + "rpi5": "deimv2-atto" + }, + "note": "Speed winner between DEIMv2-Atto and DEIMv2-Femto flips across hardware: NVIDIA RTX 5070 Ti: DEIMv2-Femto, Raspberry Pi 5: DEIMv2-Atto" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-atto", + "picodet-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-atto", + "nvidia_geforce_rtx_5070_ti": "picodet-s", + "rpi5": "deimv2-atto" + }, + "note": "Speed winner between DEIMv2-Atto and PicoDet-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-Atto, NVIDIA RTX 5070 Ti: PicoDet-S, Raspberry Pi 5: DEIMv2-Atto" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-atto", + "yolox-nano" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-nano", + "nvidia_geforce_rtx_5070_ti": "yolox-nano", + "rpi5": "deimv2-atto" + }, + "note": "Speed winner between DEIMv2-Atto and YOLOX-Nano flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-Nano, NVIDIA RTX 5070 Ti: YOLOX-Nano, Raspberry Pi 5: DEIMv2-Atto" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-femto", + "picodet-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-femto", + "nvidia_geforce_rtx_5070_ti": "picodet-s", + "rpi5": "picodet-s" + }, + "note": "Speed winner between DEIMv2-Femto and PicoDet-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-Femto, NVIDIA RTX 5070 Ti: PicoDet-S, Raspberry Pi 5: PicoDet-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-l", + "ec-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-l", + "nvidia_geforce_rtx_5070_ti": "ec-x", + "rpi5": "deimv2-l" + }, + "note": "Speed winner between DEIMv2-L and ECDet-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-L, NVIDIA RTX 5070 Ti: ECDet-X, Raspberry Pi 5: DEIMv2-L" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-l", + "rtdetr-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r101" + }, + "note": "Speed winner between DEIMv2-L and RT-DETR-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-L, NVIDIA RTX 5070 Ti: RT-DETR-R101" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-l", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "deimv2-l" + }, + "note": "Speed winner between DEIMv2-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: DEIMv2-L" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-l", + "rtdetr-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-x" + }, + "note": "Speed winner between DEIMv2-L and RT-DETR-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-L, NVIDIA RTX 5070 Ti: RT-DETR-X" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-l", + "rtdetrv2-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r101" + }, + "note": "Speed winner between DEIMv2-L and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R101" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-l", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "deimv2-l" + }, + "note": "Speed winner between DEIMv2-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: DEIMv2-L" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "ec-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-m", + "nvidia_geforce_rtx_5070_ti": "ec-l", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and ECDet-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-M, NVIDIA RTX 5070 Ti: ECDet-L, Raspberry Pi 5: DEIMv2-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rfdetr-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-m", + "nvidia_geforce_rtx_5070_ti": "rfdetr-l", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RF-DETR-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-M, NVIDIA RTX 5070 Ti: RF-DETR-L, Raspberry Pi 5: DEIMv2-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rtdetr-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetr-l", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RT-DETR-L flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-L, Raspberry Pi 5: DEIMv2-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-M, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: DEIMv2-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: DEIMv2-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: DEIMv2-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: DEIMv2-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rtdetrv4-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv4-l", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RT-DETRv4-L flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv4-L, Raspberry Pi 5: DEIMv2-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9c", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-C, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: DEIMv2-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-n", + "picodet-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "picodet-l", + "rpi5": "deimv2-n" + }, + "note": "Speed winner between DEIMv2-N and PicoDet-L flips across hardware: NVIDIA RTX 5070 Ti: PicoDet-L, Raspberry Pi 5: DEIMv2-N" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-n", + "yolov9t" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-n", + "nvidia_geforce_rtx_5070_ti": "deimv2-n", + "rpi5": "yolov9t" + }, + "note": "Speed winner between DEIMv2-N and YOLOv9-T flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-N, NVIDIA RTX 5070 Ti: DEIMv2-N, Raspberry Pi 5: YOLOv9-T" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-n", + "yolox-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-s", + "nvidia_geforce_rtx_5070_ti": "yolox-s", + "rpi5": "deimv2-n" + }, + "note": "Speed winner between DEIMv2-N and YOLOX-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-S, NVIDIA RTX 5070 Ti: YOLOX-S, Raspberry Pi 5: DEIMv2-N" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-pico", + "picodet-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-pico", + "nvidia_geforce_rtx_5070_ti": "picodet-m", + "rpi5": "picodet-m" + }, + "note": "Speed winner between DEIMv2-Pico and PicoDet-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-Pico, NVIDIA RTX 5070 Ti: PicoDet-M, Raspberry Pi 5: PicoDet-M" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-s", + "dfine-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-m", + "nvidia_geforce_rtx_5070_ti": "dfine-m", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIMv2-S and D-FINE-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: D-FINE-M, Raspberry Pi 5: DEIMv2-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-s", + "ec-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-s", + "nvidia_geforce_rtx_5070_ti": "ec-m", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIMv2-S and ECDet-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-S, NVIDIA RTX 5070 Ti: ECDet-M, Raspberry Pi 5: DEIMv2-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-s", + "rtdetr-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r18", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r18", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIMv2-S and RT-DETR-R18 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R18, NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: DEIMv2-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-s", + "rtdetrv2-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r18", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r18", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIMv2-S and RT-DETRv2-R18 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R18, NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: DEIMv2-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-s", + "rtdetrv4-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv4-m", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIMv2-S and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETRv4-M, Raspberry Pi 5: DEIMv2-S" + }, + { + "kind": "ranking_flip", + "models": [ + "deimv2-s", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9m", + "nvidia_geforce_rtx_5070_ti": "yolov9m", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIMv2-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: DEIMv2-S" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "ec-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-l", + "nvidia_geforce_rtx_5070_ti": "ec-l", + "rpi5": "dfine-l" + }, + "note": "Speed winner between D-FINE-L and ECDet-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: ECDet-L, Raspberry Pi 5: D-FINE-L" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rfdetr-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-l", + "nvidia_geforce_rtx_5070_ti": "rfdetr-l", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between D-FINE-L and RF-DETR-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RF-DETR-L, Raspberry Pi 5: RF-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rtdetr-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r101", + "jetson_orin": "dfine-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r101" + }, + "note": "Speed winner between D-FINE-L and RT-DETR-R101 flips across hardware: NVIDIA A100: RT-DETR-R101, NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RT-DETR-R101" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50", + "jetson_orin": "dfine-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "dfine-l" + }, + "note": "Speed winner between D-FINE-L and RT-DETR-R50 flips across hardware: NVIDIA A100: RT-DETR-R50, NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: D-FINE-L" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50m", + "jetson_orin": "rtdetr-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "dfine-l" + }, + "note": "Speed winner between D-FINE-L and RT-DETR-R50m flips across hardware: NVIDIA A100: RT-DETR-R50m, NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: D-FINE-L" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rtdetrv2-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r101" + }, + "note": "Speed winner between D-FINE-L and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R101" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "dfine-l" + }, + "note": "Speed winner between D-FINE-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: D-FINE-L" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "dfine-l" + }, + "note": "Speed winner between D-FINE-L and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: D-FINE-L" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-l", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "dfine-l" + }, + "note": "Speed winner between D-FINE-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: D-FINE-L" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "ec-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-m", + "rpi5": "ec-s" + }, + "note": "Speed winner between D-FINE-M and ECDet-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, Raspberry Pi 5: ECDet-S" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "rfdetr-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-m", + "nvidia_geforce_rtx_5070_ti": "rfdetr-m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between D-FINE-M and RF-DETR-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: RF-DETR-M, Raspberry Pi 5: RF-DETR-M" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "rtdetr-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r34", + "jetson_orin": "rtdetr-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r34", + "rpi5": "dfine-m" + }, + "note": "Speed winner between D-FINE-M and RT-DETR-R34 flips across hardware: NVIDIA A100: RT-DETR-R34, NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: RT-DETR-R34, Raspberry Pi 5: D-FINE-M" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50m", + "jetson_orin": "dfine-m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "dfine-m" + }, + "note": "Speed winner between D-FINE-M and RT-DETR-R50m flips across hardware: NVIDIA A100: RT-DETR-R50m, NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: D-FINE-M" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "rtdetrv2-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r34", + "rpi5": "dfine-m" + }, + "note": "Speed winner between D-FINE-M and RT-DETRv2-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: RT-DETRv2-R34, Raspberry Pi 5: D-FINE-M" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "dfine-m" + }, + "note": "Speed winner between D-FINE-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: D-FINE-M" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "dfine-m" + }, + "note": "Speed winner between D-FINE-M and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: D-FINE-M" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "yolonas-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "dfine-m", + "jetson_orin": "yolonas-s", + "rpi5": "yolonas-s" + }, + "note": "Speed winner between D-FINE-M and YOLO-NAS-S flips across hardware: NVIDIA A100: D-FINE-M, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: YOLO-NAS-S" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-m", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "dfine-m" + }, + "note": "Speed winner between D-FINE-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: D-FINE-M" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-n", + "picodet-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-n", + "nvidia_geforce_rtx_5070_ti": "picodet-l", + "rpi5": "dfine-n" + }, + "note": "Speed winner between D-FINE-N and PicoDet-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-N, NVIDIA RTX 5070 Ti: PicoDet-L, Raspberry Pi 5: D-FINE-N" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-n", + "yolov9t" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-n", + "rpi5": "yolov9t" + }, + "note": "Speed winner between D-FINE-N and YOLOv9-T flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-N, Raspberry Pi 5: YOLOv9-T" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-n", + "yolox-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-s", + "nvidia_geforce_rtx_5070_ti": "yolox-s", + "rpi5": "dfine-n" + }, + "note": "Speed winner between D-FINE-N and YOLOX-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-S, NVIDIA RTX 5070 Ti: YOLOX-S, Raspberry Pi 5: D-FINE-N" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-s", + "rtdetr-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r18", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r18", + "rpi5": "dfine-s" + }, + "note": "Speed winner between D-FINE-S and RT-DETR-R18 flips across hardware: NVIDIA A100: RT-DETR-R18, NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: D-FINE-S" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-s", + "rtdetrv2-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r18", + "rpi5": "dfine-s" + }, + "note": "Speed winner between D-FINE-S and RT-DETRv2-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: D-FINE-S" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-s", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-s", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "dfine-s" + }, + "note": "Speed winner between D-FINE-S and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-S, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: D-FINE-S" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-s", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-s", + "nvidia_geforce_rtx_5070_ti": "yolov9m", + "rpi5": "dfine-s" + }, + "note": "Speed winner between D-FINE-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-S, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: D-FINE-S" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-s", + "yolov9s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "dfine-s", + "rpi5": "yolov9s" + }, + "note": "Speed winner between D-FINE-S and YOLOv9-S flips across hardware: NVIDIA RTX 5070 Ti: D-FINE-S, Raspberry Pi 5: YOLOv9-S" + }, + { + "kind": "ranking_flip", + "models": [ + "dfine-s", + "yolox-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-s", + "nvidia_geforce_rtx_5070_ti": "yolox-m", + "rpi5": "dfine-s" + }, + "note": "Speed winner between D-FINE-S and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-S, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: D-FINE-S" + }, + { + "kind": "ranking_flip", + "models": [ + "ec-l", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "ec-l" + }, + "note": "Speed winner between ECDet-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: ECDet-L" + }, + { + "kind": "ranking_flip", + "models": [ + "ec-l", + "rtdetrv2-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "ec-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r101" + }, + "note": "Speed winner between ECDet-L and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: ECDet-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R101" + }, + { + "kind": "ranking_flip", + "models": [ + "ec-l", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "ec-l" + }, + "note": "Speed winner between ECDet-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: ECDet-L" + }, + { + "kind": "ranking_flip", + "models": [ + "ec-m", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "ec-m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "ec-m" + }, + "note": "Speed winner between ECDet-M and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: ECDet-M, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: ECDet-M" + }, + { + "kind": "ranking_flip", + "models": [ + "ec-m", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "ec-m" + }, + "note": "Speed winner between ECDet-M and RT-DETR-R50m flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: ECDet-M" + }, + { + "kind": "ranking_flip", + "models": [ + "ec-m", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "ec-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "ec-m" + }, + "note": "Speed winner between ECDet-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: ECDet-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: ECDet-M" + }, + { + "kind": "ranking_flip", + "models": [ + "ec-m", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "ec-m" + }, + "note": "Speed winner between ECDet-M and RT-DETRv2-R50m flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: ECDet-M" + }, + { + "kind": "ranking_flip", + "models": [ + "ec-m", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9c", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "ec-m" + }, + "note": "Speed winner between ECDet-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-C, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: ECDet-M" + }, + { + "kind": "ranking_flip", + "models": [ + "ec-s", + "rtdetrv4-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv4-m", + "rpi5": "ec-s" + }, + "note": "Speed winner between ECDet-S and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETRv4-M, Raspberry Pi 5: ECDet-S" + }, + { + "kind": "ranking_flip", + "models": [ + "ec-s", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9m", + "nvidia_geforce_rtx_5070_ti": "yolov9m", + "rpi5": "ec-s" + }, + "note": "Speed winner between ECDet-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: ECDet-S" + }, + { + "kind": "ranking_flip", + "models": [ + "picodet-l", + "yolov9t" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "picodet-l", + "nvidia_geforce_rtx_5070_ti": "picodet-l", + "rpi5": "yolov9t" + }, + "note": "Speed winner between PicoDet-L and YOLOv9-T flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: PicoDet-L, NVIDIA RTX 5070 Ti: PicoDet-L, Raspberry Pi 5: YOLOv9-T" + }, + { + "kind": "ranking_flip", + "models": [ + "picodet-s", + "yolox-nano" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-nano", + "nvidia_geforce_rtx_5070_ti": "yolox-nano", + "rpi5": "picodet-s" + }, + "note": "Speed winner between PicoDet-S and YOLOX-Nano flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-Nano, NVIDIA RTX 5070 Ti: YOLOX-Nano, Raspberry Pi 5: PicoDet-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "rtdetr-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-l", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and RT-DETR-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-L, NVIDIA RTX 5070 Ti: RT-DETR-L, Raspberry Pi 5: RF-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: RF-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RF-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: RF-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RF-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "rtdetrv4-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-l", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-L, Raspberry Pi 5: RF-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-l", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RF-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "rtdetr-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r34", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and RT-DETR-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: RT-DETR-R34, Raspberry Pi 5: RF-DETR-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-M, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RF-DETR-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "rtdetrv2-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r34", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and RT-DETRv2-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: RT-DETRv2-R34, Raspberry Pi 5: RF-DETR-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: RF-DETR-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RF-DETR-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "rtdetrv4-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv4-m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETRv4-M, Raspberry Pi 5: RF-DETR-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "yolonas-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolonas-m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and YOLO-NAS-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-M, Raspberry Pi 5: RF-DETR-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9c", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-C, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RF-DETR-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9m", + "nvidia_geforce_rtx_5070_ti": "yolov9m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: RF-DETR-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and YOLOX-L flips across hardware: NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RF-DETR-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-n", + "rtdetr-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetr-r18", + "rpi5": "rfdetr-n" + }, + "note": "Speed winner between RF-DETR-N and RT-DETR-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: RF-DETR-N" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-n", + "rtdetrv2-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r18", + "rpi5": "rfdetr-n" + }, + "note": "Speed winner between RF-DETR-N and RT-DETRv2-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: RF-DETR-N" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-n", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-n", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "rfdetr-n" + }, + "note": "Speed winner between RF-DETR-N and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-N, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RF-DETR-N" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-n", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-n", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rfdetr-n" + }, + "note": "Speed winner between RF-DETR-N and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-N, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RF-DETR-N" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-n", + "yolox-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-n", + "nvidia_geforce_rtx_5070_ti": "yolox-m", + "rpi5": "rfdetr-n" + }, + "note": "Speed winner between RF-DETR-N and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-N, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: RF-DETR-N" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "rtdetr-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r18", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r18", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and RT-DETR-R18 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R18, NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: RF-DETR-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "rtdetr-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r34", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and RT-DETR-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: RT-DETR-R34, Raspberry Pi 5: RF-DETR-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "rtdetrv2-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r18", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r18", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and RT-DETRv2-R18 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R18, NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: RF-DETR-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "rtdetrv2-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r34", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and RT-DETRv2-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: RT-DETRv2-R34, Raspberry Pi 5: RF-DETR-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-s", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-S, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RF-DETR-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "yolonas-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolonas-s", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and YOLO-NAS-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: RF-DETR-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-s", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-S, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RF-DETR-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9m", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, Raspberry Pi 5: RF-DETR-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-s", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-S, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RF-DETR-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "yolox-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-m", + "nvidia_geforce_rtx_5070_ti": "yolox-m", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-M, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: RF-DETR-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-l", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "rtdetr-l" + }, + "note": "Speed winner between RT-DETR-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-L, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: RT-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-l", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "rtdetr-l" + }, + "note": "Speed winner between RT-DETR-L and RT-DETR-R50m flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RT-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-l", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "rtdetr-l" + }, + "note": "Speed winner between RT-DETR-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: RT-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-l", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "rtdetr-l" + }, + "note": "Speed winner between RT-DETR-L and RT-DETRv2-R50m flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RT-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-l", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-l", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rtdetr-l" + }, + "note": "Speed winner between RT-DETR-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RT-DETR-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r101", + "rtdetr-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-x", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r101" + }, + "note": "Speed winner between RT-DETR-R101 and RT-DETR-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-X, NVIDIA RTX 5070 Ti: RT-DETR-R101" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r101", + "yolonas-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r101", + "jetson_orin": "yolonas-l" + }, + "note": "Speed winner between RT-DETR-R101 and YOLO-NAS-L flips across hardware: NVIDIA A100: RT-DETR-R101, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r101", + "yolonas-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r101", + "jetson_orin": "yolonas-m" + }, + "note": "Speed winner between RT-DETR-R101 and YOLO-NAS-M flips across hardware: NVIDIA A100: RT-DETR-R101, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r18", + "rtdetrv4-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetr-r18", + "rpi5": "rtdetrv4-s" + }, + "note": "Speed winner between RT-DETR-R18 and RT-DETRv4-S flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: RT-DETRv4-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r18", + "yolonas-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r18", + "rpi5": "yolonas-s" + }, + "note": "Speed winner between RT-DETR-R18 and YOLO-NAS-S flips across hardware: NVIDIA A100: RT-DETR-R18, Raspberry Pi 5: YOLO-NAS-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r34", + "yolonas-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r34", + "jetson_orin": "yolonas-s", + "rpi5": "yolonas-s" + }, + "note": "Speed winner between RT-DETR-R34 and YOLO-NAS-S flips across hardware: NVIDIA A100: RT-DETR-R34, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: YOLO-NAS-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r34", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r34", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "rtdetr-r34" + }, + "note": "Speed winner between RT-DETR-R34 and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RT-DETR-R34" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r34", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r34", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rtdetr-r34" + }, + "note": "Speed winner between RT-DETR-R34 and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RT-DETR-R34" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50", + "rtdetrv4-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "rtdetrv4-l" + }, + "note": "Speed winner between RT-DETR-R50 and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-L, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: RT-DETRv4-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50", + "yolonas-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50", + "jetson_orin": "yolonas-l" + }, + "note": "Speed winner between RT-DETR-R50 and YOLO-NAS-L flips across hardware: NVIDIA A100: RT-DETR-R50, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50", + "yolonas-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50", + "jetson_orin": "yolonas-m", + "rpi5": "yolonas-m" + }, + "note": "Speed winner between RT-DETR-R50 and YOLO-NAS-M flips across hardware: NVIDIA A100: RT-DETR-R50, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-M, Raspberry Pi 5: YOLO-NAS-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50", + "yolonas-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50", + "jetson_orin": "yolonas-s", + "rpi5": "yolonas-s" + }, + "note": "Speed winner between RT-DETR-R50 and YOLO-NAS-S flips across hardware: NVIDIA A100: RT-DETR-R50, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: YOLO-NAS-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50", + "yolox-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50", + "nvidia_geforce_rtx_5070_ti": "yolox-x" + }, + "note": "Speed winner between RT-DETR-R50 and YOLOX-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50, NVIDIA RTX 5070 Ti: YOLOX-X" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50m", + "rtdetrv4-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "rtdetrv4-l" + }, + "note": "Speed winner between RT-DETR-R50m and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RT-DETRv4-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50m", + "rtdetrv4-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "rtdetrv4-m" + }, + "note": "Speed winner between RT-DETR-R50m and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RT-DETRv4-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50m", + "yolonas-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50m", + "jetson_orin": "yolonas-m", + "rpi5": "yolonas-m" + }, + "note": "Speed winner between RT-DETR-R50m and YOLO-NAS-M flips across hardware: NVIDIA A100: RT-DETR-R50m, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-M, Raspberry Pi 5: YOLO-NAS-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50m", + "yolonas-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50m", + "jetson_orin": "yolonas-s", + "rpi5": "yolonas-s" + }, + "note": "Speed winner between RT-DETR-R50m and YOLO-NAS-S flips across hardware: NVIDIA A100: RT-DETR-R50m, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: YOLO-NAS-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetr-x", + "rtdetrv2-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-x", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r101" + }, + "note": "Speed winner between RT-DETR-X and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-X, NVIDIA RTX 5070 Ti: RT-DETRv2-R101" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r18", + "rtdetrv4-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r18", + "rpi5": "rtdetrv4-s" + }, + "note": "Speed winner between RT-DETRv2-R18 and RT-DETRv4-S flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: RT-DETRv4-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r34", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r34", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rtdetrv2-r34" + }, + "note": "Speed winner between RT-DETRv2-R34 and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RT-DETRv2-R34" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r50", + "rtdetrv4-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "rtdetrv4-l" + }, + "note": "Speed winner between RT-DETRv2-R50 and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: RT-DETRv4-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r50", + "yolox-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50", + "nvidia_geforce_rtx_5070_ti": "yolox-x" + }, + "note": "Speed winner between RT-DETRv2-R50 and YOLOX-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50, NVIDIA RTX 5070 Ti: YOLOX-X" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r50m", + "rtdetrv4-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "rtdetrv4-l" + }, + "note": "Speed winner between RT-DETRv2-R50m and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RT-DETRv4-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r50m", + "rtdetrv4-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "rtdetrv4-m" + }, + "note": "Speed winner between RT-DETRv2-R50m and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RT-DETRv4-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r50m", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "yolov9m" + }, + "note": "Speed winner between RT-DETRv2-R50m and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: YOLOv9-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetrv4-l", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-l", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rtdetrv4-l" + }, + "note": "Speed winner between RT-DETRv4-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RT-DETRv4-L" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetrv4-m", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-m", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "rtdetrv4-m" + }, + "note": "Speed winner between RT-DETRv4-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RT-DETRv4-M" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetrv4-s", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-s", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "rtdetrv4-s" + }, + "note": "Speed winner between RT-DETRv4-S and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-S, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RT-DETRv4-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetrv4-s", + "yolov9s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv4-s", + "rpi5": "yolov9s" + }, + "note": "Speed winner between RT-DETRv4-S and YOLOv9-S flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv4-S, Raspberry Pi 5: YOLOv9-S" + }, + { + "kind": "ranking_flip", + "models": [ + "rtdetrv4-s", + "yolox-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-s", + "nvidia_geforce_rtx_5070_ti": "yolox-m", + "rpi5": "rtdetrv4-s" + }, + "note": "Speed winner between RT-DETRv4-S and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-S, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: RT-DETRv4-S" + }, + { + "kind": "ranking_flip", + "models": [ + "yolov9c", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9m", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "yolov9m" + }, + "note": "Speed winner between YOLOv9-C and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: YOLOv9-M" + } + ] + }, + "claims": [ + { + "id": "finding_0", + "kind": "jitter", + "statement": "RF-DETR-N on NVIDIA RTX 5070 Ti/onnx_fp32: fps_mean 65.7 vs fps_p50 94.0 (-30.08% gap).", + "values": { + "kind": "jitter", + "model": "rfdetr-n", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "onnx_fp32", + "fps_mean": 65.7, + "fps_p50": 94.0, + "gap_pct": -30.08, + "note": "RF-DETR-N on NVIDIA RTX 5070 Ti/onnx_fp32: fps_mean 65.7 vs fps_p50 94.0 (-30.08% gap)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_1", + "kind": "jitter", + "statement": "RF-DETR-N on NVIDIA RTX 5070 Ti/tensorrt_fp16: fps_mean 58.3 vs fps_p50 99.3 (-41.23% gap).", + "values": { + "kind": "jitter", + "model": "rfdetr-n", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "tensorrt_fp16", + "fps_mean": 58.3, + "fps_p50": 99.3, + "gap_pct": -41.23, + "note": "RF-DETR-N on NVIDIA RTX 5070 Ti/tensorrt_fp16: fps_mean 58.3 vs fps_p50 99.3 (-41.23% gap)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_2", + "kind": "jitter", + "statement": "RF-DETR-N on NVIDIA Jetson Orin Nano Super 8GB/tensorrt_fp32: fps_mean 23.7 vs fps_p50 27.2 (-12.76% gap).", + "values": { + "kind": "jitter", + "model": "rfdetr-n", + "hardware": "jetson_orin", + "runtime": "tensorrt_fp32", + "fps_mean": 23.7, + "fps_p50": 27.2, + "gap_pct": -12.76, + "note": "RF-DETR-N on NVIDIA Jetson Orin Nano Super 8GB/tensorrt_fp32: fps_mean 23.7 vs fps_p50 27.2 (-12.76% gap)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_3", + "kind": "jitter", + "statement": "RF-DETR-N on NVIDIA RTX 5070 Ti/tensorrt_fp32: fps_mean 67.6 vs fps_p50 108.1 (-37.46% gap).", + "values": { + "kind": "jitter", + "model": "rfdetr-n", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "tensorrt_fp32", + "fps_mean": 67.6, + "fps_p50": 108.1, + "gap_pct": -37.46, + "note": "RF-DETR-N on NVIDIA RTX 5070 Ti/tensorrt_fp32: fps_mean 67.6 vs fps_p50 108.1 (-37.46% gap)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_4", + "kind": "jitter", + "statement": "RF-DETR-N on NVIDIA Jetson Orin Nano Super 8GB/tensorrt_fp16: fps_mean 25.4 vs fps_p50 29.5 (-13.81% gap).", + "values": { + "kind": "jitter", + "model": "rfdetr-n", + "hardware": "jetson_orin", + "runtime": "tensorrt_fp16", + "fps_mean": 25.4, + "fps_p50": 29.5, + "gap_pct": -13.81, + "note": "RF-DETR-N on NVIDIA Jetson Orin Nano Super 8GB/tensorrt_fp16: fps_mean 25.4 vs fps_p50 29.5 (-13.81% gap)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_5", + "kind": "trt_fp16_slower", + "statement": "YOLOv9-T on NVIDIA RTX 5070 Ti: TensorRT FP16 (59.0 FPS) is slower than FP32 (72.0 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "yolov9t", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 59.0, + "fp32_fps": 72.0, + "note": "YOLOv9-T on NVIDIA RTX 5070 Ti: TensorRT FP16 (59.0 FPS) is slower than FP32 (72.0 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_6", + "kind": "trt_fp16_slower", + "statement": "YOLOv9-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (63.8 FPS) is slower than FP32 (75.1 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "yolov9s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 63.8, + "fp32_fps": 75.1, + "note": "YOLOv9-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (63.8 FPS) is slower than FP32 (75.1 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_7", + "kind": "trt_fp16_slower", + "statement": "YOLOv9-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (72.3 FPS) is slower than FP32 (77.5 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "yolov9m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 72.3, + "fp32_fps": 77.5, + "note": "YOLOv9-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (72.3 FPS) is slower than FP32 (77.5 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_8", + "kind": "trt_fp16_slower", + "statement": "YOLOv9-C on NVIDIA RTX 5070 Ti: TensorRT FP16 (74.7 FPS) is slower than FP32 (76.7 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "yolov9c", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 74.7, + "fp32_fps": 76.7, + "note": "YOLOv9-C on NVIDIA RTX 5070 Ti: TensorRT FP16 (74.7 FPS) is slower than FP32 (76.7 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_9", + "kind": "trt_fp16_slower", + "statement": "RF-DETR-N on NVIDIA RTX 5070 Ti: TensorRT FP16 (58.3 FPS) is slower than FP32 (67.6 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "rfdetr-n", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 58.3, + "fp32_fps": 67.6, + "note": "RF-DETR-N on NVIDIA RTX 5070 Ti: TensorRT FP16 (58.3 FPS) is slower than FP32 (67.6 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_10", + "kind": "trt_fp16_slower", + "statement": "RF-DETR-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (71.6 FPS) is slower than FP32 (77.5 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "rfdetr-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 71.6, + "fp32_fps": 77.5, + "note": "RF-DETR-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (71.6 FPS) is slower than FP32 (77.5 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_11", + "kind": "trt_fp16_slower", + "statement": "RF-DETR-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (62.9 FPS) is slower than FP32 (63.7 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "rfdetr-m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 62.9, + "fp32_fps": 63.7, + "note": "RF-DETR-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (62.9 FPS) is slower than FP32 (63.7 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_12", + "kind": "trt_fp16_slower", + "statement": "RT-DETR-R18 on NVIDIA RTX 5070 Ti: TensorRT FP16 (113.4 FPS) is slower than FP32 (120.6 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "rtdetr-r18", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 113.4, + "fp32_fps": 120.6, + "note": "RT-DETR-R18 on NVIDIA RTX 5070 Ti: TensorRT FP16 (113.4 FPS) is slower than FP32 (120.6 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_13", + "kind": "trt_fp16_slower", + "statement": "RT-DETRv2-R18 on NVIDIA RTX 5070 Ti: TensorRT FP16 (117.8 FPS) is slower than FP32 (125.5 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "rtdetrv2-r18", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 117.8, + "fp32_fps": 125.5, + "note": "RT-DETRv2-R18 on NVIDIA RTX 5070 Ti: TensorRT FP16 (117.8 FPS) is slower than FP32 (125.5 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_14", + "kind": "trt_fp16_slower", + "statement": "D-FINE-N on NVIDIA RTX 5070 Ti: TensorRT FP16 (79.7 FPS) is slower than FP32 (98.7 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "dfine-n", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 79.7, + "fp32_fps": 98.7, + "note": "D-FINE-N on NVIDIA RTX 5070 Ti: TensorRT FP16 (79.7 FPS) is slower than FP32 (98.7 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_15", + "kind": "trt_fp16_slower", + "statement": "YOLOX-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (70.9 FPS) is slower than FP32 (74.5 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "yolox-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 70.9, + "fp32_fps": 74.5, + "note": "YOLOX-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (70.9 FPS) is slower than FP32 (74.5 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_16", + "kind": "fp16_accuracy_cliff", + "statement": "YOLOX-S loses 0.9 mAP points going FP32 to FP16 on NVIDIA RTX 5070 Ti.", + "values": { + "kind": "fp16_accuracy_cliff", + "model": "yolox-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "drop_pts": 0.9, + "fp32_mAP": 44.3, + "fp16_mAP": 43.4, + "note": "YOLOX-S loses 0.9 mAP points going FP32 to FP16 on NVIDIA RTX 5070 Ti." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_17", + "kind": "fp16_accuracy_cliff", + "statement": "DEIMv2-X loses 0.6 mAP points going FP32 to FP16 on NVIDIA Jetson Orin Nano Super 8GB.", + "values": { + "kind": "fp16_accuracy_cliff", + "model": "deimv2-x", + "hardware": "jetson_orin", + "drop_pts": 0.6, + "fp32_mAP": 61.3, + "fp16_mAP": 60.7, + "note": "DEIMv2-X loses 0.6 mAP points going FP32 to FP16 on NVIDIA Jetson Orin Nano Super 8GB." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_18", + "kind": "trt_fp16_slower", + "statement": "RT-DETRv4-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (92.3 FPS) is slower than FP32 (97.1 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "rtdetrv4-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 92.3, + "fp32_fps": 97.1, + "note": "RT-DETRv4-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (92.3 FPS) is slower than FP32 (97.1 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_19", + "kind": "trt_fp16_slower", + "statement": "DEIM-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (82.4 FPS) is slower than FP32 (97.6 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "deim-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 82.4, + "fp32_fps": 97.6, + "note": "DEIM-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (82.4 FPS) is slower than FP32 (97.6 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_20", + "kind": "trt_fp16_slower", + "statement": "DEIM-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (83.3 FPS) is slower than FP32 (85.4 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "deim-m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 83.3, + "fp32_fps": 85.4, + "note": "DEIM-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (83.3 FPS) is slower than FP32 (85.4 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_21", + "kind": "trt_fp16_slower", + "statement": "DEIMv2-Atto on NVIDIA RTX 5070 Ti: TensorRT FP16 (132.1 FPS) is slower than FP32 (137.9 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "deimv2-atto", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 132.1, + "fp32_fps": 137.9, + "note": "DEIMv2-Atto on NVIDIA RTX 5070 Ti: TensorRT FP16 (132.1 FPS) is slower than FP32 (137.9 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_22", + "kind": "fp16_accuracy_cliff", + "statement": "DEIMv2-Atto loses 1.7 mAP points going FP32 to FP16 on NVIDIA RTX 5070 Ti.", + "values": { + "kind": "fp16_accuracy_cliff", + "model": "deimv2-atto", + "hardware": "nvidia_geforce_rtx_5070_ti", + "drop_pts": 1.7, + "fp32_mAP": 27.5, + "fp16_mAP": 25.8, + "note": "DEIMv2-Atto loses 1.7 mAP points going FP32 to FP16 on NVIDIA RTX 5070 Ti." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_23", + "kind": "trt_fp16_slower", + "statement": "DEIMv2-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (50.4 FPS) is slower than FP32 (52.5 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "deimv2-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 50.4, + "fp32_fps": 52.5, + "note": "DEIMv2-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (50.4 FPS) is slower than FP32 (52.5 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_24", + "kind": "trt_fp16_slower", + "statement": "DEIMv2-L on NVIDIA RTX 5070 Ti: TensorRT FP16 (38.7 FPS) is slower than FP32 (43.2 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "deimv2-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 38.7, + "fp32_fps": 43.2, + "note": "DEIMv2-L on NVIDIA RTX 5070 Ti: TensorRT FP16 (38.7 FPS) is slower than FP32 (43.2 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_25", + "kind": "trt_fp16_slower", + "statement": "DEIMv2-X on NVIDIA RTX 5070 Ti: TensorRT FP16 (37.9 FPS) is slower than FP32 (38.1 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "deimv2-x", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 37.9, + "fp32_fps": 38.1, + "note": "DEIMv2-X on NVIDIA RTX 5070 Ti: TensorRT FP16 (37.9 FPS) is slower than FP32 (38.1 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_26", + "kind": "fp16_accuracy_cliff", + "statement": "DEIMv2-X loses 0.5 mAP points going FP32 to FP16 on NVIDIA RTX 5070 Ti.", + "values": { + "kind": "fp16_accuracy_cliff", + "model": "deimv2-x", + "hardware": "nvidia_geforce_rtx_5070_ti", + "drop_pts": 0.5, + "fp32_mAP": 61.3, + "fp16_mAP": 60.8, + "note": "DEIMv2-X loses 0.5 mAP points going FP32 to FP16 on NVIDIA RTX 5070 Ti." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_27", + "kind": "trt_fp16_slower", + "statement": "D-FINE-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (86.5 FPS) is slower than FP32 (98.7 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "dfine-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 86.5, + "fp32_fps": 98.7, + "note": "D-FINE-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (86.5 FPS) is slower than FP32 (98.7 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_28", + "kind": "trt_fp16_slower", + "statement": "D-FINE-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (74.5 FPS) is slower than FP32 (85.3 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "dfine-m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 74.5, + "fp32_fps": 85.3, + "note": "D-FINE-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (74.5 FPS) is slower than FP32 (85.3 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_29", + "kind": "trt_fp16_slower", + "statement": "D-FINE-L on NVIDIA RTX 5070 Ti: TensorRT FP16 (69.3 FPS) is slower than FP32 (72.3 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "dfine-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 69.3, + "fp32_fps": 72.3, + "note": "D-FINE-L on NVIDIA RTX 5070 Ti: TensorRT FP16 (69.3 FPS) is slower than FP32 (72.3 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_30", + "kind": "trt_fp16_slower", + "statement": "ECDet-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (43.5 FPS) is slower than FP32 (63.4 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "ec-s", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 43.5, + "fp32_fps": 63.4, + "note": "ECDet-S on NVIDIA RTX 5070 Ti: TensorRT FP16 (43.5 FPS) is slower than FP32 (63.4 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_31", + "kind": "trt_fp16_slower", + "statement": "ECDet-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (48.9 FPS) is slower than FP32 (59.2 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "ec-m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 48.9, + "fp32_fps": 59.2, + "note": "ECDet-M on NVIDIA RTX 5070 Ti: TensorRT FP16 (48.9 FPS) is slower than FP32 (59.2 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_32", + "kind": "trt_fp16_slower", + "statement": "ECDet-L on NVIDIA RTX 5070 Ti: TensorRT FP16 (41.7 FPS) is slower than FP32 (49.0 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "ec-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 41.7, + "fp32_fps": 49.0, + "note": "ECDet-L on NVIDIA RTX 5070 Ti: TensorRT FP16 (41.7 FPS) is slower than FP32 (49.0 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_33", + "kind": "trt_fp16_slower", + "statement": "ECDet-X on NVIDIA RTX 5070 Ti: TensorRT FP16 (41.3 FPS) is slower than FP32 (45.6 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "ec-x", + "hardware": "nvidia_geforce_rtx_5070_ti", + "fp16_fps": 41.3, + "fp32_fps": 45.6, + "note": "ECDet-X on NVIDIA RTX 5070 Ti: TensorRT FP16 (41.3 FPS) is slower than FP32 (45.6 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_34", + "kind": "trt_fp16_slower", + "statement": "DEIMv2-M on NVIDIA Jetson Orin Nano Super 8GB: TensorRT FP16 (7.9 FPS) is slower than FP32 (8.0 FPS).", + "values": { + "kind": "trt_fp16_slower", + "model": "deimv2-m", + "hardware": "jetson_orin", + "fp16_fps": 7.9, + "fp32_fps": 8.0, + "note": "DEIMv2-M on NVIDIA Jetson Orin Nano Super 8GB: TensorRT FP16 (7.9 FPS) is slower than FP32 (8.0 FPS)." + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_35", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-L and DEIMv2-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: DEIMv2-L, Raspberry Pi 5: DEIM-L", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-l", + "deimv2-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "deimv2-l", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and DEIMv2-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: DEIMv2-L, Raspberry Pi 5: DEIM-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_36", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-L and ECDet-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: ECDet-L, Raspberry Pi 5: DEIM-L", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-l", + "ec-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "ec-l", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and ECDet-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: ECDet-L, Raspberry Pi 5: DEIM-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_37", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-L and ECDet-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: ECDet-X, Raspberry Pi 5: DEIM-L", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-l", + "ec-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "ec-x", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and ECDet-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: ECDet-X, Raspberry Pi 5: DEIM-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_38", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-L and RF-DETR-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RF-DETR-L, Raspberry Pi 5: RF-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rfdetr-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "rfdetr-l", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between DEIM-L and RF-DETR-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RF-DETR-L, Raspberry Pi 5: RF-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_39", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-L and RT-DETR-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETR-R101", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetr-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r101" + }, + "note": "Speed winner between DEIM-L and RT-DETR-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETR-R101" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_40", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: DEIM-L", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: DEIM-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_41", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-L and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: DEIM-L", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: DEIM-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_42", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-L and RT-DETR-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETR-X", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetr-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-x" + }, + "note": "Speed winner between DEIM-L and RT-DETR-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETR-X" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_43", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-L and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R101", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetrv2-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r101" + }, + "note": "Speed winner between DEIM-L and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R101" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_44", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: DEIM-L", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: DEIM-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_45", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-L and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: DEIM-L", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-l", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: DEIM-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_46", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: DEIM-L", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-l", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-l", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "deim-l" + }, + "note": "Speed winner between DEIM-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: DEIM-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_47", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-M and DEIMv2-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: DEIM-M, Raspberry Pi 5: DEIMv2-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-m", + "deimv2-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "nvidia_geforce_rtx_5070_ti": "deim-m", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIM-M and DEIMv2-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: DEIM-M, Raspberry Pi 5: DEIMv2-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_48", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-M and ECDet-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, Raspberry Pi 5: ECDet-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-m", + "ec-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "rpi5": "ec-s" + }, + "note": "Speed winner between DEIM-M and ECDet-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, Raspberry Pi 5: ECDet-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_49", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-M and RF-DETR-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: RF-DETR-M, Raspberry Pi 5: RF-DETR-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-m", + "rfdetr-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "nvidia_geforce_rtx_5070_ti": "rfdetr-m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between DEIM-M and RF-DETR-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: RF-DETR-M, Raspberry Pi 5: RF-DETR-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_50", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-M and RT-DETR-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: RT-DETR-R34, Raspberry Pi 5: DEIM-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-m", + "rtdetr-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r34", + "rpi5": "deim-m" + }, + "note": "Speed winner between DEIM-M and RT-DETR-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: RT-DETR-R34, Raspberry Pi 5: DEIM-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_51", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-M and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: DEIM-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-m", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "deim-m" + }, + "note": "Speed winner between DEIM-M and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: DEIM-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_52", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-M and RT-DETRv2-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: RT-DETRv2-R34, Raspberry Pi 5: DEIM-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-m", + "rtdetrv2-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r34", + "rpi5": "deim-m" + }, + "note": "Speed winner between DEIM-M and RT-DETRv2-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: RT-DETRv2-R34, Raspberry Pi 5: DEIM-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_53", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: DEIM-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-m", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "deim-m" + }, + "note": "Speed winner between DEIM-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: DEIM-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_54", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-M and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: DEIM-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-m", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "deim-m" + }, + "note": "Speed winner between DEIM-M and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: DEIM-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_55", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: DEIM-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-m", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-m", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "deim-m" + }, + "note": "Speed winner between DEIM-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-M, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: DEIM-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_56", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-N and PicoDet-L flips across hardware: NVIDIA RTX 5070 Ti: PicoDet-L, Raspberry Pi 5: DEIM-N", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-n", + "picodet-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "picodet-l", + "rpi5": "deim-n" + }, + "note": "Speed winner between DEIM-N and PicoDet-L flips across hardware: NVIDIA RTX 5070 Ti: PicoDet-L, Raspberry Pi 5: DEIM-N" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_57", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-N and YOLOv9-T flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-N, NVIDIA RTX 5070 Ti: DEIM-N, Raspberry Pi 5: YOLOv9-T", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-n", + "yolov9t" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-n", + "nvidia_geforce_rtx_5070_ti": "deim-n", + "rpi5": "yolov9t" + }, + "note": "Speed winner between DEIM-N and YOLOv9-T flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-N, NVIDIA RTX 5070 Ti: DEIM-N, Raspberry Pi 5: YOLOv9-T" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_58", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-N and YOLOX-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-S, NVIDIA RTX 5070 Ti: YOLOX-S, Raspberry Pi 5: DEIM-N", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-n", + "yolox-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-s", + "nvidia_geforce_rtx_5070_ti": "yolox-s", + "rpi5": "deim-n" + }, + "note": "Speed winner between DEIM-N and YOLOX-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-S, NVIDIA RTX 5070 Ti: YOLOX-S, Raspberry Pi 5: DEIM-N" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_59", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-S and RT-DETR-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: DEIM-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-s", + "rtdetr-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetr-r18", + "rpi5": "deim-s" + }, + "note": "Speed winner between DEIM-S and RT-DETR-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: DEIM-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_60", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-S and RT-DETRv2-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: DEIM-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-s", + "rtdetrv2-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r18", + "rpi5": "deim-s" + }, + "note": "Speed winner between DEIM-S and RT-DETRv2-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: DEIM-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_61", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-S and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-S, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: DEIM-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-s", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-s", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "deim-s" + }, + "note": "Speed winner between DEIM-S and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-S, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: DEIM-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_62", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-S, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: DEIM-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-s", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-s", + "nvidia_geforce_rtx_5070_ti": "yolov9m", + "rpi5": "deim-s" + }, + "note": "Speed winner between DEIM-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-S, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: DEIM-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_63", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-S and YOLOv9-S flips across hardware: NVIDIA RTX 5070 Ti: DEIM-S, Raspberry Pi 5: YOLOv9-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-s", + "yolov9s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "deim-s", + "rpi5": "yolov9s" + }, + "note": "Speed winner between DEIM-S and YOLOv9-S flips across hardware: NVIDIA RTX 5070 Ti: DEIM-S, Raspberry Pi 5: YOLOv9-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_64", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-S and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-S, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: DEIM-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-s", + "yolox-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-s", + "nvidia_geforce_rtx_5070_ti": "yolox-m", + "rpi5": "deim-s" + }, + "note": "Speed winner between DEIM-S and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-S, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: DEIM-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_65", + "kind": "ranking_flip", + "statement": "Speed winner between DEIM-X and DEIMv2-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-X, NVIDIA RTX 5070 Ti: DEIMv2-X", + "values": { + "kind": "ranking_flip", + "models": [ + "deim-x", + "deimv2-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deim-x", + "nvidia_geforce_rtx_5070_ti": "deimv2-x" + }, + "note": "Speed winner between DEIM-X and DEIMv2-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIM-X, NVIDIA RTX 5070 Ti: DEIMv2-X" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_66", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-Atto and DEIMv2-Femto flips across hardware: NVIDIA RTX 5070 Ti: DEIMv2-Femto, Raspberry Pi 5: DEIMv2-Atto", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-atto", + "deimv2-femto" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "deimv2-femto", + "rpi5": "deimv2-atto" + }, + "note": "Speed winner between DEIMv2-Atto and DEIMv2-Femto flips across hardware: NVIDIA RTX 5070 Ti: DEIMv2-Femto, Raspberry Pi 5: DEIMv2-Atto" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_67", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-Atto and PicoDet-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-Atto, NVIDIA RTX 5070 Ti: PicoDet-S, Raspberry Pi 5: DEIMv2-Atto", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-atto", + "picodet-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-atto", + "nvidia_geforce_rtx_5070_ti": "picodet-s", + "rpi5": "deimv2-atto" + }, + "note": "Speed winner between DEIMv2-Atto and PicoDet-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-Atto, NVIDIA RTX 5070 Ti: PicoDet-S, Raspberry Pi 5: DEIMv2-Atto" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_68", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-Atto and YOLOX-Nano flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-Nano, NVIDIA RTX 5070 Ti: YOLOX-Nano, Raspberry Pi 5: DEIMv2-Atto", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-atto", + "yolox-nano" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-nano", + "nvidia_geforce_rtx_5070_ti": "yolox-nano", + "rpi5": "deimv2-atto" + }, + "note": "Speed winner between DEIMv2-Atto and YOLOX-Nano flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-Nano, NVIDIA RTX 5070 Ti: YOLOX-Nano, Raspberry Pi 5: DEIMv2-Atto" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_69", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-Femto and PicoDet-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-Femto, NVIDIA RTX 5070 Ti: PicoDet-S, Raspberry Pi 5: PicoDet-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-femto", + "picodet-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-femto", + "nvidia_geforce_rtx_5070_ti": "picodet-s", + "rpi5": "picodet-s" + }, + "note": "Speed winner between DEIMv2-Femto and PicoDet-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-Femto, NVIDIA RTX 5070 Ti: PicoDet-S, Raspberry Pi 5: PicoDet-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_70", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-L and ECDet-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-L, NVIDIA RTX 5070 Ti: ECDet-X, Raspberry Pi 5: DEIMv2-L", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-l", + "ec-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-l", + "nvidia_geforce_rtx_5070_ti": "ec-x", + "rpi5": "deimv2-l" + }, + "note": "Speed winner between DEIMv2-L and ECDet-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-L, NVIDIA RTX 5070 Ti: ECDet-X, Raspberry Pi 5: DEIMv2-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_71", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-L and RT-DETR-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-L, NVIDIA RTX 5070 Ti: RT-DETR-R101", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-l", + "rtdetr-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r101" + }, + "note": "Speed winner between DEIMv2-L and RT-DETR-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-L, NVIDIA RTX 5070 Ti: RT-DETR-R101" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_72", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: DEIMv2-L", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-l", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "deimv2-l" + }, + "note": "Speed winner between DEIMv2-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: DEIMv2-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_73", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-L and RT-DETR-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-L, NVIDIA RTX 5070 Ti: RT-DETR-X", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-l", + "rtdetr-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-x" + }, + "note": "Speed winner between DEIMv2-L and RT-DETR-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-L, NVIDIA RTX 5070 Ti: RT-DETR-X" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_74", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-L and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R101", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-l", + "rtdetrv2-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r101" + }, + "note": "Speed winner between DEIMv2-L and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R101" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_75", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: DEIMv2-L", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-l", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "deimv2-l" + }, + "note": "Speed winner between DEIMv2-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: DEIMv2-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_76", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-M and ECDet-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-M, NVIDIA RTX 5070 Ti: ECDet-L, Raspberry Pi 5: DEIMv2-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "ec-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-m", + "nvidia_geforce_rtx_5070_ti": "ec-l", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and ECDet-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-M, NVIDIA RTX 5070 Ti: ECDet-L, Raspberry Pi 5: DEIMv2-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_77", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-M and RF-DETR-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-M, NVIDIA RTX 5070 Ti: RF-DETR-L, Raspberry Pi 5: DEIMv2-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rfdetr-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-m", + "nvidia_geforce_rtx_5070_ti": "rfdetr-l", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RF-DETR-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-M, NVIDIA RTX 5070 Ti: RF-DETR-L, Raspberry Pi 5: DEIMv2-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_78", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-M and RT-DETR-L flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-L, Raspberry Pi 5: DEIMv2-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rtdetr-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetr-l", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RT-DETR-L flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-L, Raspberry Pi 5: DEIMv2-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_79", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-M and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-M, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: DEIMv2-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-M, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: DEIMv2-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_80", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-M and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: DEIMv2-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: DEIMv2-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_81", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: DEIMv2-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: DEIMv2-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_82", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-M and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: DEIMv2-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: DEIMv2-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_83", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-M and RT-DETRv4-L flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv4-L, Raspberry Pi 5: DEIMv2-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "rtdetrv4-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv4-l", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and RT-DETRv4-L flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv4-L, Raspberry Pi 5: DEIMv2-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_84", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-C, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: DEIMv2-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-m", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9c", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "deimv2-m" + }, + "note": "Speed winner between DEIMv2-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-C, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: DEIMv2-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_85", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-N and PicoDet-L flips across hardware: NVIDIA RTX 5070 Ti: PicoDet-L, Raspberry Pi 5: DEIMv2-N", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-n", + "picodet-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "picodet-l", + "rpi5": "deimv2-n" + }, + "note": "Speed winner between DEIMv2-N and PicoDet-L flips across hardware: NVIDIA RTX 5070 Ti: PicoDet-L, Raspberry Pi 5: DEIMv2-N" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_86", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-N and YOLOv9-T flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-N, NVIDIA RTX 5070 Ti: DEIMv2-N, Raspberry Pi 5: YOLOv9-T", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-n", + "yolov9t" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-n", + "nvidia_geforce_rtx_5070_ti": "deimv2-n", + "rpi5": "yolov9t" + }, + "note": "Speed winner between DEIMv2-N and YOLOv9-T flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-N, NVIDIA RTX 5070 Ti: DEIMv2-N, Raspberry Pi 5: YOLOv9-T" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_87", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-N and YOLOX-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-S, NVIDIA RTX 5070 Ti: YOLOX-S, Raspberry Pi 5: DEIMv2-N", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-n", + "yolox-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-s", + "nvidia_geforce_rtx_5070_ti": "yolox-s", + "rpi5": "deimv2-n" + }, + "note": "Speed winner between DEIMv2-N and YOLOX-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-S, NVIDIA RTX 5070 Ti: YOLOX-S, Raspberry Pi 5: DEIMv2-N" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_88", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-Pico and PicoDet-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-Pico, NVIDIA RTX 5070 Ti: PicoDet-M, Raspberry Pi 5: PicoDet-M", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-pico", + "picodet-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-pico", + "nvidia_geforce_rtx_5070_ti": "picodet-m", + "rpi5": "picodet-m" + }, + "note": "Speed winner between DEIMv2-Pico and PicoDet-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-Pico, NVIDIA RTX 5070 Ti: PicoDet-M, Raspberry Pi 5: PicoDet-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_89", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-S and D-FINE-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: D-FINE-M, Raspberry Pi 5: DEIMv2-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-s", + "dfine-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-m", + "nvidia_geforce_rtx_5070_ti": "dfine-m", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIMv2-S and D-FINE-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: D-FINE-M, Raspberry Pi 5: DEIMv2-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_90", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-S and ECDet-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-S, NVIDIA RTX 5070 Ti: ECDet-M, Raspberry Pi 5: DEIMv2-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-s", + "ec-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "deimv2-s", + "nvidia_geforce_rtx_5070_ti": "ec-m", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIMv2-S and ECDet-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: DEIMv2-S, NVIDIA RTX 5070 Ti: ECDet-M, Raspberry Pi 5: DEIMv2-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_91", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-S and RT-DETR-R18 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R18, NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: DEIMv2-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-s", + "rtdetr-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r18", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r18", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIMv2-S and RT-DETR-R18 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R18, NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: DEIMv2-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_92", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-S and RT-DETRv2-R18 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R18, NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: DEIMv2-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-s", + "rtdetrv2-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r18", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r18", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIMv2-S and RT-DETRv2-R18 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R18, NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: DEIMv2-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_93", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-S and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETRv4-M, Raspberry Pi 5: DEIMv2-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-s", + "rtdetrv4-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv4-m", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIMv2-S and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETRv4-M, Raspberry Pi 5: DEIMv2-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_94", + "kind": "ranking_flip", + "statement": "Speed winner between DEIMv2-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: DEIMv2-S", + "values": { + "kind": "ranking_flip", + "models": [ + "deimv2-s", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9m", + "nvidia_geforce_rtx_5070_ti": "yolov9m", + "rpi5": "deimv2-s" + }, + "note": "Speed winner between DEIMv2-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: DEIMv2-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_95", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-L and ECDet-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: ECDet-L, Raspberry Pi 5: D-FINE-L", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "ec-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-l", + "nvidia_geforce_rtx_5070_ti": "ec-l", + "rpi5": "dfine-l" + }, + "note": "Speed winner between D-FINE-L and ECDet-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: ECDet-L, Raspberry Pi 5: D-FINE-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_96", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-L and RF-DETR-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RF-DETR-L, Raspberry Pi 5: RF-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rfdetr-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-l", + "nvidia_geforce_rtx_5070_ti": "rfdetr-l", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between D-FINE-L and RF-DETR-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RF-DETR-L, Raspberry Pi 5: RF-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_97", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-L and RT-DETR-R101 flips across hardware: NVIDIA A100: RT-DETR-R101, NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RT-DETR-R101", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rtdetr-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r101", + "jetson_orin": "dfine-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r101" + }, + "note": "Speed winner between D-FINE-L and RT-DETR-R101 flips across hardware: NVIDIA A100: RT-DETR-R101, NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RT-DETR-R101" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_98", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-L and RT-DETR-R50 flips across hardware: NVIDIA A100: RT-DETR-R50, NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: D-FINE-L", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50", + "jetson_orin": "dfine-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "dfine-l" + }, + "note": "Speed winner between D-FINE-L and RT-DETR-R50 flips across hardware: NVIDIA A100: RT-DETR-R50, NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: D-FINE-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_99", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-L and RT-DETR-R50m flips across hardware: NVIDIA A100: RT-DETR-R50m, NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: D-FINE-L", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50m", + "jetson_orin": "rtdetr-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "dfine-l" + }, + "note": "Speed winner between D-FINE-L and RT-DETR-R50m flips across hardware: NVIDIA A100: RT-DETR-R50m, NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: D-FINE-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_100", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-L and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R101", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rtdetrv2-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r101" + }, + "note": "Speed winner between D-FINE-L and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R101" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_101", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: D-FINE-L", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "dfine-l" + }, + "note": "Speed winner between D-FINE-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: D-FINE-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_102", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-L and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: D-FINE-L", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "dfine-l" + }, + "note": "Speed winner between D-FINE-L and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: D-FINE-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_103", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: D-FINE-L", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-l", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-l", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "dfine-l" + }, + "note": "Speed winner between D-FINE-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: D-FINE-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_104", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-M and ECDet-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, Raspberry Pi 5: ECDet-S", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "ec-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-m", + "rpi5": "ec-s" + }, + "note": "Speed winner between D-FINE-M and ECDet-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, Raspberry Pi 5: ECDet-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_105", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-M and RF-DETR-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: RF-DETR-M, Raspberry Pi 5: RF-DETR-M", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "rfdetr-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-m", + "nvidia_geforce_rtx_5070_ti": "rfdetr-m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between D-FINE-M and RF-DETR-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: RF-DETR-M, Raspberry Pi 5: RF-DETR-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_106", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-M and RT-DETR-R34 flips across hardware: NVIDIA A100: RT-DETR-R34, NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: RT-DETR-R34, Raspberry Pi 5: D-FINE-M", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "rtdetr-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r34", + "jetson_orin": "rtdetr-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r34", + "rpi5": "dfine-m" + }, + "note": "Speed winner between D-FINE-M and RT-DETR-R34 flips across hardware: NVIDIA A100: RT-DETR-R34, NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: RT-DETR-R34, Raspberry Pi 5: D-FINE-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_107", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-M and RT-DETR-R50m flips across hardware: NVIDIA A100: RT-DETR-R50m, NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: D-FINE-M", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50m", + "jetson_orin": "dfine-m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "dfine-m" + }, + "note": "Speed winner between D-FINE-M and RT-DETR-R50m flips across hardware: NVIDIA A100: RT-DETR-R50m, NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: D-FINE-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_108", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-M and RT-DETRv2-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: RT-DETRv2-R34, Raspberry Pi 5: D-FINE-M", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "rtdetrv2-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r34", + "rpi5": "dfine-m" + }, + "note": "Speed winner between D-FINE-M and RT-DETRv2-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: RT-DETRv2-R34, Raspberry Pi 5: D-FINE-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_109", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: D-FINE-M", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "dfine-m" + }, + "note": "Speed winner between D-FINE-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: D-FINE-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_110", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-M and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: D-FINE-M", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "dfine-m" + }, + "note": "Speed winner between D-FINE-M and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: D-FINE-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_111", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-M and YOLO-NAS-S flips across hardware: NVIDIA A100: D-FINE-M, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: YOLO-NAS-S", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "yolonas-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "dfine-m", + "jetson_orin": "yolonas-s", + "rpi5": "yolonas-s" + }, + "note": "Speed winner between D-FINE-M and YOLO-NAS-S flips across hardware: NVIDIA A100: D-FINE-M, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: YOLO-NAS-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_112", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: D-FINE-M", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-m", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-m", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "dfine-m" + }, + "note": "Speed winner between D-FINE-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-M, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: D-FINE-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_113", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-N and PicoDet-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-N, NVIDIA RTX 5070 Ti: PicoDet-L, Raspberry Pi 5: D-FINE-N", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-n", + "picodet-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-n", + "nvidia_geforce_rtx_5070_ti": "picodet-l", + "rpi5": "dfine-n" + }, + "note": "Speed winner between D-FINE-N and PicoDet-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-N, NVIDIA RTX 5070 Ti: PicoDet-L, Raspberry Pi 5: D-FINE-N" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_114", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-N and YOLOv9-T flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-N, Raspberry Pi 5: YOLOv9-T", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-n", + "yolov9t" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-n", + "rpi5": "yolov9t" + }, + "note": "Speed winner between D-FINE-N and YOLOv9-T flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-N, Raspberry Pi 5: YOLOv9-T" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_115", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-N and YOLOX-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-S, NVIDIA RTX 5070 Ti: YOLOX-S, Raspberry Pi 5: D-FINE-N", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-n", + "yolox-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-s", + "nvidia_geforce_rtx_5070_ti": "yolox-s", + "rpi5": "dfine-n" + }, + "note": "Speed winner between D-FINE-N and YOLOX-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-S, NVIDIA RTX 5070 Ti: YOLOX-S, Raspberry Pi 5: D-FINE-N" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_116", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-S and RT-DETR-R18 flips across hardware: NVIDIA A100: RT-DETR-R18, NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: D-FINE-S", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-s", + "rtdetr-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r18", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r18", + "rpi5": "dfine-s" + }, + "note": "Speed winner between D-FINE-S and RT-DETR-R18 flips across hardware: NVIDIA A100: RT-DETR-R18, NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: D-FINE-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_117", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-S and RT-DETRv2-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: D-FINE-S", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-s", + "rtdetrv2-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r18", + "rpi5": "dfine-s" + }, + "note": "Speed winner between D-FINE-S and RT-DETRv2-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: D-FINE-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_118", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-S and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-S, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: D-FINE-S", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-s", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-s", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "dfine-s" + }, + "note": "Speed winner between D-FINE-S and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-S, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: D-FINE-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_119", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-S, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: D-FINE-S", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-s", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-s", + "nvidia_geforce_rtx_5070_ti": "yolov9m", + "rpi5": "dfine-s" + }, + "note": "Speed winner between D-FINE-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-S, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: D-FINE-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_120", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-S and YOLOv9-S flips across hardware: NVIDIA RTX 5070 Ti: D-FINE-S, Raspberry Pi 5: YOLOv9-S", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-s", + "yolov9s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "dfine-s", + "rpi5": "yolov9s" + }, + "note": "Speed winner between D-FINE-S and YOLOv9-S flips across hardware: NVIDIA RTX 5070 Ti: D-FINE-S, Raspberry Pi 5: YOLOv9-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_121", + "kind": "ranking_flip", + "statement": "Speed winner between D-FINE-S and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-S, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: D-FINE-S", + "values": { + "kind": "ranking_flip", + "models": [ + "dfine-s", + "yolox-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "dfine-s", + "nvidia_geforce_rtx_5070_ti": "yolox-m", + "rpi5": "dfine-s" + }, + "note": "Speed winner between D-FINE-S and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: D-FINE-S, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: D-FINE-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_122", + "kind": "ranking_flip", + "statement": "Speed winner between ECDet-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: ECDet-L", + "values": { + "kind": "ranking_flip", + "models": [ + "ec-l", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "ec-l" + }, + "note": "Speed winner between ECDet-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: ECDet-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_123", + "kind": "ranking_flip", + "statement": "Speed winner between ECDet-L and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: ECDet-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R101", + "values": { + "kind": "ranking_flip", + "models": [ + "ec-l", + "rtdetrv2-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "ec-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r101" + }, + "note": "Speed winner between ECDet-L and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: ECDet-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R101" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_124", + "kind": "ranking_flip", + "statement": "Speed winner between ECDet-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: ECDet-L", + "values": { + "kind": "ranking_flip", + "models": [ + "ec-l", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "ec-l" + }, + "note": "Speed winner between ECDet-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: ECDet-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_125", + "kind": "ranking_flip", + "statement": "Speed winner between ECDet-M and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: ECDet-M, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: ECDet-M", + "values": { + "kind": "ranking_flip", + "models": [ + "ec-m", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "ec-m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "ec-m" + }, + "note": "Speed winner between ECDet-M and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: ECDet-M, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: ECDet-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_126", + "kind": "ranking_flip", + "statement": "Speed winner between ECDet-M and RT-DETR-R50m flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: ECDet-M", + "values": { + "kind": "ranking_flip", + "models": [ + "ec-m", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "ec-m" + }, + "note": "Speed winner between ECDet-M and RT-DETR-R50m flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: ECDet-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_127", + "kind": "ranking_flip", + "statement": "Speed winner between ECDet-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: ECDet-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: ECDet-M", + "values": { + "kind": "ranking_flip", + "models": [ + "ec-m", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "ec-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "ec-m" + }, + "note": "Speed winner between ECDet-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: ECDet-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: ECDet-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_128", + "kind": "ranking_flip", + "statement": "Speed winner between ECDet-M and RT-DETRv2-R50m flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: ECDet-M", + "values": { + "kind": "ranking_flip", + "models": [ + "ec-m", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "ec-m" + }, + "note": "Speed winner between ECDet-M and RT-DETRv2-R50m flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: ECDet-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_129", + "kind": "ranking_flip", + "statement": "Speed winner between ECDet-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-C, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: ECDet-M", + "values": { + "kind": "ranking_flip", + "models": [ + "ec-m", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9c", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "ec-m" + }, + "note": "Speed winner between ECDet-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-C, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: ECDet-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_130", + "kind": "ranking_flip", + "statement": "Speed winner between ECDet-S and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETRv4-M, Raspberry Pi 5: ECDet-S", + "values": { + "kind": "ranking_flip", + "models": [ + "ec-s", + "rtdetrv4-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv4-m", + "rpi5": "ec-s" + }, + "note": "Speed winner between ECDet-S and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETRv4-M, Raspberry Pi 5: ECDet-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_131", + "kind": "ranking_flip", + "statement": "Speed winner between ECDet-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: ECDet-S", + "values": { + "kind": "ranking_flip", + "models": [ + "ec-s", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9m", + "nvidia_geforce_rtx_5070_ti": "yolov9m", + "rpi5": "ec-s" + }, + "note": "Speed winner between ECDet-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: ECDet-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_132", + "kind": "ranking_flip", + "statement": "Speed winner between PicoDet-L and YOLOv9-T flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: PicoDet-L, NVIDIA RTX 5070 Ti: PicoDet-L, Raspberry Pi 5: YOLOv9-T", + "values": { + "kind": "ranking_flip", + "models": [ + "picodet-l", + "yolov9t" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "picodet-l", + "nvidia_geforce_rtx_5070_ti": "picodet-l", + "rpi5": "yolov9t" + }, + "note": "Speed winner between PicoDet-L and YOLOv9-T flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: PicoDet-L, NVIDIA RTX 5070 Ti: PicoDet-L, Raspberry Pi 5: YOLOv9-T" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_133", + "kind": "ranking_flip", + "statement": "Speed winner between PicoDet-S and YOLOX-Nano flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-Nano, NVIDIA RTX 5070 Ti: YOLOX-Nano, Raspberry Pi 5: PicoDet-S", + "values": { + "kind": "ranking_flip", + "models": [ + "picodet-s", + "yolox-nano" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-nano", + "nvidia_geforce_rtx_5070_ti": "yolox-nano", + "rpi5": "picodet-s" + }, + "note": "Speed winner between PicoDet-S and YOLOX-Nano flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-Nano, NVIDIA RTX 5070 Ti: YOLOX-Nano, Raspberry Pi 5: PicoDet-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_134", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-L and RT-DETR-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-L, NVIDIA RTX 5070 Ti: RT-DETR-L, Raspberry Pi 5: RF-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "rtdetr-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-l", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and RT-DETR-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-L, NVIDIA RTX 5070 Ti: RT-DETR-L, Raspberry Pi 5: RF-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_135", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: RF-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: RF-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_136", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-L and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RF-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RF-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_137", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: RF-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: RF-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_138", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-L and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RF-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RF-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_139", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-L and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-L, Raspberry Pi 5: RF-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "rtdetrv4-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-l", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-L, Raspberry Pi 5: RF-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_140", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RF-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-l", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-l", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rfdetr-l" + }, + "note": "Speed winner between RF-DETR-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RF-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_141", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-M and RT-DETR-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: RT-DETR-R34, Raspberry Pi 5: RF-DETR-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "rtdetr-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r34", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and RT-DETR-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: RT-DETR-R34, Raspberry Pi 5: RF-DETR-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_142", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-M and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-M, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RF-DETR-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and RT-DETR-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-M, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RF-DETR-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_143", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-M and RT-DETRv2-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: RT-DETRv2-R34, Raspberry Pi 5: RF-DETR-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "rtdetrv2-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r34", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and RT-DETRv2-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: RT-DETRv2-R34, Raspberry Pi 5: RF-DETR-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_144", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: RF-DETR-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: RF-DETR-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_145", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-M and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RF-DETR-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RF-DETR-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_146", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-M and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETRv4-M, Raspberry Pi 5: RF-DETR-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "rtdetrv4-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv4-m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETRv4-M, Raspberry Pi 5: RF-DETR-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_147", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-M and YOLO-NAS-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-M, Raspberry Pi 5: RF-DETR-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "yolonas-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolonas-m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and YOLO-NAS-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-M, Raspberry Pi 5: RF-DETR-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_148", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-C, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RF-DETR-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9c", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-C, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RF-DETR-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_149", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-M and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: RF-DETR-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9m", + "nvidia_geforce_rtx_5070_ti": "yolov9m", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: YOLOv9-M, Raspberry Pi 5: RF-DETR-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_150", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-M and YOLOX-L flips across hardware: NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RF-DETR-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-m", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rfdetr-m" + }, + "note": "Speed winner between RF-DETR-M and YOLOX-L flips across hardware: NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RF-DETR-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_151", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-N and RT-DETR-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: RF-DETR-N", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-n", + "rtdetr-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetr-r18", + "rpi5": "rfdetr-n" + }, + "note": "Speed winner between RF-DETR-N and RT-DETR-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: RF-DETR-N" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_152", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-N and RT-DETRv2-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: RF-DETR-N", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-n", + "rtdetrv2-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r18", + "rpi5": "rfdetr-n" + }, + "note": "Speed winner between RF-DETR-N and RT-DETRv2-R18 flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: RF-DETR-N" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_153", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-N and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-N, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RF-DETR-N", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-n", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-n", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "rfdetr-n" + }, + "note": "Speed winner between RF-DETR-N and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-N, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RF-DETR-N" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_154", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-N and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-N, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RF-DETR-N", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-n", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-n", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rfdetr-n" + }, + "note": "Speed winner between RF-DETR-N and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-N, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RF-DETR-N" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_155", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-N and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-N, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: RF-DETR-N", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-n", + "yolox-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-n", + "nvidia_geforce_rtx_5070_ti": "yolox-m", + "rpi5": "rfdetr-n" + }, + "note": "Speed winner between RF-DETR-N and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-N, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: RF-DETR-N" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_156", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-S and RT-DETR-R18 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R18, NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: RF-DETR-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "rtdetr-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r18", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r18", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and RT-DETR-R18 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R18, NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: RF-DETR-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_157", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-S and RT-DETR-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: RT-DETR-R34, Raspberry Pi 5: RF-DETR-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "rtdetr-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r34", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and RT-DETR-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: RT-DETR-R34, Raspberry Pi 5: RF-DETR-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_158", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-S and RT-DETRv2-R18 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R18, NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: RF-DETR-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "rtdetrv2-r18" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r18", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r18", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and RT-DETRv2-R18 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R18, NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: RF-DETR-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_159", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-S and RT-DETRv2-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: RT-DETRv2-R34, Raspberry Pi 5: RF-DETR-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "rtdetrv2-r34" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r34", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r34", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and RT-DETRv2-R34 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: RT-DETRv2-R34, Raspberry Pi 5: RF-DETR-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_160", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-S and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-S, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RF-DETR-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-s", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and RT-DETRv2-R50m flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-S, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RF-DETR-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_161", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-S and YOLO-NAS-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: RF-DETR-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "yolonas-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolonas-s", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and YOLO-NAS-S flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: RF-DETR-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_162", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-S and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-S, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RF-DETR-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-s", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-S, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RF-DETR-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_163", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, Raspberry Pi 5: RF-DETR-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9m", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, Raspberry Pi 5: RF-DETR-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_164", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-S and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-S, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RF-DETR-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rfdetr-s", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RF-DETR-S, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RF-DETR-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_165", + "kind": "ranking_flip", + "statement": "Speed winner between RF-DETR-S and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-M, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: RF-DETR-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rfdetr-s", + "yolox-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-m", + "nvidia_geforce_rtx_5070_ti": "yolox-m", + "rpi5": "rfdetr-s" + }, + "note": "Speed winner between RF-DETR-S and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-M, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: RF-DETR-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_166", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-L, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: RT-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-l", + "rtdetr-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "rtdetr-l" + }, + "note": "Speed winner between RT-DETR-L and RT-DETR-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-L, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: RT-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_167", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-L and RT-DETR-R50m flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RT-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-l", + "rtdetr-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "rtdetr-l" + }, + "note": "Speed winner between RT-DETR-L and RT-DETR-R50m flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RT-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_168", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: RT-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-l", + "rtdetrv2-r50" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "rtdetr-l" + }, + "note": "Speed winner between RT-DETR-L and RT-DETRv2-R50 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: RT-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_169", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-L and RT-DETRv2-R50m flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RT-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-l", + "rtdetrv2-r50m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "rtdetr-l" + }, + "note": "Speed winner between RT-DETR-L and RT-DETRv2-R50m flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RT-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_170", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RT-DETR-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-l", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-l", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rtdetr-l" + }, + "note": "Speed winner between RT-DETR-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RT-DETR-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_171", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R101 and RT-DETR-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-X, NVIDIA RTX 5070 Ti: RT-DETR-R101", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r101", + "rtdetr-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-x", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r101" + }, + "note": "Speed winner between RT-DETR-R101 and RT-DETR-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-X, NVIDIA RTX 5070 Ti: RT-DETR-R101" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_172", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R101 and YOLO-NAS-L flips across hardware: NVIDIA A100: RT-DETR-R101, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r101", + "yolonas-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r101", + "jetson_orin": "yolonas-l" + }, + "note": "Speed winner between RT-DETR-R101 and YOLO-NAS-L flips across hardware: NVIDIA A100: RT-DETR-R101, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_173", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R101 and YOLO-NAS-M flips across hardware: NVIDIA A100: RT-DETR-R101, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r101", + "yolonas-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r101", + "jetson_orin": "yolonas-m" + }, + "note": "Speed winner between RT-DETR-R101 and YOLO-NAS-M flips across hardware: NVIDIA A100: RT-DETR-R101, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_174", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R18 and RT-DETRv4-S flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: RT-DETRv4-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r18", + "rtdetrv4-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetr-r18", + "rpi5": "rtdetrv4-s" + }, + "note": "Speed winner between RT-DETR-R18 and RT-DETRv4-S flips across hardware: NVIDIA RTX 5070 Ti: RT-DETR-R18, Raspberry Pi 5: RT-DETRv4-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_175", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R18 and YOLO-NAS-S flips across hardware: NVIDIA A100: RT-DETR-R18, Raspberry Pi 5: YOLO-NAS-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r18", + "yolonas-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r18", + "rpi5": "yolonas-s" + }, + "note": "Speed winner between RT-DETR-R18 and YOLO-NAS-S flips across hardware: NVIDIA A100: RT-DETR-R18, Raspberry Pi 5: YOLO-NAS-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_176", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R34 and YOLO-NAS-S flips across hardware: NVIDIA A100: RT-DETR-R34, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: YOLO-NAS-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r34", + "yolonas-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r34", + "jetson_orin": "yolonas-s", + "rpi5": "yolonas-s" + }, + "note": "Speed winner between RT-DETR-R34 and YOLO-NAS-S flips across hardware: NVIDIA A100: RT-DETR-R34, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: YOLO-NAS-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_177", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R34 and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RT-DETR-R34", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r34", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r34", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "rtdetr-r34" + }, + "note": "Speed winner between RT-DETR-R34 and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RT-DETR-R34" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_178", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R34 and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RT-DETR-R34", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r34", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r34", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rtdetr-r34" + }, + "note": "Speed winner between RT-DETR-R34 and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R34, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RT-DETR-R34" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_179", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R50 and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-L, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: RT-DETRv4-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50", + "rtdetrv4-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-l", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50", + "rpi5": "rtdetrv4-l" + }, + "note": "Speed winner between RT-DETR-R50 and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-L, NVIDIA RTX 5070 Ti: RT-DETR-R50, Raspberry Pi 5: RT-DETRv4-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_180", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R50 and YOLO-NAS-L flips across hardware: NVIDIA A100: RT-DETR-R50, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50", + "yolonas-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50", + "jetson_orin": "yolonas-l" + }, + "note": "Speed winner between RT-DETR-R50 and YOLO-NAS-L flips across hardware: NVIDIA A100: RT-DETR-R50, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_181", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R50 and YOLO-NAS-M flips across hardware: NVIDIA A100: RT-DETR-R50, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-M, Raspberry Pi 5: YOLO-NAS-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50", + "yolonas-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50", + "jetson_orin": "yolonas-m", + "rpi5": "yolonas-m" + }, + "note": "Speed winner between RT-DETR-R50 and YOLO-NAS-M flips across hardware: NVIDIA A100: RT-DETR-R50, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-M, Raspberry Pi 5: YOLO-NAS-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_182", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R50 and YOLO-NAS-S flips across hardware: NVIDIA A100: RT-DETR-R50, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: YOLO-NAS-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50", + "yolonas-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50", + "jetson_orin": "yolonas-s", + "rpi5": "yolonas-s" + }, + "note": "Speed winner between RT-DETR-R50 and YOLO-NAS-S flips across hardware: NVIDIA A100: RT-DETR-R50, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: YOLO-NAS-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_183", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R50 and YOLOX-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50, NVIDIA RTX 5070 Ti: YOLOX-X", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50", + "yolox-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50", + "nvidia_geforce_rtx_5070_ti": "yolox-x" + }, + "note": "Speed winner between RT-DETR-R50 and YOLOX-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50, NVIDIA RTX 5070 Ti: YOLOX-X" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_184", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R50m and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RT-DETRv4-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50m", + "rtdetrv4-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "rtdetrv4-l" + }, + "note": "Speed winner between RT-DETR-R50m and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-R50m, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RT-DETRv4-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_185", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R50m and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RT-DETRv4-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50m", + "rtdetrv4-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-m", + "nvidia_geforce_rtx_5070_ti": "rtdetr-r50m", + "rpi5": "rtdetrv4-m" + }, + "note": "Speed winner between RT-DETR-R50m and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETR-R50m, Raspberry Pi 5: RT-DETRv4-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_186", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R50m and YOLO-NAS-M flips across hardware: NVIDIA A100: RT-DETR-R50m, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-M, Raspberry Pi 5: YOLO-NAS-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50m", + "yolonas-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50m", + "jetson_orin": "yolonas-m", + "rpi5": "yolonas-m" + }, + "note": "Speed winner between RT-DETR-R50m and YOLO-NAS-M flips across hardware: NVIDIA A100: RT-DETR-R50m, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-M, Raspberry Pi 5: YOLO-NAS-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_187", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-R50m and YOLO-NAS-S flips across hardware: NVIDIA A100: RT-DETR-R50m, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: YOLO-NAS-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-r50m", + "yolonas-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "a100": "rtdetr-r50m", + "jetson_orin": "yolonas-s", + "rpi5": "yolonas-s" + }, + "note": "Speed winner between RT-DETR-R50m and YOLO-NAS-S flips across hardware: NVIDIA A100: RT-DETR-R50m, NVIDIA Jetson Orin Nano Super 8GB: YOLO-NAS-S, Raspberry Pi 5: YOLO-NAS-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_188", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETR-X and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-X, NVIDIA RTX 5070 Ti: RT-DETRv2-R101", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetr-x", + "rtdetrv2-r101" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetr-x", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r101" + }, + "note": "Speed winner between RT-DETR-X and RT-DETRv2-R101 flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETR-X, NVIDIA RTX 5070 Ti: RT-DETRv2-R101" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_189", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETRv2-R18 and RT-DETRv4-S flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: RT-DETRv4-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r18", + "rtdetrv4-s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r18", + "rpi5": "rtdetrv4-s" + }, + "note": "Speed winner between RT-DETRv2-R18 and RT-DETRv4-S flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv2-R18, Raspberry Pi 5: RT-DETRv4-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_190", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETRv2-R34 and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RT-DETRv2-R34", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r34", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r34", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rtdetrv2-r34" + }, + "note": "Speed winner between RT-DETRv2-R34 and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R34, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RT-DETRv2-R34" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_191", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETRv2-R50 and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: RT-DETRv4-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r50", + "rtdetrv4-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-l", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50", + "rpi5": "rtdetrv4-l" + }, + "note": "Speed winner between RT-DETRv2-R50 and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-L, NVIDIA RTX 5070 Ti: RT-DETRv2-R50, Raspberry Pi 5: RT-DETRv4-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_192", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETRv2-R50 and YOLOX-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50, NVIDIA RTX 5070 Ti: YOLOX-X", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r50", + "yolox-x" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50", + "nvidia_geforce_rtx_5070_ti": "yolox-x" + }, + "note": "Speed winner between RT-DETRv2-R50 and YOLOX-X flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50, NVIDIA RTX 5070 Ti: YOLOX-X" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_193", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETRv2-R50m and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RT-DETRv4-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r50m", + "rtdetrv4-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv2-r50m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "rtdetrv4-l" + }, + "note": "Speed winner between RT-DETRv2-R50m and RT-DETRv4-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv2-R50m, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RT-DETRv4-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_194", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETRv2-R50m and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RT-DETRv4-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r50m", + "rtdetrv4-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "rtdetrv4-m" + }, + "note": "Speed winner between RT-DETRv2-R50m and RT-DETRv4-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: RT-DETRv4-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_195", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETRv2-R50m and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: YOLOv9-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetrv2-r50m", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9m", + "nvidia_geforce_rtx_5070_ti": "rtdetrv2-r50m", + "rpi5": "yolov9m" + }, + "note": "Speed winner between RT-DETRv2-R50m and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: RT-DETRv2-R50m, Raspberry Pi 5: YOLOv9-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_196", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETRv4-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RT-DETRv4-L", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetrv4-l", + "yolox-l" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolox-l", + "nvidia_geforce_rtx_5070_ti": "yolox-l", + "rpi5": "rtdetrv4-l" + }, + "note": "Speed winner between RT-DETRv4-L and YOLOX-L flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOX-L, NVIDIA RTX 5070 Ti: YOLOX-L, Raspberry Pi 5: RT-DETRv4-L" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_197", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETRv4-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RT-DETRv4-M", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetrv4-m", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-m", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "rtdetrv4-m" + }, + "note": "Speed winner between RT-DETRv4-M and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-M, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RT-DETRv4-M" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_198", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETRv4-S and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-S, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RT-DETRv4-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetrv4-s", + "yolov9c" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-s", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "rtdetrv4-s" + }, + "note": "Speed winner between RT-DETRv4-S and YOLOv9-C flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-S, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: RT-DETRv4-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_199", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETRv4-S and YOLOv9-S flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv4-S, Raspberry Pi 5: YOLOv9-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetrv4-s", + "yolov9s" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "nvidia_geforce_rtx_5070_ti": "rtdetrv4-s", + "rpi5": "yolov9s" + }, + "note": "Speed winner between RT-DETRv4-S and YOLOv9-S flips across hardware: NVIDIA RTX 5070 Ti: RT-DETRv4-S, Raspberry Pi 5: YOLOv9-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_200", + "kind": "ranking_flip", + "statement": "Speed winner between RT-DETRv4-S and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-S, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: RT-DETRv4-S", + "values": { + "kind": "ranking_flip", + "models": [ + "rtdetrv4-s", + "yolox-m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "rtdetrv4-s", + "nvidia_geforce_rtx_5070_ti": "yolox-m", + "rpi5": "rtdetrv4-s" + }, + "note": "Speed winner between RT-DETRv4-S and YOLOX-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: RT-DETRv4-S, NVIDIA RTX 5070 Ti: YOLOX-M, Raspberry Pi 5: RT-DETRv4-S" + }, + "passed": true, + "strength": "normal" + }, + { + "id": "finding_201", + "kind": "ranking_flip", + "statement": "Speed winner between YOLOv9-C and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: YOLOv9-M", + "values": { + "kind": "ranking_flip", + "models": [ + "yolov9c", + "yolov9m" + ], + "runtime": "pytorch_fp32", + "winners_by_hardware": { + "jetson_orin": "yolov9m", + "nvidia_geforce_rtx_5070_ti": "yolov9c", + "rpi5": "yolov9m" + }, + "note": "Speed winner between YOLOv9-C and YOLOv9-M flips across hardware: NVIDIA Jetson Orin Nano Super 8GB: YOLOv9-M, NVIDIA RTX 5070 Ti: YOLOv9-C, Raspberry Pi 5: YOLOv9-M" + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 202, + "required": 1, + "reason": null + }, + "allowed_numbers": [ + -41.23, + -37.46, + -30.08, + -13.81, + -12.76, + 0.001, + 0.5, + 0.6, + 0.9, + 1, + 1.7, + 2, + 3, + 4, + 5, + 6, + 7, + 7.9, + 8.0, + 9, + 10, + 11, + 12, + 23.7, + 25.4, + 25.8, + 27.2, + 27.5, + 29.5, + 37.9, + 38.1, + 38.7, + 41.3, + 41.7, + 43.2, + 43.4, + 43.5, + 44.3, + 45.6, + 48.9, + 49.0, + 50.0, + 50.4, + 52.5, + 58.3, + 59.0, + 59.2, + 60.0, + 60.7, + 60.8, + 61.3, + 62.9, + 63.4, + 63.7, + 63.8, + 65.7, + 67.6, + 69.3, + 70.9, + 71.6, + 72.0, + 72.3, + 74.5, + 74.7, + 75, + 75.1, + 76.7, + 77.5, + 79.7, + 82.4, + 83.3, + 85.3, + 85.4, + 86.5, + 90.0, + 92.3, + 94.0, + 95, + 97.1, + 97.6, + 98.7, + 99.3, + 100, + 108.1, + 113.4, + 117.8, + 120.6, + 125.5, + 132.1, + 137.9, + 202.0, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/ncnn-fp32-vs-onnx-fp32-rpi5.claims.json b/article-pipeline/v2/output/ncnn-fp32-vs-onnx-fp32-rpi5.claims.json new file mode 100644 index 0000000..fe774cb --- /dev/null +++ b/article-pipeline/v2/output/ncnn-fp32-vs-onnx-fp32-rpi5.claims.json @@ -0,0 +1,452 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:10:13.700407+00:00", + "slug": "ncnn-fp32-vs-onnx-fp32-rpi5", + "meta": { + "type": "runtime-guide", + "hardware": "rpi5", + "baseline": "onnx_fp32", + "target": "ncnn_fp32" + }, + "facts": { + "hardware": "rpi5", + "hardware_display": "Raspberry Pi 5", + "baseline": "onnx_fp32", + "baseline_display": "ONNX Runtime FP32", + "target": "ncnn_fp32", + "target_display": "ncnn FP32", + "n_models": 13, + "rows": [ + { + "model": "yolox-x", + "displayName": "YOLOX-X", + "family": "yolox", + "baseline_fps": 0.3, + "target_fps": 0.9, + "speedup_x": 2.87, + "mAP_delta_pts": -1.0, + "baseline_mAP": 54.9, + "target_mAP": 53.9 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "family": "yolox", + "baseline_fps": 0.5, + "target_fps": 1.5, + "speedup_x": 2.85, + "mAP_delta_pts": -1.4, + "baseline_mAP": 54.3, + "target_mAP": 52.9 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "baseline_fps": 1.1, + "target_fps": 2.9, + "speedup_x": 2.72, + "mAP_delta_pts": -1.1, + "baseline_mAP": 50.8, + "target_mAP": 49.7 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "baseline_fps": 0.7, + "target_fps": 1.9, + "speedup_x": 2.61, + "mAP_delta_pts": 0.1, + "baseline_mAP": 55.7, + "target_mAP": 55.8 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "baseline_fps": 1.0, + "target_fps": 2.3, + "speedup_x": 2.44, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.6, + "target_mAP": 53.6 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "baseline_fps": 2.5, + "target_fps": 5.7, + "speedup_x": 2.26, + "mAP_delta_pts": -1.2, + "baseline_mAP": 43.1, + "target_mAP": 41.9 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "baseline_fps": 2.4, + "target_fps": 4.9, + "speedup_x": 2.09, + "mAP_delta_pts": -0.1, + "baseline_mAP": 48.5, + "target_mAP": 48.4 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "baseline_fps": 9.4, + "target_fps": 18.4, + "speedup_x": 1.97, + "mAP_delta_pts": -0.3, + "baseline_mAP": 34.3, + "target_mAP": 34.0 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "baseline_fps": 5.9, + "target_fps": 11.0, + "speedup_x": 1.87, + "mAP_delta_pts": -0.1, + "baseline_mAP": 40.7, + "target_mAP": 40.6 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "baseline_fps": 21.3, + "target_fps": 30.7, + "speedup_x": 1.44, + "mAP_delta_pts": -0.9, + "baseline_mAP": 27.7, + "target_mAP": 26.8 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 6.2, + "target_fps": 8.5, + "speedup_x": 1.36, + "mAP_delta_pts": -0.1, + "baseline_mAP": 36.5, + "target_mAP": 36.4 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 1.7, + "target_fps": 2.3, + "speedup_x": 1.33, + "mAP_delta_pts": 0.1, + "baseline_mAP": 42.7, + "target_mAP": 42.8 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "baseline_fps": 15.0, + "target_fps": 18.8, + "speedup_x": 1.25, + "mAP_delta_pts": -0.4, + "baseline_mAP": 28.1, + "target_mAP": 27.7 + } + ], + "speedup": { + "median_x": 2.09, + "min_x": 1.25, + "max_x": 2.87 + }, + "family_median_speedup": { + "yolox": 2.72, + "yolov9": 2.44, + "picodet": 1.33 + }, + "accuracy_losers": [ + { + "model": "yolox-x", + "displayName": "YOLOX-X", + "family": "yolox", + "baseline_fps": 0.3, + "target_fps": 0.9, + "speedup_x": 2.87, + "mAP_delta_pts": -1.0, + "baseline_mAP": 54.9, + "target_mAP": 53.9 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "family": "yolox", + "baseline_fps": 0.5, + "target_fps": 1.5, + "speedup_x": 2.85, + "mAP_delta_pts": -1.4, + "baseline_mAP": 54.3, + "target_mAP": 52.9 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "baseline_fps": 1.1, + "target_fps": 2.9, + "speedup_x": 2.72, + "mAP_delta_pts": -1.1, + "baseline_mAP": 50.8, + "target_mAP": 49.7 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "baseline_fps": 2.5, + "target_fps": 5.7, + "speedup_x": 2.26, + "mAP_delta_pts": -1.2, + "baseline_mAP": 43.1, + "target_mAP": 41.9 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "baseline_fps": 21.3, + "target_fps": 30.7, + "speedup_x": 1.44, + "mAP_delta_pts": -0.9, + "baseline_mAP": 27.7, + "target_mAP": 26.8 + } + ], + "speed_regressions": [] + }, + "claims": [ + { + "id": "median_speedup", + "kind": "speed", + "statement": "Across 13 models on Raspberry Pi 5, ncnn FP32 gives a median 2.09x over ONNX Runtime FP32 (range 1.25x to 2.87x).", + "values": { + "median_x": 2.09, + "min_x": 1.25, + "max_x": 2.87 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "top_gainer", + "kind": "speed", + "statement": "Biggest gain: YOLOX-X at 2.87x (0.3 to 0.9 FPS).", + "values": { + "model": "yolox-x", + "displayName": "YOLOX-X", + "family": "yolox", + "baseline_fps": 0.3, + "target_fps": 0.9, + "speedup_x": 2.87, + "mAP_delta_pts": -1.0, + "baseline_mAP": 54.9, + "target_mAP": 53.9 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "family_spread", + "kind": "speed", + "statement": "Family medians differ: yolox gains 2.72x, picodet only 1.33x. Conversion gain is architecture-dependent.", + "values": { + "best": [ + "yolox", + 2.72 + ], + "worst": [ + "picodet", + 1.33 + ] + }, + "passed": true, + "strength": "normal" + }, + { + "id": "accuracy_cost", + "kind": "accuracy", + "statement": "5 of 13 models lose 0.5+ mAP points under ncnn FP32: YOLOX-X (-1.0), YOLOX-L (-1.4), YOLOX-M (-1.1), YOLOX-S (-1.2), YOLOX-Nano (-0.9)", + "values": { + "losers": [ + { + "model": "yolox-x", + "displayName": "YOLOX-X", + "family": "yolox", + "baseline_fps": 0.3, + "target_fps": 0.9, + "speedup_x": 2.87, + "mAP_delta_pts": -1.0, + "baseline_mAP": 54.9, + "target_mAP": 53.9 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "family": "yolox", + "baseline_fps": 0.5, + "target_fps": 1.5, + "speedup_x": 2.85, + "mAP_delta_pts": -1.4, + "baseline_mAP": 54.3, + "target_mAP": 52.9 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "baseline_fps": 1.1, + "target_fps": 2.9, + "speedup_x": 2.72, + "mAP_delta_pts": -1.1, + "baseline_mAP": 50.8, + "target_mAP": 49.7 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "baseline_fps": 2.5, + "target_fps": 5.7, + "speedup_x": 2.26, + "mAP_delta_pts": -1.2, + "baseline_mAP": 43.1, + "target_mAP": 41.9 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "baseline_fps": 21.3, + "target_fps": 30.7, + "speedup_x": 1.44, + "mAP_delta_pts": -0.9, + "baseline_mAP": 27.7, + "target_mAP": 26.8 + } + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 4, + "required": 3, + "reason": null + }, + "allowed_numbers": [ + -1.4, + -1.2, + -1.1, + -1.0, + -0.9, + -0.4, + -0.3, + -0.1, + 0.0, + 0.001, + 0.1, + 0.3, + 0.5, + 0.6, + 0.7, + 0.9, + 1.0, + 1.1, + 1.25, + 1.33, + 1.36, + 1.44, + 1.5, + 1.7, + 1.87, + 1.9, + 1.97, + 2, + 2.09, + 2.26, + 2.3, + 2.4, + 2.44, + 2.5, + 2.61, + 2.72, + 2.85, + 2.87, + 2.9, + 3, + 4, + 4.9, + 5, + 5.7, + 5.9, + 6, + 6.2, + 7, + 8, + 8.5, + 9, + 9.4, + 10.0, + 11.0, + 12, + 13.0, + 15.0, + 18.4, + 18.8, + 21.3, + 26.8, + 27.7, + 28.1, + 30.0, + 30.7, + 34.0, + 34.3, + 36.4, + 36.5, + 40.6, + 40.7, + 41.9, + 42.7, + 42.8, + 43.1, + 48.4, + 48.5, + 49.7, + 50.0, + 50.8, + 52.9, + 53.6, + 53.9, + 54.3, + 54.9, + 55.7, + 55.8, + 70.0, + 75, + 90.0, + 95, + 100.0, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/onnx-fp32-vs-pytorch-fp32-jetson-orin.claims.json b/article-pipeline/v2/output/onnx-fp32-vs-pytorch-fp32-jetson-orin.claims.json new file mode 100644 index 0000000..e0fbdc2 --- /dev/null +++ b/article-pipeline/v2/output/onnx-fp32-vs-pytorch-fp32-jetson-orin.claims.json @@ -0,0 +1,1271 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:28.571692+00:00", + "slug": "onnx-fp32-vs-pytorch-fp32-jetson-orin", + "meta": { + "type": "runtime-guide", + "hardware": "jetson_orin", + "baseline": "pytorch_fp32", + "target": "onnx_fp32" + }, + "facts": { + "hardware": "jetson_orin", + "hardware_display": "NVIDIA Jetson Orin Nano Super 8GB", + "baseline": "pytorch_fp32", + "baseline_display": "PyTorch FP32", + "target": "onnx_fp32", + "target_display": "ONNX Runtime FP32", + "n_models": 55, + "rows": [ + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "baseline_fps": 15.4, + "target_fps": 43.9, + "speedup_x": 2.85, + "mAP_delta_pts": 0.0, + "baseline_mAP": 27.5, + "target_mAP": 27.5 + }, + { + "model": "deimv2-femto", + "displayName": "DEIMv2-Femto", + "family": "deimv2", + "baseline_fps": 14.8, + "target_fps": 37.1, + "speedup_x": 2.51, + "mAP_delta_pts": 0.0, + "baseline_mAP": 34.5, + "target_mAP": 34.5 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "baseline_fps": 9.9, + "target_fps": 18.6, + "speedup_x": 1.88, + "mAP_delta_pts": 0.0, + "baseline_mAP": 41.8, + "target_mAP": 41.8 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "baseline_fps": 18.4, + "target_fps": 34.4, + "speedup_x": 1.87, + "mAP_delta_pts": 0.0, + "baseline_mAP": 28.8, + "target_mAP": 28.8 + }, + { + "model": "deimv2-n", + "displayName": "DEIMv2-N", + "family": "deimv2", + "baseline_fps": 11.1, + "target_fps": 17.6, + "speedup_x": 1.58, + "mAP_delta_pts": 0.0, + "baseline_mAP": 46.7, + "target_mAP": 46.7 + }, + { + "model": "deim-n", + "displayName": "DEIM-N", + "family": "deim", + "baseline_fps": 11.3, + "target_fps": 17.5, + "speedup_x": 1.55, + "mAP_delta_pts": 0.0, + "baseline_mAP": 46.8, + "target_mAP": 46.8 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "baseline_fps": 20.3, + "target_fps": 31.4, + "speedup_x": 1.55, + "mAP_delta_pts": 0.0, + "baseline_mAP": 35.5, + "target_mAP": 35.5 + }, + { + "model": "deimv2-pico", + "displayName": "DEIMv2-Pico", + "family": "deimv2", + "baseline_fps": 14.1, + "target_fps": 21.2, + "speedup_x": 1.51, + "mAP_delta_pts": 0.0, + "baseline_mAP": 42.2, + "target_mAP": 42.2 + }, + { + "model": "dfine-n", + "displayName": "D-FINE-N", + "family": "dfine", + "baseline_fps": 11.7, + "target_fps": 17.5, + "speedup_x": 1.5, + "mAP_delta_pts": 0.0, + "baseline_mAP": 45.8, + "target_mAP": 45.8 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "baseline_fps": 9.9, + "target_fps": 12.5, + "speedup_x": 1.27, + "mAP_delta_pts": 0.0, + "baseline_mAP": 50.4, + "target_mAP": 50.4 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "baseline_fps": 13.8, + "target_fps": 16.4, + "speedup_x": 1.18, + "mAP_delta_pts": -0.8, + "baseline_mAP": 30.4, + "target_mAP": 29.6 + }, + { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "family": "rtdetr", + "baseline_fps": 5.0, + "target_fps": 5.4, + "speedup_x": 1.08, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.8, + "target_mAP": 53.8 + }, + { + "model": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "family": "rtdetrv2", + "baseline_fps": 5.1, + "target_fps": 5.4, + "speedup_x": 1.07, + "mAP_delta_pts": 0.0, + "baseline_mAP": 54.8, + "target_mAP": 54.8 + }, + { + "model": "deim-x", + "displayName": "DEIM-X", + "family": "deim", + "baseline_fps": 2.9, + "target_fps": 3.0, + "speedup_x": 1.06, + "mAP_delta_pts": 0.0, + "baseline_mAP": 59.6, + "target_mAP": 59.6 + }, + { + "model": "dfine-l", + "displayName": "D-FINE-L", + "family": "dfine", + "baseline_fps": 4.7, + "target_fps": 5.0, + "speedup_x": 1.06, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.0, + "target_mAP": 60.0 + }, + { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "baseline_fps": 2.9, + "target_fps": 3.0, + "speedup_x": 1.06, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.4, + "target_mAP": 61.4 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "family": "yolox", + "baseline_fps": 5.8, + "target_fps": 6.2, + "speedup_x": 1.06, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.4, + "target_mAP": 55.4 + }, + { + "model": "yolox-x", + "displayName": "YOLOX-X", + "family": "yolox", + "baseline_fps": 3.5, + "target_fps": 3.6, + "speedup_x": 1.06, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.3, + "target_mAP": 56.3 + }, + { + "model": "deim-l", + "displayName": "DEIM-L", + "family": "deim", + "baseline_fps": 4.8, + "target_fps": 5.0, + "speedup_x": 1.05, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "family": "rtdetr", + "baseline_fps": 2.8, + "target_fps": 2.9, + "speedup_x": 1.05, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.8, + "target_mAP": 56.8 + }, + { + "model": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "family": "rtdetrv2", + "baseline_fps": 2.8, + "target_fps": 2.9, + "speedup_x": 1.04, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.8, + "target_mAP": 56.8 + }, + { + "model": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "family": "rtdetr", + "baseline_fps": 4.3, + "target_fps": 4.4, + "speedup_x": 1.03, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.9, + "target_mAP": 55.9 + }, + { + "model": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "family": "rtdetrv2", + "baseline_fps": 4.3, + "target_fps": 4.4, + "speedup_x": 1.03, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.7, + "target_mAP": 55.7 + }, + { + "model": "rtdetrv4-x", + "displayName": "RT-DETRv4-X", + "family": "rtdetrv4", + "baseline_fps": 2.8, + "target_fps": 2.9, + "speedup_x": 1.03, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.0, + "target_mAP": 60.0 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "baseline_fps": 9.1, + "target_fps": 9.3, + "speedup_x": 1.03, + "mAP_delta_pts": 0.1, + "baseline_mAP": 51.7, + "target_mAP": 51.8 + }, + { + "model": "deim-s", + "displayName": "DEIM-S", + "family": "deim", + "baseline_fps": 10.2, + "target_fps": 10.4, + "speedup_x": 1.02, + "mAP_delta_pts": 0.0, + "baseline_mAP": 52.1, + "target_mAP": 52.1 + }, + { + "model": "rtdetr-x", + "displayName": "RT-DETR-X", + "family": "rtdetr", + "baseline_fps": 3.0, + "target_fps": 3.0, + "speedup_x": 1.02, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.9, + "target_mAP": 57.9 + }, + { + "model": "rtdetrv4-s", + "displayName": "RT-DETRv4-S", + "family": "rtdetrv4", + "baseline_fps": 10.1, + "target_fps": 10.3, + "speedup_x": 1.02, + "mAP_delta_pts": 0.0, + "baseline_mAP": 52.8, + "target_mAP": 52.8 + }, + { + "model": "rtdetrv4-l", + "displayName": "RT-DETRv4-L", + "family": "rtdetrv4", + "baseline_fps": 4.7, + "target_fps": 4.8, + "speedup_x": 1.02, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "family": "rfdetr", + "baseline_fps": 10.2, + "target_fps": 10.4, + "speedup_x": 1.02, + "mAP_delta_pts": 0.0, + "baseline_mAP": 51.4, + "target_mAP": 51.4 + }, + { + "model": "dfine-s", + "displayName": "D-FINE-S", + "family": "dfine", + "baseline_fps": 10.3, + "target_fps": 10.4, + "speedup_x": 1.01, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.4, + "target_mAP": 53.4 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "baseline_fps": 16.9, + "target_fps": 17.1, + "speedup_x": 1.01, + "mAP_delta_pts": 0.0, + "baseline_mAP": 44.3, + "target_mAP": 44.3 + }, + { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "family": "rtdetr", + "baseline_fps": 4.9, + "target_fps": 4.8, + "speedup_x": 1.0, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.8, + "target_mAP": 55.8 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "baseline_fps": 6.4, + "target_fps": 6.4, + "speedup_x": 1.0, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.1, + "target_mAP": 57.1 + }, + { + "model": "deim-m", + "displayName": "DEIM-M", + "family": "deim", + "baseline_fps": 7.0, + "target_fps": 6.9, + "speedup_x": 0.99, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.4, + "target_mAP": 55.4 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "family": "dfine", + "baseline_fps": 7.0, + "target_fps": 6.9, + "speedup_x": 0.99, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "baseline_fps": 8.0, + "target_fps": 7.8, + "speedup_x": 0.98, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.1, + "target_mAP": 56.1 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "family": "rtdetrv4", + "baseline_fps": 7.0, + "target_fps": 6.7, + "speedup_x": 0.96, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.5, + "target_mAP": 56.5 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "family": "rtdetr", + "baseline_fps": 10.2, + "target_fps": 9.6, + "speedup_x": 0.95, + "mAP_delta_pts": 0.0, + "baseline_mAP": 49.8, + "target_mAP": 49.8 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "family": "rtdetr", + "baseline_fps": 8.0, + "target_fps": 7.6, + "speedup_x": 0.95, + "mAP_delta_pts": 0.0, + "baseline_mAP": 52.2, + "target_mAP": 52.2 + }, + { + "model": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "family": "rtdetrv2", + "baseline_fps": 10.3, + "target_fps": 9.7, + "speedup_x": 0.94, + "mAP_delta_pts": 0.0, + "baseline_mAP": 50.8, + "target_mAP": 50.8 + }, + { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "baseline_fps": 8.1, + "target_fps": 7.6, + "speedup_x": 0.94, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.2, + "target_mAP": 53.2 + }, + { + "model": "deimv2-s", + "displayName": "DEIMv2-S", + "family": "deimv2", + "baseline_fps": 5.7, + "target_fps": 5.1, + "speedup_x": 0.9, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.0, + "target_mAP": 53.0 + }, + { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "family": "rfdetr", + "baseline_fps": 6.9, + "target_fps": 6.2, + "speedup_x": 0.89, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.1, + "target_mAP": 55.1 + }, + { + "model": "ec-s", + "displayName": "ECDet-S", + "family": "ec", + "baseline_fps": 6.0, + "target_fps": 5.2, + "speedup_x": 0.87, + "mAP_delta_pts": 0.0, + "baseline_mAP": 54.3, + "target_mAP": 54.3 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 12.5, + "target_fps": 10.7, + "speedup_x": 0.86, + "mAP_delta_pts": -0.6, + "baseline_mAP": 37.9, + "target_mAP": 37.3 + }, + { + "model": "rfdetr-m", + "displayName": "RF-DETR-M", + "family": "rfdetr", + "baseline_fps": 5.7, + "target_fps": 4.9, + "speedup_x": 0.85, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.4, + "target_mAP": 57.4 + }, + { + "model": "rfdetr-l", + "displayName": "RF-DETR-L", + "family": "rfdetr", + "baseline_fps": 4.0, + "target_fps": 3.4, + "speedup_x": 0.84, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.6, + "target_mAP": 58.6 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "baseline_fps": 2.7, + "target_fps": 2.2, + "speedup_x": 0.83, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.3, + "target_mAP": 61.3 + }, + { + "model": "ec-x", + "displayName": "ECDet-X", + "family": "ec", + "baseline_fps": 2.9, + "target_fps": 2.4, + "speedup_x": 0.83, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.1, + "target_mAP": 61.1 + }, + { + "model": "deimv2-m", + "displayName": "DEIMv2-M", + "family": "deimv2", + "baseline_fps": 4.7, + "target_fps": 3.8, + "speedup_x": 0.82, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.0, + "target_mAP": 56.0 + }, + { + "model": "deimv2-l", + "displayName": "DEIMv2-L", + "family": "deimv2", + "baseline_fps": 3.3, + "target_fps": 2.7, + "speedup_x": 0.82, + "mAP_delta_pts": -0.1, + "baseline_mAP": 58.6, + "target_mAP": 58.5 + }, + { + "model": "ec-l", + "displayName": "ECDet-L", + "family": "ec", + "baseline_fps": 3.4, + "target_fps": 2.7, + "speedup_x": 0.8, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.1, + "target_mAP": 60.1 + }, + { + "model": "ec-m", + "displayName": "ECDet-M", + "family": "ec", + "baseline_fps": 5.0, + "target_fps": 3.9, + "speedup_x": 0.79, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.4, + "target_mAP": 58.4 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 11.1, + "target_fps": 4.3, + "speedup_x": 0.39, + "mAP_delta_pts": -0.5, + "baseline_mAP": 44.1, + "target_mAP": 43.6 + } + ], + "speedup": { + "median_x": 1.02, + "min_x": 0.39, + "max_x": 2.85 + }, + "family_median_speedup": { + "deimv2": 1.51, + "yolov9": 1.27, + "yolox": 1.06, + "deim": 1.05, + "dfine": 1.06, + "picodet": 0.86, + "rtdetr": 1.02, + "rtdetrv2": 1.03, + "rtdetrv4": 1.02, + "rfdetr": 0.89, + "ec": 0.83 + }, + "accuracy_losers": [ + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "baseline_fps": 13.8, + "target_fps": 16.4, + "speedup_x": 1.18, + "mAP_delta_pts": -0.8, + "baseline_mAP": 30.4, + "target_mAP": 29.6 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 12.5, + "target_fps": 10.7, + "speedup_x": 0.86, + "mAP_delta_pts": -0.6, + "baseline_mAP": 37.9, + "target_mAP": 37.3 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 11.1, + "target_fps": 4.3, + "speedup_x": 0.39, + "mAP_delta_pts": -0.5, + "baseline_mAP": 44.1, + "target_mAP": 43.6 + } + ], + "speed_regressions": [ + { + "model": "deim-m", + "displayName": "DEIM-M", + "family": "deim", + "baseline_fps": 7.0, + "target_fps": 6.9, + "speedup_x": 0.99, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.4, + "target_mAP": 55.4 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "family": "dfine", + "baseline_fps": 7.0, + "target_fps": 6.9, + "speedup_x": 0.99, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "baseline_fps": 8.0, + "target_fps": 7.8, + "speedup_x": 0.98, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.1, + "target_mAP": 56.1 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "family": "rtdetrv4", + "baseline_fps": 7.0, + "target_fps": 6.7, + "speedup_x": 0.96, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.5, + "target_mAP": 56.5 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "family": "rtdetr", + "baseline_fps": 10.2, + "target_fps": 9.6, + "speedup_x": 0.95, + "mAP_delta_pts": 0.0, + "baseline_mAP": 49.8, + "target_mAP": 49.8 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "family": "rtdetr", + "baseline_fps": 8.0, + "target_fps": 7.6, + "speedup_x": 0.95, + "mAP_delta_pts": 0.0, + "baseline_mAP": 52.2, + "target_mAP": 52.2 + }, + { + "model": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "family": "rtdetrv2", + "baseline_fps": 10.3, + "target_fps": 9.7, + "speedup_x": 0.94, + "mAP_delta_pts": 0.0, + "baseline_mAP": 50.8, + "target_mAP": 50.8 + }, + { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "baseline_fps": 8.1, + "target_fps": 7.6, + "speedup_x": 0.94, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.2, + "target_mAP": 53.2 + }, + { + "model": "deimv2-s", + "displayName": "DEIMv2-S", + "family": "deimv2", + "baseline_fps": 5.7, + "target_fps": 5.1, + "speedup_x": 0.9, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.0, + "target_mAP": 53.0 + }, + { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "family": "rfdetr", + "baseline_fps": 6.9, + "target_fps": 6.2, + "speedup_x": 0.89, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.1, + "target_mAP": 55.1 + }, + { + "model": "ec-s", + "displayName": "ECDet-S", + "family": "ec", + "baseline_fps": 6.0, + "target_fps": 5.2, + "speedup_x": 0.87, + "mAP_delta_pts": 0.0, + "baseline_mAP": 54.3, + "target_mAP": 54.3 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 12.5, + "target_fps": 10.7, + "speedup_x": 0.86, + "mAP_delta_pts": -0.6, + "baseline_mAP": 37.9, + "target_mAP": 37.3 + }, + { + "model": "rfdetr-m", + "displayName": "RF-DETR-M", + "family": "rfdetr", + "baseline_fps": 5.7, + "target_fps": 4.9, + "speedup_x": 0.85, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.4, + "target_mAP": 57.4 + }, + { + "model": "rfdetr-l", + "displayName": "RF-DETR-L", + "family": "rfdetr", + "baseline_fps": 4.0, + "target_fps": 3.4, + "speedup_x": 0.84, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.6, + "target_mAP": 58.6 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "baseline_fps": 2.7, + "target_fps": 2.2, + "speedup_x": 0.83, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.3, + "target_mAP": 61.3 + }, + { + "model": "ec-x", + "displayName": "ECDet-X", + "family": "ec", + "baseline_fps": 2.9, + "target_fps": 2.4, + "speedup_x": 0.83, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.1, + "target_mAP": 61.1 + }, + { + "model": "deimv2-m", + "displayName": "DEIMv2-M", + "family": "deimv2", + "baseline_fps": 4.7, + "target_fps": 3.8, + "speedup_x": 0.82, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.0, + "target_mAP": 56.0 + }, + { + "model": "deimv2-l", + "displayName": "DEIMv2-L", + "family": "deimv2", + "baseline_fps": 3.3, + "target_fps": 2.7, + "speedup_x": 0.82, + "mAP_delta_pts": -0.1, + "baseline_mAP": 58.6, + "target_mAP": 58.5 + }, + { + "model": "ec-l", + "displayName": "ECDet-L", + "family": "ec", + "baseline_fps": 3.4, + "target_fps": 2.7, + "speedup_x": 0.8, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.1, + "target_mAP": 60.1 + }, + { + "model": "ec-m", + "displayName": "ECDet-M", + "family": "ec", + "baseline_fps": 5.0, + "target_fps": 3.9, + "speedup_x": 0.79, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.4, + "target_mAP": 58.4 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 11.1, + "target_fps": 4.3, + "speedup_x": 0.39, + "mAP_delta_pts": -0.5, + "baseline_mAP": 44.1, + "target_mAP": 43.6 + } + ] + }, + "claims": [ + { + "id": "median_speedup", + "kind": "speed", + "statement": "Across 55 models on NVIDIA Jetson Orin Nano Super 8GB, ONNX Runtime FP32 gives a median 1.02x over PyTorch FP32 (range 0.39x to 2.85x).", + "values": { + "median_x": 1.02, + "min_x": 0.39, + "max_x": 2.85 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "top_gainer", + "kind": "speed", + "statement": "Biggest gain: DEIMv2-Atto at 2.85x (15.4 to 43.9 FPS).", + "values": { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "baseline_fps": 15.4, + "target_fps": 43.9, + "speedup_x": 2.85, + "mAP_delta_pts": 0.0, + "baseline_mAP": 27.5, + "target_mAP": 27.5 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "family_spread", + "kind": "speed", + "statement": "Family medians differ: deimv2 gains 1.51x, ec only 0.83x. Conversion gain is architecture-dependent.", + "values": { + "best": [ + "deimv2", + 1.51 + ], + "worst": [ + "ec", + 0.83 + ] + }, + "passed": true, + "strength": "normal" + }, + { + "id": "accuracy_cost", + "kind": "accuracy", + "statement": "3 of 55 models lose 0.5+ mAP points under ONNX Runtime FP32: PicoDet-S (-0.8), PicoDet-M (-0.6), PicoDet-L (-0.5)", + "values": { + "losers": [ + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "baseline_fps": 13.8, + "target_fps": 16.4, + "speedup_x": 1.18, + "mAP_delta_pts": -0.8, + "baseline_mAP": 30.4, + "target_mAP": 29.6 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 12.5, + "target_fps": 10.7, + "speedup_x": 0.86, + "mAP_delta_pts": -0.6, + "baseline_mAP": 37.9, + "target_mAP": 37.3 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 11.1, + "target_fps": 4.3, + "speedup_x": 0.39, + "mAP_delta_pts": -0.5, + "baseline_mAP": 44.1, + "target_mAP": 43.6 + } + ] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_regression", + "kind": "anomaly", + "statement": "21 models get SLOWER under ONNX Runtime FP32: DEIM-M (0.99x), D-FINE-M (0.99x), YOLOv9-M (0.98x), RT-DETRv4-M (0.96x), RT-DETR-R18 (0.95x)", + "values": { + "regressions": [ + { + "model": "deim-m", + "displayName": "DEIM-M", + "family": "deim", + "baseline_fps": 7.0, + "target_fps": 6.9, + "speedup_x": 0.99, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.4, + "target_mAP": 55.4 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "family": "dfine", + "baseline_fps": 7.0, + "target_fps": 6.9, + "speedup_x": 0.99, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "baseline_fps": 8.0, + "target_fps": 7.8, + "speedup_x": 0.98, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.1, + "target_mAP": 56.1 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "family": "rtdetrv4", + "baseline_fps": 7.0, + "target_fps": 6.7, + "speedup_x": 0.96, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.5, + "target_mAP": 56.5 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "family": "rtdetr", + "baseline_fps": 10.2, + "target_fps": 9.6, + "speedup_x": 0.95, + "mAP_delta_pts": 0.0, + "baseline_mAP": 49.8, + "target_mAP": 49.8 + } + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 3, + "reason": null + }, + "allowed_numbers": [ + -0.8, + -0.6, + -0.5, + -0.1, + 0.0, + 0.001, + 0.1, + 0.39, + 0.6, + 0.79, + 0.8, + 0.82, + 0.83, + 0.84, + 0.85, + 0.86, + 0.87, + 0.89, + 0.9, + 0.94, + 0.95, + 0.96, + 0.98, + 0.99, + 1.0, + 1.01, + 1.02, + 1.03, + 1.04, + 1.05, + 1.06, + 1.07, + 1.08, + 1.18, + 1.27, + 1.5, + 1.51, + 1.55, + 1.58, + 1.87, + 1.88, + 2, + 2.2, + 2.4, + 2.51, + 2.7, + 2.8, + 2.85, + 2.9, + 3.0, + 3.3, + 3.4, + 3.5, + 3.6, + 3.8, + 3.9, + 4.0, + 4.3, + 4.4, + 4.7, + 4.8, + 4.9, + 5.0, + 5.1, + 5.2, + 5.4, + 5.7, + 5.8, + 6.0, + 6.2, + 6.4, + 6.7, + 6.9, + 7.0, + 7.6, + 7.8, + 8.0, + 8.1, + 9, + 9.1, + 9.3, + 9.6, + 9.7, + 9.9, + 10.0, + 10.1, + 10.2, + 10.3, + 10.4, + 10.7, + 11, + 11.1, + 11.3, + 11.7, + 12, + 12.5, + 13.8, + 14.1, + 14.8, + 15.4, + 16.4, + 16.9, + 17.1, + 17.5, + 17.6, + 18.4, + 18.6, + 20.3, + 21.2, + 27.5, + 28.8, + 29.6, + 30.4, + 31.4, + 34.4, + 34.5, + 35.5, + 37.1, + 37.3, + 37.9, + 39.0, + 41.8, + 42.2, + 43.6, + 43.9, + 44.1, + 44.3, + 45.8, + 46.7, + 46.8, + 49.8, + 50, + 50.4, + 50.8, + 51.4, + 51.7, + 51.8, + 52.1, + 52.2, + 52.8, + 53.0, + 53.2, + 53.4, + 53.8, + 54.3, + 54.8, + 55.0, + 55.1, + 55.4, + 55.7, + 55.8, + 55.9, + 56.0, + 56.1, + 56.3, + 56.5, + 56.8, + 57.1, + 57.4, + 57.8, + 57.9, + 58.4, + 58.5, + 58.6, + 59.6, + 60.0, + 60.1, + 61.1, + 61.3, + 61.4, + 75, + 79.0, + 80.0, + 82.0, + 83.0, + 84.0, + 85.0, + 86.0, + 87.0, + 89.0, + 90.0, + 94.0, + 95.0, + 96.0, + 98.0, + 99.0, + 100.0, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/onnx-fp32-vs-pytorch-fp32-nvidia-geforce-rtx-5070-ti.claims.json b/article-pipeline/v2/output/onnx-fp32-vs-pytorch-fp32-nvidia-geforce-rtx-5070-ti.claims.json new file mode 100644 index 0000000..a188966 --- /dev/null +++ b/article-pipeline/v2/output/onnx-fp32-vs-pytorch-fp32-nvidia-geforce-rtx-5070-ti.claims.json @@ -0,0 +1,967 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:28.693705+00:00", + "slug": "onnx-fp32-vs-pytorch-fp32-nvidia-geforce-rtx-5070-ti", + "meta": { + "type": "runtime-guide", + "hardware": "nvidia_geforce_rtx_5070_ti", + "baseline": "pytorch_fp32", + "target": "onnx_fp32" + }, + "facts": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "baseline": "pytorch_fp32", + "baseline_display": "PyTorch FP32", + "target": "onnx_fp32", + "target_display": "ONNX Runtime FP32", + "n_models": 55, + "rows": [ + { + "model": "deim-l", + "displayName": "DEIM-L", + "family": "deim", + "baseline_fps": 18.6, + "target_fps": 50.5, + "speedup_x": 2.71, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "deim-x", + "displayName": "DEIM-X", + "family": "deim", + "baseline_fps": 15.0, + "target_fps": 39.8, + "speedup_x": 2.66, + "mAP_delta_pts": 0.0, + "baseline_mAP": 59.6, + "target_mAP": 59.6 + }, + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "baseline_fps": 40.4, + "target_fps": 102.0, + "speedup_x": 2.52, + "mAP_delta_pts": 0.0, + "baseline_mAP": 27.5, + "target_mAP": 27.5 + }, + { + "model": "dfine-n", + "displayName": "D-FINE-N", + "family": "dfine", + "baseline_fps": 32.5, + "target_fps": 78.9, + "speedup_x": 2.43, + "mAP_delta_pts": 0.0, + "baseline_mAP": 45.8, + "target_mAP": 45.8 + }, + { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "family": "rtdetr", + "baseline_fps": 27.7, + "target_fps": 64.5, + "speedup_x": 2.33, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.8, + "target_mAP": 55.8 + }, + { + "model": "dfine-l", + "displayName": "D-FINE-L", + "family": "dfine", + "baseline_fps": 21.9, + "target_fps": 50.9, + "speedup_x": 2.33, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.0, + "target_mAP": 60.0 + }, + { + "model": "rtdetrv4-x", + "displayName": "RT-DETRv4-X", + "family": "rtdetrv4", + "baseline_fps": 16.6, + "target_fps": 38.3, + "speedup_x": 2.3, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.0, + "target_mAP": 60.0 + }, + { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "baseline_fps": 18.1, + "target_fps": 41.5, + "speedup_x": 2.29, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.4, + "target_mAP": 61.4 + }, + { + "model": "rtdetr-x", + "displayName": "RT-DETR-X", + "family": "rtdetr", + "baseline_fps": 21.8, + "target_fps": 48.4, + "speedup_x": 2.22, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.9, + "target_mAP": 57.9 + }, + { + "model": "deimv2-n", + "displayName": "DEIMv2-N", + "family": "deimv2", + "baseline_fps": 35.1, + "target_fps": 76.3, + "speedup_x": 2.18, + "mAP_delta_pts": 0.0, + "baseline_mAP": 46.7, + "target_mAP": 46.7 + }, + { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "family": "rtdetr", + "baseline_fps": 35.3, + "target_fps": 76.4, + "speedup_x": 2.17, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.8, + "target_mAP": 53.8 + }, + { + "model": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "family": "rtdetr", + "baseline_fps": 29.8, + "target_fps": 62.8, + "speedup_x": 2.11, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.9, + "target_mAP": 55.9 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "family": "dfine", + "baseline_fps": 28.8, + "target_fps": 59.8, + "speedup_x": 2.08, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "dfine-s", + "displayName": "D-FINE-S", + "family": "dfine", + "baseline_fps": 34.2, + "target_fps": 70.9, + "speedup_x": 2.07, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.4, + "target_mAP": 53.4 + }, + { + "model": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "family": "rtdetrv2", + "baseline_fps": 32.1, + "target_fps": 66.0, + "speedup_x": 2.05, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.7, + "target_mAP": 55.7 + }, + { + "model": "deim-n", + "displayName": "DEIM-N", + "family": "deim", + "baseline_fps": 37.0, + "target_fps": 75.9, + "speedup_x": 2.05, + "mAP_delta_pts": 0.0, + "baseline_mAP": 46.8, + "target_mAP": 46.8 + }, + { + "model": "deim-m", + "displayName": "DEIM-M", + "family": "deim", + "baseline_fps": 28.4, + "target_fps": 57.6, + "speedup_x": 2.03, + "mAP_delta_pts": -0.1, + "baseline_mAP": 55.5, + "target_mAP": 55.4 + }, + { + "model": "deim-s", + "displayName": "DEIM-S", + "family": "deim", + "baseline_fps": 33.4, + "target_fps": 67.5, + "speedup_x": 2.02, + "mAP_delta_pts": 0.0, + "baseline_mAP": 52.1, + "target_mAP": 52.1 + }, + { + "model": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "family": "rtdetr", + "baseline_fps": 23.6, + "target_fps": 47.5, + "speedup_x": 2.01, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.8, + "target_mAP": 56.8 + }, + { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "baseline_fps": 40.4, + "target_fps": 81.1, + "speedup_x": 2.01, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.2, + "target_mAP": 53.2 + }, + { + "model": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "family": "rtdetrv2", + "baseline_fps": 39.9, + "target_fps": 79.6, + "speedup_x": 2.0, + "mAP_delta_pts": 0.0, + "baseline_mAP": 54.8, + "target_mAP": 54.8 + }, + { + "model": "deimv2-femto", + "displayName": "DEIMv2-Femto", + "family": "deimv2", + "baseline_fps": 45.3, + "target_fps": 90.2, + "speedup_x": 1.99, + "mAP_delta_pts": 0.0, + "baseline_mAP": 34.5, + "target_mAP": 34.5 + }, + { + "model": "deimv2-pico", + "displayName": "DEIMv2-Pico", + "family": "deimv2", + "baseline_fps": 40.5, + "target_fps": 80.2, + "speedup_x": 1.98, + "mAP_delta_pts": 0.0, + "baseline_mAP": 42.3, + "target_mAP": 42.3 + }, + { + "model": "rfdetr-m", + "displayName": "RF-DETR-M", + "family": "rfdetr", + "baseline_fps": 30.3, + "target_fps": 59.8, + "speedup_x": 1.97, + "mAP_delta_pts": 0.1, + "baseline_mAP": 57.3, + "target_mAP": 57.4 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "family": "rtdetr", + "baseline_fps": 47.2, + "target_fps": 92.5, + "speedup_x": 1.96, + "mAP_delta_pts": 0.0, + "baseline_mAP": 49.8, + "target_mAP": 49.8 + }, + { + "model": "rtdetrv4-l", + "displayName": "RT-DETRv4-L", + "family": "rtdetrv4", + "baseline_fps": 24.7, + "target_fps": 48.0, + "speedup_x": 1.94, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "family": "rtdetr", + "baseline_fps": 40.2, + "target_fps": 77.7, + "speedup_x": 1.93, + "mAP_delta_pts": 0.0, + "baseline_mAP": 52.2, + "target_mAP": 52.2 + }, + { + "model": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "family": "rtdetrv2", + "baseline_fps": 25.7, + "target_fps": 49.3, + "speedup_x": 1.92, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.8, + "target_mAP": 56.8 + }, + { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "family": "rfdetr", + "baseline_fps": 35.8, + "target_fps": 68.5, + "speedup_x": 1.91, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.1, + "target_mAP": 55.1 + }, + { + "model": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "family": "rtdetrv2", + "baseline_fps": 51.1, + "target_fps": 97.1, + "speedup_x": 1.9, + "mAP_delta_pts": 0.0, + "baseline_mAP": 50.7, + "target_mAP": 50.7 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "baseline_fps": 30.8, + "target_fps": 54.5, + "speedup_x": 1.77, + "mAP_delta_pts": 0.0, + "baseline_mAP": 50.4, + "target_mAP": 50.4 + }, + { + "model": "rfdetr-l", + "displayName": "RF-DETR-L", + "family": "rfdetr", + "baseline_fps": 25.1, + "target_fps": 43.8, + "speedup_x": 1.75, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.6, + "target_mAP": 58.6 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "baseline_fps": 37.1, + "target_fps": 64.5, + "speedup_x": 1.74, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.1, + "target_mAP": 56.1 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "family": "rtdetrv4", + "baseline_fps": 33.6, + "target_fps": 56.6, + "speedup_x": 1.69, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.5, + "target_mAP": 56.5 + }, + { + "model": "rtdetrv4-s", + "displayName": "RT-DETRv4-S", + "family": "rtdetrv4", + "baseline_fps": 39.8, + "target_fps": 66.7, + "speedup_x": 1.67, + "mAP_delta_pts": -0.1, + "baseline_mAP": 52.9, + "target_mAP": 52.8 + }, + { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "family": "rfdetr", + "baseline_fps": 39.5, + "target_fps": 65.7, + "speedup_x": 1.66, + "mAP_delta_pts": 0.0, + "baseline_mAP": 51.4, + "target_mAP": 51.4 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "baseline_fps": 31.9, + "target_fps": 52.1, + "speedup_x": 1.63, + "mAP_delta_pts": 0.0, + "baseline_mAP": 41.8, + "target_mAP": 41.8 + }, + { + "model": "deimv2-s", + "displayName": "DEIMv2-S", + "family": "deimv2", + "baseline_fps": 24.3, + "target_fps": 39.6, + "speedup_x": 1.63, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.0, + "target_mAP": 53.0 + }, + { + "model": "deimv2-m", + "displayName": "DEIMv2-M", + "family": "deimv2", + "baseline_fps": 22.1, + "target_fps": 36.0, + "speedup_x": 1.63, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.0, + "target_mAP": 56.0 + }, + { + "model": "deimv2-l", + "displayName": "DEIMv2-L", + "family": "deimv2", + "baseline_fps": 19.7, + "target_fps": 31.4, + "speedup_x": 1.59, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.6, + "target_mAP": 58.6 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "baseline_fps": 17.4, + "target_fps": 27.1, + "speedup_x": 1.56, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.3, + "target_mAP": 61.3 + }, + { + "model": "ec-s", + "displayName": "ECDet-S", + "family": "ec", + "baseline_fps": 27.5, + "target_fps": 42.8, + "speedup_x": 1.55, + "mAP_delta_pts": -0.1, + "baseline_mAP": 54.4, + "target_mAP": 54.3 + }, + { + "model": "ec-m", + "displayName": "ECDet-M", + "family": "ec", + "baseline_fps": 26.5, + "target_fps": 40.2, + "speedup_x": 1.51, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.4, + "target_mAP": 58.4 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "baseline_fps": 42.3, + "target_fps": 63.5, + "speedup_x": 1.5, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.1, + "target_mAP": 57.1 + }, + { + "model": "ec-l", + "displayName": "ECDet-L", + "family": "ec", + "baseline_fps": 24.2, + "target_fps": 33.6, + "speedup_x": 1.39, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.1, + "target_mAP": 60.1 + }, + { + "model": "ec-x", + "displayName": "ECDet-X", + "family": "ec", + "baseline_fps": 23.0, + "target_fps": 31.9, + "speedup_x": 1.39, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.1, + "target_mAP": 61.1 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "baseline_fps": 61.6, + "target_fps": 76.2, + "speedup_x": 1.24, + "mAP_delta_pts": 0.0, + "baseline_mAP": 35.5, + "target_mAP": 35.5 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "baseline_fps": 50.0, + "target_fps": 60.6, + "speedup_x": 1.21, + "mAP_delta_pts": 0.0, + "baseline_mAP": 44.3, + "target_mAP": 44.3 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "baseline_fps": 53.6, + "target_fps": 63.2, + "speedup_x": 1.18, + "mAP_delta_pts": 0.0, + "baseline_mAP": 28.8, + "target_mAP": 28.8 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "family": "yolox", + "baseline_fps": 42.8, + "target_fps": 50.6, + "speedup_x": 1.18, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.4, + "target_mAP": 55.4 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "baseline_fps": 48.2, + "target_fps": 55.8, + "speedup_x": 1.16, + "mAP_delta_pts": 0.1, + "baseline_mAP": 51.7, + "target_mAP": 51.8 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "baseline_fps": 50.0, + "target_fps": 55.8, + "speedup_x": 1.11, + "mAP_delta_pts": 0.0, + "baseline_mAP": 30.4, + "target_mAP": 30.4 + }, + { + "model": "yolox-x", + "displayName": "YOLOX-X", + "family": "yolox", + "baseline_fps": 39.6, + "target_fps": 41.6, + "speedup_x": 1.05, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.3, + "target_mAP": 56.3 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 45.1, + "target_fps": 46.0, + "speedup_x": 1.02, + "mAP_delta_pts": 0.0, + "baseline_mAP": 37.9, + "target_mAP": 37.9 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 39.5, + "target_fps": 34.2, + "speedup_x": 0.87, + "mAP_delta_pts": 0.0, + "baseline_mAP": 44.1, + "target_mAP": 44.1 + } + ], + "speedup": { + "median_x": 1.92, + "min_x": 0.87, + "max_x": 2.71 + }, + "family_median_speedup": { + "deim": 2.05, + "deimv2": 1.98, + "dfine": 2.29, + "rtdetr": 2.11, + "rtdetrv4": 1.94, + "rtdetrv2": 2.0, + "rfdetr": 1.91, + "yolov9": 1.74, + "ec": 1.51, + "yolox": 1.18, + "picodet": 1.02 + }, + "accuracy_losers": [], + "speed_regressions": [ + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 39.5, + "target_fps": 34.2, + "speedup_x": 0.87, + "mAP_delta_pts": 0.0, + "baseline_mAP": 44.1, + "target_mAP": 44.1 + } + ] + }, + "claims": [ + { + "id": "median_speedup", + "kind": "speed", + "statement": "Across 55 models on NVIDIA RTX 5070 Ti, ONNX Runtime FP32 gives a median 1.92x over PyTorch FP32 (range 0.87x to 2.71x).", + "values": { + "median_x": 1.92, + "min_x": 0.87, + "max_x": 2.71 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "top_gainer", + "kind": "speed", + "statement": "Biggest gain: DEIM-L at 2.71x (18.6 to 50.5 FPS).", + "values": { + "model": "deim-l", + "displayName": "DEIM-L", + "family": "deim", + "baseline_fps": 18.6, + "target_fps": 50.5, + "speedup_x": 2.71, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "family_spread", + "kind": "speed", + "statement": "Family medians differ: dfine gains 2.29x, picodet only 1.02x. Conversion gain is architecture-dependent.", + "values": { + "best": [ + "dfine", + 2.29 + ], + "worst": [ + "picodet", + 1.02 + ] + }, + "passed": true, + "strength": "normal" + }, + { + "id": "accuracy_cost", + "kind": "accuracy", + "statement": "No model loses more than 0.5 mAP points converting to ONNX Runtime FP32.", + "values": { + "losers": [] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_regression", + "kind": "anomaly", + "statement": "1 models get SLOWER under ONNX Runtime FP32: PicoDet-L (0.87x)", + "values": { + "regressions": [ + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 39.5, + "target_fps": 34.2, + "speedup_x": 0.87, + "mAP_delta_pts": 0.0, + "baseline_mAP": 44.1, + "target_mAP": 44.1 + } + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 3, + "reason": null + }, + "allowed_numbers": [ + -0.1, + 0.0, + 0.001, + 0.1, + 0.6, + 0.87, + 1, + 1.02, + 1.05, + 1.11, + 1.16, + 1.18, + 1.21, + 1.24, + 1.39, + 1.5, + 1.51, + 1.55, + 1.56, + 1.59, + 1.63, + 1.66, + 1.67, + 1.69, + 1.74, + 1.75, + 1.77, + 1.9, + 1.91, + 1.92, + 1.93, + 1.94, + 1.96, + 1.97, + 1.98, + 1.99, + 2.0, + 2.01, + 2.02, + 2.03, + 2.05, + 2.07, + 2.08, + 2.11, + 2.17, + 2.18, + 2.22, + 2.29, + 2.3, + 2.33, + 2.43, + 2.52, + 2.66, + 2.71, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10.0, + 11, + 12, + 15.0, + 16.6, + 17.4, + 18.1, + 18.6, + 19.7, + 21.8, + 21.9, + 22.1, + 23.0, + 23.6, + 24.2, + 24.3, + 24.7, + 25.1, + 25.7, + 26.5, + 27.1, + 27.5, + 27.7, + 28.4, + 28.8, + 29.8, + 30.3, + 30.4, + 30.8, + 31.4, + 31.9, + 32.1, + 32.5, + 33.4, + 33.6, + 34.2, + 34.5, + 35.1, + 35.3, + 35.5, + 35.8, + 36.0, + 37.0, + 37.1, + 37.9, + 38.3, + 39.5, + 39.6, + 39.8, + 39.9, + 40.2, + 40.4, + 40.5, + 41.5, + 41.6, + 41.8, + 42.3, + 42.8, + 43.8, + 44.1, + 44.3, + 45.1, + 45.3, + 45.8, + 46.0, + 46.7, + 46.8, + 47.2, + 47.5, + 48.0, + 48.2, + 48.4, + 49.3, + 49.8, + 50.0, + 50.4, + 50.5, + 50.6, + 50.7, + 50.9, + 51.1, + 51.4, + 51.7, + 51.8, + 52.1, + 52.2, + 52.8, + 52.9, + 53.0, + 53.2, + 53.4, + 53.6, + 53.8, + 54.3, + 54.4, + 54.5, + 54.8, + 55.0, + 55.1, + 55.4, + 55.5, + 55.7, + 55.8, + 55.9, + 56.0, + 56.1, + 56.3, + 56.5, + 56.6, + 56.8, + 57.1, + 57.3, + 57.4, + 57.6, + 57.8, + 57.9, + 58.4, + 58.6, + 59.6, + 59.8, + 60.0, + 60.1, + 60.6, + 61.1, + 61.3, + 61.4, + 61.6, + 62.8, + 63.2, + 63.5, + 64.5, + 65.7, + 66.0, + 66.7, + 67.5, + 68.5, + 70.9, + 75, + 75.9, + 76.2, + 76.3, + 76.4, + 77.7, + 78.9, + 79.6, + 80.2, + 81.1, + 87.0, + 90.2, + 92.5, + 95, + 97.1, + 100, + 102.0, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/onnx-fp32-vs-pytorch-fp32-rpi5.claims.json b/article-pipeline/v2/output/onnx-fp32-vs-pytorch-fp32-rpi5.claims.json new file mode 100644 index 0000000..da4f9a9 --- /dev/null +++ b/article-pipeline/v2/output/onnx-fp32-vs-pytorch-fp32-rpi5.claims.json @@ -0,0 +1,986 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T21:56:12.822922+00:00", + "slug": "onnx-fp32-vs-pytorch-fp32-rpi5", + "meta": { + "type": "runtime-guide", + "hardware": "rpi5", + "baseline": "pytorch_fp32", + "target": "onnx_fp32" + }, + "facts": { + "hardware": "rpi5", + "hardware_display": "Raspberry Pi 5", + "baseline": "pytorch_fp32", + "baseline_display": "PyTorch FP32", + "target": "onnx_fp32", + "target_display": "ONNX Runtime FP32", + "n_models": 48, + "rows": [ + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "baseline_fps": 8.6, + "target_fps": 21.3, + "speedup_x": 2.49, + "mAP_delta_pts": -1.1, + "baseline_mAP": 28.8, + "target_mAP": 27.7 + }, + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "baseline_fps": 12.7, + "target_fps": 30.3, + "speedup_x": 2.39, + "mAP_delta_pts": 0.0, + "baseline_mAP": 27.5, + "target_mAP": 27.5 + }, + { + "model": "deimv2-femto", + "displayName": "DEIMv2-Femto", + "family": "deimv2", + "baseline_fps": 7.4, + "target_fps": 16.7, + "speedup_x": 2.26, + "mAP_delta_pts": 0.0, + "baseline_mAP": 34.5, + "target_mAP": 34.5 + }, + { + "model": "deimv2-pico", + "displayName": "DEIMv2-Pico", + "family": "deimv2", + "baseline_fps": 2.9, + "target_fps": 6.2, + "speedup_x": 2.16, + "mAP_delta_pts": 0.0, + "baseline_mAP": 42.2, + "target_mAP": 42.2 + }, + { + "model": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "family": "rtdetrv2", + "baseline_fps": 0.4, + "target_fps": 0.8, + "speedup_x": 2.08, + "mAP_delta_pts": 0.0, + "baseline_mAP": 54.8, + "target_mAP": 54.8 + }, + { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "family": "rtdetr", + "baseline_fps": 0.4, + "target_fps": 0.8, + "speedup_x": 2.08, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.8, + "target_mAP": 53.8 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "baseline_fps": 2.9, + "target_fps": 5.9, + "speedup_x": 2.03, + "mAP_delta_pts": -0.7, + "baseline_mAP": 41.4, + "target_mAP": 40.7 + }, + { + "model": "deim-n", + "displayName": "DEIM-N", + "family": "deim", + "baseline_fps": 2.3, + "target_fps": 4.3, + "speedup_x": 1.88, + "mAP_delta_pts": 0.0, + "baseline_mAP": 46.8, + "target_mAP": 46.8 + }, + { + "model": "deimv2-n", + "displayName": "DEIMv2-N", + "family": "deimv2", + "baseline_fps": 2.4, + "target_fps": 4.4, + "speedup_x": 1.87, + "mAP_delta_pts": 0.0, + "baseline_mAP": 46.7, + "target_mAP": 46.7 + }, + { + "model": "dfine-n", + "displayName": "D-FINE-N", + "family": "dfine", + "baseline_fps": 2.3, + "target_fps": 4.3, + "speedup_x": 1.87, + "mAP_delta_pts": 0.0, + "baseline_mAP": 45.8, + "target_mAP": 45.8 + }, + { + "model": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "family": "rtdetr", + "baseline_fps": 0.3, + "target_fps": 0.6, + "speedup_x": 1.87, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.9, + "target_mAP": 55.9 + }, + { + "model": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "family": "rtdetrv2", + "baseline_fps": 0.3, + "target_fps": 0.6, + "speedup_x": 1.87, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.7, + "target_mAP": 55.7 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "baseline_fps": 5.5, + "target_fps": 9.4, + "speedup_x": 1.72, + "mAP_delta_pts": -0.8, + "baseline_mAP": 35.1, + "target_mAP": 34.3 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "baseline_fps": 1.4, + "target_fps": 2.4, + "speedup_x": 1.68, + "mAP_delta_pts": -0.9, + "baseline_mAP": 49.4, + "target_mAP": 48.5 + }, + { + "model": "dfine-l", + "displayName": "D-FINE-L", + "family": "dfine", + "baseline_fps": 0.4, + "target_fps": 0.7, + "speedup_x": 1.65, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.0, + "target_mAP": 60.0 + }, + { + "model": "rtdetrv4-l", + "displayName": "RT-DETRv4-L", + "family": "rtdetrv4", + "baseline_fps": 0.4, + "target_fps": 0.7, + "speedup_x": 1.63, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "deim-l", + "displayName": "DEIM-L", + "family": "deim", + "baseline_fps": 0.4, + "target_fps": 0.7, + "speedup_x": 1.61, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "deim-m", + "displayName": "DEIM-M", + "family": "deim", + "baseline_fps": 0.6, + "target_fps": 1.0, + "speedup_x": 1.6, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.4, + "target_mAP": 55.4 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "family": "dfine", + "baseline_fps": 0.6, + "target_fps": 1.0, + "speedup_x": 1.6, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "baseline_fps": 1.6, + "target_fps": 2.5, + "speedup_x": 1.59, + "mAP_delta_pts": 0.1, + "baseline_mAP": 43.0, + "target_mAP": 43.1 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "family": "rtdetr", + "baseline_fps": 0.8, + "target_fps": 1.3, + "speedup_x": 1.59, + "mAP_delta_pts": 0.1, + "baseline_mAP": 49.7, + "target_mAP": 49.8 + }, + { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "family": "rtdetr", + "baseline_fps": 0.4, + "target_fps": 0.7, + "speedup_x": 1.58, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.8, + "target_mAP": 55.8 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "baseline_fps": 0.6, + "target_fps": 1.0, + "speedup_x": 1.57, + "mAP_delta_pts": -1.7, + "baseline_mAP": 55.3, + "target_mAP": 53.6 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "baseline_fps": 0.5, + "target_fps": 0.7, + "speedup_x": 1.57, + "mAP_delta_pts": -0.7, + "baseline_mAP": 56.4, + "target_mAP": 55.7 + }, + { + "model": "deim-s", + "displayName": "DEIM-S", + "family": "deim", + "baseline_fps": 1.1, + "target_fps": 1.8, + "speedup_x": 1.57, + "mAP_delta_pts": 0.0, + "baseline_mAP": 52.1, + "target_mAP": 52.1 + }, + { + "model": "dfine-s", + "displayName": "D-FINE-S", + "family": "dfine", + "baseline_fps": 1.1, + "target_fps": 1.8, + "speedup_x": 1.57, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.4, + "target_mAP": 53.4 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "family": "rtdetrv4", + "baseline_fps": 0.6, + "target_fps": 1.0, + "speedup_x": 1.57, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.5, + "target_mAP": 56.5 + }, + { + "model": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "family": "rtdetrv2", + "baseline_fps": 0.8, + "target_fps": 1.3, + "speedup_x": 1.57, + "mAP_delta_pts": 0.0, + "baseline_mAP": 50.8, + "target_mAP": 50.8 + }, + { + "model": "rtdetrv4-s", + "displayName": "RT-DETRv4-S", + "family": "rtdetrv4", + "baseline_fps": 1.1, + "target_fps": 1.8, + "speedup_x": 1.56, + "mAP_delta_pts": 0.0, + "baseline_mAP": 52.8, + "target_mAP": 52.8 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "family": "rtdetr", + "baseline_fps": 0.6, + "target_fps": 0.9, + "speedup_x": 1.48, + "mAP_delta_pts": 0.0, + "baseline_mAP": 52.2, + "target_mAP": 52.2 + }, + { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "baseline_fps": 0.6, + "target_fps": 0.9, + "speedup_x": 1.48, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.2, + "target_mAP": 53.2 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "baseline_fps": 10.5, + "target_fps": 15.0, + "speedup_x": 1.42, + "mAP_delta_pts": -1.4, + "baseline_mAP": 29.5, + "target_mAP": 28.1 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "family": "yolox", + "baseline_fps": 0.4, + "target_fps": 0.5, + "speedup_x": 1.39, + "mAP_delta_pts": 0.4, + "baseline_mAP": 53.9, + "target_mAP": 54.3 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "baseline_fps": 0.8, + "target_fps": 1.1, + "speedup_x": 1.38, + "mAP_delta_pts": -0.1, + "baseline_mAP": 50.9, + "target_mAP": 50.8 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 5.0, + "target_fps": 6.2, + "speedup_x": 1.26, + "mAP_delta_pts": -1.1, + "baseline_mAP": 37.6, + "target_mAP": 36.5 + }, + { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "family": "rfdetr", + "baseline_fps": 2.1, + "target_fps": 2.4, + "speedup_x": 1.17, + "mAP_delta_pts": 0.0, + "baseline_mAP": 51.4, + "target_mAP": 51.4 + }, + { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "family": "rfdetr", + "baseline_fps": 1.0, + "target_fps": 1.2, + "speedup_x": 1.13, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.1, + "target_mAP": 55.1 + }, + { + "model": "rfdetr-m", + "displayName": "RF-DETR-M", + "family": "rfdetr", + "baseline_fps": 0.8, + "target_fps": 0.9, + "speedup_x": 1.13, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.4, + "target_mAP": 57.4 + }, + { + "model": "deimv2-m", + "displayName": "DEIMv2-M", + "family": "deimv2", + "baseline_fps": 0.5, + "target_fps": 0.6, + "speedup_x": 1.13, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.0, + "target_mAP": 56.0 + }, + { + "model": "rfdetr-l", + "displayName": "RF-DETR-L", + "family": "rfdetr", + "baseline_fps": 0.5, + "target_fps": 0.5, + "speedup_x": 1.11, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.6, + "target_mAP": 58.6 + }, + { + "model": "ec-m", + "displayName": "ECDet-M", + "family": "ec", + "baseline_fps": 0.5, + "target_fps": 0.6, + "speedup_x": 1.09, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.4, + "target_mAP": 58.4 + }, + { + "model": "ec-x", + "displayName": "ECDet-X", + "family": "ec", + "baseline_fps": 0.3, + "target_fps": 0.3, + "speedup_x": 1.07, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.1, + "target_mAP": 61.1 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "baseline_fps": 0.3, + "target_fps": 0.3, + "speedup_x": 1.07, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.3, + "target_mAP": 61.3 + }, + { + "model": "ec-l", + "displayName": "ECDet-L", + "family": "ec", + "baseline_fps": 0.3, + "target_fps": 0.4, + "speedup_x": 1.06, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.1, + "target_mAP": 60.1 + }, + { + "model": "ec-s", + "displayName": "ECDet-S", + "family": "ec", + "baseline_fps": 0.8, + "target_fps": 0.9, + "speedup_x": 1.05, + "mAP_delta_pts": 0.0, + "baseline_mAP": 54.3, + "target_mAP": 54.3 + }, + { + "model": "deimv2-l", + "displayName": "DEIMv2-L", + "family": "deimv2", + "baseline_fps": 0.4, + "target_fps": 0.4, + "speedup_x": 1.03, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.6, + "target_mAP": 58.6 + }, + { + "model": "deimv2-s", + "displayName": "DEIMv2-S", + "family": "deimv2", + "baseline_fps": 0.8, + "target_fps": 0.9, + "speedup_x": 1.02, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.0, + "target_mAP": 53.0 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 1.9, + "target_fps": 1.7, + "speedup_x": 0.92, + "mAP_delta_pts": -1.5, + "baseline_mAP": 44.2, + "target_mAP": 42.7 + } + ], + "speedup": { + "median_x": 1.57, + "min_x": 0.92, + "max_x": 2.49 + }, + "family_median_speedup": { + "yolox": 1.59, + "deimv2": 1.87, + "rtdetrv2": 1.87, + "rtdetr": 1.59, + "yolov9": 1.68, + "deim": 1.61, + "dfine": 1.65, + "rtdetrv4": 1.57, + "picodet": 1.26, + "rfdetr": 1.13, + "ec": 1.07 + }, + "accuracy_losers": [ + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "baseline_fps": 8.6, + "target_fps": 21.3, + "speedup_x": 2.49, + "mAP_delta_pts": -1.1, + "baseline_mAP": 28.8, + "target_mAP": 27.7 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "baseline_fps": 2.9, + "target_fps": 5.9, + "speedup_x": 2.03, + "mAP_delta_pts": -0.7, + "baseline_mAP": 41.4, + "target_mAP": 40.7 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "baseline_fps": 5.5, + "target_fps": 9.4, + "speedup_x": 1.72, + "mAP_delta_pts": -0.8, + "baseline_mAP": 35.1, + "target_mAP": 34.3 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "baseline_fps": 1.4, + "target_fps": 2.4, + "speedup_x": 1.68, + "mAP_delta_pts": -0.9, + "baseline_mAP": 49.4, + "target_mAP": 48.5 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "baseline_fps": 0.6, + "target_fps": 1.0, + "speedup_x": 1.57, + "mAP_delta_pts": -1.7, + "baseline_mAP": 55.3, + "target_mAP": 53.6 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "baseline_fps": 0.5, + "target_fps": 0.7, + "speedup_x": 1.57, + "mAP_delta_pts": -0.7, + "baseline_mAP": 56.4, + "target_mAP": 55.7 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "baseline_fps": 10.5, + "target_fps": 15.0, + "speedup_x": 1.42, + "mAP_delta_pts": -1.4, + "baseline_mAP": 29.5, + "target_mAP": 28.1 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 5.0, + "target_fps": 6.2, + "speedup_x": 1.26, + "mAP_delta_pts": -1.1, + "baseline_mAP": 37.6, + "target_mAP": 36.5 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 1.9, + "target_fps": 1.7, + "speedup_x": 0.92, + "mAP_delta_pts": -1.5, + "baseline_mAP": 44.2, + "target_mAP": 42.7 + } + ], + "speed_regressions": [ + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 1.9, + "target_fps": 1.7, + "speedup_x": 0.92, + "mAP_delta_pts": -1.5, + "baseline_mAP": 44.2, + "target_mAP": 42.7 + } + ] + }, + "claims": [ + { + "id": "median_speedup", + "kind": "speed", + "statement": "Across 48 models on Raspberry Pi 5, ONNX Runtime FP32 gives a median 1.57x over PyTorch FP32 (range 0.92x to 2.49x).", + "values": { + "median_x": 1.57, + "min_x": 0.92, + "max_x": 2.49 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "top_gainer", + "kind": "speed", + "statement": "Biggest gain: YOLOX-Nano at 2.49x (8.6 to 21.3 FPS).", + "values": { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "baseline_fps": 8.6, + "target_fps": 21.3, + "speedup_x": 2.49, + "mAP_delta_pts": -1.1, + "baseline_mAP": 28.8, + "target_mAP": 27.7 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "family_spread", + "kind": "speed", + "statement": "Family medians differ: deimv2 gains 1.87x, ec only 1.07x. Conversion gain is architecture-dependent.", + "values": { + "best": [ + "deimv2", + 1.87 + ], + "worst": [ + "ec", + 1.07 + ] + }, + "passed": true, + "strength": "normal" + }, + { + "id": "accuracy_cost", + "kind": "accuracy", + "statement": "9 of 48 models lose 0.5+ mAP points under ONNX Runtime FP32: YOLOX-Nano (-1.1), YOLOv9-T (-0.7), YOLOX-Tiny (-0.8), YOLOv9-S (-0.9), YOLOv9-M (-1.7)", + "values": { + "losers": [ + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "baseline_fps": 8.6, + "target_fps": 21.3, + "speedup_x": 2.49, + "mAP_delta_pts": -1.1, + "baseline_mAP": 28.8, + "target_mAP": 27.7 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "baseline_fps": 2.9, + "target_fps": 5.9, + "speedup_x": 2.03, + "mAP_delta_pts": -0.7, + "baseline_mAP": 41.4, + "target_mAP": 40.7 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "baseline_fps": 5.5, + "target_fps": 9.4, + "speedup_x": 1.72, + "mAP_delta_pts": -0.8, + "baseline_mAP": 35.1, + "target_mAP": 34.3 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "baseline_fps": 1.4, + "target_fps": 2.4, + "speedup_x": 1.68, + "mAP_delta_pts": -0.9, + "baseline_mAP": 49.4, + "target_mAP": 48.5 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "baseline_fps": 0.6, + "target_fps": 1.0, + "speedup_x": 1.57, + "mAP_delta_pts": -1.7, + "baseline_mAP": 55.3, + "target_mAP": 53.6 + } + ] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_regression", + "kind": "anomaly", + "statement": "1 models get SLOWER under ONNX Runtime FP32: PicoDet-L (0.92x)", + "values": { + "regressions": [ + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 1.9, + "target_fps": 1.7, + "speedup_x": 0.92, + "mAP_delta_pts": -1.5, + "baseline_mAP": 44.2, + "target_mAP": 42.7 + } + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 3, + "reason": null + }, + "allowed_numbers": [ + -1.7, + -1.5, + -1.4, + -1.1, + -0.9, + -0.8, + -0.7, + -0.1, + 0.0, + 0.001, + 0.1, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 0.92, + 1.0, + 1.02, + 1.03, + 1.05, + 1.06, + 1.07, + 1.09, + 1.1, + 1.11, + 1.13, + 1.17, + 1.2, + 1.26, + 1.3, + 1.38, + 1.39, + 1.4, + 1.42, + 1.48, + 1.56, + 1.57, + 1.58, + 1.59, + 1.6, + 1.61, + 1.63, + 1.65, + 1.68, + 1.7, + 1.72, + 1.8, + 1.87, + 1.88, + 1.9, + 2, + 2.03, + 2.08, + 2.1, + 2.16, + 2.26, + 2.3, + 2.39, + 2.4, + 2.49, + 2.5, + 2.9, + 3, + 4, + 4.3, + 4.4, + 5.0, + 5.5, + 5.9, + 6, + 6.2, + 7, + 7.4, + 8, + 8.6, + 9, + 9.4, + 10.0, + 10.5, + 11, + 12, + 12.7, + 15.0, + 16.7, + 21.3, + 27.5, + 27.7, + 28.1, + 28.8, + 29.5, + 30.0, + 30.3, + 34.3, + 34.5, + 35.1, + 36.5, + 37.6, + 40.0, + 40.7, + 41.4, + 42.2, + 42.7, + 43.0, + 43.1, + 44.2, + 45.8, + 46.7, + 46.8, + 48.0, + 48.5, + 49.4, + 49.7, + 49.8, + 50.0, + 50.8, + 50.9, + 51.4, + 52.1, + 52.2, + 52.8, + 53.0, + 53.2, + 53.4, + 53.6, + 53.8, + 53.9, + 54.3, + 54.8, + 55.1, + 55.3, + 55.4, + 55.7, + 55.8, + 55.9, + 56.0, + 56.4, + 56.5, + 57.4, + 57.8, + 58.4, + 58.6, + 60.0, + 60.1, + 61.1, + 61.3, + 70.0, + 75, + 80.0, + 90.0, + 92.0, + 95, + 100.0, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/permissive-license-detection-models-a100.claims.json b/article-pipeline/v2/output/permissive-license-detection-models-a100.claims.json new file mode 100644 index 0000000..4e383bf --- /dev/null +++ b/article-pipeline/v2/output/permissive-license-detection-models-a100.claims.json @@ -0,0 +1,264 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:29.019496+00:00", + "slug": "permissive-license-detection-models-a100", + "meta": { + "type": "license-guide", + "hardware": "a100", + "runtime": "pytorch_fp32" + }, + "facts": { + "hardware": "a100", + "hardware_display": "NVIDIA A100", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32", + "n_permissive": 10, + "n_nonpermissive": 3, + "matchups": [ + { + "model": "yolonas-l", + "displayName": "YOLO-NAS-L", + "license": "non-permissive", + "mAP": 51.2, + "fps": 16.6, + "alternative": { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "license": "Apache-2.0", + "mAP": 50.8, + "fps": 25.1, + "mAP_gap_pts": -0.4 + } + }, + { + "model": "yolonas-m", + "displayName": "YOLO-NAS-M", + "license": "non-permissive", + "mAP": 50.5, + "fps": 17.0, + "alternative": { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "license": "Apache-2.0", + "mAP": 50.8, + "fps": 25.1, + "mAP_gap_pts": 0.3 + } + }, + { + "model": "yolonas-s", + "displayName": "YOLO-NAS-S", + "license": "non-permissive", + "mAP": 46.5, + "fps": 16.6, + "alternative": { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "license": "Apache-2.0", + "mAP": 48.2, + "fps": 27.8, + "mAP_gap_pts": 1.7 + } + } + ], + "permissive_leaderboard": [ + { + "model": "dfine-x", + "displayName": "D-FINE-X", + "license": "Apache-2.0", + "mAP": 59.3, + "fps": 15.4 + }, + { + "model": "dfine-l", + "displayName": "D-FINE-L", + "license": "Apache-2.0", + "mAP": 57.2, + "fps": 17.1 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "license": "Apache-2.0", + "mAP": 55.1, + "fps": 21.7 + }, + { + "model": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "license": "Apache-2.0", + "mAP": 53.9, + "fps": 18.6 + }, + { + "model": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "license": "Apache-2.0", + "mAP": 52.7, + "fps": 22.7 + }, + { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "license": "Apache-2.0", + "mAP": 50.8, + "fps": 25.1 + }, + { + "model": "dfine-s", + "displayName": "D-FINE-S", + "license": "Apache-2.0", + "mAP": 50.7, + "fps": 24.0 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "license": "Apache-2.0", + "mAP": 48.2, + "fps": 27.8 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "license": "Apache-2.0", + "mAP": 45.6, + "fps": 27.4 + }, + { + "model": "dfine-n", + "displayName": "D-FINE-N", + "license": "Apache-2.0", + "mAP": 42.8, + "fps": 25.6 + } + ] + }, + "claims": [ + { + "id": "alt_yolonas-l", + "kind": "license", + "statement": "RT-DETR-R50m (Apache-2.0) matches YOLO-NAS-L (non-permissive) within 0.4 mAP points on NVIDIA A100: 50.8 vs 51.2 mAP, 25.1 vs 16.6 FPS.", + "values": { + "model": "yolonas-l", + "displayName": "YOLO-NAS-L", + "license": "non-permissive", + "mAP": 51.2, + "fps": 16.6, + "alternative": { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "license": "Apache-2.0", + "mAP": 50.8, + "fps": 25.1, + "mAP_gap_pts": -0.4 + } + }, + "passed": true, + "strength": "strong" + }, + { + "id": "alt_yolonas-m", + "kind": "license", + "statement": "RT-DETR-R50m (Apache-2.0) matches YOLO-NAS-M (non-permissive) within 0.3 mAP points on NVIDIA A100: 50.8 vs 50.5 mAP, 25.1 vs 17.0 FPS.", + "values": { + "model": "yolonas-m", + "displayName": "YOLO-NAS-M", + "license": "non-permissive", + "mAP": 50.5, + "fps": 17.0, + "alternative": { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "license": "Apache-2.0", + "mAP": 50.8, + "fps": 25.1, + "mAP_gap_pts": 0.3 + } + }, + "passed": true, + "strength": "strong" + }, + { + "id": "alt_yolonas-s", + "kind": "license", + "statement": "RT-DETR-R34 (Apache-2.0) matches YOLO-NAS-S (non-permissive) within 1.7 mAP points on NVIDIA A100: 48.2 vs 46.5 mAP, 27.8 vs 16.6 FPS.", + "values": { + "model": "yolonas-s", + "displayName": "YOLO-NAS-S", + "license": "non-permissive", + "mAP": 46.5, + "fps": 16.6, + "alternative": { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "license": "Apache-2.0", + "mAP": 48.2, + "fps": 27.8, + "mAP_gap_pts": 1.7 + } + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 3, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -0.4, + 0.001, + 0.3, + 0.6, + 1, + 1.7, + 2, + 3.0, + 4, + 5, + 6, + 7, + 8, + 9, + 10.0, + 11, + 12, + 15.4, + 16.6, + 17.0, + 17.1, + 18.6, + 21.7, + 22.7, + 24.0, + 25.1, + 25.6, + 27.4, + 27.8, + 30.0, + 42.8, + 45.6, + 46.5, + 48.2, + 50, + 50.5, + 50.7, + 50.8, + 51.2, + 52.7, + 53.9, + 55.1, + 57.2, + 59.3, + 75, + 95, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/permissive-license-detection-models-jetson-orin.claims.json b/article-pipeline/v2/output/permissive-license-detection-models-jetson-orin.claims.json new file mode 100644 index 0000000..3522c4e --- /dev/null +++ b/article-pipeline/v2/output/permissive-license-detection-models-jetson-orin.claims.json @@ -0,0 +1,637 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T21:56:13.010715+00:00", + "slug": "permissive-license-detection-models-jetson-orin", + "meta": { + "type": "license-guide", + "hardware": "jetson_orin", + "runtime": "pytorch_fp32" + }, + "facts": { + "hardware": "jetson_orin", + "hardware_display": "NVIDIA Jetson Orin Nano Super 8GB", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32", + "n_permissive": 55, + "n_nonpermissive": 3, + "matchups": [ + { + "model": "yolonas-l", + "displayName": "YOLO-NAS-L", + "license": "non-permissive", + "mAP": 56.3, + "fps": 5.0, + "alternative": { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "license": "MIT", + "mAP": 56.1, + "fps": 8.0, + "mAP_gap_pts": -0.2 + } + }, + { + "model": "yolonas-m", + "displayName": "YOLO-NAS-M", + "license": "non-permissive", + "mAP": 55.5, + "fps": 6.6, + "alternative": { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "license": "MIT", + "mAP": 56.1, + "fps": 8.0, + "mAP_gap_pts": 0.6 + } + }, + { + "model": "yolonas-s", + "displayName": "YOLO-NAS-S", + "license": "non-permissive", + "mAP": 51.8, + "fps": 10.2, + "alternative": { + "model": "dfine-s", + "displayName": "D-FINE-S", + "license": "Apache-2.0", + "mAP": 53.4, + "fps": 10.3, + "mAP_gap_pts": 1.6 + } + } + ], + "permissive_leaderboard": [ + { + "model": "dfine-x", + "displayName": "D-FINE-X", + "license": "Apache-2.0", + "mAP": 61.4, + "fps": 2.9 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "license": "Apache-2.0", + "mAP": 61.3, + "fps": 2.7 + }, + { + "model": "ec-x", + "displayName": "ECDet-X", + "license": "Apache-2.0", + "mAP": 61.1, + "fps": 2.9 + }, + { + "model": "ec-l", + "displayName": "ECDet-L", + "license": "Apache-2.0", + "mAP": 60.1, + "fps": 3.4 + }, + { + "model": "dfine-l", + "displayName": "D-FINE-L", + "license": "Apache-2.0", + "mAP": 60.0, + "fps": 4.7 + }, + { + "model": "rtdetrv4-x", + "displayName": "RT-DETRv4-X", + "license": "Apache-2.0", + "mAP": 60.0, + "fps": 2.8 + }, + { + "model": "deim-x", + "displayName": "DEIM-X", + "license": "Apache-2.0", + "mAP": 59.6, + "fps": 2.9 + }, + { + "model": "deimv2-l", + "displayName": "DEIMv2-L", + "license": "Apache-2.0", + "mAP": 58.6, + "fps": 3.3 + }, + { + "model": "rfdetr-l", + "displayName": "RF-DETR-L", + "license": "Apache-2.0", + "mAP": 58.6, + "fps": 4.0 + }, + { + "model": "ec-m", + "displayName": "ECDet-M", + "license": "Apache-2.0", + "mAP": 58.4, + "fps": 5.0 + }, + { + "model": "rtdetr-x", + "displayName": "RT-DETR-X", + "license": "Apache-2.0", + "mAP": 57.9, + "fps": 3.0 + }, + { + "model": "deim-l", + "displayName": "DEIM-L", + "license": "Apache-2.0", + "mAP": 57.8, + "fps": 4.8 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "license": "Apache-2.0", + "mAP": 57.8, + "fps": 7.0 + }, + { + "model": "rtdetrv4-l", + "displayName": "RT-DETRv4-L", + "license": "Apache-2.0", + "mAP": 57.8, + "fps": 4.7 + }, + { + "model": "rfdetr-m", + "displayName": "RF-DETR-M", + "license": "Apache-2.0", + "mAP": 57.4, + "fps": 5.7 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "license": "MIT", + "mAP": 57.1, + "fps": 6.4 + }, + { + "model": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "license": "Apache-2.0", + "mAP": 56.8, + "fps": 2.8 + }, + { + "model": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "license": "Apache-2.0", + "mAP": 56.8, + "fps": 2.8 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "license": "Apache-2.0", + "mAP": 56.5, + "fps": 7.0 + }, + { + "model": "yolox-x", + "displayName": "YOLOX-X", + "license": "Apache-2.0", + "mAP": 56.3, + "fps": 3.5 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "license": "MIT", + "mAP": 56.1, + "fps": 8.0 + }, + { + "model": "deimv2-m", + "displayName": "DEIMv2-M", + "license": "Apache-2.0", + "mAP": 56.0, + "fps": 4.7 + }, + { + "model": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "license": "Apache-2.0", + "mAP": 55.9, + "fps": 4.3 + }, + { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "license": "Apache-2.0", + "mAP": 55.8, + "fps": 4.9 + }, + { + "model": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "license": "Apache-2.0", + "mAP": 55.7, + "fps": 4.3 + }, + { + "model": "deim-m", + "displayName": "DEIM-M", + "license": "Apache-2.0", + "mAP": 55.4, + "fps": 7.0 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "license": "Apache-2.0", + "mAP": 55.4, + "fps": 5.8 + }, + { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "license": "Apache-2.0", + "mAP": 55.1, + "fps": 6.9 + }, + { + "model": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "license": "Apache-2.0", + "mAP": 54.8, + "fps": 5.1 + }, + { + "model": "ec-s", + "displayName": "ECDet-S", + "license": "Apache-2.0", + "mAP": 54.3, + "fps": 6.0 + }, + { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "license": "Apache-2.0", + "mAP": 53.8, + "fps": 5.0 + }, + { + "model": "dfine-s", + "displayName": "D-FINE-S", + "license": "Apache-2.0", + "mAP": 53.4, + "fps": 10.3 + }, + { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "license": "Apache-2.0", + "mAP": 53.2, + "fps": 8.1 + }, + { + "model": "deimv2-s", + "displayName": "DEIMv2-S", + "license": "Apache-2.0", + "mAP": 53.0, + "fps": 5.7 + }, + { + "model": "rtdetrv4-s", + "displayName": "RT-DETRv4-S", + "license": "Apache-2.0", + "mAP": 52.8, + "fps": 10.1 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "license": "Apache-2.0", + "mAP": 52.2, + "fps": 8.0 + }, + { + "model": "deim-s", + "displayName": "DEIM-S", + "license": "Apache-2.0", + "mAP": 52.1, + "fps": 10.2 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "license": "Apache-2.0", + "mAP": 51.7, + "fps": 9.1 + }, + { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "license": "Apache-2.0", + "mAP": 51.4, + "fps": 10.2 + }, + { + "model": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "license": "Apache-2.0", + "mAP": 50.8, + "fps": 10.3 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "license": "MIT", + "mAP": 50.4, + "fps": 9.9 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "license": "Apache-2.0", + "mAP": 49.8, + "fps": 10.2 + }, + { + "model": "deim-n", + "displayName": "DEIM-N", + "license": "Apache-2.0", + "mAP": 46.8, + "fps": 11.3 + }, + { + "model": "deimv2-n", + "displayName": "DEIMv2-N", + "license": "Apache-2.0", + "mAP": 46.7, + "fps": 11.1 + }, + { + "model": "dfine-n", + "displayName": "D-FINE-N", + "license": "Apache-2.0", + "mAP": 45.8, + "fps": 11.7 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "license": "Apache-2.0", + "mAP": 44.3, + "fps": 16.9 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "license": "Apache-2.0", + "mAP": 44.1, + "fps": 11.1 + }, + { + "model": "deimv2-pico", + "displayName": "DEIMv2-Pico", + "license": "Apache-2.0", + "mAP": 42.2, + "fps": 14.1 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "license": "MIT", + "mAP": 41.8, + "fps": 9.9 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "license": "Apache-2.0", + "mAP": 37.9, + "fps": 12.5 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "license": "Apache-2.0", + "mAP": 35.5, + "fps": 20.3 + }, + { + "model": "deimv2-femto", + "displayName": "DEIMv2-Femto", + "license": "Apache-2.0", + "mAP": 34.5, + "fps": 14.8 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "license": "Apache-2.0", + "mAP": 30.4, + "fps": 13.8 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "license": "Apache-2.0", + "mAP": 28.8, + "fps": 18.4 + }, + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "license": "Apache-2.0", + "mAP": 27.5, + "fps": 15.4 + } + ] + }, + "claims": [ + { + "id": "alt_yolonas-l", + "kind": "license", + "statement": "YOLOv9-M (MIT) matches YOLO-NAS-L (non-permissive) within 0.2 mAP points on NVIDIA Jetson Orin Nano Super 8GB: 56.1 vs 56.3 mAP, 8.0 vs 5.0 FPS.", + "values": { + "model": "yolonas-l", + "displayName": "YOLO-NAS-L", + "license": "non-permissive", + "mAP": 56.3, + "fps": 5.0, + "alternative": { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "license": "MIT", + "mAP": 56.1, + "fps": 8.0, + "mAP_gap_pts": -0.2 + } + }, + "passed": true, + "strength": "strong" + }, + { + "id": "alt_yolonas-m", + "kind": "license", + "statement": "YOLOv9-M (MIT) matches YOLO-NAS-M (non-permissive) within 0.6 mAP points on NVIDIA Jetson Orin Nano Super 8GB: 56.1 vs 55.5 mAP, 8.0 vs 6.6 FPS.", + "values": { + "model": "yolonas-m", + "displayName": "YOLO-NAS-M", + "license": "non-permissive", + "mAP": 55.5, + "fps": 6.6, + "alternative": { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "license": "MIT", + "mAP": 56.1, + "fps": 8.0, + "mAP_gap_pts": 0.6 + } + }, + "passed": true, + "strength": "strong" + }, + { + "id": "alt_yolonas-s", + "kind": "license", + "statement": "D-FINE-S (Apache-2.0) matches YOLO-NAS-S (non-permissive) within 1.6 mAP points on NVIDIA Jetson Orin Nano Super 8GB: 53.4 vs 51.8 mAP, 10.3 vs 10.2 FPS.", + "values": { + "model": "yolonas-s", + "displayName": "YOLO-NAS-S", + "license": "non-permissive", + "mAP": 51.8, + "fps": 10.2, + "alternative": { + "model": "dfine-s", + "displayName": "D-FINE-S", + "license": "Apache-2.0", + "mAP": 53.4, + "fps": 10.3, + "mAP_gap_pts": 1.6 + } + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 3, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -0.2, + 0.001, + 0.6, + 1, + 1.6, + 2, + 2.7, + 2.8, + 2.9, + 3.0, + 3.3, + 3.4, + 3.5, + 4.0, + 4.3, + 4.7, + 4.8, + 4.9, + 5.0, + 5.1, + 5.7, + 5.8, + 6.0, + 6.4, + 6.6, + 6.9, + 7.0, + 8.0, + 8.1, + 9, + 9.1, + 9.9, + 10, + 10.1, + 10.2, + 10.3, + 11, + 11.1, + 11.3, + 11.7, + 12, + 12.5, + 13.8, + 14.1, + 14.8, + 15.4, + 16.9, + 18.4, + 20.3, + 27.5, + 28.8, + 30.4, + 34.5, + 35.5, + 37.9, + 41.8, + 42.2, + 44.1, + 44.3, + 45.8, + 46.7, + 46.8, + 49.8, + 50, + 50.4, + 50.8, + 51.4, + 51.7, + 51.8, + 52.1, + 52.2, + 52.8, + 53.0, + 53.2, + 53.4, + 53.8, + 54.3, + 54.8, + 55.0, + 55.1, + 55.4, + 55.5, + 55.7, + 55.8, + 55.9, + 56.0, + 56.1, + 56.3, + 56.5, + 56.8, + 57.1, + 57.4, + 57.8, + 57.9, + 58.4, + 58.6, + 59.6, + 60.0, + 60.1, + 61.1, + 61.3, + 61.4, + 75, + 95, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/permissive-license-detection-models-rpi5-hailo8.claims.json b/article-pipeline/v2/output/permissive-license-detection-models-rpi5-hailo8.claims.json new file mode 100644 index 0000000..b7698fc --- /dev/null +++ b/article-pipeline/v2/output/permissive-license-detection-models-rpi5-hailo8.claims.json @@ -0,0 +1,238 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:29.262634+00:00", + "slug": "permissive-license-detection-models-rpi5-hailo8", + "meta": { + "type": "license-guide", + "hardware": "rpi5_hailo8", + "runtime": "hailo_int8" + }, + "facts": { + "hardware": "rpi5_hailo8", + "hardware_display": "Raspberry Pi 5 + Hailo-8", + "runtime": "hailo_int8", + "runtime_display": "HailoRT INT8", + "n_permissive": 7, + "n_nonpermissive": 3, + "matchups": [ + { + "model": "yolonas-l", + "displayName": "YOLO-NAS-L", + "license": "non-permissive", + "mAP": 52.4, + "fps": 15.4, + "alternative": { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "license": "MIT", + "mAP": 53.3, + "fps": 30.1, + "mAP_gap_pts": 0.9 + } + }, + { + "model": "yolonas-m", + "displayName": "YOLO-NAS-M", + "license": "non-permissive", + "mAP": 47.4, + "fps": 21.7, + "alternative": { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "license": "MIT", + "mAP": 53.3, + "fps": 30.1, + "mAP_gap_pts": 5.9 + } + }, + { + "model": "yolonas-s", + "displayName": "YOLO-NAS-S", + "license": "non-permissive", + "mAP": 44.5, + "fps": 39.7, + "alternative": { + "model": "yolox-m", + "displayName": "YOLOX-M", + "license": "Apache-2.0", + "mAP": 43.7, + "fps": 34.8, + "mAP_gap_pts": -0.8 + } + } + ], + "permissive_leaderboard": [ + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "license": "MIT", + "mAP": 54.8, + "fps": 21.2 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "license": "MIT", + "mAP": 53.3, + "fps": 30.1 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "license": "Apache-2.0", + "mAP": 43.7, + "fps": 34.8 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "license": "Apache-2.0", + "mAP": 41.1, + "fps": 95.8 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "license": "Apache-2.0", + "mAP": 33.5, + "fps": 195.6 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "license": "MIT", + "mAP": 32.0, + "fps": 36.3 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "license": "MIT", + "mAP": 25.7, + "fps": 76.5 + } + ] + }, + "claims": [ + { + "id": "alt_yolonas-l", + "kind": "license", + "statement": "YOLOv9-M (MIT) matches YOLO-NAS-L (non-permissive) within 0.9 mAP points on Raspberry Pi 5 + Hailo-8: 53.3 vs 52.4 mAP, 30.1 vs 15.4 FPS.", + "values": { + "model": "yolonas-l", + "displayName": "YOLO-NAS-L", + "license": "non-permissive", + "mAP": 52.4, + "fps": 15.4, + "alternative": { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "license": "MIT", + "mAP": 53.3, + "fps": 30.1, + "mAP_gap_pts": 0.9 + } + }, + "passed": true, + "strength": "strong" + }, + { + "id": "alt_yolonas-m", + "kind": "license", + "statement": "YOLOv9-M (MIT) matches YOLO-NAS-M (non-permissive) within 5.9 mAP points on Raspberry Pi 5 + Hailo-8: 53.3 vs 47.4 mAP, 30.1 vs 21.7 FPS.", + "values": { + "model": "yolonas-m", + "displayName": "YOLO-NAS-M", + "license": "non-permissive", + "mAP": 47.4, + "fps": 21.7, + "alternative": { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "license": "MIT", + "mAP": 53.3, + "fps": 30.1, + "mAP_gap_pts": 5.9 + } + }, + "passed": true, + "strength": "strong" + }, + { + "id": "alt_yolonas-s", + "kind": "license", + "statement": "YOLOX-M (Apache-2.0) matches YOLO-NAS-S (non-permissive) within 0.8 mAP points on Raspberry Pi 5 + Hailo-8: 43.7 vs 44.5 mAP, 34.8 vs 39.7 FPS.", + "values": { + "model": "yolonas-s", + "displayName": "YOLO-NAS-S", + "license": "non-permissive", + "mAP": 44.5, + "fps": 39.7, + "alternative": { + "model": "yolox-m", + "displayName": "YOLOX-M", + "license": "Apache-2.0", + "mAP": 43.7, + "fps": 34.8, + "mAP_gap_pts": -0.8 + } + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 3, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -0.8, + 0.001, + 0.6, + 0.9, + 1, + 2, + 3.0, + 4, + 5, + 5.9, + 6, + 7.0, + 8, + 9, + 10, + 11, + 12, + 15.4, + 21.2, + 21.7, + 25.7, + 30.1, + 32.0, + 33.5, + 34.8, + 36.3, + 39.7, + 41.1, + 43.7, + 44.5, + 47.4, + 50, + 52.4, + 53.3, + 54.8, + 75, + 76.5, + 90.0, + 95, + 95.8, + 100, + 195.6, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/permissive-license-detection-models-rpi5.claims.json b/article-pipeline/v2/output/permissive-license-detection-models-rpi5.claims.json new file mode 100644 index 0000000..7a86fd2 --- /dev/null +++ b/article-pipeline/v2/output/permissive-license-detection-models-rpi5.claims.json @@ -0,0 +1,535 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:29.134889+00:00", + "slug": "permissive-license-detection-models-rpi5", + "meta": { + "type": "license-guide", + "hardware": "rpi5", + "runtime": "pytorch_fp32" + }, + "facts": { + "hardware": "rpi5", + "hardware_display": "Raspberry Pi 5", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32", + "n_permissive": 48, + "n_nonpermissive": 2, + "matchups": [ + { + "model": "yolonas-m", + "displayName": "YOLO-NAS-M", + "license": "non-permissive", + "mAP": 55.4, + "fps": 0.5, + "alternative": { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "license": "Apache-2.0", + "mAP": 55.1, + "fps": 1.0, + "mAP_gap_pts": -0.3 + } + }, + { + "model": "yolonas-s", + "displayName": "YOLO-NAS-S", + "license": "non-permissive", + "mAP": 51.8, + "fps": 0.9, + "alternative": { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "license": "Apache-2.0", + "mAP": 51.4, + "fps": 2.1, + "mAP_gap_pts": -0.4 + } + } + ], + "permissive_leaderboard": [ + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "license": "Apache-2.0", + "mAP": 61.3, + "fps": 0.3 + }, + { + "model": "ec-x", + "displayName": "ECDet-X", + "license": "Apache-2.0", + "mAP": 61.1, + "fps": 0.3 + }, + { + "model": "ec-l", + "displayName": "ECDet-L", + "license": "Apache-2.0", + "mAP": 60.1, + "fps": 0.3 + }, + { + "model": "dfine-l", + "displayName": "D-FINE-L", + "license": "Apache-2.0", + "mAP": 60.0, + "fps": 0.4 + }, + { + "model": "rfdetr-l", + "displayName": "RF-DETR-L", + "license": "Apache-2.0", + "mAP": 58.6, + "fps": 0.5 + }, + { + "model": "deimv2-l", + "displayName": "DEIMv2-L", + "license": "Apache-2.0", + "mAP": 58.6, + "fps": 0.4 + }, + { + "model": "ec-m", + "displayName": "ECDet-M", + "license": "Apache-2.0", + "mAP": 58.4, + "fps": 0.5 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "license": "Apache-2.0", + "mAP": 57.8, + "fps": 0.6 + }, + { + "model": "deim-l", + "displayName": "DEIM-L", + "license": "Apache-2.0", + "mAP": 57.8, + "fps": 0.4 + }, + { + "model": "rtdetrv4-l", + "displayName": "RT-DETRv4-L", + "license": "Apache-2.0", + "mAP": 57.8, + "fps": 0.4 + }, + { + "model": "rfdetr-m", + "displayName": "RF-DETR-M", + "license": "Apache-2.0", + "mAP": 57.4, + "fps": 0.8 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "license": "Apache-2.0", + "mAP": 56.5, + "fps": 0.6 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "license": "MIT", + "mAP": 56.4, + "fps": 0.5 + }, + { + "model": "deimv2-m", + "displayName": "DEIMv2-M", + "license": "Apache-2.0", + "mAP": 56.0, + "fps": 0.5 + }, + { + "model": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "license": "Apache-2.0", + "mAP": 55.9, + "fps": 0.3 + }, + { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "license": "Apache-2.0", + "mAP": 55.8, + "fps": 0.4 + }, + { + "model": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "license": "Apache-2.0", + "mAP": 55.7, + "fps": 0.3 + }, + { + "model": "deim-m", + "displayName": "DEIM-M", + "license": "Apache-2.0", + "mAP": 55.4, + "fps": 0.6 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "license": "MIT", + "mAP": 55.3, + "fps": 0.6 + }, + { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "license": "Apache-2.0", + "mAP": 55.1, + "fps": 1.0 + }, + { + "model": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "license": "Apache-2.0", + "mAP": 54.8, + "fps": 0.4 + }, + { + "model": "ec-s", + "displayName": "ECDet-S", + "license": "Apache-2.0", + "mAP": 54.3, + "fps": 0.8 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "license": "Apache-2.0", + "mAP": 53.9, + "fps": 0.4 + }, + { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "license": "Apache-2.0", + "mAP": 53.8, + "fps": 0.4 + }, + { + "model": "dfine-s", + "displayName": "D-FINE-S", + "license": "Apache-2.0", + "mAP": 53.4, + "fps": 1.1 + }, + { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "license": "Apache-2.0", + "mAP": 53.2, + "fps": 0.6 + }, + { + "model": "deimv2-s", + "displayName": "DEIMv2-S", + "license": "Apache-2.0", + "mAP": 53.0, + "fps": 0.8 + }, + { + "model": "rtdetrv4-s", + "displayName": "RT-DETRv4-S", + "license": "Apache-2.0", + "mAP": 52.8, + "fps": 1.1 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "license": "Apache-2.0", + "mAP": 52.2, + "fps": 0.6 + }, + { + "model": "deim-s", + "displayName": "DEIM-S", + "license": "Apache-2.0", + "mAP": 52.1, + "fps": 1.1 + }, + { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "license": "Apache-2.0", + "mAP": 51.4, + "fps": 2.1 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "license": "Apache-2.0", + "mAP": 50.9, + "fps": 0.8 + }, + { + "model": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "license": "Apache-2.0", + "mAP": 50.8, + "fps": 0.8 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "license": "Apache-2.0", + "mAP": 49.7, + "fps": 0.8 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "license": "MIT", + "mAP": 49.4, + "fps": 1.4 + }, + { + "model": "deim-n", + "displayName": "DEIM-N", + "license": "Apache-2.0", + "mAP": 46.8, + "fps": 2.3 + }, + { + "model": "deimv2-n", + "displayName": "DEIMv2-N", + "license": "Apache-2.0", + "mAP": 46.7, + "fps": 2.4 + }, + { + "model": "dfine-n", + "displayName": "D-FINE-N", + "license": "Apache-2.0", + "mAP": 45.8, + "fps": 2.3 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "license": "Apache-2.0", + "mAP": 44.2, + "fps": 1.9 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "license": "Apache-2.0", + "mAP": 43.0, + "fps": 1.6 + }, + { + "model": "deimv2-pico", + "displayName": "DEIMv2-Pico", + "license": "Apache-2.0", + "mAP": 42.2, + "fps": 2.9 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "license": "MIT", + "mAP": 41.4, + "fps": 2.9 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "license": "Apache-2.0", + "mAP": 37.6, + "fps": 5.0 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "license": "Apache-2.0", + "mAP": 35.1, + "fps": 5.5 + }, + { + "model": "deimv2-femto", + "displayName": "DEIMv2-Femto", + "license": "Apache-2.0", + "mAP": 34.5, + "fps": 7.4 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "license": "Apache-2.0", + "mAP": 29.5, + "fps": 10.5 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "license": "Apache-2.0", + "mAP": 28.8, + "fps": 8.6 + }, + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "license": "Apache-2.0", + "mAP": 27.5, + "fps": 12.7 + } + ] + }, + "claims": [ + { + "id": "alt_yolonas-m", + "kind": "license", + "statement": "RF-DETR-S (Apache-2.0) matches YOLO-NAS-M (non-permissive) within 0.3 mAP points on Raspberry Pi 5: 55.1 vs 55.4 mAP, 1.0 vs 0.5 FPS.", + "values": { + "model": "yolonas-m", + "displayName": "YOLO-NAS-M", + "license": "non-permissive", + "mAP": 55.4, + "fps": 0.5, + "alternative": { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "license": "Apache-2.0", + "mAP": 55.1, + "fps": 1.0, + "mAP_gap_pts": -0.3 + } + }, + "passed": true, + "strength": "strong" + }, + { + "id": "alt_yolonas-s", + "kind": "license", + "statement": "RF-DETR-N (Apache-2.0) matches YOLO-NAS-S (non-permissive) within 0.4 mAP points on Raspberry Pi 5: 51.4 vs 51.8 mAP, 2.1 vs 0.9 FPS.", + "values": { + "model": "yolonas-s", + "displayName": "YOLO-NAS-S", + "license": "non-permissive", + "mAP": 51.8, + "fps": 0.9, + "alternative": { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "license": "Apache-2.0", + "mAP": 51.4, + "fps": 2.1, + "mAP_gap_pts": -0.4 + } + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 2, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -0.4, + -0.3, + 0.001, + 0.3, + 0.4, + 0.5, + 0.6, + 0.8, + 0.9, + 1.0, + 1.1, + 1.4, + 1.6, + 1.9, + 2.0, + 2.1, + 2.3, + 2.4, + 2.9, + 3, + 4, + 5.0, + 5.5, + 6, + 7, + 7.4, + 8, + 8.6, + 9, + 10, + 10.5, + 11, + 12, + 12.7, + 27.5, + 28.8, + 29.5, + 30.0, + 34.5, + 35.1, + 37.6, + 40.0, + 41.4, + 42.2, + 43.0, + 44.2, + 45.8, + 46.7, + 46.8, + 48.0, + 49.4, + 49.7, + 50.0, + 50.8, + 50.9, + 51.4, + 51.8, + 52.1, + 52.2, + 52.8, + 53.0, + 53.2, + 53.4, + 53.8, + 53.9, + 54.3, + 54.8, + 55.1, + 55.3, + 55.4, + 55.7, + 55.8, + 55.9, + 56.0, + 56.4, + 56.5, + 57.4, + 57.8, + 58.4, + 58.6, + 60.0, + 60.1, + 61.1, + 61.3, + 75, + 80.0, + 90.0, + 95, + 100.0, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/picodet-vs-yolox.claims.json b/article-pipeline/v2/output/picodet-vs-yolox.claims.json new file mode 100644 index 0000000..041fe63 --- /dev/null +++ b/article-pipeline/v2/output/picodet-vs-yolox.claims.json @@ -0,0 +1,346 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:20.153528+00:00", + "slug": "picodet-vs-yolox", + "meta": { + "type": "family-vs", + "a": "picodet", + "b": "yolox", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "picodet", + "b": "yolox", + "a_display": "PicoDet", + "b_display": "YOLOX" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "picodet-s", + "displayName": "PicoDet-S", + "params": 0.99, + "mAP": 30.4, + "fps": 50.0, + "license": "Apache-2.0" + }, + { + "id": "picodet-m", + "displayName": "PicoDet-M", + "params": 2.15, + "mAP": 37.9, + "fps": 45.1, + "license": "Apache-2.0" + }, + { + "id": "picodet-l", + "displayName": "PicoDet-L", + "params": 3.31, + "mAP": 44.1, + "fps": 39.5, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + }, + { + "id": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "params": 5.06, + "mAP": 35.5, + "fps": 61.6, + "license": "Apache-2.0" + }, + { + "id": "yolox-s", + "displayName": "YOLOX-S", + "params": 8.97, + "mAP": 44.3, + "fps": 50.0, + "license": "Apache-2.0" + }, + { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + { + "id": "yolox-l", + "displayName": "YOLOX-L", + "params": 54.21, + "mAP": 55.4, + "fps": 42.8, + "license": "Apache-2.0" + }, + { + "id": "yolox-x", + "displayName": "YOLOX-X", + "params": 99.07, + "mAP": 56.3, + "fps": 39.6, + "license": "Apache-2.0" + } + ], + "matched_pairs": [ + { + "a": { + "id": "picodet-s", + "displayName": "PicoDet-S", + "params": 0.99, + "mAP": 30.4, + "fps": 50.0, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 1.6, + "fps_delta_pct": -6.72 + }, + { + "a": { + "id": "picodet-l", + "displayName": "PicoDet-L", + "params": 3.31, + "mAP": 44.1, + "fps": 39.5, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "params": 5.06, + "mAP": 35.5, + "fps": 61.6, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 8.6, + "fps_delta_pct": -35.88 + } + ], + "aggregate": { + "n_pairs": 2, + "mean_mAP_delta_pts": 5.1, + "a_wins": 2, + "b_wins": 0, + "mean_fps_delta_pct": -21.3 + }, + "flagship": { + "a": { + "id": "picodet-l", + "displayName": "PicoDet-L", + "params": 3.31, + "mAP": 44.1, + "fps": 39.5, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-x", + "displayName": "YOLOX-X", + "params": 99.07, + "mAP": 56.3, + "fps": 39.6, + "license": "Apache-2.0" + } + }, + "smallest": { + "a": { + "id": "picodet-s", + "displayName": "PicoDet-S", + "params": 0.99, + "mAP": 30.4, + "fps": 50.0, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + } + }, + "crossover": { + "low_param_leader": "picodet", + "high_param_leader": "picodet" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "picodet-s,picodet-m,picodet-l,yolox-nano,yolox-tiny,yolox-s,yolox-m,yolox-l,yolox-x" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, PicoDet averages 5.1 mAP points higher across 2 paired variants on NVIDIA RTX 5070 Ti. PicoDet wins 2, YOLOX wins 0.", + "values": { + "mean_delta_pts": 5.1, + "a_wins": 2, + "b_wins": 0 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, YOLOX is 21.3% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 21.3 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, PicoDet's largest measured variant (PicoDet-L) reaches 44.1 mAP; YOLOX's (YOLOX-X) reaches 56.3 mAP.", + "values": { + "a": { + "id": "picodet-l", + "displayName": "PicoDet-L", + "params": 3.31, + "mAP": 44.1, + "fps": 39.5, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-x", + "displayName": "YOLOX-X", + "params": 99.07, + "mAP": 56.3, + "fps": 39.6, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, PicoDet's PicoDet-S (0.99M, 30.4 mAP, 50.0 FPS) faces YOLOX's YOLOX-Nano (0.91M, 28.8 mAP, 53.6 FPS).", + "values": { + "a": { + "id": "picodet-s", + "displayName": "PicoDet-S", + "params": 0.99, + "mAP": 30.4, + "fps": 50.0, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "PicoDet fields 3 measured variants, YOLOX fields 6, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 3, + "b_n": 6 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -35.88, + -21.3, + -6.72, + 0.0, + 0.001, + 0.6, + 0.91, + 0.99, + 1, + 1.6, + 2.0, + 2.15, + 3.0, + 3.31, + 4, + 5, + 5.06, + 5.1, + 6.0, + 7, + 8, + 8.6, + 8.97, + 9, + 10, + 11, + 12, + 21.3, + 25.33, + 28.8, + 30.4, + 35.5, + 37.9, + 39.5, + 39.6, + 42.8, + 44.1, + 44.3, + 45.1, + 48.2, + 50.0, + 51.7, + 53.6, + 54.21, + 55.4, + 56.3, + 61.6, + 75, + 91.0, + 95, + 99.0, + 99.07, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rfdetr-n-vs-rtdetr-r18.claims.json b/article-pipeline/v2/output/rfdetr-n-vs-rtdetr-r18.claims.json new file mode 100644 index 0000000..8046194 --- /dev/null +++ b/article-pipeline/v2/output/rfdetr-n-vs-rtdetr-r18.claims.json @@ -0,0 +1,385 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:42.215300+00:00", + "slug": "rfdetr-n-vs-rtdetr-r18", + "meta": { + "type": "vs", + "a": "rfdetr-n", + "b": "rtdetr-r18", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "license": "Apache-2.0", + "releaseDate": "2026-02-23", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5135247777250423, + "mAP_50": 0.6989693323963424, + "mAP_small": 0.2768082791741451, + "mAP_medium": 0.5713133502545867, + "mAP_large": 0.7180799558822147, + "fps_mean": 39.53, + "fps_p50": 41.33, + "ms_per_image": 25.295, + "inference_ms": 16.983, + "preprocess_ms": 5.999, + "postprocess_ms": 2.313, + "p95_ms": 29.239, + "peak_vram_mb": 151.0, + "peak_ram_mb": 136.7, + "params_millions": 30.47, + "gflops": 0.0, + "input_size": 384, + "config": { + "batch_size": 1, + "input_size": 384, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-17T17:13:52.106298Z" + } + }, + "b": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.49787927521150827, + "mAP_50": 0.663704009633566, + "mAP_small": 0.32710231823159713, + "mAP_medium": 0.5484637312981646, + "mAP_large": 0.6407169138607522, + "fps_mean": 47.22, + "fps_p50": 48.89, + "ms_per_image": 21.179, + "inference_ms": 16.645, + "preprocess_ms": 3.452, + "postprocess_ms": 1.082, + "p95_ms": 25.931, + "peak_vram_mb": 168.9, + "peak_ram_mb": 2.0, + "params_millions": 20.18, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:23:32.849123Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 3.14, + "fps": -16.29, + "params": 50.99, + "flops": -100.0, + "peak_vram": -10.6 + }, + "map_pts": { + "a": 51.4, + "b": 49.8, + "delta": 1.6 + }, + "pair": { + "params_ratio": 1.51, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": null, + "b_mAP_per_GFLOP": 0.0083 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 51.4, + "fps": 65.7 + }, + "b": { + "mAP": 49.8, + "fps": 92.5 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 51.4, + "fps": 39.5 + }, + "b": { + "mAP": 49.8, + "fps": 47.2 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 51.4, + "fps": 58.3 + }, + "b": { + "mAP": 49.8, + "fps": 113.4 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 51.4, + "fps": 67.6 + }, + "b": { + "mAP": 49.8, + "fps": 120.6 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 51.4, + "fps": 10.2 + }, + "b": { + "mAP": 49.8, + "fps": 10.2 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 51.4, + "fps": 39.5 + }, + "b": { + "mAP": 49.8, + "fps": 47.2 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 51.4, + "fps": 2.1 + }, + "b": { + "mAP": 49.7, + "fps": 0.8 + } + } + ], + "input_sizes": { + "a": 384, + "b": 640, + "match": false + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "RF-DETR-N measures 1.6 mAP points higher than RT-DETR-R18 on NVIDIA RTX 5070 Ti (PyTorch FP32): 51.4 vs 49.8 mAP@50-95.", + "values": { + "delta_pts": 1.6, + "a_map": 51.4, + "b_map": 49.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R18 is 16.29% faster: 39.5 vs 47.2 FPS (25.3 vs 21.18 ms/image).", + "values": { + "delta_pct": 16.29, + "a_fps": 39.5, + "b_fps": 47.2 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "RF-DETR-N uses less peak VRAM: 151 vs 169 MB.", + "values": { + "a_vram": 151, + "b_vram": 169 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "RT-DETR-R18 leads on small objects: 27.7 vs 32.7 mAP_small (15.38% relative gap).", + "values": { + "a": 27.7, + "b": 32.7, + "delta_pct": 15.38 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "input_size_caveat", + "kind": "caveat", + "statement": "MANDATORY CAVEAT: RF-DETR-N runs at 384 px and RT-DETR-R18 at 640 px (each model at its author default). Say this in Setup and the verdict.", + "values": { + "a": 384, + "b": 640 + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -100.0, + -16.29, + -10.6, + 0.0, + 0.001, + 0.0083, + 0.1, + 0.2768, + 0.3271, + 0.4979, + 0.5135, + 0.5485, + 0.5713, + 0.6, + 0.6407, + 0.6637, + 0.699, + 0.7181, + 0.8, + 0.83, + 1.0, + 1.082, + 1.51, + 1.6, + 2.0, + 2.1, + 2.313, + 3, + 3.14, + 3.452, + 4, + 5, + 5.999, + 6, + 7, + 8, + 9, + 10, + 10.2, + 11, + 12, + 15.38, + 16.29, + 16.645, + 16.983, + 20.18, + 21.179, + 25.295, + 25.931, + 27.6808, + 27.7, + 29.239, + 30.47, + 32.7, + 32.7102, + 39.5, + 39.53, + 41.33, + 47.2, + 47.22, + 48.89, + 49.7, + 49.7879, + 49.8, + 50, + 50.99, + 51.3525, + 51.4, + 54.8464, + 57.1313, + 58.3, + 60.0, + 64.0717, + 65.7, + 66.3704, + 67.6, + 69.8969, + 71.808, + 75, + 80.0, + 92.5, + 95, + 100.0, + 113.4, + 120.6, + 136.7, + 151.0, + 168.9, + 169.0, + 300.0, + 384.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rfdetr-n-vs-rtdetrv2-r18.claims.json b/article-pipeline/v2/output/rfdetr-n-vs-rtdetrv2-r18.claims.json new file mode 100644 index 0000000..591341a --- /dev/null +++ b/article-pipeline/v2/output/rfdetr-n-vs-rtdetrv2-r18.claims.json @@ -0,0 +1,387 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:44.240654+00:00", + "slug": "rfdetr-n-vs-rtdetrv2-r18", + "meta": { + "type": "vs", + "a": "rfdetr-n", + "b": "rtdetrv2-r18", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "license": "Apache-2.0", + "releaseDate": "2026-02-23", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5135247777250423, + "mAP_50": 0.6989693323963424, + "mAP_small": 0.2768082791741451, + "mAP_medium": 0.5713133502545867, + "mAP_large": 0.7180799558822147, + "fps_mean": 39.53, + "fps_p50": 41.33, + "ms_per_image": 25.295, + "inference_ms": 16.983, + "preprocess_ms": 5.999, + "postprocess_ms": 2.313, + "p95_ms": 29.239, + "peak_vram_mb": 151.0, + "peak_ram_mb": 136.7, + "params_millions": 30.47, + "gflops": 0.0, + "input_size": 384, + "config": { + "batch_size": 1, + "input_size": 384, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-17T17:13:52.106298Z" + } + }, + "b": { + "id": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "license": "Apache-2.0", + "releaseDate": "2024-05-01", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5074977298289686, + "mAP_50": 0.6738187878266525, + "mAP_small": 0.3347978553273374, + "mAP_medium": 0.5390239587831667, + "mAP_large": 0.663719218286382, + "fps_mean": 51.15, + "fps_p50": 52.26, + "ms_per_image": 19.552, + "inference_ms": 14.819, + "preprocess_ms": 3.67, + "postprocess_ms": 1.062, + "p95_ms": 23.873, + "peak_vram_mb": 169.1, + "peak_ram_mb": 0.0, + "params_millions": 20.18, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:29:32.230268Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": 1.19, + "fps": -22.72, + "params": 50.99, + "flops": -100.0, + "peak_vram": -10.7 + }, + "map_pts": { + "a": 51.4, + "b": 50.7, + "delta": 0.7 + }, + "pair": { + "params_ratio": 1.51, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": null, + "b_mAP_per_GFLOP": 0.00846 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 51.4, + "fps": 65.7 + }, + "b": { + "mAP": 50.7, + "fps": 97.1 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 51.4, + "fps": 39.5 + }, + "b": { + "mAP": 50.7, + "fps": 51.1 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 51.4, + "fps": 58.3 + }, + "b": { + "mAP": 50.8, + "fps": 117.8 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 51.4, + "fps": 67.6 + }, + "b": { + "mAP": 50.7, + "fps": 125.5 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 51.4, + "fps": 10.2 + }, + "b": { + "mAP": 50.8, + "fps": 10.3 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 51.4, + "fps": 39.5 + }, + "b": { + "mAP": 50.7, + "fps": 51.1 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 51.4, + "fps": 2.1 + }, + "b": { + "mAP": 50.8, + "fps": 0.8 + } + } + ], + "input_sizes": { + "a": 384, + "b": 640, + "match": false + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "RF-DETR-N measures 0.7 mAP points higher than RT-DETRv2-R18 on NVIDIA RTX 5070 Ti (PyTorch FP32): 51.4 vs 50.7 mAP@50-95.", + "values": { + "delta_pts": 0.7, + "a_map": 51.4, + "b_map": 50.7 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETRv2-R18 is 22.72% faster: 39.5 vs 51.1 FPS (25.3 vs 19.55 ms/image).", + "values": { + "delta_pct": 22.72, + "a_fps": 39.5, + "b_fps": 51.1 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "RF-DETR-N uses less peak VRAM: 151 vs 169 MB.", + "values": { + "a_vram": 151, + "b_vram": 169 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "RT-DETRv2-R18 leads on small objects: 27.7 vs 33.5 mAP_small (17.32% relative gap).", + "values": { + "a": 27.7, + "b": 33.5, + "delta_pct": 17.32 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "input_size_caveat", + "kind": "caveat", + "statement": "MANDATORY CAVEAT: RF-DETR-N runs at 384 px and RT-DETRv2-R18 at 640 px (each model at its author default). Say this in Setup and the verdict.", + "values": { + "a": 384, + "b": 640 + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -100.0, + -22.72, + -10.7, + 0.0, + 0.001, + 0.0085, + 0.1, + 0.2768, + 0.3348, + 0.5075, + 0.5135, + 0.539, + 0.5713, + 0.6, + 0.6637, + 0.6738, + 0.699, + 0.7, + 0.7181, + 0.8, + 0.846, + 1.0, + 1.062, + 1.19, + 1.51, + 2, + 2.1, + 2.313, + 3, + 3.67, + 4, + 5, + 5.999, + 6, + 7, + 8, + 9, + 10, + 10.2, + 10.3, + 11, + 12, + 14.819, + 16.983, + 17.32, + 19.552, + 20.18, + 22.72, + 23.873, + 25.295, + 27.6808, + 27.7, + 29.239, + 30.47, + 33.4798, + 33.5, + 39.5, + 39.53, + 41.33, + 50, + 50.7, + 50.7498, + 50.8, + 50.99, + 51.1, + 51.15, + 51.3525, + 51.4, + 52.26, + 53.9024, + 57.1313, + 58.3, + 60.0, + 65.7, + 66.3719, + 67.3819, + 67.6, + 69.8969, + 70.0, + 71.808, + 75, + 80.0, + 95, + 97.1, + 100.0, + 117.8, + 125.5, + 136.7, + 151.0, + 169.0, + 169.1, + 300.0, + 384.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rfdetr-n-vs-yolov9c.claims.json b/article-pipeline/v2/output/rfdetr-n-vs-yolov9c.claims.json new file mode 100644 index 0000000..30eaf60 --- /dev/null +++ b/article-pipeline/v2/output/rfdetr-n-vs-yolov9c.claims.json @@ -0,0 +1,389 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:42.286778+00:00", + "slug": "rfdetr-n-vs-yolov9c", + "meta": { + "type": "vs", + "a": "rfdetr-n", + "b": "yolov9c", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "license": "Apache-2.0", + "releaseDate": "2026-02-23", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5135247777250423, + "mAP_50": 0.6989693323963424, + "mAP_small": 0.2768082791741451, + "mAP_medium": 0.5713133502545867, + "mAP_large": 0.7180799558822147, + "fps_mean": 39.53, + "fps_p50": 41.33, + "ms_per_image": 25.295, + "inference_ms": 16.983, + "preprocess_ms": 5.999, + "postprocess_ms": 2.313, + "p95_ms": 29.239, + "peak_vram_mb": 151.0, + "peak_ram_mb": 136.7, + "params_millions": 30.47, + "gflops": 0.0, + "input_size": 384, + "config": { + "batch_size": 1, + "input_size": 384, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-17T17:13:52.106298Z" + } + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "license": "MIT", + "releaseDate": "2024-02-21", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5709322769395554, + "mAP_50": 0.7383277753630146, + "mAP_small": 0.39292500625214793, + "mAP_medium": 0.6303775404072186, + "mAP_large": 0.7157211255887754, + "fps_mean": 42.33, + "fps_p50": 42.78, + "ms_per_image": 23.626, + "inference_ms": 18.42, + "preprocess_ms": 3.394, + "postprocess_ms": 1.812, + "p95_ms": 29.985, + "peak_vram_mb": 240.9, + "peak_ram_mb": 0.0, + "params_millions": 25.5, + "gflops": 51.75, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:19:12.111885Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -10.06, + "fps": -6.61, + "params": 19.49, + "flops": -100.0, + "peak_vram": -37.32 + }, + "map_pts": { + "a": 51.4, + "b": 57.1, + "delta": -5.7 + }, + "pair": { + "params_ratio": 1.19, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": null, + "b_mAP_per_GFLOP": 0.01103 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 51.4, + "fps": 65.7 + }, + "b": { + "mAP": 57.1, + "fps": 63.5 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 51.4, + "fps": 39.5 + }, + "b": { + "mAP": 57.1, + "fps": 42.3 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 51.4, + "fps": 58.3 + }, + "b": { + "mAP": 57.2, + "fps": 74.7 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 51.4, + "fps": 67.6 + }, + "b": { + "mAP": 57.1, + "fps": 76.7 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 51.4, + "fps": 10.2 + }, + "b": { + "mAP": 57.1, + "fps": 6.4 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 51.4, + "fps": 39.5 + }, + "b": { + "mAP": 57.1, + "fps": 42.3 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 51.4, + "fps": 2.1 + }, + "b": { + "mAP": 56.4, + "fps": 0.5 + } + } + ], + "input_sizes": { + "a": 384, + "b": 640, + "match": false + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "YOLOv9-C measures 5.7 mAP points higher than RF-DETR-N on NVIDIA RTX 5070 Ti (PyTorch FP32): 51.4 vs 57.1 mAP@50-95.", + "values": { + "delta_pts": 5.7, + "a_map": 51.4, + "b_map": 57.1 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "YOLOv9-C is 6.61% faster: 39.5 vs 42.3 FPS (25.3 vs 23.63 ms/image).", + "values": { + "delta_pct": 6.61, + "a_fps": 39.5, + "b_fps": 42.3 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "RF-DETR-N uses less peak VRAM: 151 vs 241 MB.", + "values": { + "a_vram": 151, + "b_vram": 241 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "YOLOv9-C leads on small objects: 27.7 vs 39.3 mAP_small (29.55% relative gap).", + "values": { + "a": 27.7, + "b": 39.3, + "delta_pct": 29.55 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on hardware: NVIDIA Jetson Orin Nano Super 8GB, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "hardware:NVIDIA Jetson Orin Nano Super 8GB", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "input_size_caveat", + "kind": "caveat", + "statement": "MANDATORY CAVEAT: RF-DETR-N runs at 384 px and YOLOv9-C at 640 px (each model at its author default). Say this in Setup and the verdict.", + "values": { + "a": 384, + "b": 640 + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -100.0, + -37.32, + -10.06, + -6.61, + -5.7, + 0.0, + 0.001, + 0.011, + 0.1, + 0.2768, + 0.3929, + 0.5, + 0.5135, + 0.5709, + 0.5713, + 0.6, + 0.6304, + 0.699, + 0.7157, + 0.7181, + 0.7383, + 1.0, + 1.103, + 1.19, + 1.812, + 2, + 2.1, + 2.313, + 3, + 3.394, + 4, + 5, + 5.7, + 5.999, + 6, + 6.4, + 6.61, + 7, + 8, + 9, + 10, + 10.2, + 11, + 12, + 16.983, + 18.42, + 19.49, + 23.626, + 25.295, + 25.5, + 27.6808, + 27.7, + 29.239, + 29.55, + 29.985, + 30.47, + 39.2925, + 39.3, + 39.5, + 39.53, + 41.33, + 42.3, + 42.33, + 42.78, + 50.0, + 51.3525, + 51.4, + 51.75, + 56.4, + 57.0932, + 57.1, + 57.1313, + 57.2, + 58.3, + 60.0, + 63.0378, + 63.5, + 65.7, + 67.6, + 69.8969, + 71.5721, + 71.808, + 73.8328, + 74.7, + 75, + 76.7, + 95, + 100.0, + 136.7, + 151.0, + 240.9, + 241.0, + 300.0, + 384.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rfdetr-n-vs-yolox-l.claims.json b/article-pipeline/v2/output/rfdetr-n-vs-yolox-l.claims.json new file mode 100644 index 0000000..e60ef81 --- /dev/null +++ b/article-pipeline/v2/output/rfdetr-n-vs-yolox-l.claims.json @@ -0,0 +1,389 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:42.369853+00:00", + "slug": "rfdetr-n-vs-yolox-l", + "meta": { + "type": "vs", + "a": "rfdetr-n", + "b": "yolox-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "license": "Apache-2.0", + "releaseDate": "2026-02-23", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5135247777250423, + "mAP_50": 0.6989693323963424, + "mAP_small": 0.2768082791741451, + "mAP_medium": 0.5713133502545867, + "mAP_large": 0.7180799558822147, + "fps_mean": 39.53, + "fps_p50": 41.33, + "ms_per_image": 25.295, + "inference_ms": 16.983, + "preprocess_ms": 5.999, + "postprocess_ms": 2.313, + "p95_ms": 29.239, + "peak_vram_mb": 151.0, + "peak_ram_mb": 136.7, + "params_millions": 30.47, + "gflops": 0.0, + "input_size": 384, + "config": { + "batch_size": 1, + "input_size": 384, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-17T17:13:52.106298Z" + } + }, + "b": { + "id": "yolox-l", + "displayName": "YOLOX-L", + "license": "Apache-2.0", + "releaseDate": "2021-07-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5542460212991265, + "mAP_50": 0.7302189854897115, + "mAP_small": 0.39036786287091835, + "mAP_medium": 0.5988447680065735, + "mAP_large": 0.685177367489107, + "fps_mean": 42.84, + "fps_p50": 42.78, + "ms_per_image": 23.341, + "inference_ms": 13.911, + "preprocess_ms": 6.863, + "postprocess_ms": 2.567, + "p95_ms": 27.837, + "peak_vram_mb": 307.0, + "peak_ram_mb": 0.0, + "params_millions": 54.21, + "gflops": 78.01, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:20:38.605152Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -7.35, + "fps": -7.73, + "params": -43.79, + "flops": -100.0, + "peak_vram": -50.81 + }, + "map_pts": { + "a": 51.4, + "b": 55.4, + "delta": -4.0 + }, + "pair": { + "params_ratio": 1.78, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": null, + "b_mAP_per_GFLOP": 0.0071 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 51.4, + "fps": 65.7 + }, + "b": { + "mAP": 55.4, + "fps": 50.6 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 51.4, + "fps": 39.5 + }, + "b": { + "mAP": 55.4, + "fps": 42.8 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 51.4, + "fps": 58.3 + }, + "b": { + "mAP": 55.4, + "fps": 79.2 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 51.4, + "fps": 67.6 + }, + "b": { + "mAP": 55.4, + "fps": 61.8 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 51.4, + "fps": 10.2 + }, + "b": { + "mAP": 55.4, + "fps": 5.8 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 51.4, + "fps": 39.5 + }, + "b": { + "mAP": 55.4, + "fps": 42.8 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 51.4, + "fps": 2.1 + }, + "b": { + "mAP": 53.9, + "fps": 0.4 + } + } + ], + "input_sizes": { + "a": 384, + "b": 640, + "match": false + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "YOLOX-L measures 4.0 mAP points higher than RF-DETR-N on NVIDIA RTX 5070 Ti (PyTorch FP32): 51.4 vs 55.4 mAP@50-95.", + "values": { + "delta_pts": 4.0, + "a_map": 51.4, + "b_map": 55.4 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "YOLOX-L is 7.73% faster: 39.5 vs 42.8 FPS (25.3 vs 23.34 ms/image).", + "values": { + "delta_pct": 7.73, + "a_fps": 39.5, + "b_fps": 42.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "RF-DETR-N uses less peak VRAM: 151 vs 307 MB.", + "values": { + "a_vram": 151, + "b_vram": 307 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "YOLOX-L leads on small objects: 27.7 vs 39.0 mAP_small (29.09% relative gap).", + "values": { + "a": 27.7, + "b": 39.0, + "delta_pct": 29.09 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: ONNX Runtime FP32, runtime: TensorRT FP32, hardware: NVIDIA Jetson Orin Nano Super 8GB, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:ONNX Runtime FP32", + "runtime:TensorRT FP32", + "hardware:NVIDIA Jetson Orin Nano Super 8GB", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "input_size_caveat", + "kind": "caveat", + "statement": "MANDATORY CAVEAT: RF-DETR-N runs at 384 px and YOLOX-L at 640 px (each model at its author default). Say this in Setup and the verdict.", + "values": { + "a": 384, + "b": 640 + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -100.0, + -50.81, + -43.79, + -7.73, + -7.35, + -4.0, + 0.0, + 0.001, + 0.0071, + 0.1, + 0.2768, + 0.3904, + 0.4, + 0.5135, + 0.5542, + 0.5713, + 0.5988, + 0.6, + 0.6852, + 0.699, + 0.71, + 0.7181, + 0.7302, + 1.0, + 1.78, + 2, + 2.1, + 2.313, + 2.567, + 3, + 4.0, + 5, + 5.8, + 5.999, + 6, + 6.863, + 7, + 7.73, + 8, + 9, + 10, + 10.2, + 11, + 12, + 13.911, + 16.983, + 23.341, + 25.295, + 27.6808, + 27.7, + 27.837, + 29.09, + 29.239, + 30.47, + 39.0, + 39.0368, + 39.5, + 39.53, + 40.0, + 41.33, + 42.78, + 42.8, + 42.84, + 50, + 50.6, + 51.3525, + 51.4, + 53.9, + 54.21, + 55.4, + 55.4246, + 57.1313, + 58.3, + 59.8845, + 60.0, + 61.8, + 65.7, + 67.6, + 68.5177, + 69.8969, + 71.808, + 73.0219, + 75, + 78.01, + 79.2, + 95, + 100.0, + 136.7, + 151.0, + 300.0, + 307.0, + 384.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rfdetr-n-vs-yolox-m.claims.json b/article-pipeline/v2/output/rfdetr-n-vs-yolox-m.claims.json new file mode 100644 index 0000000..77216ef --- /dev/null +++ b/article-pipeline/v2/output/rfdetr-n-vs-yolox-m.claims.json @@ -0,0 +1,391 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.487440+00:00", + "slug": "rfdetr-n-vs-yolox-m", + "meta": { + "type": "vs", + "a": "rfdetr-n", + "b": "yolox-m", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "license": "Apache-2.0", + "releaseDate": "2026-02-23", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5135247777250423, + "mAP_50": 0.6989693323963424, + "mAP_small": 0.2768082791741451, + "mAP_medium": 0.5713133502545867, + "mAP_large": 0.7180799558822147, + "fps_mean": 39.53, + "fps_p50": 41.33, + "ms_per_image": 25.295, + "inference_ms": 16.983, + "preprocess_ms": 5.999, + "postprocess_ms": 2.313, + "p95_ms": 29.239, + "peak_vram_mb": 151.0, + "peak_ram_mb": 136.7, + "params_millions": 30.47, + "gflops": 0.0, + "input_size": 384, + "config": { + "batch_size": 1, + "input_size": 384, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-17T17:13:52.106298Z" + } + }, + "b": { + "id": "yolox-m", + "displayName": "YOLOX-M", + "license": "Apache-2.0", + "releaseDate": "2021-07-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5168754166625369, + "mAP_50": 0.7015353621799966, + "mAP_small": 0.3291253382759625, + "mAP_medium": 0.580613561189069, + "mAP_large": 0.6737382688397477, + "fps_mean": 48.21, + "fps_p50": 49.54, + "ms_per_image": 20.741, + "inference_ms": 11.382, + "preprocess_ms": 6.641, + "postprocess_ms": 2.718, + "p95_ms": 24.126, + "peak_vram_mb": 172.4, + "peak_ram_mb": 385.1, + "params_millions": 25.33, + "gflops": 36.99, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T22:49:31.649772Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -0.65, + "fps": -18.0, + "params": 20.29, + "flops": -100.0, + "peak_vram": -12.41 + }, + "map_pts": { + "a": 51.4, + "b": 51.7, + "delta": -0.3 + }, + "pair": { + "params_ratio": 1.2, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": null, + "b_mAP_per_GFLOP": 0.01397 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 51.4, + "fps": 65.7 + }, + "b": { + "mAP": 51.8, + "fps": 55.8 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 51.4, + "fps": 39.5 + }, + "b": { + "mAP": 51.7, + "fps": 48.2 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 51.4, + "fps": 58.3 + }, + "b": { + "mAP": 51.6, + "fps": 83.0 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 51.4, + "fps": 67.6 + }, + "b": { + "mAP": 51.7, + "fps": 70.1 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 51.4, + "fps": 10.2 + }, + "b": { + "mAP": 51.7, + "fps": 9.1 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 51.4, + "fps": 39.5 + }, + "b": { + "mAP": 51.7, + "fps": 48.2 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 51.4, + "fps": 2.1 + }, + "b": { + "mAP": 50.9, + "fps": 0.8 + } + } + ], + "input_sizes": { + "a": 384, + "b": 640, + "match": false + } + }, + "claims": [ + { + "id": "accuracy_tie", + "kind": "accuracy", + "statement": "Accuracy is a tie: 51.4 vs 51.7 mAP@50-95 (0.3 points apart, below the 0.5-point threshold).", + "values": { + "a_map": 51.4, + "b_map": 51.7 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "YOLOX-M is 18.0% faster: 39.5 vs 48.2 FPS (25.3 vs 20.74 ms/image).", + "values": { + "delta_pct": 18.0, + "a_fps": 39.5, + "b_fps": 48.2 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "RF-DETR-N uses less peak VRAM: 151 vs 172 MB.", + "values": { + "a_vram": 151, + "b_vram": 172 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "YOLOX-M leads on small objects: 27.7 vs 32.9 mAP_small (15.9% relative gap).", + "values": { + "a": 27.7, + "b": 32.9, + "delta_pct": 15.9 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: ONNX Runtime FP32, hardware: NVIDIA Jetson Orin Nano Super 8GB, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:ONNX Runtime FP32", + "hardware:NVIDIA Jetson Orin Nano Super 8GB", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "input_size_caveat", + "kind": "caveat", + "statement": "MANDATORY CAVEAT: RF-DETR-N runs at 384 px and YOLOX-M at 640 px (each model at its author default). Say this in Setup and the verdict.", + "values": { + "a": 384, + "b": 640 + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -100.0, + -18.0, + -12.41, + -0.65, + -0.3, + 0.0, + 0.001, + 0.014, + 0.1, + 0.2768, + 0.3291, + 0.5135, + 0.5169, + 0.5713, + 0.5806, + 0.6, + 0.6737, + 0.699, + 0.7015, + 0.7181, + 0.8, + 1.0, + 1.2, + 1.397, + 2, + 2.1, + 2.313, + 2.718, + 3, + 4, + 5, + 5.999, + 6, + 6.641, + 7, + 8, + 9, + 9.1, + 10, + 10.2, + 11, + 11.382, + 12, + 15.9, + 16.983, + 18.0, + 20.29, + 20.741, + 24.126, + 25.295, + 25.33, + 27.6808, + 27.7, + 29.239, + 30.47, + 32.9, + 32.9125, + 36.99, + 39.5, + 39.53, + 41.33, + 48.2, + 48.21, + 49.54, + 50, + 50.9, + 51.3525, + 51.4, + 51.6, + 51.6875, + 51.7, + 51.8, + 55.8, + 57.1313, + 58.0614, + 58.3, + 60.0, + 65.7, + 67.3738, + 67.6, + 69.8969, + 70.1, + 70.1535, + 71.808, + 75, + 80.0, + 83.0, + 95, + 100.0, + 136.7, + 151.0, + 172.0, + 172.4, + 300.0, + 384.0, + 385.1, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rfdetr-s-vs-yolov9c.claims.json b/article-pipeline/v2/output/rfdetr-s-vs-yolov9c.claims.json new file mode 100644 index 0000000..20fdd29 --- /dev/null +++ b/article-pipeline/v2/output/rfdetr-s-vs-yolov9c.claims.json @@ -0,0 +1,389 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:42.437809+00:00", + "slug": "rfdetr-s-vs-yolov9c", + "meta": { + "type": "vs", + "a": "rfdetr-s", + "b": "yolov9c", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "rfdetr-s", + "displayName": "RF-DETR-S", + "license": "Apache-2.0", + "releaseDate": "2026-02-23", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5511843115019394, + "mAP_50": 0.7348299034797677, + "mAP_small": 0.35128962837275934, + "mAP_medium": 0.609452999962134, + "mAP_large": 0.7384939778490184, + "fps_mean": 35.81, + "fps_p50": 36.29, + "ms_per_image": 27.925, + "inference_ms": 16.816, + "preprocess_ms": 8.892, + "postprocess_ms": 2.217, + "p95_ms": 32.024, + "peak_vram_mb": 176.1, + "peak_ram_mb": 0.0, + "params_millions": 32.11, + "gflops": 0.0, + "input_size": 512, + "config": { + "batch_size": 1, + "input_size": 512, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-17T17:14:15.043738Z" + } + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "license": "MIT", + "releaseDate": "2024-02-21", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5709322769395554, + "mAP_50": 0.7383277753630146, + "mAP_small": 0.39292500625214793, + "mAP_medium": 0.6303775404072186, + "mAP_large": 0.7157211255887754, + "fps_mean": 42.33, + "fps_p50": 42.78, + "ms_per_image": 23.626, + "inference_ms": 18.42, + "preprocess_ms": 3.394, + "postprocess_ms": 1.812, + "p95_ms": 29.985, + "peak_vram_mb": 240.9, + "peak_ram_mb": 0.0, + "params_millions": 25.5, + "gflops": 51.75, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:19:12.111885Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -3.46, + "fps": -15.4, + "params": 25.92, + "flops": -100.0, + "peak_vram": -26.9 + }, + "map_pts": { + "a": 55.1, + "b": 57.1, + "delta": -2.0 + }, + "pair": { + "params_ratio": 1.26, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": null, + "b_mAP_per_GFLOP": 0.01103 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 55.1, + "fps": 68.5 + }, + "b": { + "mAP": 57.1, + "fps": 63.5 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 55.1, + "fps": 35.8 + }, + "b": { + "mAP": 57.1, + "fps": 42.3 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 55.3, + "fps": 71.6 + }, + "b": { + "mAP": 57.2, + "fps": 74.7 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 55.1, + "fps": 77.5 + }, + "b": { + "mAP": 57.1, + "fps": 76.7 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 55.1, + "fps": 6.9 + }, + "b": { + "mAP": 57.1, + "fps": 6.4 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 55.1, + "fps": 35.8 + }, + "b": { + "mAP": 57.1, + "fps": 42.3 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 55.1, + "fps": 1.0 + }, + "b": { + "mAP": 56.4, + "fps": 0.5 + } + } + ], + "input_sizes": { + "a": 512, + "b": 640, + "match": false + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "YOLOv9-C measures 2.0 mAP points higher than RF-DETR-S on NVIDIA RTX 5070 Ti (PyTorch FP32): 55.1 vs 57.1 mAP@50-95.", + "values": { + "delta_pts": 2.0, + "a_map": 55.1, + "b_map": 57.1 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "YOLOv9-C is 15.4% faster: 35.8 vs 42.3 FPS (27.93 vs 23.63 ms/image).", + "values": { + "delta_pct": 15.4, + "a_fps": 35.8, + "b_fps": 42.3 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "RF-DETR-S uses less peak VRAM: 176 vs 241 MB.", + "values": { + "a_vram": 176, + "b_vram": 241 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "YOLOv9-C leads on small objects: 35.1 vs 39.3 mAP_small (10.6% relative gap).", + "values": { + "a": 35.1, + "b": 39.3, + "delta_pct": 10.6 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: ONNX Runtime FP32, hardware: NVIDIA Jetson Orin Nano Super 8GB, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:ONNX Runtime FP32", + "hardware:NVIDIA Jetson Orin Nano Super 8GB", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "input_size_caveat", + "kind": "caveat", + "statement": "MANDATORY CAVEAT: RF-DETR-S runs at 512 px and YOLOv9-C at 640 px (each model at its author default). Say this in Setup and the verdict.", + "values": { + "a": 512, + "b": 640 + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -100.0, + -26.9, + -15.4, + -3.46, + -2.0, + 0.0, + 0.001, + 0.011, + 0.1, + 0.3513, + 0.3929, + 0.5, + 0.5512, + 0.5709, + 0.6, + 0.6095, + 0.6304, + 0.7157, + 0.7348, + 0.7383, + 0.7385, + 1.0, + 1.103, + 1.26, + 1.812, + 2.0, + 2.217, + 3, + 3.394, + 4, + 5, + 6, + 6.4, + 6.9, + 7, + 8, + 8.892, + 9, + 10, + 10.6, + 11, + 12, + 15.4, + 16.816, + 18.42, + 23.626, + 25.5, + 25.92, + 27.925, + 29.985, + 32.024, + 32.11, + 35.1, + 35.129, + 35.8, + 35.81, + 36.29, + 39.2925, + 39.3, + 42.3, + 42.33, + 42.78, + 50.0, + 51.75, + 55.1, + 55.1184, + 55.3, + 56.4, + 57.0932, + 57.1, + 57.2, + 60.0, + 60.9453, + 63.0378, + 63.5, + 68.5, + 71.5721, + 71.6, + 73.483, + 73.8328, + 73.8494, + 74.7, + 75, + 76.7, + 77.5, + 95, + 100.0, + 176.0, + 176.1, + 240.9, + 241.0, + 300.0, + 500.0, + 512.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rfdetr-s-vs-yolox-l.claims.json b/article-pipeline/v2/output/rfdetr-s-vs-yolox-l.claims.json new file mode 100644 index 0000000..9463e28 --- /dev/null +++ b/article-pipeline/v2/output/rfdetr-s-vs-yolox-l.claims.json @@ -0,0 +1,388 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:43.561612+00:00", + "slug": "rfdetr-s-vs-yolox-l", + "meta": { + "type": "vs", + "a": "rfdetr-s", + "b": "yolox-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "rfdetr-s", + "displayName": "RF-DETR-S", + "license": "Apache-2.0", + "releaseDate": "2026-02-23", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5511843115019394, + "mAP_50": 0.7348299034797677, + "mAP_small": 0.35128962837275934, + "mAP_medium": 0.609452999962134, + "mAP_large": 0.7384939778490184, + "fps_mean": 35.81, + "fps_p50": 36.29, + "ms_per_image": 27.925, + "inference_ms": 16.816, + "preprocess_ms": 8.892, + "postprocess_ms": 2.217, + "p95_ms": 32.024, + "peak_vram_mb": 176.1, + "peak_ram_mb": 0.0, + "params_millions": 32.11, + "gflops": 0.0, + "input_size": 512, + "config": { + "batch_size": 1, + "input_size": 512, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-17T17:14:15.043738Z" + } + }, + "b": { + "id": "yolox-l", + "displayName": "YOLOX-L", + "license": "Apache-2.0", + "releaseDate": "2021-07-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5542460212991265, + "mAP_50": 0.7302189854897115, + "mAP_small": 0.39036786287091835, + "mAP_medium": 0.5988447680065735, + "mAP_large": 0.685177367489107, + "fps_mean": 42.84, + "fps_p50": 42.78, + "ms_per_image": 23.341, + "inference_ms": 13.911, + "preprocess_ms": 6.863, + "postprocess_ms": 2.567, + "p95_ms": 27.837, + "peak_vram_mb": 307.0, + "peak_ram_mb": 0.0, + "params_millions": 54.21, + "gflops": 78.01, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:20:38.605152Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -0.55, + "fps": -16.41, + "params": -40.77, + "flops": -100.0, + "peak_vram": -42.64 + }, + "map_pts": { + "a": 55.1, + "b": 55.4, + "delta": -0.3 + }, + "pair": { + "params_ratio": 1.69, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": null, + "b_mAP_per_GFLOP": 0.0071 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 55.1, + "fps": 68.5 + }, + "b": { + "mAP": 55.4, + "fps": 50.6 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 55.1, + "fps": 35.8 + }, + "b": { + "mAP": 55.4, + "fps": 42.8 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 55.3, + "fps": 71.6 + }, + "b": { + "mAP": 55.4, + "fps": 79.2 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 55.1, + "fps": 77.5 + }, + "b": { + "mAP": 55.4, + "fps": 61.8 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 55.1, + "fps": 6.9 + }, + "b": { + "mAP": 55.4, + "fps": 5.8 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 55.1, + "fps": 35.8 + }, + "b": { + "mAP": 55.4, + "fps": 42.8 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 55.1, + "fps": 1.0 + }, + "b": { + "mAP": 53.9, + "fps": 0.4 + } + } + ], + "input_sizes": { + "a": 512, + "b": 640, + "match": false + } + }, + "claims": [ + { + "id": "accuracy_tie", + "kind": "accuracy", + "statement": "Accuracy is a tie: 55.1 vs 55.4 mAP@50-95 (0.3 points apart, below the 0.5-point threshold).", + "values": { + "a_map": 55.1, + "b_map": 55.4 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "YOLOX-L is 16.41% faster: 35.8 vs 42.8 FPS (27.93 vs 23.34 ms/image).", + "values": { + "delta_pct": 16.41, + "a_fps": 35.8, + "b_fps": 42.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "RF-DETR-S uses less peak VRAM: 176 vs 307 MB.", + "values": { + "a_vram": 176, + "b_vram": 307 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "YOLOX-L leads on small objects: 35.1 vs 39.0 mAP_small (10.01% relative gap).", + "values": { + "a": 35.1, + "b": 39.0, + "delta_pct": 10.01 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: ONNX Runtime FP32, runtime: TensorRT FP32, hardware: NVIDIA Jetson Orin Nano Super 8GB, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:ONNX Runtime FP32", + "runtime:TensorRT FP32", + "hardware:NVIDIA Jetson Orin Nano Super 8GB", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "input_size_caveat", + "kind": "caveat", + "statement": "MANDATORY CAVEAT: RF-DETR-S runs at 512 px and YOLOX-L at 640 px (each model at its author default). Say this in Setup and the verdict.", + "values": { + "a": 512, + "b": 640 + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -100.0, + -42.64, + -40.77, + -16.41, + -0.55, + -0.3, + 0.0, + 0.001, + 0.0071, + 0.1, + 0.3513, + 0.3904, + 0.4, + 0.5512, + 0.5542, + 0.5988, + 0.6, + 0.6095, + 0.6852, + 0.71, + 0.7302, + 0.7348, + 0.7385, + 1.0, + 1.69, + 2, + 2.217, + 2.567, + 3, + 4, + 5, + 5.8, + 6, + 6.863, + 6.9, + 7, + 8, + 8.892, + 9, + 10, + 10.01, + 11, + 12, + 13.911, + 16.41, + 16.816, + 23.341, + 27.837, + 27.925, + 32.024, + 32.11, + 35.1, + 35.129, + 35.8, + 35.81, + 36.29, + 39.0, + 39.0368, + 40.0, + 42.78, + 42.8, + 42.84, + 50, + 50.6, + 53.9, + 54.21, + 55.1, + 55.1184, + 55.3, + 55.4, + 55.4246, + 59.8845, + 60.0, + 60.9453, + 61.8, + 68.5, + 68.5177, + 71.6, + 73.0219, + 73.483, + 73.8494, + 75, + 77.5, + 78.01, + 79.2, + 95, + 100.0, + 176.0, + 176.1, + 300.0, + 307.0, + 500.0, + 512.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rfdetr-vs-rtdetr.claims.json b/article-pipeline/v2/output/rfdetr-vs-rtdetr.claims.json new file mode 100644 index 0000000..5a0cd8c --- /dev/null +++ b/article-pipeline/v2/output/rfdetr-vs-rtdetr.claims.json @@ -0,0 +1,400 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:19.322060+00:00", + "slug": "rfdetr-vs-rtdetr", + "meta": { + "type": "family-vs", + "a": "rfdetr", + "b": "rtdetr", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "rfdetr", + "b": "rtdetr", + "a_display": "RF-DETR", + "b_display": "RT-DETR" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + { + "id": "rfdetr-s", + "displayName": "RF-DETR-S", + "params": 32.11, + "mAP": 55.1, + "fps": 35.8, + "license": "Apache-2.0" + }, + { + "id": "rfdetr-m", + "displayName": "RF-DETR-M", + "params": 33.69, + "mAP": 57.3, + "fps": 30.3, + "license": "Apache-2.0" + }, + { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-l", + "displayName": "RT-DETR-L", + "params": 32.93, + "mAP": 55.8, + "fps": 27.7, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "params": 36.59, + "mAP": 53.8, + "fps": 35.3, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "params": 42.89, + "mAP": 55.9, + "fps": 29.8, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-x", + "displayName": "RT-DETR-X", + "params": 67.37, + "mAP": 57.9, + "fps": 21.8, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + } + ], + "matched_pairs": [ + { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -0.8, + "fps_delta_pct": -1.74 + }, + { + "a": { + "id": "rfdetr-s", + "displayName": "RF-DETR-S", + "params": 32.11, + "mAP": 55.1, + "fps": 35.8, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 2.9, + "fps_delta_pct": -10.95 + }, + { + "a": { + "id": "rfdetr-m", + "displayName": "RF-DETR-M", + "params": 33.69, + "mAP": 57.3, + "fps": 30.3, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-l", + "displayName": "RT-DETR-L", + "params": 32.93, + "mAP": 55.8, + "fps": 27.7, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 1.5, + "fps_delta_pct": 9.39 + }, + { + "a": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-l", + "displayName": "RT-DETR-L", + "params": 32.93, + "mAP": 55.8, + "fps": 27.7, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 2.8, + "fps_delta_pct": -9.39 + } + ], + "aggregate": { + "n_pairs": 4, + "mean_mAP_delta_pts": 1.6, + "a_wins": 3, + "b_wins": 1, + "mean_fps_delta_pct": -3.2 + }, + "flagship": { + "a": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + } + }, + "smallest": { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + } + }, + "crossover": { + "low_param_leader": "rfdetr", + "high_param_leader": "rfdetr" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "rfdetr-n,rfdetr-s,rfdetr-m,rfdetr-l,rtdetr-r18,rtdetr-r34,rtdetr-l,rtdetr-r50m,rtdetr-r50,rtdetr-x,rtdetr-r101" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, RF-DETR averages 1.6 mAP points higher across 4 paired variants on NVIDIA RTX 5070 Ti. RF-DETR wins 3, RT-DETR wins 1.", + "values": { + "mean_delta_pts": 1.6, + "a_wins": 3, + "b_wins": 1 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, RF-DETR's largest measured variant (RF-DETR-L) reaches 58.6 mAP; RT-DETR's (RT-DETR-R101) reaches 56.8 mAP.", + "values": { + "a": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, RF-DETR's RF-DETR-N (30.47M, 51.4 mAP, 39.5 FPS) faces RT-DETR's RT-DETR-R18 (20.18M, 49.8 mAP, 47.2 FPS).", + "values": { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "RF-DETR fields 4 measured variants, RT-DETR fields 7, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 4, + "b_n": 7 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 4, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -10.95, + -9.39, + -3.2, + -1.74, + -0.8, + 0.001, + 0.6, + 1.0, + 1.5, + 1.6, + 2, + 2.8, + 2.9, + 3.0, + 4.0, + 5, + 6, + 7.0, + 8, + 9, + 9.39, + 10, + 11, + 12, + 20.18, + 21.8, + 23.6, + 25.1, + 27.7, + 29.8, + 30.3, + 30.47, + 31.44, + 32.11, + 32.93, + 33.69, + 33.93, + 35.3, + 35.8, + 36.59, + 39.5, + 40.2, + 42.89, + 47.2, + 49.8, + 50, + 51.4, + 52.2, + 53.8, + 55.1, + 55.8, + 55.9, + 56.8, + 57.3, + 57.9, + 58.6, + 67.37, + 75, + 76.56, + 95, + 100.0, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rfdetr-vs-yolov9.claims.json b/article-pipeline/v2/output/rfdetr-vs-yolov9.claims.json new file mode 100644 index 0000000..2ed8e60 --- /dev/null +++ b/article-pipeline/v2/output/rfdetr-vs-yolov9.claims.json @@ -0,0 +1,390 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:17.882483+00:00", + "slug": "rfdetr-vs-yolov9", + "meta": { + "type": "family-vs", + "a": "rfdetr", + "b": "yolov9", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "rfdetr", + "b": "yolov9", + "a_display": "RF-DETR", + "b_display": "YOLOv9" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + { + "id": "rfdetr-s", + "displayName": "RF-DETR-S", + "params": 32.11, + "mAP": 55.1, + "fps": 35.8, + "license": "Apache-2.0" + }, + { + "id": "rfdetr-m", + "displayName": "RF-DETR-M", + "params": 33.69, + "mAP": 57.3, + "fps": 30.3, + "license": "Apache-2.0" + }, + { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "params": 2.04, + "mAP": 41.8, + "fps": 31.9, + "license": "MIT" + }, + { + "id": "yolov9s", + "displayName": "YOLOv9-S", + "params": 7.23, + "mAP": 50.4, + "fps": 30.8, + "license": "MIT" + }, + { + "id": "yolov9m", + "displayName": "YOLOv9-M", + "params": 20.12, + "mAP": 56.1, + "fps": 37.1, + "license": "MIT" + }, + { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + } + ], + "matched_pairs": [ + { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + }, + "mAP_delta_pts": -5.7, + "fps_delta_pct": -6.62 + }, + { + "a": { + "id": "rfdetr-s", + "displayName": "RF-DETR-S", + "params": 32.11, + "mAP": 55.1, + "fps": 35.8, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + }, + "mAP_delta_pts": -2.0, + "fps_delta_pct": -15.37 + }, + { + "a": { + "id": "rfdetr-m", + "displayName": "RF-DETR-M", + "params": 33.69, + "mAP": 57.3, + "fps": 30.3, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + }, + "mAP_delta_pts": 0.2, + "fps_delta_pct": -28.37 + }, + { + "a": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + }, + "mAP_delta_pts": 1.5, + "fps_delta_pct": -40.66 + } + ], + "aggregate": { + "n_pairs": 4, + "mean_mAP_delta_pts": -1.5, + "a_wins": 2, + "b_wins": 2, + "mean_fps_delta_pct": -22.8 + }, + "flagship": { + "a": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + } + }, + "smallest": { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "params": 2.04, + "mAP": 41.8, + "fps": 31.9, + "license": "MIT" + } + }, + "crossover": { + "low_param_leader": "yolov9", + "high_param_leader": "rfdetr" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "rfdetr-n,rfdetr-s,rfdetr-m,rfdetr-l,yolov9t,yolov9s,yolov9m,yolov9c" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, YOLOv9 averages 1.5 mAP points higher across 4 paired variants on NVIDIA RTX 5070 Ti. RF-DETR wins 2, YOLOv9 wins 2.", + "values": { + "mean_delta_pts": 1.5, + "a_wins": 2, + "b_wins": 2 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, YOLOv9 is 22.8% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 22.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_crossover", + "kind": "robustness", + "statement": "The frontier crosses: YOLOv9 leads at the small-model end, RF-DETR leads at the large-model end. There is no single winner; the choice depends on your size class.", + "values": { + "low": "yolov9", + "high": "rfdetr" + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, RF-DETR's largest measured variant (RF-DETR-L) reaches 58.6 mAP; YOLOv9's (YOLOv9-C) reaches 57.1 mAP.", + "values": { + "a": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, RF-DETR's RF-DETR-N (30.47M, 51.4 mAP, 39.5 FPS) faces YOLOv9's YOLOv9-T (2.04M, 41.8 mAP, 31.9 FPS).", + "values": { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "params": 2.04, + "mAP": 41.8, + "fps": 31.9, + "license": "MIT" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "RF-DETR fields 4 measured variants, YOLOv9 fields 4, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 4, + "b_n": 4 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -40.66, + -28.37, + -22.8, + -15.37, + -6.62, + -5.7, + -2.0, + -1.5, + 0.001, + 0.2, + 0.6, + 1, + 1.5, + 2.0, + 2.04, + 3, + 4.0, + 5, + 6, + 7, + 7.23, + 8, + 9, + 10, + 11, + 12, + 20.0, + 20.12, + 22.8, + 25.1, + 25.5, + 30.3, + 30.47, + 30.8, + 31.9, + 32.11, + 33.69, + 33.93, + 35.8, + 37.1, + 39.5, + 41.8, + 42.3, + 50, + 50.4, + 51.4, + 55.1, + 56.1, + 57.1, + 57.3, + 58.6, + 75, + 95, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rfdetr-vs-yolox.claims.json b/article-pipeline/v2/output/rfdetr-vs-yolox.claims.json new file mode 100644 index 0000000..2431d54 --- /dev/null +++ b/article-pipeline/v2/output/rfdetr-vs-yolox.claims.json @@ -0,0 +1,400 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:19.811488+00:00", + "slug": "rfdetr-vs-yolox", + "meta": { + "type": "family-vs", + "a": "rfdetr", + "b": "yolox", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "rfdetr", + "b": "yolox", + "a_display": "RF-DETR", + "b_display": "YOLOX" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + { + "id": "rfdetr-s", + "displayName": "RF-DETR-S", + "params": 32.11, + "mAP": 55.1, + "fps": 35.8, + "license": "Apache-2.0" + }, + { + "id": "rfdetr-m", + "displayName": "RF-DETR-M", + "params": 33.69, + "mAP": 57.3, + "fps": 30.3, + "license": "Apache-2.0" + }, + { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + }, + { + "id": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "params": 5.06, + "mAP": 35.5, + "fps": 61.6, + "license": "Apache-2.0" + }, + { + "id": "yolox-s", + "displayName": "YOLOX-S", + "params": 8.97, + "mAP": 44.3, + "fps": 50.0, + "license": "Apache-2.0" + }, + { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + { + "id": "yolox-l", + "displayName": "YOLOX-L", + "params": 54.21, + "mAP": 55.4, + "fps": 42.8, + "license": "Apache-2.0" + }, + { + "id": "yolox-x", + "displayName": "YOLOX-X", + "params": 99.07, + "mAP": 56.3, + "fps": 39.6, + "license": "Apache-2.0" + } + ], + "matched_pairs": [ + { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -0.3, + "fps_delta_pct": -18.05 + }, + { + "a": { + "id": "rfdetr-s", + "displayName": "RF-DETR-S", + "params": 32.11, + "mAP": 55.1, + "fps": 35.8, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 3.4, + "fps_delta_pct": -25.73 + }, + { + "a": { + "id": "rfdetr-m", + "displayName": "RF-DETR-M", + "params": 33.69, + "mAP": 57.3, + "fps": 30.3, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 5.6, + "fps_delta_pct": -37.14 + }, + { + "a": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 6.9, + "fps_delta_pct": -47.93 + } + ], + "aggregate": { + "n_pairs": 4, + "mean_mAP_delta_pts": 3.9, + "a_wins": 3, + "b_wins": 1, + "mean_fps_delta_pct": -32.2 + }, + "flagship": { + "a": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-x", + "displayName": "YOLOX-X", + "params": 99.07, + "mAP": 56.3, + "fps": 39.6, + "license": "Apache-2.0" + } + }, + "smallest": { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + } + }, + "crossover": { + "low_param_leader": "rfdetr", + "high_param_leader": "rfdetr" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "rfdetr-n,rfdetr-s,rfdetr-m,rfdetr-l,yolox-nano,yolox-tiny,yolox-s,yolox-m,yolox-l,yolox-x" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, RF-DETR averages 3.9 mAP points higher across 4 paired variants on NVIDIA RTX 5070 Ti. RF-DETR wins 3, YOLOX wins 1.", + "values": { + "mean_delta_pts": 3.9, + "a_wins": 3, + "b_wins": 1 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, YOLOX is 32.2% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 32.2 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, RF-DETR's largest measured variant (RF-DETR-L) reaches 58.6 mAP; YOLOX's (YOLOX-X) reaches 56.3 mAP.", + "values": { + "a": { + "id": "rfdetr-l", + "displayName": "RF-DETR-L", + "params": 33.93, + "mAP": 58.6, + "fps": 25.1, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-x", + "displayName": "YOLOX-X", + "params": 99.07, + "mAP": 56.3, + "fps": 39.6, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, RF-DETR's RF-DETR-N (30.47M, 51.4 mAP, 39.5 FPS) faces YOLOX's YOLOX-Nano (0.91M, 28.8 mAP, 53.6 FPS).", + "values": { + "a": { + "id": "rfdetr-n", + "displayName": "RF-DETR-N", + "params": 30.47, + "mAP": 51.4, + "fps": 39.5, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "RF-DETR fields 4 measured variants, YOLOX fields 6, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 4, + "b_n": 6 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -47.93, + -37.14, + -32.2, + -25.73, + -18.05, + -0.3, + 0.001, + 0.6, + 0.91, + 1.0, + 2, + 3.0, + 3.4, + 3.9, + 4.0, + 5, + 5.06, + 5.6, + 6.0, + 6.9, + 7, + 8, + 8.97, + 9, + 10, + 11, + 12, + 25.1, + 25.33, + 28.8, + 30.3, + 30.47, + 32.11, + 32.2, + 33.69, + 33.93, + 35.5, + 35.8, + 39.5, + 39.6, + 42.8, + 44.3, + 48.2, + 50.0, + 51.4, + 51.7, + 53.6, + 54.21, + 55.1, + 55.4, + 56.3, + 57.3, + 58.6, + 61.6, + 75, + 91.0, + 95, + 99.07, + 100.0, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rtdetr-r18-vs-rtdetrv2-r18.claims.json b/article-pipeline/v2/output/rtdetr-r18-vs-rtdetrv2-r18.claims.json new file mode 100644 index 0000000..59ad5e3 --- /dev/null +++ b/article-pipeline/v2/output/rtdetr-r18-vs-rtdetrv2-r18.claims.json @@ -0,0 +1,344 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T21:56:14.161317+00:00", + "slug": "rtdetr-r18-vs-rtdetrv2-r18", + "meta": { + "type": "vs", + "a": "rtdetr-r18", + "b": "rtdetrv2-r18", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.49787927521150827, + "mAP_50": 0.663704009633566, + "mAP_small": 0.32710231823159713, + "mAP_medium": 0.5484637312981646, + "mAP_large": 0.6407169138607522, + "fps_mean": 47.22, + "fps_p50": 48.89, + "ms_per_image": 21.179, + "inference_ms": 16.645, + "preprocess_ms": 3.452, + "postprocess_ms": 1.082, + "p95_ms": 25.931, + "peak_vram_mb": 168.9, + "peak_ram_mb": 2.0, + "params_millions": 20.18, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:23:32.849123Z" + } + }, + "b": { + "id": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "license": "Apache-2.0", + "releaseDate": "2024-05-01", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5074977298289686, + "mAP_50": 0.6738187878266525, + "mAP_small": 0.3347978553273374, + "mAP_medium": 0.5390239587831667, + "mAP_large": 0.663719218286382, + "fps_mean": 51.15, + "fps_p50": 52.26, + "ms_per_image": 19.552, + "inference_ms": 14.819, + "preprocess_ms": 3.67, + "postprocess_ms": 1.062, + "p95_ms": 23.873, + "peak_vram_mb": 169.1, + "peak_ram_mb": 0.0, + "params_millions": 20.18, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:29:32.230268Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -1.9, + "fps": -7.68, + "params": 0.0, + "flops": 0.0, + "peak_vram": -0.12 + }, + "map_pts": { + "a": 49.8, + "b": 50.7, + "delta": -0.9 + }, + "pair": { + "params_ratio": 1.0, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.0083, + "b_mAP_per_GFLOP": 0.00846 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 49.8, + "fps": 92.5 + }, + "b": { + "mAP": 50.7, + "fps": 97.1 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 49.8, + "fps": 47.2 + }, + "b": { + "mAP": 50.7, + "fps": 51.1 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 49.8, + "fps": 113.4 + }, + "b": { + "mAP": 50.8, + "fps": 117.8 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 49.8, + "fps": 120.6 + }, + "b": { + "mAP": 50.7, + "fps": 125.5 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 49.8, + "fps": 10.2 + }, + "b": { + "mAP": 50.8, + "fps": 10.3 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 49.8, + "fps": 47.2 + }, + "b": { + "mAP": 50.7, + "fps": 51.1 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 49.7, + "fps": 0.8 + }, + "b": { + "mAP": 50.8, + "fps": 0.8 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "RT-DETRv2-R18 measures 0.9 mAP points higher than RT-DETR-R18 on NVIDIA RTX 5070 Ti (PyTorch FP32): 49.8 vs 50.7 mAP@50-95.", + "values": { + "delta_pts": 0.9, + "a_map": 49.8, + "b_map": 50.7 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETRv2-R18 is 7.68% faster: 47.2 vs 51.1 FPS (21.18 vs 19.55 ms/image).", + "values": { + "delta_pct": 7.68, + "a_fps": 47.2, + "b_fps": 51.1 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "ranking_stable", + "kind": "robustness", + "statement": "The verdict holds across 4 runtimes and 3 hardware platforms with verified data.", + "values": { + "n_runtimes": 4, + "n_hardware": 3 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 3, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -7.68, + -1.9, + -0.9, + -0.12, + 0.0, + 0.001, + 0.0083, + 0.0085, + 0.1, + 0.3271, + 0.3348, + 0.4979, + 0.5075, + 0.539, + 0.5485, + 0.6, + 0.6407, + 0.6637, + 0.6738, + 0.8, + 0.83, + 0.846, + 0.9, + 1.0, + 1.062, + 1.082, + 2.0, + 3.0, + 3.452, + 3.67, + 4.0, + 5, + 6, + 7, + 7.68, + 8, + 9, + 10, + 10.2, + 10.3, + 11, + 12, + 14.819, + 16.645, + 19.552, + 20.18, + 21.179, + 23.873, + 25.931, + 32.7102, + 33.4798, + 47.2, + 47.22, + 48.89, + 49.7, + 49.7879, + 49.8, + 50, + 50.7, + 50.7498, + 50.8, + 51.1, + 51.15, + 52.26, + 53.9024, + 54.8464, + 60.0, + 64.0717, + 66.3704, + 66.3719, + 67.3819, + 75, + 80.0, + 90.0, + 92.5, + 95, + 97.1, + 100.0, + 113.4, + 117.8, + 120.6, + 125.5, + 168.9, + 169.1, + 300.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rtdetr-r18-vs-yolov9c.claims.json b/article-pipeline/v2/output/rtdetr-r18-vs-yolov9c.claims.json new file mode 100644 index 0000000..056fa5b --- /dev/null +++ b/article-pipeline/v2/output/rtdetr-r18-vs-yolov9c.claims.json @@ -0,0 +1,391 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T21:56:13.498634+00:00", + "slug": "rtdetr-r18-vs-yolov9c", + "meta": { + "type": "vs", + "a": "rtdetr-r18", + "b": "yolov9c", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.49787927521150827, + "mAP_50": 0.663704009633566, + "mAP_small": 0.32710231823159713, + "mAP_medium": 0.5484637312981646, + "mAP_large": 0.6407169138607522, + "fps_mean": 47.22, + "fps_p50": 48.89, + "ms_per_image": 21.179, + "inference_ms": 16.645, + "preprocess_ms": 3.452, + "postprocess_ms": 1.082, + "p95_ms": 25.931, + "peak_vram_mb": 168.9, + "peak_ram_mb": 2.0, + "params_millions": 20.18, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:23:32.849123Z" + } + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "license": "MIT", + "releaseDate": "2024-02-21", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5709322769395554, + "mAP_50": 0.7383277753630146, + "mAP_small": 0.39292500625214793, + "mAP_medium": 0.6303775404072186, + "mAP_large": 0.7157211255887754, + "fps_mean": 42.33, + "fps_p50": 42.78, + "ms_per_image": 23.626, + "inference_ms": 18.42, + "preprocess_ms": 3.394, + "postprocess_ms": 1.812, + "p95_ms": 29.985, + "peak_vram_mb": 240.9, + "peak_ram_mb": 0.0, + "params_millions": 25.5, + "gflops": 51.75, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:19:12.111885Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -12.8, + "fps": 11.55, + "params": -20.86, + "flops": 15.94, + "peak_vram": -29.89 + }, + "map_pts": { + "a": 49.8, + "b": 57.1, + "delta": -7.3 + }, + "pair": { + "params_ratio": 1.26, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.0083, + "b_mAP_per_GFLOP": 0.01103 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 49.8, + "fps": 92.5 + }, + "b": { + "mAP": 57.1, + "fps": 63.5 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 49.8, + "fps": 47.2 + }, + "b": { + "mAP": 57.1, + "fps": 42.3 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 49.8, + "fps": 113.4 + }, + "b": { + "mAP": 57.2, + "fps": 74.7 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 49.8, + "fps": 120.6 + }, + "b": { + "mAP": 57.1, + "fps": 76.7 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 49.8, + "fps": 10.2 + }, + "b": { + "mAP": 57.1, + "fps": 6.4 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 49.8, + "fps": 47.2 + }, + "b": { + "mAP": 57.1, + "fps": 42.3 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 49.7, + "fps": 0.8 + }, + "b": { + "mAP": 56.4, + "fps": 0.5 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "YOLOv9-C measures 7.3 mAP points higher than RT-DETR-R18 on NVIDIA RTX 5070 Ti (PyTorch FP32): 49.8 vs 57.1 mAP@50-95.", + "values": { + "delta_pts": 7.3, + "a_map": 49.8, + "b_map": 57.1 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "RT-DETR-R18 is 11.55% faster: 47.2 vs 42.3 FPS (21.18 vs 23.63 ms/image).", + "values": { + "delta_pct": 11.55, + "a_fps": 47.2, + "b_fps": 42.3 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "YOLOv9-C extracts 24.75% more mAP per GFLOP (0.0083 vs 0.01103).", + "values": { + "a": 0.0083, + "b": 0.01103, + "delta_pct": 24.75 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "RT-DETR-R18 uses less peak VRAM: 169 vs 241 MB.", + "values": { + "a_vram": 169, + "b_vram": 241 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "YOLOv9-C leads on small objects: 32.7 vs 39.3 mAP_small (16.75% relative gap).", + "values": { + "a": 32.7, + "b": 39.3, + "delta_pct": 16.75 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_stable", + "kind": "robustness", + "statement": "The verdict holds across 4 runtimes and 3 hardware platforms with verified data.", + "values": { + "n_runtimes": 4, + "n_hardware": 3 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -29.89, + -20.86, + -12.8, + -7.3, + 0.0, + 0.001, + 0.0083, + 0.011, + 0.1, + 0.3271, + 0.3929, + 0.4979, + 0.5, + 0.5485, + 0.5709, + 0.6, + 0.6304, + 0.6407, + 0.6637, + 0.7157, + 0.7383, + 0.8, + 0.83, + 1.0, + 1.082, + 1.103, + 1.26, + 1.812, + 2.0, + 3.0, + 3.394, + 3.452, + 4.0, + 5, + 6, + 6.4, + 7, + 7.3, + 8, + 9, + 10, + 10.2, + 11, + 11.55, + 12, + 15.94, + 16.645, + 16.75, + 18.42, + 20.18, + 21.179, + 23.626, + 24.75, + 25.5, + 25.931, + 29.985, + 32.7, + 32.7102, + 39.2925, + 39.3, + 42.3, + 42.33, + 42.78, + 47.2, + 47.22, + 48.89, + 49.7, + 49.7879, + 49.8, + 50.0, + 51.75, + 54.8464, + 56.4, + 57.0932, + 57.1, + 57.2, + 60.0, + 63.0378, + 63.5, + 64.0717, + 66.3704, + 71.5721, + 73.8328, + 74.7, + 75, + 76.7, + 80.0, + 92.5, + 95, + 100.0, + 113.4, + 120.6, + 168.9, + 169.0, + 240.9, + 241.0, + 300.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rtdetr-r34-vs-yolov9c.claims.json b/article-pipeline/v2/output/rtdetr-r34-vs-yolov9c.claims.json new file mode 100644 index 0000000..f718b23 --- /dev/null +++ b/article-pipeline/v2/output/rtdetr-r34-vs-yolov9c.claims.json @@ -0,0 +1,394 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:42.060790+00:00", + "slug": "rtdetr-r34-vs-yolov9c", + "meta": { + "type": "vs", + "a": "rtdetr-r34", + "b": "yolov9c", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5222774052742704, + "mAP_50": 0.6960054462347887, + "mAP_small": 0.34944822231901834, + "mAP_medium": 0.5588794736724594, + "mAP_large": 0.6678097562687866, + "fps_mean": 40.16, + "fps_p50": 41.3, + "ms_per_image": 24.903, + "inference_ms": 20.296, + "preprocess_ms": 3.515, + "postprocess_ms": 1.092, + "p95_ms": 29.392, + "peak_vram_mb": 212.9, + "peak_ram_mb": 0.0, + "params_millions": 31.44, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:23:56.394255Z" + } + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "license": "MIT", + "releaseDate": "2024-02-21", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5709322769395554, + "mAP_50": 0.7383277753630146, + "mAP_small": 0.39292500625214793, + "mAP_medium": 0.6303775404072186, + "mAP_large": 0.7157211255887754, + "fps_mean": 42.33, + "fps_p50": 42.78, + "ms_per_image": 23.626, + "inference_ms": 18.42, + "preprocess_ms": 3.394, + "postprocess_ms": 1.812, + "p95_ms": 29.985, + "peak_vram_mb": 240.9, + "peak_ram_mb": 0.0, + "params_millions": 25.5, + "gflops": 51.75, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:19:12.111885Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -8.52, + "fps": -5.13, + "params": 23.29, + "flops": 75.85, + "peak_vram": -11.62 + }, + "map_pts": { + "a": 52.2, + "b": 57.1, + "delta": -4.9 + }, + "pair": { + "params_ratio": 1.23, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00574, + "b_mAP_per_GFLOP": 0.01103 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 52.2, + "fps": 77.7 + }, + "b": { + "mAP": 57.1, + "fps": 63.5 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 52.2, + "fps": 40.2 + }, + "b": { + "mAP": 57.1, + "fps": 42.3 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 52.3, + "fps": 108.4 + }, + "b": { + "mAP": 57.2, + "fps": 74.7 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 52.2, + "fps": 98.1 + }, + "b": { + "mAP": 57.1, + "fps": 76.7 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 52.2, + "fps": 8.0 + }, + "b": { + "mAP": 57.1, + "fps": 6.4 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 52.2, + "fps": 40.2 + }, + "b": { + "mAP": 57.1, + "fps": 42.3 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 52.2, + "fps": 0.6 + }, + "b": { + "mAP": 56.4, + "fps": 0.5 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "YOLOv9-C measures 4.9 mAP points higher than RT-DETR-R34 on NVIDIA RTX 5070 Ti (PyTorch FP32): 52.2 vs 57.1 mAP@50-95.", + "values": { + "delta_pts": 4.9, + "a_map": 52.2, + "b_map": 57.1 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "YOLOv9-C is 5.13% faster: 40.2 vs 42.3 FPS (24.9 vs 23.63 ms/image).", + "values": { + "delta_pct": 5.13, + "a_fps": 40.2, + "b_fps": 42.3 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "YOLOv9-C extracts 47.96% more mAP per GFLOP (0.00574 vs 0.01103).", + "values": { + "a": 0.00574, + "b": 0.01103, + "delta_pct": 47.96 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "RT-DETR-R34 uses less peak VRAM: 213 vs 241 MB.", + "values": { + "a_vram": 213, + "b_vram": 241 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "YOLOv9-C leads on small objects: 34.9 vs 39.3 mAP_small (11.06% relative gap).", + "values": { + "a": 34.9, + "b": 39.3, + "delta_pct": 11.06 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: ONNX Runtime FP32, runtime: TensorRT FP16, runtime: TensorRT FP32, hardware: NVIDIA Jetson Orin Nano Super 8GB, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:ONNX Runtime FP32", + "runtime:TensorRT FP16", + "runtime:TensorRT FP32", + "hardware:NVIDIA Jetson Orin Nano Super 8GB", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -11.62, + -8.52, + -5.13, + -4.9, + 0.0, + 0.001, + 0.0057, + 0.011, + 0.1, + 0.3494, + 0.3929, + 0.5, + 0.5223, + 0.5589, + 0.5709, + 0.574, + 0.6, + 0.6304, + 0.6678, + 0.696, + 0.7157, + 0.7383, + 1.0, + 1.092, + 1.103, + 1.23, + 1.812, + 2, + 3, + 3.394, + 3.515, + 4, + 4.9, + 5, + 5.13, + 6, + 6.4, + 7, + 8.0, + 9, + 10, + 11, + 11.06, + 12, + 18.42, + 20.296, + 23.29, + 23.626, + 24.903, + 25.5, + 29.392, + 29.985, + 31.44, + 34.9, + 34.9448, + 39.2925, + 39.3, + 40.16, + 40.2, + 41.3, + 42.3, + 42.33, + 42.78, + 47.96, + 50.0, + 51.75, + 52.2, + 52.2277, + 52.3, + 55.8879, + 56.4, + 57.0932, + 57.1, + 57.2, + 60.0, + 63.0378, + 63.5, + 66.781, + 69.6005, + 71.5721, + 73.8328, + 74.7, + 75, + 75.85, + 76.7, + 77.7, + 95, + 98.1, + 100.0, + 108.4, + 212.9, + 213.0, + 240.9, + 241.0, + 300.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rtdetr-r34-vs-yolox-l.claims.json b/article-pipeline/v2/output/rtdetr-r34-vs-yolox-l.claims.json new file mode 100644 index 0000000..ddd02b8 --- /dev/null +++ b/article-pipeline/v2/output/rtdetr-r34-vs-yolox-l.claims.json @@ -0,0 +1,393 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:42.137094+00:00", + "slug": "rtdetr-r34-vs-yolox-l", + "meta": { + "type": "vs", + "a": "rtdetr-r34", + "b": "yolox-l", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "a": { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "license": "Apache-2.0", + "releaseDate": "2023-04-17", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5222774052742704, + "mAP_50": 0.6960054462347887, + "mAP_small": 0.34944822231901834, + "mAP_medium": 0.5588794736724594, + "mAP_large": 0.6678097562687866, + "fps_mean": 40.16, + "fps_p50": 41.3, + "ms_per_image": 24.903, + "inference_ms": 20.296, + "preprocess_ms": 3.515, + "postprocess_ms": 1.092, + "p95_ms": 29.392, + "peak_vram_mb": 212.9, + "peak_ram_mb": 0.0, + "params_millions": 31.44, + "gflops": 0.0, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:23:56.394255Z" + } + }, + "b": { + "id": "yolox-l", + "displayName": "YOLOX-L", + "license": "Apache-2.0", + "releaseDate": "2021-07-18", + "inLibreYOLO": true, + "bench": { + "mAP_50_95": 0.5542460212991265, + "mAP_50": 0.7302189854897115, + "mAP_small": 0.39036786287091835, + "mAP_medium": 0.5988447680065735, + "mAP_large": 0.685177367489107, + "fps_mean": 42.84, + "fps_p50": 42.78, + "ms_per_image": 23.341, + "inference_ms": 13.911, + "preprocess_ms": 6.863, + "postprocess_ms": 2.567, + "p95_ms": 27.837, + "peak_vram_mb": 307.0, + "peak_ram_mb": 0.0, + "params_millions": 54.21, + "gflops": 78.01, + "input_size": 640, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "benchmark_date": "2026-06-16T21:20:38.605152Z" + } + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "deltas_pct": { + "mAP_50_95": -5.77, + "fps": -6.26, + "params": -42.0, + "flops": 16.65, + "peak_vram": -30.65 + }, + "map_pts": { + "a": 52.2, + "b": 55.4, + "delta": -3.2 + }, + "pair": { + "params_ratio": 1.72, + "meaningful": true + }, + "efficiency": { + "a_mAP_per_GFLOP": 0.00574, + "b_mAP_per_GFLOP": 0.0071 + }, + "cross_runtime": [ + { + "runtime": "onnx_fp32", + "display": "ONNX Runtime FP32", + "a": { + "mAP": 52.2, + "fps": 77.7 + }, + "b": { + "mAP": 55.4, + "fps": 50.6 + } + }, + { + "runtime": "pytorch_fp32", + "display": "PyTorch FP32", + "a": { + "mAP": 52.2, + "fps": 40.2 + }, + "b": { + "mAP": 55.4, + "fps": 42.8 + } + }, + { + "runtime": "tensorrt_fp16", + "display": "TensorRT FP16", + "a": { + "mAP": 52.3, + "fps": 108.4 + }, + "b": { + "mAP": 55.4, + "fps": 79.2 + } + }, + { + "runtime": "tensorrt_fp32", + "display": "TensorRT FP32", + "a": { + "mAP": 52.2, + "fps": 98.1 + }, + "b": { + "mAP": 55.4, + "fps": 61.8 + } + } + ], + "cross_hardware": [ + { + "hardware": "jetson_orin", + "display": "NVIDIA Jetson Orin Nano Super 8GB", + "a": { + "mAP": 52.2, + "fps": 8.0 + }, + "b": { + "mAP": 55.4, + "fps": 5.8 + } + }, + { + "hardware": "nvidia_geforce_rtx_5070_ti", + "display": "NVIDIA RTX 5070 Ti", + "a": { + "mAP": 52.2, + "fps": 40.2 + }, + "b": { + "mAP": 55.4, + "fps": 42.8 + } + }, + { + "hardware": "rpi5", + "display": "Raspberry Pi 5", + "a": { + "mAP": 52.2, + "fps": 0.6 + }, + "b": { + "mAP": 53.9, + "fps": 0.4 + } + } + ], + "input_sizes": { + "a": 640, + "b": 640, + "match": true + } + }, + "claims": [ + { + "id": "accuracy_winner", + "kind": "accuracy", + "statement": "YOLOX-L measures 3.2 mAP points higher than RT-DETR-R34 on NVIDIA RTX 5070 Ti (PyTorch FP32): 52.2 vs 55.4 mAP@50-95.", + "values": { + "delta_pts": 3.2, + "a_map": 52.2, + "b_map": 55.4 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_winner", + "kind": "speed", + "statement": "YOLOX-L is 6.26% faster: 40.2 vs 42.8 FPS (24.9 vs 23.34 ms/image).", + "values": { + "delta_pct": 6.26, + "a_fps": 40.2, + "b_fps": 42.8 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "efficiency_winner", + "kind": "efficiency", + "statement": "YOLOX-L extracts 19.15% more mAP per GFLOP (0.00574 vs 0.0071).", + "values": { + "a": 0.00574, + "b": 0.0071, + "delta_pct": 19.15 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "memory_winner", + "kind": "memory", + "statement": "RT-DETR-R34 uses less peak VRAM: 213 vs 307 MB.", + "values": { + "a_vram": 213, + "b_vram": 307 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "small_object_gap", + "kind": "accuracy", + "statement": "YOLOX-L leads on small objects: 34.9 vs 39.0 mAP_small (10.48% relative gap).", + "values": { + "a": 34.9, + "b": 39.0, + "delta_pct": 10.48 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "ranking_flip", + "kind": "robustness", + "statement": "The speed verdict does NOT survive everywhere. It flips on runtime: ONNX Runtime FP32, runtime: TensorRT FP16, runtime: TensorRT FP32, hardware: NVIDIA Jetson Orin Nano Super 8GB, hardware: Raspberry Pi 5. State this; it is the most useful fact in the article.", + "values": { + "flips": [ + "runtime:ONNX Runtime FP32", + "runtime:TensorRT FP16", + "runtime:TensorRT FP32", + "hardware:NVIDIA Jetson Orin Nano Super 8GB", + "hardware:Raspberry Pi 5" + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 6, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -42.0, + -30.65, + -6.26, + -5.77, + -3.2, + 0.0, + 0.001, + 0.0057, + 0.0071, + 0.1, + 0.3494, + 0.3904, + 0.4, + 0.5223, + 0.5542, + 0.5589, + 0.574, + 0.5988, + 0.6, + 0.6678, + 0.6852, + 0.696, + 0.71, + 0.7302, + 1.0, + 1.092, + 1.72, + 2, + 2.567, + 3, + 3.2, + 3.515, + 4, + 5, + 5.8, + 6, + 6.26, + 6.863, + 7, + 8.0, + 9, + 10, + 10.48, + 11, + 12, + 13.911, + 16.65, + 19.15, + 20.296, + 23.341, + 24.903, + 27.837, + 29.392, + 31.44, + 34.9, + 34.9448, + 39.0, + 39.0368, + 40.0, + 40.16, + 40.2, + 41.3, + 42.78, + 42.8, + 42.84, + 50, + 50.6, + 52.2, + 52.2277, + 52.3, + 53.9, + 54.21, + 55.4, + 55.4246, + 55.8879, + 59.8845, + 60.0, + 61.8, + 66.781, + 68.5177, + 69.6005, + 73.0219, + 75, + 77.7, + 78.01, + 79.2, + 95, + 98.1, + 100.0, + 108.4, + 212.9, + 213.0, + 300.0, + 307.0, + 500.0, + 640.0, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rtdetr-vs-rtdetrv2.claims.json b/article-pipeline/v2/output/rtdetr-vs-rtdetrv2.claims.json new file mode 100644 index 0000000..73be9dc --- /dev/null +++ b/article-pipeline/v2/output/rtdetr-vs-rtdetrv2.claims.json @@ -0,0 +1,491 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:18.194363+00:00", + "slug": "rtdetr-vs-rtdetrv2", + "meta": { + "type": "family-vs", + "a": "rtdetr", + "b": "rtdetrv2", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "rtdetr", + "b": "rtdetrv2", + "a_display": "RT-DETR", + "b_display": "RT-DETRv2" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-l", + "displayName": "RT-DETR-L", + "params": 32.93, + "mAP": 55.8, + "fps": 27.7, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "params": 36.59, + "mAP": 53.8, + "fps": 35.3, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "params": 42.89, + "mAP": 55.9, + "fps": 29.8, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-x", + "displayName": "RT-DETR-X", + "params": 67.37, + "mAP": 57.9, + "fps": 21.8, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "params": 20.18, + "mAP": 50.7, + "fps": 51.1, + "license": "Apache-2.0" + }, + { + "id": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "params": 31.44, + "mAP": 53.2, + "fps": 40.4, + "license": "Apache-2.0" + }, + { + "id": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "params": 36.59, + "mAP": 54.8, + "fps": 39.9, + "license": "Apache-2.0" + }, + { + "id": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "params": 42.89, + "mAP": 55.7, + "fps": 32.1, + "license": "Apache-2.0" + }, + { + "id": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 25.7, + "license": "Apache-2.0" + } + ], + "matched_pairs": [ + { + "a": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "params": 20.18, + "mAP": 50.7, + "fps": 51.1, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -0.9, + "fps_delta_pct": -7.63 + }, + { + "a": { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "params": 31.44, + "mAP": 53.2, + "fps": 40.4, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -1.0, + "fps_delta_pct": -0.5 + }, + { + "a": { + "id": "rtdetr-l", + "displayName": "RT-DETR-L", + "params": 32.93, + "mAP": 55.8, + "fps": 27.7, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "params": 31.44, + "mAP": 53.2, + "fps": 40.4, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 2.6, + "fps_delta_pct": -31.44 + }, + { + "a": { + "id": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "params": 36.59, + "mAP": 53.8, + "fps": 35.3, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "params": 36.59, + "mAP": 54.8, + "fps": 39.9, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -1.0, + "fps_delta_pct": -11.53 + }, + { + "a": { + "id": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "params": 42.89, + "mAP": 55.9, + "fps": 29.8, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "params": 42.89, + "mAP": 55.7, + "fps": 32.1, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 0.2, + "fps_delta_pct": -7.17 + }, + { + "a": { + "id": "rtdetr-x", + "displayName": "RT-DETR-X", + "params": 67.37, + "mAP": 57.9, + "fps": 21.8, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 25.7, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 1.1, + "fps_delta_pct": -15.18 + }, + { + "a": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 25.7, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 0.0, + "fps_delta_pct": -8.17 + } + ], + "aggregate": { + "n_pairs": 7, + "mean_mAP_delta_pts": 0.1, + "a_wins": 3, + "b_wins": 3, + "mean_fps_delta_pct": -11.7 + }, + "flagship": { + "a": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 25.7, + "license": "Apache-2.0" + } + }, + "smallest": { + "a": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "params": 20.18, + "mAP": 50.7, + "fps": 51.1, + "license": "Apache-2.0" + } + }, + "crossover": { + "low_param_leader": "rtdetrv2", + "high_param_leader": "rtdetr" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "rtdetr-r18,rtdetr-r34,rtdetr-l,rtdetr-r50m,rtdetr-r50,rtdetr-x,rtdetr-r101,rtdetrv2-r18,rtdetrv2-r34,rtdetrv2-r50m,rtdetrv2-r50,rtdetrv2-r101" + }, + "claims": [ + { + "id": "family_accuracy_tie", + "kind": "accuracy", + "statement": "At matched compute the two families are even: mean gap 0.1 mAP points across 7 pairs (RT-DETR wins 3, RT-DETRv2 wins 3).", + "values": { + "mean_delta_pts": 0.1 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, RT-DETRv2 is 11.7% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 11.7 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_crossover", + "kind": "robustness", + "statement": "The frontier crosses: RT-DETRv2 leads at the small-model end, RT-DETR leads at the large-model end. There is no single winner; the choice depends on your size class.", + "values": { + "low": "rtdetrv2", + "high": "rtdetr" + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, RT-DETR's largest measured variant (RT-DETR-R101) reaches 56.8 mAP; RT-DETRv2's (RT-DETRv2-R101) reaches 56.8 mAP.", + "values": { + "a": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 25.7, + "license": "Apache-2.0" + } + }, + "passed": false, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, RT-DETR's RT-DETR-R18 (20.18M, 49.8 mAP, 47.2 FPS) faces RT-DETRv2's RT-DETRv2-R18 (20.18M, 50.7 mAP, 51.1 FPS).", + "values": { + "a": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + "b": { + "id": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "params": 20.18, + "mAP": 50.7, + "fps": 51.1, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "RT-DETR fields 7 measured variants, RT-DETRv2 fields 5, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 7, + "b_n": 5 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -31.44, + -15.18, + -11.7, + -11.53, + -8.17, + -7.63, + -7.17, + -1.0, + -0.9, + -0.5, + 0.0, + 0.001, + 0.1, + 0.2, + 0.6, + 1, + 1.1, + 2, + 2.6, + 3.0, + 4, + 5.0, + 6, + 7.0, + 8, + 9, + 10.0, + 11, + 11.7, + 12, + 20.0, + 20.18, + 21.8, + 23.6, + 25.7, + 27.7, + 29.8, + 31.44, + 32.1, + 32.93, + 35.3, + 36.59, + 39.9, + 40.2, + 40.4, + 42.89, + 47.2, + 49.8, + 50, + 50.7, + 51.1, + 52.2, + 53.2, + 53.8, + 54.8, + 55.7, + 55.8, + 55.9, + 56.8, + 57.9, + 67.37, + 75, + 76.56, + 95, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rtdetr-vs-yolov9.claims.json b/article-pipeline/v2/output/rtdetr-vs-yolov9.claims.json new file mode 100644 index 0000000..11acde3 --- /dev/null +++ b/article-pipeline/v2/output/rtdetr-vs-yolov9.claims.json @@ -0,0 +1,412 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:17.723280+00:00", + "slug": "rtdetr-vs-yolov9", + "meta": { + "type": "family-vs", + "a": "rtdetr", + "b": "yolov9", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "rtdetr", + "b": "yolov9", + "a_display": "RT-DETR", + "b_display": "YOLOv9" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-l", + "displayName": "RT-DETR-L", + "params": 32.93, + "mAP": 55.8, + "fps": 27.7, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "params": 36.59, + "mAP": 53.8, + "fps": 35.3, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "params": 42.89, + "mAP": 55.9, + "fps": 29.8, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-x", + "displayName": "RT-DETR-X", + "params": 67.37, + "mAP": 57.9, + "fps": 21.8, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "params": 2.04, + "mAP": 41.8, + "fps": 31.9, + "license": "MIT" + }, + { + "id": "yolov9s", + "displayName": "YOLOv9-S", + "params": 7.23, + "mAP": 50.4, + "fps": 30.8, + "license": "MIT" + }, + { + "id": "yolov9m", + "displayName": "YOLOv9-M", + "params": 20.12, + "mAP": 56.1, + "fps": 37.1, + "license": "MIT" + }, + { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + } + ], + "matched_pairs": [ + { + "a": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9m", + "displayName": "YOLOv9-M", + "params": 20.12, + "mAP": 56.1, + "fps": 37.1, + "license": "MIT" + }, + "mAP_delta_pts": -6.3, + "fps_delta_pct": 27.22 + }, + { + "a": { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + }, + "mAP_delta_pts": -4.9, + "fps_delta_pct": -4.96 + }, + { + "a": { + "id": "rtdetr-l", + "displayName": "RT-DETR-L", + "params": 32.93, + "mAP": 55.8, + "fps": 27.7, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + }, + "mAP_delta_pts": -1.3, + "fps_delta_pct": -34.52 + }, + { + "a": { + "id": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "params": 36.59, + "mAP": 53.8, + "fps": 35.3, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + }, + "mAP_delta_pts": -3.3, + "fps_delta_pct": -16.55 + } + ], + "aggregate": { + "n_pairs": 4, + "mean_mAP_delta_pts": -4.0, + "a_wins": 0, + "b_wins": 4, + "mean_fps_delta_pct": -7.2 + }, + "flagship": { + "a": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + } + }, + "smallest": { + "a": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "params": 2.04, + "mAP": 41.8, + "fps": 31.9, + "license": "MIT" + } + }, + "crossover": { + "low_param_leader": "yolov9", + "high_param_leader": "yolov9" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "rtdetr-r18,rtdetr-r34,rtdetr-l,rtdetr-r50m,rtdetr-r50,rtdetr-x,rtdetr-r101,yolov9t,yolov9s,yolov9m,yolov9c" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, YOLOv9 averages 4.0 mAP points higher across 4 paired variants on NVIDIA RTX 5070 Ti. RT-DETR wins 0, YOLOv9 wins 4.", + "values": { + "mean_delta_pts": 4.0, + "a_wins": 0, + "b_wins": 4 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, YOLOv9 is 7.2% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 7.2 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, RT-DETR's largest measured variant (RT-DETR-R101) reaches 56.8 mAP; YOLOv9's (YOLOv9-C) reaches 57.1 mAP.", + "values": { + "a": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + } + }, + "passed": false, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, RT-DETR's RT-DETR-R18 (20.18M, 49.8 mAP, 47.2 FPS) faces YOLOv9's YOLOv9-T (2.04M, 41.8 mAP, 31.9 FPS).", + "values": { + "a": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + "b": { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "params": 2.04, + "mAP": 41.8, + "fps": 31.9, + "license": "MIT" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "RT-DETR fields 7 measured variants, YOLOv9 fields 4, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 7, + "b_n": 4 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 4, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -34.52, + -16.55, + -7.2, + -6.3, + -4.96, + -4.9, + -4.0, + -3.3, + -1.3, + 0.0, + 0.001, + 0.6, + 1, + 2, + 2.04, + 3, + 4.0, + 5, + 6, + 7.0, + 7.2, + 7.23, + 8, + 9, + 10, + 11, + 12, + 20.12, + 20.18, + 21.8, + 23.6, + 25.5, + 27.22, + 27.7, + 29.8, + 30.8, + 31.44, + 31.9, + 32.93, + 35.3, + 36.59, + 37.1, + 40.2, + 41.8, + 42.3, + 42.89, + 47.2, + 49.8, + 50, + 50.4, + 52.2, + 53.8, + 55.8, + 55.9, + 56.1, + 56.8, + 57.1, + 57.9, + 67.37, + 75, + 76.56, + 95, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/rtdetr-vs-yolox.claims.json b/article-pipeline/v2/output/rtdetr-vs-yolox.claims.json new file mode 100644 index 0000000..36cc5ae --- /dev/null +++ b/article-pipeline/v2/output/rtdetr-vs-yolox.claims.json @@ -0,0 +1,498 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:18.038622+00:00", + "slug": "rtdetr-vs-yolox", + "meta": { + "type": "family-vs", + "a": "rtdetr", + "b": "yolox", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "rtdetr", + "b": "yolox", + "a_display": "RT-DETR", + "b_display": "YOLOX" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-l", + "displayName": "RT-DETR-L", + "params": 32.93, + "mAP": 55.8, + "fps": 27.7, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "params": 36.59, + "mAP": 53.8, + "fps": 35.3, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "params": 42.89, + "mAP": 55.9, + "fps": 29.8, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-x", + "displayName": "RT-DETR-X", + "params": 67.37, + "mAP": 57.9, + "fps": 21.8, + "license": "Apache-2.0" + }, + { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + } + ], + "b_frontier": [ + { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + }, + { + "id": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "params": 5.06, + "mAP": 35.5, + "fps": 61.6, + "license": "Apache-2.0" + }, + { + "id": "yolox-s", + "displayName": "YOLOX-S", + "params": 8.97, + "mAP": 44.3, + "fps": 50.0, + "license": "Apache-2.0" + }, + { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + { + "id": "yolox-l", + "displayName": "YOLOX-L", + "params": 54.21, + "mAP": 55.4, + "fps": 42.8, + "license": "Apache-2.0" + }, + { + "id": "yolox-x", + "displayName": "YOLOX-X", + "params": 99.07, + "mAP": 56.3, + "fps": 39.6, + "license": "Apache-2.0" + } + ], + "matched_pairs": [ + { + "a": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": -1.9, + "fps_delta_pct": -2.07 + }, + { + "a": { + "id": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "params": 31.44, + "mAP": 52.2, + "fps": 40.2, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 0.5, + "fps_delta_pct": -16.6 + }, + { + "a": { + "id": "rtdetr-l", + "displayName": "RT-DETR-L", + "params": 32.93, + "mAP": 55.8, + "fps": 27.7, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 4.1, + "fps_delta_pct": -42.53 + }, + { + "a": { + "id": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "params": 36.59, + "mAP": 53.8, + "fps": 35.3, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 2.1, + "fps_delta_pct": -26.76 + }, + { + "a": { + "id": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "params": 42.89, + "mAP": 55.9, + "fps": 29.8, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-l", + "displayName": "YOLOX-L", + "params": 54.21, + "mAP": 55.4, + "fps": 42.8, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 0.5, + "fps_delta_pct": -30.37 + }, + { + "a": { + "id": "rtdetr-x", + "displayName": "RT-DETR-X", + "params": 67.37, + "mAP": 57.9, + "fps": 21.8, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-l", + "displayName": "YOLOX-L", + "params": 54.21, + "mAP": 55.4, + "fps": 42.8, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 2.5, + "fps_delta_pct": -49.07 + }, + { + "a": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-l", + "displayName": "YOLOX-L", + "params": 54.21, + "mAP": 55.4, + "fps": 42.8, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 1.4, + "fps_delta_pct": -44.86 + } + ], + "aggregate": { + "n_pairs": 7, + "mean_mAP_delta_pts": 1.3, + "a_wins": 6, + "b_wins": 1, + "mean_fps_delta_pct": -30.3 + }, + "flagship": { + "a": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-x", + "displayName": "YOLOX-X", + "params": 99.07, + "mAP": 56.3, + "fps": 39.6, + "license": "Apache-2.0" + } + }, + "smallest": { + "a": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + } + }, + "crossover": { + "low_param_leader": "rtdetr", + "high_param_leader": "rtdetr" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "rtdetr-r18,rtdetr-r34,rtdetr-l,rtdetr-r50m,rtdetr-r50,rtdetr-x,rtdetr-r101,yolox-nano,yolox-tiny,yolox-s,yolox-m,yolox-l,yolox-x" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, RT-DETR averages 1.3 mAP points higher across 7 paired variants on NVIDIA RTX 5070 Ti. RT-DETR wins 6, YOLOX wins 1.", + "values": { + "mean_delta_pts": 1.3, + "a_wins": 6, + "b_wins": 1 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, YOLOX is 30.3% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 30.3 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, RT-DETR's largest measured variant (RT-DETR-R101) reaches 56.8 mAP; YOLOX's (YOLOX-X) reaches 56.3 mAP.", + "values": { + "a": { + "id": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "params": 76.56, + "mAP": 56.8, + "fps": 23.6, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-x", + "displayName": "YOLOX-X", + "params": 99.07, + "mAP": 56.3, + "fps": 39.6, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, RT-DETR's RT-DETR-R18 (20.18M, 49.8 mAP, 47.2 FPS) faces YOLOX's YOLOX-Nano (0.91M, 28.8 mAP, 53.6 FPS).", + "values": { + "a": { + "id": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "params": 20.18, + "mAP": 49.8, + "fps": 47.2, + "license": "Apache-2.0" + }, + "b": { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "RT-DETR fields 7 measured variants, YOLOX fields 6, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 7, + "b_n": 6 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -49.07, + -44.86, + -42.53, + -30.37, + -30.3, + -26.76, + -16.6, + -2.07, + -1.9, + 0.001, + 0.5, + 0.6, + 0.91, + 1.0, + 1.3, + 1.4, + 2, + 2.1, + 2.5, + 3, + 4, + 4.1, + 5, + 5.06, + 6.0, + 7.0, + 8, + 8.97, + 9, + 10, + 11, + 12, + 20.18, + 21.8, + 23.6, + 25.33, + 27.7, + 28.8, + 29.8, + 30.3, + 31.44, + 32.93, + 35.3, + 35.5, + 36.59, + 39.6, + 40.2, + 42.8, + 42.89, + 44.3, + 47.2, + 48.2, + 49.8, + 50.0, + 51.7, + 52.2, + 53.6, + 53.8, + 54.21, + 55.4, + 55.8, + 55.9, + 56.3, + 56.8, + 57.9, + 61.6, + 67.37, + 75, + 76.56, + 91.0, + 95, + 99.07, + 100.0, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/tensorrt-fp16-vs-pytorch-fp32-jetson-orin.claims.json b/article-pipeline/v2/output/tensorrt-fp16-vs-pytorch-fp32-jetson-orin.claims.json new file mode 100644 index 0000000..0cf69af --- /dev/null +++ b/article-pipeline/v2/output/tensorrt-fp16-vs-pytorch-fp32-jetson-orin.claims.json @@ -0,0 +1,961 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T21:56:12.625770+00:00", + "slug": "tensorrt-fp16-vs-pytorch-fp32-jetson-orin", + "meta": { + "type": "runtime-guide", + "hardware": "jetson_orin", + "baseline": "pytorch_fp32", + "target": "tensorrt_fp16" + }, + "facts": { + "hardware": "jetson_orin", + "hardware_display": "NVIDIA Jetson Orin Nano Super 8GB", + "baseline": "pytorch_fp32", + "baseline_display": "PyTorch FP32", + "target": "tensorrt_fp16", + "target_display": "TensorRT FP16", + "n_models": 55, + "rows": [ + { + "model": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "family": "rtdetrv2", + "baseline_fps": 2.8, + "target_fps": 17.3, + "speedup_x": 6.21, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.8, + "target_mAP": 56.8 + }, + { + "model": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "family": "rtdetr", + "baseline_fps": 2.8, + "target_fps": 17.0, + "speedup_x": 6.12, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.8, + "target_mAP": 56.8 + }, + { + "model": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "family": "rtdetrv2", + "baseline_fps": 5.1, + "target_fps": 29.7, + "speedup_x": 5.85, + "mAP_delta_pts": 0.0, + "baseline_mAP": 54.8, + "target_mAP": 54.8 + }, + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "baseline_fps": 15.4, + "target_fps": 89.1, + "speedup_x": 5.78, + "mAP_delta_pts": 0.0, + "baseline_mAP": 27.5, + "target_mAP": 27.5 + }, + { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "family": "rtdetr", + "baseline_fps": 5.0, + "target_fps": 29.1, + "speedup_x": 5.78, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.8, + "target_mAP": 53.8 + }, + { + "model": "rtdetr-x", + "displayName": "RT-DETR-X", + "family": "rtdetr", + "baseline_fps": 3.0, + "target_fps": 17.0, + "speedup_x": 5.74, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.9, + "target_mAP": 57.9 + }, + { + "model": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "family": "rtdetrv2", + "baseline_fps": 4.3, + "target_fps": 24.2, + "speedup_x": 5.66, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.7, + "target_mAP": 55.7 + }, + { + "model": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "family": "rtdetr", + "baseline_fps": 4.3, + "target_fps": 23.6, + "speedup_x": 5.54, + "mAP_delta_pts": -0.2, + "baseline_mAP": 55.9, + "target_mAP": 55.7 + }, + { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "baseline_fps": 2.9, + "target_fps": 14.2, + "speedup_x": 4.96, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.4, + "target_mAP": 61.4 + }, + { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "family": "rtdetr", + "baseline_fps": 4.9, + "target_fps": 24.1, + "speedup_x": 4.96, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.8, + "target_mAP": 55.8 + }, + { + "model": "deim-x", + "displayName": "DEIM-X", + "family": "deim", + "baseline_fps": 2.9, + "target_fps": 14.1, + "speedup_x": 4.93, + "mAP_delta_pts": 0.0, + "baseline_mAP": 59.6, + "target_mAP": 59.6 + }, + { + "model": "rtdetrv4-x", + "displayName": "RT-DETRv4-X", + "family": "rtdetrv4", + "baseline_fps": 2.8, + "target_fps": 13.8, + "speedup_x": 4.85, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.0, + "target_mAP": 60.0 + }, + { + "model": "deimv2-femto", + "displayName": "DEIMv2-Femto", + "family": "deimv2", + "baseline_fps": 14.8, + "target_fps": 65.7, + "speedup_x": 4.43, + "mAP_delta_pts": 0.0, + "baseline_mAP": 34.5, + "target_mAP": 34.5 + }, + { + "model": "yolox-x", + "displayName": "YOLOX-X", + "family": "yolox", + "baseline_fps": 3.5, + "target_fps": 14.6, + "speedup_x": 4.22, + "mAP_delta_pts": -0.1, + "baseline_mAP": 56.3, + "target_mAP": 56.2 + }, + { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "baseline_fps": 8.1, + "target_fps": 32.7, + "speedup_x": 4.03, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.2, + "target_mAP": 53.2 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "family": "rtdetr", + "baseline_fps": 8.0, + "target_fps": 31.9, + "speedup_x": 4.01, + "mAP_delta_pts": 0.1, + "baseline_mAP": 52.2, + "target_mAP": 52.3 + }, + { + "model": "dfine-l", + "displayName": "D-FINE-L", + "family": "dfine", + "baseline_fps": 4.7, + "target_fps": 18.6, + "speedup_x": 3.95, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.0, + "target_mAP": 60.0 + }, + { + "model": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "family": "rtdetrv2", + "baseline_fps": 10.3, + "target_fps": 39.9, + "speedup_x": 3.89, + "mAP_delta_pts": 0.0, + "baseline_mAP": 50.8, + "target_mAP": 50.8 + }, + { + "model": "deim-l", + "displayName": "DEIM-L", + "family": "deim", + "baseline_fps": 4.8, + "target_fps": 18.5, + "speedup_x": 3.88, + "mAP_delta_pts": 0.1, + "baseline_mAP": 57.8, + "target_mAP": 57.9 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "baseline_fps": 6.4, + "target_fps": 24.9, + "speedup_x": 3.88, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.1, + "target_mAP": 57.1 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "family": "rtdetr", + "baseline_fps": 10.2, + "target_fps": 38.8, + "speedup_x": 3.82, + "mAP_delta_pts": -0.1, + "baseline_mAP": 49.8, + "target_mAP": 49.7 + }, + { + "model": "rtdetrv4-l", + "displayName": "RT-DETRv4-L", + "family": "rtdetrv4", + "baseline_fps": 4.7, + "target_fps": 18.1, + "speedup_x": 3.82, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "deim-n", + "displayName": "DEIM-N", + "family": "deim", + "baseline_fps": 11.3, + "target_fps": 41.8, + "speedup_x": 3.7, + "mAP_delta_pts": 0.0, + "baseline_mAP": 46.8, + "target_mAP": 46.8 + }, + { + "model": "deimv2-n", + "displayName": "DEIMv2-N", + "family": "deimv2", + "baseline_fps": 11.1, + "target_fps": 41.2, + "speedup_x": 3.7, + "mAP_delta_pts": -0.1, + "baseline_mAP": 46.7, + "target_mAP": 46.6 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "family": "yolox", + "baseline_fps": 5.8, + "target_fps": 21.6, + "speedup_x": 3.7, + "mAP_delta_pts": 0.1, + "baseline_mAP": 55.4, + "target_mAP": 55.5 + }, + { + "model": "dfine-n", + "displayName": "D-FINE-N", + "family": "dfine", + "baseline_fps": 11.7, + "target_fps": 42.1, + "speedup_x": 3.6, + "mAP_delta_pts": 0.0, + "baseline_mAP": 45.8, + "target_mAP": 45.8 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "baseline_fps": 9.9, + "target_fps": 34.3, + "speedup_x": 3.47, + "mAP_delta_pts": 0.0, + "baseline_mAP": 41.8, + "target_mAP": 41.8 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "family": "dfine", + "baseline_fps": 7.0, + "target_fps": 23.7, + "speedup_x": 3.39, + "mAP_delta_pts": 0.1, + "baseline_mAP": 57.8, + "target_mAP": 57.9 + }, + { + "model": "deim-m", + "displayName": "DEIM-M", + "family": "deim", + "baseline_fps": 7.0, + "target_fps": 23.5, + "speedup_x": 3.38, + "mAP_delta_pts": 0.1, + "baseline_mAP": 55.4, + "target_mAP": 55.5 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "baseline_fps": 9.9, + "target_fps": 33.2, + "speedup_x": 3.36, + "mAP_delta_pts": 0.0, + "baseline_mAP": 50.4, + "target_mAP": 50.4 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "family": "rtdetrv4", + "baseline_fps": 7.0, + "target_fps": 23.1, + "speedup_x": 3.3, + "mAP_delta_pts": 0.1, + "baseline_mAP": 56.5, + "target_mAP": 56.6 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "baseline_fps": 8.0, + "target_fps": 25.8, + "speedup_x": 3.22, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.1, + "target_mAP": 56.1 + }, + { + "model": "deimv2-pico", + "displayName": "DEIMv2-Pico", + "family": "deimv2", + "baseline_fps": 14.1, + "target_fps": 44.5, + "speedup_x": 3.16, + "mAP_delta_pts": 0.1, + "baseline_mAP": 42.2, + "target_mAP": 42.3 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "baseline_fps": 9.1, + "target_fps": 28.0, + "speedup_x": 3.08, + "mAP_delta_pts": 0.1, + "baseline_mAP": 51.7, + "target_mAP": 51.8 + }, + { + "model": "rtdetrv4-s", + "displayName": "RT-DETRv4-S", + "family": "rtdetrv4", + "baseline_fps": 10.1, + "target_fps": 29.6, + "speedup_x": 2.94, + "mAP_delta_pts": 0.1, + "baseline_mAP": 52.8, + "target_mAP": 52.9 + }, + { + "model": "deim-s", + "displayName": "DEIM-S", + "family": "deim", + "baseline_fps": 10.2, + "target_fps": 29.9, + "speedup_x": 2.93, + "mAP_delta_pts": -0.1, + "baseline_mAP": 52.1, + "target_mAP": 52.0 + }, + { + "model": "dfine-s", + "displayName": "D-FINE-S", + "family": "dfine", + "baseline_fps": 10.3, + "target_fps": 30.2, + "speedup_x": 2.92, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.4, + "target_mAP": 53.4 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "baseline_fps": 13.8, + "target_fps": 39.6, + "speedup_x": 2.86, + "mAP_delta_pts": 0.0, + "baseline_mAP": 30.4, + "target_mAP": 30.4 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "baseline_fps": 18.4, + "target_fps": 51.0, + "speedup_x": 2.78, + "mAP_delta_pts": 0.0, + "baseline_mAP": 28.8, + "target_mAP": 28.8 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 12.5, + "target_fps": 32.4, + "speedup_x": 2.6, + "mAP_delta_pts": 0.0, + "baseline_mAP": 37.9, + "target_mAP": 37.9 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "baseline_fps": 20.3, + "target_fps": 51.6, + "speedup_x": 2.54, + "mAP_delta_pts": 0.0, + "baseline_mAP": 35.5, + "target_mAP": 35.5 + }, + { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "family": "rfdetr", + "baseline_fps": 10.2, + "target_fps": 25.4, + "speedup_x": 2.5, + "mAP_delta_pts": -0.1, + "baseline_mAP": 51.4, + "target_mAP": 51.3 + }, + { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "family": "rfdetr", + "baseline_fps": 6.9, + "target_fps": 15.5, + "speedup_x": 2.25, + "mAP_delta_pts": 0.1, + "baseline_mAP": 55.1, + "target_mAP": 55.2 + }, + { + "model": "rfdetr-m", + "displayName": "RF-DETR-M", + "family": "rfdetr", + "baseline_fps": 5.7, + "target_fps": 12.5, + "speedup_x": 2.19, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.4, + "target_mAP": 57.4 + }, + { + "model": "rfdetr-l", + "displayName": "RF-DETR-L", + "family": "rfdetr", + "baseline_fps": 4.0, + "target_fps": 8.4, + "speedup_x": 2.08, + "mAP_delta_pts": 0.1, + "baseline_mAP": 58.6, + "target_mAP": 58.7 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "baseline_fps": 16.9, + "target_fps": 34.9, + "speedup_x": 2.07, + "mAP_delta_pts": -0.1, + "baseline_mAP": 44.3, + "target_mAP": 44.2 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "baseline_fps": 2.7, + "target_fps": 5.4, + "speedup_x": 2.01, + "mAP_delta_pts": -0.6, + "baseline_mAP": 61.3, + "target_mAP": 60.7 + }, + { + "model": "ec-x", + "displayName": "ECDet-X", + "family": "ec", + "baseline_fps": 2.9, + "target_fps": 5.7, + "speedup_x": 2.0, + "mAP_delta_pts": -0.1, + "baseline_mAP": 61.1, + "target_mAP": 61.0 + }, + { + "model": "deimv2-l", + "displayName": "DEIMv2-L", + "family": "deimv2", + "baseline_fps": 3.3, + "target_fps": 6.2, + "speedup_x": 1.89, + "mAP_delta_pts": -0.1, + "baseline_mAP": 58.6, + "target_mAP": 58.5 + }, + { + "model": "deimv2-s", + "displayName": "DEIMv2-S", + "family": "deimv2", + "baseline_fps": 5.7, + "target_fps": 10.5, + "speedup_x": 1.84, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.0, + "target_mAP": 53.0 + }, + { + "model": "ec-l", + "displayName": "ECDet-L", + "family": "ec", + "baseline_fps": 3.4, + "target_fps": 6.2, + "speedup_x": 1.83, + "mAP_delta_pts": -0.1, + "baseline_mAP": 60.1, + "target_mAP": 60.0 + }, + { + "model": "ec-s", + "displayName": "ECDet-S", + "family": "ec", + "baseline_fps": 6.0, + "target_fps": 10.6, + "speedup_x": 1.77, + "mAP_delta_pts": 0.0, + "baseline_mAP": 54.3, + "target_mAP": 54.3 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 11.1, + "target_fps": 19.1, + "speedup_x": 1.72, + "mAP_delta_pts": -0.1, + "baseline_mAP": 44.1, + "target_mAP": 44.0 + }, + { + "model": "deimv2-m", + "displayName": "DEIMv2-M", + "family": "deimv2", + "baseline_fps": 4.7, + "target_fps": 7.9, + "speedup_x": 1.69, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.0, + "target_mAP": 56.0 + }, + { + "model": "ec-m", + "displayName": "ECDet-M", + "family": "ec", + "baseline_fps": 5.0, + "target_fps": 8.3, + "speedup_x": 1.68, + "mAP_delta_pts": -0.1, + "baseline_mAP": 58.4, + "target_mAP": 58.3 + } + ], + "speedup": { + "median_x": 3.39, + "min_x": 1.68, + "max_x": 6.21 + }, + "family_median_speedup": { + "rtdetrv2": 5.66, + "rtdetr": 5.54, + "deimv2": 3.16, + "dfine": 3.6, + "deim": 3.7, + "rtdetrv4": 3.82, + "yolox": 3.08, + "yolov9": 3.47, + "picodet": 2.6, + "rfdetr": 2.25, + "ec": 1.83 + }, + "accuracy_losers": [ + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "baseline_fps": 2.7, + "target_fps": 5.4, + "speedup_x": 2.01, + "mAP_delta_pts": -0.6, + "baseline_mAP": 61.3, + "target_mAP": 60.7 + } + ], + "speed_regressions": [] + }, + "claims": [ + { + "id": "median_speedup", + "kind": "speed", + "statement": "Across 55 models on NVIDIA Jetson Orin Nano Super 8GB, TensorRT FP16 gives a median 3.39x over PyTorch FP32 (range 1.68x to 6.21x).", + "values": { + "median_x": 3.39, + "min_x": 1.68, + "max_x": 6.21 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "top_gainer", + "kind": "speed", + "statement": "Biggest gain: RT-DETRv2-R101 at 6.21x (2.8 to 17.3 FPS).", + "values": { + "model": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "family": "rtdetrv2", + "baseline_fps": 2.8, + "target_fps": 17.3, + "speedup_x": 6.21, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.8, + "target_mAP": 56.8 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "family_spread", + "kind": "speed", + "statement": "Family medians differ: rtdetrv2 gains 5.66x, ec only 1.83x. Conversion gain is architecture-dependent.", + "values": { + "best": [ + "rtdetrv2", + 5.66 + ], + "worst": [ + "ec", + 1.83 + ] + }, + "passed": true, + "strength": "normal" + }, + { + "id": "accuracy_cost", + "kind": "accuracy", + "statement": "1 of 55 models lose 0.5+ mAP points under TensorRT FP16: DEIMv2-X (-0.6)", + "values": { + "losers": [ + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "baseline_fps": 2.7, + "target_fps": 5.4, + "speedup_x": 2.01, + "mAP_delta_pts": -0.6, + "baseline_mAP": 61.3, + "target_mAP": 60.7 + } + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 4, + "required": 3, + "reason": null + }, + "allowed_numbers": [ + -0.6, + -0.2, + -0.1, + 0.0, + 0.001, + 0.1, + 0.6, + 1, + 1.68, + 1.69, + 1.72, + 1.77, + 1.83, + 1.84, + 1.89, + 2.0, + 2.01, + 2.07, + 2.08, + 2.19, + 2.25, + 2.5, + 2.54, + 2.6, + 2.7, + 2.78, + 2.8, + 2.86, + 2.9, + 2.92, + 2.93, + 2.94, + 3.0, + 3.08, + 3.16, + 3.22, + 3.3, + 3.36, + 3.38, + 3.39, + 3.4, + 3.47, + 3.5, + 3.6, + 3.7, + 3.82, + 3.88, + 3.89, + 3.95, + 4.0, + 4.01, + 4.03, + 4.22, + 4.3, + 4.43, + 4.7, + 4.8, + 4.85, + 4.9, + 4.93, + 4.96, + 5.0, + 5.1, + 5.4, + 5.54, + 5.66, + 5.7, + 5.74, + 5.78, + 5.8, + 5.85, + 6.0, + 6.12, + 6.2, + 6.21, + 6.4, + 6.9, + 7.0, + 7.9, + 8.0, + 8.1, + 8.3, + 8.4, + 9, + 9.1, + 9.9, + 10.0, + 10.1, + 10.2, + 10.3, + 10.5, + 10.6, + 11, + 11.1, + 11.3, + 11.7, + 12, + 12.5, + 13.8, + 14.1, + 14.2, + 14.6, + 14.8, + 15.4, + 15.5, + 16.9, + 17.0, + 17.3, + 18.1, + 18.4, + 18.5, + 18.6, + 19.1, + 20.3, + 21.6, + 23.1, + 23.5, + 23.6, + 23.7, + 24.1, + 24.2, + 24.9, + 25.4, + 25.8, + 27.5, + 28.0, + 28.8, + 29.1, + 29.6, + 29.7, + 29.9, + 30.2, + 30.4, + 31.9, + 32.4, + 32.7, + 33.2, + 34.3, + 34.5, + 34.9, + 35.5, + 37.9, + 38.8, + 39.6, + 39.9, + 41.2, + 41.8, + 42.1, + 42.2, + 42.3, + 44.0, + 44.1, + 44.2, + 44.3, + 44.5, + 45.8, + 46.6, + 46.7, + 46.8, + 49.7, + 49.8, + 50, + 50.4, + 50.8, + 51.0, + 51.3, + 51.4, + 51.6, + 51.7, + 51.8, + 52.0, + 52.1, + 52.2, + 52.3, + 52.8, + 52.9, + 53.0, + 53.2, + 53.4, + 53.8, + 54.3, + 54.8, + 55.0, + 55.1, + 55.2, + 55.4, + 55.5, + 55.7, + 55.8, + 55.9, + 56.0, + 56.1, + 56.2, + 56.3, + 56.5, + 56.6, + 56.8, + 57.1, + 57.4, + 57.8, + 57.9, + 58.3, + 58.4, + 58.5, + 58.6, + 58.7, + 59.6, + 60.0, + 60.1, + 60.7, + 61.0, + 61.1, + 61.3, + 61.4, + 65.7, + 75, + 89.1, + 95, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/tensorrt-fp16-vs-pytorch-fp32-nvidia-geforce-rtx-5070-ti.claims.json b/article-pipeline/v2/output/tensorrt-fp16-vs-pytorch-fp32-nvidia-geforce-rtx-5070-ti.claims.json new file mode 100644 index 0000000..2ab5343 --- /dev/null +++ b/article-pipeline/v2/output/tensorrt-fp16-vs-pytorch-fp32-nvidia-geforce-rtx-5070-ti.claims.json @@ -0,0 +1,1042 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:28.325973+00:00", + "slug": "tensorrt-fp16-vs-pytorch-fp32-nvidia-geforce-rtx-5070-ti", + "meta": { + "type": "runtime-guide", + "hardware": "nvidia_geforce_rtx_5070_ti", + "baseline": "pytorch_fp32", + "target": "tensorrt_fp16" + }, + "facts": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "baseline": "pytorch_fp32", + "baseline_display": "PyTorch FP32", + "target": "tensorrt_fp16", + "target_display": "TensorRT FP16", + "n_models": 51, + "rows": [ + { + "model": "deim-x", + "displayName": "DEIM-X", + "family": "deim", + "baseline_fps": 15.0, + "target_fps": 70.7, + "speedup_x": 4.72, + "mAP_delta_pts": -0.1, + "baseline_mAP": 59.6, + "target_mAP": 59.5 + }, + { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "family": "rtdetr", + "baseline_fps": 27.7, + "target_fps": 122.0, + "speedup_x": 4.4, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.8, + "target_mAP": 55.8 + }, + { + "model": "rtdetrv4-x", + "displayName": "RT-DETRv4-X", + "family": "rtdetrv4", + "baseline_fps": 16.6, + "target_fps": 71.0, + "speedup_x": 4.27, + "mAP_delta_pts": -0.1, + "baseline_mAP": 60.0, + "target_mAP": 59.9 + }, + { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "baseline_fps": 18.1, + "target_fps": 73.0, + "speedup_x": 4.02, + "mAP_delta_pts": 0.1, + "baseline_mAP": 61.4, + "target_mAP": 61.5 + }, + { + "model": "rtdetr-x", + "displayName": "RT-DETR-X", + "family": "rtdetr", + "baseline_fps": 21.8, + "target_fps": 82.3, + "speedup_x": 3.78, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.9, + "target_mAP": 57.9 + }, + { + "model": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "family": "rtdetr", + "baseline_fps": 23.6, + "target_fps": 88.0, + "speedup_x": 3.74, + "mAP_delta_pts": -0.1, + "baseline_mAP": 56.8, + "target_mAP": 56.7 + }, + { + "model": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "family": "rtdetrv2", + "baseline_fps": 25.7, + "target_fps": 87.4, + "speedup_x": 3.4, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.8, + "target_mAP": 56.8 + }, + { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "baseline_fps": 40.4, + "target_fps": 132.9, + "speedup_x": 3.29, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.2, + "target_mAP": 53.2 + }, + { + "model": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "family": "rtdetr", + "baseline_fps": 29.8, + "target_fps": 97.4, + "speedup_x": 3.27, + "mAP_delta_pts": -0.1, + "baseline_mAP": 55.9, + "target_mAP": 55.8 + }, + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "baseline_fps": 40.4, + "target_fps": 132.1, + "speedup_x": 3.27, + "mAP_delta_pts": -1.7, + "baseline_mAP": 27.5, + "target_mAP": 25.8 + }, + { + "model": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "family": "rtdetrv2", + "baseline_fps": 32.1, + "target_fps": 103.6, + "speedup_x": 3.23, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.7, + "target_mAP": 55.7 + }, + { + "model": "dfine-l", + "displayName": "D-FINE-L", + "family": "dfine", + "baseline_fps": 21.9, + "target_fps": 69.3, + "speedup_x": 3.17, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.0, + "target_mAP": 60.0 + }, + { + "model": "rtdetrv4-l", + "displayName": "RT-DETRv4-L", + "family": "rtdetrv4", + "baseline_fps": 24.7, + "target_fps": 78.0, + "speedup_x": 3.15, + "mAP_delta_pts": -0.1, + "baseline_mAP": 57.8, + "target_mAP": 57.7 + }, + { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "family": "rtdetr", + "baseline_fps": 35.3, + "target_fps": 110.1, + "speedup_x": 3.12, + "mAP_delta_pts": 0.1, + "baseline_mAP": 53.8, + "target_mAP": 53.9 + }, + { + "model": "deim-m", + "displayName": "DEIM-M", + "family": "deim", + "baseline_fps": 28.4, + "target_fps": 83.3, + "speedup_x": 2.94, + "mAP_delta_pts": -0.1, + "baseline_mAP": 55.5, + "target_mAP": 55.4 + }, + { + "model": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "family": "rtdetrv2", + "baseline_fps": 39.9, + "target_fps": 113.0, + "speedup_x": 2.83, + "mAP_delta_pts": -0.1, + "baseline_mAP": 54.8, + "target_mAP": 54.7 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "family": "rtdetr", + "baseline_fps": 40.2, + "target_fps": 108.4, + "speedup_x": 2.7, + "mAP_delta_pts": 0.1, + "baseline_mAP": 52.2, + "target_mAP": 52.3 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "family": "dfine", + "baseline_fps": 28.8, + "target_fps": 74.5, + "speedup_x": 2.59, + "mAP_delta_pts": -0.2, + "baseline_mAP": 57.8, + "target_mAP": 57.6 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "family": "rtdetrv4", + "baseline_fps": 33.6, + "target_fps": 86.7, + "speedup_x": 2.58, + "mAP_delta_pts": -0.1, + "baseline_mAP": 56.5, + "target_mAP": 56.4 + }, + { + "model": "dfine-s", + "displayName": "D-FINE-S", + "family": "dfine", + "baseline_fps": 34.2, + "target_fps": 86.5, + "speedup_x": 2.53, + "mAP_delta_pts": -0.2, + "baseline_mAP": 53.4, + "target_mAP": 53.2 + }, + { + "model": "deim-s", + "displayName": "DEIM-S", + "family": "deim", + "baseline_fps": 33.4, + "target_fps": 82.4, + "speedup_x": 2.47, + "mAP_delta_pts": -0.2, + "baseline_mAP": 52.1, + "target_mAP": 51.9 + }, + { + "model": "dfine-n", + "displayName": "D-FINE-N", + "family": "dfine", + "baseline_fps": 32.5, + "target_fps": 79.7, + "speedup_x": 2.45, + "mAP_delta_pts": 0.0, + "baseline_mAP": 45.8, + "target_mAP": 45.8 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "family": "rtdetr", + "baseline_fps": 47.2, + "target_fps": 113.4, + "speedup_x": 2.4, + "mAP_delta_pts": 0.0, + "baseline_mAP": 49.8, + "target_mAP": 49.8 + }, + { + "model": "rtdetrv4-s", + "displayName": "RT-DETRv4-S", + "family": "rtdetrv4", + "baseline_fps": 39.8, + "target_fps": 92.3, + "speedup_x": 2.32, + "mAP_delta_pts": -0.1, + "baseline_mAP": 52.9, + "target_mAP": 52.8 + }, + { + "model": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "family": "rtdetrv2", + "baseline_fps": 51.1, + "target_fps": 117.8, + "speedup_x": 2.3, + "mAP_delta_pts": 0.1, + "baseline_mAP": 50.7, + "target_mAP": 50.8 + }, + { + "model": "deimv2-n", + "displayName": "DEIMv2-N", + "family": "deimv2", + "baseline_fps": 35.1, + "target_fps": 78.0, + "speedup_x": 2.22, + "mAP_delta_pts": 0.0, + "baseline_mAP": 46.7, + "target_mAP": 46.7 + }, + { + "model": "deim-n", + "displayName": "DEIM-N", + "family": "deim", + "baseline_fps": 37.0, + "target_fps": 81.3, + "speedup_x": 2.2, + "mAP_delta_pts": -0.2, + "baseline_mAP": 46.8, + "target_mAP": 46.6 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "baseline_fps": 17.4, + "target_fps": 37.9, + "speedup_x": 2.18, + "mAP_delta_pts": -0.5, + "baseline_mAP": 61.3, + "target_mAP": 60.8 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "baseline_fps": 30.8, + "target_fps": 63.8, + "speedup_x": 2.08, + "mAP_delta_pts": 0.1, + "baseline_mAP": 50.4, + "target_mAP": 50.5 + }, + { + "model": "deimv2-s", + "displayName": "DEIMv2-S", + "family": "deimv2", + "baseline_fps": 24.3, + "target_fps": 50.4, + "speedup_x": 2.08, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.0, + "target_mAP": 53.0 + }, + { + "model": "rfdetr-m", + "displayName": "RF-DETR-M", + "family": "rfdetr", + "baseline_fps": 30.3, + "target_fps": 62.9, + "speedup_x": 2.07, + "mAP_delta_pts": 0.1, + "baseline_mAP": 57.3, + "target_mAP": 57.4 + }, + { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "family": "rfdetr", + "baseline_fps": 35.8, + "target_fps": 71.6, + "speedup_x": 2.0, + "mAP_delta_pts": 0.2, + "baseline_mAP": 55.1, + "target_mAP": 55.3 + }, + { + "model": "deimv2-l", + "displayName": "DEIMv2-L", + "family": "deimv2", + "baseline_fps": 19.7, + "target_fps": 38.7, + "speedup_x": 1.96, + "mAP_delta_pts": -0.1, + "baseline_mAP": 58.6, + "target_mAP": 58.5 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "baseline_fps": 37.1, + "target_fps": 72.3, + "speedup_x": 1.95, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.1, + "target_mAP": 56.1 + }, + { + "model": "rfdetr-l", + "displayName": "RF-DETR-L", + "family": "rfdetr", + "baseline_fps": 25.1, + "target_fps": 47.6, + "speedup_x": 1.9, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.6, + "target_mAP": 58.6 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "baseline_fps": 53.6, + "target_fps": 100.0, + "speedup_x": 1.87, + "mAP_delta_pts": -0.1, + "baseline_mAP": 28.8, + "target_mAP": 28.7 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "baseline_fps": 31.9, + "target_fps": 59.0, + "speedup_x": 1.85, + "mAP_delta_pts": 0.0, + "baseline_mAP": 41.8, + "target_mAP": 41.8 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "family": "yolox", + "baseline_fps": 42.8, + "target_fps": 79.2, + "speedup_x": 1.85, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.4, + "target_mAP": 55.4 + }, + { + "model": "ec-m", + "displayName": "ECDet-M", + "family": "ec", + "baseline_fps": 26.5, + "target_fps": 48.9, + "speedup_x": 1.84, + "mAP_delta_pts": -0.2, + "baseline_mAP": 58.4, + "target_mAP": 58.2 + }, + { + "model": "yolox-x", + "displayName": "YOLOX-X", + "family": "yolox", + "baseline_fps": 39.6, + "target_fps": 72.1, + "speedup_x": 1.82, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.3, + "target_mAP": 56.3 + }, + { + "model": "ec-x", + "displayName": "ECDet-X", + "family": "ec", + "baseline_fps": 23.0, + "target_fps": 41.3, + "speedup_x": 1.8, + "mAP_delta_pts": -0.1, + "baseline_mAP": 61.1, + "target_mAP": 61.0 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "baseline_fps": 42.3, + "target_fps": 74.7, + "speedup_x": 1.76, + "mAP_delta_pts": 0.1, + "baseline_mAP": 57.1, + "target_mAP": 57.2 + }, + { + "model": "ec-l", + "displayName": "ECDet-L", + "family": "ec", + "baseline_fps": 24.2, + "target_fps": 41.7, + "speedup_x": 1.72, + "mAP_delta_pts": -0.1, + "baseline_mAP": 60.1, + "target_mAP": 60.0 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "baseline_fps": 48.2, + "target_fps": 83.0, + "speedup_x": 1.72, + "mAP_delta_pts": -0.1, + "baseline_mAP": 51.7, + "target_mAP": 51.6 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "baseline_fps": 61.6, + "target_fps": 97.7, + "speedup_x": 1.59, + "mAP_delta_pts": -0.1, + "baseline_mAP": 35.5, + "target_mAP": 35.4 + }, + { + "model": "ec-s", + "displayName": "ECDet-S", + "family": "ec", + "baseline_fps": 27.5, + "target_fps": 43.5, + "speedup_x": 1.58, + "mAP_delta_pts": 0.0, + "baseline_mAP": 54.4, + "target_mAP": 54.4 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "baseline_fps": 50.0, + "target_fps": 78.6, + "speedup_x": 1.57, + "mAP_delta_pts": 0.0, + "baseline_mAP": 30.4, + "target_mAP": 30.4 + }, + { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "family": "rfdetr", + "baseline_fps": 39.5, + "target_fps": 58.3, + "speedup_x": 1.48, + "mAP_delta_pts": 0.0, + "baseline_mAP": 51.4, + "target_mAP": 51.4 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 45.1, + "target_fps": 65.7, + "speedup_x": 1.46, + "mAP_delta_pts": -0.6, + "baseline_mAP": 37.9, + "target_mAP": 37.3 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "baseline_fps": 50.0, + "target_fps": 70.9, + "speedup_x": 1.42, + "mAP_delta_pts": -0.9, + "baseline_mAP": 44.3, + "target_mAP": 43.4 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 39.5, + "target_fps": 38.0, + "speedup_x": 0.96, + "mAP_delta_pts": -0.1, + "baseline_mAP": 44.1, + "target_mAP": 44.0 + } + ], + "speedup": { + "median_x": 2.22, + "min_x": 0.96, + "max_x": 4.72 + }, + "family_median_speedup": { + "deim": 2.94, + "rtdetr": 3.27, + "rtdetrv4": 3.15, + "dfine": 2.59, + "rtdetrv2": 3.23, + "deimv2": 2.18, + "yolov9": 1.95, + "rfdetr": 2.0, + "yolox": 1.82, + "ec": 1.8, + "picodet": 1.46 + }, + "accuracy_losers": [ + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "baseline_fps": 40.4, + "target_fps": 132.1, + "speedup_x": 3.27, + "mAP_delta_pts": -1.7, + "baseline_mAP": 27.5, + "target_mAP": 25.8 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "baseline_fps": 17.4, + "target_fps": 37.9, + "speedup_x": 2.18, + "mAP_delta_pts": -0.5, + "baseline_mAP": 61.3, + "target_mAP": 60.8 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 45.1, + "target_fps": 65.7, + "speedup_x": 1.46, + "mAP_delta_pts": -0.6, + "baseline_mAP": 37.9, + "target_mAP": 37.3 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "baseline_fps": 50.0, + "target_fps": 70.9, + "speedup_x": 1.42, + "mAP_delta_pts": -0.9, + "baseline_mAP": 44.3, + "target_mAP": 43.4 + } + ], + "speed_regressions": [ + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 39.5, + "target_fps": 38.0, + "speedup_x": 0.96, + "mAP_delta_pts": -0.1, + "baseline_mAP": 44.1, + "target_mAP": 44.0 + } + ] + }, + "claims": [ + { + "id": "median_speedup", + "kind": "speed", + "statement": "Across 51 models on NVIDIA RTX 5070 Ti, TensorRT FP16 gives a median 2.22x over PyTorch FP32 (range 0.96x to 4.72x).", + "values": { + "median_x": 2.22, + "min_x": 0.96, + "max_x": 4.72 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "top_gainer", + "kind": "speed", + "statement": "Biggest gain: DEIM-X at 4.72x (15.0 to 70.7 FPS).", + "values": { + "model": "deim-x", + "displayName": "DEIM-X", + "family": "deim", + "baseline_fps": 15.0, + "target_fps": 70.7, + "speedup_x": 4.72, + "mAP_delta_pts": -0.1, + "baseline_mAP": 59.6, + "target_mAP": 59.5 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "family_spread", + "kind": "speed", + "statement": "Family medians differ: rtdetr gains 3.27x, picodet only 1.46x. Conversion gain is architecture-dependent.", + "values": { + "best": [ + "rtdetr", + 3.27 + ], + "worst": [ + "picodet", + 1.46 + ] + }, + "passed": true, + "strength": "normal" + }, + { + "id": "accuracy_cost", + "kind": "accuracy", + "statement": "4 of 51 models lose 0.5+ mAP points under TensorRT FP16: DEIMv2-Atto (-1.7), DEIMv2-X (-0.5), PicoDet-M (-0.6), YOLOX-S (-0.9)", + "values": { + "losers": [ + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "baseline_fps": 40.4, + "target_fps": 132.1, + "speedup_x": 3.27, + "mAP_delta_pts": -1.7, + "baseline_mAP": 27.5, + "target_mAP": 25.8 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "baseline_fps": 17.4, + "target_fps": 37.9, + "speedup_x": 2.18, + "mAP_delta_pts": -0.5, + "baseline_mAP": 61.3, + "target_mAP": 60.8 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 45.1, + "target_fps": 65.7, + "speedup_x": 1.46, + "mAP_delta_pts": -0.6, + "baseline_mAP": 37.9, + "target_mAP": 37.3 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "baseline_fps": 50.0, + "target_fps": 70.9, + "speedup_x": 1.42, + "mAP_delta_pts": -0.9, + "baseline_mAP": 44.3, + "target_mAP": 43.4 + } + ] + }, + "passed": true, + "strength": "strong" + }, + { + "id": "speed_regression", + "kind": "anomaly", + "statement": "1 models get SLOWER under TensorRT FP16: PicoDet-L (0.96x)", + "values": { + "regressions": [ + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 39.5, + "target_fps": 38.0, + "speedup_x": 0.96, + "mAP_delta_pts": -0.1, + "baseline_mAP": 44.1, + "target_mAP": 44.0 + } + ] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 3, + "reason": null + }, + "allowed_numbers": [ + -1.7, + -0.9, + -0.6, + -0.5, + -0.2, + -0.1, + 0.0, + 0.001, + 0.1, + 0.2, + 0.6, + 0.96, + 1, + 1.42, + 1.46, + 1.48, + 1.57, + 1.58, + 1.59, + 1.72, + 1.76, + 1.8, + 1.82, + 1.84, + 1.85, + 1.87, + 1.9, + 1.95, + 1.96, + 2.0, + 2.07, + 2.08, + 2.18, + 2.2, + 2.22, + 2.3, + 2.32, + 2.4, + 2.45, + 2.47, + 2.53, + 2.58, + 2.59, + 2.7, + 2.83, + 2.94, + 3, + 3.12, + 3.15, + 3.17, + 3.23, + 3.27, + 3.29, + 3.4, + 3.74, + 3.78, + 4, + 4.02, + 4.27, + 4.4, + 4.72, + 5, + 6, + 7, + 8, + 9, + 10.0, + 11, + 12, + 15.0, + 16.6, + 17.4, + 18.1, + 19.7, + 20.0, + 21.8, + 21.9, + 23.0, + 23.6, + 24.2, + 24.3, + 24.7, + 25.1, + 25.7, + 25.8, + 26.5, + 27.5, + 27.7, + 28.4, + 28.7, + 28.8, + 29.8, + 30.3, + 30.4, + 30.8, + 31.9, + 32.1, + 32.5, + 33.4, + 33.6, + 34.2, + 35.1, + 35.3, + 35.4, + 35.5, + 35.8, + 37.0, + 37.1, + 37.3, + 37.9, + 38.0, + 38.7, + 39.5, + 39.6, + 39.8, + 39.9, + 40.2, + 40.4, + 41.3, + 41.7, + 41.8, + 42.3, + 42.8, + 43.4, + 43.5, + 44.0, + 44.1, + 44.3, + 45.1, + 45.8, + 46.6, + 46.7, + 46.8, + 47.2, + 47.6, + 48.2, + 48.9, + 49.8, + 50.0, + 50.4, + 50.5, + 50.7, + 50.8, + 51.0, + 51.1, + 51.4, + 51.6, + 51.7, + 51.9, + 52.1, + 52.2, + 52.3, + 52.8, + 52.9, + 53.0, + 53.2, + 53.4, + 53.6, + 53.8, + 53.9, + 54.4, + 54.7, + 54.8, + 55.1, + 55.3, + 55.4, + 55.5, + 55.7, + 55.8, + 55.9, + 56.1, + 56.3, + 56.4, + 56.5, + 56.7, + 56.8, + 57.1, + 57.2, + 57.3, + 57.4, + 57.6, + 57.7, + 57.8, + 57.9, + 58.2, + 58.3, + 58.4, + 58.5, + 58.6, + 59.0, + 59.5, + 59.6, + 59.9, + 60.0, + 60.1, + 60.8, + 61.0, + 61.1, + 61.3, + 61.4, + 61.5, + 61.6, + 62.9, + 63.8, + 65.7, + 69.3, + 70.7, + 70.9, + 71.0, + 71.6, + 72.1, + 72.3, + 73.0, + 74.5, + 74.7, + 75, + 78.0, + 78.6, + 79.2, + 79.7, + 81.3, + 82.3, + 82.4, + 83.0, + 83.3, + 86.5, + 86.7, + 87.4, + 88.0, + 92.3, + 95, + 96.0, + 97.4, + 97.7, + 100.0, + 103.6, + 108.4, + 110.1, + 113.0, + 113.4, + 117.8, + 122.0, + 132.1, + 132.9, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/tensorrt-fp32-vs-pytorch-fp32-jetson-orin.claims.json b/article-pipeline/v2/output/tensorrt-fp32-vs-pytorch-fp32-jetson-orin.claims.json new file mode 100644 index 0000000..c00fddc --- /dev/null +++ b/article-pipeline/v2/output/tensorrt-fp32-vs-pytorch-fp32-jetson-orin.claims.json @@ -0,0 +1,919 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:09:28.441658+00:00", + "slug": "tensorrt-fp32-vs-pytorch-fp32-jetson-orin", + "meta": { + "type": "runtime-guide", + "hardware": "jetson_orin", + "baseline": "pytorch_fp32", + "target": "tensorrt_fp32" + }, + "facts": { + "hardware": "jetson_orin", + "hardware_display": "NVIDIA Jetson Orin Nano Super 8GB", + "baseline": "pytorch_fp32", + "baseline_display": "PyTorch FP32", + "target": "tensorrt_fp32", + "target_display": "TensorRT FP32", + "n_models": 55, + "rows": [ + { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "baseline_fps": 15.4, + "target_fps": 81.4, + "speedup_x": 5.28, + "mAP_delta_pts": 0.0, + "baseline_mAP": 27.5, + "target_mAP": 27.5 + }, + { + "model": "deimv2-femto", + "displayName": "DEIMv2-Femto", + "family": "deimv2", + "baseline_fps": 14.8, + "target_fps": 60.0, + "speedup_x": 4.05, + "mAP_delta_pts": 0.0, + "baseline_mAP": 34.5, + "target_mAP": 34.5 + }, + { + "model": "rtdetr-r50m", + "displayName": "RT-DETR-R50m", + "family": "rtdetr", + "baseline_fps": 5.0, + "target_fps": 15.7, + "speedup_x": 3.12, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.8, + "target_mAP": 53.8 + }, + { + "model": "rtdetrv2-r50m", + "displayName": "RT-DETRv2-R50m", + "family": "rtdetrv2", + "baseline_fps": 5.1, + "target_fps": 15.8, + "speedup_x": 3.11, + "mAP_delta_pts": 0.0, + "baseline_mAP": 54.8, + "target_mAP": 54.8 + }, + { + "model": "rtdetr-r101", + "displayName": "RT-DETR-R101", + "family": "rtdetr", + "baseline_fps": 2.8, + "target_fps": 8.4, + "speedup_x": 3.04, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.8, + "target_mAP": 56.8 + }, + { + "model": "rtdetrv2-r101", + "displayName": "RT-DETRv2-R101", + "family": "rtdetrv2", + "baseline_fps": 2.8, + "target_fps": 8.5, + "speedup_x": 3.03, + "mAP_delta_pts": -0.1, + "baseline_mAP": 56.8, + "target_mAP": 56.7 + }, + { + "model": "deimv2-n", + "displayName": "DEIMv2-N", + "family": "deimv2", + "baseline_fps": 11.1, + "target_fps": 33.0, + "speedup_x": 2.97, + "mAP_delta_pts": 0.0, + "baseline_mAP": 46.7, + "target_mAP": 46.7 + }, + { + "model": "rtdetr-r50", + "displayName": "RT-DETR-R50", + "family": "rtdetr", + "baseline_fps": 4.3, + "target_fps": 12.6, + "speedup_x": 2.97, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.9, + "target_mAP": 55.9 + }, + { + "model": "rtdetrv2-r50", + "displayName": "RT-DETRv2-R50", + "family": "rtdetrv2", + "baseline_fps": 4.3, + "target_fps": 12.7, + "speedup_x": 2.97, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.7, + "target_mAP": 55.7 + }, + { + "model": "deim-n", + "displayName": "DEIM-N", + "family": "deim", + "baseline_fps": 11.3, + "target_fps": 33.4, + "speedup_x": 2.95, + "mAP_delta_pts": 0.0, + "baseline_mAP": 46.8, + "target_mAP": 46.8 + }, + { + "model": "rtdetr-x", + "displayName": "RT-DETR-X", + "family": "rtdetr", + "baseline_fps": 3.0, + "target_fps": 8.6, + "speedup_x": 2.92, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.9, + "target_mAP": 57.9 + }, + { + "model": "dfine-n", + "displayName": "D-FINE-N", + "family": "dfine", + "baseline_fps": 11.7, + "target_fps": 33.5, + "speedup_x": 2.86, + "mAP_delta_pts": 0.0, + "baseline_mAP": 45.8, + "target_mAP": 45.8 + }, + { + "model": "yolov9t", + "displayName": "YOLOv9-T", + "family": "yolov9", + "baseline_fps": 9.9, + "target_fps": 27.8, + "speedup_x": 2.81, + "mAP_delta_pts": 0.0, + "baseline_mAP": 41.8, + "target_mAP": 41.8 + }, + { + "model": "dfine-x", + "displayName": "D-FINE-X", + "family": "dfine", + "baseline_fps": 2.9, + "target_fps": 8.0, + "speedup_x": 2.8, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.4, + "target_mAP": 61.4 + }, + { + "model": "rtdetr-l", + "displayName": "RT-DETR-L", + "family": "rtdetr", + "baseline_fps": 4.9, + "target_fps": 13.6, + "speedup_x": 2.8, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.8, + "target_mAP": 55.8 + }, + { + "model": "deim-x", + "displayName": "DEIM-X", + "family": "deim", + "baseline_fps": 2.9, + "target_fps": 8.0, + "speedup_x": 2.79, + "mAP_delta_pts": 0.0, + "baseline_mAP": 59.6, + "target_mAP": 59.6 + }, + { + "model": "picodet-s", + "displayName": "PicoDet-S", + "family": "picodet", + "baseline_fps": 13.8, + "target_fps": 37.9, + "speedup_x": 2.74, + "mAP_delta_pts": -0.1, + "baseline_mAP": 30.4, + "target_mAP": 30.3 + }, + { + "model": "rtdetrv4-x", + "displayName": "RT-DETRv4-X", + "family": "rtdetrv4", + "baseline_fps": 2.8, + "target_fps": 7.8, + "speedup_x": 2.74, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.0, + "target_mAP": 60.0 + }, + { + "model": "deimv2-pico", + "displayName": "DEIMv2-Pico", + "family": "deimv2", + "baseline_fps": 14.1, + "target_fps": 37.7, + "speedup_x": 2.68, + "mAP_delta_pts": 0.1, + "baseline_mAP": 42.2, + "target_mAP": 42.3 + }, + { + "model": "dfine-l", + "displayName": "D-FINE-L", + "family": "dfine", + "baseline_fps": 4.7, + "target_fps": 12.5, + "speedup_x": 2.64, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.0, + "target_mAP": 60.0 + }, + { + "model": "deim-l", + "displayName": "DEIM-L", + "family": "deim", + "baseline_fps": 4.8, + "target_fps": 12.4, + "speedup_x": 2.6, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "yolox-nano", + "displayName": "YOLOX-Nano", + "family": "yolox", + "baseline_fps": 18.4, + "target_fps": 47.4, + "speedup_x": 2.58, + "mAP_delta_pts": 0.0, + "baseline_mAP": 28.8, + "target_mAP": 28.8 + }, + { + "model": "rtdetrv4-l", + "displayName": "RT-DETRv4-L", + "family": "rtdetrv4", + "baseline_fps": 4.7, + "target_fps": 12.1, + "speedup_x": 2.56, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.8, + "target_mAP": 57.8 + }, + { + "model": "picodet-m", + "displayName": "PicoDet-M", + "family": "picodet", + "baseline_fps": 12.5, + "target_fps": 29.6, + "speedup_x": 2.38, + "mAP_delta_pts": 0.0, + "baseline_mAP": 37.9, + "target_mAP": 37.9 + }, + { + "model": "rfdetr-n", + "displayName": "RF-DETR-N", + "family": "rfdetr", + "baseline_fps": 10.2, + "target_fps": 23.7, + "speedup_x": 2.33, + "mAP_delta_pts": 0.0, + "baseline_mAP": 51.4, + "target_mAP": 51.4 + }, + { + "model": "dfine-m", + "displayName": "D-FINE-M", + "family": "dfine", + "baseline_fps": 7.0, + "target_fps": 15.8, + "speedup_x": 2.26, + "mAP_delta_pts": -0.1, + "baseline_mAP": 57.8, + "target_mAP": 57.7 + }, + { + "model": "deim-m", + "displayName": "DEIM-M", + "family": "deim", + "baseline_fps": 7.0, + "target_fps": 15.7, + "speedup_x": 2.25, + "mAP_delta_pts": 0.1, + "baseline_mAP": 55.4, + "target_mAP": 55.5 + }, + { + "model": "rtdetrv4-m", + "displayName": "RT-DETRv4-M", + "family": "rtdetrv4", + "baseline_fps": 7.0, + "target_fps": 15.4, + "speedup_x": 2.2, + "mAP_delta_pts": -0.1, + "baseline_mAP": 56.5, + "target_mAP": 56.4 + }, + { + "model": "rtdetrv4-s", + "displayName": "RT-DETRv4-S", + "family": "rtdetrv4", + "baseline_fps": 10.1, + "target_fps": 22.1, + "speedup_x": 2.19, + "mAP_delta_pts": 0.0, + "baseline_mAP": 52.8, + "target_mAP": 52.8 + }, + { + "model": "yolov9s", + "displayName": "YOLOv9-S", + "family": "yolov9", + "baseline_fps": 9.9, + "target_fps": 21.6, + "speedup_x": 2.19, + "mAP_delta_pts": 0.0, + "baseline_mAP": 50.4, + "target_mAP": 50.4 + }, + { + "model": "deim-s", + "displayName": "DEIM-S", + "family": "deim", + "baseline_fps": 10.2, + "target_fps": 22.2, + "speedup_x": 2.18, + "mAP_delta_pts": 0.0, + "baseline_mAP": 52.1, + "target_mAP": 52.1 + }, + { + "model": "rtdetr-r18", + "displayName": "RT-DETR-R18", + "family": "rtdetr", + "baseline_fps": 10.2, + "target_fps": 22.1, + "speedup_x": 2.17, + "mAP_delta_pts": 0.0, + "baseline_mAP": 49.8, + "target_mAP": 49.8 + }, + { + "model": "rtdetrv2-r18", + "displayName": "RT-DETRv2-R18", + "family": "rtdetrv2", + "baseline_fps": 10.3, + "target_fps": 22.3, + "speedup_x": 2.17, + "mAP_delta_pts": -0.1, + "baseline_mAP": 50.8, + "target_mAP": 50.7 + }, + { + "model": "dfine-s", + "displayName": "D-FINE-S", + "family": "dfine", + "baseline_fps": 10.3, + "target_fps": 22.4, + "speedup_x": 2.16, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.4, + "target_mAP": 53.4 + }, + { + "model": "rtdetr-r34", + "displayName": "RT-DETR-R34", + "family": "rtdetr", + "baseline_fps": 8.0, + "target_fps": 17.1, + "speedup_x": 2.15, + "mAP_delta_pts": 0.0, + "baseline_mAP": 52.2, + "target_mAP": 52.2 + }, + { + "model": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "family": "yolox", + "baseline_fps": 20.3, + "target_fps": 43.6, + "speedup_x": 2.14, + "mAP_delta_pts": 0.0, + "baseline_mAP": 35.5, + "target_mAP": 35.5 + }, + { + "model": "rtdetrv2-r34", + "displayName": "RT-DETRv2-R34", + "family": "rtdetrv2", + "baseline_fps": 8.1, + "target_fps": 17.2, + "speedup_x": 2.12, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.2, + "target_mAP": 53.2 + }, + { + "model": "rfdetr-s", + "displayName": "RF-DETR-S", + "family": "rfdetr", + "baseline_fps": 6.9, + "target_fps": 14.5, + "speedup_x": 2.09, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.1, + "target_mAP": 55.1 + }, + { + "model": "yolov9c", + "displayName": "YOLOv9-C", + "family": "yolov9", + "baseline_fps": 6.4, + "target_fps": 13.2, + "speedup_x": 2.05, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.1, + "target_mAP": 57.1 + }, + { + "model": "rfdetr-m", + "displayName": "RF-DETR-M", + "family": "rfdetr", + "baseline_fps": 5.7, + "target_fps": 11.6, + "speedup_x": 2.03, + "mAP_delta_pts": 0.0, + "baseline_mAP": 57.4, + "target_mAP": 57.4 + }, + { + "model": "yolox-l", + "displayName": "YOLOX-L", + "family": "yolox", + "baseline_fps": 5.8, + "target_fps": 11.4, + "speedup_x": 1.96, + "mAP_delta_pts": 0.0, + "baseline_mAP": 55.4, + "target_mAP": 55.4 + }, + { + "model": "rfdetr-l", + "displayName": "RF-DETR-L", + "family": "rfdetr", + "baseline_fps": 4.0, + "target_fps": 7.8, + "speedup_x": 1.95, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.6, + "target_mAP": 58.6 + }, + { + "model": "yolox-x", + "displayName": "YOLOX-X", + "family": "yolox", + "baseline_fps": 3.5, + "target_fps": 6.7, + "speedup_x": 1.94, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.3, + "target_mAP": 56.3 + }, + { + "model": "deimv2-x", + "displayName": "DEIMv2-X", + "family": "deimv2", + "baseline_fps": 2.7, + "target_fps": 5.2, + "speedup_x": 1.91, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.3, + "target_mAP": 61.3 + }, + { + "model": "ec-x", + "displayName": "ECDet-X", + "family": "ec", + "baseline_fps": 2.9, + "target_fps": 5.4, + "speedup_x": 1.87, + "mAP_delta_pts": 0.0, + "baseline_mAP": 61.1, + "target_mAP": 61.1 + }, + { + "model": "deimv2-l", + "displayName": "DEIMv2-L", + "family": "deimv2", + "baseline_fps": 3.3, + "target_fps": 6.1, + "speedup_x": 1.86, + "mAP_delta_pts": 0.0, + "baseline_mAP": 58.6, + "target_mAP": 58.6 + }, + { + "model": "deimv2-s", + "displayName": "DEIMv2-S", + "family": "deimv2", + "baseline_fps": 5.7, + "target_fps": 10.4, + "speedup_x": 1.84, + "mAP_delta_pts": 0.0, + "baseline_mAP": 53.0, + "target_mAP": 53.0 + }, + { + "model": "yolov9m", + "displayName": "YOLOv9-M", + "family": "yolov9", + "baseline_fps": 8.0, + "target_fps": 14.5, + "speedup_x": 1.81, + "mAP_delta_pts": 0.0, + "baseline_mAP": 56.1, + "target_mAP": 56.1 + }, + { + "model": "ec-l", + "displayName": "ECDet-L", + "family": "ec", + "baseline_fps": 3.4, + "target_fps": 6.1, + "speedup_x": 1.8, + "mAP_delta_pts": 0.0, + "baseline_mAP": 60.1, + "target_mAP": 60.1 + }, + { + "model": "yolox-m", + "displayName": "YOLOX-M", + "family": "yolox", + "baseline_fps": 9.1, + "target_fps": 16.4, + "speedup_x": 1.8, + "mAP_delta_pts": 0.1, + "baseline_mAP": 51.7, + "target_mAP": 51.8 + }, + { + "model": "ec-s", + "displayName": "ECDet-S", + "family": "ec", + "baseline_fps": 6.0, + "target_fps": 10.6, + "speedup_x": 1.77, + "mAP_delta_pts": 0.1, + "baseline_mAP": 54.3, + "target_mAP": 54.4 + }, + { + "model": "deimv2-m", + "displayName": "DEIMv2-M", + "family": "deimv2", + "baseline_fps": 4.7, + "target_fps": 8.0, + "speedup_x": 1.71, + "mAP_delta_pts": -0.1, + "baseline_mAP": 56.0, + "target_mAP": 55.9 + }, + { + "model": "ec-m", + "displayName": "ECDet-M", + "family": "ec", + "baseline_fps": 5.0, + "target_fps": 8.2, + "speedup_x": 1.65, + "mAP_delta_pts": -0.1, + "baseline_mAP": 58.4, + "target_mAP": 58.3 + }, + { + "model": "yolox-s", + "displayName": "YOLOX-S", + "family": "yolox", + "baseline_fps": 16.9, + "target_fps": 26.1, + "speedup_x": 1.55, + "mAP_delta_pts": 0.0, + "baseline_mAP": 44.3, + "target_mAP": 44.3 + }, + { + "model": "picodet-l", + "displayName": "PicoDet-L", + "family": "picodet", + "baseline_fps": 11.1, + "target_fps": 16.4, + "speedup_x": 1.48, + "mAP_delta_pts": 0.0, + "baseline_mAP": 44.1, + "target_mAP": 44.1 + } + ], + "speedup": { + "median_x": 2.2, + "min_x": 1.48, + "max_x": 5.28 + }, + "family_median_speedup": { + "deimv2": 2.68, + "rtdetr": 2.92, + "rtdetrv2": 2.97, + "deim": 2.6, + "dfine": 2.64, + "yolov9": 2.19, + "picodet": 2.38, + "rtdetrv4": 2.56, + "yolox": 1.96, + "rfdetr": 2.09, + "ec": 1.8 + }, + "accuracy_losers": [], + "speed_regressions": [] + }, + "claims": [ + { + "id": "median_speedup", + "kind": "speed", + "statement": "Across 55 models on NVIDIA Jetson Orin Nano Super 8GB, TensorRT FP32 gives a median 2.2x over PyTorch FP32 (range 1.48x to 5.28x).", + "values": { + "median_x": 2.2, + "min_x": 1.48, + "max_x": 5.28 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "top_gainer", + "kind": "speed", + "statement": "Biggest gain: DEIMv2-Atto at 5.28x (15.4 to 81.4 FPS).", + "values": { + "model": "deimv2-atto", + "displayName": "DEIMv2-Atto", + "family": "deimv2", + "baseline_fps": 15.4, + "target_fps": 81.4, + "speedup_x": 5.28, + "mAP_delta_pts": 0.0, + "baseline_mAP": 27.5, + "target_mAP": 27.5 + }, + "passed": true, + "strength": "normal" + }, + { + "id": "family_spread", + "kind": "speed", + "statement": "Family medians differ: rtdetrv2 gains 2.97x, ec only 1.8x. Conversion gain is architecture-dependent.", + "values": { + "best": [ + "rtdetrv2", + 2.97 + ], + "worst": [ + "ec", + 1.8 + ] + }, + "passed": true, + "strength": "normal" + }, + { + "id": "accuracy_cost", + "kind": "accuracy", + "statement": "No model loses more than 0.5 mAP points converting to TensorRT FP32.", + "values": { + "losers": [] + }, + "passed": true, + "strength": "strong" + } + ], + "gate": { + "passed": true, + "passed_claims": 4, + "required": 3, + "reason": null + }, + "allowed_numbers": [ + -0.1, + 0.0, + 0.001, + 0.1, + 0.6, + 1, + 1.48, + 1.55, + 1.65, + 1.71, + 1.77, + 1.8, + 1.81, + 1.84, + 1.86, + 1.87, + 1.91, + 1.94, + 1.95, + 1.96, + 2, + 2.03, + 2.05, + 2.09, + 2.12, + 2.14, + 2.15, + 2.16, + 2.17, + 2.18, + 2.19, + 2.2, + 2.25, + 2.26, + 2.33, + 2.38, + 2.56, + 2.58, + 2.6, + 2.64, + 2.68, + 2.7, + 2.74, + 2.79, + 2.8, + 2.81, + 2.86, + 2.9, + 2.92, + 2.95, + 2.97, + 3.0, + 3.03, + 3.04, + 3.11, + 3.12, + 3.3, + 3.4, + 3.5, + 4.0, + 4.05, + 4.3, + 4.7, + 4.8, + 4.9, + 5.0, + 5.1, + 5.2, + 5.28, + 5.4, + 5.7, + 5.8, + 6.0, + 6.1, + 6.4, + 6.7, + 6.9, + 7.0, + 7.8, + 8.0, + 8.1, + 8.2, + 8.4, + 8.5, + 8.6, + 9, + 9.1, + 9.9, + 10.0, + 10.1, + 10.2, + 10.3, + 10.4, + 10.6, + 11, + 11.1, + 11.3, + 11.4, + 11.6, + 11.7, + 12, + 12.1, + 12.4, + 12.5, + 12.6, + 12.7, + 13.2, + 13.6, + 13.8, + 14.1, + 14.5, + 14.8, + 15.4, + 15.7, + 15.8, + 16.4, + 16.9, + 17.1, + 17.2, + 18.4, + 20.3, + 21.6, + 22.1, + 22.2, + 22.3, + 22.4, + 23.7, + 26.1, + 27.5, + 27.8, + 28.8, + 29.6, + 30.3, + 30.4, + 33.0, + 33.4, + 33.5, + 34.5, + 35.5, + 37.7, + 37.9, + 41.8, + 42.2, + 42.3, + 43.6, + 44.1, + 44.3, + 45.8, + 46.7, + 46.8, + 47.4, + 49.8, + 50, + 50.4, + 50.7, + 50.8, + 51.4, + 51.7, + 51.8, + 52.1, + 52.2, + 52.8, + 53.0, + 53.2, + 53.4, + 53.8, + 54.3, + 54.4, + 54.8, + 55.0, + 55.1, + 55.4, + 55.5, + 55.7, + 55.8, + 55.9, + 56.0, + 56.1, + 56.3, + 56.4, + 56.5, + 56.7, + 56.8, + 57.1, + 57.4, + 57.7, + 57.8, + 57.9, + 58.3, + 58.4, + 58.6, + 59.6, + 60.0, + 60.1, + 61.1, + 61.3, + 61.4, + 75, + 81.4, + 95, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/output/yolov9-vs-yolox.claims.json b/article-pipeline/v2/output/yolov9-vs-yolox.claims.json new file mode 100644 index 0000000..578bb6c --- /dev/null +++ b/article-pipeline/v2/output/yolov9-vs-yolox.claims.json @@ -0,0 +1,379 @@ +{ + "engine": "article-pipeline-v2", + "generated_at": "2026-07-03T22:28:17.502283+00:00", + "slug": "yolov9-vs-yolox", + "meta": { + "type": "family-vs", + "a": "yolov9", + "b": "yolox", + "hardware": "nvidia_geforce_rtx_5070_ti", + "runtime": "pytorch_fp32" + }, + "facts": { + "families": { + "a": "yolov9", + "b": "yolox", + "a_display": "YOLOv9", + "b_display": "YOLOX" + }, + "primary": { + "hardware": "nvidia_geforce_rtx_5070_ti", + "hardware_display": "NVIDIA RTX 5070 Ti", + "runtime": "pytorch_fp32", + "runtime_display": "PyTorch FP32" + }, + "a_frontier": [ + { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "params": 2.04, + "mAP": 41.8, + "fps": 31.9, + "license": "MIT" + }, + { + "id": "yolov9s", + "displayName": "YOLOv9-S", + "params": 7.23, + "mAP": 50.4, + "fps": 30.8, + "license": "MIT" + }, + { + "id": "yolov9m", + "displayName": "YOLOv9-M", + "params": 20.12, + "mAP": 56.1, + "fps": 37.1, + "license": "MIT" + }, + { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + } + ], + "b_frontier": [ + { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + }, + { + "id": "yolox-tiny", + "displayName": "YOLOX-Tiny", + "params": 5.06, + "mAP": 35.5, + "fps": 61.6, + "license": "Apache-2.0" + }, + { + "id": "yolox-s", + "displayName": "YOLOX-S", + "params": 8.97, + "mAP": 44.3, + "fps": 50.0, + "license": "Apache-2.0" + }, + { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + { + "id": "yolox-l", + "displayName": "YOLOX-L", + "params": 54.21, + "mAP": 55.4, + "fps": 42.8, + "license": "Apache-2.0" + }, + { + "id": "yolox-x", + "displayName": "YOLOX-X", + "params": 99.07, + "mAP": 56.3, + "fps": 39.6, + "license": "Apache-2.0" + } + ], + "matched_pairs": [ + { + "a": { + "id": "yolov9s", + "displayName": "YOLOv9-S", + "params": 7.23, + "mAP": 50.4, + "fps": 30.8, + "license": "MIT" + }, + "b": { + "id": "yolox-s", + "displayName": "YOLOX-S", + "params": 8.97, + "mAP": 44.3, + "fps": 50.0, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 6.1, + "fps_delta_pct": -38.4 + }, + { + "a": { + "id": "yolov9m", + "displayName": "YOLOv9-M", + "params": 20.12, + "mAP": 56.1, + "fps": 37.1, + "license": "MIT" + }, + "b": { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 4.4, + "fps_delta_pct": -23.03 + }, + { + "a": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + }, + "b": { + "id": "yolox-m", + "displayName": "YOLOX-M", + "params": 25.33, + "mAP": 51.7, + "fps": 48.2, + "license": "Apache-2.0" + }, + "mAP_delta_pts": 5.4, + "fps_delta_pct": -12.24 + } + ], + "aggregate": { + "n_pairs": 3, + "mean_mAP_delta_pts": 5.3, + "a_wins": 3, + "b_wins": 0, + "mean_fps_delta_pct": -24.6 + }, + "flagship": { + "a": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + }, + "b": { + "id": "yolox-x", + "displayName": "YOLOX-X", + "params": 99.07, + "mAP": 56.3, + "fps": 39.6, + "license": "Apache-2.0" + } + }, + "smallest": { + "a": { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "params": 2.04, + "mAP": 41.8, + "fps": 31.9, + "license": "MIT" + }, + "b": { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + } + }, + "crossover": { + "low_param_leader": "yolov9", + "high_param_leader": "yolov9" + }, + "license": { + "a_all_permissive": true, + "b_all_permissive": true + }, + "chart_highlight": "yolov9t,yolov9s,yolov9m,yolov9c,yolox-nano,yolox-tiny,yolox-s,yolox-m,yolox-l,yolox-x" + }, + "claims": [ + { + "id": "family_accuracy_edge", + "kind": "accuracy", + "statement": "At matched compute, YOLOv9 averages 5.3 mAP points higher across 3 paired variants on NVIDIA RTX 5070 Ti. YOLOv9 wins 3, YOLOX wins 0.", + "values": { + "mean_delta_pts": 5.3, + "a_wins": 3, + "b_wins": 0 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "family_speed_edge", + "kind": "speed", + "statement": "At matched compute, YOLOX is 24.6% faster on average on NVIDIA RTX 5070 Ti (PyTorch FP32).", + "values": { + "mean_delta_pct": 24.6 + }, + "passed": true, + "strength": "strong" + }, + { + "id": "flagship", + "kind": "accuracy", + "statement": "At the top end, YOLOv9's largest measured variant (YOLOv9-C) reaches 57.1 mAP; YOLOX's (YOLOX-X) reaches 56.3 mAP.", + "values": { + "a": { + "id": "yolov9c", + "displayName": "YOLOv9-C", + "params": 25.5, + "mAP": 57.1, + "fps": 42.3, + "license": "MIT" + }, + "b": { + "id": "yolox-x", + "displayName": "YOLOX-X", + "params": 99.07, + "mAP": 56.3, + "fps": 39.6, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "efficient_end", + "kind": "efficiency", + "statement": "At the small end, YOLOv9's YOLOv9-T (2.04M, 41.8 mAP, 31.9 FPS) faces YOLOX's YOLOX-Nano (0.91M, 28.8 mAP, 53.6 FPS).", + "values": { + "a": { + "id": "yolov9t", + "displayName": "YOLOv9-T", + "params": 2.04, + "mAP": 41.8, + "fps": 31.9, + "license": "MIT" + }, + "b": { + "id": "yolox-nano", + "displayName": "YOLOX-Nano", + "params": 0.91, + "mAP": 28.8, + "fps": 53.6, + "license": "Apache-2.0" + } + }, + "passed": true, + "strength": "normal" + }, + { + "id": "coverage", + "kind": "ranking", + "statement": "YOLOv9 fields 4 measured variants, YOLOX fields 6, all on the same protocol on NVIDIA RTX 5070 Ti.", + "values": { + "a_n": 4, + "b_n": 6 + }, + "passed": true, + "strength": "normal" + } + ], + "gate": { + "passed": true, + "passed_claims": 5, + "required": 2, + "reason": null + }, + "allowed_numbers": [ + -38.4, + -24.6, + -23.03, + -12.24, + 0.0, + 0.001, + 0.6, + 0.91, + 1, + 2, + 2.04, + 3.0, + 4.0, + 4.4, + 5, + 5.06, + 5.3, + 5.4, + 6.0, + 6.1, + 7, + 7.23, + 8, + 8.97, + 9, + 10, + 11, + 12, + 20.12, + 24.6, + 25.33, + 25.5, + 28.8, + 30.8, + 31.9, + 35.5, + 37.1, + 39.6, + 41.8, + 42.3, + 42.8, + 44.3, + 48.2, + 50.0, + 50.4, + 51.7, + 53.6, + 54.21, + 55.4, + 56.1, + 56.3, + 57.1, + 61.6, + 75, + 91.0, + 95, + 99.07, + 100, + 300, + 500, + 640, + 5000 + ] +} \ No newline at end of file diff --git a/article-pipeline/v2/validate.py b/article-pipeline/v2/validate.py new file mode 100644 index 0000000..966d495 --- /dev/null +++ b/article-pipeline/v2/validate.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Validator for v2 article content files. + +Usage: + validate.py [claims.json] + +If claims.json is omitted it is resolved from the content file's "claimsFile" +field (path relative to repo root). + +Checks: + 1. Schema: required metadata fields, known block kinds, per-kind required keys. + 2. Gate: the claims file gate must have passed. + 3. Numbers: every numeric token in prose/verdict/callout text must exist in + the claims file's allowed_numbers (exact match after normalization). + 4. Style: no em dashes, no banned marketing words, no forbidden sections, + no paper-mAP comparisons, dek/title length bounds for SEO. + 5. Slug/status sanity, ISO date, tags present. + +Exit 0 = publishable. Exit 1 = fix and rerun. +""" +import json +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + +BLOCK_KINDS = { + "prose": {"text"}, + "verdict": {"text"}, + "heading": {"text"}, + "callout": {"text"}, + "chart": {"src", "caption"}, + "code": {"language", "text"}, + "kv": {"title", "items"}, + "comparison-table": {"models", "hardware", "runtime"}, + "ranking-table": {"hardware", "runtime", "metric"}, + "speedup-table": {"hardware", "baseline", "target"}, + "family-frontier": {"familyA", "familyB", "hardware", "runtime"}, + "methodology": set(), +} + +TEXT_KINDS = {"prose", "verdict", "callout"} + +ARTICLE_TYPES = {"vs", "family-vs", "hardware-guide", "runtime-guide", "license-guide", "finding"} + +BANNED_PATTERNS = [ + (r"—", "em dash (use colon, comma, or period)"), + (r"–", "en dash (use hyphen)"), + (r"(?i)\b(blazing|crushes|crushing|destroys|insane|game.chang\w+|revolution\w+|" + r"unleash\w*|supercharge\w*|delve|elevate|seamless\w*|cutting.edge|" + r"state.of.the.art|best.in.class)\b", "marketing superlative"), + (r"(?i)\b(really|quite|arguably|very|extremely)\b", "hedge word"), + (r"(?i)paper[- ]reported|published mAP|paper value|paperReportedMap|claims? in the paper", + "paper-mAP comparison (forbidden)"), + (r"(?i)^#*\s*(conclusion|bottom line)", "Conclusion/Bottom-line section (verdict block covers this)"), + (r"(?i)\b(it'?s worth noting|in today'?s world|in the realm of|" + r"let'?s dive|dive into|landscape of)\b", "AI-slop phrase"), +] + +# tokens like 46.8, 0.468, 1,234, 3x, 45%, 12.5ms +NUMBER_RE = re.compile(r"(? 2 * n_data + 4: + errors.append( + f"prose budget exceeded: {n_text} text blocks vs {n_data} data blocks. " + "Cut prose or add rendered data.") + + words = sum(len(str(b.get("text", "")).split()) for b in blocks if b.get("kind") in TEXT_KINDS) + if words > 900: + errors.append(f"total prose is {words} words (max 900). This engine renders data; it does not narrate it.") + + +def check_style(doc, errors): + for i, b in enumerate(doc.get("blocks", [])): + text = b.get("text", "") if b.get("kind") in TEXT_KINDS else "" + if not text: + continue + for pattern, why in BANNED_PATTERNS: + m = re.search(pattern, text) + if m: + errors.append(f"block[{i}]: {why}: {m.group(0)!r}") + for field in ("title", "dek"): + for pattern, why in BANNED_PATTERNS: + m = re.search(pattern, doc.get(field, "")) + if m: + errors.append(f"{field}: {why}: {m.group(0)!r}") + + +def check_numbers(doc, claims, errors): + allowed = set() + for n in claims.get("allowed_numbers", []): + allowed.add(round(float(n), 4)) + if not allowed: + errors.append("claims file has no allowed_numbers; regenerate with claims.py") + return + + def ok(value): + v = float(value.replace(",", "")) + if v in allowed or round(v, 4) in allowed: + return True + # a prose number may be a rounding of an allowed number + for decimals in (0, 1, 2): + for a in allowed: + if round(a, decimals) == v: + return True + return False + + for i, b in enumerate(doc.get("blocks", [])): + if b.get("kind") not in TEXT_KINDS: + continue + text = STRIP_RE.sub(" ", b.get("text", "")) + for m in NUMBER_RE.finditer(text): + if not ok(m.group(1)): + errors.append( + f"block[{i}]: number {m.group(1)} not in claims allowed_numbers. " + "Every prose number must come from the claims file.") + + +def check_gate(doc, claims, errors): + gate = claims.get("gate", {}) + if not gate.get("passed"): + errors.append(f"claims gate failed: {gate.get('reason')}") + if claims.get("slug") and doc.get("slug") != claims["slug"] and doc.get("type") != "finding": + errors.append(f"slug mismatch: content={doc.get('slug')} claims={claims['slug']}") + if doc.get("type") != claims.get("meta", {}).get("type"): + errors.append(f"type mismatch: content={doc.get('type')} claims={claims.get('meta', {}).get('type')}") + + # mandatory caveats must surface in prose + all_text = " ".join(str(b.get("text", "")) for b in doc.get("blocks", [])).lower() + for c in claims.get("claims", []): + if c.get("kind") == "caveat" and c.get("passed") and c.get("strength") == "strong": + if c["id"] == "input_size_caveat": + sizes = [str(v) for v in c.get("values", {}).values() if v] + if not all(s in all_text for s in sizes): + errors.append( + "mandatory input_size_caveat not surfaced: the two models run at " + "different input sizes and the prose must say so.") + + +def main(): + if len(sys.argv) < 2: + print(__doc__) + sys.exit(1) + content_path = Path(sys.argv[1]) + doc = json.loads(content_path.read_text(encoding="utf-8")) + + if len(sys.argv) > 2: + claims_path = Path(sys.argv[2]) + else: + claims_path = ROOT / doc.get("claimsFile", "") + if not claims_path.exists(): + fail([f"claims file not found: {claims_path}"]) + claims = json.loads(claims_path.read_text(encoding="utf-8")) + + errors = [] + check_schema(doc, errors) + check_gate(doc, claims, errors) + check_style(doc, errors) + check_numbers(doc, claims, errors) + + if errors: + fail(errors) + words = sum(len(str(b.get("text", "")).split()) + for b in doc["blocks"] if b.get("kind") in TEXT_KINDS) + print(f"OK: {content_path.name} is publishable " + f"({len(doc['blocks'])} blocks, {words} prose words, gate passed).") + + +if __name__ == "__main__": + main() diff --git a/generated/verified-results.v1.json b/generated/verified-results.v1.json index e4481d4..88ab817 100644 --- a/generated/verified-results.v1.json +++ b/generated/verified-results.v1.json @@ -49392,6 +49392,14346 @@ "source_file": "rfdetr-l__pytorch__cuda__nvidia_geforce_rtx_5070_ti__20260617T171509275538Z.json", "verified_at": "2026-07-06T00:14:49.916092Z" }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9t-pytorch-cpu-rpi5-20260627T174600Z", + "created_at": "2026-06-27T17:46:00.644311Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9t", + "name": "yolov9-t", + "family": "yolov9", + "variant": "t", + "source": "libreyolo", + "weights": "LibreYOLO9t.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4142886655085496, + "mAP_50": 0.561828226291405, + "mAP_75": 0.44681530144982345, + "mAP_small": 0.21504784164132595, + "mAP_medium": 0.4639682903623852, + "mAP_large": 0.5600930967573488, + "AR1": 0.35160654286643267, + "AR10": 0.5545386454848746, + "AR100": 0.5872618193555333, + "AR_small": 0.34655592324253437, + "AR_medium": 0.648047133626687, + "AR_large": 0.7212515580394837 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 346.255, + "std": 3.038, + "p50": 345.943, + "p95": 350.352, + "p99": 353.337, + "preprocess_ms": 3.504, + "inference_ms": 340.048, + "postprocess_ms": 2.703 + } + }, + "throughput": { + "fps_mean": 2.89, + "fps_p50": 2.89 + }, + "model_stats": { + "params_millions": 2.04, + "gflops": 3.98 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 82.6 + }, + "metadata": { + "benchmark_date": "2026-06-27T17:46:00.644311Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolov9t__pytorch__cpu__rpi5__20260627T174600644311Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9s-pytorch-cpu-rpi5-20260627T175420Z", + "created_at": "2026-06-27T17:54:20.938854Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9s", + "name": "yolov9-s", + "family": "yolov9", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLO9s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4944675127323235, + "mAP_50": 0.6596658528528612, + "mAP_75": 0.5413431837383176, + "mAP_small": 0.2975432516124148, + "mAP_medium": 0.5565341085804606, + "mAP_large": 0.6390638954417918, + "AR1": 0.3748582894884518, + "AR10": 0.6048835920165866, + "AR100": 0.6334458960835037, + "AR_small": 0.4098745352838049, + "AR_medium": 0.6915979672968395, + "AR_large": 0.7829655837727637 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 716.73, + "std": 8.597, + "p50": 719.061, + "p95": 726.876, + "p99": 730.016, + "preprocess_ms": 3.57, + "inference_ms": 710.747, + "postprocess_ms": 2.413 + } + }, + "throughput": { + "fps_mean": 1.4, + "fps_p50": 1.39 + }, + "model_stats": { + "params_millions": 7.23, + "gflops": 13.52 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 30.5 + }, + "metadata": { + "benchmark_date": "2026-06-27T17:54:20.938854Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolov9s__pytorch__cpu__rpi5__20260627T175420938854Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9m-pytorch-cpu-rpi5-20260627T181011Z", + "created_at": "2026-06-27T18:10:11.500592Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9m", + "name": "yolov9-m", + "family": "yolov9", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLO9m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5528302643352359, + "mAP_50": 0.7169861926817778, + "mAP_75": 0.600945163591593, + "mAP_small": 0.3632950248731157, + "mAP_medium": 0.6176426703980745, + "mAP_large": 0.6922671306992221, + "AR1": 0.42118616021280153, + "AR10": 0.6615014970561075, + "AR100": 0.687016360397431, + "AR_small": 0.490465192200465, + "AR_medium": 0.7390452575430414, + "AR_large": 0.7928921026725747 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1638.037, + "std": 11.717, + "p50": 1640.403, + "p95": 1651.406, + "p99": 1655.153, + "preprocess_ms": 3.432, + "inference_ms": 1632.286, + "postprocess_ms": 2.32 + } + }, + "throughput": { + "fps_mean": 0.61, + "fps_p50": 0.61 + }, + "model_stats": { + "params_millions": 20.12, + "gflops": 38.68 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:10:11.500592Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolov9m__pytorch__cpu__rpi5__20260627T181011500592Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9c-pytorch-cpu-rpi5-20260627T183042Z", + "created_at": "2026-06-27T18:30:42.773905Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9c", + "name": "yolov9-c", + "family": "yolov9", + "variant": "c", + "source": "libreyolo", + "weights": "LibreYOLO9c.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5644543593876598, + "mAP_50": 0.7300005402529022, + "mAP_75": 0.6145330016470393, + "mAP_small": 0.37838306469498856, + "mAP_medium": 0.6229321401810518, + "mAP_large": 0.7093135785059931, + "AR1": 0.4204609989789378, + "AR10": 0.6671566582359321, + "AR100": 0.6939913561307999, + "AR_small": 0.5068377158440566, + "AR_medium": 0.7340935279923338, + "AR_large": 0.8209435571618553 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2153.804, + "std": 14.662, + "p50": 2155.616, + "p95": 2165.246, + "p99": 2173.649, + "preprocess_ms": 3.366, + "inference_ms": 2148.2, + "postprocess_ms": 2.238 + } + }, + "throughput": { + "fps_mean": 0.46, + "fps_p50": 0.46 + }, + "model_stats": { + "params_millions": 25.5, + "gflops": 51.75 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 5.5 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:30:42.773905Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolov9c__pytorch__cpu__rpi5__20260627T183042773905Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9t-onnx-cpu-rpi5-20260627T183220Z", + "created_at": "2026-06-27T18:32:20.946977Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9t", + "name": "yolov9-t", + "family": "yolov9", + "variant": "t", + "source": "libreyolo", + "weights": "LibreYOLO9t.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4074648393937907, + "mAP_50": 0.5506893479126529, + "mAP_75": 0.44026169451437647, + "mAP_small": 0.2090285418847601, + "mAP_medium": 0.462185513260405, + "mAP_large": 0.5385469455363298, + "AR1": 0.33865172239561997, + "AR10": 0.5262971308176277, + "AR100": 0.5548826838590432, + "AR_small": 0.3055872980978682, + "AR_medium": 0.6215627307649114, + "AR_large": 0.6643449226322229 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 169.966, + "std": 5.736, + "p50": 168.092, + "p95": 179.504, + "p99": 196.557, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 5.88, + "fps_p50": 5.95 + }, + "model_stats": { + "params_millions": 2.03, + "gflops": 3.98 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 48.2 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:32:20.946977Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolov9t__onnx__cpu__rpi5__20260627T183220946977Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9s-onnx-cpu-rpi5-20260627T183602Z", + "created_at": "2026-06-27T18:36:02.767730Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9s", + "name": "yolov9-s", + "family": "yolov9", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLO9s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4853996159820961, + "mAP_50": 0.6461150697986132, + "mAP_75": 0.5309630273094694, + "mAP_small": 0.293103152155661, + "mAP_medium": 0.5371168693341506, + "mAP_large": 0.637471842387726, + "AR1": 0.3653049250326364, + "AR10": 0.5835361493751406, + "AR100": 0.608027908594639, + "AR_small": 0.3921281074987864, + "AR_medium": 0.6537540973193435, + "AR_large": 0.7603834052649736 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 425.079, + "std": 7.453, + "p50": 422.447, + "p95": 439.647, + "p99": 453.722, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 2.35, + "fps_p50": 2.37 + }, + "model_stats": { + "params_millions": 7.21, + "gflops": 13.52 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:36:02.767730Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolov9s__onnx__cpu__rpi5__20260627T183602767730Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9m-onnx-cpu-rpi5-20260627T184453Z", + "created_at": "2026-06-27T18:44:53.208676Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9m", + "name": "yolov9-m", + "family": "yolov9", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLO9m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5362561688863842, + "mAP_50": 0.6984629810034418, + "mAP_75": 0.5822802169919253, + "mAP_small": 0.34856523008321005, + "mAP_medium": 0.6028613173004446, + "mAP_large": 0.6797255613398119, + "AR1": 0.4022525302184688, + "AR10": 0.6360745900137863, + "AR100": 0.6603222993849666, + "AR_small": 0.4691277418037462, + "AR_medium": 0.7129589768007857, + "AR_large": 0.7715388557377723 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1039.926, + "std": 6.587, + "p50": 1037.314, + "p95": 1053.842, + "p99": 1065.368, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.96, + "fps_p50": 0.96 + }, + "model_stats": { + "params_millions": 20.1, + "gflops": 38.68 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:44:53.208676Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolov9m__onnx__cpu__rpi5__20260627T184453208676Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9c-onnx-cpu-rpi5-20260627T185643Z", + "created_at": "2026-06-27T18:56:43.908084Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9c", + "name": "yolov9-c", + "family": "yolov9", + "variant": "c", + "source": "libreyolo", + "weights": "LibreYOLO9c.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5566300259948382, + "mAP_50": 0.7189050949764564, + "mAP_75": 0.605865824410832, + "mAP_small": 0.3652740059450509, + "mAP_medium": 0.6174037066617017, + "mAP_large": 0.6961826666388315, + "AR1": 0.4118876606519492, + "AR10": 0.6507793746815861, + "AR100": 0.6759077751224216, + "AR_small": 0.4710651153951669, + "AR_medium": 0.7204474161385015, + "AR_large": 0.7981423351926766 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1398.298, + "std": 8.211, + "p50": 1395.518, + "p95": 1412.814, + "p99": 1425.463, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.72, + "fps_p50": 0.72 + }, + "model_stats": { + "params_millions": 25.47, + "gflops": 51.75 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:56:43.908084Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolov9c__onnx__cpu__rpi5__20260627T185643908084Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9t-ncnn-cpu-rpi5-20260627T185812Z", + "created_at": "2026-06-27T18:58:12.539348Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9t", + "name": "yolov9-t", + "family": "yolov9", + "variant": "t", + "source": "libreyolo", + "weights": "LibreYOLO9t.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4058640221288353, + "mAP_50": 0.5475924958482162, + "mAP_75": 0.43728430457276396, + "mAP_small": 0.20796364489459593, + "mAP_medium": 0.45828521694993446, + "mAP_large": 0.5382840184674554, + "AR1": 0.33371038683374427, + "AR10": 0.5219582085682064, + "AR100": 0.55158805287851, + "AR_small": 0.29685765329229263, + "AR_medium": 0.618996539352521, + "AR_large": 0.6659664303018029 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 90.959, + "std": 4.968, + "p50": 89.417, + "p95": 99.815, + "p99": 114.633, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 10.99, + "fps_p50": 11.18 + }, + "model_stats": { + "params_millions": 2.04, + "gflops": 3.98 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 33.2 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:58:12.539348Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolov9t__ncnn__cpu__rpi5__20260627T185812539348Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9s-ncnn-cpu-rpi5-20260627T190002Z", + "created_at": "2026-06-27T19:00:02.561194Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9s", + "name": "yolov9-s", + "family": "yolov9", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLO9s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.484190091628974, + "mAP_50": 0.6451728654247788, + "mAP_75": 0.5296253601274965, + "mAP_small": 0.29249693304875785, + "mAP_medium": 0.5368658238613542, + "mAP_large": 0.6352314909128711, + "AR1": 0.3637650968327946, + "AR10": 0.5825791209645957, + "AR100": 0.6067629306798582, + "AR_small": 0.39021001116429194, + "AR_medium": 0.6547917139111623, + "AR_large": 0.7576447505953106 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 203.84, + "std": 3.823, + "p50": 202.602, + "p95": 211.321, + "p99": 219.656, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 4.91, + "fps_p50": 4.94 + }, + "model_stats": { + "params_millions": 7.23, + "gflops": 13.52 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.4 + }, + "metadata": { + "benchmark_date": "2026-06-27T19:00:02.561194Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolov9s__ncnn__cpu__rpi5__20260627T190002561194Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9m-ncnn-cpu-rpi5-20260627T190343Z", + "created_at": "2026-06-27T19:03:43.992851Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9m", + "name": "yolov9-m", + "family": "yolov9", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLO9m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5360877614676471, + "mAP_50": 0.6986857575481058, + "mAP_75": 0.5821722325108789, + "mAP_small": 0.3496126726985973, + "mAP_medium": 0.6043786920558012, + "mAP_large": 0.678102458800428, + "AR1": 0.4018644485947828, + "AR10": 0.6361830406470051, + "AR100": 0.6598288998441495, + "AR_small": 0.4671601187146544, + "AR_medium": 0.7133224921300235, + "AR_large": 0.7709461748018833 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 426.456, + "std": 3.536, + "p50": 425.372, + "p95": 433.219, + "p99": 442.629, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 2.34, + "fps_p50": 2.35 + }, + "model_stats": { + "params_millions": 20.12, + "gflops": 38.68 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T19:03:43.992851Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolov9m__ncnn__cpu__rpi5__20260627T190343992851Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9c-ncnn-cpu-rpi5-20260627T190818Z", + "created_at": "2026-06-27T19:08:18.435183Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9c", + "name": "yolov9-c", + "family": "yolov9", + "variant": "c", + "source": "libreyolo", + "weights": "LibreYOLO9c.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5583977509379243, + "mAP_50": 0.7213695204157715, + "mAP_75": 0.6068137998649635, + "mAP_small": 0.3665642619867098, + "mAP_medium": 0.6182217354453128, + "mAP_large": 0.6968989740677778, + "AR1": 0.4111350174840171, + "AR10": 0.6509927883417154, + "AR100": 0.676280013725787, + "AR_small": 0.47127765853283904, + "AR_medium": 0.7208207352066598, + "AR_large": 0.7991152862312051 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 531.781, + "std": 3.314, + "p50": 530.818, + "p95": 537.626, + "p99": 546.149, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.88, + "fps_p50": 1.88 + }, + "model_stats": { + "params_millions": 25.5, + "gflops": 51.75 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T19:08:18.435183Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolov9c__ncnn__cpu__rpi5__20260627T190818435183Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-n-onnx-cpu-rpi5-20260627T214203Z", + "created_at": "2026-06-27T21:42:03.728445Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-n", + "name": "rf-detr-n", + "family": "rfdetr", + "variant": "n", + "source": "libreyolo", + "weights": "LibreRFDETRn.pt", + "input_size": 384 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 384, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5138715096714886, + "mAP_50": 0.6990689274876704, + "mAP_75": 0.5545178686601185, + "mAP_small": 0.2775638537589648, + "mAP_medium": 0.571678381229741, + "mAP_large": 0.7184524786335472, + "AR1": 0.39943654428373826, + "AR10": 0.630384871636969, + "AR100": 0.6724047346083549, + "AR_small": 0.3890767161821573, + "AR_medium": 0.7463711052931866, + "AR_large": 0.8836870549836006 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 409.243, + "std": 64.229, + "p50": 405.173, + "p95": 410.455, + "p99": 411.233, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 2.44, + "fps_p50": 2.47 + }, + "model_stats": { + "params_millions": 26.86, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 97.8 + }, + "metadata": { + "benchmark_date": "2026-06-27T21:42:03.728445Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rfdetr-n__onnx__cpu__rpi5__20260627T214203728445Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-s-onnx-cpu-rpi5-20260627T214926Z", + "created_at": "2026-06-27T21:49:26.399429Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-s", + "name": "rf-detr-s", + "family": "rfdetr", + "variant": "s", + "source": "libreyolo", + "weights": "LibreRFDETRs.pt", + "input_size": 512 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 512, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5514059726096994, + "mAP_50": 0.7347662100308575, + "mAP_75": 0.5942836611341867, + "mAP_small": 0.35185487254383585, + "mAP_medium": 0.6087277147212882, + "mAP_large": 0.7391800435640334, + "AR1": 0.41629806425979743, + "AR10": 0.681382009005631, + "AR100": 0.7248709221123524, + "AR_small": 0.49441840322082514, + "AR_medium": 0.7941331959800668, + "AR_large": 0.8966372240704275 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 850.101, + "std": 1.262, + "p50": 849.938, + "p95": 852.051, + "p99": 854.343, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.18, + "fps_p50": 1.18 + }, + "model_stats": { + "params_millions": 28.51, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T21:49:26.399429Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rfdetr-s__onnx__cpu__rpi5__20260627T214926399429Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-m-onnx-cpu-rpi5-20260627T215916Z", + "created_at": "2026-06-27T21:59:16.603731Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-m", + "name": "rf-detr-m", + "family": "rfdetr", + "variant": "m", + "source": "libreyolo", + "weights": "LibreRFDETRm.pt", + "input_size": 576 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 576, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.574212802247223, + "mAP_50": 0.7532792126972107, + "mAP_75": 0.624341870173254, + "mAP_small": 0.3839341593907846, + "mAP_medium": 0.6332878141582797, + "mAP_large": 0.7435058935858826, + "AR1": 0.4296453683453095, + "AR10": 0.6988397888599172, + "AR100": 0.7430848130856929, + "AR_small": 0.5319319331259718, + "AR_medium": 0.8035658603539425, + "AR_large": 0.9124160945152083 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1142.177, + "std": 3.034, + "p50": 1141.894, + "p95": 1143.956, + "p99": 1147.717, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.88, + "fps_p50": 0.88 + }, + "model_stats": { + "params_millions": 30.08, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.2 + }, + "metadata": { + "benchmark_date": "2026-06-27T21:59:16.603731Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rfdetr-m__onnx__cpu__rpi5__20260627T215916603731Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-l-onnx-cpu-rpi5-20260627T221539Z", + "created_at": "2026-06-27T22:15:39.909474Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-l", + "name": "rf-detr-l", + "family": "rfdetr", + "variant": "l", + "source": "libreyolo", + "weights": "LibreRFDETRl.pt", + "input_size": 704 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 704, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5859589819852379, + "mAP_50": 0.7607832992022558, + "mAP_75": 0.6430713642386356, + "mAP_small": 0.40893469014459016, + "mAP_medium": 0.6424580981621197, + "mAP_large": 0.7466233369396336, + "AR1": 0.4381431421070228, + "AR10": 0.7121681420603849, + "AR100": 0.759825162772023, + "AR_small": 0.5765694118901368, + "AR_medium": 0.8072927342544284, + "AR_large": 0.9056357201257855 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1924.498, + "std": 2.497, + "p50": 1924.122, + "p95": 1928.538, + "p99": 1931.583, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.52, + "fps_p50": 0.52 + }, + "model_stats": { + "params_millions": 30.33, + "gflops": 340.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T22:15:39.909474Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rfdetr-l__onnx__cpu__rpi5__20260627T221539909474Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-n-pytorch-cpu-rpi5-20260627T222407Z", + "created_at": "2026-06-27T22:24:07.937201Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-n", + "name": "rf-detr-n", + "family": "rfdetr", + "variant": "n", + "source": "libreyolo", + "weights": "LibreRFDETRn.pt", + "input_size": 384 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 384, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5138105965112096, + "mAP_50": 0.6990724692418401, + "mAP_75": 0.5546099900861416, + "mAP_small": 0.2775598218124653, + "mAP_medium": 0.5716791277445037, + "mAP_large": 0.7181729926162229, + "AR1": 0.3993445310338302, + "AR10": 0.6302980005689852, + "AR100": 0.6725590862593693, + "AR_small": 0.38905253248055427, + "AR_medium": 0.7468215621290925, + "AR_large": 0.8836471237540516 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 479.574, + "std": 10.559, + "p50": 481.213, + "p95": 485.016, + "p99": 486.741, + "preprocess_ms": 8.109, + "inference_ms": 468.973, + "postprocess_ms": 2.492 + } + }, + "throughput": { + "fps_mean": 2.09, + "fps_p50": 2.08 + }, + "model_stats": { + "params_millions": 30.47, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 63.2 + }, + "metadata": { + "benchmark_date": "2026-06-27T22:24:07.937201Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rfdetr-n__pytorch__cpu__rpi5__20260627T222407937201Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-s-pytorch-cpu-rpi5-20260627T223330Z", + "created_at": "2026-06-27T22:33:30.205175Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-s", + "name": "rf-detr-s", + "family": "rfdetr", + "variant": "s", + "source": "libreyolo", + "weights": "LibreRFDETRs.pt", + "input_size": 512 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 512, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5512802743036784, + "mAP_50": 0.7347705308004343, + "mAP_75": 0.5940212137659786, + "mAP_small": 0.3517797197118371, + "mAP_medium": 0.6084994790061564, + "mAP_large": 0.7391852638939048, + "AR1": 0.41656386423356995, + "AR10": 0.6812778522573595, + "AR100": 0.7239444499085594, + "AR_small": 0.49427310407552594, + "AR_medium": 0.7925295775590141, + "AR_large": 0.8963737384572662 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 959.447, + "std": 13.842, + "p50": 960.131, + "p95": 969.088, + "p99": 971.164, + "preprocess_ms": 12.312, + "inference_ms": 944.538, + "postprocess_ms": 2.597 + } + }, + "throughput": { + "fps_mean": 1.04, + "fps_p50": 1.04 + }, + "model_stats": { + "params_millions": 32.11, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T22:33:30.205175Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rfdetr-s__pytorch__cpu__rpi5__20260627T223330205175Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-m-pytorch-cpu-rpi5-20260627T224658Z", + "created_at": "2026-06-27T22:46:58.295133Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-m", + "name": "rf-detr-m", + "family": "rfdetr", + "variant": "m", + "source": "libreyolo", + "weights": "LibreRFDETRm.pt", + "input_size": 576 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 576, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5739331874129254, + "mAP_50": 0.7532911305959131, + "mAP_75": 0.6243325497808554, + "mAP_small": 0.3839298566399166, + "mAP_medium": 0.633282711912449, + "mAP_large": 0.7432463466158686, + "AR1": 0.4293236562251876, + "AR10": 0.6988373902606991, + "AR100": 0.7430289499869026, + "AR_small": 0.5319135953843089, + "AR_medium": 0.8035621009554462, + "AR_large": 0.9120736287617837 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1281.026, + "std": 15.647, + "p50": 1282.338, + "p95": 1290.531, + "p99": 1296.884, + "preprocess_ms": 14.846, + "inference_ms": 1263.522, + "postprocess_ms": 2.658 + } + }, + "throughput": { + "fps_mean": 0.78, + "fps_p50": 0.78 + }, + "model_stats": { + "params_millions": 33.69, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T22:46:58.295133Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rfdetr-m__pytorch__cpu__rpi5__20260627T224658295133Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-l-pytorch-cpu-rpi5-20260627T230925Z", + "created_at": "2026-06-27T23:09:25.397322Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-l", + "name": "rf-detr-l", + "family": "rfdetr", + "variant": "l", + "source": "libreyolo", + "weights": "LibreRFDETRl.pt", + "input_size": 704 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 704, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5859255759140363, + "mAP_50": 0.7608500969953391, + "mAP_75": 0.6431066159618175, + "mAP_small": 0.40890502726004985, + "mAP_medium": 0.6424034930844548, + "mAP_large": 0.7466197007070701, + "AR1": 0.43805125196352723, + "AR10": 0.7120575227968798, + "AR100": 0.7595246001966368, + "AR_small": 0.5749240860020073, + "AR_medium": 0.8069345471199254, + "AR_large": 0.9054636091317568 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2110.219, + "std": 19.52, + "p50": 2111.73, + "p95": 2127.169, + "p99": 2134.949, + "preprocess_ms": 21.459, + "inference_ms": 2086.167, + "postprocess_ms": 2.593 + } + }, + "throughput": { + "fps_mean": 0.47, + "fps_p50": 0.47 + }, + "model_stats": { + "params_millions": 33.93, + "gflops": 340.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T23:09:25.397322Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rfdetr-l__pytorch__cpu__rpi5__20260627T230925397322Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-nano-onnx-cpu-rpi5-20260627T231834Z", + "created_at": "2026-06-27T23:18:34.551500Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-nano", + "name": "yolox-nano", + "family": "yolox", + "variant": "nano", + "source": "libreyolo", + "weights": "LibreYOLOXn.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.27685857961952604, + "mAP_50": 0.42734128303256097, + "mAP_75": 0.29140239953452074, + "mAP_small": 0.11728903036893894, + "mAP_medium": 0.27800536997957237, + "mAP_large": 0.41207893068827733, + "AR1": 0.241041546973221, + "AR10": 0.37212501436783635, + "AR100": 0.3934431803759116, + "AR_small": 0.16388973271510549, + "AR_medium": 0.4107943957825836, + "AR_large": 0.5465296093259006 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 46.98, + "std": 3.404, + "p50": 45.973, + "p95": 53.848, + "p99": 59.242, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 21.29, + "fps_p50": 21.75 + }, + "model_stats": { + "params_millions": 0.9, + "gflops": 1.32 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 35.1 + }, + "metadata": { + "benchmark_date": "2026-06-27T23:18:34.551500Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-nano__onnx__cpu__rpi5__20260627T231834551500Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-tiny-onnx-cpu-rpi5-20260627T231939Z", + "created_at": "2026-06-27T23:19:39.650560Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-tiny", + "name": "yolox-tiny", + "family": "yolox", + "variant": "tiny", + "source": "libreyolo", + "weights": "LibreYOLOXt.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.3430169990073264, + "mAP_50": 0.515721093844349, + "mAP_75": 0.3589411422898518, + "mAP_small": 0.15805005951198056, + "mAP_medium": 0.360053827095217, + "mAP_large": 0.49948878510205447, + "AR1": 0.2795115290778376, + "AR10": 0.4362892776562205, + "AR100": 0.4596545269028024, + "AR_small": 0.23112833784085282, + "AR_medium": 0.49827732201225117, + "AR_large": 0.6230383106180426 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 106.688, + "std": 3.05, + "p50": 105.842, + "p95": 112.705, + "p99": 117.871, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 9.37, + "fps_p50": 9.45 + }, + "model_stats": { + "params_millions": 5.05, + "gflops": 7.68 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 27.4 + }, + "metadata": { + "benchmark_date": "2026-06-27T23:19:39.650560Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-tiny__onnx__cpu__rpi5__20260627T231939650560Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-s-onnx-cpu-rpi5-20260627T232309Z", + "created_at": "2026-06-27T23:23:09.330878Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-s", + "name": "yolox-s", + "family": "yolox", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLOXs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4310803617516668, + "mAP_50": 0.6150036757369061, + "mAP_75": 0.47318084172273933, + "mAP_small": 0.25959567816358403, + "mAP_medium": 0.48433772755387117, + "mAP_large": 0.5546052620673176, + "AR1": 0.3444410874562318, + "AR10": 0.538269640202279, + "AR100": 0.5611962334744903, + "AR_small": 0.3706899883081812, + "AR_medium": 0.5986010377258627, + "AR_large": 0.661479601812048 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 393.522, + "std": 5.28, + "p50": 392.898, + "p95": 402.547, + "p99": 414.172, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 2.54, + "fps_p50": 2.55 + }, + "model_stats": { + "params_millions": 8.96, + "gflops": 13.46 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 19.9 + }, + "metadata": { + "benchmark_date": "2026-06-27T23:23:09.330878Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-s__onnx__cpu__rpi5__20260627T232309330878Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-m-onnx-cpu-rpi5-20260627T233104Z", + "created_at": "2026-06-27T23:31:04.111286Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-m", + "name": "yolox-m", + "family": "yolox", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLOXm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5079809277791495, + "mAP_50": 0.6885030787752504, + "mAP_75": 0.5561999863864477, + "mAP_small": 0.3169944618897781, + "mAP_medium": 0.5756125968123131, + "mAP_large": 0.6494321665777568, + "AR1": 0.38473953732302707, + "AR10": 0.6148418795289804, + "AR100": 0.6380360590060202, + "AR_small": 0.42389438889250686, + "AR_medium": 0.6846285912520943, + "AR_large": 0.7565182102322142 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 922.046, + "std": 3.757, + "p50": 921.247, + "p95": 929.027, + "p99": 935.563, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.08, + "fps_p50": 1.09 + }, + "model_stats": { + "params_millions": 25.3, + "gflops": 36.99 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 15.4 + }, + "metadata": { + "benchmark_date": "2026-06-27T23:31:04.111286Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-m__onnx__cpu__rpi5__20260627T233104111286Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-l-onnx-cpu-rpi5-20260627T234701Z", + "created_at": "2026-06-27T23:47:01.760600Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-l", + "name": "yolox-l", + "family": "yolox", + "variant": "l", + "source": "libreyolo", + "weights": "LibreYOLOXl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5425723281364085, + "mAP_50": 0.7148532433913983, + "mAP_75": 0.5900762125505338, + "mAP_small": 0.3732762177642842, + "mAP_medium": 0.5899402317718047, + "mAP_large": 0.6632153806471186, + "AR1": 0.40963947368940234, + "AR10": 0.6314039307956981, + "AR100": 0.6523117132579677, + "AR_small": 0.4700331331237939, + "AR_medium": 0.6937800073605596, + "AR_large": 0.7703388497310525 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1882.396, + "std": 4.506, + "p50": 1881.406, + "p95": 1890.615, + "p99": 1897.52, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.53, + "fps_p50": 0.53 + }, + "model_stats": { + "params_millions": 54.17, + "gflops": 78.01 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 14.8 + }, + "metadata": { + "benchmark_date": "2026-06-27T23:47:01.760600Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-l__onnx__cpu__rpi5__20260627T234701760600Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-x-onnx-cpu-rpi5-20260628T001402Z", + "created_at": "2026-06-28T00:14:02.617033Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-x", + "name": "yolox-x", + "family": "yolox", + "variant": "x", + "source": "libreyolo", + "weights": "LibreYOLOXx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5486653429671409, + "mAP_50": 0.7238862739256607, + "mAP_75": 0.5983595432292893, + "mAP_small": 0.365999601171563, + "mAP_medium": 0.6026995218171797, + "mAP_large": 0.6743854440852284, + "AR1": 0.41002385524288115, + "AR10": 0.6358502590432511, + "AR100": 0.6558722179507147, + "AR_small": 0.46583175901590107, + "AR_medium": 0.6997710103321302, + "AR_large": 0.7807043628363978 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3200.47, + "std": 5.253, + "p50": 3199.437, + "p95": 3210.456, + "p99": 3217.408, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.31, + "fps_p50": 0.31 + }, + "model_stats": { + "params_millions": 99.02, + "gflops": 141.23 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 21.7 + }, + "metadata": { + "benchmark_date": "2026-06-28T00:14:02.617033Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-x__onnx__cpu__rpi5__20260628T001402617033Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "picodet-s-onnx-cpu-rpi5-20260628T001452Z", + "created_at": "2026-06-28T00:14:52.461958Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-s", + "name": "picodet-s", + "family": "picodet", + "variant": "s", + "source": "libreyolo", + "weights": "LibrePICODETs.pt", + "input_size": 320 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 320, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.2813929119899346, + "mAP_50": 0.41334677585508217, + "mAP_75": 0.2914752838468096, + "mAP_small": 0.10473734019737763, + "mAP_medium": 0.2847293168548949, + "mAP_large": 0.45544267933014243, + "AR1": 0.2507679660084101, + "AR10": 0.3812605234864112, + "AR100": 0.40703112811926967, + "AR_small": 0.13952395056463135, + "AR_medium": 0.44171381382718916, + "AR_large": 0.6117060183789583 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 66.772, + "std": 6.813, + "p50": 66.223, + "p95": 77.632, + "p99": 83.263, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 14.98, + "fps_p50": 15.1 + }, + "model_stats": { + "params_millions": 0.98, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 77.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T00:14:52.461958Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "picodet-s__onnx__cpu__rpi5__20260628T001452461958Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "picodet-m-onnx-cpu-rpi5-20260628T001629Z", + "created_at": "2026-06-28T00:16:29.175431Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-m", + "name": "picodet-m", + "family": "picodet", + "variant": "m", + "source": "libreyolo", + "weights": "LibrePICODETm.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.36518214747227806, + "mAP_50": 0.5150924681561029, + "mAP_75": 0.39078171842360454, + "mAP_small": 0.15625223645887754, + "mAP_medium": 0.4044241653234226, + "mAP_large": 0.5489585415064024, + "AR1": 0.29175206643260554, + "AR10": 0.4556965436968646, + "AR100": 0.48529481762450577, + "AR_small": 0.20353234109992768, + "AR_medium": 0.541150288949737, + "AR_large": 0.6780719958670148 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 160.153, + "std": 13.454, + "p50": 159.777, + "p95": 181.497, + "p99": 193.048, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 6.24, + "fps_p50": 6.26 + }, + "model_stats": { + "params_millions": 2.14, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 76.4 + }, + "metadata": { + "benchmark_date": "2026-06-28T00:16:29.175431Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "picodet-m__onnx__cpu__rpi5__20260628T001629175431Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "picodet-l-onnx-cpu-rpi5-20260628T002141Z", + "created_at": "2026-06-28T00:21:41.253882Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-l", + "name": "picodet-l", + "family": "picodet", + "variant": "l", + "source": "libreyolo", + "weights": "LibrePICODETl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.42670912044377685, + "mAP_50": 0.5936773089219671, + "mAP_75": 0.4612754592116292, + "mAP_small": 0.23112272997138533, + "mAP_medium": 0.4779040568106618, + "mAP_large": 0.5654200117039199, + "AR1": 0.3396315456658758, + "AR10": 0.5344037912182181, + "AR100": 0.5662394426425511, + "AR_small": 0.3343903110723805, + "AR_medium": 0.6121755479346273, + "AR_large": 0.6979159624415997 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 589.472, + "std": 39.594, + "p50": 589.567, + "p95": 654.582, + "p99": 686.693, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.7, + "fps_p50": 1.7 + }, + "model_stats": { + "params_millions": 3.29, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 68.4 + }, + "metadata": { + "benchmark_date": "2026-06-28T00:21:41.253882Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "picodet-l__onnx__cpu__rpi5__20260628T002141253882Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "ec-s-onnx-cpu-rpi5-20260628T003144Z", + "created_at": "2026-06-28T00:31:44.772328Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-s", + "name": "ec-s", + "family": "ec", + "variant": "s", + "source": "libreyolo", + "weights": "LibreECs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5432194719471461, + "mAP_50": 0.7140171161058909, + "mAP_75": 0.5923210155991958, + "mAP_small": 0.3666505350937702, + "mAP_medium": 0.6000801241816464, + "mAP_large": 0.7143119547282032, + "AR1": 0.42118346438115173, + "AR10": 0.6849257047218618, + "AR100": 0.7312556280979691, + "AR_small": 0.5461653398174465, + "AR_medium": 0.7835733460879325, + "AR_large": 0.8876717561873844 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1154.939, + "std": 7.895, + "p50": 1155.213, + "p95": 1159.207, + "p99": 1165.138, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.87, + "fps_p50": 0.87 + }, + "model_stats": { + "params_millions": 9.84, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T00:31:44.772328Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "ec-s__onnx__cpu__rpi5__20260628T003144772328Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "ec-m-onnx-cpu-rpi5-20260628T004627Z", + "created_at": "2026-06-28T00:46:27.037156Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-m", + "name": "ec-m", + "family": "ec", + "variant": "m", + "source": "libreyolo", + "weights": "LibreECm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5835379539521929, + "mAP_50": 0.757759379520078, + "mAP_75": 0.6350328729707719, + "mAP_small": 0.40672995162395603, + "mAP_medium": 0.6388688026086551, + "mAP_large": 0.7501653928844247, + "AR1": 0.43343530313979, + "AR10": 0.7050906565517581, + "AR100": 0.7503141190923781, + "AR_small": 0.5728496539199226, + "AR_medium": 0.7894453607495379, + "AR_large": 0.8827568118289735 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1708.186, + "std": 8.115, + "p50": 1707.671, + "p95": 1713.206, + "p99": 1762.731, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.59, + "fps_p50": 0.59 + }, + "model_stats": { + "params_millions": 19.23, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T00:46:27.037156Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "ec-m__onnx__cpu__rpi5__20260628T004627037156Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "ec-l-onnx-cpu-rpi5-20260628T010919Z", + "created_at": "2026-06-28T01:09:19.277114Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-l", + "name": "ec-l", + "family": "ec", + "variant": "l", + "source": "libreyolo", + "weights": "LibreECl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.6007115783135872, + "mAP_50": 0.7749472354400845, + "mAP_75": 0.6523345711566724, + "mAP_small": 0.43393729112076873, + "mAP_medium": 0.6628074404971023, + "mAP_large": 0.7572737906104186, + "AR1": 0.4375595411244058, + "AR10": 0.7201219301492249, + "AR100": 0.759011988294033, + "AR_small": 0.5846511426384247, + "AR_medium": 0.8139374671717773, + "AR_large": 0.8965742116512401 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2681.797, + "std": 9.081, + "p50": 2680.598, + "p95": 2688.627, + "p99": 2736.943, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.37, + "fps_p50": 0.37 + }, + "model_stats": { + "params_millions": 32.69, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T01:09:19.277114Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "ec-l__onnx__cpu__rpi5__20260628T010919277114Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "ec-x-onnx-cpu-rpi5-20260628T013738Z", + "created_at": "2026-06-28T01:37:38.374712Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-x", + "name": "ec-x", + "family": "ec", + "variant": "x", + "source": "libreyolo", + "weights": "LibreECx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.6113462650555552, + "mAP_50": 0.7856324800583312, + "mAP_75": 0.6655222088638381, + "mAP_small": 0.4359469005574484, + "mAP_medium": 0.676115542202016, + "mAP_large": 0.7790291400209074, + "AR1": 0.4380420119244032, + "AR10": 0.7264733654815458, + "AR100": 0.7625240302226778, + "AR_small": 0.5840608876384857, + "AR_medium": 0.8115087135596564, + "AR_large": 0.9073395719728262 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3331.418, + "std": 10.066, + "p50": 3330.154, + "p95": 3339.355, + "p99": 3387.449, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.3, + "fps_p50": 0.3 + }, + "model_stats": { + "params_millions": 48.99, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.4 + }, + "metadata": { + "benchmark_date": "2026-06-28T01:37:38.374712Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "ec-x__onnx__cpu__rpi5__20260628T013738374712Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "dfine-n-onnx-cpu-rpi5-20260628T013952Z", + "created_at": "2026-06-28T01:39:52.354694Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-n", + "name": "d-fine-n", + "family": "dfine", + "variant": "n", + "source": "libreyolo", + "weights": "LibreDFINEn.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4577159174725761, + "mAP_50": 0.6276159010117028, + "mAP_75": 0.4936186188949874, + "mAP_small": 0.2582314909429722, + "mAP_medium": 0.5148568614812048, + "mAP_large": 0.6164865684657567, + "AR1": 0.3662054162313099, + "AR10": 0.5972180880952037, + "AR100": 0.64941207677179, + "AR_small": 0.4247151375969516, + "AR_medium": 0.7131204074253821, + "AR_large": 0.8464105287851524 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 229.676, + "std": 2.711, + "p50": 229.555, + "p95": 231.445, + "p99": 233.676, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 4.35, + "fps_p50": 4.36 + }, + "model_stats": { + "params_millions": 3.75, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 71.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T01:39:52.354694Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "dfine-n__onnx__cpu__rpi5__20260628T013952354694Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "dfine-s-onnx-cpu-rpi5-20260628T014453Z", + "created_at": "2026-06-28T01:44:53.094863Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-s", + "name": "d-fine-s", + "family": "dfine", + "variant": "s", + "source": "libreyolo", + "weights": "LibreDFINEs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.533843125935515, + "mAP_50": 0.6978897743414051, + "mAP_75": 0.5815144847483065, + "mAP_small": 0.37951088111313763, + "mAP_medium": 0.5786108122038341, + "mAP_large": 0.6712528670597301, + "AR1": 0.40805078036777326, + "AR10": 0.684853567531656, + "AR100": 0.7344177724592036, + "AR_small": 0.5816815390659993, + "AR_medium": 0.7768773182389905, + "AR_large": 0.8765596543653289 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 559.268, + "std": 4.922, + "p50": 559.498, + "p95": 562.537, + "p99": 564.198, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.79, + "fps_p50": 1.79 + }, + "model_stats": { + "params_millions": 10.25, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T01:44:53.094863Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "dfine-s__onnx__cpu__rpi5__20260628T014453094863Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "dfine-m-onnx-cpu-rpi5-20260628T015332Z", + "created_at": "2026-06-28T01:53:32.710054Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-m", + "name": "d-fine-m", + "family": "dfine", + "variant": "m", + "source": "libreyolo", + "weights": "LibreDFINEm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5783006671575653, + "mAP_50": 0.7466815879044602, + "mAP_75": 0.6299110302822055, + "mAP_small": 0.43019262435533856, + "mAP_medium": 0.6140427646392412, + "mAP_large": 0.7292981794777466, + "AR1": 0.4258019573210324, + "AR10": 0.7202899110812604, + "AR100": 0.771521209128276, + "AR_small": 0.6401413616788801, + "AR_medium": 0.7966833420700948, + "AR_large": 0.89146658543394 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 992.427, + "std": 5.017, + "p50": 992.29, + "p95": 996.902, + "p99": 1017.562, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.01, + "fps_p50": 1.01 + }, + "model_stats": { + "params_millions": 19.26, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T01:53:32.710054Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "dfine-m__onnx__cpu__rpi5__20260628T015332710054Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "dfine-l-onnx-cpu-rpi5-20260628T020541Z", + "created_at": "2026-06-28T02:05:41.646388Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-l", + "name": "d-fine-l", + "family": "dfine", + "variant": "l", + "source": "libreyolo", + "weights": "LibreDFINEl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5998638361378038, + "mAP_50": 0.7714133253871575, + "mAP_75": 0.6545275995421832, + "mAP_small": 0.46006035897968195, + "mAP_medium": 0.651843563065576, + "mAP_large": 0.7519489970046525, + "AR1": 0.4438129932529035, + "AR10": 0.7301763319531722, + "AR100": 0.7823271050417004, + "AR_small": 0.6336871651097996, + "AR_medium": 0.8251280046501868, + "AR_large": 0.902008282481585 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1402.913, + "std": 8.691, + "p50": 1402.601, + "p95": 1408.151, + "p99": 1459.521, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.71, + "fps_p50": 0.71 + }, + "model_stats": { + "params_millions": 30.79, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T02:05:41.646388Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "dfine-l__onnx__cpu__rpi5__20260628T020541646388Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "dfine-x-onnx-cpu-rpi5-20260628T022812Z", + "created_at": "2026-06-28T02:28:12.543209Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-x", + "name": "d-fine-x", + "family": "dfine", + "variant": "x", + "source": "libreyolo", + "weights": "LibreDFINEx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.6142388237060353, + "mAP_50": 0.7798523770021832, + "mAP_75": 0.6720056040137532, + "mAP_small": 0.478398640099852, + "mAP_medium": 0.673787867178134, + "mAP_large": 0.7694334953082029, + "AR1": 0.44763907531490205, + "AR10": 0.7492636927367275, + "AR100": 0.79469353444029, + "AR_small": 0.6545081595760803, + "AR_medium": 0.8381553060186311, + "AR_large": 0.9082575658932268 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2638.747, + "std": 32.632, + "p50": 2636.49, + "p95": 2645.381, + "p99": 2700.959, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.38, + "fps_p50": 0.38 + }, + "model_stats": { + "params_millions": 61.71, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T02:28:12.543209Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "dfine-x__onnx__cpu__rpi5__20260628T022812543209Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deim-n-onnx-cpu-rpi5-20260628T023026Z", + "created_at": "2026-06-28T02:30:26.752367Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-n", + "name": "deim-n", + "family": "deim", + "variant": "n", + "source": "libreyolo", + "weights": "LibreDEIMn.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4679439393783107, + "mAP_50": 0.6384369065119964, + "mAP_75": 0.5065966451768882, + "mAP_small": 0.2827452405332132, + "mAP_medium": 0.5239486171961336, + "mAP_large": 0.649292759072888, + "AR1": 0.37475703908357355, + "AR10": 0.6089594734177107, + "AR100": 0.6622745544514113, + "AR_small": 0.4596949926557759, + "AR_medium": 0.7285437095660443, + "AR_large": 0.8547273276141655 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 229.711, + "std": 2.113, + "p50": 229.43, + "p95": 231.366, + "p99": 233.022, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 4.35, + "fps_p50": 4.36 + }, + "model_stats": { + "params_millions": 3.75, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 71.8 + }, + "metadata": { + "benchmark_date": "2026-06-28T02:30:26.752367Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deim-n__onnx__cpu__rpi5__20260628T023026752367Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deim-s-onnx-cpu-rpi5-20260628T023527Z", + "created_at": "2026-06-28T02:35:27.204132Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-s", + "name": "deim-s", + "family": "deim", + "variant": "s", + "source": "libreyolo", + "weights": "LibreDEIMs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5209717892441037, + "mAP_50": 0.6833211071300828, + "mAP_75": 0.5700182819277311, + "mAP_small": 0.3596241748448038, + "mAP_medium": 0.5639596802427718, + "mAP_large": 0.6702399089962287, + "AR1": 0.3980327093777471, + "AR10": 0.6838315860340944, + "AR100": 0.735977169586032, + "AR_small": 0.5491536948619029, + "AR_medium": 0.7814739019262885, + "AR_large": 0.879701448382054 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 559.177, + "std": 4.83, + "p50": 559.209, + "p95": 562.544, + "p99": 568.306, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.79, + "fps_p50": 1.79 + }, + "model_stats": { + "params_millions": 10.25, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T02:35:27.204132Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deim-s__onnx__cpu__rpi5__20260628T023527204132Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deim-m-onnx-cpu-rpi5-20260628T024405Z", + "created_at": "2026-06-28T02:44:05.862998Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-m", + "name": "deim-m", + "family": "deim", + "variant": "m", + "source": "libreyolo", + "weights": "LibreDEIMm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.554380350639803, + "mAP_50": 0.7201200948623354, + "mAP_75": 0.6059586355391078, + "mAP_small": 0.38938230326988665, + "mAP_medium": 0.6047536503404021, + "mAP_large": 0.7154140931221996, + "AR1": 0.4270136586370724, + "AR10": 0.6977532541784438, + "AR100": 0.7412542504872445, + "AR_small": 0.5772622767167025, + "AR_medium": 0.7771076238238673, + "AR_large": 0.8916083181230134 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 990.654, + "std": 6.591, + "p50": 990.591, + "p95": 994.601, + "p99": 1020.687, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.01, + "fps_p50": 1.01 + }, + "model_stats": { + "params_millions": 19.26, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T02:44:05.862998Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deim-m__onnx__cpu__rpi5__20260628T024405862998Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deim-l-onnx-cpu-rpi5-20260628T025613Z", + "created_at": "2026-06-28T02:56:13.259095Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-l", + "name": "deim-l", + "family": "deim", + "variant": "l", + "source": "libreyolo", + "weights": "LibreDEIMl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5784101734004303, + "mAP_50": 0.7554938545399417, + "mAP_75": 0.6393003920178135, + "mAP_small": 0.44463414701490817, + "mAP_medium": 0.6354539408240266, + "mAP_large": 0.7458486451096473, + "AR1": 0.43152856098801823, + "AR10": 0.7128187787446858, + "AR100": 0.7592639126055808, + "AR_small": 0.6187280480226055, + "AR_medium": 0.794876872302488, + "AR_large": 0.8895979531006462 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1400.237, + "std": 7.405, + "p50": 1400.382, + "p95": 1406.819, + "p99": 1422.474, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.71, + "fps_p50": 0.71 + }, + "model_stats": { + "params_millions": 30.79, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T02:56:13.259095Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deim-l__onnx__cpu__rpi5__20260628T025613259095Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deim-x-onnx-cpu-rpi5-20260628T031842Z", + "created_at": "2026-06-28T03:18:42.468270Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-x", + "name": "deim-x", + "family": "deim", + "variant": "x", + "source": "libreyolo", + "weights": "LibreDEIMx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5961960308925123, + "mAP_50": 0.7700141120836566, + "mAP_75": 0.6562959986729319, + "mAP_small": 0.4559131099923876, + "mAP_medium": 0.6569733791673753, + "mAP_large": 0.7668436019384222, + "AR1": 0.4445621917232395, + "AR10": 0.7250154079634621, + "AR100": 0.7705422586062296, + "AR_small": 0.6212383765698645, + "AR_medium": 0.81244331327091, + "AR_large": 0.9076892664639177 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2635.563, + "std": 10.373, + "p50": 2634.795, + "p95": 2643.715, + "p99": 2697.274, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.38, + "fps_p50": 0.38 + }, + "model_stats": { + "params_millions": 61.71, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:18:42.468270Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deim-x__onnx__cpu__rpi5__20260628T031842468270Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-atto-onnx-cpu-rpi5-20260628T031917Z", + "created_at": "2026-06-28T03:19:17.141261Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-atto", + "name": "deimv2-atto", + "family": "deimv2", + "variant": "atto", + "source": "libreyolo", + "weights": "LibreDEIMv2atto.pt", + "input_size": 320 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 320, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.2748653297512432, + "mAP_50": 0.4030736022619007, + "mAP_75": 0.29403623096869114, + "mAP_small": 0.0812842855548296, + "mAP_medium": 0.3051796806318778, + "mAP_large": 0.4290115385111966, + "AR1": 0.2591026420470867, + "AR10": 0.4353053743391541, + "AR100": 0.469344423710311, + "AR_small": 0.134311368254856, + "AR_medium": 0.5430335808039892, + "AR_large": 0.7063349233643775 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 33.024, + "std": 0.75, + "p50": 33.067, + "p95": 34.206, + "p99": 34.749, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 30.28, + "fps_p50": 30.24 + }, + "model_stats": { + "params_millions": 0.49, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 75.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:19:17.141261Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-atto__onnx__cpu__rpi5__20260628T031917141261Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-femto-onnx-cpu-rpi5-20260628T032005Z", + "created_at": "2026-06-28T03:20:05.516190Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-femto", + "name": "deimv2-femto", + "family": "deimv2", + "variant": "femto", + "source": "libreyolo", + "weights": "LibreDEIMv2femto.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.3450873229627451, + "mAP_50": 0.4889703158548694, + "mAP_75": 0.37426314484979606, + "mAP_small": 0.16186424592475104, + "mAP_medium": 0.3767184765437616, + "mAP_large": 0.5210417081877327, + "AR1": 0.30103618458050724, + "AR10": 0.5085654916447838, + "AR100": 0.5517491879578669, + "AR_small": 0.2854844141323229, + "AR_medium": 0.6055178230259108, + "AR_large": 0.7861551867618176 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 59.787, + "std": 1.787, + "p50": 59.634, + "p95": 61.294, + "p99": 62.42, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 16.73, + "fps_p50": 16.77 + }, + "model_stats": { + "params_millions": 0.96, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 73.4 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:20:05.516190Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-femto__onnx__cpu__rpi5__20260628T032005516190Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-pico-onnx-cpu-rpi5-20260628T032145Z", + "created_at": "2026-06-28T03:21:45.228502Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-pico", + "name": "deimv2-pico", + "family": "deimv2", + "variant": "pico", + "source": "libreyolo", + "weights": "LibreDEIMv2pico.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4224854946316034, + "mAP_50": 0.5929091004343798, + "mAP_75": 0.45616100220090755, + "mAP_small": 0.26532966904860034, + "mAP_medium": 0.4668541396371766, + "mAP_large": 0.5649540356513841, + "AR1": 0.3475074208313628, + "AR10": 0.5775063386549875, + "AR100": 0.631145447447673, + "AR_small": 0.3960639942633518, + "AR_medium": 0.6938846662696448, + "AR_large": 0.8284719440079611 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 161.527, + "std": 1.661, + "p50": 162.106, + "p95": 163.395, + "p99": 165.149, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 6.19, + "fps_p50": 6.17 + }, + "model_stats": { + "params_millions": 1.51, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 76.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:21:45.228502Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-pico__onnx__cpu__rpi5__20260628T032145228502Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-n-onnx-cpu-rpi5-20260628T032357Z", + "created_at": "2026-06-28T03:23:57.810370Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-n", + "name": "deimv2-n", + "family": "deimv2", + "variant": "n", + "source": "libreyolo", + "weights": "LibreDEIMv2n.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.46702654729561943, + "mAP_50": 0.6399052553560688, + "mAP_75": 0.5054600012954302, + "mAP_small": 0.28811860923654115, + "mAP_medium": 0.5139653255366282, + "mAP_large": 0.6324793538325342, + "AR1": 0.38158229768106366, + "AR10": 0.60698925868329, + "AR100": 0.6653036888284473, + "AR_small": 0.43434618816664816, + "AR_medium": 0.7275925487620608, + "AR_large": 0.8386129454975145 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 226.26, + "std": 1.702, + "p50": 226.489, + "p95": 228.449, + "p99": 229.931, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 4.42, + "fps_p50": 4.42 + }, + "model_stats": { + "params_millions": 3.56, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 71.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:23:57.810370Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-n__onnx__cpu__rpi5__20260628T032357810370Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-s-onnx-cpu-rpi5-20260628T033405Z", + "created_at": "2026-06-28T03:34:05.534056Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-s", + "name": "deimv2-s", + "family": "deimv2", + "variant": "s", + "source": "libreyolo", + "weights": "LibreDEIMv2s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5300700271711015, + "mAP_50": 0.6976653903900368, + "mAP_75": 0.5751331009055647, + "mAP_small": 0.3500893223080657, + "mAP_medium": 0.5804289777484958, + "mAP_large": 0.7299875093807994, + "AR1": 0.4065742563193488, + "AR10": 0.6752756526406756, + "AR100": 0.7241649181407003, + "AR_small": 0.5444134233119735, + "AR_medium": 0.7724843253587157, + "AR_large": 0.8780156970954397 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1162.774, + "std": 6.066, + "p50": 1162.821, + "p95": 1167.175, + "p99": 1173.671, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.86, + "fps_p50": 0.86 + }, + "model_stats": { + "params_millions": 9.73, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:34:05.534056Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-s__onnx__cpu__rpi5__20260628T033405534056Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-m-onnx-cpu-rpi5-20260628T034843Z", + "created_at": "2026-06-28T03:48:43.353689Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-m", + "name": "deimv2-m", + "family": "deimv2", + "variant": "m", + "source": "libreyolo", + "weights": "LibreDEIMv2m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5601263467392655, + "mAP_50": 0.7292297490227051, + "mAP_75": 0.6153556104085471, + "mAP_small": 0.38810204225815637, + "mAP_medium": 0.6023119533013779, + "mAP_large": 0.7215611191519414, + "AR1": 0.42507423593101595, + "AR10": 0.6943088231150157, + "AR100": 0.7368570541319415, + "AR_small": 0.5641964407869676, + "AR_medium": 0.774128610329005, + "AR_large": 0.8771622951355976 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1699.669, + "std": 15.315, + "p50": 1698.565, + "p95": 1704.212, + "p99": 1755.324, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.59, + "fps_p50": 0.59 + }, + "model_stats": { + "params_millions": 18.13, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:48:43.353689Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-m__onnx__cpu__rpi5__20260628T034843353689Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-l-onnx-cpu-rpi5-20260628T041134Z", + "created_at": "2026-06-28T04:11:34.309534Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-l", + "name": "deimv2-l", + "family": "deimv2", + "variant": "l", + "source": "libreyolo", + "weights": "LibreDEIMv2l.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5857056480114364, + "mAP_50": 0.756122518317119, + "mAP_75": 0.6423894397306, + "mAP_small": 0.4237311254486432, + "mAP_medium": 0.6448895711462656, + "mAP_large": 0.755484145872126, + "AR1": 0.43021983010021314, + "AR10": 0.7119366639781503, + "AR100": 0.7559290439408436, + "AR_small": 0.6041215166550952, + "AR_medium": 0.7913091585542085, + "AR_large": 0.883534820509406 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2678.98, + "std": 27.251, + "p50": 2676.912, + "p95": 2683.508, + "p99": 2732.584, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.37, + "fps_p50": 0.37 + }, + "model_stats": { + "params_millions": 32.19, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T04:11:34.309534Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-l__onnx__cpu__rpi5__20260628T041134309534Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-x-onnx-cpu-rpi5-20260628T044058Z", + "created_at": "2026-06-28T04:40:58.635016Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-x", + "name": "deimv2-x", + "family": "deimv2", + "variant": "x", + "source": "libreyolo", + "weights": "LibreDEIMv2x.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.6133233772282629, + "mAP_50": 0.7894398948856872, + "mAP_75": 0.6727366504784924, + "mAP_small": 0.4339764082361583, + "mAP_medium": 0.6737693410194326, + "mAP_large": 0.7656487806361741, + "AR1": 0.4453632538830613, + "AR10": 0.7287634403924469, + "AR100": 0.7705460192274229, + "AR_small": 0.6192627518682189, + "AR_medium": 0.8184399305818361, + "AR_large": 0.8827316250588547 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3453.025, + "std": 11.258, + "p50": 3451.917, + "p95": 3461.03, + "p99": 3514.227, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.29, + "fps_p50": 0.29 + }, + "model_stats": { + "params_millions": 50.27, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T04:40:58.635016Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-x__onnx__cpu__rpi5__20260628T044058635016Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r18-onnx-cpu-rpi5-20260628T044752Z", + "created_at": "2026-06-28T04:47:52.276336Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r18", + "name": "rt-detr-r18", + "family": "rtdetr", + "variant": "r18", + "source": "libreyolo", + "weights": "LibreRTDETRr18.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.49774256714404225, + "mAP_50": 0.6634992177231226, + "mAP_75": 0.5400484427487215, + "mAP_small": 0.3261582121122263, + "mAP_medium": 0.548562025815511, + "mAP_large": 0.6408920088998157, + "AR1": 0.39604668829512685, + "AR10": 0.664007747539048, + "AR100": 0.7230544546382387, + "AR_small": 0.5380604638906232, + "AR_medium": 0.7700151640261632, + "AR_large": 0.8588714644727942 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 786.553, + "std": 8.765, + "p50": 782.758, + "p95": 803.468, + "p99": 819.171, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.27, + "fps_p50": 1.28 + }, + "model_stats": { + "params_millions": 20.11, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 70.8 + }, + "metadata": { + "benchmark_date": "2026-06-28T04:47:52.276336Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetr-r18__onnx__cpu__rpi5__20260628T044752276336Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r34-onnx-cpu-rpi5-20260628T045733Z", + "created_at": "2026-06-28T04:57:33.192435Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r34", + "name": "rt-detr-r34", + "family": "rtdetr", + "variant": "r34", + "source": "libreyolo", + "weights": "LibreRTDETRr34.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.522144868701534, + "mAP_50": 0.6962817889643671, + "mAP_75": 0.5673028292737368, + "mAP_small": 0.34963723196781965, + "mAP_medium": 0.5588247724308356, + "mAP_large": 0.6674805428302818, + "AR1": 0.409745659514483, + "AR10": 0.668504721728897, + "AR100": 0.7220680557653572, + "AR_small": 0.54067270344758, + "AR_medium": 0.7543034638396552, + "AR_large": 0.8734680809296462 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1118.681, + "std": 11.285, + "p50": 1114.813, + "p95": 1136.01, + "p99": 1169.664, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.89, + "fps_p50": 0.9 + }, + "model_stats": { + "params_millions": 31.34, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 70.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T04:57:33.192435Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetr-r34__onnx__cpu__rpi5__20260628T045733192435Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r50-onnx-cpu-rpi5-20260628T051244Z", + "created_at": "2026-06-28T05:12:44.059755Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r50", + "name": "rt-detr-r50", + "family": "rtdetr", + "variant": "r50", + "source": "libreyolo", + "weights": "LibreRTDETRr50.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5586474163642929, + "mAP_50": 0.7304232780632504, + "mAP_75": 0.6166275682858972, + "mAP_small": 0.38254731498765926, + "mAP_medium": 0.6108448756694206, + "mAP_large": 0.7033294309172166, + "AR1": 0.42091531825728223, + "AR10": 0.6986371536602564, + "AR100": 0.7468353502304153, + "AR_small": 0.5821806901578028, + "AR_medium": 0.7880349995633875, + "AR_large": 0.8820373961923116 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1772.895, + "std": 9.627, + "p50": 1770.223, + "p95": 1788.033, + "p99": 1818.512, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.56, + "fps_p50": 0.56 + }, + "model_stats": { + "params_millions": 42.81, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T05:12:44.059755Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetr-r50__onnx__cpu__rpi5__20260628T051244059755Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r50m-onnx-cpu-rpi5-20260628T052410Z", + "created_at": "2026-06-28T05:24:10.847249Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r50m", + "name": "rt-detr-r50m", + "family": "rtdetr", + "variant": "r50m", + "source": "libreyolo", + "weights": "LibreRTDETRr50m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5381186274201499, + "mAP_50": 0.7122493094497363, + "mAP_75": 0.5848035966081955, + "mAP_small": 0.3763542586544715, + "mAP_medium": 0.5938425477922029, + "mAP_large": 0.6904993799904184, + "AR1": 0.417359729259557, + "AR10": 0.6780930703428721, + "AR100": 0.7300127669264833, + "AR_small": 0.5776721345625127, + "AR_medium": 0.7669397771663522, + "AR_large": 0.8806901143915012 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1329.009, + "std": 10.193, + "p50": 1326.111, + "p95": 1343.352, + "p99": 1371.406, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.75, + "fps_p50": 0.75 + }, + "model_stats": { + "params_millions": 33.13, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.2 + }, + "metadata": { + "benchmark_date": "2026-06-28T05:24:10.847249Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetr-r50m__onnx__cpu__rpi5__20260628T052410847249Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r101-onnx-cpu-rpi5-20260628T054948Z", + "created_at": "2026-06-28T05:49:48.877275Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r101", + "name": "rt-detr-r101", + "family": "rtdetr", + "variant": "r101", + "source": "libreyolo", + "weights": "LibreRTDETRr101.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5674425526984037, + "mAP_50": 0.7395636718774186, + "mAP_75": 0.6178049152885773, + "mAP_small": 0.3919726645932447, + "mAP_medium": 0.6204339342414583, + "mAP_large": 0.7171567835394471, + "AR1": 0.4252468321189817, + "AR10": 0.707393435689367, + "AR100": 0.7551884403445683, + "AR_small": 0.5989601353874838, + "AR_medium": 0.7916745721758722, + "AR_large": 0.8760709999518368 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3019.38, + "std": 13.842, + "p50": 3016.969, + "p95": 3035.793, + "p99": 3091.323, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.33, + "fps_p50": 0.33 + }, + "model_stats": { + "params_millions": 76.52, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.2 + }, + "metadata": { + "benchmark_date": "2026-06-28T05:49:48.877275Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetr-r101__onnx__cpu__rpi5__20260628T054948877275Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-l-onnx-cpu-rpi5-20260628T060222Z", + "created_at": "2026-06-28T06:02:22.619689Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-l", + "name": "rt-detr-l", + "family": "rtdetr", + "variant": "l", + "source": "libreyolo", + "weights": "LibreRTDETRl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5577690093694818, + "mAP_50": 0.7370566478233471, + "mAP_75": 0.611902700368415, + "mAP_small": 0.39158012790306695, + "mAP_medium": 0.5925269295111903, + "mAP_large": 0.7300745279615439, + "AR1": 0.4180303327747421, + "AR10": 0.6967882828105793, + "AR100": 0.7420949806172361, + "AR_small": 0.5930121962336482, + "AR_medium": 0.7707117941164834, + "AR_large": 0.8881804848784471 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1460.995, + "std": 11.908, + "p50": 1457.491, + "p95": 1478.655, + "p99": 1514.473, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.68, + "fps_p50": 0.69 + }, + "model_stats": { + "params_millions": 32.84, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T06:02:22.619689Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetr-l__onnx__cpu__rpi5__20260628T060222619689Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-x-onnx-cpu-rpi5-20260628T062611Z", + "created_at": "2026-06-28T06:26:11.065238Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-x", + "name": "rt-detr-x", + "family": "rtdetr", + "variant": "x", + "source": "libreyolo", + "weights": "LibreRTDETRx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5794524359370877, + "mAP_50": 0.760353887037982, + "mAP_75": 0.6333719411593034, + "mAP_small": 0.3976324086492176, + "mAP_medium": 0.6439313968855763, + "mAP_large": 0.7254609070440775, + "AR1": 0.432528983655378, + "AR10": 0.7126273184401586, + "AR100": 0.7616617701826264, + "AR_small": 0.6011353676420481, + "AR_medium": 0.805626655986392, + "AR_large": 0.8834554777655631 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2801.481, + "std": 14.959, + "p50": 2798.131, + "p95": 2828.051, + "p99": 2862.928, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.36, + "fps_p50": 0.36 + }, + "model_stats": { + "params_millions": 67.33, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.2 + }, + "metadata": { + "benchmark_date": "2026-06-28T06:26:11.065238Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetr-x__onnx__cpu__rpi5__20260628T062611065238Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r18-onnx-cpu-rpi5-20260628T063306Z", + "created_at": "2026-06-28T06:33:06.884082Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r18", + "name": "rt-detrv2-r18", + "family": "rtdetrv2", + "variant": "r18", + "source": "libreyolo", + "weights": "LibreRTDETRv2r18.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5075814715671121, + "mAP_50": 0.6738984732532675, + "mAP_75": 0.5495938940196975, + "mAP_small": 0.33501169940322056, + "mAP_medium": 0.5391037819664849, + "mAP_large": 0.6640383390452526, + "AR1": 0.3949345357955365, + "AR10": 0.6750040230328294, + "AR100": 0.7279190894521826, + "AR_small": 0.5576572291262796, + "AR_medium": 0.7556398310786503, + "AR_large": 0.8596387462926546 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 791.233, + "std": 9.419, + "p50": 787.849, + "p95": 807.113, + "p99": 833.335, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.26, + "fps_p50": 1.27 + }, + "model_stats": { + "params_millions": 20.15, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 70.8 + }, + "metadata": { + "benchmark_date": "2026-06-28T06:33:06.884082Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv2-r18__onnx__cpu__rpi5__20260628T063306884082Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r34-onnx-cpu-rpi5-20260628T064249Z", + "created_at": "2026-06-28T06:42:49.407646Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r34", + "name": "rt-detrv2-r34", + "family": "rtdetrv2", + "variant": "r34", + "source": "libreyolo", + "weights": "LibreRTDETRv2r34.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5318119413094168, + "mAP_50": 0.7018691549392625, + "mAP_75": 0.5781553635714163, + "mAP_small": 0.35549933093655, + "mAP_medium": 0.5762248642955541, + "mAP_large": 0.6739853756707758, + "AR1": 0.4055715211295446, + "AR10": 0.6645740019115106, + "AR100": 0.7204275773514845, + "AR_small": 0.5658065214832655, + "AR_medium": 0.7545120243239197, + "AR_large": 0.8550972711754022 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1122.153, + "std": 11.941, + "p50": 1117.447, + "p95": 1143.4, + "p99": 1166.716, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.89, + "fps_p50": 0.89 + }, + "model_stats": { + "params_millions": 31.38, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 70.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T06:42:49.407646Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv2-r34__onnx__cpu__rpi5__20260628T064249407646Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r50-onnx-cpu-rpi5-20260628T065806Z", + "created_at": "2026-06-28T06:58:06.042017Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r50", + "name": "rt-detrv2-r50", + "family": "rtdetrv2", + "variant": "r50", + "source": "libreyolo", + "weights": "LibreRTDETRv2r50.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5569738100824244, + "mAP_50": 0.728395197258972, + "mAP_75": 0.6056225695539527, + "mAP_small": 0.3896959314468154, + "mAP_medium": 0.6038698896109826, + "mAP_large": 0.718780877749079, + "AR1": 0.41980268254801273, + "AR10": 0.6880983552288026, + "AR100": 0.7403090572956712, + "AR_small": 0.5962663669882283, + "AR_medium": 0.778614386279377, + "AR_large": 0.8723433753727128 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1785.296, + "std": 12.194, + "p50": 1782.492, + "p95": 1801.697, + "p99": 1845.503, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.56, + "fps_p50": 0.56 + }, + "model_stats": { + "params_millions": 42.85, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T06:58:06.042017Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv2-r50__onnx__cpu__rpi5__20260628T065806042017Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r50m-onnx-cpu-rpi5-20260628T070931Z", + "created_at": "2026-06-28T07:09:31.410491Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r50m", + "name": "rt-detrv2-r50m", + "family": "rtdetrv2", + "variant": "r50m", + "source": "libreyolo", + "weights": "LibreRTDETRv2r50m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5476645770436055, + "mAP_50": 0.7176598191789246, + "mAP_75": 0.5991054847047382, + "mAP_small": 0.37988141287131805, + "mAP_medium": 0.6013683390782228, + "mAP_large": 0.6934279367189854, + "AR1": 0.41156813370207107, + "AR10": 0.6841501952835427, + "AR100": 0.7383814121531431, + "AR_small": 0.5873760190188806, + "AR_medium": 0.7797841209975578, + "AR_large": 0.8766520295342554 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1326.47, + "std": 9.967, + "p50": 1323.441, + "p95": 1340.463, + "p99": 1364.549, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.75, + "fps_p50": 0.76 + }, + "model_stats": { + "params_millions": 33.17, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T07:09:31.410491Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv2-r50m__onnx__cpu__rpi5__20260628T070931410491Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r101-onnx-cpu-rpi5-20260628T073520Z", + "created_at": "2026-06-28T07:35:20.903988Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r101", + "name": "rt-detrv2-r101", + "family": "rtdetrv2", + "variant": "r101", + "source": "libreyolo", + "weights": "LibreRTDETRv2r101.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5674425531042802, + "mAP_50": 0.7395636718774186, + "mAP_75": 0.6178049152885773, + "mAP_small": 0.3919726676626625, + "mAP_medium": 0.6204339342414583, + "mAP_large": 0.7171567835394471, + "AR1": 0.4252468321189817, + "AR10": 0.707393435689367, + "AR100": 0.7551884403445683, + "AR_small": 0.5989601353874838, + "AR_medium": 0.7916745721758722, + "AR_large": 0.8760709999518368 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3042.678, + "std": 13.364, + "p50": 3040.4, + "p95": 3058.157, + "p99": 3112.795, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.33, + "fps_p50": 0.33 + }, + "model_stats": { + "params_millions": 76.56, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T07:35:20.903988Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv2-r101__onnx__cpu__rpi5__20260628T073520903988Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-s-onnx-cpu-rpi5-20260628T074023Z", + "created_at": "2026-06-28T07:40:23.398916Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-s", + "name": "rt-detrv4-s", + "family": "rtdetrv4", + "variant": "s", + "source": "libreyolo", + "weights": "LibreRTDETRv4s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5282822600337147, + "mAP_50": 0.6994595490451228, + "mAP_75": 0.5726015461943361, + "mAP_small": 0.34987476785569926, + "mAP_medium": 0.5703623254772469, + "mAP_large": 0.6837499344307502, + "AR1": 0.408956514400302, + "AR10": 0.6752355179616181, + "AR100": 0.7297513713228271, + "AR_small": 0.5313628472015742, + "AR_medium": 0.7731283222221518, + "AR_large": 0.880861480555594 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 562.506, + "std": 4.006, + "p50": 562.616, + "p95": 566.024, + "p99": 568.437, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.78, + "fps_p50": 1.78 + }, + "model_stats": { + "params_millions": 10.29, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T07:40:23.398916Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv4-s__onnx__cpu__rpi5__20260628T074023398916Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-m-onnx-cpu-rpi5-20260628T074910Z", + "created_at": "2026-06-28T07:49:10.221350Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-m", + "name": "rt-detrv4-m", + "family": "rtdetrv4", + "variant": "m", + "source": "libreyolo", + "weights": "LibreRTDETRv4m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5651023536810272, + "mAP_50": 0.7295076922166904, + "mAP_75": 0.6213825312048349, + "mAP_small": 0.3946394777715095, + "mAP_medium": 0.6083528341161982, + "mAP_large": 0.7333848441893696, + "AR1": 0.4209735862228707, + "AR10": 0.7026556941131794, + "AR100": 0.7485848272493711, + "AR_small": 0.5869271822546797, + "AR_medium": 0.787922916284933, + "AR_large": 0.8845628241437012 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1006.491, + "std": 6.366, + "p50": 1006.394, + "p95": 1010.511, + "p99": 1013.926, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.99, + "fps_p50": 0.99 + }, + "model_stats": { + "params_millions": 19.52, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T07:49:10.221350Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv4-m__onnx__cpu__rpi5__20260628T074910221350Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-l-onnx-cpu-rpi5-20260628T080131Z", + "created_at": "2026-06-28T08:01:31.290707Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-l", + "name": "rt-detrv4-l", + "family": "rtdetrv4", + "variant": "l", + "source": "libreyolo", + "weights": "LibreRTDETRv4l.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5778216814272358, + "mAP_50": 0.7442366630340385, + "mAP_75": 0.6334281763674426, + "mAP_small": 0.4212308683801451, + "mAP_medium": 0.6364238083168491, + "mAP_large": 0.7449359377371503, + "AR1": 0.4357236915112565, + "AR10": 0.7176614125783658, + "AR100": 0.7650717295248021, + "AR_small": 0.6090221864066407, + "AR_medium": 0.8011127339685521, + "AR_large": 0.9102406922726382 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1427.403, + "std": 28.953, + "p50": 1426.034, + "p95": 1431.835, + "p99": 1476.507, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.7, + "fps_p50": 0.7 + }, + "model_stats": { + "params_millions": 31.19, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:01:31.290707Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv4-l__onnx__cpu__rpi5__20260628T080131290707Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-x-onnx-cpu-rpi5-20260628T082425Z", + "created_at": "2026-06-28T08:24:25.591936Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-x", + "name": "rt-detrv4-x", + "family": "rtdetrv4", + "variant": "x", + "source": "libreyolo", + "weights": "LibreRTDETRv4x.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5999021600251853, + "mAP_50": 0.7715643159289338, + "mAP_75": 0.6587334327811359, + "mAP_small": 0.44167293302601235, + "mAP_medium": 0.6528721192223313, + "mAP_large": 0.7413916074464995, + "AR1": 0.4446619265526469, + "AR10": 0.7257632575706068, + "AR100": 0.775710265507648, + "AR_small": 0.6309413433178404, + "AR_medium": 0.8141752623679169, + "AR_large": 0.8955351216107611 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2685.676, + "std": 11.64, + "p50": 2685.113, + "p95": 2692.773, + "p99": 2750.138, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.37, + "fps_p50": 0.37 + }, + "model_stats": { + "params_millions": 62.6, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:24:25.591936Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv4-x__onnx__cpu__rpi5__20260628T082425591936Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-nano-ncnn-cpu-rpi5-20260628T083125Z", + "created_at": "2026-06-28T08:31:25.436356Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-nano", + "name": "yolox-nano", + "family": "yolox", + "variant": "nano", + "source": "libreyolo", + "weights": "LibreYOLOXn.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.26778154230434653, + "mAP_50": 0.4133182842310963, + "mAP_75": 0.27991306032274327, + "mAP_small": 0.1112425154779287, + "mAP_medium": 0.2604675357298222, + "mAP_large": 0.4036815524243401, + "AR1": 0.23620923484312897, + "AR10": 0.3646138903045786, + "AR100": 0.38732800969923664, + "AR_small": 0.17069738657984965, + "AR_medium": 0.40244485710832145, + "AR_large": 0.5372784676862987 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 32.581, + "std": 3.705, + "p50": 31.564, + "p95": 39.828, + "p99": 46.408, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 30.69, + "fps_p50": 31.68 + }, + "model_stats": { + "params_millions": 0.91, + "gflops": 1.32 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 31.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:31:25.436356Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-nano__ncnn__cpu__rpi5__20260628T083125436356Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-tiny-ncnn-cpu-rpi5-20260628T083235Z", + "created_at": "2026-06-28T08:32:35.583918Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-tiny", + "name": "yolox-tiny", + "family": "yolox", + "variant": "tiny", + "source": "libreyolo", + "weights": "LibreYOLOXt.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.34018710868068724, + "mAP_50": 0.515197185375451, + "mAP_75": 0.35530815204888655, + "mAP_small": 0.1624423684857395, + "mAP_medium": 0.35967145151208013, + "mAP_large": 0.48609589032951545, + "AR1": 0.27907472341163525, + "AR10": 0.43768103350807686, + "AR100": 0.46013456915204903, + "AR_small": 0.24814593704685403, + "AR_medium": 0.4975632162219642, + "AR_large": 0.6246945608892681 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 54.281, + "std": 3.4, + "p50": 53.36, + "p95": 61.778, + "p99": 66.277, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 18.42, + "fps_p50": 18.74 + }, + "model_stats": { + "params_millions": 5.06, + "gflops": 7.68 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 20.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:32:35.583918Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-tiny__ncnn__cpu__rpi5__20260628T083235583918Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-s-ncnn-cpu-rpi5-20260628T083436Z", + "created_at": "2026-06-28T08:34:36.014518Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-s", + "name": "yolox-s", + "family": "yolox", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLOXs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4185662911362574, + "mAP_50": 0.6069472657595824, + "mAP_75": 0.4579735482952389, + "mAP_small": 0.24747404414796903, + "mAP_medium": 0.4530628388606526, + "mAP_large": 0.5451185371505978, + "AR1": 0.3388265661320211, + "AR10": 0.5256455820552539, + "AR100": 0.5539892971320937, + "AR_small": 0.37409886749020593, + "AR_medium": 0.5888357441443937, + "AR_large": 0.6705650528404357 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 174.294, + "std": 6.457, + "p50": 172.492, + "p95": 186.237, + "p99": 198.885, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 5.74, + "fps_p50": 5.8 + }, + "model_stats": { + "params_millions": 8.97, + "gflops": 13.46 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 35.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:34:36.014518Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-s__ncnn__cpu__rpi5__20260628T083436014518Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "picodet-s-ncnn-cpu-rpi5-20260628T084729Z", + "created_at": "2026-06-28T08:47:29.796838Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-s", + "name": "picodet-s", + "family": "picodet", + "variant": "s", + "source": "libreyolo", + "weights": "LibrePICODETs.pt", + "input_size": 320 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 320, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.27668476253288005, + "mAP_50": 0.4081527727174869, + "mAP_75": 0.2872617489863077, + "mAP_small": 0.10346510842730021, + "mAP_medium": 0.28694097782031575, + "mAP_large": 0.450153507877427, + "AR1": 0.24787276519511137, + "AR10": 0.3773122272971989, + "AR100": 0.4010293646530582, + "AR_small": 0.1368677435465607, + "AR_medium": 0.4374935072503024, + "AR_large": 0.6070085634595899 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 53.34, + "std": 6.732, + "p50": 52.9, + "p95": 63.992, + "p99": 69.984, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 18.75, + "fps_p50": 18.9 + }, + "model_stats": { + "params_millions": 0.0, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 73.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:47:29.796838Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "picodet-s__ncnn__cpu__rpi5__20260628T084729796838Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "picodet-m-ncnn-cpu-rpi5-20260628T084859Z", + "created_at": "2026-06-28T08:48:59.231781Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-m", + "name": "picodet-m", + "family": "picodet", + "variant": "m", + "source": "libreyolo", + "weights": "LibrePICODETm.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.3644691634603584, + "mAP_50": 0.5152405901703948, + "mAP_75": 0.38933970019931985, + "mAP_small": 0.15878492232794844, + "mAP_medium": 0.3980997373437907, + "mAP_large": 0.546612926981912, + "AR1": 0.2918425050346936, + "AR10": 0.45549597432797595, + "AR100": 0.48503044648524485, + "AR_small": 0.2113143560310581, + "AR_medium": 0.5345845813031153, + "AR_large": 0.689623196708951 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 117.78, + "std": 13.634, + "p50": 117.036, + "p95": 138.628, + "p99": 151.139, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 8.49, + "fps_p50": 8.54 + }, + "model_stats": { + "params_millions": 0.0, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 76.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:48:59.231781Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "picodet-m__ncnn__cpu__rpi5__20260628T084859231781Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "picodet-l-ncnn-cpu-rpi5-20260628T091505Z", + "created_at": "2026-06-28T09:15:05.814759Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-l", + "name": "picodet-l", + "family": "picodet", + "variant": "l", + "source": "libreyolo", + "weights": "LibrePICODETl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.42770296733531016, + "mAP_50": 0.5944168351919669, + "mAP_75": 0.46411415029057845, + "mAP_small": 0.23192866375154145, + "mAP_medium": 0.47685405613134785, + "mAP_large": 0.5661665410866924, + "AR1": 0.3405386329048061, + "AR10": 0.5360245143308368, + "AR100": 0.5679978126498423, + "AR_small": 0.33515737612793667, + "AR_medium": 0.6114916185523256, + "AR_large": 0.699992934376178 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 441.829, + "std": 39.479, + "p50": 443.541, + "p95": 506.988, + "p99": 537.778, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 2.26, + "fps_p50": 2.25 + }, + "model_stats": { + "params_millions": 0.0, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.2 + }, + "metadata": { + "benchmark_date": "2026-06-28T09:15:05.814759Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "picodet-l__ncnn__cpu__rpi5__20260628T091505814759Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-m-ncnn-cpu-rpi5-20260628T093306Z", + "created_at": "2026-06-28T09:33:06.098833Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-m", + "name": "yolox-m", + "family": "yolox", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLOXm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4973637579387121, + "mAP_50": 0.6778704655888971, + "mAP_75": 0.5436906487170675, + "mAP_small": 0.31767712052616864, + "mAP_medium": 0.5632789389188162, + "mAP_large": 0.6350061189476125, + "AR1": 0.3775142117444723, + "AR10": 0.6106086415880172, + "AR100": 0.6354199496981533, + "AR_small": 0.4326868277422986, + "AR_medium": 0.6788045896586901, + "AR_large": 0.7524180746189737 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 340.284, + "std": 7.7, + "p50": 338.348, + "p95": 351.333, + "p99": 372.308, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 2.94, + "fps_p50": 2.96 + }, + "model_stats": { + "params_millions": 25.33, + "gflops": 36.99 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T09:33:06.098833Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-m__ncnn__cpu__rpi5__20260628T093306098833Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolonas-s-pytorch-cuda-jetson_orin-20260628T094453Z", + "created_at": "2026-06-28T09:44:53.408459Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0.dev0", + "libreyolo_commit": "5f12ee96d5f3099eaede265187a5df70a5f70c08" + }, + "model": { + "id": "yolonas-s", + "name": "yolo-nas-s", + "family": "yolonas", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLONASs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Orin (nvgpu)", + "gpu_memory_gb": 0.0, + "driver_version": "595.78", + "cuda_version": "13.0", + "cpu": "Unknown", + "cpu_cores": 6, + "ram_gb": 7, + "id": "jetson_orin" + }, + "software": { + "python": "3.12.3", + "torch": "2.12.0+cu130", + "libreyolo": "1.2.0.dev0", + "libreyolo_commit": "5f12ee96d5f3099eaede265187a5df70a5f70c08", + "onnxruntime": "1.24.0" + }, + "accuracy": { + "mAP_50_95": 0.5175123196146053, + "mAP_50": 0.6899003979069985, + "mAP_75": 0.565893842401729, + "mAP_small": 0.3121718608086119, + "mAP_medium": 0.5805600595913436, + "mAP_large": 0.6776601702081991, + "AR1": 0.3996041197390947, + "AR10": 0.6268054217417672, + "AR100": 0.658152170495466, + "AR_small": 0.425049605010031, + "AR_medium": 0.7156647993447107, + "AR_large": 0.8077122859175214 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 98.002, + "std": 20.79, + "p50": 96.547, + "p95": 98.904, + "p99": 101.173, + "preprocess_ms": 15.853, + "inference_ms": 75.994, + "postprocess_ms": 6.155 + } + }, + "throughput": { + "fps_mean": 10.2, + "fps_p50": 10.36 + }, + "model_stats": { + "params_millions": 19.05, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 172.7, + "peak_ram_mb": 227.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T09:44:53.408459Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0.dev0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cuda", + "device": "gpu" + }, + "source_file": "yolonas-s__pytorch__cuda__jetson_orin__20260628T094453408459Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolonas-m-pytorch-cuda-jetson_orin-20260628T094638Z", + "created_at": "2026-06-28T09:46:38.919768Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0.dev0", + "libreyolo_commit": "5f12ee96d5f3099eaede265187a5df70a5f70c08" + }, + "model": { + "id": "yolonas-m", + "name": "yolo-nas-m", + "family": "yolonas", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLONASm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Orin (nvgpu)", + "gpu_memory_gb": 0.0, + "driver_version": "595.78", + "cuda_version": "13.0", + "cpu": "Unknown", + "cpu_cores": 6, + "ram_gb": 7, + "id": "jetson_orin" + }, + "software": { + "python": "3.12.3", + "torch": "2.12.0+cu130", + "libreyolo": "1.2.0.dev0", + "libreyolo_commit": "5f12ee96d5f3099eaede265187a5df70a5f70c08", + "onnxruntime": "1.24.0" + }, + "accuracy": { + "mAP_50_95": 0.5545068064245974, + "mAP_50": 0.7225798762638183, + "mAP_75": 0.6034660037662518, + "mAP_small": 0.3763163218088265, + "mAP_medium": 0.6291118510748668, + "mAP_large": 0.6893367086338652, + "AR1": 0.42140201995331994, + "AR10": 0.6641307164205555, + "AR100": 0.6932570917806772, + "AR_small": 0.49073760887096063, + "AR_medium": 0.7478990607811923, + "AR_large": 0.8538268909166743 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 152.018, + "std": 14.286, + "p50": 151.357, + "p95": 153.812, + "p99": 154.873, + "preprocess_ms": 12.572, + "inference_ms": 133.209, + "postprocess_ms": 6.237 + } + }, + "throughput": { + "fps_mean": 6.58, + "fps_p50": 6.61 + }, + "model_stats": { + "params_millions": 51.18, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 364.5, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T09:46:38.919768Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0.dev0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cuda", + "device": "gpu" + }, + "source_file": "yolonas-m__pytorch__cuda__jetson_orin__20260628T094638919768Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolonas-l-pytorch-cuda-jetson_orin-20260628T094851Z", + "created_at": "2026-06-28T09:48:51.464380Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0.dev0", + "libreyolo_commit": "5f12ee96d5f3099eaede265187a5df70a5f70c08" + }, + "model": { + "id": "yolonas-l", + "name": "yolo-nas-l", + "family": "yolonas", + "variant": "l", + "source": "libreyolo", + "weights": "LibreYOLONASl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Orin (nvgpu)", + "gpu_memory_gb": 0.0, + "driver_version": "595.78", + "cuda_version": "13.0", + "cpu": "Unknown", + "cpu_cores": 6, + "ram_gb": 7, + "id": "jetson_orin" + }, + "software": { + "python": "3.12.3", + "torch": "2.12.0+cu130", + "libreyolo": "1.2.0.dev0", + "libreyolo_commit": "5f12ee96d5f3099eaede265187a5df70a5f70c08", + "onnxruntime": "1.24.0" + }, + "accuracy": { + "mAP_50_95": 0.5631728310107038, + "mAP_50": 0.7326899962305481, + "mAP_75": 0.6197943523935017, + "mAP_small": 0.36144187071523554, + "mAP_medium": 0.6306335421543419, + "mAP_large": 0.6962833299114103, + "AR1": 0.41836905497431415, + "AR10": 0.6600220531833608, + "AR100": 0.6918215543277225, + "AR_small": 0.47509709353249796, + "AR_medium": 0.7430166700634716, + "AR_large": 0.8318025907979362 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 200.166, + "std": 1.644, + "p50": 199.994, + "p95": 202.637, + "p99": 205.456, + "preprocess_ms": 13.381, + "inference_ms": 180.523, + "postprocess_ms": 6.262 + } + }, + "throughput": { + "fps_mean": 5.0, + "fps_p50": 5.0 + }, + "model_stats": { + "params_millions": 66.98, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 469.9, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T09:48:51.464380Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0.dev0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cuda", + "device": "gpu" + }, + "source_file": "yolonas-l__pytorch__cuda__jetson_orin__20260628T094851464380Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-l-ncnn-cpu-rpi5-20260628T095530Z", + "created_at": "2026-06-28T09:55:30.857573Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-l", + "name": "yolox-l", + "family": "yolox", + "variant": "l", + "source": "libreyolo", + "weights": "LibreYOLOXl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.528503230832125, + "mAP_50": 0.7047107466545172, + "mAP_75": 0.5797387403130669, + "mAP_small": 0.3641973658775863, + "mAP_medium": 0.593339115805217, + "mAP_large": 0.655551455750752, + "AR1": 0.40605083051426444, + "AR10": 0.6293511801499894, + "AR100": 0.6519720370803261, + "AR_small": 0.4738208902298428, + "AR_medium": 0.6996843841570429, + "AR_large": 0.7667428430878959 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 662.417, + "std": 15.182, + "p50": 662.514, + "p95": 677.071, + "p99": 737.746, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.51, + "fps_p50": 1.51 + }, + "model_stats": { + "params_millions": 54.21, + "gflops": 78.01 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 24.2 + }, + "metadata": { + "benchmark_date": "2026-06-28T09:55:30.857573Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-l__ncnn__cpu__rpi5__20260628T095530857573Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-x-ncnn-cpu-rpi5-20260628T101844Z", + "created_at": "2026-06-28T10:18:44.235212Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-x", + "name": "yolox-x", + "family": "yolox", + "variant": "x", + "source": "libreyolo", + "weights": "LibreYOLOXx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5392783045915617, + "mAP_50": 0.713395311782745, + "mAP_75": 0.5857402354568811, + "mAP_small": 0.3639028164271833, + "mAP_medium": 0.6023587897373351, + "mAP_large": 0.6647190792773235, + "AR1": 0.4060455451363752, + "AR10": 0.636941478990372, + "AR100": 0.6597413549532147, + "AR_small": 0.47090377473567496, + "AR_medium": 0.6990418161236918, + "AR_large": 0.7816580652957302 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1122.797, + "std": 21.501, + "p50": 1122.824, + "p95": 1144.462, + "p99": 1209.295, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.89, + "fps_p50": 0.89 + }, + "model_stats": { + "params_millions": 99.07, + "gflops": 141.23 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 6.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T10:18:44.235212Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-x__ncnn__cpu__rpi5__20260628T101844235212Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9t-hailo-int8-hailort-rpi5_hailo8-20260628T134218779826Z", + "created_at": "2026-06-28T13:42:18.779870Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9t", + "name": "yolov9t", + "family": "yolov9", + "variant": "t", + "source": "libreyolo", + "weights": "LibreYOLO9t.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.2574812551731687, + "mAP_50": 0.3724033083680573, + "mAP_75": 0.26901622770320943, + "mAP_small": 0.1810754891905198, + "mAP_medium": 0.2300524545800218, + "mAP_large": 0.38513007996398724, + "AR1": 0.2640856426966872, + "AR10": 0.4639695273232776, + "AR100": 0.500326398525334, + "AR_small": 0.31131925240285746, + "AR_medium": 0.5204978214354378, + "AR_large": 0.6177960497236653 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 13.077, + "std": 0.12, + "p50": 13.058, + "p95": 13.259, + "p99": 13.424, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 76.47, + "fps_p50": 76.58 + }, + "model_stats": { + "params_millions": 2.04, + "gflops": 3.98 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T13:42:18.779870Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + }, + "source_file": "yolov9t_hailo_sub.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9c-hailo-int8-hailort-rpi5_hailo8-20260628T134302307079Z", + "created_at": "2026-06-28T13:43:02.307140Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9c", + "name": "yolov9c", + "family": "yolov9", + "variant": "c", + "source": "libreyolo", + "weights": "LibreYOLO9c.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.5476913920616249, + "mAP_50": 0.7131098144960689, + "mAP_75": 0.5939205638251501, + "mAP_small": 0.35394835635034233, + "mAP_medium": 0.6029809562957014, + "mAP_large": 0.6933464388665393, + "AR1": 0.411439527476965, + "AR10": 0.6524148628406007, + "AR100": 0.6819507435947298, + "AR_small": 0.4954902209154678, + "AR_medium": 0.7211944153664459, + "AR_large": 0.8003639237722002 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 47.079, + "std": 0.16, + "p50": 47.057, + "p95": 47.249, + "p99": 47.375, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 21.24, + "fps_p50": 21.25 + }, + "model_stats": { + "params_millions": 25.5, + "gflops": 51.75 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T13:43:02.307140Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + }, + "source_file": "yolov9c_hailo_sub.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9s-hailo-int8-hailort-rpi5_hailo8-20260628T143918676128Z", + "created_at": "2026-06-28T14:39:18.676175Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9s", + "name": "yolov9s", + "family": "yolov9", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLO9s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.3202846787855532, + "mAP_50": 0.45209050901722025, + "mAP_75": 0.345745204202997, + "mAP_small": 0.24456901999951858, + "mAP_medium": 0.287868241763752, + "mAP_large": 0.5383565118707007, + "AR1": 0.3050424135140193, + "AR10": 0.5258970743061131, + "AR100": 0.5746515426102451, + "AR_small": 0.38826074471458855, + "AR_medium": 0.6140913568484321, + "AR_large": 0.6804843133291116 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 27.517, + "std": 0.167, + "p50": 27.497, + "p95": 27.686, + "p99": 27.759, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 36.34, + "fps_p50": 36.37 + }, + "model_stats": { + "params_millions": 7.23, + "gflops": 13.52 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T14:39:18.676175Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + }, + "source_file": "yolov9s_hailo_sub.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolov9m-hailo-int8-hailort-rpi5_hailo8-20260628T143953705472Z", + "created_at": "2026-06-28T14:39:53.705530Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9m", + "name": "yolov9m", + "family": "yolov9", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLO9m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.5329770770002916, + "mAP_50": 0.6983906983089887, + "mAP_75": 0.5812418637026137, + "mAP_small": 0.35497461017547266, + "mAP_medium": 0.6027547293345632, + "mAP_large": 0.6758022076684784, + "AR1": 0.40010749483340924, + "AR10": 0.6388201868343779, + "AR100": 0.668386838759305, + "AR_small": 0.49351004442454804, + "AR_medium": 0.7171895263421209, + "AR_large": 0.7890398835778198 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 33.232, + "std": 0.083, + "p50": 33.222, + "p95": 33.384, + "p99": 33.489, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 30.09, + "fps_p50": 30.1 + }, + "model_stats": { + "params_millions": 20.12, + "gflops": 38.68 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T14:39:53.705530Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + }, + "source_file": "yolov9m_hailo_sub.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-tiny-hailo-int8-hailort-rpi5_hailo8-20260628T150112939893Z", + "created_at": "2026-06-28T15:01:12.939934Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-tiny", + "name": "yolox-tiny", + "family": "yolox", + "variant": "tiny", + "source": "libreyolo", + "weights": "yolox-tiny", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.65, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.3353106440385515, + "mAP_50": 0.5189426748353215, + "mAP_75": 0.3641099820138371, + "mAP_small": 0.17764741992028685, + "mAP_medium": 0.36505107259605907, + "mAP_large": 0.49343300017658587, + "AR1": 0.2893957905970191, + "AR10": 0.4696391008304537, + "AR100": 0.5116869077190426, + "AR_small": 0.2877205160943997, + "AR_medium": 0.5603867562198416, + "AR_large": 0.6986791375911993 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 5.112, + "std": 0.063, + "p50": 5.109, + "p95": 5.223, + "p99": 5.273, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 195.62, + "fps_p50": 195.73 + }, + "model_stats": { + "params_millions": 5.06, + "gflops": 7.68 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T15:01:12.939934Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + }, + "source_file": "yolox-tiny_hailo_int8.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-s-hailo-int8-hailort-rpi5_hailo8-20260628T153957476167Z", + "created_at": "2026-06-28T15:39:57.476206Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-s", + "name": "yolox-s", + "family": "yolox", + "variant": "s", + "source": "libreyolo", + "weights": "yolox-s", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.65, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.4108814968955979, + "mAP_50": 0.6064624459610067, + "mAP_75": 0.45019380792723657, + "mAP_small": 0.24086973713406004, + "mAP_medium": 0.45198968047342614, + "mAP_large": 0.5516861215849379, + "AR1": 0.33068144790244186, + "AR10": 0.5487670148495044, + "AR100": 0.5951574328491301, + "AR_small": 0.4066083827833327, + "AR_medium": 0.6326720962572658, + "AR_large": 0.7265084581631085 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 10.437, + "std": 0.133, + "p50": 10.426, + "p95": 10.555, + "p99": 10.658, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 95.81, + "fps_p50": 95.91 + }, + "model_stats": { + "params_millions": 8.97, + "gflops": 13.46 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T15:39:57.476206Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + }, + "source_file": "yolox-s_hailo_int8.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolonas-s-hailo-int8-hailort-rpi5_hailo8-20260628T165354261494Z", + "created_at": "2026-06-28T16:53:54.261533Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolonas-s", + "name": "yolonas-s", + "family": "yolonas", + "variant": "s", + "source": "libreyolo", + "weights": "yolonas-s", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.4446089067793858, + "mAP_50": 0.6136841620622177, + "mAP_75": 0.47159018033704103, + "mAP_small": 0.28340429645318765, + "mAP_medium": 0.5084353950730437, + "mAP_large": 0.552387659507169, + "AR1": 0.3510863125136369, + "AR10": 0.5645162152320146, + "AR100": 0.6026627926794943, + "AR_small": 0.4250443242017818, + "AR_medium": 0.6519618056463824, + "AR_large": 0.716713529654017 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 25.2, + "std": 0.134, + "p50": 25.165, + "p95": 25.472, + "p99": 25.584, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 39.68, + "fps_p50": 39.74 + }, + "model_stats": { + "params_millions": 19.02, + "gflops": 32.8 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T16:53:54.261533Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + }, + "source_file": "yolonas-s_hailo_int8.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-m-hailo-int8-hailort-rpi5_hailo8-20260628T165553195225Z", + "created_at": "2026-06-28T16:55:53.195266Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-m", + "name": "yolox-m", + "family": "yolox", + "variant": "m", + "source": "libreyolo", + "weights": "yolox-m", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.65, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.43675730077050046, + "mAP_50": 0.672517622406605, + "mAP_75": 0.487206700674435, + "mAP_small": 0.29748043110546074, + "mAP_medium": 0.5033715842973834, + "mAP_large": 0.5434731037743242, + "AR1": 0.3448179409504896, + "AR10": 0.5760628996119347, + "AR100": 0.617170327621538, + "AR_small": 0.4455076890917315, + "AR_medium": 0.6734527396795311, + "AR_large": 0.7079739394568797 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 28.76, + "std": 0.296, + "p50": 28.737, + "p95": 28.829, + "p99": 28.9, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 34.77, + "fps_p50": 34.8 + }, + "model_stats": { + "params_millions": 25.33, + "gflops": 36.99 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T16:55:53.195266Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + }, + "source_file": "yolox-m_hailo_int8.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolonas-m-hailo-int8-hailort-rpi5_hailo8-20260628T181411310711Z", + "created_at": "2026-06-28T18:14:11.310749Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolonas-m", + "name": "yolonas-m", + "family": "yolonas", + "variant": "m", + "source": "libreyolo", + "weights": "yolonas-m", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.47352629105744415, + "mAP_50": 0.6376564877227787, + "mAP_75": 0.5187175003491332, + "mAP_small": 0.29475744384964614, + "mAP_medium": 0.5482709458514573, + "mAP_large": 0.5764728560539091, + "AR1": 0.3640817386206549, + "AR10": 0.5910485022165374, + "AR100": 0.6300280220448617, + "AR_small": 0.4309091807908955, + "AR_medium": 0.6943580826835967, + "AR_large": 0.7590375638022909 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 46.047, + "std": 0.258, + "p50": 46.026, + "p95": 46.147, + "p99": 46.214, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 21.72, + "fps_p50": 21.73 + }, + "model_stats": { + "params_millions": 51.1, + "gflops": 88.9 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T18:14:11.310749Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + }, + "source_file": "yolonas-m_hailo_int8.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolonas-l-hailo-int8-hailort-rpi5_hailo8-20260628T195258766098Z", + "created_at": "2026-06-28T19:52:58.766135Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolonas-l", + "name": "yolonas-l", + "family": "yolonas", + "variant": "l", + "source": "libreyolo", + "weights": "yolonas-l", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.5242338911343088, + "mAP_50": 0.6935547237267466, + "mAP_75": 0.5649020873573805, + "mAP_small": 0.35182186649309055, + "mAP_medium": 0.5936105569738327, + "mAP_large": 0.6245882897512551, + "AR1": 0.39886015400859043, + "AR10": 0.6312355092169804, + "AR100": 0.6715977250187926, + "AR_small": 0.4977650642226191, + "AR_medium": 0.7201306551961083, + "AR_large": 0.790381063565836 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 64.892, + "std": 0.128, + "p50": 64.88, + "p95": 64.985, + "p99": 65.038, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 15.41, + "fps_p50": 15.41 + }, + "model_stats": { + "params_millions": 66.97, + "gflops": 116.6 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T19:52:58.766135Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + }, + "source_file": "yolonas-l_hailo_int8.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-atto-pytorch-cpu-rpi5-20260628T211422Z", + "created_at": "2026-06-28T21:14:22.281141Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-atto", + "name": "deimv2-atto", + "family": "deimv2", + "variant": "atto", + "source": "libreyolo", + "weights": "LibreDEIMv2atto.pt", + "input_size": 320 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 320, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.27482484560388637, + "mAP_50": 0.4030546265739421, + "mAP_75": 0.2939224609772488, + "mAP_small": 0.08127671069299465, + "mAP_medium": 0.3051240582960075, + "mAP_large": 0.428892694234773, + "AR1": 0.25899522556340965, + "AR10": 0.43518621941320745, + "AR100": 0.46923174643854004, + "AR_small": 0.13428594082868414, + "AR_medium": 0.5427429323077486, + "AR_large": 0.7063253102905971 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 79.064, + "std": 1.254, + "p50": 78.982, + "p95": 80.756, + "p99": 82.098, + "preprocess_ms": 4.933, + "inference_ms": 73.594, + "postprocess_ms": 0.536 + } + }, + "throughput": { + "fps_mean": 12.65, + "fps_p50": 12.66 + }, + "model_stats": { + "params_millions": 0.51, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 80.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:14:22.281141Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-atto__pytorch__cpu__rpi5__20260628T211422281141Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-nano-pytorch-cpu-rpi5-20260628T211630Z", + "created_at": "2026-06-28T21:16:30.082561Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-nano", + "name": "yolox-nano", + "family": "yolox", + "variant": "nano", + "source": "libreyolo", + "weights": "LibreYOLOXn.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.28777402963982734, + "mAP_50": 0.4451673550790783, + "mAP_75": 0.3028944469746292, + "mAP_small": 0.12006471690282278, + "mAP_medium": 0.29773356865571005, + "mAP_large": 0.42695198658971256, + "AR1": 0.2590170696484389, + "AR10": 0.4129019655382761, + "AR100": 0.44096391074417346, + "AR_small": 0.18192961060887416, + "AR_medium": 0.4803201787960003, + "AR_large": 0.6389866972976415 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 117.019, + "std": 2.05, + "p50": 116.703, + "p95": 119.738, + "p99": 123.358, + "preprocess_ms": 6.813, + "inference_ms": 106.927, + "postprocess_ms": 3.279 + } + }, + "throughput": { + "fps_mean": 8.55, + "fps_p50": 8.57 + }, + "model_stats": { + "params_millions": 0.91, + "gflops": 1.32 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 54.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:16:30.082561Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-nano__pytorch__cpu__rpi5__20260628T211630082561Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-femto-pytorch-cpu-rpi5-20260628T211756Z", + "created_at": "2026-06-28T21:17:56.579613Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-femto", + "name": "deimv2-femto", + "family": "deimv2", + "variant": "femto", + "source": "libreyolo", + "weights": "LibreDEIMv2femto.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.34501855839690715, + "mAP_50": 0.48896797791936086, + "mAP_75": 0.3742550702979947, + "mAP_small": 0.16178949601222695, + "mAP_medium": 0.3764979706066898, + "mAP_large": 0.5209814392741269, + "AR1": 0.30087553093763414, + "AR10": 0.5082598470174468, + "AR100": 0.5514464481741097, + "AR_small": 0.285370460098146, + "AR_medium": 0.6053972089908232, + "AR_large": 0.7852369269394482 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 135.282, + "std": 1.525, + "p50": 135.102, + "p95": 136.84, + "p99": 138.77, + "preprocess_ms": 6.497, + "inference_ms": 128.201, + "postprocess_ms": 0.584 + } + }, + "throughput": { + "fps_mean": 7.39, + "fps_p50": 7.4 + }, + "model_stats": { + "params_millions": 0.98, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 82.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:17:56.579613Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-femto__pytorch__cpu__rpi5__20260628T211756579613Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "picodet-s-pytorch-cpu-rpi5-20260628T211903Z", + "created_at": "2026-06-28T21:19:03.354874Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-s", + "name": "picodet-s", + "family": "picodet", + "variant": "s", + "source": "libreyolo", + "weights": "LibrePICODETs.pt", + "input_size": 320 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 320, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.2948574387762822, + "mAP_50": 0.4317496219042625, + "mAP_75": 0.3066188182975883, + "mAP_small": 0.11303066903813051, + "mAP_medium": 0.3009440965032811, + "mAP_large": 0.4801055275972295, + "AR1": 0.27111322725289133, + "AR10": 0.44450670126411373, + "AR100": 0.47922131990213757, + "AR_small": 0.18632252231207877, + "AR_medium": 0.5368430036125633, + "AR_large": 0.7142117190467947 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 95.043, + "std": 9.215, + "p50": 94.542, + "p95": 96.326, + "p99": 101.109, + "preprocess_ms": 6.029, + "inference_ms": 67.874, + "postprocess_ms": 21.14 + } + }, + "throughput": { + "fps_mean": 10.52, + "fps_p50": 10.58 + }, + "model_stats": { + "params_millions": 0.99, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 96.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:19:03.354874Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "picodet-s__pytorch__cpu__rpi5__20260628T211903354874Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-pico-pytorch-cpu-rpi5-20260628T212218Z", + "created_at": "2026-06-28T21:22:18.081947Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-pico", + "name": "deimv2-pico", + "family": "deimv2", + "variant": "pico", + "source": "libreyolo", + "weights": "LibreDEIMv2pico.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.42236369083316583, + "mAP_50": 0.5929095585840876, + "mAP_75": 0.456151853998398, + "mAP_small": 0.26531258037819605, + "mAP_medium": 0.46683143988252807, + "mAP_large": 0.5647437896527114, + "AR1": 0.3473883401228512, + "AR10": 0.5773718981092814, + "AR100": 0.6310409380751396, + "AR_small": 0.39603272087723973, + "AR_medium": 0.6937328805553592, + "AR_large": 0.8282697261345104 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 349.801, + "std": 3.128, + "p50": 349.295, + "p95": 356.684, + "p99": 360.238, + "preprocess_ms": 8.952, + "inference_ms": 340.204, + "postprocess_ms": 0.645 + } + }, + "throughput": { + "fps_mean": 2.86, + "fps_p50": 2.86 + }, + "model_stats": { + "params_millions": 1.54, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 60.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:22:18.081947Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-pico__pytorch__cpu__rpi5__20260628T212218081947Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "picodet-m-pytorch-cpu-rpi5-20260628T212418Z", + "created_at": "2026-06-28T21:24:18.426331Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-m", + "name": "picodet-m", + "family": "picodet", + "variant": "m", + "source": "libreyolo", + "weights": "LibrePICODETm.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.37622258563330857, + "mAP_50": 0.5302670327093372, + "mAP_75": 0.4031927733445861, + "mAP_small": 0.1774283925337442, + "mAP_medium": 0.40932920767836517, + "mAP_large": 0.5730505210815525, + "AR1": 0.31237085609548815, + "AR10": 0.5225526014836394, + "AR100": 0.5596398389459811, + "AR_small": 0.2782176748170048, + "AR_medium": 0.6178179023642811, + "AR_large": 0.783357700560243 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 201.122, + "std": 9.322, + "p50": 200.585, + "p95": 204.204, + "p99": 208.875, + "preprocess_ms": 8.212, + "inference_ms": 169.492, + "postprocess_ms": 23.418 + } + }, + "throughput": { + "fps_mean": 4.97, + "fps_p50": 4.99 + }, + "model_stats": { + "params_millions": 2.15, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 82.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:24:18.426331Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "picodet-m__pytorch__cpu__rpi5__20260628T212418426331Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "picodet-l-pytorch-cpu-rpi5-20260628T212909Z", + "created_at": "2026-06-28T21:29:09.759037Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-l", + "name": "picodet-l", + "family": "picodet", + "variant": "l", + "source": "libreyolo", + "weights": "LibrePICODETl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.44180297724122514, + "mAP_50": 0.6123659448116612, + "mAP_75": 0.47989821583822584, + "mAP_small": 0.25367398272057823, + "mAP_medium": 0.48866727706769647, + "mAP_large": 0.6026292859455765, + "AR1": 0.35750711092113147, + "AR10": 0.585816152664366, + "AR100": 0.6265703629517813, + "AR_small": 0.4101739411217021, + "AR_medium": 0.686802522647409, + "AR_large": 0.7950929986667825 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 540.311, + "std": 12.087, + "p50": 539.77, + "p95": 551.2, + "p99": 555.588, + "preprocess_ms": 14.362, + "inference_ms": 494.842, + "postprocess_ms": 31.106 + } + }, + "throughput": { + "fps_mean": 1.85, + "fps_p50": 1.85 + }, + "model_stats": { + "params_millions": 3.31, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 87.4 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:29:09.759037Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "picodet-l__pytorch__cpu__rpi5__20260628T212909759037Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-n-pytorch-cpu-rpi5-20260628T213301Z", + "created_at": "2026-06-28T21:33:01.315421Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-n", + "name": "deimv2-n", + "family": "deimv2", + "variant": "n", + "source": "libreyolo", + "weights": "LibreDEIMv2n.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.4670213335038969, + "mAP_50": 0.639905576947522, + "mAP_75": 0.5054596477905674, + "mAP_small": 0.28810252375241885, + "mAP_medium": 0.5139622089791221, + "mAP_large": 0.6324757403885736, + "AR1": 0.38157578980653545, + "AR10": 0.6069542738223006, + "AR100": 0.6652687039674579, + "AR_small": 0.4343253981458581, + "AR_medium": 0.7275327401496207, + "AR_large": 0.8386081389606242 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 423.001, + "std": 11.372, + "p50": 424.559, + "p95": 430.496, + "p99": 433.575, + "preprocess_ms": 9.066, + "inference_ms": 413.181, + "postprocess_ms": 0.754 + } + }, + "throughput": { + "fps_mean": 2.36, + "fps_p50": 2.36 + }, + "model_stats": { + "params_millions": 3.6, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 73.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:33:01.315421Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-n__pytorch__cpu__rpi5__20260628T213301315421Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deim-n-pytorch-cpu-rpi5-20260628T213704Z", + "created_at": "2026-06-28T21:37:04.426434Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-n", + "name": "deim-n", + "family": "deim", + "variant": "n", + "source": "libreyolo", + "weights": "LibreDEIMn.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.46790377719107634, + "mAP_50": 0.6384362340573561, + "mAP_75": 0.5065071383556861, + "mAP_small": 0.28272666254163575, + "mAP_medium": 0.5239486087186582, + "mAP_large": 0.6492162454283862, + "AR1": 0.37465778703821373, + "AR10": 0.6088876938998233, + "AR100": 0.6621982004214751, + "AR_small": 0.45967862604366466, + "AR_medium": 0.7285399501675481, + "AR_large": 0.8544990171118824 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 430.954, + "std": 5.886, + "p50": 433.292, + "p95": 436.616, + "p99": 438.543, + "preprocess_ms": 7.724, + "inference_ms": 422.475, + "postprocess_ms": 0.755 + } + }, + "throughput": { + "fps_mean": 2.32, + "fps_p50": 2.31 + }, + "model_stats": { + "params_millions": 3.78, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 110.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:37:04.426434Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deim-n__pytorch__cpu__rpi5__20260628T213704426434Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "dfine-n-pytorch-cpu-rpi5-20260628T214059Z", + "created_at": "2026-06-28T21:40:59.576864Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-n", + "name": "d-fine-n", + "family": "dfine", + "variant": "n", + "source": "libreyolo", + "weights": "LibreDFINEn.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.4576828753960137, + "mAP_50": 0.627611672354216, + "mAP_75": 0.4936120506981263, + "mAP_small": 0.25813013104993077, + "mAP_medium": 0.5148552892369967, + "mAP_large": 0.6164531686764337, + "AR1": 0.3661276595022023, + "AR10": 0.597139132066487, + "AR100": 0.6492993825514405, + "AR_small": 0.4246210950171405, + "AR_medium": 0.7131204074253821, + "AR_large": 0.846219767993009 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 429.079, + "std": 5.636, + "p50": 431.126, + "p95": 435.518, + "p99": 436.986, + "preprocess_ms": 7.795, + "inference_ms": 420.516, + "postprocess_ms": 0.769 + } + }, + "throughput": { + "fps_mean": 2.33, + "fps_p50": 2.32 + }, + "model_stats": { + "params_millions": 3.78, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 91.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:40:59.576864Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "dfine-n__pytorch__cpu__rpi5__20260628T214059576864Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-tiny-pytorch-cpu-rpi5-20260628T214304Z", + "created_at": "2026-06-28T21:43:04.549414Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-tiny", + "name": "yolox-tiny", + "family": "yolox", + "variant": "tiny", + "source": "libreyolo", + "weights": "LibreYOLOXt.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.3507077718360378, + "mAP_50": 0.5299376329995034, + "mAP_75": 0.3699625988162548, + "mAP_small": 0.17195484437178557, + "mAP_medium": 0.37105117781877367, + "mAP_large": 0.5024188305234134, + "AR1": 0.29771079852091176, + "AR10": 0.48312923523876466, + "AR100": 0.5087055145491408, + "AR_small": 0.26969197340962586, + "AR_medium": 0.5545428732852953, + "AR_large": 0.6856424677582275 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 183.538, + "std": 1.79, + "p50": 183.316, + "p95": 186.528, + "p99": 188.541, + "preprocess_ms": 6.828, + "inference_ms": 173.282, + "postprocess_ms": 3.428 + } + }, + "throughput": { + "fps_mean": 5.45, + "fps_p50": 5.46 + }, + "model_stats": { + "params_millions": 5.06, + "gflops": 7.68 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 45.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:43:04.549414Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-tiny__pytorch__cpu__rpi5__20260628T214304549414Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-s-pytorch-cpu-rpi5-20260628T214837Z", + "created_at": "2026-06-28T21:48:37.471331Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-s", + "name": "yolox-s", + "family": "yolox", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLOXs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.43010515097930546, + "mAP_50": 0.6220141448765621, + "mAP_75": 0.46897993598051824, + "mAP_small": 0.2599367080177599, + "mAP_medium": 0.4656542485238322, + "mAP_large": 0.5685465617255948, + "AR1": 0.35492883202242, + "AR10": 0.5655097103344773, + "AR100": 0.5969756922378401, + "AR_small": 0.40518655835245926, + "AR_medium": 0.6354007584389023, + "AR_large": 0.7251227937433352 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 626.603, + "std": 6.253, + "p50": 627.342, + "p95": 635.033, + "p99": 639.983, + "preprocess_ms": 8.29, + "inference_ms": 612.634, + "postprocess_ms": 5.679 + } + }, + "throughput": { + "fps_mean": 1.6, + "fps_p50": 1.59 + }, + "model_stats": { + "params_millions": 8.97, + "gflops": 13.46 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 97.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:48:37.471331Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-s__pytorch__cpu__rpi5__20260628T214837471331Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-s-pytorch-cpu-rpi5-20260628T220533Z", + "created_at": "2026-06-28T22:05:33.872596Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-s", + "name": "deimv2-s", + "family": "deimv2", + "variant": "s", + "source": "libreyolo", + "weights": "LibreDEIMv2s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5300705352326833, + "mAP_50": 0.6976651497162647, + "mAP_75": 0.5751333008654972, + "mAP_small": 0.3500893991731694, + "mAP_medium": 0.5804291582052804, + "mAP_large": 0.7299874007669309, + "AR1": 0.4065742563193488, + "AR10": 0.6752756526406756, + "AR100": 0.7241649181407003, + "AR_small": 0.5444134233119735, + "AR_medium": 0.7724843253587157, + "AR_large": 0.8780156970954397 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1190.937, + "std": 12.062, + "p50": 1194.707, + "p95": 1206.359, + "p99": 1218.328, + "preprocess_ms": 17.284, + "inference_ms": 1172.895, + "postprocess_ms": 0.758 + } + }, + "throughput": { + "fps_mean": 0.84, + "fps_p50": 0.84 + }, + "model_stats": { + "params_millions": 9.78, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 50.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T22:05:33.872596Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-s__pytorch__cpu__rpi5__20260628T220533872596Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "ec-s-pytorch-cpu-rpi5-20260628T221558Z", + "created_at": "2026-06-28T22:15:58.217609Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-s", + "name": "ec-s", + "family": "ec", + "variant": "s", + "source": "libreyolo", + "weights": "LibreECs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5432219445119648, + "mAP_50": 0.714017633533976, + "mAP_75": 0.5923221234790975, + "mAP_small": 0.3666503591643378, + "mAP_medium": 0.6000808289526866, + "mAP_large": 0.7143113755076108, + "AR1": 0.42118346438115173, + "AR10": 0.6849257047218618, + "AR100": 0.7312556280979691, + "AR_small": 0.5461653398174465, + "AR_medium": 0.7835733460879325, + "AR_large": 0.8876717561873844 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1200.969, + "std": 8.783, + "p50": 1200.297, + "p95": 1217.586, + "p99": 1224.614, + "preprocess_ms": 15.796, + "inference_ms": 1184.428, + "postprocess_ms": 0.745 + } + }, + "throughput": { + "fps_mean": 0.83, + "fps_p50": 0.83 + }, + "model_stats": { + "params_millions": 9.88, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 94.3 + }, + "metadata": { + "benchmark_date": "2026-06-28T22:15:58.217609Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "ec-s__pytorch__cpu__rpi5__20260628T221558217609Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deim-s-pytorch-cpu-rpi5-20260628T222358Z", + "created_at": "2026-06-28T22:23:58.258295Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-s", + "name": "deim-s", + "family": "deim", + "variant": "s", + "source": "libreyolo", + "weights": "LibreDEIMs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5209501200526508, + "mAP_50": 0.6833215973288361, + "mAP_75": 0.5700182129622119, + "mAP_small": 0.3595557837023218, + "mAP_medium": 0.5639603145573738, + "mAP_large": 0.6702396634323782, + "AR1": 0.3980327093777471, + "AR10": 0.6837733109758194, + "AR100": 0.7359188945277568, + "AR_small": 0.5490511307593388, + "AR_medium": 0.7814739019262885, + "AR_large": 0.879701448382054 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 878.248, + "std": 7.305, + "p50": 878.229, + "p95": 888.075, + "p99": 898.123, + "preprocess_ms": 7.794, + "inference_ms": 869.689, + "postprocess_ms": 0.765 + } + }, + "throughput": { + "fps_mean": 1.14, + "fps_p50": 1.14 + }, + "model_stats": { + "params_millions": 10.32, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 109.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T22:23:58.258295Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deim-s__pytorch__cpu__rpi5__20260628T222358258295Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "dfine-s-pytorch-cpu-rpi5-20260628T223148Z", + "created_at": "2026-06-28T22:31:48.995294Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-s", + "name": "d-fine-s", + "family": "dfine", + "variant": "s", + "source": "libreyolo", + "weights": "LibreDFINEs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.533816550656966, + "mAP_50": 0.6978934949475171, + "mAP_75": 0.5815146677466713, + "mAP_small": 0.3794386468726935, + "mAP_medium": 0.5786169334696805, + "mAP_large": 0.6712528261067662, + "AR1": 0.40807927039626324, + "AR10": 0.6848022854803739, + "AR100": 0.7343664904079216, + "AR_small": 0.5815105988950591, + "AR_medium": 0.7768773182389905, + "AR_large": 0.8765596543653289 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 874.322, + "std": 7.672, + "p50": 874.788, + "p95": 886.507, + "p99": 894.322, + "preprocess_ms": 7.773, + "inference_ms": 865.785, + "postprocess_ms": 0.765 + } + }, + "throughput": { + "fps_mean": 1.14, + "fps_p50": 1.14 + }, + "model_stats": { + "params_millions": 10.32, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 109.7 + }, + "metadata": { + "benchmark_date": "2026-06-28T22:31:48.995294Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "dfine-s__pytorch__cpu__rpi5__20260628T223148995294Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-s-pytorch-cpu-rpi5-20260628T223931Z", + "created_at": "2026-06-28T22:39:31.540013Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-s", + "name": "rt-detrv4-s", + "family": "rtdetrv4", + "variant": "s", + "source": "libreyolo", + "weights": "LibreRTDETRv4s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5282737967407499, + "mAP_50": 0.6994596509087028, + "mAP_75": 0.572601548852765, + "mAP_small": 0.34987337823602815, + "mAP_medium": 0.5703579990666772, + "mAP_large": 0.6837293185691692, + "AR1": 0.40892277620866907, + "AR10": 0.6752355179616181, + "AR100": 0.729750172023218, + "AR_small": 0.5313593023593197, + "AR_medium": 0.7731283222221518, + "AR_large": 0.880861480555594 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 879.882, + "std": 6.882, + "p50": 879.81, + "p95": 890.186, + "p99": 902.855, + "preprocess_ms": 7.831, + "inference_ms": 871.284, + "postprocess_ms": 0.767 + } + }, + "throughput": { + "fps_mean": 1.14, + "fps_p50": 1.14 + }, + "model_stats": { + "params_millions": 10.32, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 112.2 + }, + "metadata": { + "benchmark_date": "2026-06-28T22:39:31.540013Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv4-s__pytorch__cpu__rpi5__20260628T223931540013Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "ec-m-pytorch-cpu-rpi5-20260628T230648Z", + "created_at": "2026-06-28T23:06:48.254971Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-m", + "name": "ec-m", + "family": "ec", + "variant": "m", + "source": "libreyolo", + "weights": "LibreECm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5835333586126803, + "mAP_50": 0.7577586900096812, + "mAP_75": 0.6350329290731235, + "mAP_small": 0.40672990096458406, + "mAP_medium": 0.6388487836333429, + "mAP_large": 0.7501653894101447, + "AR1": 0.43343530313979, + "AR10": 0.7050750217800258, + "AR100": 0.7502984843206456, + "AR_small": 0.5728496539199226, + "AR_medium": 0.7893795712758537, + "AR_large": 0.8827568118289735 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1868.727, + "std": 16.432, + "p50": 1874.334, + "p95": 1883.764, + "p99": 1886.904, + "preprocess_ms": 14.959, + "inference_ms": 1853.027, + "postprocess_ms": 0.742 + } + }, + "throughput": { + "fps_mean": 0.54, + "fps_p50": 0.53 + }, + "model_stats": { + "params_millions": 19.43, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 97.3 + }, + "metadata": { + "benchmark_date": "2026-06-28T23:06:48.254971Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "ec-m__pytorch__cpu__rpi5__20260628T230648254971Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-m-pytorch-cpu-rpi5-20260628T232355Z", + "created_at": "2026-06-28T23:23:55.345875Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-m", + "name": "deimv2-m", + "family": "deimv2", + "variant": "m", + "source": "libreyolo", + "weights": "LibreDEIMv2m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5601229905454717, + "mAP_50": 0.7292297655671566, + "mAP_75": 0.6153556732298998, + "mAP_small": 0.38810271496061227, + "mAP_medium": 0.6022996527298319, + "mAP_large": 0.7215611210879361, + "AR1": 0.42507423593101595, + "AR10": 0.6942931883432835, + "AR100": 0.7368414193602092, + "AR_small": 0.5641964407869676, + "AR_medium": 0.7740628208553209, + "AR_large": 0.8771622951355976 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1913.191, + "std": 9.537, + "p50": 1914.334, + "p95": 1923.924, + "p99": 1929.852, + "preprocess_ms": 16.634, + "inference_ms": 1895.804, + "postprocess_ms": 0.754 + } + }, + "throughput": { + "fps_mean": 0.52, + "fps_p50": 0.52 + }, + "model_stats": { + "params_millions": 18.36, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 106.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T23:23:55.345875Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-m__pytorch__cpu__rpi5__20260628T232355345875Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deim-m-pytorch-cpu-rpi5-20260628T233813Z", + "created_at": "2026-06-28T23:38:13.060016Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-m", + "name": "deim-m", + "family": "deim", + "variant": "m", + "source": "libreyolo", + "weights": "LibreDEIMm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.554380329873063, + "mAP_50": 0.7201200159471879, + "mAP_75": 0.6059586355391078, + "mAP_small": 0.38938229790680734, + "mAP_medium": 0.6047535480560435, + "mAP_large": 0.715414025259707, + "AR1": 0.4270136586370724, + "AR10": 0.6977532541784438, + "AR100": 0.7412542504872445, + "AR_small": 0.5772622767167025, + "AR_medium": 0.7771076238238673, + "AR_large": 0.8916083181230134 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1582.835, + "std": 13.037, + "p50": 1582.748, + "p95": 1595.245, + "p99": 1602.267, + "preprocess_ms": 7.68, + "inference_ms": 1574.397, + "postprocess_ms": 0.758 + } + }, + "throughput": { + "fps_mean": 0.63, + "fps_p50": 0.63 + }, + "model_stats": { + "params_millions": 19.59, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 97.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T23:38:13.060016Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deim-m__pytorch__cpu__rpi5__20260628T233813060016Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "dfine-m-pytorch-cpu-rpi5-20260628T235147Z", + "created_at": "2026-06-28T23:51:47.949418Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-m", + "name": "d-fine-m", + "family": "dfine", + "variant": "m", + "source": "libreyolo", + "weights": "LibreDFINEm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5782994902642316, + "mAP_50": 0.7466821563973365, + "mAP_75": 0.6299117815035142, + "mAP_small": 0.4301932955022992, + "mAP_medium": 0.614041717660949, + "mAP_large": 0.729298114060577, + "AR1": 0.4258019573210324, + "AR10": 0.7202899110812604, + "AR100": 0.7715135321744913, + "AR_small": 0.6401413616788801, + "AR_medium": 0.7966559297893929, + "AR_large": 0.89146658543394 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1577.225, + "std": 13.329, + "p50": 1576.609, + "p95": 1590.509, + "p99": 1597.548, + "preprocess_ms": 7.752, + "inference_ms": 1568.725, + "postprocess_ms": 0.748 + } + }, + "throughput": { + "fps_mean": 0.63, + "fps_p50": 0.63 + }, + "model_stats": { + "params_millions": 19.59, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 79.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T23:51:47.949418Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "dfine-m__pytorch__cpu__rpi5__20260628T235147949418Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-m-pytorch-cpu-rpi5-20260629T000528Z", + "created_at": "2026-06-29T00:05:28.347300Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-m", + "name": "rt-detrv4-m", + "family": "rtdetrv4", + "variant": "m", + "source": "libreyolo", + "weights": "LibreRTDETRv4m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5650964727884, + "mAP_50": 0.729508246830964, + "mAP_75": 0.6213825976722152, + "mAP_small": 0.3946348695064786, + "mAP_medium": 0.6083527271023137, + "mAP_large": 0.7333839664388817, + "AR1": 0.4209735862228707, + "AR10": 0.7026438232679751, + "AR100": 0.748571757104558, + "AR_small": 0.5869023683588981, + "AR_medium": 0.7879191568864369, + "AR_large": 0.8845628241437012 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1588.449, + "std": 8.848, + "p50": 1588.688, + "p95": 1601.154, + "p99": 1606.572, + "preprocess_ms": 7.72, + "inference_ms": 1579.98, + "postprocess_ms": 0.749 + } + }, + "throughput": { + "fps_mean": 0.63, + "fps_p50": 0.63 + }, + "model_stats": { + "params_millions": 19.59, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 85.1 + }, + "metadata": { + "benchmark_date": "2026-06-29T00:05:28.347300Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv4-m__pytorch__cpu__rpi5__20260629T000528347300Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolonas-s-pytorch-cpu-rpi5-20260629T001434Z", + "created_at": "2026-06-29T00:14:34.373358Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolonas-s", + "name": "yolo-nas-s", + "family": "yolonas", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLONASs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5175187804596387, + "mAP_50": 0.68985349636812, + "mAP_75": 0.5658402328317349, + "mAP_small": 0.31195812978907683, + "mAP_medium": 0.5807006918363328, + "mAP_large": 0.6776626884024056, + "AR1": 0.3996041197390947, + "AR10": 0.6268720617180102, + "AR100": 0.6582188104717092, + "AR_small": 0.4251034101792207, + "AR_medium": 0.7157421987255158, + "AR_large": 0.8077122859175214 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1049.54, + "std": 13.242, + "p50": 1055.903, + "p95": 1066.118, + "p99": 1069.991, + "preprocess_ms": 9.364, + "inference_ms": 1037.35, + "postprocess_ms": 2.825 + } + }, + "throughput": { + "fps_mean": 0.95, + "fps_p50": 0.95 + }, + "model_stats": { + "params_millions": 19.05, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 81.2 + }, + "metadata": { + "benchmark_date": "2026-06-29T00:14:34.373358Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolonas-s__pytorch__cpu__rpi5__20260629T001434373358Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r18-pytorch-cpu-rpi5-20260629T002539Z", + "created_at": "2026-06-29T00:25:39.420656Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r18", + "name": "rt-detr-r18", + "family": "rtdetr", + "variant": "r18", + "source": "libreyolo", + "weights": "LibreRTDETRr18.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.4974385575772144, + "mAP_50": 0.6634990211631825, + "mAP_75": 0.5400385660072728, + "mAP_small": 0.3259147207054876, + "mAP_medium": 0.5483387545699251, + "mAP_large": 0.6406918404250833, + "AR1": 0.3957277587834025, + "AR10": 0.6636363747426361, + "AR100": 0.7226404916875274, + "AR_small": 0.5376757775735183, + "AR_medium": 0.7695527656896355, + "AR_large": 0.8587953609720331 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1245.405, + "std": 9.825, + "p50": 1247.454, + "p95": 1253.143, + "p99": 1260.762, + "preprocess_ms": 3.601, + "inference_ms": 1241.054, + "postprocess_ms": 0.749 + } + }, + "throughput": { + "fps_mean": 0.8, + "fps_p50": 0.8 + }, + "model_stats": { + "params_millions": 20.18, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 92.0 + }, + "metadata": { + "benchmark_date": "2026-06-29T00:25:39.420656Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetr-r18__pytorch__cpu__rpi5__20260629T002539420656Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r18-pytorch-cpu-rpi5-20260629T003710Z", + "created_at": "2026-06-29T00:37:10.952196Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r18", + "name": "rt-detrv2-r18", + "family": "rtdetrv2", + "variant": "r18", + "source": "libreyolo", + "weights": "LibreRTDETRv2r18.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5075295513304796, + "mAP_50": 0.6738984732532675, + "mAP_75": 0.5495400880228422, + "mAP_small": 0.33493026397740544, + "mAP_medium": 0.5390480085426296, + "mAP_large": 0.6640351147172935, + "AR1": 0.39492685884175177, + "AR10": 0.6748887578268217, + "AR100": 0.7277599874225195, + "AR_small": 0.5575095523214917, + "AR_medium": 0.7554613421447167, + "AR_large": 0.8595988150631054 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1244.505, + "std": 9.581, + "p50": 1246.635, + "p95": 1254.867, + "p99": 1257.566, + "preprocess_ms": 3.581, + "inference_ms": 1240.157, + "postprocess_ms": 0.766 + } + }, + "throughput": { + "fps_mean": 0.8, + "fps_p50": 0.8 + }, + "model_stats": { + "params_millions": 20.18, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 102.0 + }, + "metadata": { + "benchmark_date": "2026-06-29T00:37:10.952196Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv2-r18__pytorch__cpu__rpi5__20260629T003710952196Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-m-pytorch-cpu-rpi5-20260629T005027Z", + "created_at": "2026-06-29T00:50:27.897974Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-m", + "name": "yolox-m", + "family": "yolox", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLOXm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5091614002893265, + "mAP_50": 0.6937390846005986, + "mAP_75": 0.5567399898210998, + "mAP_small": 0.3183261816380624, + "mAP_medium": 0.5753011971934548, + "mAP_large": 0.6645057221435278, + "AR1": 0.3939314117959562, + "AR10": 0.638739175548791, + "AR100": 0.6643246891329869, + "AR_small": 0.4485149876142434, + "AR_medium": 0.7048254369364738, + "AR_large": 0.8146288687115271 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1284.111, + "std": 11.267, + "p50": 1286.967, + "p95": 1296.347, + "p99": 1302.237, + "preprocess_ms": 8.274, + "inference_ms": 1270.559, + "postprocess_ms": 5.278 + } + }, + "throughput": { + "fps_mean": 0.78, + "fps_p50": 0.78 + }, + "model_stats": { + "params_millions": 25.33, + "gflops": 36.99 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 40.5 + }, + "metadata": { + "benchmark_date": "2026-06-29T00:50:27.897974Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-m__pytorch__cpu__rpi5__20260629T005027897974Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deim-l-pytorch-cpu-rpi5-20260629T011106Z", + "created_at": "2026-06-29T01:11:06.433509Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-l", + "name": "deim-l", + "family": "deim", + "variant": "l", + "source": "libreyolo", + "weights": "LibreDEIMl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5784020658941241, + "mAP_50": 0.7554947331574471, + "mAP_75": 0.6393013469861818, + "mAP_small": 0.44458639801354877, + "mAP_medium": 0.6354404061583653, + "mAP_large": 0.7458486122504594, + "AR1": 0.43152856098801823, + "AR10": 0.7128187787446858, + "AR100": 0.759162698030682, + "AR_small": 0.617189586484144, + "AR_medium": 0.7947756577275892, + "AR_large": 0.8895979531006462 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2295.545, + "std": 15.395, + "p50": 2298.32, + "p95": 2311.261, + "p99": 2315.066, + "preprocess_ms": 7.412, + "inference_ms": 2287.367, + "postprocess_ms": 0.766 + } + }, + "throughput": { + "fps_mean": 0.44, + "fps_p50": 0.44 + }, + "model_stats": { + "params_millions": 31.24, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 72.2 + }, + "metadata": { + "benchmark_date": "2026-06-29T01:11:06.433509Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deim-l__pytorch__cpu__rpi5__20260629T011106433509Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "dfine-l-pytorch-cpu-rpi5-20260629T013105Z", + "created_at": "2026-06-29T01:31:05.259823Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-l", + "name": "d-fine-l", + "family": "dfine", + "variant": "l", + "source": "libreyolo", + "weights": "LibreDFINEl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5998637327996268, + "mAP_50": 0.771413290072788, + "mAP_75": 0.6545274000895153, + "mAP_small": 0.4600603643885379, + "mAP_medium": 0.651843569917136, + "mAP_large": 0.7519470808192028, + "AR1": 0.4438129932529035, + "AR10": 0.7301763319531722, + "AR100": 0.7823205971671722, + "AR_small": 0.6336871651097996, + "AR_medium": 0.8251280046501868, + "AR_large": 0.901957546814411 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2305.176, + "std": 13.155, + "p50": 2308.045, + "p95": 2318.262, + "p99": 2325.132, + "preprocess_ms": 7.361, + "inference_ms": 2297.051, + "postprocess_ms": 0.764 + } + }, + "throughput": { + "fps_mean": 0.43, + "fps_p50": 0.43 + }, + "model_stats": { + "params_millions": 31.24, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 89.3 + }, + "metadata": { + "benchmark_date": "2026-06-29T01:31:05.259823Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "dfine-l__pytorch__cpu__rpi5__20260629T013105259823Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "ec-l-pytorch-cpu-rpi5-20260629T015522Z", + "created_at": "2026-06-29T01:55:22.978697Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-l", + "name": "ec-l", + "family": "ec", + "variant": "l", + "source": "libreyolo", + "weights": "LibreECl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.6007011060957481, + "mAP_50": 0.7749474593822242, + "mAP_75": 0.6523360761497325, + "mAP_small": 0.4339405977702752, + "mAP_medium": 0.6628074293970688, + "mAP_large": 0.7572514538766986, + "AR1": 0.43754390635267343, + "AR10": 0.7201062953774926, + "AR100": 0.7589963535223007, + "AR_small": 0.5846511426384247, + "AR_medium": 0.8139374671717773, + "AR_large": 0.8965390869585811 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2850.252, + "std": 11.02, + "p50": 2852.271, + "p95": 2864.132, + "p99": 2870.324, + "preprocess_ms": 15.0, + "inference_ms": 2834.501, + "postprocess_ms": 0.751 + } + }, + "throughput": { + "fps_mean": 0.35, + "fps_p50": 0.35 + }, + "model_stats": { + "params_millions": 32.97, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 73.0 + }, + "metadata": { + "benchmark_date": "2026-06-29T01:55:22.978697Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "ec-l__pytorch__cpu__rpi5__20260629T015522978697Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r34-pytorch-cpu-rpi5-20260629T021016Z", + "created_at": "2026-06-29T02:10:16.897798Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r34", + "name": "rt-detr-r34", + "family": "rtdetr", + "variant": "r34", + "source": "libreyolo", + "weights": "LibreRTDETRr34.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5220705477893457, + "mAP_50": 0.6956972783119411, + "mAP_75": 0.5672764157874404, + "mAP_small": 0.3496314468621894, + "mAP_medium": 0.558792600550835, + "mAP_large": 0.6673601126544302, + "AR1": 0.4096667574894731, + "AR10": 0.6683935712842712, + "AR100": 0.7219515967458123, + "AR_small": 0.5406543657059172, + "AR_medium": 0.7541078184762466, + "AR_large": 0.8733822205698133 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1669.041, + "std": 8.411, + "p50": 1668.949, + "p95": 1680.314, + "p99": 1686.699, + "preprocess_ms": 3.605, + "inference_ms": 1664.643, + "postprocess_ms": 0.793 + } + }, + "throughput": { + "fps_mean": 0.6, + "fps_p50": 0.6 + }, + "model_stats": { + "params_millions": 31.44, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 107.3 + }, + "metadata": { + "benchmark_date": "2026-06-29T02:10:16.897798Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetr-r34__pytorch__cpu__rpi5__20260629T021016897798Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r34-pytorch-cpu-rpi5-20260629T022501Z", + "created_at": "2026-06-29T02:25:01.471763Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r34", + "name": "rt-detrv2-r34", + "family": "rtdetrv2", + "variant": "r34", + "source": "libreyolo", + "weights": "LibreRTDETRv2r34.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5317866682163993, + "mAP_50": 0.7018630480731898, + "mAP_75": 0.5781560934160433, + "mAP_small": 0.3554951561526654, + "mAP_medium": 0.5762138931529338, + "mAP_large": 0.673940756314738, + "AR1": 0.40553267109069463, + "AR10": 0.6645274749188759, + "AR100": 0.7202744486197602, + "AR_small": 0.5656720704641991, + "AR_medium": 0.7545120243239197, + "AR_large": 0.8549994238172808 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1664.933, + "std": 13.327, + "p50": 1665.81, + "p95": 1674.354, + "p99": 1678.704, + "preprocess_ms": 3.561, + "inference_ms": 1660.622, + "postprocess_ms": 0.75 + } + }, + "throughput": { + "fps_mean": 0.6, + "fps_p50": 0.6 + }, + "model_stats": { + "params_millions": 31.44, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 96.7 + }, + "metadata": { + "benchmark_date": "2026-06-29T02:25:01.471763Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv2-r34__pytorch__cpu__rpi5__20260629T022501471763Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-l-pytorch-cpu-rpi5-20260629T024444Z", + "created_at": "2026-06-29T02:44:44.005586Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-l", + "name": "rt-detrv4-l", + "family": "rtdetrv4", + "variant": "l", + "source": "libreyolo", + "weights": "LibreRTDETRv4l.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5778213004784958, + "mAP_50": 0.7442362572121634, + "mAP_75": 0.6334282910545552, + "mAP_small": 0.42123051715546067, + "mAP_medium": 0.6364248441671155, + "mAP_large": 0.7449359377371503, + "AR1": 0.4357236915112565, + "AR10": 0.7176614125783658, + "AR100": 0.765070530225193, + "AR_small": 0.6090186415643862, + "AR_medium": 0.8011127339685521, + "AR_large": 0.9102406922726382 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2303.768, + "std": 9.83, + "p50": 2303.617, + "p95": 2315.565, + "p99": 2330.775, + "preprocess_ms": 7.396, + "inference_ms": 2295.617, + "postprocess_ms": 0.755 + } + }, + "throughput": { + "fps_mean": 0.43, + "fps_p50": 0.43 + }, + "model_stats": { + "params_millions": 31.24, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 132.5 + }, + "metadata": { + "benchmark_date": "2026-06-29T02:44:44.005586Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv4-l__pytorch__cpu__rpi5__20260629T024444005586Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-l-pytorch-cpu-rpi5-20260629T030515Z", + "created_at": "2026-06-29T03:05:15.993467Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-l", + "name": "rt-detr-l", + "family": "rtdetr", + "variant": "l", + "source": "libreyolo", + "weights": "LibreRTDETRl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5575479992934428, + "mAP_50": 0.7370539207052157, + "mAP_75": 0.6119027301701989, + "mAP_small": 0.3915772453802141, + "mAP_medium": 0.5924982987913687, + "mAP_large": 0.729726871134273, + "AR1": 0.4178675326119419, + "AR10": 0.6965772852777472, + "AR100": 0.7418839830844038, + "AR_small": 0.5928839911054432, + "AR_medium": 0.7705208239024869, + "AR_large": 0.8880044454648446 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2345.193, + "std": 16.061, + "p50": 2347.699, + "p95": 2356.526, + "p99": 2362.263, + "preprocess_ms": 3.279, + "inference_ms": 2341.163, + "postprocess_ms": 0.75 + } + }, + "throughput": { + "fps_mean": 0.43, + "fps_p50": 0.43 + }, + "model_stats": { + "params_millions": 32.93, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 78.5 + }, + "metadata": { + "benchmark_date": "2026-06-29T03:05:15.993467Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetr-l__pytorch__cpu__rpi5__20260629T030515993467Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-l-pytorch-cpu-rpi5-20260629T033028Z", + "created_at": "2026-06-29T03:30:28.978384Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-l", + "name": "deimv2-l", + "family": "deimv2", + "variant": "l", + "source": "libreyolo", + "weights": "LibreDEIMv2l.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5857229926291182, + "mAP_50": 0.7561055145330008, + "mAP_75": 0.6423712369191432, + "mAP_small": 0.4237277438008416, + "mAP_medium": 0.6446915283665434, + "mAP_large": 0.7558959123772737, + "AR1": 0.43021983010021314, + "AR10": 0.7119366639781503, + "AR100": 0.7559768066356651, + "AR_small": 0.6040310189175387, + "AR_medium": 0.7913091585542085, + "AR_large": 0.8838087931121458 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2802.229, + "std": 13.313, + "p50": 2803.933, + "p95": 2815.509, + "p99": 2820.016, + "preprocess_ms": 17.375, + "inference_ms": 2784.119, + "postprocess_ms": 0.736 + } + }, + "throughput": { + "fps_mean": 0.36, + "fps_p50": 0.36 + }, + "model_stats": { + "params_millions": 32.55, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 118.0 + }, + "metadata": { + "benchmark_date": "2026-06-29T03:30:28.978384Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-l__pytorch__cpu__rpi5__20260629T033028978384Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r50m-pytorch-cpu-rpi5-20260629T035550Z", + "created_at": "2026-06-29T03:55:50.107846Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r50m", + "name": "rt-detrv2-r50m", + "family": "rtdetrv2", + "variant": "r50m", + "source": "libreyolo", + "weights": "LibreRTDETRv2r50m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5475701026232949, + "mAP_50": 0.7176596185245316, + "mAP_75": 0.5991013990587835, + "mAP_small": 0.3789979568982936, + "mAP_medium": 0.6013682226961227, + "mAP_large": 0.69338885373, + "AR1": 0.4115040311379684, + "AR10": 0.6839278724348254, + "AR100": 0.7381671290997697, + "AR_small": 0.586667067152034, + "AR_medium": 0.7797841209975578, + "AR_large": 0.8765759260334942 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2810.255, + "std": 16.144, + "p50": 2811.264, + "p95": 2827.445, + "p99": 2831.547, + "preprocess_ms": 3.289, + "inference_ms": 2806.208, + "postprocess_ms": 0.758 + } + }, + "throughput": { + "fps_mean": 0.36, + "fps_p50": 0.36 + }, + "model_stats": { + "params_millions": 36.59, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 91.5 + }, + "metadata": { + "benchmark_date": "2026-06-29T03:55:50.107846Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv2-r50m__pytorch__cpu__rpi5__20260629T035550107846Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r50m-pytorch-cpu-rpi5-20260629T042008Z", + "created_at": "2026-06-29T04:20:08.262043Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r50m", + "name": "rt-detr-r50m", + "family": "rtdetr", + "variant": "r50m", + "source": "libreyolo", + "weights": "LibreRTDETRr50m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5380213245334988, + "mAP_50": 0.7122448267576749, + "mAP_75": 0.5848047008866352, + "mAP_small": 0.37633171209032285, + "mAP_medium": 0.5939333438455466, + "mAP_large": 0.6899231369387736, + "AR1": 0.41720418627313166, + "AR10": 0.677884852236364, + "AR100": 0.7298682016629048, + "AR_small": 0.5776550405454185, + "AR_medium": 0.766607070399435, + "AR_large": 0.8797720658454787 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2801.975, + "std": 20.116, + "p50": 2803.997, + "p95": 2816.569, + "p99": 2829.904, + "preprocess_ms": 3.287, + "inference_ms": 2797.93, + "postprocess_ms": 0.758 + } + }, + "throughput": { + "fps_mean": 0.36, + "fps_p50": 0.36 + }, + "model_stats": { + "params_millions": 36.59, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 68.6 + }, + "metadata": { + "benchmark_date": "2026-06-29T04:20:08.262043Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetr-r50m__pytorch__cpu__rpi5__20260629T042008262043Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r50-pytorch-cpu-rpi5-20260629T044842Z", + "created_at": "2026-06-29T04:48:42.968753Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r50", + "name": "rt-detr-r50", + "family": "rtdetr", + "variant": "r50", + "source": "libreyolo", + "weights": "LibreRTDETRr50.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5586219562510433, + "mAP_50": 0.7304228810571854, + "mAP_75": 0.6166276372667306, + "mAP_small": 0.3825271464442928, + "mAP_medium": 0.6108065605086318, + "mAP_large": 0.7033225957178774, + "AR1": 0.42085895754874764, + "AR10": 0.6985677228069086, + "AR100": 0.7467315408224947, + "AR_small": 0.5821257451028578, + "AR_medium": 0.7878582128853165, + "AR_large": 0.8820373961923116 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3317.573, + "std": 18.91, + "p50": 3319.276, + "p95": 3335.005, + "p99": 3337.755, + "preprocess_ms": 3.303, + "inference_ms": 3313.502, + "postprocess_ms": 0.768 + } + }, + "throughput": { + "fps_mean": 0.3, + "fps_p50": 0.3 + }, + "model_stats": { + "params_millions": 42.89, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 70.3 + }, + "metadata": { + "benchmark_date": "2026-06-29T04:48:42.968753Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetr-r50__pytorch__cpu__rpi5__20260629T044842968753Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r50-pytorch-cpu-rpi5-20260629T051812Z", + "created_at": "2026-06-29T05:18:12.080119Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r50", + "name": "rt-detrv2-r50", + "family": "rtdetrv2", + "variant": "r50", + "source": "libreyolo", + "weights": "LibreRTDETRv2r50.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5569306661618519, + "mAP_50": 0.7283953697118257, + "mAP_75": 0.6056222550072237, + "mAP_small": 0.38962732253938737, + "mAP_medium": 0.6038589595446148, + "mAP_large": 0.7187812766701257, + "AR1": 0.41980268254801273, + "AR10": 0.6879503522170225, + "AR100": 0.7401610542838911, + "AR_small": 0.5961381618600232, + "AR_medium": 0.7784381644748659, + "AR_large": 0.8723433753727128 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3317.275, + "std": 19.91, + "p50": 3321.673, + "p95": 3333.853, + "p99": 3337.86, + "preprocess_ms": 3.294, + "inference_ms": 3313.204, + "postprocess_ms": 0.777 + } + }, + "throughput": { + "fps_mean": 0.3, + "fps_p50": 0.3 + }, + "model_stats": { + "params_millions": 42.89, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 94.3 + }, + "metadata": { + "benchmark_date": "2026-06-29T05:18:12.080119Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "rtdetrv2-r50__pytorch__cpu__rpi5__20260629T051812080119Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "ec-x-pytorch-cpu-rpi5-20260629T055009Z", + "created_at": "2026-06-29T05:50:09.308891Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-x", + "name": "ec-x", + "family": "ec", + "variant": "x", + "source": "libreyolo", + "weights": "LibreECx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.6113461416284465, + "mAP_50": 0.7856329163312351, + "mAP_75": 0.665522214313548, + "mAP_small": 0.43594677616719263, + "mAP_medium": 0.6761155368028872, + "mAP_large": 0.7790291400209074, + "AR1": 0.4380420119244032, + "AR10": 0.7264733654815458, + "AR100": 0.7625240302226778, + "AR_small": 0.5840608876384857, + "AR_medium": 0.8115087135596564, + "AR_large": 0.9073395719728262 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3635.078, + "std": 17.441, + "p50": 3637.342, + "p95": 3653.653, + "p99": 3660.413, + "preprocess_ms": 14.978, + "inference_ms": 3619.34, + "postprocess_ms": 0.76 + } + }, + "throughput": { + "fps_mean": 0.28, + "fps_p50": 0.27 + }, + "model_stats": { + "params_millions": 49.94, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 91.3 + }, + "metadata": { + "benchmark_date": "2026-06-29T05:50:09.308891Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "ec-x__pytorch__cpu__rpi5__20260629T055009308891Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "deimv2-x-pytorch-cpu-rpi5-20260629T062402Z", + "created_at": "2026-06-29T06:24:02.415113Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-x", + "name": "deimv2-x", + "family": "deimv2", + "variant": "x", + "source": "libreyolo", + "weights": "LibreDEIMv2x.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.6133221273087972, + "mAP_50": 0.7894407109124595, + "mAP_75": 0.6727361018617944, + "mAP_small": 0.4339747185867168, + "mAP_medium": 0.6737668436781252, + "mAP_large": 0.7656487457031492, + "AR1": 0.4453632538830613, + "AR10": 0.7287634403924469, + "AR100": 0.7705342572890554, + "AR_small": 0.6192456578511248, + "AR_medium": 0.8184399305818361, + "AR_large": 0.8827316250588547 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3711.783, + "std": 16.813, + "p50": 3712.827, + "p95": 3729.248, + "p99": 3737.808, + "preprocess_ms": 17.408, + "inference_ms": 3693.633, + "postprocess_ms": 0.742 + } + }, + "throughput": { + "fps_mean": 0.27, + "fps_p50": 0.27 + }, + "model_stats": { + "params_millions": 51.21, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 45.7 + }, + "metadata": { + "benchmark_date": "2026-06-29T06:24:02.415113Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "deimv2-x__pytorch__cpu__rpi5__20260629T062402415113Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolonas-m-pytorch-cpu-rpi5-20260629T064125Z", + "created_at": "2026-06-29T06:41:25.448789Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolonas-m", + "name": "yolo-nas-m", + "family": "yolonas", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLONASm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5544547683777413, + "mAP_50": 0.722620824576428, + "mAP_75": 0.6035577827392103, + "mAP_small": 0.3762689309932977, + "mAP_medium": 0.6289804777038956, + "mAP_large": 0.6892941564923878, + "AR1": 0.42135623240753245, + "AR10": 0.6639889766461968, + "AR100": 0.6931054991397029, + "AR_small": 0.49060743009913144, + "AR_medium": 0.7477457628729601, + "AR_large": 0.8538316974535644 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2028.618, + "std": 7.754, + "p50": 2027.951, + "p95": 2041.683, + "p99": 2050.422, + "preprocess_ms": 9.397, + "inference_ms": 2016.355, + "postprocess_ms": 2.867 + } + }, + "throughput": { + "fps_mean": 0.49, + "fps_p50": 0.49 + }, + "model_stats": { + "params_millions": 51.18, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 82.1 + }, + "metadata": { + "benchmark_date": "2026-06-29T06:41:25.448789Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolonas-m__pytorch__cpu__rpi5__20260629T064125448789Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, + { + "schema_version": "va.submission.v1", + "submission_id": "yolox-l-pytorch-cpu-rpi5-20260629T071142Z", + "created_at": "2026-06-29T07:11:42.195596Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-l", + "name": "yolox-l", + "family": "yolox", + "variant": "l", + "source": "libreyolo", + "weights": "LibreYOLOXl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5391250645636839, + "mAP_50": 0.7166949761716915, + "mAP_75": 0.5910098562529215, + "mAP_small": 0.38166727366157893, + "mAP_medium": 0.6005190613243097, + "mAP_large": 0.6736901380015644, + "AR1": 0.41972797189119304, + "AR10": 0.6602132206987458, + "AR100": 0.684190638681071, + "AR_small": 0.5029510237520646, + "AR_medium": 0.7297849558476158, + "AR_large": 0.826473447231166 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2627.003, + "std": 17.989, + "p50": 2632.621, + "p95": 2644.28, + "p99": 2650.036, + "preprocess_ms": 8.016, + "inference_ms": 2613.758, + "postprocess_ms": 5.229 + } + }, + "throughput": { + "fps_mean": 0.38, + "fps_p50": 0.38 + }, + "model_stats": { + "params_millions": 54.21, + "gflops": 78.01 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 55.1 + }, + "metadata": { + "benchmark_date": "2026-06-29T07:11:42.195596Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + }, + "source_file": "yolox-l__pytorch__cpu__rpi5__20260629T071142195596Z.json", + "verified_at": "2026-07-06T00:14:49.916092Z" + }, { "schema_version": "va.submission.v1", "submission_id": "yolo11n-pytorch-cpu-rpi5-20260705T111735Z", diff --git a/scripts/test_verified_results_compat.py b/scripts/test_verified_results_compat.py index 290f9a7..ec25f4b 100644 --- a/scripts/test_verified_results_compat.py +++ b/scripts/test_verified_results_compat.py @@ -12,8 +12,6 @@ GENERATED_RESULTS_PATH = ROOT / "generated" / "verified-results.v1.json" MODELS_METADATA_PATH = ROOT / "website" / "src" / "data" / "metadata" / "models.json" -EXPECTED_RESULT_COUNT = 74 - OLD_HARDWARE_MAP = [ ("a100", "a100"), ("raspberry pi 5", "rpi5"), @@ -213,9 +211,7 @@ def main() -> int: model["id"] for model in load_json(MODELS_METADATA_PATH)["models"] } - assert len(submissions) == EXPECTED_RESULT_COUNT, ( - f"expected {EXPECTED_RESULT_COUNT} verified results, got {len(submissions)}" - ) + assert submissions, "expected at least one verified result" missing_metadata = sorted( {new_model_id(submission) for submission in submissions} - metadata_model_ids @@ -223,6 +219,28 @@ def main() -> int: assert not missing_metadata, f"models missing from models.json: {missing_metadata}" for submission in submissions: + source = submission["source_file"] + assert new_model_id(submission), f"{source}: expected a model id" + assert new_hardware_id(submission), f"{source}: expected a hardware id" + assert num_images(submission) > 0, f"{source}: expected a non-empty evaluation set" + assert batch_size(submission) == 1, f"{source}: expected batch size 1" + + # Coordinate-aware loading must retain every canonical row, including + # full-val and subset runs that share a model/hardware/runtime tuple. + new_rows_by_coordinate = latest_by_key(submissions, new_coordinate) + assert len(new_rows_by_coordinate) == len(submissions), ( + "canonical results contain duplicate benchmark coordinates" + ) + + # The original loader only represented one full-val row per + # model/hardware/runtime. Keep its parity check scoped to that legacy cohort; + # subset runs are intentionally distinct in the coordinate-aware loader. + legacy_submissions = [ + submission for submission in submissions if num_images(submission) == 5000 + ] + assert legacy_submissions, "expected at least one full-val legacy result" + + for submission in legacy_submissions: source = submission["source_file"] assert old_model_id(submission) == new_model_id(submission), ( f"{source}: model.id no longer matches legacy normalization" @@ -230,18 +248,16 @@ def main() -> int: assert old_hardware_id(submission) == new_hardware_id(submission), ( f"{source}: hardware id changed from legacy detection" ) - assert num_images(submission) == 5000, f"{source}: expected 5000 images" - assert batch_size(submission) == 1, f"{source}: expected batch size 1" old_rows_by_model = latest_by_key( - submissions, + legacy_submissions, lambda submission: ( old_hardware_id(submission), runtime_id(submission), old_model_id(submission), ), ) - new_rows_by_coordinate = latest_by_key(submissions, new_coordinate) + legacy_rows_by_coordinate = latest_by_key(legacy_submissions, new_coordinate) old_rows = sorted( row_identity(submission, old_hardware_id(submission)) @@ -249,20 +265,14 @@ def main() -> int: ) new_rows = sorted( row_identity(submission, new_hardware_id(submission)) - for submission in new_rows_by_coordinate.values() + for submission in legacy_rows_by_coordinate.values() ) - assert len(old_rows) == EXPECTED_RESULT_COUNT, ( - f"legacy model-keyed loader would keep {len(old_rows)} rows" - ) - assert len(new_rows) == EXPECTED_RESULT_COUNT, ( - f"coordinate-keyed loader keeps {len(new_rows)} rows" - ) assert old_rows == new_rows, "coordinate-keyed loader changed the current row set" print( - f"Verified Step 1 compatibility: {len(new_rows)} rows, " - "unique per model/hardware/runtime, all full-val batch=1." + f"Verified coordinate compatibility: {len(submissions)} canonical rows, " + f"{len(new_rows)} legacy full-val rows, no duplicate coordinates." ) return 0 diff --git a/submissions/deim-l__onnx__cpu__rpi5__20260628T025613259095Z.json b/submissions/deim-l__onnx__cpu__rpi5__20260628T025613259095Z.json new file mode 100644 index 0000000..9737671 --- /dev/null +++ b/submissions/deim-l__onnx__cpu__rpi5__20260628T025613259095Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deim-l-onnx-cpu-rpi5-20260628T025613Z", + "created_at": "2026-06-28T02:56:13.259095Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-l", + "name": "deim-l", + "family": "deim", + "variant": "l", + "source": "libreyolo", + "weights": "LibreDEIMl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5784101734004303, + "mAP_50": 0.7554938545399417, + "mAP_75": 0.6393003920178135, + "mAP_small": 0.44463414701490817, + "mAP_medium": 0.6354539408240266, + "mAP_large": 0.7458486451096473, + "AR1": 0.43152856098801823, + "AR10": 0.7128187787446858, + "AR100": 0.7592639126055808, + "AR_small": 0.6187280480226055, + "AR_medium": 0.794876872302488, + "AR_large": 0.8895979531006462 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1400.237, + "std": 7.405, + "p50": 1400.382, + "p95": 1406.819, + "p99": 1422.474, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.71, + "fps_p50": 0.71 + }, + "model_stats": { + "params_millions": 30.79, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T02:56:13.259095Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deim-l__pytorch__cpu__rpi5__20260629T011106433509Z.json b/submissions/deim-l__pytorch__cpu__rpi5__20260629T011106433509Z.json new file mode 100644 index 0000000..38cddd6 --- /dev/null +++ b/submissions/deim-l__pytorch__cpu__rpi5__20260629T011106433509Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deim-l-pytorch-cpu-rpi5-20260629T011106Z", + "created_at": "2026-06-29T01:11:06.433509Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-l", + "name": "deim-l", + "family": "deim", + "variant": "l", + "source": "libreyolo", + "weights": "LibreDEIMl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5784020658941241, + "mAP_50": 0.7554947331574471, + "mAP_75": 0.6393013469861818, + "mAP_small": 0.44458639801354877, + "mAP_medium": 0.6354404061583653, + "mAP_large": 0.7458486122504594, + "AR1": 0.43152856098801823, + "AR10": 0.7128187787446858, + "AR100": 0.759162698030682, + "AR_small": 0.617189586484144, + "AR_medium": 0.7947756577275892, + "AR_large": 0.8895979531006462 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2295.545, + "std": 15.395, + "p50": 2298.32, + "p95": 2311.261, + "p99": 2315.066, + "preprocess_ms": 7.412, + "inference_ms": 2287.367, + "postprocess_ms": 0.766 + } + }, + "throughput": { + "fps_mean": 0.44, + "fps_p50": 0.44 + }, + "model_stats": { + "params_millions": 31.24, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 72.2 + }, + "metadata": { + "benchmark_date": "2026-06-29T01:11:06.433509Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deim-m__onnx__cpu__rpi5__20260628T024405862998Z.json b/submissions/deim-m__onnx__cpu__rpi5__20260628T024405862998Z.json new file mode 100644 index 0000000..08dbf6c --- /dev/null +++ b/submissions/deim-m__onnx__cpu__rpi5__20260628T024405862998Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deim-m-onnx-cpu-rpi5-20260628T024405Z", + "created_at": "2026-06-28T02:44:05.862998Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-m", + "name": "deim-m", + "family": "deim", + "variant": "m", + "source": "libreyolo", + "weights": "LibreDEIMm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.554380350639803, + "mAP_50": 0.7201200948623354, + "mAP_75": 0.6059586355391078, + "mAP_small": 0.38938230326988665, + "mAP_medium": 0.6047536503404021, + "mAP_large": 0.7154140931221996, + "AR1": 0.4270136586370724, + "AR10": 0.6977532541784438, + "AR100": 0.7412542504872445, + "AR_small": 0.5772622767167025, + "AR_medium": 0.7771076238238673, + "AR_large": 0.8916083181230134 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 990.654, + "std": 6.591, + "p50": 990.591, + "p95": 994.601, + "p99": 1020.687, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.01, + "fps_p50": 1.01 + }, + "model_stats": { + "params_millions": 19.26, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T02:44:05.862998Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deim-m__pytorch__cpu__rpi5__20260628T233813060016Z.json b/submissions/deim-m__pytorch__cpu__rpi5__20260628T233813060016Z.json new file mode 100644 index 0000000..150a6b0 --- /dev/null +++ b/submissions/deim-m__pytorch__cpu__rpi5__20260628T233813060016Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deim-m-pytorch-cpu-rpi5-20260628T233813Z", + "created_at": "2026-06-28T23:38:13.060016Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-m", + "name": "deim-m", + "family": "deim", + "variant": "m", + "source": "libreyolo", + "weights": "LibreDEIMm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.554380329873063, + "mAP_50": 0.7201200159471879, + "mAP_75": 0.6059586355391078, + "mAP_small": 0.38938229790680734, + "mAP_medium": 0.6047535480560435, + "mAP_large": 0.715414025259707, + "AR1": 0.4270136586370724, + "AR10": 0.6977532541784438, + "AR100": 0.7412542504872445, + "AR_small": 0.5772622767167025, + "AR_medium": 0.7771076238238673, + "AR_large": 0.8916083181230134 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1582.835, + "std": 13.037, + "p50": 1582.748, + "p95": 1595.245, + "p99": 1602.267, + "preprocess_ms": 7.68, + "inference_ms": 1574.397, + "postprocess_ms": 0.758 + } + }, + "throughput": { + "fps_mean": 0.63, + "fps_p50": 0.63 + }, + "model_stats": { + "params_millions": 19.59, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 97.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T23:38:13.060016Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deim-n__onnx__cpu__rpi5__20260628T023026752367Z.json b/submissions/deim-n__onnx__cpu__rpi5__20260628T023026752367Z.json new file mode 100644 index 0000000..869069e --- /dev/null +++ b/submissions/deim-n__onnx__cpu__rpi5__20260628T023026752367Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deim-n-onnx-cpu-rpi5-20260628T023026Z", + "created_at": "2026-06-28T02:30:26.752367Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-n", + "name": "deim-n", + "family": "deim", + "variant": "n", + "source": "libreyolo", + "weights": "LibreDEIMn.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4679439393783107, + "mAP_50": 0.6384369065119964, + "mAP_75": 0.5065966451768882, + "mAP_small": 0.2827452405332132, + "mAP_medium": 0.5239486171961336, + "mAP_large": 0.649292759072888, + "AR1": 0.37475703908357355, + "AR10": 0.6089594734177107, + "AR100": 0.6622745544514113, + "AR_small": 0.4596949926557759, + "AR_medium": 0.7285437095660443, + "AR_large": 0.8547273276141655 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 229.711, + "std": 2.113, + "p50": 229.43, + "p95": 231.366, + "p99": 233.022, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 4.35, + "fps_p50": 4.36 + }, + "model_stats": { + "params_millions": 3.75, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 71.8 + }, + "metadata": { + "benchmark_date": "2026-06-28T02:30:26.752367Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deim-n__pytorch__cpu__rpi5__20260628T213704426434Z.json b/submissions/deim-n__pytorch__cpu__rpi5__20260628T213704426434Z.json new file mode 100644 index 0000000..292555a --- /dev/null +++ b/submissions/deim-n__pytorch__cpu__rpi5__20260628T213704426434Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deim-n-pytorch-cpu-rpi5-20260628T213704Z", + "created_at": "2026-06-28T21:37:04.426434Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-n", + "name": "deim-n", + "family": "deim", + "variant": "n", + "source": "libreyolo", + "weights": "LibreDEIMn.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.46790377719107634, + "mAP_50": 0.6384362340573561, + "mAP_75": 0.5065071383556861, + "mAP_small": 0.28272666254163575, + "mAP_medium": 0.5239486087186582, + "mAP_large": 0.6492162454283862, + "AR1": 0.37465778703821373, + "AR10": 0.6088876938998233, + "AR100": 0.6621982004214751, + "AR_small": 0.45967862604366466, + "AR_medium": 0.7285399501675481, + "AR_large": 0.8544990171118824 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 430.954, + "std": 5.886, + "p50": 433.292, + "p95": 436.616, + "p99": 438.543, + "preprocess_ms": 7.724, + "inference_ms": 422.475, + "postprocess_ms": 0.755 + } + }, + "throughput": { + "fps_mean": 2.32, + "fps_p50": 2.31 + }, + "model_stats": { + "params_millions": 3.78, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 110.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:37:04.426434Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deim-s__onnx__cpu__rpi5__20260628T023527204132Z.json b/submissions/deim-s__onnx__cpu__rpi5__20260628T023527204132Z.json new file mode 100644 index 0000000..a25c554 --- /dev/null +++ b/submissions/deim-s__onnx__cpu__rpi5__20260628T023527204132Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deim-s-onnx-cpu-rpi5-20260628T023527Z", + "created_at": "2026-06-28T02:35:27.204132Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-s", + "name": "deim-s", + "family": "deim", + "variant": "s", + "source": "libreyolo", + "weights": "LibreDEIMs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5209717892441037, + "mAP_50": 0.6833211071300828, + "mAP_75": 0.5700182819277311, + "mAP_small": 0.3596241748448038, + "mAP_medium": 0.5639596802427718, + "mAP_large": 0.6702399089962287, + "AR1": 0.3980327093777471, + "AR10": 0.6838315860340944, + "AR100": 0.735977169586032, + "AR_small": 0.5491536948619029, + "AR_medium": 0.7814739019262885, + "AR_large": 0.879701448382054 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 559.177, + "std": 4.83, + "p50": 559.209, + "p95": 562.544, + "p99": 568.306, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.79, + "fps_p50": 1.79 + }, + "model_stats": { + "params_millions": 10.25, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T02:35:27.204132Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deim-s__pytorch__cpu__rpi5__20260628T222358258295Z.json b/submissions/deim-s__pytorch__cpu__rpi5__20260628T222358258295Z.json new file mode 100644 index 0000000..1b922b1 --- /dev/null +++ b/submissions/deim-s__pytorch__cpu__rpi5__20260628T222358258295Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deim-s-pytorch-cpu-rpi5-20260628T222358Z", + "created_at": "2026-06-28T22:23:58.258295Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-s", + "name": "deim-s", + "family": "deim", + "variant": "s", + "source": "libreyolo", + "weights": "LibreDEIMs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5209501200526508, + "mAP_50": 0.6833215973288361, + "mAP_75": 0.5700182129622119, + "mAP_small": 0.3595557837023218, + "mAP_medium": 0.5639603145573738, + "mAP_large": 0.6702396634323782, + "AR1": 0.3980327093777471, + "AR10": 0.6837733109758194, + "AR100": 0.7359188945277568, + "AR_small": 0.5490511307593388, + "AR_medium": 0.7814739019262885, + "AR_large": 0.879701448382054 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 878.248, + "std": 7.305, + "p50": 878.229, + "p95": 888.075, + "p99": 898.123, + "preprocess_ms": 7.794, + "inference_ms": 869.689, + "postprocess_ms": 0.765 + } + }, + "throughput": { + "fps_mean": 1.14, + "fps_p50": 1.14 + }, + "model_stats": { + "params_millions": 10.32, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 109.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T22:23:58.258295Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deim-x__onnx__cpu__rpi5__20260628T031842468270Z.json b/submissions/deim-x__onnx__cpu__rpi5__20260628T031842468270Z.json new file mode 100644 index 0000000..24acae6 --- /dev/null +++ b/submissions/deim-x__onnx__cpu__rpi5__20260628T031842468270Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deim-x-onnx-cpu-rpi5-20260628T031842Z", + "created_at": "2026-06-28T03:18:42.468270Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deim-x", + "name": "deim-x", + "family": "deim", + "variant": "x", + "source": "libreyolo", + "weights": "LibreDEIMx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5961960308925123, + "mAP_50": 0.7700141120836566, + "mAP_75": 0.6562959986729319, + "mAP_small": 0.4559131099923876, + "mAP_medium": 0.6569733791673753, + "mAP_large": 0.7668436019384222, + "AR1": 0.4445621917232395, + "AR10": 0.7250154079634621, + "AR100": 0.7705422586062296, + "AR_small": 0.6212383765698645, + "AR_medium": 0.81244331327091, + "AR_large": 0.9076892664639177 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2635.563, + "std": 10.373, + "p50": 2634.795, + "p95": 2643.715, + "p99": 2697.274, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.38, + "fps_p50": 0.38 + }, + "model_stats": { + "params_millions": 61.71, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:18:42.468270Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-atto__onnx__cpu__rpi5__20260628T031917141261Z.json b/submissions/deimv2-atto__onnx__cpu__rpi5__20260628T031917141261Z.json new file mode 100644 index 0000000..f1c1748 --- /dev/null +++ b/submissions/deimv2-atto__onnx__cpu__rpi5__20260628T031917141261Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-atto-onnx-cpu-rpi5-20260628T031917Z", + "created_at": "2026-06-28T03:19:17.141261Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-atto", + "name": "deimv2-atto", + "family": "deimv2", + "variant": "atto", + "source": "libreyolo", + "weights": "LibreDEIMv2atto.pt", + "input_size": 320 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 320, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.2748653297512432, + "mAP_50": 0.4030736022619007, + "mAP_75": 0.29403623096869114, + "mAP_small": 0.0812842855548296, + "mAP_medium": 0.3051796806318778, + "mAP_large": 0.4290115385111966, + "AR1": 0.2591026420470867, + "AR10": 0.4353053743391541, + "AR100": 0.469344423710311, + "AR_small": 0.134311368254856, + "AR_medium": 0.5430335808039892, + "AR_large": 0.7063349233643775 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 33.024, + "std": 0.75, + "p50": 33.067, + "p95": 34.206, + "p99": 34.749, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 30.28, + "fps_p50": 30.24 + }, + "model_stats": { + "params_millions": 0.49, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 75.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:19:17.141261Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-atto__pytorch__cpu__rpi5__20260628T211422281141Z.json b/submissions/deimv2-atto__pytorch__cpu__rpi5__20260628T211422281141Z.json new file mode 100644 index 0000000..228b34c --- /dev/null +++ b/submissions/deimv2-atto__pytorch__cpu__rpi5__20260628T211422281141Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-atto-pytorch-cpu-rpi5-20260628T211422Z", + "created_at": "2026-06-28T21:14:22.281141Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-atto", + "name": "deimv2-atto", + "family": "deimv2", + "variant": "atto", + "source": "libreyolo", + "weights": "LibreDEIMv2atto.pt", + "input_size": 320 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 320, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.27482484560388637, + "mAP_50": 0.4030546265739421, + "mAP_75": 0.2939224609772488, + "mAP_small": 0.08127671069299465, + "mAP_medium": 0.3051240582960075, + "mAP_large": 0.428892694234773, + "AR1": 0.25899522556340965, + "AR10": 0.43518621941320745, + "AR100": 0.46923174643854004, + "AR_small": 0.13428594082868414, + "AR_medium": 0.5427429323077486, + "AR_large": 0.7063253102905971 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 79.064, + "std": 1.254, + "p50": 78.982, + "p95": 80.756, + "p99": 82.098, + "preprocess_ms": 4.933, + "inference_ms": 73.594, + "postprocess_ms": 0.536 + } + }, + "throughput": { + "fps_mean": 12.65, + "fps_p50": 12.66 + }, + "model_stats": { + "params_millions": 0.51, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 80.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:14:22.281141Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-femto__onnx__cpu__rpi5__20260628T032005516190Z.json b/submissions/deimv2-femto__onnx__cpu__rpi5__20260628T032005516190Z.json new file mode 100644 index 0000000..cff195c --- /dev/null +++ b/submissions/deimv2-femto__onnx__cpu__rpi5__20260628T032005516190Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-femto-onnx-cpu-rpi5-20260628T032005Z", + "created_at": "2026-06-28T03:20:05.516190Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-femto", + "name": "deimv2-femto", + "family": "deimv2", + "variant": "femto", + "source": "libreyolo", + "weights": "LibreDEIMv2femto.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.3450873229627451, + "mAP_50": 0.4889703158548694, + "mAP_75": 0.37426314484979606, + "mAP_small": 0.16186424592475104, + "mAP_medium": 0.3767184765437616, + "mAP_large": 0.5210417081877327, + "AR1": 0.30103618458050724, + "AR10": 0.5085654916447838, + "AR100": 0.5517491879578669, + "AR_small": 0.2854844141323229, + "AR_medium": 0.6055178230259108, + "AR_large": 0.7861551867618176 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 59.787, + "std": 1.787, + "p50": 59.634, + "p95": 61.294, + "p99": 62.42, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 16.73, + "fps_p50": 16.77 + }, + "model_stats": { + "params_millions": 0.96, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 73.4 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:20:05.516190Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-femto__pytorch__cpu__rpi5__20260628T211756579613Z.json b/submissions/deimv2-femto__pytorch__cpu__rpi5__20260628T211756579613Z.json new file mode 100644 index 0000000..f1e09a4 --- /dev/null +++ b/submissions/deimv2-femto__pytorch__cpu__rpi5__20260628T211756579613Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-femto-pytorch-cpu-rpi5-20260628T211756Z", + "created_at": "2026-06-28T21:17:56.579613Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-femto", + "name": "deimv2-femto", + "family": "deimv2", + "variant": "femto", + "source": "libreyolo", + "weights": "LibreDEIMv2femto.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.34501855839690715, + "mAP_50": 0.48896797791936086, + "mAP_75": 0.3742550702979947, + "mAP_small": 0.16178949601222695, + "mAP_medium": 0.3764979706066898, + "mAP_large": 0.5209814392741269, + "AR1": 0.30087553093763414, + "AR10": 0.5082598470174468, + "AR100": 0.5514464481741097, + "AR_small": 0.285370460098146, + "AR_medium": 0.6053972089908232, + "AR_large": 0.7852369269394482 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 135.282, + "std": 1.525, + "p50": 135.102, + "p95": 136.84, + "p99": 138.77, + "preprocess_ms": 6.497, + "inference_ms": 128.201, + "postprocess_ms": 0.584 + } + }, + "throughput": { + "fps_mean": 7.39, + "fps_p50": 7.4 + }, + "model_stats": { + "params_millions": 0.98, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 82.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:17:56.579613Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-l__onnx__cpu__rpi5__20260628T041134309534Z.json b/submissions/deimv2-l__onnx__cpu__rpi5__20260628T041134309534Z.json new file mode 100644 index 0000000..48b021c --- /dev/null +++ b/submissions/deimv2-l__onnx__cpu__rpi5__20260628T041134309534Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-l-onnx-cpu-rpi5-20260628T041134Z", + "created_at": "2026-06-28T04:11:34.309534Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-l", + "name": "deimv2-l", + "family": "deimv2", + "variant": "l", + "source": "libreyolo", + "weights": "LibreDEIMv2l.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5857056480114364, + "mAP_50": 0.756122518317119, + "mAP_75": 0.6423894397306, + "mAP_small": 0.4237311254486432, + "mAP_medium": 0.6448895711462656, + "mAP_large": 0.755484145872126, + "AR1": 0.43021983010021314, + "AR10": 0.7119366639781503, + "AR100": 0.7559290439408436, + "AR_small": 0.6041215166550952, + "AR_medium": 0.7913091585542085, + "AR_large": 0.883534820509406 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2678.98, + "std": 27.251, + "p50": 2676.912, + "p95": 2683.508, + "p99": 2732.584, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.37, + "fps_p50": 0.37 + }, + "model_stats": { + "params_millions": 32.19, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T04:11:34.309534Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-l__pytorch__cpu__rpi5__20260629T033028978384Z.json b/submissions/deimv2-l__pytorch__cpu__rpi5__20260629T033028978384Z.json new file mode 100644 index 0000000..96a599d --- /dev/null +++ b/submissions/deimv2-l__pytorch__cpu__rpi5__20260629T033028978384Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-l-pytorch-cpu-rpi5-20260629T033028Z", + "created_at": "2026-06-29T03:30:28.978384Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-l", + "name": "deimv2-l", + "family": "deimv2", + "variant": "l", + "source": "libreyolo", + "weights": "LibreDEIMv2l.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5857229926291182, + "mAP_50": 0.7561055145330008, + "mAP_75": 0.6423712369191432, + "mAP_small": 0.4237277438008416, + "mAP_medium": 0.6446915283665434, + "mAP_large": 0.7558959123772737, + "AR1": 0.43021983010021314, + "AR10": 0.7119366639781503, + "AR100": 0.7559768066356651, + "AR_small": 0.6040310189175387, + "AR_medium": 0.7913091585542085, + "AR_large": 0.8838087931121458 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2802.229, + "std": 13.313, + "p50": 2803.933, + "p95": 2815.509, + "p99": 2820.016, + "preprocess_ms": 17.375, + "inference_ms": 2784.119, + "postprocess_ms": 0.736 + } + }, + "throughput": { + "fps_mean": 0.36, + "fps_p50": 0.36 + }, + "model_stats": { + "params_millions": 32.55, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 118.0 + }, + "metadata": { + "benchmark_date": "2026-06-29T03:30:28.978384Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-m__onnx__cpu__rpi5__20260628T034843353689Z.json b/submissions/deimv2-m__onnx__cpu__rpi5__20260628T034843353689Z.json new file mode 100644 index 0000000..cd61330 --- /dev/null +++ b/submissions/deimv2-m__onnx__cpu__rpi5__20260628T034843353689Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-m-onnx-cpu-rpi5-20260628T034843Z", + "created_at": "2026-06-28T03:48:43.353689Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-m", + "name": "deimv2-m", + "family": "deimv2", + "variant": "m", + "source": "libreyolo", + "weights": "LibreDEIMv2m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5601263467392655, + "mAP_50": 0.7292297490227051, + "mAP_75": 0.6153556104085471, + "mAP_small": 0.38810204225815637, + "mAP_medium": 0.6023119533013779, + "mAP_large": 0.7215611191519414, + "AR1": 0.42507423593101595, + "AR10": 0.6943088231150157, + "AR100": 0.7368570541319415, + "AR_small": 0.5641964407869676, + "AR_medium": 0.774128610329005, + "AR_large": 0.8771622951355976 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1699.669, + "std": 15.315, + "p50": 1698.565, + "p95": 1704.212, + "p99": 1755.324, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.59, + "fps_p50": 0.59 + }, + "model_stats": { + "params_millions": 18.13, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:48:43.353689Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-m__pytorch__cpu__rpi5__20260628T232355345875Z.json b/submissions/deimv2-m__pytorch__cpu__rpi5__20260628T232355345875Z.json new file mode 100644 index 0000000..177c3ef --- /dev/null +++ b/submissions/deimv2-m__pytorch__cpu__rpi5__20260628T232355345875Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-m-pytorch-cpu-rpi5-20260628T232355Z", + "created_at": "2026-06-28T23:23:55.345875Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-m", + "name": "deimv2-m", + "family": "deimv2", + "variant": "m", + "source": "libreyolo", + "weights": "LibreDEIMv2m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5601229905454717, + "mAP_50": 0.7292297655671566, + "mAP_75": 0.6153556732298998, + "mAP_small": 0.38810271496061227, + "mAP_medium": 0.6022996527298319, + "mAP_large": 0.7215611210879361, + "AR1": 0.42507423593101595, + "AR10": 0.6942931883432835, + "AR100": 0.7368414193602092, + "AR_small": 0.5641964407869676, + "AR_medium": 0.7740628208553209, + "AR_large": 0.8771622951355976 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1913.191, + "std": 9.537, + "p50": 1914.334, + "p95": 1923.924, + "p99": 1929.852, + "preprocess_ms": 16.634, + "inference_ms": 1895.804, + "postprocess_ms": 0.754 + } + }, + "throughput": { + "fps_mean": 0.52, + "fps_p50": 0.52 + }, + "model_stats": { + "params_millions": 18.36, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 106.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T23:23:55.345875Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-n__onnx__cpu__rpi5__20260628T032357810370Z.json b/submissions/deimv2-n__onnx__cpu__rpi5__20260628T032357810370Z.json new file mode 100644 index 0000000..ce840d1 --- /dev/null +++ b/submissions/deimv2-n__onnx__cpu__rpi5__20260628T032357810370Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-n-onnx-cpu-rpi5-20260628T032357Z", + "created_at": "2026-06-28T03:23:57.810370Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-n", + "name": "deimv2-n", + "family": "deimv2", + "variant": "n", + "source": "libreyolo", + "weights": "LibreDEIMv2n.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.46702654729561943, + "mAP_50": 0.6399052553560688, + "mAP_75": 0.5054600012954302, + "mAP_small": 0.28811860923654115, + "mAP_medium": 0.5139653255366282, + "mAP_large": 0.6324793538325342, + "AR1": 0.38158229768106366, + "AR10": 0.60698925868329, + "AR100": 0.6653036888284473, + "AR_small": 0.43434618816664816, + "AR_medium": 0.7275925487620608, + "AR_large": 0.8386129454975145 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 226.26, + "std": 1.702, + "p50": 226.489, + "p95": 228.449, + "p99": 229.931, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 4.42, + "fps_p50": 4.42 + }, + "model_stats": { + "params_millions": 3.56, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 71.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:23:57.810370Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-n__pytorch__cpu__rpi5__20260628T213301315421Z.json b/submissions/deimv2-n__pytorch__cpu__rpi5__20260628T213301315421Z.json new file mode 100644 index 0000000..e2e0a35 --- /dev/null +++ b/submissions/deimv2-n__pytorch__cpu__rpi5__20260628T213301315421Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-n-pytorch-cpu-rpi5-20260628T213301Z", + "created_at": "2026-06-28T21:33:01.315421Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-n", + "name": "deimv2-n", + "family": "deimv2", + "variant": "n", + "source": "libreyolo", + "weights": "LibreDEIMv2n.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.4670213335038969, + "mAP_50": 0.639905576947522, + "mAP_75": 0.5054596477905674, + "mAP_small": 0.28810252375241885, + "mAP_medium": 0.5139622089791221, + "mAP_large": 0.6324757403885736, + "AR1": 0.38157578980653545, + "AR10": 0.6069542738223006, + "AR100": 0.6652687039674579, + "AR_small": 0.4343253981458581, + "AR_medium": 0.7275327401496207, + "AR_large": 0.8386081389606242 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 423.001, + "std": 11.372, + "p50": 424.559, + "p95": 430.496, + "p99": 433.575, + "preprocess_ms": 9.066, + "inference_ms": 413.181, + "postprocess_ms": 0.754 + } + }, + "throughput": { + "fps_mean": 2.36, + "fps_p50": 2.36 + }, + "model_stats": { + "params_millions": 3.6, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 73.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:33:01.315421Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-pico__onnx__cpu__rpi5__20260628T032145228502Z.json b/submissions/deimv2-pico__onnx__cpu__rpi5__20260628T032145228502Z.json new file mode 100644 index 0000000..d8ee6b2 --- /dev/null +++ b/submissions/deimv2-pico__onnx__cpu__rpi5__20260628T032145228502Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-pico-onnx-cpu-rpi5-20260628T032145Z", + "created_at": "2026-06-28T03:21:45.228502Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-pico", + "name": "deimv2-pico", + "family": "deimv2", + "variant": "pico", + "source": "libreyolo", + "weights": "LibreDEIMv2pico.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4224854946316034, + "mAP_50": 0.5929091004343798, + "mAP_75": 0.45616100220090755, + "mAP_small": 0.26532966904860034, + "mAP_medium": 0.4668541396371766, + "mAP_large": 0.5649540356513841, + "AR1": 0.3475074208313628, + "AR10": 0.5775063386549875, + "AR100": 0.631145447447673, + "AR_small": 0.3960639942633518, + "AR_medium": 0.6938846662696448, + "AR_large": 0.8284719440079611 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 161.527, + "std": 1.661, + "p50": 162.106, + "p95": 163.395, + "p99": 165.149, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 6.19, + "fps_p50": 6.17 + }, + "model_stats": { + "params_millions": 1.51, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 76.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:21:45.228502Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-pico__pytorch__cpu__rpi5__20260628T212218081947Z.json b/submissions/deimv2-pico__pytorch__cpu__rpi5__20260628T212218081947Z.json new file mode 100644 index 0000000..65b77ed --- /dev/null +++ b/submissions/deimv2-pico__pytorch__cpu__rpi5__20260628T212218081947Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-pico-pytorch-cpu-rpi5-20260628T212218Z", + "created_at": "2026-06-28T21:22:18.081947Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-pico", + "name": "deimv2-pico", + "family": "deimv2", + "variant": "pico", + "source": "libreyolo", + "weights": "LibreDEIMv2pico.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.42236369083316583, + "mAP_50": 0.5929095585840876, + "mAP_75": 0.456151853998398, + "mAP_small": 0.26531258037819605, + "mAP_medium": 0.46683143988252807, + "mAP_large": 0.5647437896527114, + "AR1": 0.3473883401228512, + "AR10": 0.5773718981092814, + "AR100": 0.6310409380751396, + "AR_small": 0.39603272087723973, + "AR_medium": 0.6937328805553592, + "AR_large": 0.8282697261345104 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 349.801, + "std": 3.128, + "p50": 349.295, + "p95": 356.684, + "p99": 360.238, + "preprocess_ms": 8.952, + "inference_ms": 340.204, + "postprocess_ms": 0.645 + } + }, + "throughput": { + "fps_mean": 2.86, + "fps_p50": 2.86 + }, + "model_stats": { + "params_millions": 1.54, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 60.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:22:18.081947Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-s__onnx__cpu__rpi5__20260628T033405534056Z.json b/submissions/deimv2-s__onnx__cpu__rpi5__20260628T033405534056Z.json new file mode 100644 index 0000000..4b0bf9c --- /dev/null +++ b/submissions/deimv2-s__onnx__cpu__rpi5__20260628T033405534056Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-s-onnx-cpu-rpi5-20260628T033405Z", + "created_at": "2026-06-28T03:34:05.534056Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-s", + "name": "deimv2-s", + "family": "deimv2", + "variant": "s", + "source": "libreyolo", + "weights": "LibreDEIMv2s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5300700271711015, + "mAP_50": 0.6976653903900368, + "mAP_75": 0.5751331009055647, + "mAP_small": 0.3500893223080657, + "mAP_medium": 0.5804289777484958, + "mAP_large": 0.7299875093807994, + "AR1": 0.4065742563193488, + "AR10": 0.6752756526406756, + "AR100": 0.7241649181407003, + "AR_small": 0.5444134233119735, + "AR_medium": 0.7724843253587157, + "AR_large": 0.8780156970954397 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1162.774, + "std": 6.066, + "p50": 1162.821, + "p95": 1167.175, + "p99": 1173.671, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.86, + "fps_p50": 0.86 + }, + "model_stats": { + "params_millions": 9.73, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T03:34:05.534056Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-s__pytorch__cpu__rpi5__20260628T220533872596Z.json b/submissions/deimv2-s__pytorch__cpu__rpi5__20260628T220533872596Z.json new file mode 100644 index 0000000..bc1368d --- /dev/null +++ b/submissions/deimv2-s__pytorch__cpu__rpi5__20260628T220533872596Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-s-pytorch-cpu-rpi5-20260628T220533Z", + "created_at": "2026-06-28T22:05:33.872596Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-s", + "name": "deimv2-s", + "family": "deimv2", + "variant": "s", + "source": "libreyolo", + "weights": "LibreDEIMv2s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5300705352326833, + "mAP_50": 0.6976651497162647, + "mAP_75": 0.5751333008654972, + "mAP_small": 0.3500893991731694, + "mAP_medium": 0.5804291582052804, + "mAP_large": 0.7299874007669309, + "AR1": 0.4065742563193488, + "AR10": 0.6752756526406756, + "AR100": 0.7241649181407003, + "AR_small": 0.5444134233119735, + "AR_medium": 0.7724843253587157, + "AR_large": 0.8780156970954397 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1190.937, + "std": 12.062, + "p50": 1194.707, + "p95": 1206.359, + "p99": 1218.328, + "preprocess_ms": 17.284, + "inference_ms": 1172.895, + "postprocess_ms": 0.758 + } + }, + "throughput": { + "fps_mean": 0.84, + "fps_p50": 0.84 + }, + "model_stats": { + "params_millions": 9.78, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 50.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T22:05:33.872596Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-x__onnx__cpu__rpi5__20260628T044058635016Z.json b/submissions/deimv2-x__onnx__cpu__rpi5__20260628T044058635016Z.json new file mode 100644 index 0000000..395919c --- /dev/null +++ b/submissions/deimv2-x__onnx__cpu__rpi5__20260628T044058635016Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-x-onnx-cpu-rpi5-20260628T044058Z", + "created_at": "2026-06-28T04:40:58.635016Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-x", + "name": "deimv2-x", + "family": "deimv2", + "variant": "x", + "source": "libreyolo", + "weights": "LibreDEIMv2x.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.6133233772282629, + "mAP_50": 0.7894398948856872, + "mAP_75": 0.6727366504784924, + "mAP_small": 0.4339764082361583, + "mAP_medium": 0.6737693410194326, + "mAP_large": 0.7656487806361741, + "AR1": 0.4453632538830613, + "AR10": 0.7287634403924469, + "AR100": 0.7705460192274229, + "AR_small": 0.6192627518682189, + "AR_medium": 0.8184399305818361, + "AR_large": 0.8827316250588547 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3453.025, + "std": 11.258, + "p50": 3451.917, + "p95": 3461.03, + "p99": 3514.227, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.29, + "fps_p50": 0.29 + }, + "model_stats": { + "params_millions": 50.27, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T04:40:58.635016Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/deimv2-x__pytorch__cpu__rpi5__20260629T062402415113Z.json b/submissions/deimv2-x__pytorch__cpu__rpi5__20260629T062402415113Z.json new file mode 100644 index 0000000..e7c19c8 --- /dev/null +++ b/submissions/deimv2-x__pytorch__cpu__rpi5__20260629T062402415113Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "deimv2-x-pytorch-cpu-rpi5-20260629T062402Z", + "created_at": "2026-06-29T06:24:02.415113Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "deimv2-x", + "name": "deimv2-x", + "family": "deimv2", + "variant": "x", + "source": "libreyolo", + "weights": "LibreDEIMv2x.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.6133221273087972, + "mAP_50": 0.7894407109124595, + "mAP_75": 0.6727361018617944, + "mAP_small": 0.4339747185867168, + "mAP_medium": 0.6737668436781252, + "mAP_large": 0.7656487457031492, + "AR1": 0.4453632538830613, + "AR10": 0.7287634403924469, + "AR100": 0.7705342572890554, + "AR_small": 0.6192456578511248, + "AR_medium": 0.8184399305818361, + "AR_large": 0.8827316250588547 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3711.783, + "std": 16.813, + "p50": 3712.827, + "p95": 3729.248, + "p99": 3737.808, + "preprocess_ms": 17.408, + "inference_ms": 3693.633, + "postprocess_ms": 0.742 + } + }, + "throughput": { + "fps_mean": 0.27, + "fps_p50": 0.27 + }, + "model_stats": { + "params_millions": 51.21, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 45.7 + }, + "metadata": { + "benchmark_date": "2026-06-29T06:24:02.415113Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/dfine-l__onnx__cpu__rpi5__20260628T020541646388Z.json b/submissions/dfine-l__onnx__cpu__rpi5__20260628T020541646388Z.json new file mode 100644 index 0000000..a8d3e81 --- /dev/null +++ b/submissions/dfine-l__onnx__cpu__rpi5__20260628T020541646388Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "dfine-l-onnx-cpu-rpi5-20260628T020541Z", + "created_at": "2026-06-28T02:05:41.646388Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-l", + "name": "d-fine-l", + "family": "dfine", + "variant": "l", + "source": "libreyolo", + "weights": "LibreDFINEl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5998638361378038, + "mAP_50": 0.7714133253871575, + "mAP_75": 0.6545275995421832, + "mAP_small": 0.46006035897968195, + "mAP_medium": 0.651843563065576, + "mAP_large": 0.7519489970046525, + "AR1": 0.4438129932529035, + "AR10": 0.7301763319531722, + "AR100": 0.7823271050417004, + "AR_small": 0.6336871651097996, + "AR_medium": 0.8251280046501868, + "AR_large": 0.902008282481585 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1402.913, + "std": 8.691, + "p50": 1402.601, + "p95": 1408.151, + "p99": 1459.521, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.71, + "fps_p50": 0.71 + }, + "model_stats": { + "params_millions": 30.79, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T02:05:41.646388Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/dfine-l__pytorch__cpu__rpi5__20260629T013105259823Z.json b/submissions/dfine-l__pytorch__cpu__rpi5__20260629T013105259823Z.json new file mode 100644 index 0000000..c082ad7 --- /dev/null +++ b/submissions/dfine-l__pytorch__cpu__rpi5__20260629T013105259823Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "dfine-l-pytorch-cpu-rpi5-20260629T013105Z", + "created_at": "2026-06-29T01:31:05.259823Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-l", + "name": "d-fine-l", + "family": "dfine", + "variant": "l", + "source": "libreyolo", + "weights": "LibreDFINEl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5998637327996268, + "mAP_50": 0.771413290072788, + "mAP_75": 0.6545274000895153, + "mAP_small": 0.4600603643885379, + "mAP_medium": 0.651843569917136, + "mAP_large": 0.7519470808192028, + "AR1": 0.4438129932529035, + "AR10": 0.7301763319531722, + "AR100": 0.7823205971671722, + "AR_small": 0.6336871651097996, + "AR_medium": 0.8251280046501868, + "AR_large": 0.901957546814411 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2305.176, + "std": 13.155, + "p50": 2308.045, + "p95": 2318.262, + "p99": 2325.132, + "preprocess_ms": 7.361, + "inference_ms": 2297.051, + "postprocess_ms": 0.764 + } + }, + "throughput": { + "fps_mean": 0.43, + "fps_p50": 0.43 + }, + "model_stats": { + "params_millions": 31.24, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 89.3 + }, + "metadata": { + "benchmark_date": "2026-06-29T01:31:05.259823Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/dfine-m__onnx__cpu__rpi5__20260628T015332710054Z.json b/submissions/dfine-m__onnx__cpu__rpi5__20260628T015332710054Z.json new file mode 100644 index 0000000..6f31df5 --- /dev/null +++ b/submissions/dfine-m__onnx__cpu__rpi5__20260628T015332710054Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "dfine-m-onnx-cpu-rpi5-20260628T015332Z", + "created_at": "2026-06-28T01:53:32.710054Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-m", + "name": "d-fine-m", + "family": "dfine", + "variant": "m", + "source": "libreyolo", + "weights": "LibreDFINEm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5783006671575653, + "mAP_50": 0.7466815879044602, + "mAP_75": 0.6299110302822055, + "mAP_small": 0.43019262435533856, + "mAP_medium": 0.6140427646392412, + "mAP_large": 0.7292981794777466, + "AR1": 0.4258019573210324, + "AR10": 0.7202899110812604, + "AR100": 0.771521209128276, + "AR_small": 0.6401413616788801, + "AR_medium": 0.7966833420700948, + "AR_large": 0.89146658543394 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 992.427, + "std": 5.017, + "p50": 992.29, + "p95": 996.902, + "p99": 1017.562, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.01, + "fps_p50": 1.01 + }, + "model_stats": { + "params_millions": 19.26, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T01:53:32.710054Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/dfine-m__pytorch__cpu__rpi5__20260628T235147949418Z.json b/submissions/dfine-m__pytorch__cpu__rpi5__20260628T235147949418Z.json new file mode 100644 index 0000000..9db0d6d --- /dev/null +++ b/submissions/dfine-m__pytorch__cpu__rpi5__20260628T235147949418Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "dfine-m-pytorch-cpu-rpi5-20260628T235147Z", + "created_at": "2026-06-28T23:51:47.949418Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-m", + "name": "d-fine-m", + "family": "dfine", + "variant": "m", + "source": "libreyolo", + "weights": "LibreDFINEm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5782994902642316, + "mAP_50": 0.7466821563973365, + "mAP_75": 0.6299117815035142, + "mAP_small": 0.4301932955022992, + "mAP_medium": 0.614041717660949, + "mAP_large": 0.729298114060577, + "AR1": 0.4258019573210324, + "AR10": 0.7202899110812604, + "AR100": 0.7715135321744913, + "AR_small": 0.6401413616788801, + "AR_medium": 0.7966559297893929, + "AR_large": 0.89146658543394 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1577.225, + "std": 13.329, + "p50": 1576.609, + "p95": 1590.509, + "p99": 1597.548, + "preprocess_ms": 7.752, + "inference_ms": 1568.725, + "postprocess_ms": 0.748 + } + }, + "throughput": { + "fps_mean": 0.63, + "fps_p50": 0.63 + }, + "model_stats": { + "params_millions": 19.59, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 79.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T23:51:47.949418Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/dfine-n__onnx__cpu__rpi5__20260628T013952354694Z.json b/submissions/dfine-n__onnx__cpu__rpi5__20260628T013952354694Z.json new file mode 100644 index 0000000..0273d98 --- /dev/null +++ b/submissions/dfine-n__onnx__cpu__rpi5__20260628T013952354694Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "dfine-n-onnx-cpu-rpi5-20260628T013952Z", + "created_at": "2026-06-28T01:39:52.354694Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-n", + "name": "d-fine-n", + "family": "dfine", + "variant": "n", + "source": "libreyolo", + "weights": "LibreDFINEn.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4577159174725761, + "mAP_50": 0.6276159010117028, + "mAP_75": 0.4936186188949874, + "mAP_small": 0.2582314909429722, + "mAP_medium": 0.5148568614812048, + "mAP_large": 0.6164865684657567, + "AR1": 0.3662054162313099, + "AR10": 0.5972180880952037, + "AR100": 0.64941207677179, + "AR_small": 0.4247151375969516, + "AR_medium": 0.7131204074253821, + "AR_large": 0.8464105287851524 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 229.676, + "std": 2.711, + "p50": 229.555, + "p95": 231.445, + "p99": 233.676, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 4.35, + "fps_p50": 4.36 + }, + "model_stats": { + "params_millions": 3.75, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 71.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T01:39:52.354694Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/dfine-n__pytorch__cpu__rpi5__20260628T214059576864Z.json b/submissions/dfine-n__pytorch__cpu__rpi5__20260628T214059576864Z.json new file mode 100644 index 0000000..66b5f70 --- /dev/null +++ b/submissions/dfine-n__pytorch__cpu__rpi5__20260628T214059576864Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "dfine-n-pytorch-cpu-rpi5-20260628T214059Z", + "created_at": "2026-06-28T21:40:59.576864Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-n", + "name": "d-fine-n", + "family": "dfine", + "variant": "n", + "source": "libreyolo", + "weights": "LibreDFINEn.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.4576828753960137, + "mAP_50": 0.627611672354216, + "mAP_75": 0.4936120506981263, + "mAP_small": 0.25813013104993077, + "mAP_medium": 0.5148552892369967, + "mAP_large": 0.6164531686764337, + "AR1": 0.3661276595022023, + "AR10": 0.597139132066487, + "AR100": 0.6492993825514405, + "AR_small": 0.4246210950171405, + "AR_medium": 0.7131204074253821, + "AR_large": 0.846219767993009 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 429.079, + "std": 5.636, + "p50": 431.126, + "p95": 435.518, + "p99": 436.986, + "preprocess_ms": 7.795, + "inference_ms": 420.516, + "postprocess_ms": 0.769 + } + }, + "throughput": { + "fps_mean": 2.33, + "fps_p50": 2.32 + }, + "model_stats": { + "params_millions": 3.78, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 91.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:40:59.576864Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/dfine-s__onnx__cpu__rpi5__20260628T014453094863Z.json b/submissions/dfine-s__onnx__cpu__rpi5__20260628T014453094863Z.json new file mode 100644 index 0000000..ae4f1b1 --- /dev/null +++ b/submissions/dfine-s__onnx__cpu__rpi5__20260628T014453094863Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "dfine-s-onnx-cpu-rpi5-20260628T014453Z", + "created_at": "2026-06-28T01:44:53.094863Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-s", + "name": "d-fine-s", + "family": "dfine", + "variant": "s", + "source": "libreyolo", + "weights": "LibreDFINEs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.533843125935515, + "mAP_50": 0.6978897743414051, + "mAP_75": 0.5815144847483065, + "mAP_small": 0.37951088111313763, + "mAP_medium": 0.5786108122038341, + "mAP_large": 0.6712528670597301, + "AR1": 0.40805078036777326, + "AR10": 0.684853567531656, + "AR100": 0.7344177724592036, + "AR_small": 0.5816815390659993, + "AR_medium": 0.7768773182389905, + "AR_large": 0.8765596543653289 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 559.268, + "std": 4.922, + "p50": 559.498, + "p95": 562.537, + "p99": 564.198, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.79, + "fps_p50": 1.79 + }, + "model_stats": { + "params_millions": 10.25, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T01:44:53.094863Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/dfine-s__pytorch__cpu__rpi5__20260628T223148995294Z.json b/submissions/dfine-s__pytorch__cpu__rpi5__20260628T223148995294Z.json new file mode 100644 index 0000000..796f00c --- /dev/null +++ b/submissions/dfine-s__pytorch__cpu__rpi5__20260628T223148995294Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "dfine-s-pytorch-cpu-rpi5-20260628T223148Z", + "created_at": "2026-06-28T22:31:48.995294Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-s", + "name": "d-fine-s", + "family": "dfine", + "variant": "s", + "source": "libreyolo", + "weights": "LibreDFINEs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.533816550656966, + "mAP_50": 0.6978934949475171, + "mAP_75": 0.5815146677466713, + "mAP_small": 0.3794386468726935, + "mAP_medium": 0.5786169334696805, + "mAP_large": 0.6712528261067662, + "AR1": 0.40807927039626324, + "AR10": 0.6848022854803739, + "AR100": 0.7343664904079216, + "AR_small": 0.5815105988950591, + "AR_medium": 0.7768773182389905, + "AR_large": 0.8765596543653289 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 874.322, + "std": 7.672, + "p50": 874.788, + "p95": 886.507, + "p99": 894.322, + "preprocess_ms": 7.773, + "inference_ms": 865.785, + "postprocess_ms": 0.765 + } + }, + "throughput": { + "fps_mean": 1.14, + "fps_p50": 1.14 + }, + "model_stats": { + "params_millions": 10.32, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 109.7 + }, + "metadata": { + "benchmark_date": "2026-06-28T22:31:48.995294Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/dfine-x__onnx__cpu__rpi5__20260628T022812543209Z.json b/submissions/dfine-x__onnx__cpu__rpi5__20260628T022812543209Z.json new file mode 100644 index 0000000..f3006bc --- /dev/null +++ b/submissions/dfine-x__onnx__cpu__rpi5__20260628T022812543209Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "dfine-x-onnx-cpu-rpi5-20260628T022812Z", + "created_at": "2026-06-28T02:28:12.543209Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "dfine-x", + "name": "d-fine-x", + "family": "dfine", + "variant": "x", + "source": "libreyolo", + "weights": "LibreDFINEx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.6142388237060353, + "mAP_50": 0.7798523770021832, + "mAP_75": 0.6720056040137532, + "mAP_small": 0.478398640099852, + "mAP_medium": 0.673787867178134, + "mAP_large": 0.7694334953082029, + "AR1": 0.44763907531490205, + "AR10": 0.7492636927367275, + "AR100": 0.79469353444029, + "AR_small": 0.6545081595760803, + "AR_medium": 0.8381553060186311, + "AR_large": 0.9082575658932268 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2638.747, + "std": 32.632, + "p50": 2636.49, + "p95": 2645.381, + "p99": 2700.959, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.38, + "fps_p50": 0.38 + }, + "model_stats": { + "params_millions": 61.71, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T02:28:12.543209Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/ec-l__onnx__cpu__rpi5__20260628T010919277114Z.json b/submissions/ec-l__onnx__cpu__rpi5__20260628T010919277114Z.json new file mode 100644 index 0000000..23c12bd --- /dev/null +++ b/submissions/ec-l__onnx__cpu__rpi5__20260628T010919277114Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "ec-l-onnx-cpu-rpi5-20260628T010919Z", + "created_at": "2026-06-28T01:09:19.277114Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-l", + "name": "ec-l", + "family": "ec", + "variant": "l", + "source": "libreyolo", + "weights": "LibreECl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.6007115783135872, + "mAP_50": 0.7749472354400845, + "mAP_75": 0.6523345711566724, + "mAP_small": 0.43393729112076873, + "mAP_medium": 0.6628074404971023, + "mAP_large": 0.7572737906104186, + "AR1": 0.4375595411244058, + "AR10": 0.7201219301492249, + "AR100": 0.759011988294033, + "AR_small": 0.5846511426384247, + "AR_medium": 0.8139374671717773, + "AR_large": 0.8965742116512401 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2681.797, + "std": 9.081, + "p50": 2680.598, + "p95": 2688.627, + "p99": 2736.943, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.37, + "fps_p50": 0.37 + }, + "model_stats": { + "params_millions": 32.69, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T01:09:19.277114Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/ec-l__pytorch__cpu__rpi5__20260629T015522978697Z.json b/submissions/ec-l__pytorch__cpu__rpi5__20260629T015522978697Z.json new file mode 100644 index 0000000..1b7f608 --- /dev/null +++ b/submissions/ec-l__pytorch__cpu__rpi5__20260629T015522978697Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "ec-l-pytorch-cpu-rpi5-20260629T015522Z", + "created_at": "2026-06-29T01:55:22.978697Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-l", + "name": "ec-l", + "family": "ec", + "variant": "l", + "source": "libreyolo", + "weights": "LibreECl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.6007011060957481, + "mAP_50": 0.7749474593822242, + "mAP_75": 0.6523360761497325, + "mAP_small": 0.4339405977702752, + "mAP_medium": 0.6628074293970688, + "mAP_large": 0.7572514538766986, + "AR1": 0.43754390635267343, + "AR10": 0.7201062953774926, + "AR100": 0.7589963535223007, + "AR_small": 0.5846511426384247, + "AR_medium": 0.8139374671717773, + "AR_large": 0.8965390869585811 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2850.252, + "std": 11.02, + "p50": 2852.271, + "p95": 2864.132, + "p99": 2870.324, + "preprocess_ms": 15.0, + "inference_ms": 2834.501, + "postprocess_ms": 0.751 + } + }, + "throughput": { + "fps_mean": 0.35, + "fps_p50": 0.35 + }, + "model_stats": { + "params_millions": 32.97, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 73.0 + }, + "metadata": { + "benchmark_date": "2026-06-29T01:55:22.978697Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/ec-m__onnx__cpu__rpi5__20260628T004627037156Z.json b/submissions/ec-m__onnx__cpu__rpi5__20260628T004627037156Z.json new file mode 100644 index 0000000..82aade9 --- /dev/null +++ b/submissions/ec-m__onnx__cpu__rpi5__20260628T004627037156Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "ec-m-onnx-cpu-rpi5-20260628T004627Z", + "created_at": "2026-06-28T00:46:27.037156Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-m", + "name": "ec-m", + "family": "ec", + "variant": "m", + "source": "libreyolo", + "weights": "LibreECm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5835379539521929, + "mAP_50": 0.757759379520078, + "mAP_75": 0.6350328729707719, + "mAP_small": 0.40672995162395603, + "mAP_medium": 0.6388688026086551, + "mAP_large": 0.7501653928844247, + "AR1": 0.43343530313979, + "AR10": 0.7050906565517581, + "AR100": 0.7503141190923781, + "AR_small": 0.5728496539199226, + "AR_medium": 0.7894453607495379, + "AR_large": 0.8827568118289735 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1708.186, + "std": 8.115, + "p50": 1707.671, + "p95": 1713.206, + "p99": 1762.731, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.59, + "fps_p50": 0.59 + }, + "model_stats": { + "params_millions": 19.23, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T00:46:27.037156Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/ec-m__pytorch__cpu__rpi5__20260628T230648254971Z.json b/submissions/ec-m__pytorch__cpu__rpi5__20260628T230648254971Z.json new file mode 100644 index 0000000..59d1f9c --- /dev/null +++ b/submissions/ec-m__pytorch__cpu__rpi5__20260628T230648254971Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "ec-m-pytorch-cpu-rpi5-20260628T230648Z", + "created_at": "2026-06-28T23:06:48.254971Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-m", + "name": "ec-m", + "family": "ec", + "variant": "m", + "source": "libreyolo", + "weights": "LibreECm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5835333586126803, + "mAP_50": 0.7577586900096812, + "mAP_75": 0.6350329290731235, + "mAP_small": 0.40672990096458406, + "mAP_medium": 0.6388487836333429, + "mAP_large": 0.7501653894101447, + "AR1": 0.43343530313979, + "AR10": 0.7050750217800258, + "AR100": 0.7502984843206456, + "AR_small": 0.5728496539199226, + "AR_medium": 0.7893795712758537, + "AR_large": 0.8827568118289735 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1868.727, + "std": 16.432, + "p50": 1874.334, + "p95": 1883.764, + "p99": 1886.904, + "preprocess_ms": 14.959, + "inference_ms": 1853.027, + "postprocess_ms": 0.742 + } + }, + "throughput": { + "fps_mean": 0.54, + "fps_p50": 0.53 + }, + "model_stats": { + "params_millions": 19.43, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 97.3 + }, + "metadata": { + "benchmark_date": "2026-06-28T23:06:48.254971Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/ec-s__onnx__cpu__rpi5__20260628T003144772328Z.json b/submissions/ec-s__onnx__cpu__rpi5__20260628T003144772328Z.json new file mode 100644 index 0000000..1180154 --- /dev/null +++ b/submissions/ec-s__onnx__cpu__rpi5__20260628T003144772328Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "ec-s-onnx-cpu-rpi5-20260628T003144Z", + "created_at": "2026-06-28T00:31:44.772328Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-s", + "name": "ec-s", + "family": "ec", + "variant": "s", + "source": "libreyolo", + "weights": "LibreECs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5432194719471461, + "mAP_50": 0.7140171161058909, + "mAP_75": 0.5923210155991958, + "mAP_small": 0.3666505350937702, + "mAP_medium": 0.6000801241816464, + "mAP_large": 0.7143119547282032, + "AR1": 0.42118346438115173, + "AR10": 0.6849257047218618, + "AR100": 0.7312556280979691, + "AR_small": 0.5461653398174465, + "AR_medium": 0.7835733460879325, + "AR_large": 0.8876717561873844 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1154.939, + "std": 7.895, + "p50": 1155.213, + "p95": 1159.207, + "p99": 1165.138, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.87, + "fps_p50": 0.87 + }, + "model_stats": { + "params_millions": 9.84, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T00:31:44.772328Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/ec-s__pytorch__cpu__rpi5__20260628T221558217609Z.json b/submissions/ec-s__pytorch__cpu__rpi5__20260628T221558217609Z.json new file mode 100644 index 0000000..c3510a6 --- /dev/null +++ b/submissions/ec-s__pytorch__cpu__rpi5__20260628T221558217609Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "ec-s-pytorch-cpu-rpi5-20260628T221558Z", + "created_at": "2026-06-28T22:15:58.217609Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-s", + "name": "ec-s", + "family": "ec", + "variant": "s", + "source": "libreyolo", + "weights": "LibreECs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5432219445119648, + "mAP_50": 0.714017633533976, + "mAP_75": 0.5923221234790975, + "mAP_small": 0.3666503591643378, + "mAP_medium": 0.6000808289526866, + "mAP_large": 0.7143113755076108, + "AR1": 0.42118346438115173, + "AR10": 0.6849257047218618, + "AR100": 0.7312556280979691, + "AR_small": 0.5461653398174465, + "AR_medium": 0.7835733460879325, + "AR_large": 0.8876717561873844 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1200.969, + "std": 8.783, + "p50": 1200.297, + "p95": 1217.586, + "p99": 1224.614, + "preprocess_ms": 15.796, + "inference_ms": 1184.428, + "postprocess_ms": 0.745 + } + }, + "throughput": { + "fps_mean": 0.83, + "fps_p50": 0.83 + }, + "model_stats": { + "params_millions": 9.88, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 94.3 + }, + "metadata": { + "benchmark_date": "2026-06-28T22:15:58.217609Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/ec-x__onnx__cpu__rpi5__20260628T013738374712Z.json b/submissions/ec-x__onnx__cpu__rpi5__20260628T013738374712Z.json new file mode 100644 index 0000000..7f3abeb --- /dev/null +++ b/submissions/ec-x__onnx__cpu__rpi5__20260628T013738374712Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "ec-x-onnx-cpu-rpi5-20260628T013738Z", + "created_at": "2026-06-28T01:37:38.374712Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-x", + "name": "ec-x", + "family": "ec", + "variant": "x", + "source": "libreyolo", + "weights": "LibreECx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.6113462650555552, + "mAP_50": 0.7856324800583312, + "mAP_75": 0.6655222088638381, + "mAP_small": 0.4359469005574484, + "mAP_medium": 0.676115542202016, + "mAP_large": 0.7790291400209074, + "AR1": 0.4380420119244032, + "AR10": 0.7264733654815458, + "AR100": 0.7625240302226778, + "AR_small": 0.5840608876384857, + "AR_medium": 0.8115087135596564, + "AR_large": 0.9073395719728262 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3331.418, + "std": 10.066, + "p50": 3330.154, + "p95": 3339.355, + "p99": 3387.449, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.3, + "fps_p50": 0.3 + }, + "model_stats": { + "params_millions": 48.99, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.4 + }, + "metadata": { + "benchmark_date": "2026-06-28T01:37:38.374712Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/ec-x__pytorch__cpu__rpi5__20260629T055009308891Z.json b/submissions/ec-x__pytorch__cpu__rpi5__20260629T055009308891Z.json new file mode 100644 index 0000000..142b77e --- /dev/null +++ b/submissions/ec-x__pytorch__cpu__rpi5__20260629T055009308891Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "ec-x-pytorch-cpu-rpi5-20260629T055009Z", + "created_at": "2026-06-29T05:50:09.308891Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "ec-x", + "name": "ec-x", + "family": "ec", + "variant": "x", + "source": "libreyolo", + "weights": "LibreECx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.6113461416284465, + "mAP_50": 0.7856329163312351, + "mAP_75": 0.665522214313548, + "mAP_small": 0.43594677616719263, + "mAP_medium": 0.6761155368028872, + "mAP_large": 0.7790291400209074, + "AR1": 0.4380420119244032, + "AR10": 0.7264733654815458, + "AR100": 0.7625240302226778, + "AR_small": 0.5840608876384857, + "AR_medium": 0.8115087135596564, + "AR_large": 0.9073395719728262 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3635.078, + "std": 17.441, + "p50": 3637.342, + "p95": 3653.653, + "p99": 3660.413, + "preprocess_ms": 14.978, + "inference_ms": 3619.34, + "postprocess_ms": 0.76 + } + }, + "throughput": { + "fps_mean": 0.28, + "fps_p50": 0.27 + }, + "model_stats": { + "params_millions": 49.94, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 91.3 + }, + "metadata": { + "benchmark_date": "2026-06-29T05:50:09.308891Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/picodet-l__ncnn__cpu__rpi5__20260628T091505814759Z.json b/submissions/picodet-l__ncnn__cpu__rpi5__20260628T091505814759Z.json new file mode 100644 index 0000000..c28b18a --- /dev/null +++ b/submissions/picodet-l__ncnn__cpu__rpi5__20260628T091505814759Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "picodet-l-ncnn-cpu-rpi5-20260628T091505Z", + "created_at": "2026-06-28T09:15:05.814759Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-l", + "name": "picodet-l", + "family": "picodet", + "variant": "l", + "source": "libreyolo", + "weights": "LibrePICODETl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.42770296733531016, + "mAP_50": 0.5944168351919669, + "mAP_75": 0.46411415029057845, + "mAP_small": 0.23192866375154145, + "mAP_medium": 0.47685405613134785, + "mAP_large": 0.5661665410866924, + "AR1": 0.3405386329048061, + "AR10": 0.5360245143308368, + "AR100": 0.5679978126498423, + "AR_small": 0.33515737612793667, + "AR_medium": 0.6114916185523256, + "AR_large": 0.699992934376178 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 441.829, + "std": 39.479, + "p50": 443.541, + "p95": 506.988, + "p99": 537.778, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 2.26, + "fps_p50": 2.25 + }, + "model_stats": { + "params_millions": 0.0, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.2 + }, + "metadata": { + "benchmark_date": "2026-06-28T09:15:05.814759Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/picodet-l__onnx__cpu__rpi5__20260628T002141253882Z.json b/submissions/picodet-l__onnx__cpu__rpi5__20260628T002141253882Z.json new file mode 100644 index 0000000..e5208ea --- /dev/null +++ b/submissions/picodet-l__onnx__cpu__rpi5__20260628T002141253882Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "picodet-l-onnx-cpu-rpi5-20260628T002141Z", + "created_at": "2026-06-28T00:21:41.253882Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-l", + "name": "picodet-l", + "family": "picodet", + "variant": "l", + "source": "libreyolo", + "weights": "LibrePICODETl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.42670912044377685, + "mAP_50": 0.5936773089219671, + "mAP_75": 0.4612754592116292, + "mAP_small": 0.23112272997138533, + "mAP_medium": 0.4779040568106618, + "mAP_large": 0.5654200117039199, + "AR1": 0.3396315456658758, + "AR10": 0.5344037912182181, + "AR100": 0.5662394426425511, + "AR_small": 0.3343903110723805, + "AR_medium": 0.6121755479346273, + "AR_large": 0.6979159624415997 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 589.472, + "std": 39.594, + "p50": 589.567, + "p95": 654.582, + "p99": 686.693, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.7, + "fps_p50": 1.7 + }, + "model_stats": { + "params_millions": 3.29, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 68.4 + }, + "metadata": { + "benchmark_date": "2026-06-28T00:21:41.253882Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/picodet-l__pytorch__cpu__rpi5__20260628T212909759037Z.json b/submissions/picodet-l__pytorch__cpu__rpi5__20260628T212909759037Z.json new file mode 100644 index 0000000..8870fe3 --- /dev/null +++ b/submissions/picodet-l__pytorch__cpu__rpi5__20260628T212909759037Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "picodet-l-pytorch-cpu-rpi5-20260628T212909Z", + "created_at": "2026-06-28T21:29:09.759037Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-l", + "name": "picodet-l", + "family": "picodet", + "variant": "l", + "source": "libreyolo", + "weights": "LibrePICODETl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.44180297724122514, + "mAP_50": 0.6123659448116612, + "mAP_75": 0.47989821583822584, + "mAP_small": 0.25367398272057823, + "mAP_medium": 0.48866727706769647, + "mAP_large": 0.6026292859455765, + "AR1": 0.35750711092113147, + "AR10": 0.585816152664366, + "AR100": 0.6265703629517813, + "AR_small": 0.4101739411217021, + "AR_medium": 0.686802522647409, + "AR_large": 0.7950929986667825 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 540.311, + "std": 12.087, + "p50": 539.77, + "p95": 551.2, + "p99": 555.588, + "preprocess_ms": 14.362, + "inference_ms": 494.842, + "postprocess_ms": 31.106 + } + }, + "throughput": { + "fps_mean": 1.85, + "fps_p50": 1.85 + }, + "model_stats": { + "params_millions": 3.31, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 87.4 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:29:09.759037Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/picodet-m__ncnn__cpu__rpi5__20260628T084859231781Z.json b/submissions/picodet-m__ncnn__cpu__rpi5__20260628T084859231781Z.json new file mode 100644 index 0000000..2a559c8 --- /dev/null +++ b/submissions/picodet-m__ncnn__cpu__rpi5__20260628T084859231781Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "picodet-m-ncnn-cpu-rpi5-20260628T084859Z", + "created_at": "2026-06-28T08:48:59.231781Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-m", + "name": "picodet-m", + "family": "picodet", + "variant": "m", + "source": "libreyolo", + "weights": "LibrePICODETm.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.3644691634603584, + "mAP_50": 0.5152405901703948, + "mAP_75": 0.38933970019931985, + "mAP_small": 0.15878492232794844, + "mAP_medium": 0.3980997373437907, + "mAP_large": 0.546612926981912, + "AR1": 0.2918425050346936, + "AR10": 0.45549597432797595, + "AR100": 0.48503044648524485, + "AR_small": 0.2113143560310581, + "AR_medium": 0.5345845813031153, + "AR_large": 0.689623196708951 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 117.78, + "std": 13.634, + "p50": 117.036, + "p95": 138.628, + "p99": 151.139, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 8.49, + "fps_p50": 8.54 + }, + "model_stats": { + "params_millions": 0.0, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 76.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:48:59.231781Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/picodet-m__onnx__cpu__rpi5__20260628T001629175431Z.json b/submissions/picodet-m__onnx__cpu__rpi5__20260628T001629175431Z.json new file mode 100644 index 0000000..70e47bd --- /dev/null +++ b/submissions/picodet-m__onnx__cpu__rpi5__20260628T001629175431Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "picodet-m-onnx-cpu-rpi5-20260628T001629Z", + "created_at": "2026-06-28T00:16:29.175431Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-m", + "name": "picodet-m", + "family": "picodet", + "variant": "m", + "source": "libreyolo", + "weights": "LibrePICODETm.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.36518214747227806, + "mAP_50": 0.5150924681561029, + "mAP_75": 0.39078171842360454, + "mAP_small": 0.15625223645887754, + "mAP_medium": 0.4044241653234226, + "mAP_large": 0.5489585415064024, + "AR1": 0.29175206643260554, + "AR10": 0.4556965436968646, + "AR100": 0.48529481762450577, + "AR_small": 0.20353234109992768, + "AR_medium": 0.541150288949737, + "AR_large": 0.6780719958670148 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 160.153, + "std": 13.454, + "p50": 159.777, + "p95": 181.497, + "p99": 193.048, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 6.24, + "fps_p50": 6.26 + }, + "model_stats": { + "params_millions": 2.14, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 76.4 + }, + "metadata": { + "benchmark_date": "2026-06-28T00:16:29.175431Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/picodet-m__pytorch__cpu__rpi5__20260628T212418426331Z.json b/submissions/picodet-m__pytorch__cpu__rpi5__20260628T212418426331Z.json new file mode 100644 index 0000000..7802b23 --- /dev/null +++ b/submissions/picodet-m__pytorch__cpu__rpi5__20260628T212418426331Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "picodet-m-pytorch-cpu-rpi5-20260628T212418Z", + "created_at": "2026-06-28T21:24:18.426331Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-m", + "name": "picodet-m", + "family": "picodet", + "variant": "m", + "source": "libreyolo", + "weights": "LibrePICODETm.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.37622258563330857, + "mAP_50": 0.5302670327093372, + "mAP_75": 0.4031927733445861, + "mAP_small": 0.1774283925337442, + "mAP_medium": 0.40932920767836517, + "mAP_large": 0.5730505210815525, + "AR1": 0.31237085609548815, + "AR10": 0.5225526014836394, + "AR100": 0.5596398389459811, + "AR_small": 0.2782176748170048, + "AR_medium": 0.6178179023642811, + "AR_large": 0.783357700560243 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 201.122, + "std": 9.322, + "p50": 200.585, + "p95": 204.204, + "p99": 208.875, + "preprocess_ms": 8.212, + "inference_ms": 169.492, + "postprocess_ms": 23.418 + } + }, + "throughput": { + "fps_mean": 4.97, + "fps_p50": 4.99 + }, + "model_stats": { + "params_millions": 2.15, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 82.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:24:18.426331Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/picodet-s__ncnn__cpu__rpi5__20260628T084729796838Z.json b/submissions/picodet-s__ncnn__cpu__rpi5__20260628T084729796838Z.json new file mode 100644 index 0000000..7a5f017 --- /dev/null +++ b/submissions/picodet-s__ncnn__cpu__rpi5__20260628T084729796838Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "picodet-s-ncnn-cpu-rpi5-20260628T084729Z", + "created_at": "2026-06-28T08:47:29.796838Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-s", + "name": "picodet-s", + "family": "picodet", + "variant": "s", + "source": "libreyolo", + "weights": "LibrePICODETs.pt", + "input_size": 320 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 320, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.27668476253288005, + "mAP_50": 0.4081527727174869, + "mAP_75": 0.2872617489863077, + "mAP_small": 0.10346510842730021, + "mAP_medium": 0.28694097782031575, + "mAP_large": 0.450153507877427, + "AR1": 0.24787276519511137, + "AR10": 0.3773122272971989, + "AR100": 0.4010293646530582, + "AR_small": 0.1368677435465607, + "AR_medium": 0.4374935072503024, + "AR_large": 0.6070085634595899 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 53.34, + "std": 6.732, + "p50": 52.9, + "p95": 63.992, + "p99": 69.984, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 18.75, + "fps_p50": 18.9 + }, + "model_stats": { + "params_millions": 0.0, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 73.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:47:29.796838Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/picodet-s__onnx__cpu__rpi5__20260628T001452461958Z.json b/submissions/picodet-s__onnx__cpu__rpi5__20260628T001452461958Z.json new file mode 100644 index 0000000..1a1bee5 --- /dev/null +++ b/submissions/picodet-s__onnx__cpu__rpi5__20260628T001452461958Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "picodet-s-onnx-cpu-rpi5-20260628T001452Z", + "created_at": "2026-06-28T00:14:52.461958Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-s", + "name": "picodet-s", + "family": "picodet", + "variant": "s", + "source": "libreyolo", + "weights": "LibrePICODETs.pt", + "input_size": 320 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 320, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.2813929119899346, + "mAP_50": 0.41334677585508217, + "mAP_75": 0.2914752838468096, + "mAP_small": 0.10473734019737763, + "mAP_medium": 0.2847293168548949, + "mAP_large": 0.45544267933014243, + "AR1": 0.2507679660084101, + "AR10": 0.3812605234864112, + "AR100": 0.40703112811926967, + "AR_small": 0.13952395056463135, + "AR_medium": 0.44171381382718916, + "AR_large": 0.6117060183789583 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 66.772, + "std": 6.813, + "p50": 66.223, + "p95": 77.632, + "p99": 83.263, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 14.98, + "fps_p50": 15.1 + }, + "model_stats": { + "params_millions": 0.98, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 77.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T00:14:52.461958Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/picodet-s__pytorch__cpu__rpi5__20260628T211903354874Z.json b/submissions/picodet-s__pytorch__cpu__rpi5__20260628T211903354874Z.json new file mode 100644 index 0000000..4dcea8d --- /dev/null +++ b/submissions/picodet-s__pytorch__cpu__rpi5__20260628T211903354874Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "picodet-s-pytorch-cpu-rpi5-20260628T211903Z", + "created_at": "2026-06-28T21:19:03.354874Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "picodet-s", + "name": "picodet-s", + "family": "picodet", + "variant": "s", + "source": "libreyolo", + "weights": "LibrePICODETs.pt", + "input_size": 320 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 320, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.2948574387762822, + "mAP_50": 0.4317496219042625, + "mAP_75": 0.3066188182975883, + "mAP_small": 0.11303066903813051, + "mAP_medium": 0.3009440965032811, + "mAP_large": 0.4801055275972295, + "AR1": 0.27111322725289133, + "AR10": 0.44450670126411373, + "AR100": 0.47922131990213757, + "AR_small": 0.18632252231207877, + "AR_medium": 0.5368430036125633, + "AR_large": 0.7142117190467947 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 95.043, + "std": 9.215, + "p50": 94.542, + "p95": 96.326, + "p99": 101.109, + "preprocess_ms": 6.029, + "inference_ms": 67.874, + "postprocess_ms": 21.14 + } + }, + "throughput": { + "fps_mean": 10.52, + "fps_p50": 10.58 + }, + "model_stats": { + "params_millions": 0.99, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 96.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:19:03.354874Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rfdetr-l__onnx__cpu__rpi5__20260627T221539909474Z.json b/submissions/rfdetr-l__onnx__cpu__rpi5__20260627T221539909474Z.json new file mode 100644 index 0000000..3b0fbab --- /dev/null +++ b/submissions/rfdetr-l__onnx__cpu__rpi5__20260627T221539909474Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-l-onnx-cpu-rpi5-20260627T221539Z", + "created_at": "2026-06-27T22:15:39.909474Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-l", + "name": "rf-detr-l", + "family": "rfdetr", + "variant": "l", + "source": "libreyolo", + "weights": "LibreRFDETRl.pt", + "input_size": 704 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 704, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5859589819852379, + "mAP_50": 0.7607832992022558, + "mAP_75": 0.6430713642386356, + "mAP_small": 0.40893469014459016, + "mAP_medium": 0.6424580981621197, + "mAP_large": 0.7466233369396336, + "AR1": 0.4381431421070228, + "AR10": 0.7121681420603849, + "AR100": 0.759825162772023, + "AR_small": 0.5765694118901368, + "AR_medium": 0.8072927342544284, + "AR_large": 0.9056357201257855 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1924.498, + "std": 2.497, + "p50": 1924.122, + "p95": 1928.538, + "p99": 1931.583, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.52, + "fps_p50": 0.52 + }, + "model_stats": { + "params_millions": 30.33, + "gflops": 340.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T22:15:39.909474Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rfdetr-l__pytorch__cpu__rpi5__20260627T230925397322Z.json b/submissions/rfdetr-l__pytorch__cpu__rpi5__20260627T230925397322Z.json new file mode 100644 index 0000000..38acf1e --- /dev/null +++ b/submissions/rfdetr-l__pytorch__cpu__rpi5__20260627T230925397322Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-l-pytorch-cpu-rpi5-20260627T230925Z", + "created_at": "2026-06-27T23:09:25.397322Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-l", + "name": "rf-detr-l", + "family": "rfdetr", + "variant": "l", + "source": "libreyolo", + "weights": "LibreRFDETRl.pt", + "input_size": 704 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 704, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5859255759140363, + "mAP_50": 0.7608500969953391, + "mAP_75": 0.6431066159618175, + "mAP_small": 0.40890502726004985, + "mAP_medium": 0.6424034930844548, + "mAP_large": 0.7466197007070701, + "AR1": 0.43805125196352723, + "AR10": 0.7120575227968798, + "AR100": 0.7595246001966368, + "AR_small": 0.5749240860020073, + "AR_medium": 0.8069345471199254, + "AR_large": 0.9054636091317568 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2110.219, + "std": 19.52, + "p50": 2111.73, + "p95": 2127.169, + "p99": 2134.949, + "preprocess_ms": 21.459, + "inference_ms": 2086.167, + "postprocess_ms": 2.593 + } + }, + "throughput": { + "fps_mean": 0.47, + "fps_p50": 0.47 + }, + "model_stats": { + "params_millions": 33.93, + "gflops": 340.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T23:09:25.397322Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rfdetr-m__onnx__cpu__rpi5__20260627T215916603731Z.json b/submissions/rfdetr-m__onnx__cpu__rpi5__20260627T215916603731Z.json new file mode 100644 index 0000000..7748de9 --- /dev/null +++ b/submissions/rfdetr-m__onnx__cpu__rpi5__20260627T215916603731Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-m-onnx-cpu-rpi5-20260627T215916Z", + "created_at": "2026-06-27T21:59:16.603731Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-m", + "name": "rf-detr-m", + "family": "rfdetr", + "variant": "m", + "source": "libreyolo", + "weights": "LibreRFDETRm.pt", + "input_size": 576 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 576, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.574212802247223, + "mAP_50": 0.7532792126972107, + "mAP_75": 0.624341870173254, + "mAP_small": 0.3839341593907846, + "mAP_medium": 0.6332878141582797, + "mAP_large": 0.7435058935858826, + "AR1": 0.4296453683453095, + "AR10": 0.6988397888599172, + "AR100": 0.7430848130856929, + "AR_small": 0.5319319331259718, + "AR_medium": 0.8035658603539425, + "AR_large": 0.9124160945152083 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1142.177, + "std": 3.034, + "p50": 1141.894, + "p95": 1143.956, + "p99": 1147.717, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.88, + "fps_p50": 0.88 + }, + "model_stats": { + "params_millions": 30.08, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.2 + }, + "metadata": { + "benchmark_date": "2026-06-27T21:59:16.603731Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rfdetr-m__pytorch__cpu__rpi5__20260627T224658295133Z.json b/submissions/rfdetr-m__pytorch__cpu__rpi5__20260627T224658295133Z.json new file mode 100644 index 0000000..621d47f --- /dev/null +++ b/submissions/rfdetr-m__pytorch__cpu__rpi5__20260627T224658295133Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-m-pytorch-cpu-rpi5-20260627T224658Z", + "created_at": "2026-06-27T22:46:58.295133Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-m", + "name": "rf-detr-m", + "family": "rfdetr", + "variant": "m", + "source": "libreyolo", + "weights": "LibreRFDETRm.pt", + "input_size": 576 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 576, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5739331874129254, + "mAP_50": 0.7532911305959131, + "mAP_75": 0.6243325497808554, + "mAP_small": 0.3839298566399166, + "mAP_medium": 0.633282711912449, + "mAP_large": 0.7432463466158686, + "AR1": 0.4293236562251876, + "AR10": 0.6988373902606991, + "AR100": 0.7430289499869026, + "AR_small": 0.5319135953843089, + "AR_medium": 0.8035621009554462, + "AR_large": 0.9120736287617837 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1281.026, + "std": 15.647, + "p50": 1282.338, + "p95": 1290.531, + "p99": 1296.884, + "preprocess_ms": 14.846, + "inference_ms": 1263.522, + "postprocess_ms": 2.658 + } + }, + "throughput": { + "fps_mean": 0.78, + "fps_p50": 0.78 + }, + "model_stats": { + "params_millions": 33.69, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T22:46:58.295133Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rfdetr-n__onnx__cpu__rpi5__20260627T214203728445Z.json b/submissions/rfdetr-n__onnx__cpu__rpi5__20260627T214203728445Z.json new file mode 100644 index 0000000..872a75a --- /dev/null +++ b/submissions/rfdetr-n__onnx__cpu__rpi5__20260627T214203728445Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-n-onnx-cpu-rpi5-20260627T214203Z", + "created_at": "2026-06-27T21:42:03.728445Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-n", + "name": "rf-detr-n", + "family": "rfdetr", + "variant": "n", + "source": "libreyolo", + "weights": "LibreRFDETRn.pt", + "input_size": 384 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 384, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5138715096714886, + "mAP_50": 0.6990689274876704, + "mAP_75": 0.5545178686601185, + "mAP_small": 0.2775638537589648, + "mAP_medium": 0.571678381229741, + "mAP_large": 0.7184524786335472, + "AR1": 0.39943654428373826, + "AR10": 0.630384871636969, + "AR100": 0.6724047346083549, + "AR_small": 0.3890767161821573, + "AR_medium": 0.7463711052931866, + "AR_large": 0.8836870549836006 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 409.243, + "std": 64.229, + "p50": 405.173, + "p95": 410.455, + "p99": 411.233, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 2.44, + "fps_p50": 2.47 + }, + "model_stats": { + "params_millions": 26.86, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 97.8 + }, + "metadata": { + "benchmark_date": "2026-06-27T21:42:03.728445Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rfdetr-n__pytorch__cpu__rpi5__20260627T222407937201Z.json b/submissions/rfdetr-n__pytorch__cpu__rpi5__20260627T222407937201Z.json new file mode 100644 index 0000000..b07d788 --- /dev/null +++ b/submissions/rfdetr-n__pytorch__cpu__rpi5__20260627T222407937201Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-n-pytorch-cpu-rpi5-20260627T222407Z", + "created_at": "2026-06-27T22:24:07.937201Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-n", + "name": "rf-detr-n", + "family": "rfdetr", + "variant": "n", + "source": "libreyolo", + "weights": "LibreRFDETRn.pt", + "input_size": 384 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 384, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5138105965112096, + "mAP_50": 0.6990724692418401, + "mAP_75": 0.5546099900861416, + "mAP_small": 0.2775598218124653, + "mAP_medium": 0.5716791277445037, + "mAP_large": 0.7181729926162229, + "AR1": 0.3993445310338302, + "AR10": 0.6302980005689852, + "AR100": 0.6725590862593693, + "AR_small": 0.38905253248055427, + "AR_medium": 0.7468215621290925, + "AR_large": 0.8836471237540516 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 479.574, + "std": 10.559, + "p50": 481.213, + "p95": 485.016, + "p99": 486.741, + "preprocess_ms": 8.109, + "inference_ms": 468.973, + "postprocess_ms": 2.492 + } + }, + "throughput": { + "fps_mean": 2.09, + "fps_p50": 2.08 + }, + "model_stats": { + "params_millions": 30.47, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 63.2 + }, + "metadata": { + "benchmark_date": "2026-06-27T22:24:07.937201Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rfdetr-s__onnx__cpu__rpi5__20260627T214926399429Z.json b/submissions/rfdetr-s__onnx__cpu__rpi5__20260627T214926399429Z.json new file mode 100644 index 0000000..1345a80 --- /dev/null +++ b/submissions/rfdetr-s__onnx__cpu__rpi5__20260627T214926399429Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-s-onnx-cpu-rpi5-20260627T214926Z", + "created_at": "2026-06-27T21:49:26.399429Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-s", + "name": "rf-detr-s", + "family": "rfdetr", + "variant": "s", + "source": "libreyolo", + "weights": "LibreRFDETRs.pt", + "input_size": 512 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 512, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5514059726096994, + "mAP_50": 0.7347662100308575, + "mAP_75": 0.5942836611341867, + "mAP_small": 0.35185487254383585, + "mAP_medium": 0.6087277147212882, + "mAP_large": 0.7391800435640334, + "AR1": 0.41629806425979743, + "AR10": 0.681382009005631, + "AR100": 0.7248709221123524, + "AR_small": 0.49441840322082514, + "AR_medium": 0.7941331959800668, + "AR_large": 0.8966372240704275 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 850.101, + "std": 1.262, + "p50": 849.938, + "p95": 852.051, + "p99": 854.343, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.18, + "fps_p50": 1.18 + }, + "model_stats": { + "params_millions": 28.51, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T21:49:26.399429Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rfdetr-s__pytorch__cpu__rpi5__20260627T223330205175Z.json b/submissions/rfdetr-s__pytorch__cpu__rpi5__20260627T223330205175Z.json new file mode 100644 index 0000000..4f2d921 --- /dev/null +++ b/submissions/rfdetr-s__pytorch__cpu__rpi5__20260627T223330205175Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rfdetr-s-pytorch-cpu-rpi5-20260627T223330Z", + "created_at": "2026-06-27T22:33:30.205175Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rfdetr-s", + "name": "rf-detr-s", + "family": "rfdetr", + "variant": "s", + "source": "libreyolo", + "weights": "LibreRFDETRs.pt", + "input_size": 512 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 512, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5512802743036784, + "mAP_50": 0.7347705308004343, + "mAP_75": 0.5940212137659786, + "mAP_small": 0.3517797197118371, + "mAP_medium": 0.6084994790061564, + "mAP_large": 0.7391852638939048, + "AR1": 0.41656386423356995, + "AR10": 0.6812778522573595, + "AR100": 0.7239444499085594, + "AR_small": 0.49427310407552594, + "AR_medium": 0.7925295775590141, + "AR_large": 0.8963737384572662 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 959.447, + "std": 13.842, + "p50": 960.131, + "p95": 969.088, + "p99": 971.164, + "preprocess_ms": 12.312, + "inference_ms": 944.538, + "postprocess_ms": 2.597 + } + }, + "throughput": { + "fps_mean": 1.04, + "fps_p50": 1.04 + }, + "model_stats": { + "params_millions": 32.11, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T22:33:30.205175Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetr-l__onnx__cpu__rpi5__20260628T060222619689Z.json b/submissions/rtdetr-l__onnx__cpu__rpi5__20260628T060222619689Z.json new file mode 100644 index 0000000..49731cd --- /dev/null +++ b/submissions/rtdetr-l__onnx__cpu__rpi5__20260628T060222619689Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-l-onnx-cpu-rpi5-20260628T060222Z", + "created_at": "2026-06-28T06:02:22.619689Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-l", + "name": "rt-detr-l", + "family": "rtdetr", + "variant": "l", + "source": "libreyolo", + "weights": "LibreRTDETRl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5577690093694818, + "mAP_50": 0.7370566478233471, + "mAP_75": 0.611902700368415, + "mAP_small": 0.39158012790306695, + "mAP_medium": 0.5925269295111903, + "mAP_large": 0.7300745279615439, + "AR1": 0.4180303327747421, + "AR10": 0.6967882828105793, + "AR100": 0.7420949806172361, + "AR_small": 0.5930121962336482, + "AR_medium": 0.7707117941164834, + "AR_large": 0.8881804848784471 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1460.995, + "std": 11.908, + "p50": 1457.491, + "p95": 1478.655, + "p99": 1514.473, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.68, + "fps_p50": 0.69 + }, + "model_stats": { + "params_millions": 32.84, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T06:02:22.619689Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetr-l__pytorch__cpu__rpi5__20260629T030515993467Z.json b/submissions/rtdetr-l__pytorch__cpu__rpi5__20260629T030515993467Z.json new file mode 100644 index 0000000..483b425 --- /dev/null +++ b/submissions/rtdetr-l__pytorch__cpu__rpi5__20260629T030515993467Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-l-pytorch-cpu-rpi5-20260629T030515Z", + "created_at": "2026-06-29T03:05:15.993467Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-l", + "name": "rt-detr-l", + "family": "rtdetr", + "variant": "l", + "source": "libreyolo", + "weights": "LibreRTDETRl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5575479992934428, + "mAP_50": 0.7370539207052157, + "mAP_75": 0.6119027301701989, + "mAP_small": 0.3915772453802141, + "mAP_medium": 0.5924982987913687, + "mAP_large": 0.729726871134273, + "AR1": 0.4178675326119419, + "AR10": 0.6965772852777472, + "AR100": 0.7418839830844038, + "AR_small": 0.5928839911054432, + "AR_medium": 0.7705208239024869, + "AR_large": 0.8880044454648446 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2345.193, + "std": 16.061, + "p50": 2347.699, + "p95": 2356.526, + "p99": 2362.263, + "preprocess_ms": 3.279, + "inference_ms": 2341.163, + "postprocess_ms": 0.75 + } + }, + "throughput": { + "fps_mean": 0.43, + "fps_p50": 0.43 + }, + "model_stats": { + "params_millions": 32.93, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 78.5 + }, + "metadata": { + "benchmark_date": "2026-06-29T03:05:15.993467Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetr-r101__onnx__cpu__rpi5__20260628T054948877275Z.json b/submissions/rtdetr-r101__onnx__cpu__rpi5__20260628T054948877275Z.json new file mode 100644 index 0000000..3a98299 --- /dev/null +++ b/submissions/rtdetr-r101__onnx__cpu__rpi5__20260628T054948877275Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r101-onnx-cpu-rpi5-20260628T054948Z", + "created_at": "2026-06-28T05:49:48.877275Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r101", + "name": "rt-detr-r101", + "family": "rtdetr", + "variant": "r101", + "source": "libreyolo", + "weights": "LibreRTDETRr101.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5674425526984037, + "mAP_50": 0.7395636718774186, + "mAP_75": 0.6178049152885773, + "mAP_small": 0.3919726645932447, + "mAP_medium": 0.6204339342414583, + "mAP_large": 0.7171567835394471, + "AR1": 0.4252468321189817, + "AR10": 0.707393435689367, + "AR100": 0.7551884403445683, + "AR_small": 0.5989601353874838, + "AR_medium": 0.7916745721758722, + "AR_large": 0.8760709999518368 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3019.38, + "std": 13.842, + "p50": 3016.969, + "p95": 3035.793, + "p99": 3091.323, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.33, + "fps_p50": 0.33 + }, + "model_stats": { + "params_millions": 76.52, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.2 + }, + "metadata": { + "benchmark_date": "2026-06-28T05:49:48.877275Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetr-r18__onnx__cpu__rpi5__20260628T044752276336Z.json b/submissions/rtdetr-r18__onnx__cpu__rpi5__20260628T044752276336Z.json new file mode 100644 index 0000000..fbd7c99 --- /dev/null +++ b/submissions/rtdetr-r18__onnx__cpu__rpi5__20260628T044752276336Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r18-onnx-cpu-rpi5-20260628T044752Z", + "created_at": "2026-06-28T04:47:52.276336Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r18", + "name": "rt-detr-r18", + "family": "rtdetr", + "variant": "r18", + "source": "libreyolo", + "weights": "LibreRTDETRr18.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.49774256714404225, + "mAP_50": 0.6634992177231226, + "mAP_75": 0.5400484427487215, + "mAP_small": 0.3261582121122263, + "mAP_medium": 0.548562025815511, + "mAP_large": 0.6408920088998157, + "AR1": 0.39604668829512685, + "AR10": 0.664007747539048, + "AR100": 0.7230544546382387, + "AR_small": 0.5380604638906232, + "AR_medium": 0.7700151640261632, + "AR_large": 0.8588714644727942 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 786.553, + "std": 8.765, + "p50": 782.758, + "p95": 803.468, + "p99": 819.171, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.27, + "fps_p50": 1.28 + }, + "model_stats": { + "params_millions": 20.11, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 70.8 + }, + "metadata": { + "benchmark_date": "2026-06-28T04:47:52.276336Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetr-r18__pytorch__cpu__rpi5__20260629T002539420656Z.json b/submissions/rtdetr-r18__pytorch__cpu__rpi5__20260629T002539420656Z.json new file mode 100644 index 0000000..9975e6a --- /dev/null +++ b/submissions/rtdetr-r18__pytorch__cpu__rpi5__20260629T002539420656Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r18-pytorch-cpu-rpi5-20260629T002539Z", + "created_at": "2026-06-29T00:25:39.420656Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r18", + "name": "rt-detr-r18", + "family": "rtdetr", + "variant": "r18", + "source": "libreyolo", + "weights": "LibreRTDETRr18.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.4974385575772144, + "mAP_50": 0.6634990211631825, + "mAP_75": 0.5400385660072728, + "mAP_small": 0.3259147207054876, + "mAP_medium": 0.5483387545699251, + "mAP_large": 0.6406918404250833, + "AR1": 0.3957277587834025, + "AR10": 0.6636363747426361, + "AR100": 0.7226404916875274, + "AR_small": 0.5376757775735183, + "AR_medium": 0.7695527656896355, + "AR_large": 0.8587953609720331 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1245.405, + "std": 9.825, + "p50": 1247.454, + "p95": 1253.143, + "p99": 1260.762, + "preprocess_ms": 3.601, + "inference_ms": 1241.054, + "postprocess_ms": 0.749 + } + }, + "throughput": { + "fps_mean": 0.8, + "fps_p50": 0.8 + }, + "model_stats": { + "params_millions": 20.18, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 92.0 + }, + "metadata": { + "benchmark_date": "2026-06-29T00:25:39.420656Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetr-r34__onnx__cpu__rpi5__20260628T045733192435Z.json b/submissions/rtdetr-r34__onnx__cpu__rpi5__20260628T045733192435Z.json new file mode 100644 index 0000000..dfb717b --- /dev/null +++ b/submissions/rtdetr-r34__onnx__cpu__rpi5__20260628T045733192435Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r34-onnx-cpu-rpi5-20260628T045733Z", + "created_at": "2026-06-28T04:57:33.192435Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r34", + "name": "rt-detr-r34", + "family": "rtdetr", + "variant": "r34", + "source": "libreyolo", + "weights": "LibreRTDETRr34.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.522144868701534, + "mAP_50": 0.6962817889643671, + "mAP_75": 0.5673028292737368, + "mAP_small": 0.34963723196781965, + "mAP_medium": 0.5588247724308356, + "mAP_large": 0.6674805428302818, + "AR1": 0.409745659514483, + "AR10": 0.668504721728897, + "AR100": 0.7220680557653572, + "AR_small": 0.54067270344758, + "AR_medium": 0.7543034638396552, + "AR_large": 0.8734680809296462 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1118.681, + "std": 11.285, + "p50": 1114.813, + "p95": 1136.01, + "p99": 1169.664, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.89, + "fps_p50": 0.9 + }, + "model_stats": { + "params_millions": 31.34, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 70.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T04:57:33.192435Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetr-r34__pytorch__cpu__rpi5__20260629T021016897798Z.json b/submissions/rtdetr-r34__pytorch__cpu__rpi5__20260629T021016897798Z.json new file mode 100644 index 0000000..7262d12 --- /dev/null +++ b/submissions/rtdetr-r34__pytorch__cpu__rpi5__20260629T021016897798Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r34-pytorch-cpu-rpi5-20260629T021016Z", + "created_at": "2026-06-29T02:10:16.897798Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r34", + "name": "rt-detr-r34", + "family": "rtdetr", + "variant": "r34", + "source": "libreyolo", + "weights": "LibreRTDETRr34.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5220705477893457, + "mAP_50": 0.6956972783119411, + "mAP_75": 0.5672764157874404, + "mAP_small": 0.3496314468621894, + "mAP_medium": 0.558792600550835, + "mAP_large": 0.6673601126544302, + "AR1": 0.4096667574894731, + "AR10": 0.6683935712842712, + "AR100": 0.7219515967458123, + "AR_small": 0.5406543657059172, + "AR_medium": 0.7541078184762466, + "AR_large": 0.8733822205698133 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1669.041, + "std": 8.411, + "p50": 1668.949, + "p95": 1680.314, + "p99": 1686.699, + "preprocess_ms": 3.605, + "inference_ms": 1664.643, + "postprocess_ms": 0.793 + } + }, + "throughput": { + "fps_mean": 0.6, + "fps_p50": 0.6 + }, + "model_stats": { + "params_millions": 31.44, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 107.3 + }, + "metadata": { + "benchmark_date": "2026-06-29T02:10:16.897798Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetr-r50__onnx__cpu__rpi5__20260628T051244059755Z.json b/submissions/rtdetr-r50__onnx__cpu__rpi5__20260628T051244059755Z.json new file mode 100644 index 0000000..dcdf66c --- /dev/null +++ b/submissions/rtdetr-r50__onnx__cpu__rpi5__20260628T051244059755Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r50-onnx-cpu-rpi5-20260628T051244Z", + "created_at": "2026-06-28T05:12:44.059755Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r50", + "name": "rt-detr-r50", + "family": "rtdetr", + "variant": "r50", + "source": "libreyolo", + "weights": "LibreRTDETRr50.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5586474163642929, + "mAP_50": 0.7304232780632504, + "mAP_75": 0.6166275682858972, + "mAP_small": 0.38254731498765926, + "mAP_medium": 0.6108448756694206, + "mAP_large": 0.7033294309172166, + "AR1": 0.42091531825728223, + "AR10": 0.6986371536602564, + "AR100": 0.7468353502304153, + "AR_small": 0.5821806901578028, + "AR_medium": 0.7880349995633875, + "AR_large": 0.8820373961923116 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1772.895, + "std": 9.627, + "p50": 1770.223, + "p95": 1788.033, + "p99": 1818.512, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.56, + "fps_p50": 0.56 + }, + "model_stats": { + "params_millions": 42.81, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T05:12:44.059755Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetr-r50__pytorch__cpu__rpi5__20260629T044842968753Z.json b/submissions/rtdetr-r50__pytorch__cpu__rpi5__20260629T044842968753Z.json new file mode 100644 index 0000000..6419faa --- /dev/null +++ b/submissions/rtdetr-r50__pytorch__cpu__rpi5__20260629T044842968753Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r50-pytorch-cpu-rpi5-20260629T044842Z", + "created_at": "2026-06-29T04:48:42.968753Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r50", + "name": "rt-detr-r50", + "family": "rtdetr", + "variant": "r50", + "source": "libreyolo", + "weights": "LibreRTDETRr50.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5586219562510433, + "mAP_50": 0.7304228810571854, + "mAP_75": 0.6166276372667306, + "mAP_small": 0.3825271464442928, + "mAP_medium": 0.6108065605086318, + "mAP_large": 0.7033225957178774, + "AR1": 0.42085895754874764, + "AR10": 0.6985677228069086, + "AR100": 0.7467315408224947, + "AR_small": 0.5821257451028578, + "AR_medium": 0.7878582128853165, + "AR_large": 0.8820373961923116 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3317.573, + "std": 18.91, + "p50": 3319.276, + "p95": 3335.005, + "p99": 3337.755, + "preprocess_ms": 3.303, + "inference_ms": 3313.502, + "postprocess_ms": 0.768 + } + }, + "throughput": { + "fps_mean": 0.3, + "fps_p50": 0.3 + }, + "model_stats": { + "params_millions": 42.89, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 70.3 + }, + "metadata": { + "benchmark_date": "2026-06-29T04:48:42.968753Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetr-r50m__onnx__cpu__rpi5__20260628T052410847249Z.json b/submissions/rtdetr-r50m__onnx__cpu__rpi5__20260628T052410847249Z.json new file mode 100644 index 0000000..6a60e3b --- /dev/null +++ b/submissions/rtdetr-r50m__onnx__cpu__rpi5__20260628T052410847249Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r50m-onnx-cpu-rpi5-20260628T052410Z", + "created_at": "2026-06-28T05:24:10.847249Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r50m", + "name": "rt-detr-r50m", + "family": "rtdetr", + "variant": "r50m", + "source": "libreyolo", + "weights": "LibreRTDETRr50m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5381186274201499, + "mAP_50": 0.7122493094497363, + "mAP_75": 0.5848035966081955, + "mAP_small": 0.3763542586544715, + "mAP_medium": 0.5938425477922029, + "mAP_large": 0.6904993799904184, + "AR1": 0.417359729259557, + "AR10": 0.6780930703428721, + "AR100": 0.7300127669264833, + "AR_small": 0.5776721345625127, + "AR_medium": 0.7669397771663522, + "AR_large": 0.8806901143915012 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1329.009, + "std": 10.193, + "p50": 1326.111, + "p95": 1343.352, + "p99": 1371.406, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.75, + "fps_p50": 0.75 + }, + "model_stats": { + "params_millions": 33.13, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.2 + }, + "metadata": { + "benchmark_date": "2026-06-28T05:24:10.847249Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetr-r50m__pytorch__cpu__rpi5__20260629T042008262043Z.json b/submissions/rtdetr-r50m__pytorch__cpu__rpi5__20260629T042008262043Z.json new file mode 100644 index 0000000..2c69b55 --- /dev/null +++ b/submissions/rtdetr-r50m__pytorch__cpu__rpi5__20260629T042008262043Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-r50m-pytorch-cpu-rpi5-20260629T042008Z", + "created_at": "2026-06-29T04:20:08.262043Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-r50m", + "name": "rt-detr-r50m", + "family": "rtdetr", + "variant": "r50m", + "source": "libreyolo", + "weights": "LibreRTDETRr50m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5380213245334988, + "mAP_50": 0.7122448267576749, + "mAP_75": 0.5848047008866352, + "mAP_small": 0.37633171209032285, + "mAP_medium": 0.5939333438455466, + "mAP_large": 0.6899231369387736, + "AR1": 0.41720418627313166, + "AR10": 0.677884852236364, + "AR100": 0.7298682016629048, + "AR_small": 0.5776550405454185, + "AR_medium": 0.766607070399435, + "AR_large": 0.8797720658454787 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2801.975, + "std": 20.116, + "p50": 2803.997, + "p95": 2816.569, + "p99": 2829.904, + "preprocess_ms": 3.287, + "inference_ms": 2797.93, + "postprocess_ms": 0.758 + } + }, + "throughput": { + "fps_mean": 0.36, + "fps_p50": 0.36 + }, + "model_stats": { + "params_millions": 36.59, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 68.6 + }, + "metadata": { + "benchmark_date": "2026-06-29T04:20:08.262043Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetr-x__onnx__cpu__rpi5__20260628T062611065238Z.json b/submissions/rtdetr-x__onnx__cpu__rpi5__20260628T062611065238Z.json new file mode 100644 index 0000000..3305b6e --- /dev/null +++ b/submissions/rtdetr-x__onnx__cpu__rpi5__20260628T062611065238Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetr-x-onnx-cpu-rpi5-20260628T062611Z", + "created_at": "2026-06-28T06:26:11.065238Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetr-x", + "name": "rt-detr-x", + "family": "rtdetr", + "variant": "x", + "source": "libreyolo", + "weights": "LibreRTDETRx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5794524359370877, + "mAP_50": 0.760353887037982, + "mAP_75": 0.6333719411593034, + "mAP_small": 0.3976324086492176, + "mAP_medium": 0.6439313968855763, + "mAP_large": 0.7254609070440775, + "AR1": 0.432528983655378, + "AR10": 0.7126273184401586, + "AR100": 0.7616617701826264, + "AR_small": 0.6011353676420481, + "AR_medium": 0.805626655986392, + "AR_large": 0.8834554777655631 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2801.481, + "std": 14.959, + "p50": 2798.131, + "p95": 2828.051, + "p99": 2862.928, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.36, + "fps_p50": 0.36 + }, + "model_stats": { + "params_millions": 67.33, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.2 + }, + "metadata": { + "benchmark_date": "2026-06-28T06:26:11.065238Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv2-r101__onnx__cpu__rpi5__20260628T073520903988Z.json b/submissions/rtdetrv2-r101__onnx__cpu__rpi5__20260628T073520903988Z.json new file mode 100644 index 0000000..80d1493 --- /dev/null +++ b/submissions/rtdetrv2-r101__onnx__cpu__rpi5__20260628T073520903988Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r101-onnx-cpu-rpi5-20260628T073520Z", + "created_at": "2026-06-28T07:35:20.903988Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r101", + "name": "rt-detrv2-r101", + "family": "rtdetrv2", + "variant": "r101", + "source": "libreyolo", + "weights": "LibreRTDETRv2r101.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5674425531042802, + "mAP_50": 0.7395636718774186, + "mAP_75": 0.6178049152885773, + "mAP_small": 0.3919726676626625, + "mAP_medium": 0.6204339342414583, + "mAP_large": 0.7171567835394471, + "AR1": 0.4252468321189817, + "AR10": 0.707393435689367, + "AR100": 0.7551884403445683, + "AR_small": 0.5989601353874838, + "AR_medium": 0.7916745721758722, + "AR_large": 0.8760709999518368 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3042.678, + "std": 13.364, + "p50": 3040.4, + "p95": 3058.157, + "p99": 3112.795, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.33, + "fps_p50": 0.33 + }, + "model_stats": { + "params_millions": 76.56, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T07:35:20.903988Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv2-r18__onnx__cpu__rpi5__20260628T063306884082Z.json b/submissions/rtdetrv2-r18__onnx__cpu__rpi5__20260628T063306884082Z.json new file mode 100644 index 0000000..6b833b2 --- /dev/null +++ b/submissions/rtdetrv2-r18__onnx__cpu__rpi5__20260628T063306884082Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r18-onnx-cpu-rpi5-20260628T063306Z", + "created_at": "2026-06-28T06:33:06.884082Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r18", + "name": "rt-detrv2-r18", + "family": "rtdetrv2", + "variant": "r18", + "source": "libreyolo", + "weights": "LibreRTDETRv2r18.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5075814715671121, + "mAP_50": 0.6738984732532675, + "mAP_75": 0.5495938940196975, + "mAP_small": 0.33501169940322056, + "mAP_medium": 0.5391037819664849, + "mAP_large": 0.6640383390452526, + "AR1": 0.3949345357955365, + "AR10": 0.6750040230328294, + "AR100": 0.7279190894521826, + "AR_small": 0.5576572291262796, + "AR_medium": 0.7556398310786503, + "AR_large": 0.8596387462926546 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 791.233, + "std": 9.419, + "p50": 787.849, + "p95": 807.113, + "p99": 833.335, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.26, + "fps_p50": 1.27 + }, + "model_stats": { + "params_millions": 20.15, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 70.8 + }, + "metadata": { + "benchmark_date": "2026-06-28T06:33:06.884082Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv2-r18__pytorch__cpu__rpi5__20260629T003710952196Z.json b/submissions/rtdetrv2-r18__pytorch__cpu__rpi5__20260629T003710952196Z.json new file mode 100644 index 0000000..b5eb769 --- /dev/null +++ b/submissions/rtdetrv2-r18__pytorch__cpu__rpi5__20260629T003710952196Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r18-pytorch-cpu-rpi5-20260629T003710Z", + "created_at": "2026-06-29T00:37:10.952196Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r18", + "name": "rt-detrv2-r18", + "family": "rtdetrv2", + "variant": "r18", + "source": "libreyolo", + "weights": "LibreRTDETRv2r18.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5075295513304796, + "mAP_50": 0.6738984732532675, + "mAP_75": 0.5495400880228422, + "mAP_small": 0.33493026397740544, + "mAP_medium": 0.5390480085426296, + "mAP_large": 0.6640351147172935, + "AR1": 0.39492685884175177, + "AR10": 0.6748887578268217, + "AR100": 0.7277599874225195, + "AR_small": 0.5575095523214917, + "AR_medium": 0.7554613421447167, + "AR_large": 0.8595988150631054 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1244.505, + "std": 9.581, + "p50": 1246.635, + "p95": 1254.867, + "p99": 1257.566, + "preprocess_ms": 3.581, + "inference_ms": 1240.157, + "postprocess_ms": 0.766 + } + }, + "throughput": { + "fps_mean": 0.8, + "fps_p50": 0.8 + }, + "model_stats": { + "params_millions": 20.18, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 102.0 + }, + "metadata": { + "benchmark_date": "2026-06-29T00:37:10.952196Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv2-r34__onnx__cpu__rpi5__20260628T064249407646Z.json b/submissions/rtdetrv2-r34__onnx__cpu__rpi5__20260628T064249407646Z.json new file mode 100644 index 0000000..9504c94 --- /dev/null +++ b/submissions/rtdetrv2-r34__onnx__cpu__rpi5__20260628T064249407646Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r34-onnx-cpu-rpi5-20260628T064249Z", + "created_at": "2026-06-28T06:42:49.407646Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r34", + "name": "rt-detrv2-r34", + "family": "rtdetrv2", + "variant": "r34", + "source": "libreyolo", + "weights": "LibreRTDETRv2r34.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5318119413094168, + "mAP_50": 0.7018691549392625, + "mAP_75": 0.5781553635714163, + "mAP_small": 0.35549933093655, + "mAP_medium": 0.5762248642955541, + "mAP_large": 0.6739853756707758, + "AR1": 0.4055715211295446, + "AR10": 0.6645740019115106, + "AR100": 0.7204275773514845, + "AR_small": 0.5658065214832655, + "AR_medium": 0.7545120243239197, + "AR_large": 0.8550972711754022 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1122.153, + "std": 11.941, + "p50": 1117.447, + "p95": 1143.4, + "p99": 1166.716, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.89, + "fps_p50": 0.89 + }, + "model_stats": { + "params_millions": 31.38, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 70.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T06:42:49.407646Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv2-r34__pytorch__cpu__rpi5__20260629T022501471763Z.json b/submissions/rtdetrv2-r34__pytorch__cpu__rpi5__20260629T022501471763Z.json new file mode 100644 index 0000000..26e1ada --- /dev/null +++ b/submissions/rtdetrv2-r34__pytorch__cpu__rpi5__20260629T022501471763Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r34-pytorch-cpu-rpi5-20260629T022501Z", + "created_at": "2026-06-29T02:25:01.471763Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r34", + "name": "rt-detrv2-r34", + "family": "rtdetrv2", + "variant": "r34", + "source": "libreyolo", + "weights": "LibreRTDETRv2r34.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5317866682163993, + "mAP_50": 0.7018630480731898, + "mAP_75": 0.5781560934160433, + "mAP_small": 0.3554951561526654, + "mAP_medium": 0.5762138931529338, + "mAP_large": 0.673940756314738, + "AR1": 0.40553267109069463, + "AR10": 0.6645274749188759, + "AR100": 0.7202744486197602, + "AR_small": 0.5656720704641991, + "AR_medium": 0.7545120243239197, + "AR_large": 0.8549994238172808 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1664.933, + "std": 13.327, + "p50": 1665.81, + "p95": 1674.354, + "p99": 1678.704, + "preprocess_ms": 3.561, + "inference_ms": 1660.622, + "postprocess_ms": 0.75 + } + }, + "throughput": { + "fps_mean": 0.6, + "fps_p50": 0.6 + }, + "model_stats": { + "params_millions": 31.44, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 96.7 + }, + "metadata": { + "benchmark_date": "2026-06-29T02:25:01.471763Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv2-r50__onnx__cpu__rpi5__20260628T065806042017Z.json b/submissions/rtdetrv2-r50__onnx__cpu__rpi5__20260628T065806042017Z.json new file mode 100644 index 0000000..ffcada5 --- /dev/null +++ b/submissions/rtdetrv2-r50__onnx__cpu__rpi5__20260628T065806042017Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r50-onnx-cpu-rpi5-20260628T065806Z", + "created_at": "2026-06-28T06:58:06.042017Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r50", + "name": "rt-detrv2-r50", + "family": "rtdetrv2", + "variant": "r50", + "source": "libreyolo", + "weights": "LibreRTDETRv2r50.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5569738100824244, + "mAP_50": 0.728395197258972, + "mAP_75": 0.6056225695539527, + "mAP_small": 0.3896959314468154, + "mAP_medium": 0.6038698896109826, + "mAP_large": 0.718780877749079, + "AR1": 0.41980268254801273, + "AR10": 0.6880983552288026, + "AR100": 0.7403090572956712, + "AR_small": 0.5962663669882283, + "AR_medium": 0.778614386279377, + "AR_large": 0.8723433753727128 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1785.296, + "std": 12.194, + "p50": 1782.492, + "p95": 1801.697, + "p99": 1845.503, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.56, + "fps_p50": 0.56 + }, + "model_stats": { + "params_millions": 42.85, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T06:58:06.042017Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv2-r50__pytorch__cpu__rpi5__20260629T051812080119Z.json b/submissions/rtdetrv2-r50__pytorch__cpu__rpi5__20260629T051812080119Z.json new file mode 100644 index 0000000..4330e63 --- /dev/null +++ b/submissions/rtdetrv2-r50__pytorch__cpu__rpi5__20260629T051812080119Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r50-pytorch-cpu-rpi5-20260629T051812Z", + "created_at": "2026-06-29T05:18:12.080119Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r50", + "name": "rt-detrv2-r50", + "family": "rtdetrv2", + "variant": "r50", + "source": "libreyolo", + "weights": "LibreRTDETRv2r50.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5569306661618519, + "mAP_50": 0.7283953697118257, + "mAP_75": 0.6056222550072237, + "mAP_small": 0.38962732253938737, + "mAP_medium": 0.6038589595446148, + "mAP_large": 0.7187812766701257, + "AR1": 0.41980268254801273, + "AR10": 0.6879503522170225, + "AR100": 0.7401610542838911, + "AR_small": 0.5961381618600232, + "AR_medium": 0.7784381644748659, + "AR_large": 0.8723433753727128 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3317.275, + "std": 19.91, + "p50": 3321.673, + "p95": 3333.853, + "p99": 3337.86, + "preprocess_ms": 3.294, + "inference_ms": 3313.204, + "postprocess_ms": 0.777 + } + }, + "throughput": { + "fps_mean": 0.3, + "fps_p50": 0.3 + }, + "model_stats": { + "params_millions": 42.89, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 94.3 + }, + "metadata": { + "benchmark_date": "2026-06-29T05:18:12.080119Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv2-r50m__onnx__cpu__rpi5__20260628T070931410491Z.json b/submissions/rtdetrv2-r50m__onnx__cpu__rpi5__20260628T070931410491Z.json new file mode 100644 index 0000000..a2964e2 --- /dev/null +++ b/submissions/rtdetrv2-r50m__onnx__cpu__rpi5__20260628T070931410491Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r50m-onnx-cpu-rpi5-20260628T070931Z", + "created_at": "2026-06-28T07:09:31.410491Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r50m", + "name": "rt-detrv2-r50m", + "family": "rtdetrv2", + "variant": "r50m", + "source": "libreyolo", + "weights": "LibreRTDETRv2r50m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5476645770436055, + "mAP_50": 0.7176598191789246, + "mAP_75": 0.5991054847047382, + "mAP_small": 0.37988141287131805, + "mAP_medium": 0.6013683390782228, + "mAP_large": 0.6934279367189854, + "AR1": 0.41156813370207107, + "AR10": 0.6841501952835427, + "AR100": 0.7383814121531431, + "AR_small": 0.5873760190188806, + "AR_medium": 0.7797841209975578, + "AR_large": 0.8766520295342554 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1326.47, + "std": 9.967, + "p50": 1323.441, + "p95": 1340.463, + "p99": 1364.549, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.75, + "fps_p50": 0.76 + }, + "model_stats": { + "params_millions": 33.17, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 66.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T07:09:31.410491Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv2-r50m__pytorch__cpu__rpi5__20260629T035550107846Z.json b/submissions/rtdetrv2-r50m__pytorch__cpu__rpi5__20260629T035550107846Z.json new file mode 100644 index 0000000..c0eb5a3 --- /dev/null +++ b/submissions/rtdetrv2-r50m__pytorch__cpu__rpi5__20260629T035550107846Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv2-r50m-pytorch-cpu-rpi5-20260629T035550Z", + "created_at": "2026-06-29T03:55:50.107846Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv2-r50m", + "name": "rt-detrv2-r50m", + "family": "rtdetrv2", + "variant": "r50m", + "source": "libreyolo", + "weights": "LibreRTDETRv2r50m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5475701026232949, + "mAP_50": 0.7176596185245316, + "mAP_75": 0.5991013990587835, + "mAP_small": 0.3789979568982936, + "mAP_medium": 0.6013682226961227, + "mAP_large": 0.69338885373, + "AR1": 0.4115040311379684, + "AR10": 0.6839278724348254, + "AR100": 0.7381671290997697, + "AR_small": 0.586667067152034, + "AR_medium": 0.7797841209975578, + "AR_large": 0.8765759260334942 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2810.255, + "std": 16.144, + "p50": 2811.264, + "p95": 2827.445, + "p99": 2831.547, + "preprocess_ms": 3.289, + "inference_ms": 2806.208, + "postprocess_ms": 0.758 + } + }, + "throughput": { + "fps_mean": 0.36, + "fps_p50": 0.36 + }, + "model_stats": { + "params_millions": 36.59, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 91.5 + }, + "metadata": { + "benchmark_date": "2026-06-29T03:55:50.107846Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv4-l__onnx__cpu__rpi5__20260628T080131290707Z.json b/submissions/rtdetrv4-l__onnx__cpu__rpi5__20260628T080131290707Z.json new file mode 100644 index 0000000..741c973 --- /dev/null +++ b/submissions/rtdetrv4-l__onnx__cpu__rpi5__20260628T080131290707Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-l-onnx-cpu-rpi5-20260628T080131Z", + "created_at": "2026-06-28T08:01:31.290707Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-l", + "name": "rt-detrv4-l", + "family": "rtdetrv4", + "variant": "l", + "source": "libreyolo", + "weights": "LibreRTDETRv4l.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5778216814272358, + "mAP_50": 0.7442366630340385, + "mAP_75": 0.6334281763674426, + "mAP_small": 0.4212308683801451, + "mAP_medium": 0.6364238083168491, + "mAP_large": 0.7449359377371503, + "AR1": 0.4357236915112565, + "AR10": 0.7176614125783658, + "AR100": 0.7650717295248021, + "AR_small": 0.6090221864066407, + "AR_medium": 0.8011127339685521, + "AR_large": 0.9102406922726382 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1427.403, + "std": 28.953, + "p50": 1426.034, + "p95": 1431.835, + "p99": 1476.507, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.7, + "fps_p50": 0.7 + }, + "model_stats": { + "params_millions": 31.19, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:01:31.290707Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv4-l__pytorch__cpu__rpi5__20260629T024444005586Z.json b/submissions/rtdetrv4-l__pytorch__cpu__rpi5__20260629T024444005586Z.json new file mode 100644 index 0000000..0dc2100 --- /dev/null +++ b/submissions/rtdetrv4-l__pytorch__cpu__rpi5__20260629T024444005586Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-l-pytorch-cpu-rpi5-20260629T024444Z", + "created_at": "2026-06-29T02:44:44.005586Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-l", + "name": "rt-detrv4-l", + "family": "rtdetrv4", + "variant": "l", + "source": "libreyolo", + "weights": "LibreRTDETRv4l.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5778213004784958, + "mAP_50": 0.7442362572121634, + "mAP_75": 0.6334282910545552, + "mAP_small": 0.42123051715546067, + "mAP_medium": 0.6364248441671155, + "mAP_large": 0.7449359377371503, + "AR1": 0.4357236915112565, + "AR10": 0.7176614125783658, + "AR100": 0.765070530225193, + "AR_small": 0.6090186415643862, + "AR_medium": 0.8011127339685521, + "AR_large": 0.9102406922726382 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2303.768, + "std": 9.83, + "p50": 2303.617, + "p95": 2315.565, + "p99": 2330.775, + "preprocess_ms": 7.396, + "inference_ms": 2295.617, + "postprocess_ms": 0.755 + } + }, + "throughput": { + "fps_mean": 0.43, + "fps_p50": 0.43 + }, + "model_stats": { + "params_millions": 31.24, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 132.5 + }, + "metadata": { + "benchmark_date": "2026-06-29T02:44:44.005586Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv4-m__onnx__cpu__rpi5__20260628T074910221350Z.json b/submissions/rtdetrv4-m__onnx__cpu__rpi5__20260628T074910221350Z.json new file mode 100644 index 0000000..7cea898 --- /dev/null +++ b/submissions/rtdetrv4-m__onnx__cpu__rpi5__20260628T074910221350Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-m-onnx-cpu-rpi5-20260628T074910Z", + "created_at": "2026-06-28T07:49:10.221350Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-m", + "name": "rt-detrv4-m", + "family": "rtdetrv4", + "variant": "m", + "source": "libreyolo", + "weights": "LibreRTDETRv4m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5651023536810272, + "mAP_50": 0.7295076922166904, + "mAP_75": 0.6213825312048349, + "mAP_small": 0.3946394777715095, + "mAP_medium": 0.6083528341161982, + "mAP_large": 0.7333848441893696, + "AR1": 0.4209735862228707, + "AR10": 0.7026556941131794, + "AR100": 0.7485848272493711, + "AR_small": 0.5869271822546797, + "AR_medium": 0.787922916284933, + "AR_large": 0.8845628241437012 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1006.491, + "std": 6.366, + "p50": 1006.394, + "p95": 1010.511, + "p99": 1013.926, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.99, + "fps_p50": 0.99 + }, + "model_stats": { + "params_millions": 19.52, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T07:49:10.221350Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv4-m__pytorch__cpu__rpi5__20260629T000528347300Z.json b/submissions/rtdetrv4-m__pytorch__cpu__rpi5__20260629T000528347300Z.json new file mode 100644 index 0000000..4a65317 --- /dev/null +++ b/submissions/rtdetrv4-m__pytorch__cpu__rpi5__20260629T000528347300Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-m-pytorch-cpu-rpi5-20260629T000528Z", + "created_at": "2026-06-29T00:05:28.347300Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-m", + "name": "rt-detrv4-m", + "family": "rtdetrv4", + "variant": "m", + "source": "libreyolo", + "weights": "LibreRTDETRv4m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5650964727884, + "mAP_50": 0.729508246830964, + "mAP_75": 0.6213825976722152, + "mAP_small": 0.3946348695064786, + "mAP_medium": 0.6083527271023137, + "mAP_large": 0.7333839664388817, + "AR1": 0.4209735862228707, + "AR10": 0.7026438232679751, + "AR100": 0.748571757104558, + "AR_small": 0.5869023683588981, + "AR_medium": 0.7879191568864369, + "AR_large": 0.8845628241437012 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1588.449, + "std": 8.848, + "p50": 1588.688, + "p95": 1601.154, + "p99": 1606.572, + "preprocess_ms": 7.72, + "inference_ms": 1579.98, + "postprocess_ms": 0.749 + } + }, + "throughput": { + "fps_mean": 0.63, + "fps_p50": 0.63 + }, + "model_stats": { + "params_millions": 19.59, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 85.1 + }, + "metadata": { + "benchmark_date": "2026-06-29T00:05:28.347300Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv4-s__onnx__cpu__rpi5__20260628T074023398916Z.json b/submissions/rtdetrv4-s__onnx__cpu__rpi5__20260628T074023398916Z.json new file mode 100644 index 0000000..4a94c14 --- /dev/null +++ b/submissions/rtdetrv4-s__onnx__cpu__rpi5__20260628T074023398916Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-s-onnx-cpu-rpi5-20260628T074023Z", + "created_at": "2026-06-28T07:40:23.398916Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-s", + "name": "rt-detrv4-s", + "family": "rtdetrv4", + "variant": "s", + "source": "libreyolo", + "weights": "LibreRTDETRv4s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5282822600337147, + "mAP_50": 0.6994595490451228, + "mAP_75": 0.5726015461943361, + "mAP_small": 0.34987476785569926, + "mAP_medium": 0.5703623254772469, + "mAP_large": 0.6837499344307502, + "AR1": 0.408956514400302, + "AR10": 0.6752355179616181, + "AR100": 0.7297513713228271, + "AR_small": 0.5313628472015742, + "AR_medium": 0.7731283222221518, + "AR_large": 0.880861480555594 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 562.506, + "std": 4.006, + "p50": 562.616, + "p95": 566.024, + "p99": 568.437, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.78, + "fps_p50": 1.78 + }, + "model_stats": { + "params_millions": 10.29, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T07:40:23.398916Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv4-s__pytorch__cpu__rpi5__20260628T223931540013Z.json b/submissions/rtdetrv4-s__pytorch__cpu__rpi5__20260628T223931540013Z.json new file mode 100644 index 0000000..5e2b116 --- /dev/null +++ b/submissions/rtdetrv4-s__pytorch__cpu__rpi5__20260628T223931540013Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-s-pytorch-cpu-rpi5-20260628T223931Z", + "created_at": "2026-06-28T22:39:31.540013Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-s", + "name": "rt-detrv4-s", + "family": "rtdetrv4", + "variant": "s", + "source": "libreyolo", + "weights": "LibreRTDETRv4s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5282737967407499, + "mAP_50": 0.6994596509087028, + "mAP_75": 0.572601548852765, + "mAP_small": 0.34987337823602815, + "mAP_medium": 0.5703579990666772, + "mAP_large": 0.6837293185691692, + "AR1": 0.40892277620866907, + "AR10": 0.6752355179616181, + "AR100": 0.729750172023218, + "AR_small": 0.5313593023593197, + "AR_medium": 0.7731283222221518, + "AR_large": 0.880861480555594 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 879.882, + "std": 6.882, + "p50": 879.81, + "p95": 890.186, + "p99": 902.855, + "preprocess_ms": 7.831, + "inference_ms": 871.284, + "postprocess_ms": 0.767 + } + }, + "throughput": { + "fps_mean": 1.14, + "fps_p50": 1.14 + }, + "model_stats": { + "params_millions": 10.32, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 112.2 + }, + "metadata": { + "benchmark_date": "2026-06-28T22:39:31.540013Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/rtdetrv4-x__onnx__cpu__rpi5__20260628T082425591936Z.json b/submissions/rtdetrv4-x__onnx__cpu__rpi5__20260628T082425591936Z.json new file mode 100644 index 0000000..f3ab9ba --- /dev/null +++ b/submissions/rtdetrv4-x__onnx__cpu__rpi5__20260628T082425591936Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "rtdetrv4-x-onnx-cpu-rpi5-20260628T082425Z", + "created_at": "2026-06-28T08:24:25.591936Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "rtdetrv4-x", + "name": "rt-detrv4-x", + "family": "rtdetrv4", + "variant": "x", + "source": "libreyolo", + "weights": "LibreRTDETRv4x.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5999021600251853, + "mAP_50": 0.7715643159289338, + "mAP_75": 0.6587334327811359, + "mAP_small": 0.44167293302601235, + "mAP_medium": 0.6528721192223313, + "mAP_large": 0.7413916074464995, + "AR1": 0.4446619265526469, + "AR10": 0.7257632575706068, + "AR100": 0.775710265507648, + "AR_small": 0.6309413433178404, + "AR_medium": 0.8141752623679169, + "AR_large": 0.8955351216107611 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2685.676, + "std": 11.64, + "p50": 2685.113, + "p95": 2692.773, + "p99": 2750.138, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.37, + "fps_p50": 0.37 + }, + "model_stats": { + "params_millions": 62.6, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 65.6 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:24:25.591936Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolonas-l__pytorch__cuda__jetson_orin__20260628T094851464380Z.json b/submissions/yolonas-l__pytorch__cuda__jetson_orin__20260628T094851464380Z.json new file mode 100644 index 0000000..c3bb483 --- /dev/null +++ b/submissions/yolonas-l__pytorch__cuda__jetson_orin__20260628T094851464380Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolonas-l-pytorch-cuda-jetson_orin-20260628T094851Z", + "created_at": "2026-06-28T09:48:51.464380Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0.dev0", + "libreyolo_commit": "5f12ee96d5f3099eaede265187a5df70a5f70c08" + }, + "model": { + "id": "yolonas-l", + "name": "yolo-nas-l", + "family": "yolonas", + "variant": "l", + "source": "libreyolo", + "weights": "LibreYOLONASl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Orin (nvgpu)", + "gpu_memory_gb": 0.0, + "driver_version": "595.78", + "cuda_version": "13.0", + "cpu": "Unknown", + "cpu_cores": 6, + "ram_gb": 7, + "id": "jetson_orin" + }, + "software": { + "python": "3.12.3", + "torch": "2.12.0+cu130", + "libreyolo": "1.2.0.dev0", + "libreyolo_commit": "5f12ee96d5f3099eaede265187a5df70a5f70c08", + "onnxruntime": "1.24.0" + }, + "accuracy": { + "mAP_50_95": 0.5631728310107038, + "mAP_50": 0.7326899962305481, + "mAP_75": 0.6197943523935017, + "mAP_small": 0.36144187071523554, + "mAP_medium": 0.6306335421543419, + "mAP_large": 0.6962833299114103, + "AR1": 0.41836905497431415, + "AR10": 0.6600220531833608, + "AR100": 0.6918215543277225, + "AR_small": 0.47509709353249796, + "AR_medium": 0.7430166700634716, + "AR_large": 0.8318025907979362 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 200.166, + "std": 1.644, + "p50": 199.994, + "p95": 202.637, + "p99": 205.456, + "preprocess_ms": 13.381, + "inference_ms": 180.523, + "postprocess_ms": 6.262 + } + }, + "throughput": { + "fps_mean": 5.0, + "fps_p50": 5.0 + }, + "model_stats": { + "params_millions": 66.98, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 469.9, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T09:48:51.464380Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0.dev0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cuda", + "device": "gpu" + } +} \ No newline at end of file diff --git a/submissions/yolonas-l_hailo_int8.json b/submissions/yolonas-l_hailo_int8.json new file mode 100644 index 0000000..476398b --- /dev/null +++ b/submissions/yolonas-l_hailo_int8.json @@ -0,0 +1,101 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolonas-l-hailo-int8-hailort-rpi5_hailo8-20260628T195258766098Z", + "created_at": "2026-06-28T19:52:58.766135Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolonas-l", + "name": "yolonas-l", + "family": "yolonas", + "variant": "l", + "source": "libreyolo", + "weights": "yolonas-l", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.5242338911343088, + "mAP_50": 0.6935547237267466, + "mAP_75": 0.5649020873573805, + "mAP_small": 0.35182186649309055, + "mAP_medium": 0.5936105569738327, + "mAP_large": 0.6245882897512551, + "AR1": 0.39886015400859043, + "AR10": 0.6312355092169804, + "AR100": 0.6715977250187926, + "AR_small": 0.4977650642226191, + "AR_medium": 0.7201306551961083, + "AR_large": 0.790381063565836 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 64.892, + "std": 0.128, + "p50": 64.88, + "p95": 64.985, + "p99": 65.038, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 15.41, + "fps_p50": 15.41 + }, + "model_stats": { + "params_millions": 66.97, + "gflops": 116.6 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T19:52:58.766135Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + } +} \ No newline at end of file diff --git a/submissions/yolonas-m__pytorch__cpu__rpi5__20260629T064125448789Z.json b/submissions/yolonas-m__pytorch__cpu__rpi5__20260629T064125448789Z.json new file mode 100644 index 0000000..8cfb707 --- /dev/null +++ b/submissions/yolonas-m__pytorch__cpu__rpi5__20260629T064125448789Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolonas-m-pytorch-cpu-rpi5-20260629T064125Z", + "created_at": "2026-06-29T06:41:25.448789Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolonas-m", + "name": "yolo-nas-m", + "family": "yolonas", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLONASm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5544547683777413, + "mAP_50": 0.722620824576428, + "mAP_75": 0.6035577827392103, + "mAP_small": 0.3762689309932977, + "mAP_medium": 0.6289804777038956, + "mAP_large": 0.6892941564923878, + "AR1": 0.42135623240753245, + "AR10": 0.6639889766461968, + "AR100": 0.6931054991397029, + "AR_small": 0.49060743009913144, + "AR_medium": 0.7477457628729601, + "AR_large": 0.8538316974535644 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2028.618, + "std": 7.754, + "p50": 2027.951, + "p95": 2041.683, + "p99": 2050.422, + "preprocess_ms": 9.397, + "inference_ms": 2016.355, + "postprocess_ms": 2.867 + } + }, + "throughput": { + "fps_mean": 0.49, + "fps_p50": 0.49 + }, + "model_stats": { + "params_millions": 51.18, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 82.1 + }, + "metadata": { + "benchmark_date": "2026-06-29T06:41:25.448789Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolonas-m__pytorch__cuda__jetson_orin__20260628T094638919768Z.json b/submissions/yolonas-m__pytorch__cuda__jetson_orin__20260628T094638919768Z.json new file mode 100644 index 0000000..e40723d --- /dev/null +++ b/submissions/yolonas-m__pytorch__cuda__jetson_orin__20260628T094638919768Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolonas-m-pytorch-cuda-jetson_orin-20260628T094638Z", + "created_at": "2026-06-28T09:46:38.919768Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0.dev0", + "libreyolo_commit": "5f12ee96d5f3099eaede265187a5df70a5f70c08" + }, + "model": { + "id": "yolonas-m", + "name": "yolo-nas-m", + "family": "yolonas", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLONASm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Orin (nvgpu)", + "gpu_memory_gb": 0.0, + "driver_version": "595.78", + "cuda_version": "13.0", + "cpu": "Unknown", + "cpu_cores": 6, + "ram_gb": 7, + "id": "jetson_orin" + }, + "software": { + "python": "3.12.3", + "torch": "2.12.0+cu130", + "libreyolo": "1.2.0.dev0", + "libreyolo_commit": "5f12ee96d5f3099eaede265187a5df70a5f70c08", + "onnxruntime": "1.24.0" + }, + "accuracy": { + "mAP_50_95": 0.5545068064245974, + "mAP_50": 0.7225798762638183, + "mAP_75": 0.6034660037662518, + "mAP_small": 0.3763163218088265, + "mAP_medium": 0.6291118510748668, + "mAP_large": 0.6893367086338652, + "AR1": 0.42140201995331994, + "AR10": 0.6641307164205555, + "AR100": 0.6932570917806772, + "AR_small": 0.49073760887096063, + "AR_medium": 0.7478990607811923, + "AR_large": 0.8538268909166743 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 152.018, + "std": 14.286, + "p50": 151.357, + "p95": 153.812, + "p99": 154.873, + "preprocess_ms": 12.572, + "inference_ms": 133.209, + "postprocess_ms": 6.237 + } + }, + "throughput": { + "fps_mean": 6.58, + "fps_p50": 6.61 + }, + "model_stats": { + "params_millions": 51.18, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 364.5, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T09:46:38.919768Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0.dev0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cuda", + "device": "gpu" + } +} \ No newline at end of file diff --git a/submissions/yolonas-m_hailo_int8.json b/submissions/yolonas-m_hailo_int8.json new file mode 100644 index 0000000..f3ce577 --- /dev/null +++ b/submissions/yolonas-m_hailo_int8.json @@ -0,0 +1,101 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolonas-m-hailo-int8-hailort-rpi5_hailo8-20260628T181411310711Z", + "created_at": "2026-06-28T18:14:11.310749Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolonas-m", + "name": "yolonas-m", + "family": "yolonas", + "variant": "m", + "source": "libreyolo", + "weights": "yolonas-m", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.47352629105744415, + "mAP_50": 0.6376564877227787, + "mAP_75": 0.5187175003491332, + "mAP_small": 0.29475744384964614, + "mAP_medium": 0.5482709458514573, + "mAP_large": 0.5764728560539091, + "AR1": 0.3640817386206549, + "AR10": 0.5910485022165374, + "AR100": 0.6300280220448617, + "AR_small": 0.4309091807908955, + "AR_medium": 0.6943580826835967, + "AR_large": 0.7590375638022909 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 46.047, + "std": 0.258, + "p50": 46.026, + "p95": 46.147, + "p99": 46.214, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 21.72, + "fps_p50": 21.73 + }, + "model_stats": { + "params_millions": 51.1, + "gflops": 88.9 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T18:14:11.310749Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + } +} \ No newline at end of file diff --git a/submissions/yolonas-s__pytorch__cpu__rpi5__20260629T001434373358Z.json b/submissions/yolonas-s__pytorch__cpu__rpi5__20260629T001434373358Z.json new file mode 100644 index 0000000..d9564b4 --- /dev/null +++ b/submissions/yolonas-s__pytorch__cpu__rpi5__20260629T001434373358Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolonas-s-pytorch-cpu-rpi5-20260629T001434Z", + "created_at": "2026-06-29T00:14:34.373358Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolonas-s", + "name": "yolo-nas-s", + "family": "yolonas", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLONASs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5175187804596387, + "mAP_50": 0.68985349636812, + "mAP_75": 0.5658402328317349, + "mAP_small": 0.31195812978907683, + "mAP_medium": 0.5807006918363328, + "mAP_large": 0.6776626884024056, + "AR1": 0.3996041197390947, + "AR10": 0.6268720617180102, + "AR100": 0.6582188104717092, + "AR_small": 0.4251034101792207, + "AR_medium": 0.7157421987255158, + "AR_large": 0.8077122859175214 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1049.54, + "std": 13.242, + "p50": 1055.903, + "p95": 1066.118, + "p99": 1069.991, + "preprocess_ms": 9.364, + "inference_ms": 1037.35, + "postprocess_ms": 2.825 + } + }, + "throughput": { + "fps_mean": 0.95, + "fps_p50": 0.95 + }, + "model_stats": { + "params_millions": 19.05, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 81.2 + }, + "metadata": { + "benchmark_date": "2026-06-29T00:14:34.373358Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolonas-s__pytorch__cuda__jetson_orin__20260628T094453408459Z.json b/submissions/yolonas-s__pytorch__cuda__jetson_orin__20260628T094453408459Z.json new file mode 100644 index 0000000..696bc21 --- /dev/null +++ b/submissions/yolonas-s__pytorch__cuda__jetson_orin__20260628T094453408459Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolonas-s-pytorch-cuda-jetson_orin-20260628T094453Z", + "created_at": "2026-06-28T09:44:53.408459Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0.dev0", + "libreyolo_commit": "5f12ee96d5f3099eaede265187a5df70a5f70c08" + }, + "model": { + "id": "yolonas-s", + "name": "yolo-nas-s", + "family": "yolonas", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLONASs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Orin (nvgpu)", + "gpu_memory_gb": 0.0, + "driver_version": "595.78", + "cuda_version": "13.0", + "cpu": "Unknown", + "cpu_cores": 6, + "ram_gb": 7, + "id": "jetson_orin" + }, + "software": { + "python": "3.12.3", + "torch": "2.12.0+cu130", + "libreyolo": "1.2.0.dev0", + "libreyolo_commit": "5f12ee96d5f3099eaede265187a5df70a5f70c08", + "onnxruntime": "1.24.0" + }, + "accuracy": { + "mAP_50_95": 0.5175123196146053, + "mAP_50": 0.6899003979069985, + "mAP_75": 0.565893842401729, + "mAP_small": 0.3121718608086119, + "mAP_medium": 0.5805600595913436, + "mAP_large": 0.6776601702081991, + "AR1": 0.3996041197390947, + "AR10": 0.6268054217417672, + "AR100": 0.658152170495466, + "AR_small": 0.425049605010031, + "AR_medium": 0.7156647993447107, + "AR_large": 0.8077122859175214 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 98.002, + "std": 20.79, + "p50": 96.547, + "p95": 98.904, + "p99": 101.173, + "preprocess_ms": 15.853, + "inference_ms": 75.994, + "postprocess_ms": 6.155 + } + }, + "throughput": { + "fps_mean": 10.2, + "fps_p50": 10.36 + }, + "model_stats": { + "params_millions": 19.05, + "gflops": 0.0 + }, + "memory": { + "peak_vram_mb": 172.7, + "peak_ram_mb": 227.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T09:44:53.408459Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0.dev0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cuda", + "device": "gpu" + } +} \ No newline at end of file diff --git a/submissions/yolonas-s_hailo_int8.json b/submissions/yolonas-s_hailo_int8.json new file mode 100644 index 0000000..6b53405 --- /dev/null +++ b/submissions/yolonas-s_hailo_int8.json @@ -0,0 +1,101 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolonas-s-hailo-int8-hailort-rpi5_hailo8-20260628T165354261494Z", + "created_at": "2026-06-28T16:53:54.261533Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolonas-s", + "name": "yolonas-s", + "family": "yolonas", + "variant": "s", + "source": "libreyolo", + "weights": "yolonas-s", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.4446089067793858, + "mAP_50": 0.6136841620622177, + "mAP_75": 0.47159018033704103, + "mAP_small": 0.28340429645318765, + "mAP_medium": 0.5084353950730437, + "mAP_large": 0.552387659507169, + "AR1": 0.3510863125136369, + "AR10": 0.5645162152320146, + "AR100": 0.6026627926794943, + "AR_small": 0.4250443242017818, + "AR_medium": 0.6519618056463824, + "AR_large": 0.716713529654017 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 25.2, + "std": 0.134, + "p50": 25.165, + "p95": 25.472, + "p99": 25.584, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 39.68, + "fps_p50": 39.74 + }, + "model_stats": { + "params_millions": 19.02, + "gflops": 32.8 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T16:53:54.261533Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + } +} \ No newline at end of file diff --git a/submissions/yolov9c__ncnn__cpu__rpi5__20260627T190818435183Z.json b/submissions/yolov9c__ncnn__cpu__rpi5__20260627T190818435183Z.json new file mode 100644 index 0000000..67f4753 --- /dev/null +++ b/submissions/yolov9c__ncnn__cpu__rpi5__20260627T190818435183Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9c-ncnn-cpu-rpi5-20260627T190818Z", + "created_at": "2026-06-27T19:08:18.435183Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9c", + "name": "yolov9-c", + "family": "yolov9", + "variant": "c", + "source": "libreyolo", + "weights": "LibreYOLO9c.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5583977509379243, + "mAP_50": 0.7213695204157715, + "mAP_75": 0.6068137998649635, + "mAP_small": 0.3665642619867098, + "mAP_medium": 0.6182217354453128, + "mAP_large": 0.6968989740677778, + "AR1": 0.4111350174840171, + "AR10": 0.6509927883417154, + "AR100": 0.676280013725787, + "AR_small": 0.47127765853283904, + "AR_medium": 0.7208207352066598, + "AR_large": 0.7991152862312051 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 531.781, + "std": 3.314, + "p50": 530.818, + "p95": 537.626, + "p99": 546.149, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.88, + "fps_p50": 1.88 + }, + "model_stats": { + "params_millions": 25.5, + "gflops": 51.75 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T19:08:18.435183Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolov9c__onnx__cpu__rpi5__20260627T185643908084Z.json b/submissions/yolov9c__onnx__cpu__rpi5__20260627T185643908084Z.json new file mode 100644 index 0000000..180415d --- /dev/null +++ b/submissions/yolov9c__onnx__cpu__rpi5__20260627T185643908084Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9c-onnx-cpu-rpi5-20260627T185643Z", + "created_at": "2026-06-27T18:56:43.908084Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9c", + "name": "yolov9-c", + "family": "yolov9", + "variant": "c", + "source": "libreyolo", + "weights": "LibreYOLO9c.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5566300259948382, + "mAP_50": 0.7189050949764564, + "mAP_75": 0.605865824410832, + "mAP_small": 0.3652740059450509, + "mAP_medium": 0.6174037066617017, + "mAP_large": 0.6961826666388315, + "AR1": 0.4118876606519492, + "AR10": 0.6507793746815861, + "AR100": 0.6759077751224216, + "AR_small": 0.4710651153951669, + "AR_medium": 0.7204474161385015, + "AR_large": 0.7981423351926766 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1398.298, + "std": 8.211, + "p50": 1395.518, + "p95": 1412.814, + "p99": 1425.463, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.72, + "fps_p50": 0.72 + }, + "model_stats": { + "params_millions": 25.47, + "gflops": 51.75 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:56:43.908084Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolov9c__pytorch__cpu__rpi5__20260627T183042773905Z.json b/submissions/yolov9c__pytorch__cpu__rpi5__20260627T183042773905Z.json new file mode 100644 index 0000000..8162f10 --- /dev/null +++ b/submissions/yolov9c__pytorch__cpu__rpi5__20260627T183042773905Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9c-pytorch-cpu-rpi5-20260627T183042Z", + "created_at": "2026-06-27T18:30:42.773905Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9c", + "name": "yolov9-c", + "family": "yolov9", + "variant": "c", + "source": "libreyolo", + "weights": "LibreYOLO9c.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5644543593876598, + "mAP_50": 0.7300005402529022, + "mAP_75": 0.6145330016470393, + "mAP_small": 0.37838306469498856, + "mAP_medium": 0.6229321401810518, + "mAP_large": 0.7093135785059931, + "AR1": 0.4204609989789378, + "AR10": 0.6671566582359321, + "AR100": 0.6939913561307999, + "AR_small": 0.5068377158440566, + "AR_medium": 0.7340935279923338, + "AR_large": 0.8209435571618553 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2153.804, + "std": 14.662, + "p50": 2155.616, + "p95": 2165.246, + "p99": 2173.649, + "preprocess_ms": 3.366, + "inference_ms": 2148.2, + "postprocess_ms": 2.238 + } + }, + "throughput": { + "fps_mean": 0.46, + "fps_p50": 0.46 + }, + "model_stats": { + "params_millions": 25.5, + "gflops": 51.75 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 5.5 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:30:42.773905Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolov9c_hailo_sub.json b/submissions/yolov9c_hailo_sub.json new file mode 100644 index 0000000..d0d88f0 --- /dev/null +++ b/submissions/yolov9c_hailo_sub.json @@ -0,0 +1,101 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9c-hailo-int8-hailort-rpi5_hailo8-20260628T134302307079Z", + "created_at": "2026-06-28T13:43:02.307140Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9c", + "name": "yolov9c", + "family": "yolov9", + "variant": "c", + "source": "libreyolo", + "weights": "LibreYOLO9c.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.5476913920616249, + "mAP_50": 0.7131098144960689, + "mAP_75": 0.5939205638251501, + "mAP_small": 0.35394835635034233, + "mAP_medium": 0.6029809562957014, + "mAP_large": 0.6933464388665393, + "AR1": 0.411439527476965, + "AR10": 0.6524148628406007, + "AR100": 0.6819507435947298, + "AR_small": 0.4954902209154678, + "AR_medium": 0.7211944153664459, + "AR_large": 0.8003639237722002 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 47.079, + "std": 0.16, + "p50": 47.057, + "p95": 47.249, + "p99": 47.375, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 21.24, + "fps_p50": 21.25 + }, + "model_stats": { + "params_millions": 25.5, + "gflops": 51.75 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T13:43:02.307140Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + } +} \ No newline at end of file diff --git a/submissions/yolov9m__ncnn__cpu__rpi5__20260627T190343992851Z.json b/submissions/yolov9m__ncnn__cpu__rpi5__20260627T190343992851Z.json new file mode 100644 index 0000000..b16d78d --- /dev/null +++ b/submissions/yolov9m__ncnn__cpu__rpi5__20260627T190343992851Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9m-ncnn-cpu-rpi5-20260627T190343Z", + "created_at": "2026-06-27T19:03:43.992851Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9m", + "name": "yolov9-m", + "family": "yolov9", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLO9m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5360877614676471, + "mAP_50": 0.6986857575481058, + "mAP_75": 0.5821722325108789, + "mAP_small": 0.3496126726985973, + "mAP_medium": 0.6043786920558012, + "mAP_large": 0.678102458800428, + "AR1": 0.4018644485947828, + "AR10": 0.6361830406470051, + "AR100": 0.6598288998441495, + "AR_small": 0.4671601187146544, + "AR_medium": 0.7133224921300235, + "AR_large": 0.7709461748018833 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 426.456, + "std": 3.536, + "p50": 425.372, + "p95": 433.219, + "p99": 442.629, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 2.34, + "fps_p50": 2.35 + }, + "model_stats": { + "params_millions": 20.12, + "gflops": 38.68 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T19:03:43.992851Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolov9m__onnx__cpu__rpi5__20260627T184453208676Z.json b/submissions/yolov9m__onnx__cpu__rpi5__20260627T184453208676Z.json new file mode 100644 index 0000000..df69e6c --- /dev/null +++ b/submissions/yolov9m__onnx__cpu__rpi5__20260627T184453208676Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9m-onnx-cpu-rpi5-20260627T184453Z", + "created_at": "2026-06-27T18:44:53.208676Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9m", + "name": "yolov9-m", + "family": "yolov9", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLO9m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5362561688863842, + "mAP_50": 0.6984629810034418, + "mAP_75": 0.5822802169919253, + "mAP_small": 0.34856523008321005, + "mAP_medium": 0.6028613173004446, + "mAP_large": 0.6797255613398119, + "AR1": 0.4022525302184688, + "AR10": 0.6360745900137863, + "AR100": 0.6603222993849666, + "AR_small": 0.4691277418037462, + "AR_medium": 0.7129589768007857, + "AR_large": 0.7715388557377723 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1039.926, + "std": 6.587, + "p50": 1037.314, + "p95": 1053.842, + "p99": 1065.368, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.96, + "fps_p50": 0.96 + }, + "model_stats": { + "params_millions": 20.1, + "gflops": 38.68 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:44:53.208676Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolov9m__pytorch__cpu__rpi5__20260627T181011500592Z.json b/submissions/yolov9m__pytorch__cpu__rpi5__20260627T181011500592Z.json new file mode 100644 index 0000000..c511e77 --- /dev/null +++ b/submissions/yolov9m__pytorch__cpu__rpi5__20260627T181011500592Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9m-pytorch-cpu-rpi5-20260627T181011Z", + "created_at": "2026-06-27T18:10:11.500592Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9m", + "name": "yolov9-m", + "family": "yolov9", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLO9m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5528302643352359, + "mAP_50": 0.7169861926817778, + "mAP_75": 0.600945163591593, + "mAP_small": 0.3632950248731157, + "mAP_medium": 0.6176426703980745, + "mAP_large": 0.6922671306992221, + "AR1": 0.42118616021280153, + "AR10": 0.6615014970561075, + "AR100": 0.687016360397431, + "AR_small": 0.490465192200465, + "AR_medium": 0.7390452575430414, + "AR_large": 0.7928921026725747 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1638.037, + "std": 11.717, + "p50": 1640.403, + "p95": 1651.406, + "p99": 1655.153, + "preprocess_ms": 3.432, + "inference_ms": 1632.286, + "postprocess_ms": 2.32 + } + }, + "throughput": { + "fps_mean": 0.61, + "fps_p50": 0.61 + }, + "model_stats": { + "params_millions": 20.12, + "gflops": 38.68 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:10:11.500592Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolov9m_hailo_sub.json b/submissions/yolov9m_hailo_sub.json new file mode 100644 index 0000000..b92b3c2 --- /dev/null +++ b/submissions/yolov9m_hailo_sub.json @@ -0,0 +1,101 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9m-hailo-int8-hailort-rpi5_hailo8-20260628T143953705472Z", + "created_at": "2026-06-28T14:39:53.705530Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9m", + "name": "yolov9m", + "family": "yolov9", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLO9m.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.5329770770002916, + "mAP_50": 0.6983906983089887, + "mAP_75": 0.5812418637026137, + "mAP_small": 0.35497461017547266, + "mAP_medium": 0.6027547293345632, + "mAP_large": 0.6758022076684784, + "AR1": 0.40010749483340924, + "AR10": 0.6388201868343779, + "AR100": 0.668386838759305, + "AR_small": 0.49351004442454804, + "AR_medium": 0.7171895263421209, + "AR_large": 0.7890398835778198 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 33.232, + "std": 0.083, + "p50": 33.222, + "p95": 33.384, + "p99": 33.489, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 30.09, + "fps_p50": 30.1 + }, + "model_stats": { + "params_millions": 20.12, + "gflops": 38.68 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T14:39:53.705530Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + } +} \ No newline at end of file diff --git a/submissions/yolov9s__ncnn__cpu__rpi5__20260627T190002561194Z.json b/submissions/yolov9s__ncnn__cpu__rpi5__20260627T190002561194Z.json new file mode 100644 index 0000000..d6849bf --- /dev/null +++ b/submissions/yolov9s__ncnn__cpu__rpi5__20260627T190002561194Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9s-ncnn-cpu-rpi5-20260627T190002Z", + "created_at": "2026-06-27T19:00:02.561194Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9s", + "name": "yolov9-s", + "family": "yolov9", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLO9s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.484190091628974, + "mAP_50": 0.6451728654247788, + "mAP_75": 0.5296253601274965, + "mAP_small": 0.29249693304875785, + "mAP_medium": 0.5368658238613542, + "mAP_large": 0.6352314909128711, + "AR1": 0.3637650968327946, + "AR10": 0.5825791209645957, + "AR100": 0.6067629306798582, + "AR_small": 0.39021001116429194, + "AR_medium": 0.6547917139111623, + "AR_large": 0.7576447505953106 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 203.84, + "std": 3.823, + "p50": 202.602, + "p95": 211.321, + "p99": 219.656, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 4.91, + "fps_p50": 4.94 + }, + "model_stats": { + "params_millions": 7.23, + "gflops": 13.52 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.4 + }, + "metadata": { + "benchmark_date": "2026-06-27T19:00:02.561194Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolov9s__onnx__cpu__rpi5__20260627T183602767730Z.json b/submissions/yolov9s__onnx__cpu__rpi5__20260627T183602767730Z.json new file mode 100644 index 0000000..053fcc1 --- /dev/null +++ b/submissions/yolov9s__onnx__cpu__rpi5__20260627T183602767730Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9s-onnx-cpu-rpi5-20260627T183602Z", + "created_at": "2026-06-27T18:36:02.767730Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9s", + "name": "yolov9-s", + "family": "yolov9", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLO9s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4853996159820961, + "mAP_50": 0.6461150697986132, + "mAP_75": 0.5309630273094694, + "mAP_small": 0.293103152155661, + "mAP_medium": 0.5371168693341506, + "mAP_large": 0.637471842387726, + "AR1": 0.3653049250326364, + "AR10": 0.5835361493751406, + "AR100": 0.608027908594639, + "AR_small": 0.3921281074987864, + "AR_medium": 0.6537540973193435, + "AR_large": 0.7603834052649736 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 425.079, + "std": 7.453, + "p50": 422.447, + "p95": 439.647, + "p99": 453.722, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 2.35, + "fps_p50": 2.37 + }, + "model_stats": { + "params_millions": 7.21, + "gflops": 13.52 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:36:02.767730Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolov9s__pytorch__cpu__rpi5__20260627T175420938854Z.json b/submissions/yolov9s__pytorch__cpu__rpi5__20260627T175420938854Z.json new file mode 100644 index 0000000..a4aa534 --- /dev/null +++ b/submissions/yolov9s__pytorch__cpu__rpi5__20260627T175420938854Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9s-pytorch-cpu-rpi5-20260627T175420Z", + "created_at": "2026-06-27T17:54:20.938854Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9s", + "name": "yolov9-s", + "family": "yolov9", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLO9s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4944675127323235, + "mAP_50": 0.6596658528528612, + "mAP_75": 0.5413431837383176, + "mAP_small": 0.2975432516124148, + "mAP_medium": 0.5565341085804606, + "mAP_large": 0.6390638954417918, + "AR1": 0.3748582894884518, + "AR10": 0.6048835920165866, + "AR100": 0.6334458960835037, + "AR_small": 0.4098745352838049, + "AR_medium": 0.6915979672968395, + "AR_large": 0.7829655837727637 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 716.73, + "std": 8.597, + "p50": 719.061, + "p95": 726.876, + "p99": 730.016, + "preprocess_ms": 3.57, + "inference_ms": 710.747, + "postprocess_ms": 2.413 + } + }, + "throughput": { + "fps_mean": 1.4, + "fps_p50": 1.39 + }, + "model_stats": { + "params_millions": 7.23, + "gflops": 13.52 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 30.5 + }, + "metadata": { + "benchmark_date": "2026-06-27T17:54:20.938854Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolov9s_hailo_sub.json b/submissions/yolov9s_hailo_sub.json new file mode 100644 index 0000000..7bfc008 --- /dev/null +++ b/submissions/yolov9s_hailo_sub.json @@ -0,0 +1,101 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9s-hailo-int8-hailort-rpi5_hailo8-20260628T143918676128Z", + "created_at": "2026-06-28T14:39:18.676175Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9s", + "name": "yolov9s", + "family": "yolov9", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLO9s.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.3202846787855532, + "mAP_50": 0.45209050901722025, + "mAP_75": 0.345745204202997, + "mAP_small": 0.24456901999951858, + "mAP_medium": 0.287868241763752, + "mAP_large": 0.5383565118707007, + "AR1": 0.3050424135140193, + "AR10": 0.5258970743061131, + "AR100": 0.5746515426102451, + "AR_small": 0.38826074471458855, + "AR_medium": 0.6140913568484321, + "AR_large": 0.6804843133291116 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 27.517, + "std": 0.167, + "p50": 27.497, + "p95": 27.686, + "p99": 27.759, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 36.34, + "fps_p50": 36.37 + }, + "model_stats": { + "params_millions": 7.23, + "gflops": 13.52 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T14:39:18.676175Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + } +} \ No newline at end of file diff --git a/submissions/yolov9t__ncnn__cpu__rpi5__20260627T185812539348Z.json b/submissions/yolov9t__ncnn__cpu__rpi5__20260627T185812539348Z.json new file mode 100644 index 0000000..7126108 --- /dev/null +++ b/submissions/yolov9t__ncnn__cpu__rpi5__20260627T185812539348Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9t-ncnn-cpu-rpi5-20260627T185812Z", + "created_at": "2026-06-27T18:58:12.539348Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9t", + "name": "yolov9-t", + "family": "yolov9", + "variant": "t", + "source": "libreyolo", + "weights": "LibreYOLO9t.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4058640221288353, + "mAP_50": 0.5475924958482162, + "mAP_75": 0.43728430457276396, + "mAP_small": 0.20796364489459593, + "mAP_medium": 0.45828521694993446, + "mAP_large": 0.5382840184674554, + "AR1": 0.33371038683374427, + "AR10": 0.5219582085682064, + "AR100": 0.55158805287851, + "AR_small": 0.29685765329229263, + "AR_medium": 0.618996539352521, + "AR_large": 0.6659664303018029 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 90.959, + "std": 4.968, + "p50": 89.417, + "p95": 99.815, + "p99": 114.633, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 10.99, + "fps_p50": 11.18 + }, + "model_stats": { + "params_millions": 2.04, + "gflops": 3.98 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 33.2 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:58:12.539348Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolov9t__onnx__cpu__rpi5__20260627T183220946977Z.json b/submissions/yolov9t__onnx__cpu__rpi5__20260627T183220946977Z.json new file mode 100644 index 0000000..48d277c --- /dev/null +++ b/submissions/yolov9t__onnx__cpu__rpi5__20260627T183220946977Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9t-onnx-cpu-rpi5-20260627T183220Z", + "created_at": "2026-06-27T18:32:20.946977Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9t", + "name": "yolov9-t", + "family": "yolov9", + "variant": "t", + "source": "libreyolo", + "weights": "LibreYOLO9t.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4074648393937907, + "mAP_50": 0.5506893479126529, + "mAP_75": 0.44026169451437647, + "mAP_small": 0.2090285418847601, + "mAP_medium": 0.462185513260405, + "mAP_large": 0.5385469455363298, + "AR1": 0.33865172239561997, + "AR10": 0.5262971308176277, + "AR100": 0.5548826838590432, + "AR_small": 0.3055872980978682, + "AR_medium": 0.6215627307649114, + "AR_large": 0.6643449226322229 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 169.966, + "std": 5.736, + "p50": 168.092, + "p95": 179.504, + "p99": 196.557, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 5.88, + "fps_p50": 5.95 + }, + "model_stats": { + "params_millions": 2.03, + "gflops": 3.98 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 48.2 + }, + "metadata": { + "benchmark_date": "2026-06-27T18:32:20.946977Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolov9t__pytorch__cpu__rpi5__20260627T174600644311Z.json b/submissions/yolov9t__pytorch__cpu__rpi5__20260627T174600644311Z.json new file mode 100644 index 0000000..009e1ec --- /dev/null +++ b/submissions/yolov9t__pytorch__cpu__rpi5__20260627T174600644311Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9t-pytorch-cpu-rpi5-20260627T174600Z", + "created_at": "2026-06-27T17:46:00.644311Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9t", + "name": "yolov9-t", + "family": "yolov9", + "variant": "t", + "source": "libreyolo", + "weights": "LibreYOLO9t.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4142886655085496, + "mAP_50": 0.561828226291405, + "mAP_75": 0.44681530144982345, + "mAP_small": 0.21504784164132595, + "mAP_medium": 0.4639682903623852, + "mAP_large": 0.5600930967573488, + "AR1": 0.35160654286643267, + "AR10": 0.5545386454848746, + "AR100": 0.5872618193555333, + "AR_small": 0.34655592324253437, + "AR_medium": 0.648047133626687, + "AR_large": 0.7212515580394837 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 346.255, + "std": 3.038, + "p50": 345.943, + "p95": 350.352, + "p99": 353.337, + "preprocess_ms": 3.504, + "inference_ms": 340.048, + "postprocess_ms": 2.703 + } + }, + "throughput": { + "fps_mean": 2.89, + "fps_p50": 2.89 + }, + "model_stats": { + "params_millions": 2.04, + "gflops": 3.98 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 82.6 + }, + "metadata": { + "benchmark_date": "2026-06-27T17:46:00.644311Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolov9t_hailo_sub.json b/submissions/yolov9t_hailo_sub.json new file mode 100644 index 0000000..1299f13 --- /dev/null +++ b/submissions/yolov9t_hailo_sub.json @@ -0,0 +1,101 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolov9t-hailo-int8-hailort-rpi5_hailo8-20260628T134218779826Z", + "created_at": "2026-06-28T13:42:18.779870Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolov9t", + "name": "yolov9t", + "family": "yolov9", + "variant": "t", + "source": "libreyolo", + "weights": "LibreYOLO9t.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.2574812551731687, + "mAP_50": 0.3724033083680573, + "mAP_75": 0.26901622770320943, + "mAP_small": 0.1810754891905198, + "mAP_medium": 0.2300524545800218, + "mAP_large": 0.38513007996398724, + "AR1": 0.2640856426966872, + "AR10": 0.4639695273232776, + "AR100": 0.500326398525334, + "AR_small": 0.31131925240285746, + "AR_medium": 0.5204978214354378, + "AR_large": 0.6177960497236653 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 13.077, + "std": 0.12, + "p50": 13.058, + "p95": 13.259, + "p99": 13.424, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 76.47, + "fps_p50": 76.58 + }, + "model_stats": { + "params_millions": 2.04, + "gflops": 3.98 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T13:42:18.779870Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + } +} \ No newline at end of file diff --git a/submissions/yolox-l__ncnn__cpu__rpi5__20260628T095530857573Z.json b/submissions/yolox-l__ncnn__cpu__rpi5__20260628T095530857573Z.json new file mode 100644 index 0000000..a08f191 --- /dev/null +++ b/submissions/yolox-l__ncnn__cpu__rpi5__20260628T095530857573Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-l-ncnn-cpu-rpi5-20260628T095530Z", + "created_at": "2026-06-28T09:55:30.857573Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-l", + "name": "yolox-l", + "family": "yolox", + "variant": "l", + "source": "libreyolo", + "weights": "LibreYOLOXl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.528503230832125, + "mAP_50": 0.7047107466545172, + "mAP_75": 0.5797387403130669, + "mAP_small": 0.3641973658775863, + "mAP_medium": 0.593339115805217, + "mAP_large": 0.655551455750752, + "AR1": 0.40605083051426444, + "AR10": 0.6293511801499894, + "AR100": 0.6519720370803261, + "AR_small": 0.4738208902298428, + "AR_medium": 0.6996843841570429, + "AR_large": 0.7667428430878959 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 662.417, + "std": 15.182, + "p50": 662.514, + "p95": 677.071, + "p99": 737.746, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.51, + "fps_p50": 1.51 + }, + "model_stats": { + "params_millions": 54.21, + "gflops": 78.01 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 24.2 + }, + "metadata": { + "benchmark_date": "2026-06-28T09:55:30.857573Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-l__onnx__cpu__rpi5__20260627T234701760600Z.json b/submissions/yolox-l__onnx__cpu__rpi5__20260627T234701760600Z.json new file mode 100644 index 0000000..a7fbb33 --- /dev/null +++ b/submissions/yolox-l__onnx__cpu__rpi5__20260627T234701760600Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-l-onnx-cpu-rpi5-20260627T234701Z", + "created_at": "2026-06-27T23:47:01.760600Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-l", + "name": "yolox-l", + "family": "yolox", + "variant": "l", + "source": "libreyolo", + "weights": "LibreYOLOXl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5425723281364085, + "mAP_50": 0.7148532433913983, + "mAP_75": 0.5900762125505338, + "mAP_small": 0.3732762177642842, + "mAP_medium": 0.5899402317718047, + "mAP_large": 0.6632153806471186, + "AR1": 0.40963947368940234, + "AR10": 0.6314039307956981, + "AR100": 0.6523117132579677, + "AR_small": 0.4700331331237939, + "AR_medium": 0.6937800073605596, + "AR_large": 0.7703388497310525 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1882.396, + "std": 4.506, + "p50": 1881.406, + "p95": 1890.615, + "p99": 1897.52, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.53, + "fps_p50": 0.53 + }, + "model_stats": { + "params_millions": 54.17, + "gflops": 78.01 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 14.8 + }, + "metadata": { + "benchmark_date": "2026-06-27T23:47:01.760600Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-l__pytorch__cpu__rpi5__20260629T071142195596Z.json b/submissions/yolox-l__pytorch__cpu__rpi5__20260629T071142195596Z.json new file mode 100644 index 0000000..2804544 --- /dev/null +++ b/submissions/yolox-l__pytorch__cpu__rpi5__20260629T071142195596Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-l-pytorch-cpu-rpi5-20260629T071142Z", + "created_at": "2026-06-29T07:11:42.195596Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-l", + "name": "yolox-l", + "family": "yolox", + "variant": "l", + "source": "libreyolo", + "weights": "LibreYOLOXl.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5391250645636839, + "mAP_50": 0.7166949761716915, + "mAP_75": 0.5910098562529215, + "mAP_small": 0.38166727366157893, + "mAP_medium": 0.6005190613243097, + "mAP_large": 0.6736901380015644, + "AR1": 0.41972797189119304, + "AR10": 0.6602132206987458, + "AR100": 0.684190638681071, + "AR_small": 0.5029510237520646, + "AR_medium": 0.7297849558476158, + "AR_large": 0.826473447231166 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 2627.003, + "std": 17.989, + "p50": 2632.621, + "p95": 2644.28, + "p99": 2650.036, + "preprocess_ms": 8.016, + "inference_ms": 2613.758, + "postprocess_ms": 5.229 + } + }, + "throughput": { + "fps_mean": 0.38, + "fps_p50": 0.38 + }, + "model_stats": { + "params_millions": 54.21, + "gflops": 78.01 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 55.1 + }, + "metadata": { + "benchmark_date": "2026-06-29T07:11:42.195596Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-m__ncnn__cpu__rpi5__20260628T093306098833Z.json b/submissions/yolox-m__ncnn__cpu__rpi5__20260628T093306098833Z.json new file mode 100644 index 0000000..a6ee9f5 --- /dev/null +++ b/submissions/yolox-m__ncnn__cpu__rpi5__20260628T093306098833Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-m-ncnn-cpu-rpi5-20260628T093306Z", + "created_at": "2026-06-28T09:33:06.098833Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-m", + "name": "yolox-m", + "family": "yolox", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLOXm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4973637579387121, + "mAP_50": 0.6778704655888971, + "mAP_75": 0.5436906487170675, + "mAP_small": 0.31767712052616864, + "mAP_medium": 0.5632789389188162, + "mAP_large": 0.6350061189476125, + "AR1": 0.3775142117444723, + "AR10": 0.6106086415880172, + "AR100": 0.6354199496981533, + "AR_small": 0.4326868277422986, + "AR_medium": 0.6788045896586901, + "AR_large": 0.7524180746189737 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 340.284, + "std": 7.7, + "p50": 338.348, + "p95": 351.333, + "p99": 372.308, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 2.94, + "fps_p50": 2.96 + }, + "model_stats": { + "params_millions": 25.33, + "gflops": 36.99 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 0.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T09:33:06.098833Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-m__onnx__cpu__rpi5__20260627T233104111286Z.json b/submissions/yolox-m__onnx__cpu__rpi5__20260627T233104111286Z.json new file mode 100644 index 0000000..69ed9bd --- /dev/null +++ b/submissions/yolox-m__onnx__cpu__rpi5__20260627T233104111286Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-m-onnx-cpu-rpi5-20260627T233104Z", + "created_at": "2026-06-27T23:31:04.111286Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-m", + "name": "yolox-m", + "family": "yolox", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLOXm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5079809277791495, + "mAP_50": 0.6885030787752504, + "mAP_75": 0.5561999863864477, + "mAP_small": 0.3169944618897781, + "mAP_medium": 0.5756125968123131, + "mAP_large": 0.6494321665777568, + "AR1": 0.38473953732302707, + "AR10": 0.6148418795289804, + "AR100": 0.6380360590060202, + "AR_small": 0.42389438889250686, + "AR_medium": 0.6846285912520943, + "AR_large": 0.7565182102322142 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 922.046, + "std": 3.757, + "p50": 921.247, + "p95": 929.027, + "p99": 935.563, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 1.08, + "fps_p50": 1.09 + }, + "model_stats": { + "params_millions": 25.3, + "gflops": 36.99 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 15.4 + }, + "metadata": { + "benchmark_date": "2026-06-27T23:31:04.111286Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-m__pytorch__cpu__rpi5__20260629T005027897974Z.json b/submissions/yolox-m__pytorch__cpu__rpi5__20260629T005027897974Z.json new file mode 100644 index 0000000..6e82186 --- /dev/null +++ b/submissions/yolox-m__pytorch__cpu__rpi5__20260629T005027897974Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-m-pytorch-cpu-rpi5-20260629T005027Z", + "created_at": "2026-06-29T00:50:27.897974Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-m", + "name": "yolox-m", + "family": "yolox", + "variant": "m", + "source": "libreyolo", + "weights": "LibreYOLOXm.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.5091614002893265, + "mAP_50": 0.6937390846005986, + "mAP_75": 0.5567399898210998, + "mAP_small": 0.3183261816380624, + "mAP_medium": 0.5753011971934548, + "mAP_large": 0.6645057221435278, + "AR1": 0.3939314117959562, + "AR10": 0.638739175548791, + "AR100": 0.6643246891329869, + "AR_small": 0.4485149876142434, + "AR_medium": 0.7048254369364738, + "AR_large": 0.8146288687115271 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1284.111, + "std": 11.267, + "p50": 1286.967, + "p95": 1296.347, + "p99": 1302.237, + "preprocess_ms": 8.274, + "inference_ms": 1270.559, + "postprocess_ms": 5.278 + } + }, + "throughput": { + "fps_mean": 0.78, + "fps_p50": 0.78 + }, + "model_stats": { + "params_millions": 25.33, + "gflops": 36.99 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 40.5 + }, + "metadata": { + "benchmark_date": "2026-06-29T00:50:27.897974Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-m_hailo_int8.json b/submissions/yolox-m_hailo_int8.json new file mode 100644 index 0000000..e090461 --- /dev/null +++ b/submissions/yolox-m_hailo_int8.json @@ -0,0 +1,101 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-m-hailo-int8-hailort-rpi5_hailo8-20260628T165553195225Z", + "created_at": "2026-06-28T16:55:53.195266Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-m", + "name": "yolox-m", + "family": "yolox", + "variant": "m", + "source": "libreyolo", + "weights": "yolox-m", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.65, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.43675730077050046, + "mAP_50": 0.672517622406605, + "mAP_75": 0.487206700674435, + "mAP_small": 0.29748043110546074, + "mAP_medium": 0.5033715842973834, + "mAP_large": 0.5434731037743242, + "AR1": 0.3448179409504896, + "AR10": 0.5760628996119347, + "AR100": 0.617170327621538, + "AR_small": 0.4455076890917315, + "AR_medium": 0.6734527396795311, + "AR_large": 0.7079739394568797 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 28.76, + "std": 0.296, + "p50": 28.737, + "p95": 28.829, + "p99": 28.9, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 34.77, + "fps_p50": 34.8 + }, + "model_stats": { + "params_millions": 25.33, + "gflops": 36.99 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T16:55:53.195266Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + } +} \ No newline at end of file diff --git a/submissions/yolox-nano__ncnn__cpu__rpi5__20260628T083125436356Z.json b/submissions/yolox-nano__ncnn__cpu__rpi5__20260628T083125436356Z.json new file mode 100644 index 0000000..2c8de60 --- /dev/null +++ b/submissions/yolox-nano__ncnn__cpu__rpi5__20260628T083125436356Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-nano-ncnn-cpu-rpi5-20260628T083125Z", + "created_at": "2026-06-28T08:31:25.436356Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-nano", + "name": "yolox-nano", + "family": "yolox", + "variant": "nano", + "source": "libreyolo", + "weights": "LibreYOLOXn.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.26778154230434653, + "mAP_50": 0.4133182842310963, + "mAP_75": 0.27991306032274327, + "mAP_small": 0.1112425154779287, + "mAP_medium": 0.2604675357298222, + "mAP_large": 0.4036815524243401, + "AR1": 0.23620923484312897, + "AR10": 0.3646138903045786, + "AR100": 0.38732800969923664, + "AR_small": 0.17069738657984965, + "AR_medium": 0.40244485710832145, + "AR_large": 0.5372784676862987 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 32.581, + "std": 3.705, + "p50": 31.564, + "p95": 39.828, + "p99": 46.408, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 30.69, + "fps_p50": 31.68 + }, + "model_stats": { + "params_millions": 0.91, + "gflops": 1.32 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 31.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:31:25.436356Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-nano__onnx__cpu__rpi5__20260627T231834551500Z.json b/submissions/yolox-nano__onnx__cpu__rpi5__20260627T231834551500Z.json new file mode 100644 index 0000000..16c9ebf --- /dev/null +++ b/submissions/yolox-nano__onnx__cpu__rpi5__20260627T231834551500Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-nano-onnx-cpu-rpi5-20260627T231834Z", + "created_at": "2026-06-27T23:18:34.551500Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-nano", + "name": "yolox-nano", + "family": "yolox", + "variant": "nano", + "source": "libreyolo", + "weights": "LibreYOLOXn.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.27685857961952604, + "mAP_50": 0.42734128303256097, + "mAP_75": 0.29140239953452074, + "mAP_small": 0.11728903036893894, + "mAP_medium": 0.27800536997957237, + "mAP_large": 0.41207893068827733, + "AR1": 0.241041546973221, + "AR10": 0.37212501436783635, + "AR100": 0.3934431803759116, + "AR_small": 0.16388973271510549, + "AR_medium": 0.4107943957825836, + "AR_large": 0.5465296093259006 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 46.98, + "std": 3.404, + "p50": 45.973, + "p95": 53.848, + "p99": 59.242, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 21.29, + "fps_p50": 21.75 + }, + "model_stats": { + "params_millions": 0.9, + "gflops": 1.32 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 35.1 + }, + "metadata": { + "benchmark_date": "2026-06-27T23:18:34.551500Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-nano__pytorch__cpu__rpi5__20260628T211630082561Z.json b/submissions/yolox-nano__pytorch__cpu__rpi5__20260628T211630082561Z.json new file mode 100644 index 0000000..2522001 --- /dev/null +++ b/submissions/yolox-nano__pytorch__cpu__rpi5__20260628T211630082561Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-nano-pytorch-cpu-rpi5-20260628T211630Z", + "created_at": "2026-06-28T21:16:30.082561Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-nano", + "name": "yolox-nano", + "family": "yolox", + "variant": "nano", + "source": "libreyolo", + "weights": "LibreYOLOXn.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.28777402963982734, + "mAP_50": 0.4451673550790783, + "mAP_75": 0.3028944469746292, + "mAP_small": 0.12006471690282278, + "mAP_medium": 0.29773356865571005, + "mAP_large": 0.42695198658971256, + "AR1": 0.2590170696484389, + "AR10": 0.4129019655382761, + "AR100": 0.44096391074417346, + "AR_small": 0.18192961060887416, + "AR_medium": 0.4803201787960003, + "AR_large": 0.6389866972976415 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 117.019, + "std": 2.05, + "p50": 116.703, + "p95": 119.738, + "p99": 123.358, + "preprocess_ms": 6.813, + "inference_ms": 106.927, + "postprocess_ms": 3.279 + } + }, + "throughput": { + "fps_mean": 8.55, + "fps_p50": 8.57 + }, + "model_stats": { + "params_millions": 0.91, + "gflops": 1.32 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 54.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:16:30.082561Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-s__ncnn__cpu__rpi5__20260628T083436014518Z.json b/submissions/yolox-s__ncnn__cpu__rpi5__20260628T083436014518Z.json new file mode 100644 index 0000000..0ff62fb --- /dev/null +++ b/submissions/yolox-s__ncnn__cpu__rpi5__20260628T083436014518Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-s-ncnn-cpu-rpi5-20260628T083436Z", + "created_at": "2026-06-28T08:34:36.014518Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-s", + "name": "yolox-s", + "family": "yolox", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLOXs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4185662911362574, + "mAP_50": 0.6069472657595824, + "mAP_75": 0.4579735482952389, + "mAP_small": 0.24747404414796903, + "mAP_medium": 0.4530628388606526, + "mAP_large": 0.5451185371505978, + "AR1": 0.3388265661320211, + "AR10": 0.5256455820552539, + "AR100": 0.5539892971320937, + "AR_small": 0.37409886749020593, + "AR_medium": 0.5888357441443937, + "AR_large": 0.6705650528404357 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 174.294, + "std": 6.457, + "p50": 172.492, + "p95": 186.237, + "p99": 198.885, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 5.74, + "fps_p50": 5.8 + }, + "model_stats": { + "params_millions": 8.97, + "gflops": 13.46 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 35.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:34:36.014518Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-s__onnx__cpu__rpi5__20260627T232309330878Z.json b/submissions/yolox-s__onnx__cpu__rpi5__20260627T232309330878Z.json new file mode 100644 index 0000000..91d50aa --- /dev/null +++ b/submissions/yolox-s__onnx__cpu__rpi5__20260627T232309330878Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-s-onnx-cpu-rpi5-20260627T232309Z", + "created_at": "2026-06-27T23:23:09.330878Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-s", + "name": "yolox-s", + "family": "yolox", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLOXs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.4310803617516668, + "mAP_50": 0.6150036757369061, + "mAP_75": 0.47318084172273933, + "mAP_small": 0.25959567816358403, + "mAP_medium": 0.48433772755387117, + "mAP_large": 0.5546052620673176, + "AR1": 0.3444410874562318, + "AR10": 0.538269640202279, + "AR100": 0.5611962334744903, + "AR_small": 0.3706899883081812, + "AR_medium": 0.5986010377258627, + "AR_large": 0.661479601812048 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 393.522, + "std": 5.28, + "p50": 392.898, + "p95": 402.547, + "p99": 414.172, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 2.54, + "fps_p50": 2.55 + }, + "model_stats": { + "params_millions": 8.96, + "gflops": 13.46 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 19.9 + }, + "metadata": { + "benchmark_date": "2026-06-27T23:23:09.330878Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-s__pytorch__cpu__rpi5__20260628T214837471331Z.json b/submissions/yolox-s__pytorch__cpu__rpi5__20260628T214837471331Z.json new file mode 100644 index 0000000..28eb4f0 --- /dev/null +++ b/submissions/yolox-s__pytorch__cpu__rpi5__20260628T214837471331Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-s-pytorch-cpu-rpi5-20260628T214837Z", + "created_at": "2026-06-28T21:48:37.471331Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-s", + "name": "yolox-s", + "family": "yolox", + "variant": "s", + "source": "libreyolo", + "weights": "LibreYOLOXs.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.43010515097930546, + "mAP_50": 0.6220141448765621, + "mAP_75": 0.46897993598051824, + "mAP_small": 0.2599367080177599, + "mAP_medium": 0.4656542485238322, + "mAP_large": 0.5685465617255948, + "AR1": 0.35492883202242, + "AR10": 0.5655097103344773, + "AR100": 0.5969756922378401, + "AR_small": 0.40518655835245926, + "AR_medium": 0.6354007584389023, + "AR_large": 0.7251227937433352 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 626.603, + "std": 6.253, + "p50": 627.342, + "p95": 635.033, + "p99": 639.983, + "preprocess_ms": 8.29, + "inference_ms": 612.634, + "postprocess_ms": 5.679 + } + }, + "throughput": { + "fps_mean": 1.6, + "fps_p50": 1.59 + }, + "model_stats": { + "params_millions": 8.97, + "gflops": 13.46 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 97.5 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:48:37.471331Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-s_hailo_int8.json b/submissions/yolox-s_hailo_int8.json new file mode 100644 index 0000000..8a89a1e --- /dev/null +++ b/submissions/yolox-s_hailo_int8.json @@ -0,0 +1,101 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-s-hailo-int8-hailort-rpi5_hailo8-20260628T153957476167Z", + "created_at": "2026-06-28T15:39:57.476206Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-s", + "name": "yolox-s", + "family": "yolox", + "variant": "s", + "source": "libreyolo", + "weights": "yolox-s", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.65, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.4108814968955979, + "mAP_50": 0.6064624459610067, + "mAP_75": 0.45019380792723657, + "mAP_small": 0.24086973713406004, + "mAP_medium": 0.45198968047342614, + "mAP_large": 0.5516861215849379, + "AR1": 0.33068144790244186, + "AR10": 0.5487670148495044, + "AR100": 0.5951574328491301, + "AR_small": 0.4066083827833327, + "AR_medium": 0.6326720962572658, + "AR_large": 0.7265084581631085 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 10.437, + "std": 0.133, + "p50": 10.426, + "p95": 10.555, + "p99": 10.658, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 95.81, + "fps_p50": 95.91 + }, + "model_stats": { + "params_millions": 8.97, + "gflops": 13.46 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T15:39:57.476206Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + } +} \ No newline at end of file diff --git a/submissions/yolox-tiny__ncnn__cpu__rpi5__20260628T083235583918Z.json b/submissions/yolox-tiny__ncnn__cpu__rpi5__20260628T083235583918Z.json new file mode 100644 index 0000000..12a72d2 --- /dev/null +++ b/submissions/yolox-tiny__ncnn__cpu__rpi5__20260628T083235583918Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-tiny-ncnn-cpu-rpi5-20260628T083235Z", + "created_at": "2026-06-28T08:32:35.583918Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-tiny", + "name": "yolox-tiny", + "family": "yolox", + "variant": "tiny", + "source": "libreyolo", + "weights": "LibreYOLOXt.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.34018710868068724, + "mAP_50": 0.515197185375451, + "mAP_75": 0.35530815204888655, + "mAP_small": 0.1624423684857395, + "mAP_medium": 0.35967145151208013, + "mAP_large": 0.48609589032951545, + "AR1": 0.27907472341163525, + "AR10": 0.43768103350807686, + "AR100": 0.46013456915204903, + "AR_small": 0.24814593704685403, + "AR_medium": 0.4975632162219642, + "AR_large": 0.6246945608892681 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 54.281, + "std": 3.4, + "p50": 53.36, + "p95": 61.778, + "p99": 66.277, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 18.42, + "fps_p50": 18.74 + }, + "model_stats": { + "params_millions": 5.06, + "gflops": 7.68 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 20.9 + }, + "metadata": { + "benchmark_date": "2026-06-28T08:32:35.583918Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-tiny__onnx__cpu__rpi5__20260627T231939650560Z.json b/submissions/yolox-tiny__onnx__cpu__rpi5__20260627T231939650560Z.json new file mode 100644 index 0000000..5d979e5 --- /dev/null +++ b/submissions/yolox-tiny__onnx__cpu__rpi5__20260627T231939650560Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-tiny-onnx-cpu-rpi5-20260627T231939Z", + "created_at": "2026-06-27T23:19:39.650560Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-tiny", + "name": "yolox-tiny", + "family": "yolox", + "variant": "tiny", + "source": "libreyolo", + "weights": "LibreYOLOXt.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.3430169990073264, + "mAP_50": 0.515721093844349, + "mAP_75": 0.3589411422898518, + "mAP_small": 0.15805005951198056, + "mAP_medium": 0.360053827095217, + "mAP_large": 0.49948878510205447, + "AR1": 0.2795115290778376, + "AR10": 0.4362892776562205, + "AR100": 0.4596545269028024, + "AR_small": 0.23112833784085282, + "AR_medium": 0.49827732201225117, + "AR_large": 0.6230383106180426 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 106.688, + "std": 3.05, + "p50": 105.842, + "p95": 112.705, + "p99": 117.871, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 9.37, + "fps_p50": 9.45 + }, + "model_stats": { + "params_millions": 5.05, + "gflops": 7.68 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 27.4 + }, + "metadata": { + "benchmark_date": "2026-06-27T23:19:39.650560Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-tiny__pytorch__cpu__rpi5__20260628T214304549414Z.json b/submissions/yolox-tiny__pytorch__cpu__rpi5__20260628T214304549414Z.json new file mode 100644 index 0000000..a231f35 --- /dev/null +++ b/submissions/yolox-tiny__pytorch__cpu__rpi5__20260628T214304549414Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-tiny-pytorch-cpu-rpi5-20260628T214304Z", + "created_at": "2026-06-28T21:43:04.549414Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-tiny", + "name": "yolox-tiny", + "family": "yolox", + "variant": "tiny", + "source": "libreyolo", + "weights": "LibreYOLOXt.pt", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "not-installed" + }, + "accuracy": { + "mAP_50_95": 0.3507077718360378, + "mAP_50": 0.5299376329995034, + "mAP_75": 0.3699625988162548, + "mAP_small": 0.17195484437178557, + "mAP_medium": 0.37105117781877367, + "mAP_large": 0.5024188305234134, + "AR1": 0.29771079852091176, + "AR10": 0.48312923523876466, + "AR100": 0.5087055145491408, + "AR_small": 0.26969197340962586, + "AR_medium": 0.5545428732852953, + "AR_large": 0.6856424677582275 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 183.538, + "std": 1.79, + "p50": 183.316, + "p95": 186.528, + "p99": 188.541, + "preprocess_ms": 6.828, + "inference_ms": 173.282, + "postprocess_ms": 3.428 + } + }, + "throughput": { + "fps_mean": 5.45, + "fps_p50": 5.46 + }, + "model_stats": { + "params_millions": 5.06, + "gflops": 7.68 + }, + "memory": { + "peak_vram_mb": 0.0, + "peak_ram_mb": 45.1 + }, + "metadata": { + "benchmark_date": "2026-06-28T21:43:04.549414Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "pytorch", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-tiny_hailo_int8.json b/submissions/yolox-tiny_hailo_int8.json new file mode 100644 index 0000000..435c0da --- /dev/null +++ b/submissions/yolox-tiny_hailo_int8.json @@ -0,0 +1,101 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-tiny-hailo-int8-hailort-rpi5_hailo8-20260628T150112939893Z", + "created_at": "2026-06-28T15:01:12.939934Z", + "benchmark": { + "harness": "hailo-kit", + "harness_version": "1.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-tiny", + "name": "yolox-tiny", + "family": "yolox", + "variant": "tiny", + "source": "libreyolo", + "weights": "yolox-tiny", + "input_size": 416 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 416, + "conf": 0.001, + "iou": 0.65, + "max_det": 300 + }, + "hardware": { + "gpu": "Hailo-8", + "gpu_memory_gb": 0, + "driver_version": "4.23.0", + "cuda_version": "N/A", + "cpu": "Broadcom BCM2712", + "cpu_cores": 4, + "ram_gb": 16, + "id": "rpi5_hailo8" + }, + "accuracy": { + "mAP_50_95": 0.3353106440385515, + "mAP_50": 0.5189426748353215, + "mAP_75": 0.3641099820138371, + "mAP_small": 0.17764741992028685, + "mAP_medium": 0.36505107259605907, + "mAP_large": 0.49343300017658587, + "AR1": 0.2893957905970191, + "AR10": 0.4696391008304537, + "AR100": 0.5116869077190426, + "AR_small": 0.2877205160943997, + "AR_medium": 0.5603867562198416, + "AR_large": 0.6986791375911993 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 5.112, + "std": 0.063, + "p50": 5.109, + "p95": 5.223, + "p99": 5.273, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 195.62, + "fps_p50": 195.73 + }, + "model_stats": { + "params_millions": 5.06, + "gflops": 7.68 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": null + }, + "metadata": { + "benchmark_date": "2026-06-28T15:01:12.939934Z", + "benchmark_version": "1.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "hailo", + "precision": "int8", + "provider": "hailort", + "device": "npu" + } +} \ No newline at end of file diff --git a/submissions/yolox-x__ncnn__cpu__rpi5__20260628T101844235212Z.json b/submissions/yolox-x__ncnn__cpu__rpi5__20260628T101844235212Z.json new file mode 100644 index 0000000..ca48efc --- /dev/null +++ b/submissions/yolox-x__ncnn__cpu__rpi5__20260628T101844235212Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-x-ncnn-cpu-rpi5-20260628T101844Z", + "created_at": "2026-06-28T10:18:44.235212Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-x", + "name": "yolox-x", + "family": "yolox", + "variant": "x", + "source": "libreyolo", + "weights": "LibreYOLOXx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5392783045915617, + "mAP_50": 0.713395311782745, + "mAP_75": 0.5857402354568811, + "mAP_small": 0.3639028164271833, + "mAP_medium": 0.6023587897373351, + "mAP_large": 0.6647190792773235, + "AR1": 0.4060455451363752, + "AR10": 0.636941478990372, + "AR100": 0.6597413549532147, + "AR_small": 0.47090377473567496, + "AR_medium": 0.6990418161236918, + "AR_large": 0.7816580652957302 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 1122.797, + "std": 21.501, + "p50": 1122.824, + "p95": 1144.462, + "p99": 1209.295, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.89, + "fps_p50": 0.89 + }, + "model_stats": { + "params_millions": 99.07, + "gflops": 141.23 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 6.0 + }, + "metadata": { + "benchmark_date": "2026-06-28T10:18:44.235212Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "ncnn", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/submissions/yolox-x__onnx__cpu__rpi5__20260628T001402617033Z.json b/submissions/yolox-x__onnx__cpu__rpi5__20260628T001402617033Z.json new file mode 100644 index 0000000..72d04ed --- /dev/null +++ b/submissions/yolox-x__onnx__cpu__rpi5__20260628T001402617033Z.json @@ -0,0 +1,108 @@ +{ + "schema_version": "va.submission.v1", + "submission_id": "yolox-x-onnx-cpu-rpi5-20260628T001402Z", + "created_at": "2026-06-28T00:14:02.617033Z", + "benchmark": { + "harness": "vision-analysis-benchmark", + "harness_version": "2.0.0", + "libreyolo_version": "1.2.0", + "libreyolo_commit": "unknown" + }, + "model": { + "id": "yolox-x", + "name": "yolox-x", + "family": "yolox", + "variant": "x", + "source": "libreyolo", + "weights": "LibreYOLOXx.pt", + "input_size": 640 + }, + "dataset": { + "id": "coco2017", + "split": "val2017", + "num_images": 500 + }, + "config": { + "batch_size": 1, + "input_size": 640, + "conf": 0.001, + "iou": 0.6, + "max_det": 300 + }, + "hardware": { + "gpu": "Raspberry Pi 5 Model B Rev 1.1", + "gpu_memory_gb": 0.0, + "driver_version": "N/A", + "cuda_version": "N/A", + "cpu": "Unknown", + "cpu_cores": 4, + "ram_gb": 15, + "id": "rpi5" + }, + "software": { + "python": "3.13.5", + "torch": "2.12.1+cpu", + "libreyolo": "1.2.0", + "libreyolo_commit": "unknown", + "onnxruntime": "1.27.0" + }, + "accuracy": { + "mAP_50_95": 0.5486653429671409, + "mAP_50": 0.7238862739256607, + "mAP_75": 0.5983595432292893, + "mAP_small": 0.365999601171563, + "mAP_medium": 0.6026995218171797, + "mAP_large": 0.6743854440852284, + "AR1": 0.41002385524288115, + "AR10": 0.6358502590432511, + "AR100": 0.6558722179507147, + "AR_small": 0.46583175901590107, + "AR_medium": 0.6997710103321302, + "AR_large": 0.7807043628363978 + }, + "timing": { + "batch_size": 1, + "num_images": 500, + "total_ms": { + "mean": 3200.47, + "std": 5.253, + "p50": 3199.437, + "p95": 3210.456, + "p99": 3217.408, + "preprocess_ms": null, + "inference_ms": null, + "postprocess_ms": null + } + }, + "throughput": { + "fps_mean": 0.31, + "fps_p50": 0.31 + }, + "model_stats": { + "params_millions": 99.02, + "gflops": 141.23 + }, + "memory": { + "peak_vram_mb": null, + "peak_ram_mb": 21.7 + }, + "metadata": { + "benchmark_date": "2026-06-28T00:14:02.617033Z", + "benchmark_version": "2.0.0" + }, + "eval": { + "dataset": "coco", + "split": "val2017", + "numImages": 500 + }, + "implementation": { + "provider": "libreyolo", + "version": "1.2.0" + }, + "runtime": { + "format": "onnx", + "precision": "fp32", + "provider": "cpu", + "device": "cpu" + } +} \ No newline at end of file diff --git a/website/next.config.ts b/website/next.config.ts index fa09489..12147a7 100644 --- a/website/next.config.ts +++ b/website/next.config.ts @@ -1,6 +1,13 @@ import type { NextConfig } from "next"; +import path from "path"; const nextConfig: NextConfig = { + // Pin the workspace root to this app. Stray package-lock.json files higher up + // (e.g. in the home directory) otherwise make Next infer the wrong root, which + // breaks module resolution and makes the watcher scan the whole home tree. + turbopack: { + root: path.resolve(__dirname), + }, async redirects() { return [ { diff --git a/website/package-lock.json b/website/package-lock.json index 1409a1d..ab9b730 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -19,6 +19,7 @@ "@radix-ui/react-switch": "^1.2.6", "@radix-ui/react-tabs": "^1.1.13", "@radix-ui/react-tooltip": "^1.2.8", + "@vercel/analytics": "^2.0.1", "@vercel/og": "^0.8.6", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -87,7 +88,6 @@ "integrity": "sha512-H3mcG6ZDLTlYfaSNi0iOKkigqMFvkTKlGUYlD8GW7nNOYRrevuA46iTypPyv+06V3fEmvvazfntkBU34L0azAw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.28.6", "@babel/generator": "^7.28.6", @@ -2641,7 +2641,6 @@ "integrity": "sha512-Lpo8kgb/igvMIPeNV2rsYKTgaORYdO1XGVZ4Qz3akwOj0ySGYMPlQWa8BaLn0G63D1aSaAQ5ldR06wCpChQCjA==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -2652,7 +2651,6 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -2702,7 +2700,6 @@ "integrity": "sha512-nm3cvFN9SqZGXjmw5bZ6cGmvJSyJPn0wU9gHAZZHDnZl2wF9PhHv78Xf06E0MaNk4zLVHL8hb2/c32XvyJOLQg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.53.1", "@typescript-eslint/types": "8.53.1", @@ -3196,6 +3193,48 @@ "win32" ] }, + "node_modules/@vercel/analytics": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@vercel/analytics/-/analytics-2.0.1.tgz", + "integrity": "sha512-MTQG6V9qQrt1tsDeF+2Uoo5aPjqbVPys1xvnIftXSJYG2SrwXRHnqEvVoYID7BTruDz4lCd2Z7rM1BdkUehk2g==", + "license": "MIT", + "peerDependencies": { + "@remix-run/react": "^2", + "@sveltejs/kit": "^1 || ^2", + "next": ">= 13", + "nuxt": ">= 3", + "react": "^18 || ^19 || ^19.0.0-rc", + "svelte": ">= 4", + "vue": "^3", + "vue-router": "^4" + }, + "peerDependenciesMeta": { + "@remix-run/react": { + "optional": true + }, + "@sveltejs/kit": { + "optional": true + }, + "next": { + "optional": true + }, + "nuxt": { + "optional": true + }, + "react": { + "optional": true + }, + "svelte": { + "optional": true + }, + "vue": { + "optional": true + }, + "vue-router": { + "optional": true + } + } + }, "node_modules/@vercel/og": { "version": "0.8.6", "resolved": "https://registry.npmjs.org/@vercel/og/-/og-0.8.6.tgz", @@ -3215,7 +3254,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3583,7 +3621,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -4153,7 +4190,6 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", - "peer": true, "engines": { "node": ">=12" } @@ -4662,7 +4698,6 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -4848,7 +4883,6 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -7135,7 +7169,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.3.tgz", "integrity": "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7145,7 +7178,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg==", "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -7971,7 +8003,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -8144,7 +8175,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -8479,7 +8509,6 @@ "integrity": "sha512-k7Nwx6vuWx1IJ9Bjuf4Zt1PEllcwe7cls3VNzm4CQ1/hgtFUK2bRNG3rvnpPUhFjmqJKAKtjV576KnUkHocg/g==", "dev": true, "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/website/package.json b/website/package.json index b3c546c..5c6e466 100644 --- a/website/package.json +++ b/website/package.json @@ -4,10 +4,12 @@ "private": true, "scripts": { "dev": "next dev", + "prebuild": "npm run verify:analytics", "build": "next build", - "vercel-build": "next build", + "vercel-build": "npm run verify:analytics && next build", "start": "next start", - "lint": "eslint" + "lint": "eslint", + "verify:analytics": "node scripts/verify-analytics.mjs" }, "dependencies": { "@observablehq/plot": "^0.6.17", @@ -21,6 +23,7 @@ "@radix-ui/react-switch": "^1.2.6", "@radix-ui/react-tabs": "^1.1.13", "@radix-ui/react-tooltip": "^1.2.8", + "@vercel/analytics": "^2.0.1", "@vercel/og": "^0.8.6", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", diff --git a/website/scripts/generate-draft.ts b/website/scripts/generate-draft.ts index 3397439..adaa83f 100644 --- a/website/scripts/generate-draft.ts +++ b/website/scripts/generate-draft.ts @@ -173,8 +173,6 @@ function generateDraft( // Compute deltas if we have data const mapDelta = hasData ? benchB.mAP_50_95 - benchA.mAP_50_95 : 0; - const fpsDelta = hasData ? benchB.throughputFps / benchA.throughputFps : 0; - return ` diff --git a/website/scripts/verify-analytics.mjs b/website/scripts/verify-analytics.mjs new file mode 100644 index 0000000..e1ba423 --- /dev/null +++ b/website/scripts/verify-analytics.mjs @@ -0,0 +1,29 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +const root = dirname(dirname(fileURLToPath(import.meta.url))); +const packageJson = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); +const layout = readFileSync(join(root, "src", "app", "layout.tsx"), "utf8"); + +const failures = []; + +if (!packageJson.dependencies?.["@vercel/analytics"]) { + failures.push("@vercel/analytics is missing from dependencies"); +} + +if (!layout.includes('from "@vercel/analytics/next"')) { + failures.push("the root layout does not import @vercel/analytics/next"); +} + +if (!layout.includes(" 0) { + console.error("Vercel Web Analytics integration check failed:"); + for (const failure of failures) console.error(`- ${failure}`); + process.exit(1); +} + +console.log("Vercel Web Analytics integration is present."); diff --git a/website/src/app/articles/[slug]/page.tsx b/website/src/app/articles/[slug]/page.tsx new file mode 100644 index 0000000..972c23d --- /dev/null +++ b/website/src/app/articles/[slug]/page.tsx @@ -0,0 +1,226 @@ +import { Metadata } from "next"; +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { ArrowLeft } from "lucide-react"; +import { + getArticle, + getArticleContent, + publishedArticles, + formatArticleDate, +} from "@/lib/articles"; +import { ArticleRenderer } from "@/components/articles/ArticleRenderer"; +import { siteConfig } from "@/config/site"; + +interface Props { + params: Promise<{ slug: string }>; +} + +export function generateStaticParams() { + return publishedArticles().map((a) => ({ slug: a.slug })); +} + +export async function generateMetadata({ params }: Props): Promise { + const { slug } = await params; + const article = getArticle(slug); + if (!article) return { title: "Article Not Found" }; + return { + title: article.title, + description: article.dek, + alternates: { canonical: `/articles/${slug}` }, + openGraph: { + title: `${article.title} | ${siteConfig.name}`, + description: article.dek, + type: "article", + }, + }; +} + +// Legacy bodies: hand-written React articles that predate the v2 engine. +// New articles are JSON content files under src/content/articles rendered by +// ArticleRenderer; do not add entries here. +const BODIES: Record React.ReactNode> = { + "yolov9s-vs-yolox-s": Yolov9sVsYoloxS, +}; + +function ArticleJsonLd({ slug }: { slug: string }) { + const article = getArticle(slug); + if (!article) return null; + const jsonLd = { + "@context": "https://schema.org", + "@type": "TechArticle", + headline: article.title, + description: article.dek, + datePublished: article.date, + dateModified: article.date, + author: { "@type": "Organization", name: article.author, url: siteConfig.url }, + publisher: { "@type": "Organization", name: siteConfig.name, url: siteConfig.url }, + mainEntityOfPage: `${siteConfig.url}/articles/${slug}`, + isBasedOn: `${siteConfig.url}/methodology`, + }; + return ( +