diff --git a/docs/comparison-systems.md b/docs/comparison-systems.md index 663fb1f..5e52402 100644 --- a/docs/comparison-systems.md +++ b/docs/comparison-systems.md @@ -60,13 +60,17 @@ Computed directly against the corpus and freshly-fetched peer content with the s ## External benchmarks (not yet run by us) -CodeWikiBench (ACL Findings 2026) publishes its own evaluator and competitor scores. doc0 has not been run through it yet — Family 2 of this benchmark suite (see `docs/DESIGN.md`) targets exactly that. Listed here for reference only; **do not compare these numbers directly to the claim-support numbers above** — different corpus (21 repos, 7 languages), different judge, different metric definition entirely. +CodeWikiBench (ACL Findings 2026) publishes its own evaluator and competitor scores, including for [CodeWiki](https://github.com/FSoft-AI4Code/CodeWiki), the paper authors' own generator. doc0 has not been run through it yet — Family 2 of this benchmark suite (see `docs/DESIGN.md`) targets exactly that. Listed here for reference only; **do not compare these numbers directly to the claim-support numbers above** — different corpus, different judge, different metric definition entirely. -| System | CodeWikiBench score | Source | +The published averages below are the paper's Table 1 "Average" row (4 systems, 7 repositories with per-repo detail; the project page describes the same 68.79/64.06 headline as a 21-repo result — a scope discrepancy we cite as-published without resolving, see `docs/codewikibench-pinned.md` §(f)). The Family-2 run reuses their published per-repo rubrics and their exact judge panel (Gemini 2.5 Flash, GPT-OSS-120B, Kimi K2 Instruct, averaged), so doc0's row will be comparable per-repo against every published row, CodeWiki's included, without re-running any peer. + +| System | CodeWikiBench score (avg) | Source | |---|---|---| -| CodeWiki-Sonnet-4 | 68.8% | published, external | -| DeepWiki | 64.1% | published, external | -| doc0 | — | not yet run | +| CodeWiki (Sonnet-4) — the benchmark authors' generator | 68.79% | published, Table 1 | +| DeepWiki | 64.06% | published, Table 1 | +| deepwiki-open | 50.05% | published, Table 1 | +| OpenDeepWiki | 47.13% | published, Table 1 | +| doc0 | — | not yet run (Family 2) | ## Change log diff --git a/runners/codewikibench.test.ts b/runners/codewikibench.test.ts index 9c8889c..80d7bb6 100644 --- a/runners/codewikibench.test.ts +++ b/runners/codewikibench.test.ts @@ -1,5 +1,8 @@ import { describe, it, expect } from "vitest"; import { + hoistPreambles, + neutralizeNonFinalLists, + normalizeHeadingLevels, planEvaluatorInput, stripLeadingDetailsBlock, transpileMdx, @@ -283,3 +286,183 @@ describe("planEvaluatorInput — merged with user_guide", () => { expect(new Set(slugs).size).toBe(slugs.length); }); }); + +// --------------------------------------------------------------------------- +// hoistPreambles +// --------------------------------------------------------------------------- + +describe("hoistPreambles", () => { + it("hoists an intro paragraph under an H1 with H2 children into ## Overview", () => { + const out = hoistPreambles("# Page\n\nIntro words.\n\n## First Section\n\nBody.\n"); + expect(out).toContain("# Page\n\n## Overview\n\nIntro words."); + expect(out.indexOf("## Overview")).toBeLessThan(out.indexOf("## First Section")); + }); + + it("hoists a preamble under an H2 with H3 children into ### Overview", () => { + const out = hoistPreambles("# P\n\n## Section\n\nPreamble.\n\n### Child\n\nLeaf.\n"); + expect(out).toContain("## Section\n\n### Overview\n\nPreamble."); + }); + + it("leaves leaf sections untouched", () => { + const md = "# P\n\n## Leaf\n\nJust a paragraph.\n\n1. item\n"; + expect(hoistPreambles(md)).toBe(md); + }); + + it("leaves a childful section with no preamble untouched", () => { + const md = "# P\n\n## Section\n\n### Child\n\nBody.\n"; + expect(hoistPreambles(md)).toBe(md); + }); + + it("ignores heading-looking lines inside fences, as content and as boundary", () => { + const md = "# P\n\nIntro.\n\n```sh\n# not a heading\n## also not\n```\n\n## Real Child\n\nBody.\n"; + const out = hoistPreambles(md); + expect(out).toContain("# P\n\n## Overview\n\nIntro."); + // The fence stays verbatim inside the hoisted preamble. + expect(out).toContain("```sh\n# not a heading\n## also not\n```"); + expect(out.match(/## Overview/g)).toHaveLength(1); + }); + + it("falls back to Introduction when a sibling child is already titled Overview", () => { + const out = hoistPreambles("# P\n\nIntro.\n\n## Overview\n\nReal overview.\n"); + expect(out).toContain("# P\n\n## Introduction\n\nIntro."); + }); + + it("hoists every childful section independently", () => { + const out = hoistPreambles( + "# P\n\nTop intro.\n\n## A\n\nA preamble.\n\n### A1\n\nLeaf.\n\n## B\n\nB leaf only.\n", + ); + expect(out).toContain("# P\n\n## Overview\n\nTop intro."); + expect(out).toContain("## A\n\n### Overview\n\nA preamble."); + // B is a leaf: untouched. + expect(out).toContain("## B\n\nB leaf only."); + }); + + // The pinned parser splits each section on headings at EXACTLY parent+1 and + // discards everything before the first one it finds. A first child that + // skips a level (H1 straight to H3, the shape of the committed + // redis cluster-routing page) is never a key, so a synthetic ## Overview + // would swallow it and lose the preamble again, while a synthetic ### would + // be skipped over together with the child. The skip is normalized away + // first, so the preamble lands under a leaf key beside its real sibling. + it("hoists a preamble whose first child skips a level into a leaf sibling", () => { + const out = hoistPreambles("# P\n\nIntro.\n\n### Child\n\nBody.\n\n## Sec\n\nS.\n"); + expect(out).toBe("# P\n\n## Overview\n\nIntro.\n\n## Child\n\nBody.\n\n## Sec\n\nS.\n"); + }); + + it("checks the synthetic title against the promoted sibling", () => { + const out = hoistPreambles("# P\n\nIntro.\n\n### Overview\n\nReal.\n"); + expect(out).toBe("# P\n\n## Introduction\n\nIntro.\n\n## Overview\n\nReal.\n"); + }); +}); + +// --------------------------------------------------------------------------- +// normalizeHeadingLevels +// --------------------------------------------------------------------------- + +describe("normalizeHeadingLevels", () => { + it("promotes a child that skips a level to exactly parent+1", () => { + expect(normalizeHeadingLevels("# P\n\n### Child\n\nBody.\n\n## Sec\n\nS.\n")).toBe( + "# P\n\n## Child\n\nBody.\n\n## Sec\n\nS.\n", + ); + }); + + it("re-levels descendants relative to their promoted parent", () => { + expect(normalizeHeadingLevels("# P\n\n### A\n\n##### A1\n\n## B\n")).toBe( + "# P\n\n## A\n\n### A1\n\n## B\n", + ); + }); + + it("makes a shallower heading after a skip a sibling of the skipped one, never its parent", () => { + // H4 under H2 skips H3; the H3 that follows closes the H4 and is H2's + // child too, so both come out at level 3. + expect(normalizeHeadingLevels("# P\n\n## S\n\n#### Deep\n\nD.\n\n### Next\n\nN.\n")).toBe( + "# P\n\n## S\n\n### Deep\n\nD.\n\n### Next\n\nN.\n", + ); + }); + + it("leaves a well-formed ladder byte-identical", () => { + const md = "# P\n\nIntro.\n\n## A\n\n### A1\n\nLeaf.\n\n## B\n\nB.\n"; + expect(normalizeHeadingLevels(md)).toBe(md); + }); + + it("starts the ladder at the page's first heading level", () => { + expect(normalizeHeadingLevels("## Top\n\n#### Child\n")).toBe("## Top\n\n### Child\n"); + }); + + it("never touches heading-looking lines inside fences", () => { + const md = "# P\n\n```sh\n#### not a heading\n```\n\n### Child\n"; + expect(normalizeHeadingLevels(md)).toBe("# P\n\n```sh\n#### not a heading\n```\n\n## Child\n"); + }); +}); + +describe("neutralizeNonFinalLists", () => { + it("escapes a list followed by a paragraph, keeps the words verbatim", () => { + const out = neutralizeNonFinalLists("## S\n\n1. First step.\n2. Second step.\n\nAfter-list paragraph.\n"); + expect(out).toContain("1\\. First step."); + expect(out).toContain("2\\. Second step."); + expect(out).toContain("After-list paragraph."); + }); + + it("keeps a section-final list as a real list", () => { + const md = "## S\n\nLead-in.\n\n1. Only list.\n2. Still fine.\n"; + expect(neutralizeNonFinalLists(md)).toBe(md); + }); + + it("escapes the first of two lists, keeps the second", () => { + const out = neutralizeNonFinalLists("## S\n\n1. A.\n\nBetween.\n\n- B.\n"); + expect(out).toContain("1\\. A."); + expect(out).toContain("\nBetween."); + expect(out).toContain("\n- B."); + expect(out).not.toContain("\\- B."); + }); + + it("escapes nested markers of a non-final list block", () => { + const out = neutralizeNonFinalLists("## S\n\n- Parent.\n 1. Child.\n\nTail paragraph.\n"); + expect(out).toContain("\\- Parent."); + expect(out).toContain(" 1\\. Child."); + }); + + it("treats each heading span independently", () => { + const out = neutralizeNonFinalLists("## A\n\n1. NonFinal.\n\nTail.\n\n## B\n\n1. Final.\n"); + expect(out).toContain("1\\. NonFinal."); + expect(out).toContain("\n1. Final."); + }); + + it("never touches list-looking lines inside fences", () => { + const md = "## S\n\n```txt\n1. not a list\n- neither\n```\n\nTail.\n"; + expect(neutralizeNonFinalLists(md)).toBe(md); + }); +}); + +describe("neutralizeNonFinalLists — tight lists", () => { + it("escapes a tight list (lead-in directly above the markers) when content follows", () => { + const out = neutralizeNonFinalLists( + "## S\n\nThe check sequence is:\n1. First check.\n2. Second check.\n\nSources: [x.ts:1-2](y)\n", + ); + expect(out).toContain("The check sequence is:"); + expect(out).toContain("1\\. First check."); + expect(out).toContain("2\\. Second check."); + expect(out).toContain("Sources: [x.ts:1-2](y)"); + }); + + it("keeps a tight section-final block untouched", () => { + const md = "## S\n\nMechanism:\n1. Detect.\n2. Rewrite.\n"; + expect(neutralizeNonFinalLists(md)).toBe(md); + }); +}); + +describe("neutralizeNonFinalLists — trailing Sources line", () => { + it("escapes a span-final list whose block ends with a Sources paragraph", () => { + const out = neutralizeNonFinalLists( + "## S\n\nOnion model:\n\n1. Receive context.\n2. Dispatch next.\nSources: [x.ts:1-2](y)\n", + ); + expect(out).toContain("1\\. Receive context."); + expect(out).toContain("2\\. Dispatch next."); + expect(out).toContain("Sources: [x.ts:1-2](y)"); + }); + + it("keeps a trailing list whose continuations are indented", () => { + const md = "## S\n\nLead.\n\n- Parent.\n 1. Child one.\n 2. Child two.\n"; + expect(neutralizeNonFinalLists(md)).toBe(md); + }); +}); diff --git a/runners/codewikibench.ts b/runners/codewikibench.ts index a1de667..02e8bcd 100644 --- a/runners/codewikibench.ts +++ b/runners/codewikibench.ts @@ -313,6 +313,219 @@ export function stripLeadingDetailsBlock(md: string): string { return md.replace(/^(\s*(?:# [^\n]*\n)?\s*)
[\s\S]*?<\/details>\s*/, "$1"); } +// --------------------------------------------------------------------------- +// normalizeHeadingLevels — parser-safe heading ladder +// --------------------------------------------------------------------------- + +const ATX_HEADING_RE = /^(#{1,6})\s+(.*)$/; + +interface HeadingLine { + idx: number; + level: number; + text: string; +} + +/** + * The evaluator's parser (`markdown_to_json`, `_dictify_blocks`) splits a + * section on headings at EXACTLY one level below its own and, when it finds + * any, discards every block before the first one (`dictify_list_by`). A + * heading that skips a level is therefore never a key. Measured on the pinned + * parser (2026-09-02 probe): an H3 directly under an H1 followed by an H2 + * loses the H1's preamble AND the whole H3 section, and H4s under an H2 with + * no H3 flatten into the H2's text. Its own docstring says as much ("if you + * jump ... the high-numbered headings won't be treated as keys"). + * + * The fix re-levels every heading to exactly one deeper than its nearest + * shallower predecessor (a stack walk; the first heading keeps its level, the + * parser roots at the page minimum anyway), so the ladder never skips: + * H1→H3→H2 becomes H1→H2→H2, H2→H4→H3 becomes H2→H3→H3. Heading text and + * order are untouched, and the nesting is exactly what a stack-based reader + * already infers — packaging for their parser, not editing. + * + * `hoistPreambles` runs this first so its "synthetic child one level deeper" + * invariant holds; the exported form exists for direct pinning. + */ +export function normalizeHeadingLevels(md: string): string { + const { working, fences, inlines } = maskCode(md); + const lines = working.split("\n"); + normalizeHeadingLines(lines); + return unmaskCode(lines.join("\n"), fences, inlines); +} + +/** + * Re-levels the ATX headings of a masked line array in place and returns + * them with their normalized levels. + */ +function normalizeHeadingLines(lines: string[]): HeadingLine[] { + const headings: HeadingLine[] = []; + // Ancestors still open: the raw level that opened them, and the level they + // were assigned. + const open: { raw: number; level: number }[] = []; + lines.forEach((line, idx) => { + const m = ATX_HEADING_RE.exec(line); + if (!m) return; + const raw = m[1].length; + while (open.length > 0 && (open[open.length - 1]?.raw ?? 0) >= raw) open.pop(); + const parent = open[open.length - 1]; + const level = parent ? parent.level + 1 : raw; + open.push({ raw, level }); + if (level !== raw) lines[idx] = `${"#".repeat(level)} ${m[2] ?? ""}`; + headings.push({ idx, level, text: (m[2] ?? "").trim() }); + }); + return headings; +} + +// --------------------------------------------------------------------------- +// hoistPreambles — parser-safe preamble hoisting +// --------------------------------------------------------------------------- + +/** + * The evaluator's parser (`markdown_to_json.jsonify`) maps each heading to a + * dict entry; a section that has CHILD headings becomes a dict of those + * children, and any loose content between the parent heading and its first + * child heading has nowhere to hang — it is silently dropped. Measured + * directly against the pinned parser (2026-08-31 probe, see the dry-run + * results): under a well-formed heading ladder this preamble drop is the + * ONLY construct lost. Paragraphs, ordered and unordered lists, tables, + * blockquotes, and fences under LEAF headings all survive verbatim. (A + * skipped heading level is the other loss mode; `normalizeHeadingLevels` + * above removes it first.) + * + * The fix packages each preamble as a synthetic first child heading one + * level deeper (default title "Overview" — the same key shape CodeWiki's own + * example output uses, e.g. "Purpose"), so the words their judge scores are + * the words doc0 wrote. Content is never altered, reordered, or summarized: + * this is packaging for their parser, not editing. + * + * Scope notes: + * - ATX headings only (`#`…`######`) — doc0-generated GFM never emits setext + * headings, and the corpus exporter preserves that. + * - Fences and inline code are masked first (same `maskCode` as + * `transpileMdx`), so a `#` line inside a code sample neither hoists nor + * terminates a section. + * - Heading levels are normalized before anything is hoisted: a first child + * that skips a level would otherwise nest under the synthetic heading + * (making it childful, so the parser drops the preamble again), and a + * synthetic heading placed at the child's skipped level would be skipped + * over with it. The sibling-title check runs on the normalized ladder. + * - A synthetic title colliding with a real sibling child would merge two + * dict keys in their parser, so the title falls back Overview → + * Introduction → Preamble → "Overview N". + * - Content before the file's FIRST heading is out of scope: exported corpus + * pages always open with their H1 (and `stripLeadingDetailsBlock` runs + * earlier in the chain). + */ +export function hoistPreambles(md: string): string { + const { working, fences, inlines } = maskCode(md); + const lines = working.split("\n"); + const headings = normalizeHeadingLines(lines); + + const insertions: { at: number; heading: string }[] = []; + for (let i = 0; i < headings.length; i += 1) { + const parent = headings[i]; + const next = headings[i + 1]; + // Leaf section (no following heading, or the next heading is a sibling or + // an ancestor): its content survives their parser untouched. + if (!parent || !next || next.level <= parent.level) continue; + + const preamble = lines.slice(parent.idx + 1, next.idx); + if (!preamble.some((l) => l.trim().length > 0)) continue; + + const childLevel = parent.level + 1; + const sectionEndIdx = + headings.slice(i + 1).find((h) => h.level <= parent.level)?.idx ?? lines.length; + const siblingTitles = new Set( + headings + .filter((h) => h.idx > parent.idx && h.idx < sectionEndIdx && h.level === childLevel) + .map((h) => h.text.toLowerCase()), + ); + let title = "Overview"; + if (siblingTitles.has(title.toLowerCase())) title = "Introduction"; + if (siblingTitles.has(title.toLowerCase())) title = "Preamble"; + for (let n = 2; siblingTitles.has(title.toLowerCase()); n += 1) title = `Overview ${n}`; + + insertions.push({ at: parent.idx + 1, heading: `${"#".repeat(childLevel)} ${title}` }); + } + + for (const ins of insertions.reverse()) { + lines.splice(ins.at, 0, "", ins.heading); + } + return unmaskCode(lines.join("\n"), fences, inlines); +} + +// --------------------------------------------------------------------------- +// neutralizeNonFinalLists — parser-safe list flattening +// --------------------------------------------------------------------------- + +/** + * Second measured loss mode in the pinned parser (2026-08-31 probe): within a + * section's content span, `markdown_to_json` keeps everything up to and + * including the FIRST list, then drops whatever follows it — a paragraph + * after a list, and any second list, vanish. A list that closes its section + * survives whole. + * + * Content-preserving fix: any list block that is NOT the final block of its + * span gets its markers escaped (`1.` → `1\.`, `-` → `\-`), which renders as + * the identical visible text but parses as ordinary paragraphs — and + * paragraphs always survive. The section-final list keeps real list syntax. + * No words are added, removed, or reordered. + * + * Mechanics: spans are the line ranges between ATX headings (fences masked + * first, so fence content is never touched and never splits a block). Blocks + * are blank-line-separated; a block whose first line carries a list marker is + * a list block. A loose list that blank-splits into several blocks keeps its + * final physical block as a real list and has the earlier items escaped — + * visually unchanged, and every word survives either way. + */ +export function neutralizeNonFinalLists(md: string): string { + const { working, fences, inlines } = maskCode(md); + const lines = working.split("\n"); + + const isHeading = (l: string): boolean => /^#{1,6}\s+/.test(l); + const isMarker = (l: string): boolean => /^\s*(?:[-*+]|\d+[.)])\s+/.test(l); + const isIndented = (l: string): boolean => /^\s+\S/.test(l); + const isBlank = (l: string): boolean => l.trim() === ""; + + const escapeMarker = (i: number): void => { + const line = lines[i] ?? ""; + lines[i] = line + .replace(/^(\s*)(\d+)([.)])(\s)/, "$1$2\\$3$4") + .replace(/^(\s*)([-*+])(\s)/, "$1\\$2$3"); + }; + + // Line-wise rule per inter-heading span: a list-marker line SURVIVES only + // when every non-blank line after it (to the span's end) is itself a list + // marker or an indented continuation — i.e. only the span's trailing list + // keeps real list syntax. This covers every measured death shape: a list + // followed by a paragraph, a second list, a tight lead-in block, and the + // corpus staple of a "Sources:" line directly under the final item (which + // kills the items but survives itself). + const processSpan = (start: number, end: number): void => { + // suffixClean[j]: every non-blank line in [j, end) is marker/indented. + const suffixClean: boolean[] = new Array(end - start + 1); + suffixClean[end - start] = true; + for (let j = end - 1; j >= start; j -= 1) { + const line = lines[j] ?? ""; + const rest = suffixClean[j - start + 1] ?? true; + suffixClean[j - start] = isBlank(line) ? rest : (isMarker(line) || isIndented(line)) && rest; + } + for (let j = start; j < end; j += 1) { + const line = lines[j] ?? ""; + if (isMarker(line) && !(suffixClean[j - start + 1] ?? true)) escapeMarker(j); + } + }; + + let spanStart = 0; + for (let i = 0; i <= lines.length; i += 1) { + if (i === lines.length || isHeading(lines[i] ?? "")) { + processSpan(spanStart, i); + spanStart = i + 1; + } + } + + return unmaskCode(lines.join("\n"), fences, inlines); +} + // --------------------------------------------------------------------------- // module_tree.json — hierarchy + user_guide merge // --------------------------------------------------------------------------- @@ -420,7 +633,12 @@ async function loadCorpusPages(dir: string): Promise { return Promise.all( entries.map(async (f) => ({ slug: f.slice(0, -3), - content: transpileMdx(stripLeadingDetailsBlock(await readFile(join(dir, f), "utf-8"))), + // Order matters: components demote first (Step/Accordion introduce real + // headings), preambles hoist next (this reshapes the heading spans), and + // list neutralization runs last against the final span layout. + content: neutralizeNonFinalLists( + hoistPreambles(transpileMdx(stripLeadingDetailsBlock(await readFile(join(dir, f), "utf-8")))), + ), })), ); }