fix(plugin-markdown): teach extractToc's emphasis rules CommonMark's flanking rule (objectui#7667)

.changeset/7667-toc-emphasis-flanking.md

@@ -0,0 +1,39 @@
+---
+'@object-ui/plugin-markdown': patch
+---
+
+Fix `extractToc` eating the underscores out of a `SCREAMING_SNAKE` heading, so
+its `#id` links resolve to the heading they name again (objectui#7667).
+
+`stripInline()` ended with two hand-rolled emphasis rules that gave `*` and `_`
+one shared regex — `(\*\*|__)(.*?)\1` and `(\*|_)(.*?)\1`. Neither knew
+CommonMark's flanking rule, under which a `_` run INSIDE a word opens nothing:
+it is both left- and right-flanking with no adjacent punctuation, so it may
+neither open nor close emphasis. The renderer obeys that and keeps the
+underscores; the shared rule paired the first two underscores of
+`### NON_GRID_ROW_CEILING` and ate `GRID`, then resumed and ate `ROW`. The TOC
+said `nongridrow_ceiling` while `rehype-slug` put `non_grid_row_ceiling` on the
+anchor, so the entry rendered, was clickable, and silently went nowhere.
+
+The underscore form is now its own flanking-aware rule and the asterisk rules
+are left alone, because only `_` carries the intraword exemption — giving `*`
+the same one would break `a*b*c`, which the renderer really does emphasise.
+Underscore runs are matched whole (`(?<!_)` / `(?!_)`), which is what keeps
+`x__init__y` literal, and a matched pair is dropped at whatever length it has,
+since it contributes no characters to the rendered text however it nests.
+
+The one-underscore case is a REGRESSION pin, not a repair: `## the snake_case
+name` was already correct — a lone underscore has nothing to pair with, which
+is why this went unnoticed for so long — and it still slugs
+`the-snake_case-name`. It takes two or more underscores in one heading for the
+old italic rule to find a pair.
+
+One live heading in this repository's own docs was affected
+(`packages/react/README.md:224`). Measured, not derived: the corpus sweep over
+`content/docs/**` plus every `packages/*/README.md` — 223 files, 2941 rendered
+headings — goes from 5 divergent files to 4, and the 4 that remain are a
+different, already-filed defect (objectui#7666, a heading `extractToc` lists
+that the renderer never emits under a JSX block). Pinned against the real
+render pipeline rather than a second derivation of the flanking rule: each
+heading is rendered through `MarkdownImpl` and `extractToc`'s id compared to
+the `id` attribute `rehype-slug` actually emitted.

packages/plugin-markdown/src/toc-anchor-parity.test.tsx

@@ -111,3 +111,99 @@ describe('extractToc ↔ rendered-anchor parity (objectui#7658)', () => {
     expect(tocIds(source)).toEqual(renderedHeadingIds(source));
   });
 });
+
+/**
+ * objectui#7667 — the emphasis rules and CommonMark's FLANKING rule.
+ *
+ * `*` and `_` are not interchangeable. `*` opens emphasis anywhere, including
+ * inside a word; a `_` run inside a word opens nothing, because it is both
+ * left- and right-flanking with no adjacent punctuation and CommonMark lets
+ * such a run neither open nor close. The renderer obeys that and keeps the
+ * underscores, so the ids only agree if `stripInline` does too.
+ *
+ * Same truth source as above: the `id` the real chain puts on the rendered
+ * heading, never a second derivation of the flanking rule.
+ */
+describe('extractToc ↔ rendered-anchor parity, emphasis flanking (objectui#7667)', () => {
+  /** Asserts parity AND that the shared expectation is the id named here. */
+  const bothAgree = (source: string, id: string) => {
+    // Reading the renderer is the lit control: `[]` here means the harness
+    // rendered nothing and the comparison below it would be vacuous.
+    expect(renderedHeadingIds(source)).toEqual([id]);
+    expect(tocIds(source)).toEqual([id]);
+  };
+
+  it('resolves the anchor for ### NON_GRID_ROW_CEILING (packages/react/README.md:224)', () => {
+    // The live instance. Before the fix the TOC said `nongridrow_ceiling`:
+    // the italic rule paired the 1st and 2nd underscores and ate `GRID`, then
+    // resumed past them and ate `ROW` — an id no anchor on the page carries.
+    bothAgree('### NON_GRID_ROW_CEILING\n', 'non_grid_row_ceiling');
+  });
+
+  it('leaves the ONE-underscore boundary exactly where it was', () => {
+    // Why this went unnoticed: a lone underscore has nothing to pair with, so
+    // ordinary `snake_case` prose was already correct. The fix must not move
+    // it — this case is a regression pin, not a repair.
+    bothAgree('## the snake_case name\n', 'the-snake_case-name');
+  });
+
+  it('keeps every intraword underscore run literal, whatever its length', () => {
+    for (const [md, id] of [
+      ['## A_B_C_D', 'a_b_c_d'], // 3 runs, so the naive rule paired two of them
+      ['## snake_case_word', 'snake_case_word'],
+      ['## MAX_ROWS vs MIN_ROWS', 'max_rows-vs-min_rows'],
+      ['## file_name.ts and other_name.ts', 'file_namets-and-other_namets'],
+      ['## trailing_underscore_', 'trailing_underscore_'], // nothing opened, so the closer stays
+      ['## x__init__y', 'x__init__y'], // a `__` run is intraword too
+    ] as const) {
+      bothAgree(`${md}\n`, id);
+    }
+  });
+
+  it('still strips underscore emphasis that CommonMark really opens', () => {
+    // The intraword exemption is not "underscores are inert" — a run flanked
+    // by whitespace or punctuation opens and closes exactly as before.
+    for (const [md, id] of [
+      ['## _em_ leading', 'em-leading'],
+      ['## __bold__ leading', 'bold-leading'],
+      ['## a _b_ c', 'a-b-c'],
+      ['## a __b__ c', 'a-b-c'],
+      ['## __init__', 'init'], // dunder at word boundaries DOES pair
+      ['## _leading and trailing_', 'leading-and-trailing'],
+      ['## _a_b_c_', 'a_b_c'], // outer runs pair; the inner two are intraword
+    ] as const) {
+      bothAgree(`${md}\n`, id);
+    }
+  });
+
+  it('leaves the asterisk forms alone — only `_` carries the exemption', () => {
+    // The counter-direction: a fix that gave `*` the same intraword exemption
+    // would be wrong here, because `*` opens emphasis inside a word.
+    for (const [md, id] of [
+      ['## *em* asterisk', 'em-asterisk'],
+      ['## **bold** asterisk', 'bold-asterisk'],
+      ['## a*b*c intraword asterisk', 'abc-intraword-asterisk'],
+      ['## a**b**c intraword asterisk bold', 'abc-intraword-asterisk-bold'],
+      ['## *a_b_c*', 'a_b_c'], // asterisk emphasis wrapping intraword underscores
+    ] as const) {
+      bothAgree(`${md}\n`, id);
+    }
+  });
+
+  it('agrees when the two markers meet in one heading', () => {
+    for (const [md, id] of [
+      ['## SOME_CONST and *em*', 'some_const-and-em'],
+      ['## snake_case and *em* mixed', 'snake_case-and-em-mixed'],
+      ['## snake_case *and* more_words', 'snake_case-and-more_words'],
+    ] as const) {
+      bothAgree(`${md}\n`, id);
+    }
+  });
+
+  it('control: a heading with no emphasis marker at all is untouched', () => {
+    // Lit control — a non-empty id that neither the old nor the new rule can
+    // move. A run in which this reads `[]` or drifts is a broken instrument
+    // rather than evidence about the flanking rule.
+    bothAgree('## Plain heading text\n', 'plain-heading-text');
+  });
+});

packages/plugin-markdown/src/toc.ts

@@ -31,6 +31,41 @@ const SLOT_CLOSE = "\uE001"
 const SLOT_RE = /\uE000(\d+)\uE001/g
 const SENTINEL_RE = /[\uE000\uE001]/g
 
+/**
+ * A matched pair of `_` delimiter runs, under CommonMark's FLANKING rule.
+ *
+ * `*` and `_` used to share one regex, and that is the bug: `*` opens emphasis
+ * anywhere, including inside a word, but a `_` run INSIDE a word opens nothing.
+ * It is both left- and right-flanking with no adjacent punctuation, and
+ * CommonMark lets such a run neither open nor close. The renderer obeys that
+ * and keeps the underscores, so `### NON_GRID_ROW_CEILING` is slugged
+ * `non_grid_row_ceiling`; the shared rule paired the first two underscores and
+ * ate `GRID`, then resumed and ate `ROW`, yielding `nongridrow_ceiling` — a
+ * `#id` naming an anchor the page does not carry (objectui#7667).
+ *
+ * Specialised to `_`, CommonMark's can-open / can-close conditions each reduce
+ * to one boundary test on either side of the whole RUN:
+ *
+ *   open  ⟺ preceded by start-of-text, whitespace or punctuation
+ *            AND followed by a non-whitespace character
+ *   close ⟺ followed by end-of-text, whitespace or punctuation
+ *            AND preceded by a non-whitespace character
+ *
+ * so `[^\s\p{P}\p{S}]` — neither Unicode whitespace nor CommonMark's Unicode
+ * punctuation (categories P and S) — is the character class both lookarounds
+ * negate. `(?<!_)` / `(?!_)` anchor each match to a WHOLE run, which is what
+ * keeps `x__init__y` literal instead of pairing that run's inner underscores.
+ * A run is consumed at whatever length it has, because a matched pair
+ * contributes no characters to the rendered text however it nests
+ * (`___x___` → `<em><strong>x</strong></em>` → `x`).
+ *
+ * Only the underscore form is flanking-aware: the asterisk rules above it are
+ * unchanged, since giving `*` the same exemption would break `a*b*c`, which
+ * the renderer really does emphasise.
+ */
+const UNDERSCORE_EMPHASIS_RE =
+  /(?<![^\s\p{P}\p{S}])(?<!_)(_+)(?!\s)(.+?)(?<!\s)(_+)(?!_)(?![^\s\p{P}\p{S}])/gu
+
 /**
  * Strip the inline-markdown wrappers so the text matches `rehype-slug`'s.
  *
@@ -58,8 +93,9 @@ function stripInline(s: string): string {
     }) // inline code → an opaque slot
     .replace(/!\[[^\]]*\]\([^)]*\)/g, "") // images
     .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1") // links → text
-    .replace(/(\*\*|__)(.*?)\1/g, "$2") // bold
-    .replace(/(\*|_)(.*?)\1/g, "$2") // italic
+    .replace(/\*\*(.*?)\*\*/g, "$1") // bold, asterisk form
+    .replace(/\*(.*?)\*/g, "$1") // italic, asterisk form
+    .replace(UNDERSCORE_EMPHASIS_RE, "$2") // emphasis, underscore form
     .replace(/<[^>]+>/g, "") // raw html
     .replace(SLOT_RE, (_match, index: string) => codeSpans[Number(index)]) // code spans, verbatim
     .trim()