From 6038410c4d21a99554eafaa4098c94db43183a6d Mon Sep 17 00:00:00 2001 From: Adam Banko Date: Wed, 22 Jul 2026 21:27:36 +0200 Subject: [PATCH] extradoc: word-atomic in-paragraph diff for readable redlines Diff paragraph bodies at word-token granularity so multi-word edits stay single readable chunks in the Docs Suggestions UI instead of char-level fragment-soup, with a char-level fallback for whitespace-only edits (e.g. "PARTI" -> "PART I") so single-space changes stay minimal. Adds regression tests: a one-word change stays one atomic replace opcode (char-level difflib would fragment "brown"->"red" on a spurious shared 'r'); a whitespace-only change falls back to a minimal char-level insert. --- extradoc/src/extradoc/reconcile_v3/lower.py | 72 ++++++++++++++++++++- extradoc/tests/reconcile_v3/test_lower.py | 56 ++++++++++++++++ 2 files changed, 125 insertions(+), 3 deletions(-) diff --git a/extradoc/src/extradoc/reconcile_v3/lower.py b/extradoc/src/extradoc/reconcile_v3/lower.py index d7c0b88..5681e92 100644 --- a/extradoc/src/extradoc/reconcile_v3/lower.py +++ b/extradoc/src/extradoc/reconcile_v3/lower.py @@ -32,6 +32,7 @@ import copy import difflib +import re from itertools import groupby from typing import TYPE_CHECKING @@ -2307,6 +2308,72 @@ def _abs(cp_idx: int) -> int: return result +_WORD_TOKEN_RE = re.compile(r"\w+|[^\w]") +_WHITESPACE_RE = re.compile(r"\s+") + + +def _tokenize_words(text: str) -> list[tuple[str, int, int]]: + """Split *text* into (token, start_cp, end_cp) triples. + + Maximal runs of word characters (``\\w+``, includes digits/underscore) + form one token each; every other character (whitespace, punctuation) is + its own single-character token. Offsets are code-point-based, matching + ``difflib.SequenceMatcher`` output. + """ + return [(m.group(0), m.start(), m.end()) for m in _WORD_TOKEN_RE.finditer(text)] + + +def _word_diff_opcodes( + base_body: str, desired_body: str +) -> list[tuple[str, int, int, int, int]]: + """Word-atomic diff with a char-level fallback for whitespace-only edits. + + Diffing at word-token granularity keeps multi-word inserts/deletes as + single readable chunks (no char-level fragment-soup in the Docs + Suggestions UI). But a pure word-level diff would replace an entire + token when only whitespace was inserted/removed inside it (e.g. + "PARTI" -> "PART I"), destroying the fine-grained single-space edits + downstream code (and callers) rely on. So: for any non-equal opcode + whose base/desired chunks are identical once whitespace is stripped, + fall back to a char-level sub-diff of just that chunk. + """ + base_tokens = _tokenize_words(base_body) + desired_tokens = _tokenize_words(desired_body) + base_tok_strs = [t[0] for t in base_tokens] + desired_tok_strs = [t[0] for t in desired_tokens] + + matcher = difflib.SequenceMatcher( + None, base_tok_strs, desired_tok_strs, autojunk=False + ) + + result: list[tuple[str, int, int, int, int]] = [] + for tag, ti1, ti2, tj1, tj2 in matcher.get_opcodes(): + i1 = base_tokens[ti1][1] if ti1 < len(base_tokens) else len(base_body) + i2 = base_tokens[ti2 - 1][2] if ti2 > ti1 else i1 + j1 = desired_tokens[tj1][1] if tj1 < len(desired_tokens) else len(desired_body) + j2 = desired_tokens[tj2 - 1][2] if tj2 > tj1 else j1 + + if tag == "equal": + result.append((tag, i1, i2, j1, j2)) + continue + + base_chunk = base_body[i1:i2] + desired_chunk = desired_body[j1:j2] + if _WHITESPACE_RE.sub("", base_chunk) == _WHITESPACE_RE.sub("", desired_chunk): + # Whitespace-only rearrangement: refine at char level so the + # edit stays minimal (e.g. a single inserted/removed space). + sub_matcher = difflib.SequenceMatcher( + None, base_chunk, desired_chunk, autojunk=False + ) + for stag, si1, si2, sj1, sj2 in sub_matcher.get_opcodes(): + result.append((stag, i1 + si1, i1 + si2, j1 + sj1, j1 + sj2)) + continue + + result.append((tag, i1, i2, j1, j2)) + + return result + + def _diff_paragraph_runs( *, base_para: Paragraph, @@ -2360,9 +2427,8 @@ def _diff_paragraph_runs( # (emoji, mathematical bold, etc.) are 1 code point but 2 UTF-16 units. base_cp_to_utf16 = _build_cp_to_utf16(base_body) - # Compute character-level diff - matcher = difflib.SequenceMatcher(None, base_body, desired_body, autojunk=False) - opcodes = matcher.get_opcodes() + # Compute word-atomic diff (char-level fallback for whitespace-only edits). + opcodes = _word_diff_opcodes(base_body, desired_body) # Collect pending ops as (abs_start, abs_end, kind, extra) # kind ∈ {"delete", "insert", "update_style", "replace"} diff --git a/extradoc/tests/reconcile_v3/test_lower.py b/extradoc/tests/reconcile_v3/test_lower.py index a86ff36..c651dff 100644 --- a/extradoc/tests/reconcile_v3/test_lower.py +++ b/extradoc/tests/reconcile_v3/test_lower.py @@ -1689,6 +1689,62 @@ def test_end_to_end_styled_paragraph_surgical_ops(self) -> None: ) +# =========================================================================== +# Part 7: Word-atomic diffing +# =========================================================================== + + +class TestWordAtomicDiff: + """_word_diff_opcodes: whole-word replace ops, char-level only for whitespace.""" + + def test_single_word_replace_is_one_atomic_opcode(self) -> None: + """A one-word change stays a single word-level replace opcode. + + Plain char-level ``difflib.SequenceMatcher`` (the pre-fix behavior) + finds a spurious shared 'r' between "brown" and "red" and fragments + this into three opcodes (delete "b", equal "r", replace "own"->"ed"). + The word-atomic diff must instead treat "brown" and "red" as opaque + tokens and emit exactly one replace opcode for the whole word. + """ + from extradoc.reconcile_v3.lower import _word_diff_opcodes + + base = "The quick brown fox jumps over\n" + desired = "The quick red fox jumps over\n" + + opcodes = _word_diff_opcodes(base, desired) + non_equal = [op for op in opcodes if op[0] != "equal"] + + assert len(non_equal) == 1, f"Expected one atomic word replace, got {non_equal}" + tag, i1, i2, j1, j2 = non_equal[0] + assert tag == "replace" + assert base[i1:i2] == "brown" + assert desired[j1:j2] == "red" + + def test_whitespace_only_change_falls_back_to_char_diff(self) -> None: + """Inserting a space inside a token stays a minimal char-level edit. + + A naive word-level diff would treat "PARTI" and "PART I" as wholly + different tokens and replace the whole word. Since they're identical + once whitespace is stripped, the fallback must instead emit a + minimal single-character insert of the space. + """ + from extradoc.reconcile_v3.lower import _word_diff_opcodes + + base = "PARTI\n" + desired = "PART I\n" + + opcodes = _word_diff_opcodes(base, desired) + non_equal = [op for op in opcodes if op[0] != "equal"] + + assert len(non_equal) == 1, ( + f"Expected one minimal char-level insert, got {non_equal}" + ) + tag, i1, i2, j1, j2 = non_equal[0] + assert tag == "insert" + assert base[i1:i2] == "" + assert desired[j1:j2] == " " + + # =========================================================================== # Part 7: Footnote lowering # ===========================================================================