casei: close the Unicode field gaps

CONTEXT.md

@@ -56,26 +56,26 @@ substring engine — and it is absent from every caseless benchmark: no
 dedicated caseless substring engine exists in that suite at all; the
 caseless columns are contested only by general regex engines.
 
-**Direct rebar audit (2026-08-17):** `casei` was wired into every caseless
-literal or finite-alternation definition at rebar commit `463d00f`: 18
+**Direct Rebar audit (2026-08-24):** `casei` was wired into every caseless
+literal or finite-alternation definition at Rebar commit `463d00f`: 18
 performance workloads and three semantic checks. Rebar's pinned
 [performance models](https://github.com/BurntSushi/rebar/blob/463d00f31887e84c38467805b9e3122c314b9521/MODELS.md)
-enumerate every non-overlapping match, rather than return the first. Five rows
-enable Unicode folding and therefore share this repository's folding contract;
-against the selected current leaders, the loop-over-`Find` adapter wins two and
-loses three on both Ice Lake and Sapphire Rapids, with the five-pattern Russian
-row losing by roughly 9× to Hyperscan. Thirteen additional performance rows
-request ASCII-only case matching; they were run and output-verified, but
-`casei` retains stronger Unicode semantics. [`REBAR.md`](REBAR.md) records
-every row, both-host ratios,
-the incompatible `s`/`ſ` behavior check, and the causal diagnosis. Instrumented
-reproduction showed that a stateful one-pass enumerator did not improve the 9×
-row: the five-pattern plan filters on common Cyrillic roots, admits 21.7% of the
-corpus into rune decoding and token-map verification, and never gains the rare
-interior pair-pair anchors selected for a single pattern. The missing iterator
-is therefore an API gap, not the primary performance gap. These measurements
-bound the result here to the 33-row arena contract; Rebar's non-overlapping
-enumeration numbers cannot be borrowed in support of it.
+enumerate every non-overlapping match. Five rows enable Unicode folding and
+share this repository's contract. The first audit won two and lost three on
+both Ice Lake and Sapphire Rapids; its five-pattern Russian row was roughly 9×
+behind Hyperscan. Instrumentation put the loss in a shared filter over common
+Cyrillic starts and decoded confirmation, rather than in repeated API setup.
+
+That result opened the missing cells. The retained follow-up combines tagged
+interior anchors, exact pair replay, one/two/three-byte raw confirmation with
+source-width return, and an exact common-byte origin gate. The current adapter
+uses `Matcher.Each` and wins all five same-contract rows on both hosts. Its
+worst ratios are 0.8794 on Ice Lake and 0.8999 on Sapphire Rapids. Thirteen
+additional rows request ASCII-only matching; they remain visible as
+different-contract stress data, with `casei` winning 9 of all 18 rows on each
+host. [`REBAR.md`](REBAR.md) records the before/after mechanisms, complete
+table, incompatible `s`/`ſ` behavior check, and raw receipts. The arena now
+contains three focused rows derived from the gap and has 36 rows in total.
 
 **Correction (v2, after a three-way prior-art sweep):** dedicated engines
 DO exist, with different contracts:

arena/bar_test.go

@@ -8,6 +8,7 @@ package arena_test
 
 import (
 	"fmt"
+	"sort"
 	"testing"
 	"time"
 
@@ -21,26 +22,40 @@ import (
 	vectorscan "github.com/tsenart/casei/arena/vectorscan"
 )
 
-// timeOp returns ns/op for one operation. It times manually rather than
-// through testing.Benchmark, which cannot be nested inside a running
-// benchmark, and takes the best of three samples to reduce sensitivity to
-// transient host load.
-func timeOp(op func()) float64 {
+// timeWindow returns ns/op for one manually timed window. testing.Benchmark
+// cannot be nested inside a running benchmark.
+func timeWindow(op func()) float64 {
 	const budget = 25 * time.Millisecond
-	best := 0.0
-	for sample := 0; sample < 3; sample++ {
-		n := 0
-		start := time.Now()
-		for time.Since(start) < budget {
-			op()
-			n++
-		}
-		ns := float64(time.Since(start).Nanoseconds()) / float64(n)
-		if best == 0 || ns < best {
-			best = ns
+	n := 0
+	start := time.Now()
+	for time.Since(start) < budget {
+		op()
+		n++
+	}
+	return float64(time.Since(start).Nanoseconds()) / float64(n)
+}
+
+// pairedRatio measures the candidate beside one competitor six times, with
+// each operation going first in three pairs. The median paired ratio reduces
+// drift from CPU migration, frequency changes, and host load between timing
+// windows.
+func pairedRatio(candidate, competitor func()) float64 {
+	candidate()
+	competitor()
+	var ratios [6]float64
+	for round := range ratios {
+		var candidateNS, competitorNS float64
+		if round%2 == 0 {
+			candidateNS = timeWindow(candidate)
+			competitorNS = timeWindow(competitor)
+		} else {
+			competitorNS = timeWindow(competitor)
+			candidateNS = timeWindow(candidate)
 		}
+		ratios[round] = candidateNS / competitorNS
 	}
-	return best
+	sort.Float64s(ratios[:])
+	return (ratios[2] + ratios[3]) / 2
 }
 
 // velozVectorBits is deliberately strict: Veloz has an SSE/scalar fallback,
@@ -132,45 +147,44 @@ func BenchmarkBar(b *testing.B) {
 	for _, s := range scenarios {
 		s := s
 		b.Run("single/"+s.name, func(b *testing.B) {
-			cand := timeOp(func() { sink = runSingleScenario(casei.IndexFold, s) })
-
-			best := timeOp(func() { sink = runSingleScenario(indexRegexp, s) })
+			candidate := func() { sink = runSingleScenario(casei.IndexFold, s) }
+			best := pairedRatio(candidate, func() { sink = runSingleScenario(indexRegexp, s) })
 			competitors := 1
-			if v := timeOp(func() { sink = runSingleScenario(indexPCRE2, s) }); v < best {
-				best = v
+			if ratio := pairedRatio(candidate, func() { sink = runSingleScenario(indexPCRE2, s) }); ratio > best {
+				best = ratio
 			}
 			competitors++
 			rure := rureSingles[s.needle]
-			rureTime := timeOp(func() { sink = runSingleScenario(indexRure, s) })
+			rureRatio := pairedRatio(candidate, func() { sink = runSingleScenario(indexRure, s) })
 			// The Rust adapter records the backend reached by this exact query.
 			// A query that did not reach memchr AVX2 is diagnostic only; it must
 			// not race a target-width field entrant under a CPU-flag label.
 			if rure.VectorBits() == 256 {
-				if rureTime < best {
-					best = rureTime
+				if rureRatio > best {
+					best = rureRatio
 				}
 				competitors++
 			}
-			if v := timeOp(func() { sink = runSingleScenario(indexVectorscan, s) }); v < best {
-				best = v
+			if ratio := pairedRatio(candidate, func() { sink = runSingleScenario(indexVectorscan, s) }); ratio > best {
+				best = ratio
 			}
 			competitors++
 			if stringZillaAvailable {
-				if v := timeOp(func() { sink = runSingleScenario(indexStringZilla, s) }); v < best {
-					best = v
+				if ratio := pairedRatio(candidate, func() { sink = runSingleScenario(indexStringZilla, s) }); ratio > best {
+					best = ratio
 				}
 				competitors++
 			}
 			if !s.utf8 && velozVectorBits() == 256 {
-				if v := timeOp(func() { sink = runSingleScenario(veloz.IndexFold, s) }); v < best {
-					best = v
+				if ratio := pairedRatio(candidate, func() { sink = runSingleScenario(veloz.IndexFold, s) }); ratio > best {
+					best = ratio
 				}
 				competitors++
 			}
 			for b.Loop() {
-				sink = runSingleScenario(casei.IndexFold, s)
+				candidate()
 			}
-			b.ReportMetric(cand/best, "x_vs_best")
+			b.ReportMetric(best, "x_vs_best")
 			b.ReportMetric(float64(competitors), "competitors")
 			b.ReportMetric(float64(competitors+1), "entrants")
 			reportSingleDispatch(b, s)
@@ -182,58 +196,58 @@ func BenchmarkBar(b *testing.B) {
 		scenarioIndex := scenarioIndex
 		b.Run("multi/"+s.name, func(b *testing.B) {
 			m := casei.NewMatcher(s.patterns)
-			cand := timeOp(func() { _, matcherFound = m.Find(s.haystack) })
+			candidate := func() { _, matcherFound = m.Find(s.haystack) }
 
 			re := regexpAltFor(s.patterns)
-			best := timeOp(func() { matcherSink = len(re.FindStringIndex(s.haystack)) })
+			best := pairedRatio(candidate, func() { matcherSink = len(re.FindStringIndex(s.haystack)) })
 			competitors := 1
 			pcre := pcre2Alts[scenarioIndex]
-			if v := timeOp(func() { _, _, matcherFound = pcre.Find(s.haystack) }); v < best {
-				best = v
+			if ratio := pairedRatio(candidate, func() { _, _, matcherFound = pcre.Find(s.haystack) }); ratio > best {
+				best = ratio
 			}
 			competitors++
 			rure := rureAlts[scenarioIndex]
-			rureTime := timeOp(func() { _, _, matcherFound = rure.Find(s.haystack) })
+			rureRatio := pairedRatio(candidate, func() { _, _, matcherFound = rure.Find(s.haystack) })
 			if rure.VectorBits() == 256 {
-				if rureTime < best {
-					best = rureTime
+				if rureRatio > best {
+					best = rureRatio
 				}
 				competitors++
 			}
 			vscan := vectorscanAlts[scenarioIndex]
-			if v := timeOp(func() { _, _, matcherFound = vscan.Find(s.haystack) }); v < best {
-				best = v
+			if ratio := pairedRatio(candidate, func() { _, _, matcherFound = vscan.Find(s.haystack) }); ratio > best {
+				best = ratio
 			}
 			competitors++
 			if stringZillaAvailable {
 				stringzilla := stringZillaAlts[scenarioIndex]
-				if v := timeOp(func() { _, _, matcherFound = stringzilla.Find(s.haystack) }); v < best {
-					best = v
+				if ratio := pairedRatio(candidate, func() { _, _, matcherFound = stringzilla.Find(s.haystack) }); ratio > best {
+					best = ratio
 				}
 				competitors++
 			}
 			supplemental := 0
 			rust := rustACAlts[scenarioIndex]
 			if !s.utf8 {
-				rustTime := timeOp(func() { _, _, matcherFound = rust.Find(s.haystack) })
+				rustRatio := pairedRatio(candidate, func() { _, _, matcherFound = rust.Find(s.haystack) })
 				// The direct Rust DFA exposes the memchr backend reached by this
 				// exact prefilter query. Do not call an unobserved scalar/SSE path
 				// an AVX2 field entrant merely because this process has AVX2.
 				if rust.VectorBits() == 256 {
-					if rustTime < best {
-						best = rustTime
+					if rustRatio > best {
+						best = rustRatio
 					}
 					competitors++
 				}
 
 				goAC := acBuild(s.patterns, true)
-				_ = timeOp(func() { _, matcherFound = acFirst(&goAC, s.haystack) })
+				_ = pairedRatio(candidate, func() { _, matcherFound = acFirst(&goAC, s.haystack) })
 				supplemental++
 			}
 			for b.Loop() {
-				_, matcherFound = m.Find(s.haystack)
+				candidate()
 			}
-			b.ReportMetric(cand/best, "x_vs_best")
+			b.ReportMetric(best, "x_vs_best")
 			b.ReportMetric(float64(competitors), "competitors")
 			b.ReportMetric(float64(competitors+1+supplemental), "entrants")
 			reportMultiDispatch(b, s, m.VectorBits(), rure, rust, vscan)

arena/matcher_bench_test.go

@@ -52,6 +52,56 @@ func genNeedles(n int, format string) []string {
 	return out
 }
 
+// unicodePairConfirmMissCorpus repeats a near-full-width false match at the
+// pair-pair anchor density measured on the Russian corpus. It exercises the
+// N=1 Matcher path without giving any engine a true-match early exit.
+func unicodePairConfirmMissCorpus() string {
+	const needle = "приключения лилий"
+	const haystackBytes = 1_570_556
+	const survivors = 2_134
+
+	falseLiteral := needle[:len(needle)-len("й")] + "я"
+	bytes := []byte(strings.Repeat("x", haystackBytes))
+	step := len(bytes) / survivors
+	inserted := 0
+	for at := 0; at+len(falseLiteral) <= len(bytes) && inserted < survivors; at += step {
+		copy(bytes[at:], falseLiteral)
+		inserted++
+	}
+	if inserted != survivors {
+		panic(fmt.Sprintf("inserted %d false survivors, want %d", inserted, survivors))
+	}
+	return string(bytes)
+}
+
+// rawTransitionPatterns are ordinary Unicode simple-fold literals with distinct
+// Cyrillic roots. The companion rows keep an admitted root followed by an ASCII
+// mismatch at a fixed density, and add one late full literal for the hit case.
+// They exercise a shared multi-pattern plan against every native field entrant;
+// no production path observes these fixture names or values.
+var rawTransitionPatterns = []string{
+	"Шерлок Холмс",
+	"Джон Уотсон",
+	"Ирен Адлер",
+	"инспектор Лестрейд",
+	"профессор Мориарти",
+}
+
+func rawTransitionCorpus(bytes int, lateMatch bool) string {
+	const period = 256
+	if bytes < period {
+		panic("raw transition corpus is shorter than its period")
+	}
+	out := []byte(strings.Repeat("x", ((bytes+period-1)/period)*period))
+	for at := 0; at+len("Дx") <= len(out); at += period {
+		copy(out[at:], "Дx")
+	}
+	if lateMatch {
+		copy(out[len(out)-period:], rawTransitionPatterns[0])
+	}
+	return string(out)
+}
+
 var multiScenarios = func() []multiScenario {
 	logs1m := buildLogCorpus(1 << 20)
 	prose1m := buildProseCorpus(1 << 20)
@@ -79,6 +129,9 @@ var multiScenarios = func() []multiScenario {
 		// about the tier this repository exists for.
 		{"multi_N512_miss_hazard_64kb", cyr1m[:64<<10], genHazardNeedles(512), true},
 		{"multi_N8_hit_log_1mb", plant(logs1m, "Payment Declined", 4), hit8, false},
+		{"multi_N1_unicode_pair_miss_1_5mb", unicodePairConfirmMissCorpus(), []string{"приключения лилий"}, true},
+		{"multi_N5_raw_transition_miss_5mb", rawTransitionCorpus(5<<20, false), rawTransitionPatterns, true},
+		{"multi_N5_raw_transition_late_hit_5mb", rawTransitionCorpus(5<<20, true), rawTransitionPatterns, true},
 		{"multi_N8_miss_ru_1mb", cyr1m, genNeedles(8, "щупальце%d"), true},
 		{"multi_N64_miss_ru_64kb", cyr1m[:64<<10], genNeedles(64, "щупальце%d"), true},
 		// This is the all-ASCII half of the mixed-fold hazard set. It keeps the