From 317a148fbae21ad1bc3b49084de9da709d6621db Mon Sep 17 00:00:00 2001 From: us Date: Sun, 19 Jul 2026 02:48:01 +0300 Subject: [PATCH] feat(search): passage-select + snippet-first for the answer path Add two gated levers to the /v1/search LLM answer synthesis, both default off so prod behavior is unchanged until flipped (A/B keep/revert; off = byte-identical to the prior path). - answer_bm25_select: when a scraped source exceeds the per-source byte budget, keep its query-relevant passages (sentence-chunked, BM25-ranked, reusing the existing chunker/ranker) instead of a blind head-truncation that drops answers sitting deep in the page. Two-pass with a partial-fill tail so it never feeds the model less than a head-truncation would (monotone on recall); non-adjacent passages are joined with a gap marker. - snippet_fallback is now snippet-first: prepend the SERP snippet to every answer source and block-guard the scraped body, so a fetched-but-blocked page (a "Wikimedia Error"/bot-wall shell) falls back to the clean snippet instead of grounding the answer on error text. The legacy path (flag off) is byte-identical, so the multi-round scout still fires on such sources. Also parse citations from the LAST ===CITATIONS=== marker (rsplit) so a source that quotes the marker cannot truncate the answer or divert parsing. --- crates/crw-core/src/config.rs | 29 +++- crates/crw-extract/src/answer.rs | 191 ++++++++++++++++++++++++- crates/crw-server/src/routes/search.rs | 90 ++++++++++-- 3 files changed, 287 insertions(+), 23 deletions(-) diff --git a/crates/crw-core/src/config.rs b/crates/crw-core/src/config.rs index 89f76b38..cecbbe76 100644 --- a/crates/crw-core/src/config.rs +++ b/crates/crw-core/src/config.rs @@ -311,6 +311,16 @@ pub struct SearchConfig { /// `false` (gated); answer prompt + plain path untouched. #[serde(default)] pub passage_select: bool, + /// Cheap BM25 variant of `passage_select` for the plain answer path: when a + /// scraped source exceeds `max_chars_per_source`, keep its query-relevant + /// passages (sentence-chunked, BM25-ranked, no LLM call) instead of a blind + /// head-truncation that drops answers buried deep in the page. Two-pass so it + /// NEVER feeds less than a head-truncation (fills the budget after ranking), + /// so it is monotone-safe on recall. Defaults to `false` (gated); off = + /// byte-identical head-truncation, A/B keep/revert in prod. Ignored when + /// `passage_select` is on (that path LLM-reduces sources instead). + #[serde(default)] + pub answer_bm25_select: bool, /// Page-2 fallback for the LLM answer / summarize path: if the reranked /// (junk-filtered, deduped) candidate pool comes back thinner than the /// answer needs (`< answer_top_n`), fetch the SAME query's SearXNG page 2 @@ -362,13 +372,17 @@ pub struct SearchConfig { /// the wrong-non-abstain invariant before flip. #[serde(default)] pub wikidata_lookup: bool, - /// Snippet fallback for the LLM answer path (gated): when a top-N result's - /// scrape failed (empty `markdown`), the result is normally dropped from the - /// answer pool — if it was the answer-bearing page, crw abstains though - /// retrieval succeeded (diagnosed Pattern A). With this on, such results - /// fall back to their SearXNG `description` snippet as a thin source instead - /// of vanishing. The snippet is verbatim upstream text, so it cannot inject - /// a fact not already present — near-zero INCORRECT exposure. Default false. + /// Snippet-first grounding for the LLM answer path (gated). With this on, the + /// SearXNG `description` snippet is prepended to EVERY answer source as + /// `[snippet] \n\n` (not merely a fallback for failed scrapes): + /// the snippet is the engine's own query-relevant answer passage, so putting + /// it first means it survives the per-source passage budget. It also block- + /// guards the body — a fetched-but-blocked page ("Wikimedia Error", a bot + /// wall) is dropped in favor of the clean snippet. When a scrape returned no + /// markdown at all, the snippet still keeps the result in the pool instead of + /// dropping the (possibly answer-bearing) page. The snippet is verbatim + /// upstream text, so it cannot inject a fact not already present — near-zero + /// INCORRECT exposure. Default false; off = markdown-only (legacy), A/B in prod. #[serde(default)] pub snippet_fallback: bool, /// Relevance gate for the LLM answer / summarize re-rank (gated). After the @@ -415,6 +429,7 @@ impl Default for SearchConfig { pipeline_overlap: false, multi_round: false, passage_select: false, + answer_bm25_select: false, page2_fallback: false, answer_calibrated: false, answer_guarded: false, diff --git a/crates/crw-extract/src/answer.rs b/crates/crw-extract/src/answer.rs index 73a6246e..39643c7d 100644 --- a/crates/crw-extract/src/answer.rs +++ b/crates/crw-extract/src/answer.rs @@ -8,9 +8,10 @@ use crate::llm::{self, LlmCallResult}; use crate::untrusted; +use crate::{chunking, filter}; use crw_core::config::LlmConfig; use crw_core::error::{CrwError, CrwResult}; -use crw_core::types::{Citation, LlmUsage}; +use crw_core::types::{ChunkStrategy, Citation, FilterMode, LlmUsage}; /// Per-source server-side hard ceiling. The request's /// `max_chars_per_source` is clamped to this regardless of value. @@ -208,6 +209,107 @@ fn truncate_on_char_boundary(s: &str, max_bytes: usize) -> &str { &s[..idx] } +/// Worst-case separator between two kept chunks ("\n[...]\n"). Accounted +/// conservatively so the assembled output never exceeds `cap` before the final +/// truncate; the leftover slack is reclaimed by the partial-fill step. +const GAP_MARKER: &str = "\n[...]\n"; + +/// Add whole chunk `i` to `keep` if it fits the remaining byte budget (charging +/// the worst-case separator). Whole-chunk only; the partial tail is handled by +/// the caller's fill step. +fn try_keep_chunk( + chunks: &[String], + keep: &mut std::collections::BTreeSet, + used: &mut usize, + i: usize, + cap: usize, +) { + if keep.contains(&i) { + return; + } + let clen = chunks[i].len(); + if *used + GAP_MARKER.len() + clen > cap { + return; + } + keep.insert(i); + *used += GAP_MARKER.len() + clen; +} + +/// Fit an over-budget source into `cap` bytes by RELEVANCE, not by position. +/// A blind head-truncation drops the answer when it sits deep in the page (a +/// stats table at char 12k, a fact at char 55k); scoring passages against the +/// query and keeping the best ones recovers those. Reuses the engine's sentence +/// chunker + BM25 ranker. +/// +/// Two passes, so this NEVER feeds the model less content than a plain +/// head-truncation would (the hard "don't regress recall" invariant): first pack +/// the highest-BM25 chunks (the answer-bearing passage, even if deep, gets in +/// ahead of filler), then FILL the remaining budget with the rest in original +/// order. The lead chunk (page lede / SERP snippet) is always kept. Non-adjacent +/// kept chunks are joined with a `[...]` gap marker so the model does not read two +/// distant passages as one continuous span. Falls back to head-truncation if the +/// text won't chunk. +fn select_relevant_passages(md: &str, query: &str, cap: usize) -> String { + if md.len() <= cap || query.trim().is_empty() { + return truncate_on_char_boundary(md, cap).to_string(); + } + let strategy = ChunkStrategy::Sentence { + max_chars: Some(700), + overlap_chars: None, + dedupe: Some(false), + }; + let chunks = chunking::chunk_text(md, &strategy); + if chunks.is_empty() { + return truncate_on_char_boundary(md, cap).to_string(); + } + let scored = filter::filter_chunks_scored(&chunks, query, &FilterMode::Bm25, chunks.len()); + let mut keep: std::collections::BTreeSet = std::collections::BTreeSet::new(); + keep.insert(0); // lead chunk always kept: page lede / snippet lives here + let mut used = chunks[0].len(); + // Priority pass: query-relevant chunks first (BM25 > 0), highest score first. + for sc in &scored { + if sc.score > 0.0 { + try_keep_chunk(&chunks, &mut keep, &mut used, sc.index, cap); + } + } + // Fill pass: pack the rest in original order until the budget is full, so the + // model never sees less than a head-truncation would have given it. + for i in 0..chunks.len() { + try_keep_chunk(&chunks, &mut keep, &mut used, i, cap); + } + // Join in original order; mark gaps between non-adjacent kept chunks. + let mut out = String::new(); + let mut prev: Option = None; + for &i in &keep { + if let Some(p) = prev { + out.push_str(if i == p + 1 { "\n" } else { GAP_MARKER }); + } + out.push_str(&chunks[i]); + prev = Some(i); + } + // Partial-fill: if budget remains (e.g. an unselected chunk was an unbreakable + // blob too big to keep whole, or slack from conservative separator charging), + // append a byte-slice of the best remaining chunk so we never feed LESS than a + // head-truncation would. Prefer the highest-BM25 unselected chunk, else the + // first in original order. + if out.len() + GAP_MARKER.len() < cap { + let next = scored + .iter() + .map(|s| s.index) + .chain(0..chunks.len()) + .find(|i| !keep.contains(i)); + if let Some(i) = next { + let room = cap - out.len() - GAP_MARKER.len(); + let slice = truncate_on_char_boundary(&chunks[i], room); + if !slice.is_empty() { + out.push_str(GAP_MARKER); + out.push_str(slice); + } + } + } + truncate_on_char_boundary(&out, cap).to_string() // hard-enforce the byte cap +} + /// Hard server-side cap on the caller-supplied prompt addition. See /// `crate::summary::MAX_USER_PROMPT_CHARS` for rationale. pub const MAX_USER_PROMPT_CHARS: usize = 500; @@ -226,6 +328,10 @@ pub async fn synthesize( calibrated: bool, guarded: bool, list_format: bool, + // When false, over-budget sources are head-truncated (byte-identical to the + // pre-passage-select behavior). When true, they are reduced to their + // query-relevant passages. Gated by `search.answer_bm25_select` (default off). + bm25_select: bool, ) -> CrwResult { if sources.is_empty() { return Err(CrwError::InvalidRequest( @@ -243,7 +349,14 @@ pub async fn synthesize( if was_truncated { any_truncated = true; } - let body = truncate_on_char_boundary(md, cap); + // Relevance-select passages when over budget (keeps deep answers a blind + // head-cut would drop); within budget this is a no-op passthrough. Gated: + // off = byte-identical head-truncation. + let body = if bm25_select { + select_relevant_passages(md, query, cap) + } else { + truncate_on_char_boundary(md, cap).to_string() + }; let source_block = format!("Source #{idx}\nURL: {url}\nTitle: {title}\n\n{body}"); parts.push(untrusted::wrap(&source_block, "SOURCE", &nonce, Some(idx))); } @@ -437,6 +550,7 @@ pub async fn synthesize_selected( calibrated, guarded, list_format, + false, // sources already LLM-reduced; head-truncate the remainder ) .await } @@ -446,7 +560,10 @@ fn parse_answer_and_citations( sources: &[Source], ) -> (String, Vec, Vec) { let mut warnings = Vec::new(); - let Some((answer_part, cite_part)) = raw.split_once("===CITATIONS===") else { + // rsplit: the model's citations block is always LAST, so split on the final + // marker — a source that itself quotes "===CITATIONS===" cannot then truncate + // the answer or divert the citation parse to an earlier fake block. + let Some((answer_part, cite_part)) = raw.rsplit_once("===CITATIONS===") else { warnings.push("model omitted citations marker; returning answer without citations".into()); return (raw.trim().to_string(), Vec::new(), warnings); }; @@ -661,4 +778,72 @@ mod tests { assert!(both.contains("ranked list")); assert!(both.contains("give the direct answer confidently")); } + + #[test] + fn passage_select_keeps_deep_answer_within_cap() { + let lead = "Intro paragraph about the football season."; + let filler = (0..80) + .map(|i| format!("Filler sentence {i} about various unrelated topics and clubs.")) + .collect::>() + .join(" "); + let answer = "Leeds United finished with 38 points at the end of the season."; + let page = format!("{lead} {filler} {answer}"); + let cap = 1500; + assert!(page.len() > cap); + let out = select_relevant_passages(&page, "Which team finished with 38 points", cap); + assert!(out.len() <= cap, "must respect the byte cap"); + assert!( + out.contains("Leeds United"), + "deep answer must survive relevance selection" + ); + } + + #[test] + fn passage_select_fills_budget_never_less_than_head_truncation() { + // No query-term overlap anywhere -> BM25 all-zero. The fill pass must still + // pack the budget (never feed the model less than a head-truncation would). + let page = "Completely unrelated boilerplate about widgets. ".repeat(400); + let cap = 4000; + assert!(page.len() > cap); + let out = select_relevant_passages(&page, "xyzzy plugh nonexistent", cap); + assert!(out.len() > 2000, "budget must be filled, got {}", out.len()); + assert!(out.len() <= cap, "must respect the byte cap"); + } + + #[test] + fn passage_select_fills_budget_even_when_no_whole_chunk_fits_the_tail() { + // Bin-packing remainder: after packing whole chunks, the small leftover + // budget is too tight for ANY remaining whole chunk, so whole-chunk-only + // packing would underfill (feed less than a head-truncation). The + // partial-fill step must still fill the cap AND surface relevant content. + let filler = "generic filler words about other topics here. ".repeat(120); + let blob = "answerword ".repeat(500); // ~5500 chars, query-relevant + let page = format!("Intro sentence. {filler} {blob}"); + let cap = 4000; + assert!(page.len() > cap); + let out = select_relevant_passages(&page, "answerword", cap); + assert!(out.len() <= cap, "must respect the byte cap"); + assert!( + out.len() > cap - 800, + "budget must be ~filled, got {}", + out.len() + ); + assert!( + out.contains("answerword"), + "relevant blob must be surfaced (at least partially)" + ); + } + + #[test] + fn passage_select_passthrough_when_within_budget_or_no_query() { + // within budget -> returned whole + assert_eq!( + select_relevant_passages("a short source", "some query", 8192), + "a short source" + ); + // no query -> plain head-truncation, never panics + let long = "x".repeat(20_000); + let out = select_relevant_passages(&long, " ", 100); + assert_eq!(out.len(), 100); + } } diff --git a/crates/crw-server/src/routes/search.rs b/crates/crw-server/src/routes/search.rs index 558e0a27..e69d3283 100644 --- a/crates/crw-server/src/routes/search.rs +++ b/crates/crw-server/src/routes/search.rs @@ -94,6 +94,29 @@ fn evidence_excerpt(data: &SearchData, max_sources: usize, per_chars: usize) -> out } +/// True when a short scraped body is really a block / error shell (e.g. a page +/// that 403'd to a "Wikimedia Error" datacenter-block, or a bot wall) rather +/// than content. Grounding the answer on such text is worse than useless, so +/// snippet-first drops the body and keeps only the clean SERP snippet. Size-gated +/// so a real article that merely quotes one of these phrases can't false-positive. +fn is_block_shell(md: &str) -> bool { + if md.len() >= 2000 { + return false; + } + let l = md.to_lowercase(); + [ + "wikimedia error", + "are forbidden", + "access denied", + "request blocked", + "just a moment", + "attention required", + "enable javascript and cookies", + ] + .iter() + .any(|p| l.contains(p)) +} + /// Merge freshly-scraped scout rows into the flat answer pool (dedup by URL, /// only rows that actually carry markdown). Returns true if any were added. /// Grouped data (the explicit-`sources` path) is left untouched — multi-round @@ -483,6 +506,7 @@ pub async fn search_inner( &data, &leg_cfg, state.config.search.passage_select, + state.config.search.answer_bm25_select, state.config.search.answer_calibrated, state.config.search.snippet_fallback, state.config.search.answer_guarded, @@ -566,6 +590,7 @@ pub async fn search_inner( &data, &leg_cfg, state.config.search.passage_select, + state.config.search.answer_bm25_select, state.config.search.answer_calibrated, state.config.search.snippet_fallback, state.config.search.answer_guarded, @@ -820,6 +845,7 @@ async fn synthesize_answer( data: &SearchData, cfg: &LlmConfig, passage_select: bool, + bm25_select: bool, calibrated: bool, snippet_fallback: bool, guarded: bool, @@ -863,21 +889,41 @@ async fn synthesize_answer( let scraped: Vec = pool .iter() .filter_map(|r| { - if let Some(md) = r.markdown.as_ref() { - Some((r.url.clone(), r.title.clone(), md.clone())) - } else if snippet_fallback { - // Scrape failed/empty — instead of dropping the result (which - // can lose the answer-bearing page, Pattern A), fall back to the - // SearXNG snippet. It's verbatim upstream text, so it can only - // surface a fact already present, never invent one. - let desc = r.description.trim(); - if desc.is_empty() { - None - } else { + // Legacy path (snippet_fallback off): markdown-only, BYTE-IDENTICAL to + // the pre-change behavior — a result with markdown is kept as-is (even + // a block shell), so the multi-round scout still sees it and can + // recover. Block-guard + snippet-first apply ONLY when snippet_fallback + // is on (there the clean snippet replaces the dropped block body). + if !snippet_fallback { + return r + .markdown + .as_ref() + .map(|md| (r.url.clone(), r.title.clone(), md.clone())); + } + let md = r.markdown.as_deref().map(str::trim).unwrap_or(""); + // Block-guard: a fetched-but-blocked page ("Wikimedia Error", bot wall) + // is noise, not content — drop the body and lean on the clean snippet. + let body = if md.is_empty() || is_block_shell(md) { + "" + } else { + md + }; + // Snippet-first: the SERP snippet is the engine's own query-relevant + // answer passage, so put it FIRST (it then survives the per-source + // passage budget) and always include it; append the body for depth. + // Verbatim upstream text can only surface a present fact, never invent. + let desc = r.description.trim(); + match (desc.is_empty(), body.is_empty()) { + (true, true) => None, + (false, true) => { Some((r.url.clone(), r.title.clone(), format!("[snippet] {desc}"))) } - } else { - None + (true, false) => Some((r.url.clone(), r.title.clone(), body.to_string())), + (false, false) => Some(( + r.url.clone(), + r.title.clone(), + format!("[snippet] {desc}\n\n{body}"), + )), } }) .take(top_n) @@ -918,6 +964,7 @@ async fn synthesize_answer( calibrated, guarded, list_format, + bm25_select, ) .await } @@ -1270,6 +1317,23 @@ mod tests { use super::*; use crw_core::types::SearchSource; + #[test] + fn block_shell_detected_but_not_real_articles() { + assert!(is_block_shell( + "# Wikimedia Error\nError: 403, Contabo networks are forbidden." + )); + assert!(is_block_shell("Just a moment... Attention Required")); + // a long real article that merely contains a phrase is not a block shell + let long = format!( + "Access denied is a common HTTP concept. {}", + "x".repeat(2100) + ); + assert!(!is_block_shell(&long)); + assert!(!is_block_shell( + "Radcliffe College was a women's liberal arts college." + )); + } + fn req(q: &str) -> SearchRequest { SearchRequest { query: q.into(),