From cf0322780e5ed9b867cc995aebf8b1336cb5103c Mon Sep 17 00:00:00 2001 From: Richard Hightower <382678+RichardHightower@users.noreply.github.com> Date: Sun, 30 Aug 2026 06:57:38 +0000 Subject: [PATCH] feat: run rg as retrieval-ladder step 1 rg_ask advertised ripgrep but never invoked it. When rg is on PATH (or OKF_RG_PATH), AND-search knowledge/research/**. Missing rg is not an error; pack/index steps still run. --no-rg skips the step. Fixes #1 --- AGENTS.md | 2 +- scripts/rg_ask.py | 70 ++++++++++++++++++++++++++++- skills/research-ask/SKILL.md | 2 +- tests/fixtures/fake_rg.py | 86 ++++++++++++++++++++++++++++++++++++ tests/test_ask.py | 53 ++++++++++++++++++++++ 5 files changed, 210 insertions(+), 3 deletions(-) create mode 100755 tests/fixtures/fake_rg.py create mode 100644 tests/test_ask.py diff --git a/AGENTS.md b/AGENTS.md index fc74329..3e6ee89 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,6 +3,6 @@ Layer 1. Owns **no** OKF types. Projects RKC into Agent Brain. - `/research-project` — accepted|reviewed only. `GRAPH_USE_LLM_EXTRACTION=false`. -- `/research-ask` — rg → pack → BM25/Chroma → Kuzu last. +- `/research-ask` — rg (when on PATH) → pack → BM25/Chroma → Kuzu last. Missing rg is not an error. - Never cite a vector/graph blob. Citations are OKF locators. - Destroying the index is always safe. diff --git a/scripts/rg_ask.py b/scripts/rg_ask.py index 0ad7317..3dca57e 100755 --- a/scripts/rg_ask.py +++ b/scripts/rg_ask.py @@ -1,14 +1,78 @@ #!/usr/bin/env python3 -"""Retrieval ladder stub: prefer RKC pack, then say index is unprojected.""" +"""Retrieval ladder: rg over research Markdown, then RKC pack, then unprojected index.""" from __future__ import annotations import argparse import json +import os +import re +import shutil import subprocess import sys from pathlib import Path +def find_rg() -> str | None: + for var in ("OKF_RG_PATH", "PKC_RG_PATH", "SECOND_BRAIN_RG_PATH"): + override = (os.environ.get(var) or "").strip() + if not override: + continue + p = Path(override) + if p.is_file() and os.access(p, os.X_OK): + return str(p.resolve()) + found = shutil.which(override) + if found: + return found + return shutil.which("rg") + + +def try_rg(root: Path, question: str, *, limit: int = 10) -> dict: + """Step 1 of the ladder: lexical hits over knowledge/research/**.""" + rg = find_rg() + research = root / "research" if (root / "research").is_dir() else root + if not research.exists(): + return {"engine": None, "hits": [], "note": f"no research tree at {root}"} + terms = [t for t in re.split(r"\s+", question.strip()) if t] + if not rg: + return { + "engine": None, + "hits": [], + "note": "rg not on PATH; install ripgrep or set OKF_RG_PATH. Search still works via /research-pack.", + } + if not terms: + return {"engine": "rg", "hits": []} + # AND: intersect file lists per term, then take the first `limit` paths. + matched: set[Path] | None = None + for term in terms: + cmd = [ + rg, "-l", "--no-messages", "--color", "never", + "-i", "--glob", "*.md", "--glob", "!**/source-assets/**", + "--", term, str(research), + ] + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30, check=False) + except (OSError, subprocess.TimeoutExpired): + return {"engine": None, "hits": [], "note": "rg invocation failed; falling through the ladder"} + if proc.returncode not in (0, 1): + return {"engine": None, "hits": [], "note": proc.stderr.strip() or "rg error"} + files = set() + for line in proc.stdout.splitlines(): + line = line.strip() + if line: + files.add(Path(line)) + matched = files if matched is None else (matched & files) + if not matched: + break + hits = [] + for path in sorted(matched or [])[:limit]: + try: + rel = str(path.relative_to(root)) + except ValueError: + rel = str(path) + hits.append({"path": rel}) + return {"engine": "rg", "hits": hits, "count": len(hits)} + + def try_pack(root: Path, root_id: str | None): rkc = Path(__file__).resolve().parent.parent.parent / "research-knowledge-capture" / "scripts" / "rkc_pack.py" if root_id and rkc.exists(): @@ -27,13 +91,17 @@ def main(): ap.add_argument("--root", type=Path, required=True) ap.add_argument("--question", required=True) ap.add_argument("--pack-root", default=None, help="Optional RKC node id to pack first") + ap.add_argument("--no-rg", action="store_true", help="Skip the ripgrep step") + ap.add_argument("--limit", type=int, default=10) args = ap.parse_args() + lexical = None if args.no_rg else try_rg(args.root, args.question, limit=args.limit) packed = try_pack(args.root, args.pack_root) print( json.dumps( { "question": args.question, "ladder": ["rg", "research-pack", "bm25/chroma", "kuzu"], + "rg": lexical, "pack": packed, "index": "unprojected — run /research-project. GRAPH_USE_LLM_EXTRACTION=false.", "citation_rule": "Finding → Claim → Evidence → source-asset. Never cite a blob.", diff --git a/skills/research-ask/SKILL.md b/skills/research-ask/SKILL.md index 0dbf0d9..c47c190 100644 --- a/skills/research-ask/SKILL.md +++ b/skills/research-ask/SKILL.md @@ -11,7 +11,7 @@ python3 ${CLAUDE_PLUGIN_ROOT}/scripts/rg_ask.py --root knowledge --question "... Ladder: -1. `rg` over `knowledge/research/**` +1. `rg` over `knowledge/research/**` (when ripgrep is on PATH, or `OKF_RG_PATH`). Missing rg is not an error — the rest of the ladder still runs. `--no-rg` skips this step. 2. `/research-pack` (RKC spine) 3. BM25 / Chroma over the projection 4. Kuzu last, for typed paths diff --git a/tests/fixtures/fake_rg.py b/tests/fixtures/fake_rg.py new file mode 100755 index 0000000..b64c9bd --- /dev/null +++ b/tests/fixtures/fake_rg.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Minimal `rg -l` stand-in for tests. Not a real ripgrep. + +Understands: + -l / --files-with-matches + -i / --ignore-case + -F / --fixed-strings + --glob GLOB (including !negations) + --no-messages --color never + pattern PATH +""" + +from __future__ import annotations + +import argparse +import fnmatch +import os +import re +import sys +from pathlib import Path + + +def match_globs(rel: str, globs: list[str]) -> bool: + include = [g for g in globs if not g.startswith("!")] + exclude = [g[1:] for g in globs if g.startswith("!")] + ok = True + if include: + ok = any(fnmatch.fnmatch(rel, g) or fnmatch.fnmatch(Path(rel).name, g) for g in include) + for g in exclude: + if fnmatch.fnmatch(rel, g) or fnmatch.fnmatch(Path(rel).name, g): + return False + return ok + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(add_help=False) + p.add_argument("-l", "--files-with-matches", action="store_true") + p.add_argument("-i", "--ignore-case", action="store_true") + p.add_argument("-F", "--fixed-strings", action="store_true") + p.add_argument("--glob", action="append", default=[]) + p.add_argument("--no-messages", action="store_true") + p.add_argument("--color", default="never") + p.add_argument("pattern") + p.add_argument("path", nargs="?", default=".") + args = p.parse_args(argv) + + root = Path(args.path).resolve() + flags = re.I if args.ignore_case else 0 + if args.fixed_strings: + needle = args.pattern.lower() if args.ignore_case else args.pattern + pred = lambda text: needle in (text.lower() if args.ignore_case else text) + else: + try: + rx = re.compile(args.pattern, flags) + except re.error: + return 2 + pred = lambda text: rx.search(text) is not None + + hits = 0 + if root.is_file(): + files = [root] + base = root.parent + else: + files = sorted(root.rglob("*")) + base = root + for path in files: + if not path.is_file(): + continue + try: + rel = path.relative_to(base).as_posix() + except ValueError: + rel = path.name + if args.glob and not match_globs(rel, args.glob): + continue + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + if pred(text): + print(path) + hits += 1 + return 0 if hits else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_ask.py b/tests/test_ask.py new file mode 100644 index 0000000..c894472 --- /dev/null +++ b/tests/test_ask.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import os +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO / "scripts")) +from rg_ask import try_rg # noqa: E402 + +FAKE_RG = REPO / "tests/fixtures/fake_rg.py" + + +class AskRgTests(unittest.TestCase): + def test_missing_rg_is_not_an_error(self): + env = os.environ.get("OKF_RG_PATH") + os.environ["OKF_RG_PATH"] = "/definitely/not/rg" + try: + out = try_rg(Path("/tmp"), "anything") + self.assertIsNone(out["engine"]) + self.assertEqual(out["hits"], []) + self.assertIn("rg not on PATH", out["note"]) + finally: + if env is None: + os.environ.pop("OKF_RG_PATH", None) + else: + os.environ["OKF_RG_PATH"] = env + + def test_hits_when_fake_rg_present(self): + FAKE_RG.chmod(0o755) + tmp = Path(tempfile.mkdtemp()) + try: + claims = tmp / "research" / "claims" + claims.mkdir(parents=True) + (claims / "claim.md").write_text( + "---\ntype: Claim\ntitle: loop policy\n---\nThe loop policy holds.\n", + encoding="utf-8", + ) + os.environ["OKF_RG_PATH"] = str(FAKE_RG) + out = try_rg(tmp, "loop", limit=5) + self.assertEqual(out["engine"], "rg") + self.assertGreaterEqual(out["count"], 1) + finally: + os.environ.pop("OKF_RG_PATH", None) + shutil.rmtree(tmp) + + +if __name__ == "__main__": + unittest.main()