diff --git a/api/requirements.txt b/api/requirements.txt index a63ff42..f38813f 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -10,6 +10,5 @@ sentry-sdk[fastapi]>=2.20.0 slowapi>=0.1.9 tiktoken>=0.7.0 faiss-cpu>=1.8.0 -rank-bm25>=0.2.2 filelock>=3.12.0 asyncpg>=0.29.0 diff --git a/pyproject.toml b/pyproject.toml index d7de1f2..632be9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ dependencies = [ "slowapi>=0.1.9", "tiktoken>=0.7.0", "faiss-cpu>=1.8.0", - "rank-bm25>=0.2.2", "filelock>=3.12.0", "asyncpg>=0.29.0" ] @@ -34,7 +33,8 @@ dev = [ "pytest-mock>=3.12.0", "coverage>=7.6.0", "pytest-cov>=5.0.0", - "pandas>=2.0.0" + "pandas>=2.0.0", + "bm25s>=0.3.0", ] [tool.pytest.ini_options] diff --git a/scripts/benchmark_beir.py b/scripts/benchmark_beir.py index c04add8..3ebd0ed 100644 --- a/scripts/benchmark_beir.py +++ b/scripts/benchmark_beir.py @@ -180,12 +180,17 @@ def compute_metrics( def build_lexical_index(corpus: dict[str, str]): - """Build lightweight token-based BM25 index.""" - from rank_bm25 import BM25Okapi + """Build lightweight token-based BM25 index. + + Tokenization is intentionally plain lowercase whitespace splitting so + lexical scores stay comparable across runs (no stemming). + """ + import bm25s doc_ids = list(corpus.keys()) tokenized_corpus = [corpus[did].lower().split() for did in doc_ids] - bm25 = BM25Okapi(tokenized_corpus) + bm25 = bm25s.BM25() + bm25.index(tokenized_corpus) return bm25, doc_ids