diff --git a/.changeset/feature-reorganization.md b/.changeset/feature-reorganization.md index 9c33bbee8..1703ec86e 100644 --- a/.changeset/feature-reorganization.md +++ b/.changeset/feature-reorganization.md @@ -5,7 +5,7 @@ "@launchstack/llm": minor "@launchstack/conversion": minor "@launchstack/indexing": minor -"@launchstack/search": minor +"@launchstack/retrieval": minor "@launchstack/orchestration": minor "@launchstack/editing": minor "@launchstack/collab": minor diff --git a/.changeset/retrieval-rename.md b/.changeset/retrieval-rename.md new file mode 100644 index 000000000..d217001fb --- /dev/null +++ b/.changeset/retrieval-rename.md @@ -0,0 +1,19 @@ +--- +"@launchstack/retrieval": minor +"@launchstack/engine": patch +"@launchstack/tools": patch +"@launchstack/pipelines": patch +--- + +Rename `@launchstack/search` to `@launchstack/retrieval` and consolidate +every retrieval algorithm and tool into it, organized as one documented +folder per algorithm: `algorithms/{bm25,vector,fusion,ensemble,rlm,graph, +reranking}` and `tools/{citation-builder,grounded-retrieval,rag-search-tool, +rlm-search}`. The RLM, graph, and ensemble retrievers move in from apps/web; +the predictive-analysis ANN strategies become named modules behind the +vector retriever; grounded-retrieval moves over from `@launchstack/tools` +(a re-export keeps the old path). The `RagPort` contract is unchanged; the +ensemble's env reads become `configureEnsemble()` injected by the +composition root; old subpaths (`./retrievers`, `./reranking`, +`./citation-builder`) survive one release as aliases. The old package name +is lint-banned alongside the ADR-008 legacy names. diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 034240d0c..aaa8d62ff 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -88,7 +88,7 @@ jobs: run: >- pnpm --filter @launchstack/evidence --filter @launchstack/conversion - --filter @launchstack/search + --filter @launchstack/retrieval --filter @launchstack/document-conversion-engine --filter @launchstack/worker test diff --git a/REPOSITORY.md b/REPOSITORY.md index 7df28cad0..de2786c0b 100644 --- a/REPOSITORY.md +++ b/REPOSITORY.md @@ -35,14 +35,14 @@ outright, since nothing was ever published under the old names. | `packages/orchestration` | TS library (published) | Durable work (ADR-003): the pipeline-events contract, the SKIP LOCKED outbox store, the worker tick with bounded retries, transactional source acceptance, and the stage ports. | | `packages/conversion` | TS library (published) | Any source → EvidenceDocument: per-type document converters with their wire + client, audio- and video-transcription in their own folders, OCR primitives, chunking, archive expansion, the extraction router. | | `packages/indexing` | TS library (published) | EvidenceDocument → searchable: the two-stage doc-ingestion pipeline, entity extraction, Neo4j graph sync (optional peer). | -| `packages/search` | TS library (published) | Question → cited answer: BM25 + vector ensemble behind a replaceable port, reranking, the citation builder. | +| `packages/retrieval` | TS library (published) | Question → cited answer (renamed from `search`): every retrieval algorithm as a documented folder under `src/algorithms/` (bm25, vector, fusion, ensemble, rlm, graph, reranking) behind the replaceable RagPort, plus the retrieval-facing tools under `src/tools/`. | | `packages/editing` | TS library (published) | Tracked-changes Word editing (ADR-007): the adeu wire contract + typed client. | | `packages/document-conversion-engine` | TS library (published) | PDF rendering (ADR-009): the typed client for the Gotenberg service — Office → PDF via LibreOffice, HTML/Markdown → PDF via Chromium. Imports nothing, reads no env. | | `packages/google-drive` | TS library (published) | Thin typed client for the Google Drive v3 REST API and Google OAuth 2.0 token endpoints — the wire layer for Drive-linked documents. Framework-free; credentials injected, never read from the environment. | | `packages/collab` | TS library (published) | Agent meetings in Slack-shaped channels, signed HTTP agent transport. Node built-ins only. | | `packages/engine` | TS library (published) | The one-install aggregate: `createEngine(CoreConfig)` plus re-exports of every feature surface. | | `packages/schema-generator` | TS library (published) | Walks the feature wire contracts and emits the one `schemas/v1/` bundle the Python contract tests validate against. | -| `packages/tools` | TS library | Shared, contract-typed capabilities the verticals compose (company-context, grounded-retrieval, brand-voice, persona, web-research, social-publish, platform-profiles, content-scoring, claim-evidence, stage-runner). Tools may import bricks up to `search`, never a vertical. | +| `packages/tools` | TS library | Shared, contract-typed capabilities the verticals compose (company-context, grounded-retrieval, brand-voice, persona, web-research, social-publish, platform-profiles, content-scoring, claim-evidence, stage-runner). Tools may import bricks up to `retrieval`, never a vertical. | | `packages/design-tokens` | CSS (published) | The design contract: primitives feeding semantic tokens, one file, no build step. | | `pipelines/` | TS library (published) | **The compositions tier** — nine verticals (marketing, email, founder-weekly-review, legal-templates, company-metadata, client-prospector, trend-search, connectors, repo-explainer) + the product schema they own. May import any brick; no brick may import it (lint-enforced). | | `services/document-converter` | Node/Express | Routing decisions, vision classification, PDF page rendering, docling-backed parsing → typed `EvidenceDocument`. Replaced `ocr-router` + `ocr-worker` (ADR-004). | @@ -65,7 +65,7 @@ store llm ← persistence · model calls (embeddings live here) orchestration ← events, outbox, tick, source acceptance conversion ← any source → EvidenceDocument indexing ← EvidenceDocument → chunks, vectors, graph -search ← question → cited answer +retrieval ← question → cited answer engine ← createEngine() aggregate pipelines/ apps/ ← compositions and products (never imported by bricks) ``` diff --git a/apps/web/__tests__/api/agent/references.test.ts b/apps/web/__tests__/api/agent/references.test.ts index 3e2a2ab3a..338bba74f 100644 --- a/apps/web/__tests__/api/agent/references.test.ts +++ b/apps/web/__tests__/api/agent/references.test.ts @@ -2,7 +2,7 @@ import { buildReferences, extractRecommendedPages, } from "~/app/api/agents/documentQ&A/services/references"; -import type { SearchResult } from "~/lib/tools/rag"; +import type { SearchResult } from "@launchstack/retrieval/search-types"; describe("references service", () => { it("extracts sorted unique recommended pages and ignores invalid values", () => { diff --git a/apps/web/__tests__/api/agents/documentQ&A/AIChat/query.test.ts b/apps/web/__tests__/api/agents/documentQ&A/AIChat/query.test.ts index c4897b43b..8ed5e1330 100644 --- a/apps/web/__tests__/api/agents/documentQ&A/AIChat/query.test.ts +++ b/apps/web/__tests__/api/agents/documentQ&A/AIChat/query.test.ts @@ -9,7 +9,7 @@ import { POST } from "~/app/api/agents/documentQ&A/AIChat/query/route"; import { requireWorkspaceContext } from "~/lib/require-workspace-context"; import type { WorkspaceContext } from "~/lib/require-workspace-context"; -import { companyEnsembleSearch, documentEnsembleSearch } from "~/lib/tools/rag"; +import { companyEnsembleSearch, documentEnsembleSearch } from "~/server/rag/ensemble"; jest.mock("~/lib/require-workspace-context", () => { const actual = jest.requireActual("~/lib/require-workspace-context"); @@ -78,20 +78,19 @@ jest.mock("~/server/metrics/registry", () => ({ qaRequestDuration: { startTimer: () => jest.fn() }, })); -jest.mock("~/app/api/agents/predictive-document-analysis/services/annOptimizer", () => ({ - __esModule: true, - default: class { - searchSimilarChunks = jest.fn().mockResolvedValue([]); - }, -})); - const RETRIEVED = [{ pageContent: "chunk text", metadata: { page: 1 } }]; -jest.mock("~/lib/tools/rag", () => ({ +jest.mock("~/server/rag/ensemble", () => ({ companyEnsembleSearch: jest.fn(), documentEnsembleSearch: jest.fn(), multiDocEnsembleSearch: jest.fn(), +})); + +jest.mock("@launchstack/retrieval/algorithms/vector", () => ({ createDocumentVectorRetriever: jest.fn(), + ANNOptimizer: class { + searchSimilarChunks = jest.fn().mockResolvedValue([]); + }, })); jest.mock("@launchstack/llm/embeddings", () => ({ diff --git a/apps/web/__tests__/api/predictiveDocumentAnalysis/content.test.ts b/apps/web/__tests__/api/predictiveDocumentAnalysis/content.test.ts index f21b20817..979572a16 100644 --- a/apps/web/__tests__/api/predictiveDocumentAnalysis/content.test.ts +++ b/apps/web/__tests__/api/predictiveDocumentAnalysis/content.test.ts @@ -9,8 +9,9 @@ import { } from "~/app/api/agents/predictive-document-analysis/utils/content"; import type { PdfChunk } from "~/app/api/agents/predictive-document-analysis/types"; import { db } from "~/server/db/index"; +import { getDb } from "@launchstack/store/client"; import { document, documentSections } from "@launchstack/store/schema"; -import { hybridSearchWithRRF } from "~/app/api/agents/predictive-document-analysis/services/hybridSearch"; +import { hybridSearchWithRRF } from "@launchstack/retrieval/algorithms/fusion"; import { findSuggestedCompanyDocuments } from "~/app/api/agents/predictive-document-analysis/services/documentMatcher"; jest.mock("~/server/db/index", () => ({ @@ -19,6 +20,13 @@ jest.mock("~/server/db/index", () => ({ }, })); +// The moved fusion/strategy algorithms reach the database through the store +// client, not the app's ~/server/db proxy — same chain mock, second seam. +jest.mock("@launchstack/store/client", () => ({ + getDb: jest.fn(), + toRows: (rows: unknown) => rows, +})); + jest.mock("~/app/api/agents/predictive-document-analysis/utils/embeddings", () => ({ getEmbeddings: jest.fn().mockResolvedValue([]), })); @@ -88,7 +96,7 @@ type QueryChain = { }; function mockPredictiveSelects(): void { - (db.select as jest.Mock).mockImplementation(() => { + const makeSelect = () => { let source: unknown; let condition: unknown; const query: QueryChain = { @@ -117,7 +125,9 @@ function mockPredictiveSelects(): void { }, }; return query; - }); + }; + (db.select as jest.Mock).mockImplementation(makeSelect); + (getDb as jest.Mock).mockImplementation(() => ({ select: makeSelect })); } describe("predictive current-version retrieval", () => { @@ -127,7 +137,7 @@ describe("predictive current-version retrieval", () => { }); it("excludes historical chunks from hybrid results", async () => { - const results = await hybridSearchWithRRF("schedule a", [2], 8); + const results = await hybridSearchWithRRF("schedule a", [2], 8, async () => []); expect(results).toHaveLength(1); expect(results[0]?.content).toBe(currentChunk.content); diff --git a/apps/web/__tests__/server/retrieval-golden.test.ts b/apps/web/__tests__/server/retrieval-golden.test.ts new file mode 100644 index 000000000..a5aaec3fd --- /dev/null +++ b/apps/web/__tests__/server/retrieval-golden.test.ts @@ -0,0 +1,226 @@ +/** + * Golden retrieval tests (database). + * + * A fixed corpus, a fixed query set, deterministic embeddings — the ensemble + * search's top results are pinned here so a move or "cleanup" that shifts + * relevance behavior fails loudly instead of drifting silently. If a change + * legitimately improves ranking, update the goldens in the same commit and + * say why. + */ + +import type * as CoreDb from "@launchstack/store/client"; + +import { randomUUID } from "node:crypto"; + +import { eq } from "drizzle-orm"; + +jest.mock("~/server/engine", () => { + const databaseUrl = process.env.DATABASE_URL; + const coreDb = jest.requireActual("@launchstack/store/client"); + const engineDb = databaseUrl + ? coreDb.createDb({ + url: databaseUrl, + maxConnections: 10, + }).db + : undefined; + if (engineDb) coreDb.configureDatabase(engineDb); + + return { + getEngine: jest.fn(() => ({ db: engineDb })), + }; +}); + +import type { EmbeddingsProvider } from "@launchstack/llm/embeddings"; +import { + company, + document, + documentContextChunks, + documentRetrievalChunks, + documentSections, + documentVersions, +} from "@launchstack/store/schema"; +import { documentEnsembleSearch } from "@launchstack/retrieval/algorithms/ensemble"; +import { createDocumentVectorRetriever } from "@launchstack/retrieval/algorithms/vector"; +import { resolveEmbeddingIndex } from "@launchstack/llm/embeddings"; +import { db } from "~/server/db/index"; + +const integrationDescribe = process.env.DATABASE_URL ? describe : describe.skip; + +/** One-hot 1536-dim vector — cosine distance is 0 to itself, 1 to any other axis. */ +function axis(i: number): number[] { + const v = Array(1536).fill(0); + v[i] = 1; + return v; +} + +const CHUNKS = [ + { axisIndex: 0, page: 1, content: "alpha section mentions the invoice once" }, + { axisIndex: 1, page: 2, content: "payment terms for the invoice are net thirty days" }, + { axisIndex: 2, page: 3, content: "charlie section about an unrelated appendix" }, +] as const; + +/** Deterministic embedder: every query lands exactly on axis 1 (chunk 2). */ +const stubEmbeddings: EmbeddingsProvider = { + embedQuery: async () => axis(1), + embedDocuments: async docs => docs.map(() => axis(1)), +}; + +integrationDescribe("Golden ensemble retrieval (database)", () => { + let seededCompanyId: number | undefined; + let seededDocumentId: number | undefined; + + afterAll(async () => { + if (seededDocumentId !== undefined) { + await db.delete(document).where(eq(document.id, seededDocumentId)); + } + if (seededCompanyId !== undefined) { + await db.delete(company).where(eq(company.id, seededCompanyId)); + } + }); + + beforeAll(async () => { + const suffix = randomUUID(); + const [companyRow] = await db + .insert(company) + .values({ name: `Golden retrieval ${suffix}`, numberOfEmployees: "1" }) + .returning({ id: company.id }); + if (!companyRow) throw new Error("Failed to seed golden company"); + seededCompanyId = companyRow.id; + + const [documentRow] = await db + .insert(document) + .values({ + url: `https://example.test/golden/${suffix}`, + category: "test", + title: "Golden retrieval fixture", + companyId: BigInt(companyRow.id), + }) + .returning({ id: document.id }); + if (!documentRow) throw new Error("Failed to seed golden document"); + seededDocumentId = documentRow.id; + + const [version] = await db + .insert(documentVersions) + .values({ + documentId: BigInt(documentRow.id), + versionNumber: 1, + url: `https://example.test/golden/${suffix}/v1`, + mimeType: "text/plain", + }) + .returning({ id: documentVersions.id }); + if (!version) throw new Error("Failed to seed golden version"); + + await db + .update(document) + .set({ currentVersionId: BigInt(version.id) }) + .where(eq(document.id, documentRow.id)); + + for (const chunk of CHUNKS) { + // BM25 leg reads documentSections; vector leg reads the retrieval + // chunks via their parent context chunk. Seed both sides. + await db.insert(documentSections).values({ + documentId: BigInt(documentRow.id), + versionId: BigInt(version.id), + content: chunk.content, + pageNumber: chunk.page, + embedding: axis(chunk.axisIndex), + }); + + const [context] = await db + .insert(documentContextChunks) + .values({ + documentId: BigInt(documentRow.id), + versionId: BigInt(version.id), + content: chunk.content, + pageNumber: chunk.page, + embedding: axis(chunk.axisIndex), + }) + .returning({ id: documentContextChunks.id }); + if (!context) throw new Error("Failed to seed golden context chunk"); + + await db.insert(documentRetrievalChunks).values({ + documentId: BigInt(documentRow.id), + versionId: BigInt(version.id), + contextChunkId: BigInt(context.id), + content: chunk.content, + embedding: axis(chunk.axisIndex), + embeddingShort: axis(chunk.axisIndex).slice(0, 512), + }); + } + }); + + it("golden: 'invoice payment terms' ranks the payment-terms chunk first in document scope", async () => { + const results = await documentEnsembleSearch( + "invoice payment terms", + { + documentId: seededDocumentId!, + topK: 3, + embeddingIndexKey: "legacy-openai-1536", + }, + stubEmbeddings + ); + + expect(results.length).toBeGreaterThan(0); + // Both legs agree on chunk 2: BM25 because it carries the most query + // terms, vector because the stub query embedding sits on its axis. + expect(results[0]?.pageContent).toBe(CHUNKS[1].content); + // Every hit comes from the seeded corpus — nothing leaks across scope. + const corpus = new Set(CHUNKS.map(c => c.content)); + for (const hit of results) { + expect(corpus.has(hit.pageContent)).toBe(true); + } + }); + + it("golden: RRF rewards cross-leg agreement over a single leg's top rank", async () => { + // "section" matches chunks 1 and 3 lexically; the stub embedding puts + // every query on chunk 2's axis. Chunk 1 appears in BOTH legs' lists + // while chunk 2 appears only in the vector list, so fusion ranks + // chunk 1 first — pinning the agreement-beats-solo-rank property of + // weighted RRF (k=60, weights [0.4, 0.6]). + const results = await documentEnsembleSearch( + "section", + { + documentId: seededDocumentId!, + topK: 3, + embeddingIndexKey: "legacy-openai-1536", + }, + stubEmbeddings + ); + + expect(results[0]?.pageContent).toBe(CHUNKS[0].content); + }); + + it("golden: the vector leg the ensemble builds is alive against the registry index", async () => { + // Same construction the ensemble uses internally: the registered + // legacy index, the stub embedder, the seeded retrieval chunks. The + // stub query sits on chunk 2's axis, so a live vector leg must rank + // it first — a dead leg (wrong table, wrong join, wrong dimension) + // returns something else or nothing. + const retriever = createDocumentVectorRetriever( + seededDocumentId!, + stubEmbeddings, + resolveEmbeddingIndex("legacy-openai-1536"), + 3 + ); + const docs = await retriever.getRelevantDocuments("anything"); + + expect(docs.length).toBeGreaterThan(0); + expect(docs[0]?.pageContent).toBe(CHUNKS[1].content); + }); + + it("golden: retrieval is non-empty on the seeded corpus (the silent-death canary)", async () => { + // The worst retrieval failure is empty-context-not-error. This canary + // exists so a broken leg composition or port wiring cannot pass CI by + // returning [] — see the design's §6/§7. + const results = await documentEnsembleSearch( + "appendix", + { + documentId: seededDocumentId!, + topK: 2, + embeddingIndexKey: "legacy-openai-1536", + }, + stubEmbeddings + ); + expect(results.length).toBeGreaterThan(0); + }); +}); diff --git a/apps/web/__tests__/server/services/document-creation.integration.test.ts b/apps/web/__tests__/server/services/document-creation.integration.test.ts index 06e019d3a..61d90667a 100644 --- a/apps/web/__tests__/server/services/document-creation.integration.test.ts +++ b/apps/web/__tests__/server/services/document-creation.integration.test.ts @@ -48,7 +48,7 @@ import { type SourceVersionCreatedEvent, } from "@launchstack/orchestration/pipeline-events"; import { db } from "~/server/db"; -import { getDocumentChunks, RLMRetriever } from "~/lib/tools/rag/retrievers"; +import { getDocumentChunks, RLMRetriever } from "@launchstack/retrieval/algorithms"; import { createDocumentLifecycle, createDocumentVersionLifecycle, diff --git a/apps/web/__tests__/server/vector-retriever.test.ts b/apps/web/__tests__/server/vector-retriever.test.ts index b2556bb32..cc09bf7ba 100644 --- a/apps/web/__tests__/server/vector-retriever.test.ts +++ b/apps/web/__tests__/server/vector-retriever.test.ts @@ -32,7 +32,7 @@ import { documentRetrievalChunks, documentVersions, } from "@launchstack/store/schema"; -import { createDocumentVectorRetriever } from "~/lib/tools/rag/retrievers/vector-retriever"; +import { createDocumentVectorRetriever } from "@launchstack/retrieval/algorithms/vector"; import { db } from "~/server/db/index"; const integrationDescribe = process.env.DATABASE_URL ? describe : describe.skip; diff --git a/apps/web/jest.config.js b/apps/web/jest.config.js index 885d69b19..58e7ed820 100644 --- a/apps/web/jest.config.js +++ b/apps/web/jest.config.js @@ -69,11 +69,17 @@ export const config = { "^@launchstack/indexing/knowledge-graph$": "/../../packages/indexing/src/knowledge-graph/index.ts", "^@launchstack/indexing/(.*)$": "/../../packages/indexing/src/$1", - "^@launchstack/search$": "/../../packages/search/src/index.ts", - "^@launchstack/search/retrievers$": - "/../../packages/search/src/retrievers/index.ts", - "^@launchstack/search/reranking$": "/../../packages/search/src/reranking/index.ts", - "^@launchstack/search/(.*)$": "/../../packages/search/src/$1", + "^@launchstack/retrieval$": "/../../packages/retrieval/src/index.ts", + "^@launchstack/retrieval/retrievers$": + "/../../packages/retrieval/src/algorithms/index.ts", + "^@launchstack/retrieval/reranking$": + "/../../packages/retrieval/src/algorithms/reranking/index.ts", + "^@launchstack/retrieval/citation-builder$": + "/../../packages/retrieval/src/tools/citation-builder/index.ts", + "^@launchstack/retrieval/algorithms$": + "/../../packages/retrieval/src/algorithms/index.ts", + "^@launchstack/retrieval/tools$": "/../../packages/retrieval/src/tools/index.ts", + "^@launchstack/retrieval/(.*)$": "/../../packages/retrieval/src/$1", "^@launchstack/orchestration$": "/../../packages/orchestration/src/index.ts", "^@launchstack/orchestration/pipeline-events$": "/../../packages/orchestration/src/pipeline-events.ts", diff --git a/apps/web/next.config.ts b/apps/web/next.config.ts index b8ef170f7..77074eb72 100644 --- a/apps/web/next.config.ts +++ b/apps/web/next.config.ts @@ -34,7 +34,7 @@ const config: NextConfig = { "@launchstack/llm", "@launchstack/conversion", "@launchstack/indexing", - "@launchstack/search", + "@launchstack/retrieval", "@launchstack/orchestration", "@launchstack/collab", "@launchstack/editing", diff --git a/apps/web/package.json b/apps/web/package.json index f304f18ef..f951867ef 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -54,7 +54,7 @@ "@launchstack/pipelines": "workspace:^", "@launchstack/runtime": "workspace:^", "@launchstack/schema-generator": "workspace:^", - "@launchstack/search": "workspace:^", + "@launchstack/retrieval": "workspace:^", "@launchstack/store": "workspace:^", "@launchstack/tools": "workspace:^", "@mozilla/readability": "^0.6.0", diff --git a/apps/web/src/app/api/agents/documentQ&A/AIChat/query/route.ts b/apps/web/src/app/api/agents/documentQ&A/AIChat/query/route.ts index a155d8bec..08481e72d 100644 --- a/apps/web/src/app/api/agents/documentQ&A/AIChat/query/route.ts +++ b/apps/web/src/app/api/agents/documentQ&A/AIChat/query/route.ts @@ -2,17 +2,21 @@ import { NextResponse } from "next/server"; import { SystemMessage, HumanMessage } from "@langchain/core/messages"; import { db } from "~/server/db/index"; import { and, eq, inArray } from "drizzle-orm"; -import ANNOptimizer from "~/app/api/agents/predictive-document-analysis/services/annOptimizer"; import { - companyEnsembleSearch, + ANNOptimizer, createDocumentVectorRetriever, +} from "@launchstack/retrieval/algorithms/vector"; +import { + companyEnsembleSearch, documentEnsembleSearch, multiDocEnsembleSearch, - type CompanySearchOptions, - type DocumentSearchOptions, - type MultiDocSearchOptions, - type SearchResult, -} from "~/lib/tools/rag"; +} from "~/server/rag/ensemble"; +import type { + CompanySearchOptions, + DocumentSearchOptions, + MultiDocSearchOptions, + SearchResult, +} from "@launchstack/retrieval/search-types"; import { resolveEmbeddingIndex, isLegacyEmbeddingIndex } from "@launchstack/llm/embeddings"; import { getCompanyEmbeddingConfig } from "@launchstack/llm/embeddings"; import { validateRequestBody, QuestionSchema } from "~/lib/validation"; diff --git a/apps/web/src/app/api/agents/documentQ&A/AIQuery/route.ts b/apps/web/src/app/api/agents/documentQ&A/AIQuery/route.ts index 9c9cddace..fc3e2f6f9 100644 --- a/apps/web/src/app/api/agents/documentQ&A/AIQuery/route.ts +++ b/apps/web/src/app/api/agents/documentQ&A/AIQuery/route.ts @@ -2,14 +2,16 @@ import { NextResponse } from "next/server"; import { SystemMessage, HumanMessage } from "@langchain/core/messages"; import { db } from "~/server/db/index"; import { eq } from "drizzle-orm"; -import ANNOptimizer from "~/app/api/agents/predictive-document-analysis/services/annOptimizer"; import { - documentEnsembleSearch, + ANNOptimizer, createDocumentVectorRetriever, - type RetrievalMethod, - type DocumentSearchOptions, - type SearchResult, -} from "~/lib/tools/rag"; +} from "@launchstack/retrieval/algorithms/vector"; +import { documentEnsembleSearch } from "~/server/rag/ensemble"; +import type { + RetrievalMethod, + DocumentSearchOptions, + SearchResult, +} from "@launchstack/retrieval/search-types"; import { resolveEmbeddingIndex, isLegacyEmbeddingIndex } from "@launchstack/llm/embeddings"; import { getCompanyEmbeddingConfig } from "@launchstack/llm/embeddings"; import { validateRequestBody, QuestionSchema } from "~/lib/validation"; diff --git a/apps/web/src/app/api/agents/documentQ&A/AIQueryRLM/route.ts b/apps/web/src/app/api/agents/documentQ&A/AIQueryRLM/route.ts index e8e3c11fd..449098b06 100644 --- a/apps/web/src/app/api/agents/documentQ&A/AIQueryRLM/route.ts +++ b/apps/web/src/app/api/agents/documentQ&A/AIQueryRLM/route.ts @@ -29,7 +29,7 @@ import { getWebSearchInstruction, describeChatError, } from "../services"; -import { performRLMSearch, type RLMSearchOptions } from "../services/rlmSearch"; +import { performRLMSearch, type RLMSearchOptions } from "@launchstack/retrieval/tools/rlm-search"; import { describeChatResolutionFailure, resolveConfiguredChatModel } from "~/lib/models"; import { validateDeprecatedChatSelection } from "~/server/chat-request-compat"; import type { SYSTEM_PROMPTS } from "../services/prompts"; diff --git a/apps/web/src/app/api/agents/documentQ&A/AskMyNotes/route.ts b/apps/web/src/app/api/agents/documentQ&A/AskMyNotes/route.ts index f751fd178..eef0ddcd4 100644 --- a/apps/web/src/app/api/agents/documentQ&A/AskMyNotes/route.ts +++ b/apps/web/src/app/api/agents/documentQ&A/AskMyNotes/route.ts @@ -7,17 +7,12 @@ */ import { NextResponse } from "next/server"; -import { OpenAIEmbeddings } from "@langchain/openai"; import { HumanMessage, SystemMessage } from "@langchain/core/messages"; import { withRateLimit } from "~/lib/rate-limit-middleware"; import { RateLimitPresets } from "~/lib/rate-limiter"; import { resolveConfiguredChatModel } from "~/lib/models"; -import { createUserNotesRetriever } from "~/lib/tools/rag/retrievers/notes-retriever"; -import { - EMBEDDING_DIM, - EMBEDDING_MODEL, - resolveEmbeddingConfig, -} from "~/server/notes/embedding-config"; +import { createUserNotesRetriever } from "~/server/notes/notes-retriever"; +import { createNotesEmbeddingsProvider } from "~/server/notes/embedding-config"; import { normalizeModelContent } from "../services"; import { requireWorkspaceContext } from "~/lib/require-workspace-context"; @@ -55,8 +50,8 @@ export async function POST(request: Request) { } const topK = Math.min(Math.max(body.topK ?? 8, 1), 25); - const { apiKey, baseURL } = resolveEmbeddingConfig(); - if (!apiKey) { + const embeddings = createNotesEmbeddingsProvider(); + if (!embeddings) { return NextResponse.json( { success: false, @@ -67,13 +62,6 @@ export async function POST(request: Request) { ); } - const embeddings = new OpenAIEmbeddings({ - openAIApiKey: apiKey, - modelName: EMBEDDING_MODEL, - dimensions: EMBEDDING_DIM, - ...(baseURL ? { configuration: { baseURL } } : {}), - }); - const retriever = createUserNotesRetriever( ctx.data.authUserId, String(ctx.data.companyId), diff --git a/apps/web/src/app/api/agents/documentQ&A/services/index.ts b/apps/web/src/app/api/agents/documentQ&A/services/index.ts index 4bac7feea..a1ec675f3 100644 --- a/apps/web/src/app/api/agents/documentQ&A/services/index.ts +++ b/apps/web/src/app/api/agents/documentQ&A/services/index.ts @@ -34,7 +34,7 @@ export { getSectionsByPath, type RLMSearchOptions, type RLMSearchResult, -} from "./rlmSearch"; +} from "@launchstack/retrieval/tools/rlm-search"; // Types - Centralized export from types.ts export type { diff --git a/apps/web/src/app/api/agents/documentQ&A/services/references.ts b/apps/web/src/app/api/agents/documentQ&A/services/references.ts index 8ad4aaa9f..2d1d46f0a 100644 --- a/apps/web/src/app/api/agents/documentQ&A/services/references.ts +++ b/apps/web/src/app/api/agents/documentQ&A/services/references.ts @@ -1,5 +1,5 @@ -import { buildCitations, type RetrievedEvidence } from "@launchstack/search"; -import type { SearchResult } from "~/lib/tools/rag"; +import { buildCitations, type RetrievedEvidence } from "@launchstack/retrieval"; +import type { SearchResult } from "@launchstack/retrieval/search-types"; import type { SourceReference } from "./types"; const STOPWORDS = new Set([ @@ -190,7 +190,7 @@ function getRelevance(metadata: Record): number | undefined { * compatibility but is now populated from the REAL retrieval relevance * (`relevance`, the reranker's score) and omitted entirely when no such score * exists. Rows that carry full identity (documentId + versionId) are also - * anchored via @launchstack/search's `buildCitations`, yielding a stable + * anchored via @launchstack/retrieval's `buildCitations`, yielding a stable * `anchorKey`. The frontend reads `snippet`/`documentId`/`page`, which are * unchanged. */ diff --git a/apps/web/src/app/api/agents/predictive-document-analysis/services/annOptimizer.ts b/apps/web/src/app/api/agents/predictive-document-analysis/services/annOptimizer.ts deleted file mode 100644 index 635744474..000000000 --- a/apps/web/src/app/api/agents/predictive-document-analysis/services/annOptimizer.ts +++ /dev/null @@ -1,547 +0,0 @@ -import { db } from "~/server/db/index"; -import { and, eq, inArray, sql } from "drizzle-orm"; -import { document, documentSections, documentRetrievalChunks } from "@launchstack/store/schema"; -import { sanitizeErrorMessage } from "~/app/api/agents/predictive-document-analysis/utils/logging"; - -interface ANNConfig { - strategy: "hnsw" | "ivf" | "hybrid" | "prefiltered" | "matryoshka"; - probeCount?: number; - efSearch?: number; - maxCandidates?: number; - prefilterThreshold?: number; -} - -interface ANNResult { - id: number; - content: string; - page: number; - documentId: number; - distance: number; - confidence: number; -} - -type ANNRow = { id: number; content: string; page: number; documentId: number; distance: number }; - -const documentClustersCache = new Map(); - -interface DocumentCluster { - documentId: number; - centroid: number[]; - chunkIds: number[]; - avgDistance: number; - lastUpdated: Date; -} - -export class ANNOptimizer { - private config: ANNConfig; - - constructor(config: ANNConfig = { strategy: "hybrid" }) { - this.config = config; - } - - async searchSimilarChunks( - queryEmbedding: number[], - documentIds: number[], - limit = 10, - distanceThreshold = 0.7 - ): Promise { - if (!documentIds || documentIds.length === 0) { - return []; - } - - switch (this.config.strategy) { - case "hnsw": - return this.hnswSearch(queryEmbedding, documentIds, limit, distanceThreshold); - - case "ivf": - return this.ivfSearch(queryEmbedding, documentIds, limit, distanceThreshold); - - case "prefiltered": - return this.prefilteredSearch( - queryEmbedding, - documentIds, - limit, - distanceThreshold - ); - - case "matryoshka": - return this.matryoshkaSearch(queryEmbedding, documentIds, limit, distanceThreshold); - - case "hybrid": - default: - return this.hybridSearch(queryEmbedding, documentIds, limit, distanceThreshold); - } - } - - private async hnswSearch( - queryEmbedding: number[], - documentIds: number[], - limit: number, - threshold: number - ): Promise { - try { - const embeddingStr = `[${queryEmbedding.join(",")}]`; - - const approximateLimit = Math.min(limit * 5, 100); - - const results = await db - .select({ - id: documentSections.id, - content: documentSections.content, - page: documentSections.pageNumber, - documentId: documentSections.documentId, - distance: sql`${documentSections.embedding} <=> ${embeddingStr}::vector`, - }) - .from(documentSections) - .innerJoin(document, eq(documentSections.documentId, document.id)) - .where( - and( - inArray( - documentSections.documentId, - documentIds.map(id => BigInt(id)) - ), - eq(documentSections.versionId, document.currentVersionId) - ) - ) - .orderBy(sql`${documentSections.embedding} <=> ${embeddingStr}::vector`) - .limit(approximateLimit); - - const rows: ANNRow[] = results.map(r => ({ - id: r.id, - content: r.content, - page: r.page ?? 0, - documentId: Number(r.documentId), - distance: Number(r.distance ?? 1), - })); - - const refinedResults = rows - .map(row => ({ - ...row, - confidence: Math.max(0, 1 - row.distance), - })) - .filter(r => r.distance <= threshold) - .sort((a, b) => a.distance - b.distance) - .slice(0, limit); - - return refinedResults as ANNResult[]; - } catch (error) { - console.warn("HNSW search failed:", sanitizeErrorMessage(error)); - return []; - } - } - - private async ivfSearch( - queryEmbedding: number[], - documentIds: number[], - limit: number, - threshold: number - ): Promise { - try { - const relevantClusters = await this.findRelevantDocumentClusters( - queryEmbedding, - documentIds, - this.config.probeCount ?? 3 - ); - - if (relevantClusters.length === 0) { - return this.hnswSearch(queryEmbedding, documentIds, limit, threshold); - } - - const clusterChunkIds = relevantClusters.flatMap(c => c.chunkIds); - - if (clusterChunkIds.length === 0) { - return []; - } - - const embeddingStr = `[${queryEmbedding.join(",")}]`; - - const results = await db - .select({ - id: documentSections.id, - content: documentSections.content, - page: documentSections.pageNumber, - documentId: documentSections.documentId, - distance: sql`${documentSections.embedding} <=> ${embeddingStr}::vector`, - }) - .from(documentSections) - .innerJoin(document, eq(documentSections.documentId, document.id)) - .where( - and( - inArray(documentSections.id, clusterChunkIds), - eq(documentSections.versionId, document.currentVersionId), - sql`${documentSections.embedding} <=> ${embeddingStr}::vector <= ${threshold}` - ) - ) - .orderBy(sql`${documentSections.embedding} <=> ${embeddingStr}::vector`) - .limit(limit); - - return results.map(row => ({ - id: row.id, - content: row.content, - page: row.page ?? 0, - documentId: Number(row.documentId), - distance: Number(row.distance ?? 1), - confidence: Math.max(0, 1 - Number(row.distance ?? 1)), - })); - } catch (error) { - console.warn("IVF search failed:", sanitizeErrorMessage(error)); - return []; - } - } - - private async prefilteredSearch( - queryEmbedding: number[], - documentIds: number[], - limit: number, - threshold: number - ): Promise { - try { - const docScores = await this.calculateDocumentRelevanceScores( - queryEmbedding, - documentIds - ); - - const sortedDocIds = docScores - .filter(d => d.score > (this.config.prefilterThreshold ?? 0.3)) - .sort((a, b) => b.score - a.score) - .map(d => d.documentId); - - if (sortedDocIds.length === 0) { - return this.hnswSearch(queryEmbedding, documentIds, limit, threshold); - } - - const results: ANNResult[] = []; - const embeddingStr = `[${queryEmbedding.join(",")}]`; - - for (const docId of sortedDocIds) { - if (results.length >= limit) break; - - const remaining = limit - results.length; - const docResults = await db - .select({ - id: documentSections.id, - content: documentSections.content, - page: documentSections.pageNumber, - documentId: documentSections.documentId, - distance: sql`${documentSections.embedding} <=> ${embeddingStr}::vector`, - }) - .from(documentSections) - .innerJoin(document, eq(documentSections.documentId, document.id)) - .where( - and( - eq(documentSections.documentId, BigInt(docId)), - eq(documentSections.versionId, document.currentVersionId), - sql`${documentSections.embedding} <=> ${embeddingStr}::vector <= ${threshold}` - ) - ) - .orderBy(sql`${documentSections.embedding} <=> ${embeddingStr}::vector`) - .limit(remaining * 2); - - const mappedResults: ANNResult[] = docResults.map(row => ({ - id: row.id, - content: row.content, - page: row.page ?? 0, - documentId: Number(row.documentId), - distance: Number(row.distance ?? 1), - confidence: Math.max(0, 1 - Number(row.distance ?? 1)), - })); - - results.push(...mappedResults.slice(0, remaining)); - } - - return results.sort((a, b) => a.distance - b.distance); - } catch (error) { - console.warn("Prefiltered search failed:", sanitizeErrorMessage(error)); - return []; - } - } - - /** - * Matryoshka coarse-to-fine: use 512-dim short embeddings from - * document_retrieval_chunks (HNSW-indexed) for fast candidate filtering, - * then re-rank the top candidates with full 1536-dim embeddings. - */ - private async matryoshkaSearch( - queryEmbedding: number[], - documentIds: number[], - limit: number, - threshold: number - ): Promise { - try { - const shortDim = 512; - const queryShort = queryEmbedding.slice(0, shortDim); - const shortStr = `[${queryShort.join(",")}]`; - - const coarseCandidateCount = Math.min(limit * 6, 120); - - const coarseResults = await db - .select({ - id: documentRetrievalChunks.id, - content: documentRetrievalChunks.content, - documentId: documentRetrievalChunks.documentId, - contextChunkId: documentRetrievalChunks.contextChunkId, - shortDistance: sql`${documentRetrievalChunks.embeddingShort} <=> ${shortStr}::vector`, - }) - .from(documentRetrievalChunks) - .innerJoin(document, eq(documentRetrievalChunks.documentId, document.id)) - .where( - and( - inArray( - documentRetrievalChunks.documentId, - documentIds.map(id => BigInt(id)) - ), - eq(documentRetrievalChunks.versionId, document.currentVersionId) - ) - ) - .orderBy(sql`${documentRetrievalChunks.embeddingShort} <=> ${shortStr}::vector`) - .limit(coarseCandidateCount); - - if (coarseResults.length === 0) { - return this.hnswSearch(queryEmbedding, documentIds, limit, threshold); - } - - const candidateIds = coarseResults.map(r => r.id); - const fullStr = `[${queryEmbedding.join(",")}]`; - - const refinedResults = await db - .select({ - id: documentRetrievalChunks.id, - content: documentRetrievalChunks.content, - documentId: documentRetrievalChunks.documentId, - distance: sql`${documentRetrievalChunks.embedding} <=> ${fullStr}::vector`, - }) - .from(documentRetrievalChunks) - .innerJoin(document, eq(documentRetrievalChunks.documentId, document.id)) - .where( - and( - inArray(documentRetrievalChunks.id, candidateIds), - eq(documentRetrievalChunks.versionId, document.currentVersionId) - ) - ) - .orderBy(sql`${documentRetrievalChunks.embedding} <=> ${fullStr}::vector`) - .limit(limit); - - const contextChunkIds = coarseResults - .map(r => Number(r.contextChunkId)) - .filter(id => !isNaN(id)); - - const pageMap = new Map(); - if (contextChunkIds.length > 0) { - const pages = await db - .select({ - id: documentSections.id, - page: documentSections.pageNumber, - }) - .from(documentSections) - .innerJoin(document, eq(documentSections.documentId, document.id)) - .where( - and( - inArray(documentSections.id, contextChunkIds), - eq(documentSections.versionId, document.currentVersionId) - ) - ); - - for (const p of pages) { - pageMap.set(p.id, p.page ?? 1); - } - } - - const contextIdMap = new Map(coarseResults.map(r => [r.id, Number(r.contextChunkId)])); - - return refinedResults - .map(row => { - const dist = Number(row.distance ?? 1); - const ctxId = contextIdMap.get(row.id); - return { - id: row.id, - content: row.content, - page: ctxId ? (pageMap.get(ctxId) ?? 1) : 1, - documentId: Number(row.documentId), - distance: dist, - confidence: Math.max(0, 1 - dist), - }; - }) - .filter(r => r.distance <= threshold); - } catch (error) { - console.warn("Matryoshka search failed:", sanitizeErrorMessage(error)); - return []; - } - } - - private async hybridSearch( - queryEmbedding: number[], - documentIds: number[], - limit: number, - threshold: number - ): Promise { - if (documentIds.length <= 5) { - return this.hnswSearch(queryEmbedding, documentIds, limit, threshold); - } - - if (documentIds.length <= 20) { - return this.prefilteredSearch(queryEmbedding, documentIds, limit, threshold); - } - - // For large document sets, use Matryoshka coarse-to-fine - return this.matryoshkaSearch(queryEmbedding, documentIds, limit, threshold); - } - - private async calculateDocumentRelevanceScores( - queryEmbedding: number[], - documentIds: number[] - ): Promise<{ documentId: number; score: number }[]> { - const scores: { documentId: number; score: number }[] = []; - - for (const docId of documentIds) { - let cluster = documentClustersCache.get(docId); - - if (!cluster || Date.now() - cluster.lastUpdated.getTime() > 3600000) { - cluster = await this.buildDocumentCluster(docId); - documentClustersCache.set(docId, cluster); - } - - const similarity = this.cosineSimilarity(queryEmbedding, cluster.centroid); - scores.push({ documentId: docId, score: similarity }); - } - - return scores; - } - - private async buildDocumentCluster(documentId: number): Promise { - const chunks = await db - .select({ - id: documentSections.id, - embedding: documentSections.embedding, - }) - .from(documentSections) - .innerJoin(document, eq(documentSections.documentId, document.id)) - .where( - and( - eq(documentSections.documentId, BigInt(documentId)), - eq(documentSections.versionId, document.currentVersionId) - ) - ); - - if (chunks.length === 0) { - return { - documentId, - centroid: [], - chunkIds: [], - avgDistance: 1, - lastUpdated: new Date(), - }; - } - - const dimension = chunks[0]?.embedding?.length ?? 1536; - const centroid = new Array(dimension).fill(0); - - for (const chunk of chunks) { - if (chunk.embedding) { - for (let i = 0; i < dimension; i++) { - centroid[i] += chunk.embedding[i]; - } - } - } - - for (let i = 0; i < dimension; i++) { - centroid[i] /= chunks.length; - } - - let totalDistance = 0; - let comparisons = 0; - - for (let i = 0; i < chunks.length && comparisons < 100; i++) { - for (let j = i + 1; j < chunks.length && comparisons < 100; j++) { - if (chunks[i]?.embedding && chunks[j]?.embedding) { - totalDistance += this.euclideanDistance( - chunks[i]!.embedding!, - chunks[j]!.embedding! - ); - comparisons++; - } - } - } - - const avgDistance = comparisons > 0 ? totalDistance / comparisons : 1; - - return { - documentId, - centroid: centroid as number[], - chunkIds: chunks.map(c => c.id), - avgDistance, - lastUpdated: new Date(), - }; - } - - private async findRelevantDocumentClusters( - queryEmbedding: number[], - documentIds: number[], - topK = 3 - ): Promise { - const clusters: Array<{ cluster: DocumentCluster; similarity: number }> = []; - - for (const docId of documentIds) { - let cluster = documentClustersCache.get(docId); - - if (!cluster) { - cluster = await this.buildDocumentCluster(docId); - documentClustersCache.set(docId, cluster); - } - - if (cluster.centroid.length > 0) { - const similarity = this.cosineSimilarity(queryEmbedding, cluster.centroid); - clusters.push({ cluster, similarity }); - } - } - - return clusters - .sort((a, b) => b.similarity - a.similarity) - .slice(0, topK) - .map(c => c.cluster); - } - - private cosineSimilarity(a: number[], b: number[]): number { - if (a.length !== b.length) return 0; - - let dotProduct = 0; - let normA = 0; - let normB = 0; - - for (let i = 0; i < a.length; i++) { - dotProduct += a[i]! * b[i]!; - normA += a[i]! * a[i]!; - normB += b[i]! * b[i]!; - } - - if (normA === 0 || normB === 0) return 0; - return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); - } - - private euclideanDistance(a: number[], b: number[]): number { - if (a.length !== b.length) return Infinity; - - let sum = 0; - for (let i = 0; i < a.length; i++) { - const diff = a[i]! - b[i]!; - sum += diff * diff; - } - return Math.sqrt(sum); - } - - static clearCache(): void { - documentClustersCache.clear(); - } - - static getCacheStats(): { size: number; oldestEntry: Date | null } { - const entries = Array.from(documentClustersCache.values()); - return { - size: entries.length, - oldestEntry: - entries.length > 0 - ? new Date(Math.min(...entries.map(e => e.lastUpdated.getTime()))) - : null, - }; - } -} - -export default ANNOptimizer; diff --git a/apps/web/src/app/api/agents/predictive-document-analysis/services/documentMatcher.ts b/apps/web/src/app/api/agents/predictive-document-analysis/services/documentMatcher.ts index 62af75b02..0243625f6 100644 --- a/apps/web/src/app/api/agents/predictive-document-analysis/services/documentMatcher.ts +++ b/apps/web/src/app/api/agents/predictive-document-analysis/services/documentMatcher.ts @@ -13,8 +13,8 @@ import { truncateText, } from "~/app/api/agents/predictive-document-analysis/utils/content"; import { sanitizeErrorMessage } from "~/app/api/agents/predictive-document-analysis/utils/logging"; -import ANNOptimizer from "~/app/api/agents/predictive-document-analysis/services/annOptimizer"; -import { hybridSearchWithRRF } from "~/app/api/agents/predictive-document-analysis/services/hybridSearch"; +import { ANNOptimizer } from "@launchstack/retrieval/algorithms/vector"; +import { hybridSearchWithRRF } from "@launchstack/retrieval/algorithms/fusion"; type MatchCandidate = { documentId: number; @@ -97,7 +97,9 @@ export async function findSuggestedCompanyDocuments( const searchQuery = `${missingDoc.documentType} ${missingDoc.documentName}`; const [contextMatches, hybridMatches] = await Promise.all([ findOptimizedContextualMatches(missingDoc, otherDocIds), - hybridSearchWithRRF(searchQuery, otherDocIds, 6).catch(() => [] as DocumentMatch[]), + hybridSearchWithRRF(searchQuery, otherDocIds, 6, getEmbeddings).catch( + () => [] as DocumentMatch[] + ), ]); const allContextMatches = [...contextMatches, ...hybridMatches]; diff --git a/apps/web/src/app/api/agents/predictive-document-analysis/services/hybridSearch.ts b/apps/web/src/app/api/agents/predictive-document-analysis/services/hybridSearch.ts deleted file mode 100644 index 654ceca43..000000000 --- a/apps/web/src/app/api/agents/predictive-document-analysis/services/hybridSearch.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { db } from "~/server/db/index"; -import { and, inArray, sql, eq } from "drizzle-orm"; -import { document, documentSections } from "@launchstack/store/schema"; -import { getEmbeddings } from "~/app/api/agents/predictive-document-analysis/utils/embeddings"; -import { truncateText } from "~/app/api/agents/predictive-document-analysis/utils/content"; -import type { DocumentMatch } from "~/app/api/agents/predictive-document-analysis/types"; - -interface RankedResult { - documentId: number; - page: number; - content: string; - rank: number; -} - -/** - * Full-text search using PostgreSQL's built-in ts_vector/ts_query. - * Returns results ranked by ts_rank. - */ -async function bm25Search(query: string, docIds: number[], limit = 10): Promise { - if (docIds.length === 0) return []; - - const tsQuery = query - .split(/\s+/) - .filter(w => w.length > 1) - .map(w => w.replace(/[^a-zA-Z0-9]/g, "")) - .filter(Boolean) - .join(" | "); - - if (!tsQuery) return []; - - const results = await db - .select({ - id: documentSections.id, - content: documentSections.content, - page: documentSections.pageNumber, - documentId: documentSections.documentId, - rank: sql`ts_rank(to_tsvector('english', ${documentSections.content}), to_tsquery('english', ${tsQuery}))`, - }) - .from(documentSections) - .innerJoin(document, eq(documentSections.documentId, document.id)) - .where( - and( - inArray( - documentSections.documentId, - docIds.map(id => BigInt(id)) - ), - eq(documentSections.versionId, document.currentVersionId), - sql`to_tsvector('english', ${documentSections.content}) @@ to_tsquery('english', ${tsQuery})` - ) - ) - .orderBy( - sql`ts_rank(to_tsvector('english', ${documentSections.content}), to_tsquery('english', ${tsQuery})) DESC` - ) - .limit(limit); - - return results.map((r, idx) => ({ - documentId: Number(r.documentId), - page: r.page ?? 1, - content: r.content, - rank: idx + 1, - })); -} - -/** - * Dense vector search using cosine similarity. - */ -async function vectorSearch( - query: string, - docIds: number[], - limit = 10, - threshold = 0.4 -): Promise { - if (docIds.length === 0) return []; - - const queryEmbedding = await getEmbeddings(query); - if (queryEmbedding.length === 0) return []; - - const embeddingStr = `[${queryEmbedding.join(",")}]`; - - const results = await db - .select({ - id: documentSections.id, - content: documentSections.content, - page: documentSections.pageNumber, - documentId: documentSections.documentId, - distance: sql`${documentSections.embedding} <=> ${embeddingStr}::vector`, - }) - .from(documentSections) - .innerJoin(document, eq(documentSections.documentId, document.id)) - .where( - and( - inArray( - documentSections.documentId, - docIds.map(id => BigInt(id)) - ), - eq(documentSections.versionId, document.currentVersionId), - sql`${documentSections.embedding} <=> ${embeddingStr}::vector < ${threshold}` - ) - ) - .orderBy(sql`${documentSections.embedding} <=> ${embeddingStr}::vector`) - .limit(limit); - - return results.map((r, idx) => ({ - documentId: Number(r.documentId), - page: r.page ?? 1, - content: r.content, - rank: idx + 1, - })); -} - -/** - * Reciprocal Rank Fusion: merges ranked lists from different retrieval methods. - * RRF(d) = sum( 1 / (k + rank_i(d)) ) for each list i that contains d. - * k=60 is standard (from the original Cormack et al. paper). - */ -function reciprocalRankFusion( - lists: RankedResult[][], - k = 60 -): Map { - const fused = new Map< - string, - { score: number; documentId: number; page: number; content: string } - >(); - - for (const list of lists) { - for (const item of list) { - const key = `${item.documentId}:${item.page}`; - const existing = fused.get(key); - const rrfScore = 1 / (k + item.rank); - - if (existing) { - existing.score += rrfScore; - if (item.content.length > existing.content.length) { - existing.content = item.content; - } - } else { - fused.set(key, { - score: rrfScore, - documentId: item.documentId, - page: item.page, - content: item.content, - }); - } - } - } - - return fused; -} - -/** - * Hybrid search combining BM25 full-text and vector similarity with RRF. - */ -export async function hybridSearchWithRRF( - query: string, - docIds: number[], - limit = 8 -): Promise { - if (docIds.length === 0) return []; - - const [bm25Results, vecResults] = await Promise.all([ - bm25Search(query, docIds, limit * 2).catch(() => [] as RankedResult[]), - vectorSearch(query, docIds, limit * 2).catch(() => [] as RankedResult[]), - ]); - - if (bm25Results.length === 0 && vecResults.length === 0) return []; - - const fused = reciprocalRankFusion([bm25Results, vecResults]); - - return Array.from(fused.values()) - .sort((a, b) => b.score - a.score) - .slice(0, limit) - .map(r => ({ - documentId: r.documentId, - page: r.page, - snippet: truncateText(r.content, 150), - similarity: Math.min(r.score * 60, 0.95), - content: r.content, - })); -} diff --git a/apps/web/src/app/api/agents/predictive-document-analysis/utils/embeddings.ts b/apps/web/src/app/api/agents/predictive-document-analysis/utils/embeddings.ts index 610d5b762..8eb6c77e4 100644 --- a/apps/web/src/app/api/agents/predictive-document-analysis/utils/embeddings.ts +++ b/apps/web/src/app/api/agents/predictive-document-analysis/utils/embeddings.ts @@ -1,4 +1,15 @@ -import { OpenAIEmbeddings } from "@langchain/openai"; +/** + * Query embeddings for predictive analysis, generated through + * @launchstack/llm's embedding service (no private HTTP client here — this + * file only resolves the feature's endpoint pair, caches, and sets error + * semantics). Fixed at text-embedding-3-large / 1536 dims because every + * similarity comparison this feature makes is against the legacy 1536-dim + * `documentSections` / `documentRetrievalChunks` embeddings. + */ + +import { generateEmbeddings } from "@launchstack/llm/embeddings"; +import { LRUCache } from "lru-cache"; +import { sanitizeErrorMessage } from "~/app/api/agents/predictive-document-analysis/utils/logging"; /** * Thrown when embeddings are not configured at all. Distinct from a transient @@ -13,18 +24,17 @@ export class EmbeddingConfigurationError extends Error { /** * Endpoint and credential, resolved as a PAIR. * - * `@langchain/openai` silently falls back to `api.openai.com` when - * `configuration.baseURL` is undefined, so a key without an endpoint used to - * ship document text to a vendor nothing here names. Both halves come from the - * same source or the call fails. + * The llm embedding service deliberately has no default endpoint — + * embeddings are persisted, so the provider must be named explicitly. Both + * halves come from the same source or the call fails. */ -function resolveEmbeddingEndpoint(): { apiKey: string; baseURL: string } { - const baseURL = process.env.EMBEDDING_API_BASE_URL ?? process.env.AI_BASE_URL; +function resolveEmbeddingEndpoint(): { apiKey: string; baseUrl: string } { + const baseUrl = process.env.EMBEDDING_API_BASE_URL ?? process.env.AI_BASE_URL; const apiKey = process.env.EMBEDDING_API_BASE_URL ? process.env.EMBEDDING_API_KEY : (process.env.AI_API_KEY ?? process.env.OPENAI_API_KEY); - if (!baseURL || !apiKey) { + if (!baseUrl || !apiKey) { throw new EmbeddingConfigurationError( "Embeddings are not configured. Set EMBEDDING_API_BASE_URL and " + "EMBEDDING_API_KEY (or AI_BASE_URL and AI_API_KEY). There is no " + @@ -32,12 +42,11 @@ function resolveEmbeddingEndpoint(): { apiKey: string; baseURL: string } { "must be named explicitly." ); } - return { apiKey, baseURL }; + return { apiKey, baseUrl }; } -import { LRUCache } from "lru-cache"; -import { sanitizeErrorMessage } from "~/app/api/agents/predictive-document-analysis/utils/logging"; const EMBEDDING_MODEL = "text-embedding-3-large"; +const EMBEDDING_DIMENSIONS = 1536; const MAX_CACHE_ENTRIES = 500; const embeddingCache = new LRUCache({ @@ -51,16 +60,14 @@ export async function getEmbeddings(text: string): Promise { } try { - const { apiKey, baseURL } = resolveEmbeddingEndpoint(); - const embeddings = new OpenAIEmbeddings({ - openAIApiKey: apiKey, - modelName: EMBEDDING_MODEL, - dimensions: 1536, - configuration: { baseURL }, + const { apiKey, baseUrl } = resolveEmbeddingEndpoint(); + const { embeddings } = await generateEmbeddings([text], { + apiKey, + baseUrl, + model: EMBEDDING_MODEL, + dimensions: EMBEDDING_DIMENSIONS, }); - - const [embedding] = await embeddings.embedDocuments([text]); - const result = embedding ?? []; + const result = embeddings[0] ?? []; embeddingCache.set(text, result); return result; @@ -78,16 +85,14 @@ export async function batchGetEmbeddings(texts: string[]): Promise { const uniqueTexts = [...new Set(texts)]; try { - const { apiKey, baseURL } = resolveEmbeddingEndpoint(); - const embeddings = new OpenAIEmbeddings({ - openAIApiKey: apiKey, - modelName: EMBEDDING_MODEL, - dimensions: 1536, - configuration: { baseURL }, + const { apiKey, baseUrl } = resolveEmbeddingEndpoint(); + const { embeddings } = await generateEmbeddings(uniqueTexts, { + apiKey, + baseUrl, + model: EMBEDDING_MODEL, + dimensions: EMBEDDING_DIMENSIONS, }); - - const results = await embeddings.embedDocuments(uniqueTexts); - const embeddingMap = new Map(uniqueTexts.map((text, i) => [text, results[i]])); + const embeddingMap = new Map(uniqueTexts.map((text, i) => [text, embeddings[i]])); embeddingMap.forEach((embedding, text) => { embeddingCache.set(text, embedding ?? []); diff --git a/apps/web/src/app/api/document-generator/research/route.ts b/apps/web/src/app/api/document-generator/research/route.ts index 264c72098..b14f0dda2 100644 --- a/apps/web/src/app/api/document-generator/research/route.ts +++ b/apps/web/src/app/api/document-generator/research/route.ts @@ -9,11 +9,8 @@ import { NextResponse } from "next/server"; import { z } from "zod"; -import { - companyEnsembleSearch, - type CompanySearchOptions, - type SearchResult, -} from "~/lib/tools/rag"; +import { companyEnsembleSearch } from "~/server/rag/ensemble"; +import type { CompanySearchOptions, SearchResult } from "@launchstack/retrieval/search-types"; import { performExaSearch } from "~/app/api/agents/documentQ&A/services/exaSearch"; import { getEmbeddings } from "~/app/api/agents/documentQ&A/services"; import { requireWorkspaceContext } from "~/lib/require-workspace-context"; diff --git a/apps/web/src/lib/models.ts b/apps/web/src/lib/models.ts index 497807c6c..65cdcdd68 100644 --- a/apps/web/src/lib/models.ts +++ b/apps/web/src/lib/models.ts @@ -23,7 +23,7 @@ import { import { createEmbeddingModel } from "@launchstack/llm/embeddings"; import { resolveEmbeddingIndex } from "@launchstack/llm/embeddings"; import type { CompanyEmbeddingConfig } from "@launchstack/llm/embeddings"; -import type { EmbeddingsProvider } from "~/lib/tools/rag/types"; +import type { EmbeddingsProvider } from "@launchstack/retrieval/search-types"; import { configureAppChatModels } from "~/server/chat-models"; import { env } from "~/env"; diff --git a/apps/web/src/lib/tools/index.ts b/apps/web/src/lib/tools/index.ts deleted file mode 100644 index 3c2ee3991..000000000 --- a/apps/web/src/lib/tools/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export { - runDocIngestionTool, - type DocIngestionToolInput, - type DocIngestionToolResult, - type DocIngestionToolRuntimeOptions, -} from "@launchstack/indexing/doc-ingestion"; - -export * from "./rag"; diff --git a/apps/web/src/lib/tools/rag/agentic/index.ts b/apps/web/src/lib/tools/rag/agentic/index.ts deleted file mode 100644 index bc132af3a..000000000 --- a/apps/web/src/lib/tools/rag/agentic/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { ragSearchTool, executeRAGSearch } from "./rag-search-tool"; diff --git a/apps/web/src/lib/tools/rag/index.ts b/apps/web/src/lib/tools/rag/index.ts deleted file mode 100644 index 102572070..000000000 --- a/apps/web/src/lib/tools/rag/index.ts +++ /dev/null @@ -1,72 +0,0 @@ -export type { - SearchScope, - RetrievalMethod, - BaseSearchMetadata, - SearchResult, - DocumentSearchResult, - CompanySearchResult, - MultiDocSearchResult, - EnsembleSearchOptions, - DocumentSearchOptions, - CompanySearchOptions, - MultiDocSearchOptions, - ChunkRow, - ANNResult, - ANNStrategy, - ANNConfig, - DocumentCluster, - EmbeddingsProvider, - RAGSearchInput, - RAGSearchResult, -} from "./types"; - -export { - VectorRetriever, - createDocumentVectorRetriever, - createCompanyVectorRetriever, - createMultiDocVectorRetriever, - getDocumentChunks, - getCompanyChunks, - getMultiDocChunks, - chunksToDocuments, - createDocumentBM25Retriever, - createCompanyBM25Retriever, - createMultiDocBM25Retriever, - RLMRetriever, - createRLMRetriever, - getDocumentSummary, - getStructureContent, - GraphRetriever, - createGraphRetriever, -} from "./retrievers"; - -export type { - DocumentOverview, - StructureNode, - SectionWithCost, - SectionPreview, - WorkspaceEntry, - TokenBudgetOptions, - WorkspaceStoreOptions, -} from "./retrievers"; - -export { - createOpenAIEmbeddings, - createDocumentEnsembleRetriever, - createCompanyEnsembleRetriever, - createMultiDocEnsembleRetriever, - documentEnsembleSearch, - companyEnsembleSearch, - multiDocEnsembleSearch, -} from "./search"; - -export { - validateDocumentAccess, - getUserCompanyId, - formatResultsForPrompt, - truncateText, - cosineSimilarity, - euclideanDistance, -} from "./utils"; - -export { ragSearchTool, executeRAGSearch } from "./agentic"; diff --git a/apps/web/src/lib/tools/rag/retrievers/bm25-retriever.ts b/apps/web/src/lib/tools/rag/retrievers/bm25-retriever.ts deleted file mode 100644 index e5284f25e..000000000 --- a/apps/web/src/lib/tools/rag/retrievers/bm25-retriever.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Moved to the engine: packages/core/src/rag/retrievers/bm25-retriever.ts. - * - * Kept as a re-export so the existing `~/lib/tools/rag` consumers keep working - * while the boundary inversion described in REPOSITORY.md finishes. New app code - * should import from `@launchstack/core/rag/retrievers` directly. - */ - -export { - getDocumentChunks, - getCompanyChunks, - getMultiDocChunks, - chunksToDocuments, - createDocumentBM25Retriever, - createCompanyBM25Retriever, - createMultiDocBM25Retriever, -} from "@launchstack/search/retrievers"; diff --git a/apps/web/src/lib/tools/rag/retrievers/index.ts b/apps/web/src/lib/tools/rag/retrievers/index.ts deleted file mode 100644 index 797f42a19..000000000 --- a/apps/web/src/lib/tools/rag/retrievers/index.ts +++ /dev/null @@ -1,48 +0,0 @@ -export { - VectorRetriever, - createDocumentVectorRetriever, - createCompanyVectorRetriever, - createMultiDocVectorRetriever, -} from "./vector-retriever"; - -export { - getDocumentChunks, - getCompanyChunks, - getMultiDocChunks, - chunksToDocuments, - createDocumentBM25Retriever, - createCompanyBM25Retriever, - createMultiDocBM25Retriever, -} from "./bm25-retriever"; - -export { - RLMRetriever, - createRLMRetriever, - getDocumentSummary, - getStructureContent, -} from "./rlm-retriever"; - -export type { - DocumentOverview, - StructureNode, - SectionWithCost, - SectionPreview, - WorkspaceEntry, - TokenBudgetOptions, - WorkspaceStoreOptions, -} from "./rlm-retriever"; - -export { GraphRetriever, createGraphRetriever } from "./graph-retriever"; - -export { - Neo4jGraphRetriever, - createNeo4jGraphRetriever, - shouldUseNeo4jRetriever, -} from "./neo4j-graph-retriever"; - -export { - NotesRetriever, - createDocumentNotesRetriever, - createCompanyNotesRetriever, - createMultiDocNotesRetriever, -} from "./notes-retriever"; diff --git a/apps/web/src/lib/tools/rag/retrievers/vector-retriever.ts b/apps/web/src/lib/tools/rag/retrievers/vector-retriever.ts deleted file mode 100644 index 10c6520c9..000000000 --- a/apps/web/src/lib/tools/rag/retrievers/vector-retriever.ts +++ /dev/null @@ -1,14 +0,0 @@ -/** - * Moved to the engine: packages/core/src/rag/retrievers/vector-retriever.ts. - * - * Kept as a re-export so the existing `~/lib/tools/rag` consumers keep working - * while the boundary inversion described in REPOSITORY.md finishes. New app code - * should import from `@launchstack/core/rag/retrievers` directly. - */ - -export { - VectorRetriever, - createDocumentVectorRetriever, - createCompanyVectorRetriever, - createMultiDocVectorRetriever, -} from "@launchstack/search/retrievers"; diff --git a/apps/web/src/lib/tools/rag/search/index.ts b/apps/web/src/lib/tools/rag/search/index.ts deleted file mode 100644 index 65b16b740..000000000 --- a/apps/web/src/lib/tools/rag/search/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export { - createOpenAIEmbeddings, - createDocumentEnsembleRetriever, - createCompanyEnsembleRetriever, - createMultiDocEnsembleRetriever, - documentEnsembleSearch, - companyEnsembleSearch, - multiDocEnsembleSearch, -} from "./ensemble-search"; diff --git a/apps/web/src/lib/tools/rag/types.ts b/apps/web/src/lib/tools/rag/types.ts deleted file mode 100644 index d9685c91b..000000000 --- a/apps/web/src/lib/tools/rag/types.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Moved to the engine: packages/core/src/rag/search-types.ts. - * - * Re-exported here so the existing `~/lib/tools/rag` consumers keep working - * while the boundary inversion described in REPOSITORY.md finishes. Keeping one - * definition matters more than the import path: the moved retrievers now return - * core's types, and a second structurally-identical copy in the app would drift. - */ - -export type { - SearchScope, - RetrievalMethod, - BaseSearchMetadata, - SearchResult, - DocumentSearchResult, - CompanySearchResult, - MultiDocSearchResult, - SearchFilters, - EnsembleSearchOptions, - DocumentSearchOptions, - CompanySearchOptions, - MultiDocSearchOptions, - ChunkRow, - ANNResult, - ANNStrategy, - ANNConfig, - DocumentCluster, - EmbeddingsProvider, - RAGSearchResult, - RAGSearchInput, -} from "@launchstack/search/search-types"; diff --git a/apps/web/src/lib/tools/rag/utils.ts b/apps/web/src/lib/tools/rag/utils.ts deleted file mode 100644 index bad31fba8..000000000 --- a/apps/web/src/lib/tools/rag/utils.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { db } from "~/server/db/index"; -import { eq } from "drizzle-orm"; -import { document } from "@launchstack/store/schema"; -import { users } from "~/server/db/schema"; -import type { SearchResult } from "./types"; - -export async function validateDocumentAccess( - userId: string, - requestedDocIds: (string | number)[] -): Promise<{ - validDocIds: number[]; - documentTitles: Map; - companyId: string | null; -}> { - const [userInfo] = await db.select().from(users).where(eq(users.userId, userId)); - - if (!userInfo) { - return { validDocIds: [], documentTitles: new Map(), companyId: null }; - } - - const companyId = userInfo.companyId; - const numericIds = requestedDocIds.map(id => Number(id)); - - const docs = await db - .select({ - id: document.id, - title: document.title, - }) - .from(document) - .where(eq(document.companyId, companyId)); - - const validDocIds = docs.map(d => d.id).filter(id => numericIds.includes(id)); - - const documentTitles = new Map(); - docs.forEach(d => { - if (numericIds.includes(d.id)) { - documentTitles.set(d.id, d.title); - } - }); - - return { validDocIds, documentTitles, companyId: companyId.toString() }; -} - -export async function getUserCompanyId(userId: string): Promise { - const [userInfo] = await db - .select({ companyId: users.companyId }) - .from(users) - .where(eq(users.userId, userId)); - - return userInfo?.companyId ? userInfo.companyId.toString() : null; -} - -export function formatResultsForPrompt( - results: SearchResult[], - documentTitles?: Map -): string { - if (results.length === 0) { - return ""; - } - - const byDocument = new Map(); - for (const result of results) { - const docId = result.metadata.documentId; - if (docId !== undefined) { - if (!byDocument.has(docId)) { - byDocument.set(docId, []); - } - byDocument.get(docId)!.push(result); - } - } - - const sections: string[] = []; - - for (const [docId, docResults] of byDocument.entries()) { - const title = - documentTitles?.get(docId) ?? - docResults[0]?.metadata.documentTitle ?? - `Document ${docId}`; - - docResults.sort((a, b) => (a.metadata.page ?? 0) - (b.metadata.page ?? 0)); - - const content = docResults - .map(r => { - const pageInfo = r.metadata.page ? `[Page ${r.metadata.page}]` : ""; - return `${pageInfo}\n${r.pageContent}`; - }) - .join("\n\n"); - - sections.push(`--- ${title} ---\n${content}`); - } - - return sections.join("\n\n"); -} - -export function truncateText(text: string, maxLength: number): string { - if (text.length <= maxLength) { - return text; - } - return text.substring(0, maxLength - 3) + "..."; -} - -export function cosineSimilarity(a: number[], b: number[]): number { - if (a.length !== b.length) return 0; - - let dotProduct = 0; - let normA = 0; - let normB = 0; - - for (let i = 0; i < a.length; i++) { - dotProduct += a[i]! * b[i]!; - normA += a[i]! * a[i]!; - normB += b[i]! * b[i]!; - } - - if (normA === 0 || normB === 0) return 0; - return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); -} - -export function euclideanDistance(a: number[], b: number[]): number { - if (a.length !== b.length) return Infinity; - - let sum = 0; - for (let i = 0; i < a.length; i++) { - const diff = a[i]! - b[i]!; - sum += diff * diff; - } - return Math.sqrt(sum); -} diff --git a/apps/web/src/server/notes/embed-note.ts b/apps/web/src/server/notes/embed-note.ts index 73ced7b5c..7292bbbe2 100644 --- a/apps/web/src/server/notes/embed-note.ts +++ b/apps/web/src/server/notes/embed-note.ts @@ -10,17 +10,16 @@ */ import { eq } from "drizzle-orm"; -import { OpenAIEmbeddings } from "@langchain/openai"; import { document } from "@launchstack/store/schema"; import { db } from "~/server/db"; import { type NoteAnchor } from "~/server/db/schema"; import { documentNotes, documentNoteEmbeddings } from "~/server/db/schema"; import { + createNotesEmbeddingsProvider, EMBEDDING_DIM, EMBEDDING_MODEL, EMBEDDING_SHORT_DIM, - resolveEmbeddingConfig, } from "./embedding-config"; /** @@ -71,9 +70,9 @@ export async function embedNote(noteId: number): Promise { return; } - // Both halves or neither — see resolveEmbeddingConfig. - const { apiKey, baseURL } = resolveEmbeddingConfig(); - if (!apiKey || !baseURL) { + // Both halves or neither — see createNotesEmbeddingsProvider. + const provider = createNotesEmbeddingsProvider(); + if (!provider) { console.warn( "[embedNote] no embedding endpoint configured (EMBEDDING_API_BASE_URL " + "+ EMBEDDING_API_KEY, or AI_BASE_URL + AI_API_KEY) — skipping", @@ -81,14 +80,7 @@ export async function embedNote(noteId: number): Promise { return; } - const client = new OpenAIEmbeddings({ - openAIApiKey: apiKey, - modelName: EMBEDDING_MODEL, - dimensions: EMBEDDING_DIM, - configuration: { baseURL }, - }); - - const [embedding] = await client.embedDocuments([embeddingText]); + const embedding = await provider.embedQuery(embeddingText); if (!embedding || embedding.length !== EMBEDDING_DIM) { console.warn( `[embedNote] unexpected embedding length ${embedding?.length ?? "null"}`, diff --git a/apps/web/src/server/notes/embedding-config.ts b/apps/web/src/server/notes/embedding-config.ts index 3548c072a..2d446cb72 100644 --- a/apps/web/src/server/notes/embedding-config.ts +++ b/apps/web/src/server/notes/embedding-config.ts @@ -5,6 +5,8 @@ * silently nor inconsistently. */ +import { generateEmbeddings, type EmbeddingsProvider } from "@launchstack/llm/embeddings"; + export const EMBEDDING_MODEL = "text-embedding-3-large"; export const EMBEDDING_DIM = 1536; export const EMBEDDING_SHORT_DIM = 512; @@ -42,3 +44,33 @@ export function resolveEmbeddingConfig(): EmbeddingProviderConfig { return { apiKey: undefined, baseURL: undefined }; } + +/** + * The notes pipeline's one embeddings provider, generated through + * @launchstack/llm's embedding service — no direct HTTP client here. Returns + * null when no endpoint pair is configured so each caller keeps its own + * skip/warn semantics (embedding a note is best-effort; searching without + * an endpoint just returns nothing). + */ +export function createNotesEmbeddingsProvider(): EmbeddingsProvider | null { + const { apiKey, baseURL } = resolveEmbeddingConfig(); + if (!apiKey || !baseURL) return null; + + const config = { + apiKey, + baseUrl: baseURL, + model: EMBEDDING_MODEL, + dimensions: EMBEDDING_DIM, + }; + + return { + embedQuery: async (query: string) => { + const { embeddings } = await generateEmbeddings([query], config); + return embeddings[0] ?? []; + }, + embedDocuments: async (documents: string[]) => { + const { embeddings } = await generateEmbeddings(documents, config); + return embeddings; + }, + }; +} diff --git a/apps/web/src/lib/tools/rag/retrievers/notes-retriever.ts b/apps/web/src/server/notes/notes-retriever.ts similarity index 98% rename from apps/web/src/lib/tools/rag/retrievers/notes-retriever.ts rename to apps/web/src/server/notes/notes-retriever.ts index 9092192d9..31b629178 100644 --- a/apps/web/src/lib/tools/rag/retrievers/notes-retriever.ts +++ b/apps/web/src/server/notes/notes-retriever.ts @@ -19,7 +19,7 @@ import { Document } from "@langchain/core/documents"; import type { CallbackManagerForRetrieverRun } from "@langchain/core/callbacks/manager"; import { db, toRows } from "~/server/db/index"; -import type { EmbeddingsProvider, SearchScope } from "../types"; +import type { EmbeddingsProvider, SearchScope } from "@launchstack/retrieval/search-types"; /** * Notes retrievers extend the global `SearchScope` with a "user" branch so diff --git a/apps/web/src/server/notes/search.ts b/apps/web/src/server/notes/search.ts index 48c4557e4..12896b006 100644 --- a/apps/web/src/server/notes/search.ts +++ b/apps/web/src/server/notes/search.ts @@ -7,14 +7,12 @@ import { sql } from "drizzle-orm"; import { T } from "~/server/db/tables"; -import { OpenAIEmbeddings } from "@langchain/openai"; import { db, toRows } from "~/server/db/index"; import { + createNotesEmbeddingsProvider, EMBEDDING_DIM, - EMBEDDING_MODEL, EMBEDDING_SHORT_DIM, - resolveEmbeddingConfig, } from "./embedding-config"; export type NoteSearchScope = "user" | "document" | "company"; @@ -69,19 +67,11 @@ export async function searchNotes( const trimmed = query.trim(); if (!trimmed) return []; - // Both halves or neither — an endpoint without a key, or a key without an - // endpoint, would let the SDK fall back to its own vendor default. - const { apiKey, baseURL } = resolveEmbeddingConfig(); - if (!apiKey || !baseURL) return []; + // Both halves or neither — see createNotesEmbeddingsProvider. + const provider = createNotesEmbeddingsProvider(); + if (!provider) return []; - const client = new OpenAIEmbeddings({ - openAIApiKey: apiKey, - modelName: EMBEDDING_MODEL, - dimensions: EMBEDDING_DIM, - configuration: { baseURL }, - }); - - const embedding = await client.embedQuery(trimmed); + const embedding = await provider.embedQuery(trimmed); if (!embedding || embedding.length !== EMBEDDING_DIM) return []; const short = embedding.slice(0, EMBEDDING_SHORT_DIM); diff --git a/apps/web/src/server/rag/access.ts b/apps/web/src/server/rag/access.ts new file mode 100644 index 000000000..dbf730569 --- /dev/null +++ b/apps/web/src/server/rag/access.ts @@ -0,0 +1,57 @@ +/** + * App-side access resolution for retrieval: which documents a user may + * search, and their titles. This is product-schema knowledge (the `users` + * table), so it stays in apps/web and is injected where a retrieval tool + * needs it (see @launchstack/retrieval/tools/rag-search-tool). + */ + +import { db } from "~/server/db/index"; +import { eq } from "drizzle-orm"; +import { document } from "@launchstack/store/schema"; +import { users } from "~/server/db/schema"; + +export async function validateDocumentAccess( + userId: string, + requestedDocIds: (string | number)[] +): Promise<{ + validDocIds: number[]; + documentTitles: Map; + companyId: string | null; +}> { + const [userInfo] = await db.select().from(users).where(eq(users.userId, userId)); + + if (!userInfo) { + return { validDocIds: [], documentTitles: new Map(), companyId: null }; + } + + const companyId = userInfo.companyId; + const numericIds = requestedDocIds.map(id => Number(id)); + + const docs = await db + .select({ + id: document.id, + title: document.title, + }) + .from(document) + .where(eq(document.companyId, companyId)); + + const validDocIds = docs.map(d => d.id).filter(id => numericIds.includes(id)); + + const documentTitles = new Map(); + docs.forEach(d => { + if (numericIds.includes(d.id)) { + documentTitles.set(d.id, d.title); + } + }); + + return { validDocIds, documentTitles, companyId: companyId.toString() }; +} + +export async function getUserCompanyId(userId: string): Promise { + const [userInfo] = await db + .select({ companyId: users.companyId }) + .from(users) + .where(eq(users.userId, userId)); + + return userInfo?.companyId ? userInfo.companyId.toString() : null; +} diff --git a/apps/web/src/server/rag/ensemble.ts b/apps/web/src/server/rag/ensemble.ts new file mode 100644 index 000000000..73d66f934 --- /dev/null +++ b/apps/web/src/server/rag/ensemble.ts @@ -0,0 +1,54 @@ +/** + * The app's composition seam for @launchstack/retrieval's ensemble. + * + * The package reads no env: this module translates the deployment's flags + * into ensemble config exactly once, at module load, and registers the + * app-owned notes leg (notes live in product schema, so the retriever stays + * in apps/web and joins the ensemble by injection). Route handlers import + * the search functions from here, not from the package, so retrieval can + * never run before this configuration has happened. + */ + +import { env } from "~/env"; +import { + configureEnsemble, + documentEnsembleSearch, + companyEnsembleSearch, + multiDocEnsembleSearch, + createDocumentEnsembleRetriever, + createCompanyEnsembleRetriever, + createMultiDocEnsembleRetriever, + createOpenAIEmbeddings, + createEmbeddingsForIndex, + type NotesLegProvider, +} from "@launchstack/retrieval/algorithms/ensemble"; +import { + createDocumentNotesRetriever, + createCompanyNotesRetriever, + createMultiDocNotesRetriever, +} from "~/server/notes/notes-retriever"; + +const notesLegs: NotesLegProvider = { + createDocumentLeg: (documentId, embeddings, topK) => + createDocumentNotesRetriever(documentId, embeddings, topK), + createCompanyLeg: (companyId, embeddings, topK) => + createCompanyNotesRetriever(companyId, embeddings, topK), + createMultiDocLeg: (documentIds, embeddings, topK) => + createMultiDocNotesRetriever(documentIds, embeddings, topK), +}; + +configureEnsemble({ + graphRetrieval: env.server.ENABLE_GRAPH_RETRIEVER === true, + notesLegs: env.server.ENABLE_NOTES_RETRIEVER === true ? notesLegs : null, +}); + +export { + documentEnsembleSearch, + companyEnsembleSearch, + multiDocEnsembleSearch, + createDocumentEnsembleRetriever, + createCompanyEnsembleRetriever, + createMultiDocEnsembleRetriever, + createOpenAIEmbeddings, + createEmbeddingsForIndex, +}; diff --git a/apps/web/src/server/rag/index.ts b/apps/web/src/server/rag/index.ts deleted file mode 100644 index 13d13ce4b..000000000 --- a/apps/web/src/server/rag/index.ts +++ /dev/null @@ -1,52 +0,0 @@ -export type { - SearchScope, - RetrievalMethod, - BaseSearchMetadata, - SearchResult, - DocumentSearchResult, - CompanySearchResult, - MultiDocSearchResult, - EnsembleSearchOptions, - DocumentSearchOptions, - CompanySearchOptions, - MultiDocSearchOptions, - ChunkRow, - ANNResult, - ANNStrategy, - ANNConfig, - DocumentCluster, - EmbeddingsProvider, -} from "./types"; - -export { - VectorRetriever, - createDocumentVectorRetriever, - createCompanyVectorRetriever, - createMultiDocVectorRetriever, - getDocumentChunks, - getCompanyChunks, - getMultiDocChunks, - chunksToDocuments, - createDocumentBM25Retriever, - createCompanyBM25Retriever, - createMultiDocBM25Retriever, -} from "./retrievers"; - -export { - createOpenAIEmbeddings, - createDocumentEnsembleRetriever, - createCompanyEnsembleRetriever, - createMultiDocEnsembleRetriever, - documentEnsembleSearch, - companyEnsembleSearch, - multiDocEnsembleSearch, -} from "./search"; - -export { - validateDocumentAccess, - getUserCompanyId, - formatResultsForPrompt, - truncateText, - cosineSimilarity, - euclideanDistance, -} from "./utils"; diff --git a/apps/web/src/server/rag/port.ts b/apps/web/src/server/rag/port.ts index 6946b9a90..d31a14def 100644 --- a/apps/web/src/server/rag/port.ts +++ b/apps/web/src/server/rag/port.ts @@ -1,23 +1,28 @@ /** - * Concrete RagPort implementation that wraps the app's existing ensemble - * search pipeline in ~/lib/tools/rag. This is what apps/web hands to - * createEngine so features can run retrieval queries without importing - * the RAG stack directly. + * Concrete RagPort implementation wrapping @launchstack/retrieval's ensemble + * search. This is what apps/web hands to createEngine so features can run + * retrieval queries without importing the retrieval stack directly. + * + * Imports go through ~/server/rag/ensemble (not the package) on purpose: + * that module configures the ensemble — env flags, the app's notes leg — + * at load, so a registered port implies a configured ensemble. The worst + * retrieval failure is a silent one (ragCompanySearchSafe degrades a broken + * port to empty context, not an error), so registration is logged loudly. * * The embedding model is created once per search call — the underlying * createOpenAIEmbeddings() is cheap to construct and the pipeline mutates * per-query options anyway. */ -import type { RagPort, CompanySearchOptions, RagSearchResult } from "@launchstack/search"; -import { - companyEnsembleSearch, - createOpenAIEmbeddings, - type CompanySearchOptions as AppCompanySearchOptions, - type SearchResult as AppSearchResult, -} from "~/lib/tools/rag"; +import type { RagPort, CompanySearchOptions, RagSearchResult } from "@launchstack/retrieval"; +import { companyEnsembleSearch, createOpenAIEmbeddings } from "~/server/rag/ensemble"; +import type { + CompanySearchOptions as AppCompanySearchOptions, + SearchResult as AppSearchResult, +} from "@launchstack/retrieval/search-types"; export function createAppRagPort(): RagPort { + console.log("[rag/port] RagPort registered (ensemble configured via ~/server/rag/ensemble)"); return { async companyEnsembleSearch( query: string, diff --git a/apps/web/src/server/rag/retrievers/graph-retriever.ts b/apps/web/src/server/rag/retrievers/graph-retriever.ts deleted file mode 100644 index 125bd3aad..000000000 --- a/apps/web/src/server/rag/retrievers/graph-retriever.ts +++ /dev/null @@ -1 +0,0 @@ -export { GraphRetriever, createGraphRetriever } from "~/lib/tools/rag/retrievers/graph-retriever"; diff --git a/apps/web/src/server/rag/retrievers/index.ts b/apps/web/src/server/rag/retrievers/index.ts deleted file mode 100644 index e590c66c4..000000000 --- a/apps/web/src/server/rag/retrievers/index.ts +++ /dev/null @@ -1,29 +0,0 @@ -export { - VectorRetriever, - createDocumentVectorRetriever, - createCompanyVectorRetriever, - createMultiDocVectorRetriever, - getDocumentChunks, - getCompanyChunks, - getMultiDocChunks, - chunksToDocuments, - createDocumentBM25Retriever, - createCompanyBM25Retriever, - createMultiDocBM25Retriever, - RLMRetriever, - createRLMRetriever, - getDocumentSummary, - getStructureContent, - GraphRetriever, - createGraphRetriever, -} from "~/lib/tools/rag/retrievers"; - -export type { - DocumentOverview, - StructureNode, - SectionWithCost, - SectionPreview, - WorkspaceEntry, - TokenBudgetOptions, - WorkspaceStoreOptions, -} from "~/lib/tools/rag/retrievers"; diff --git a/apps/web/src/server/rag/retrievers/rlm-retriever.ts b/apps/web/src/server/rag/retrievers/rlm-retriever.ts deleted file mode 100644 index a52e27027..000000000 --- a/apps/web/src/server/rag/retrievers/rlm-retriever.ts +++ /dev/null @@ -1,16 +0,0 @@ -export { - RLMRetriever, - createRLMRetriever, - getDocumentSummary, - getStructureContent, -} from "~/lib/tools/rag/retrievers/rlm-retriever"; - -export type { - DocumentOverview, - StructureNode, - SectionWithCost, - SectionPreview, - WorkspaceEntry, - TokenBudgetOptions, - WorkspaceStoreOptions, -} from "~/lib/tools/rag/retrievers/rlm-retriever"; diff --git a/apps/web/src/server/rag/retrievers/vector-retriever.ts b/apps/web/src/server/rag/retrievers/vector-retriever.ts deleted file mode 100644 index b874e4512..000000000 --- a/apps/web/src/server/rag/retrievers/vector-retriever.ts +++ /dev/null @@ -1,6 +0,0 @@ -export { - VectorRetriever, - createDocumentVectorRetriever, - createCompanyVectorRetriever, - createMultiDocVectorRetriever, -} from "~/lib/tools/rag/retrievers/vector-retriever"; diff --git a/apps/web/src/server/rag/search/ensemble-search.ts b/apps/web/src/server/rag/search/ensemble-search.ts deleted file mode 100644 index d76892a10..000000000 --- a/apps/web/src/server/rag/search/ensemble-search.ts +++ /dev/null @@ -1,9 +0,0 @@ -export { - createOpenAIEmbeddings, - createDocumentEnsembleRetriever, - createCompanyEnsembleRetriever, - createMultiDocEnsembleRetriever, - documentEnsembleSearch, - companyEnsembleSearch, - multiDocEnsembleSearch, -} from "~/lib/tools/rag/search/ensemble-search"; diff --git a/apps/web/src/server/rag/types.ts b/apps/web/src/server/rag/types.ts deleted file mode 100644 index c417c05e3..000000000 --- a/apps/web/src/server/rag/types.ts +++ /dev/null @@ -1,19 +0,0 @@ -export type { - SearchScope, - RetrievalMethod, - BaseSearchMetadata, - SearchResult, - DocumentSearchResult, - CompanySearchResult, - MultiDocSearchResult, - EnsembleSearchOptions, - DocumentSearchOptions, - CompanySearchOptions, - MultiDocSearchOptions, - ChunkRow, - ANNResult, - ANNStrategy, - ANNConfig, - DocumentCluster, - EmbeddingsProvider, -} from "~/lib/tools/rag/types"; diff --git a/apps/web/src/server/rag/utils.ts b/apps/web/src/server/rag/utils.ts deleted file mode 100644 index fa3151356..000000000 --- a/apps/web/src/server/rag/utils.ts +++ /dev/null @@ -1,8 +0,0 @@ -export { - validateDocumentAccess, - getUserCompanyId, - formatResultsForPrompt, - truncateText, - cosineSimilarity, - euclideanDistance, -} from "~/lib/tools/rag/utils"; diff --git a/apps/worker/package.json b/apps/worker/package.json index 9a7484801..ad066bff6 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -22,7 +22,7 @@ "@launchstack/pipelines": "workspace:^", "@launchstack/runtime": "workspace:^", "@launchstack/schema-generator": "workspace:^", - "@launchstack/search": "workspace:^", + "@launchstack/retrieval": "workspace:^", "@launchstack/store": "workspace:^", "drizzle-orm": "^0.45.1", "inngest": "^3.54.2", diff --git a/eslint.config.js b/eslint.config.js index 5b93377ef..da8f8b4e9 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -254,11 +254,14 @@ const eslintConfig = [ "@launchstack/adapters/*", "@launchstack/features", "@launchstack/features/*", + "@launchstack/search", + "@launchstack/search/*", ], message: - "Deleted package (ADR-008). Import the owning feature package " + - "instead: store/llm/conversion/indexing/search/orchestration/" + - "editing/collab/runtime/engine/pipelines.", + "Deleted package (ADR-008) or renamed brick " + + "(@launchstack/search → @launchstack/retrieval). Import the " + + "owning feature package instead: store/llm/conversion/indexing/" + + "retrieval/orchestration/editing/collab/runtime/engine/pipelines.", }; const frameworkBan = { group: ["next/*", "next", "@clerk/*", "react", "react-dom", "~/*"], @@ -281,7 +284,7 @@ const eslintConfig = [ "llm", "conversion", "indexing", - "search", + "retrieval", "orchestration", "editing", "document-conversion-engine", @@ -412,13 +415,20 @@ const eslintConfig = [ }, }, { - files: ["packages/search/src/**/*.ts"], + files: ["packages/retrieval/src/**/*.ts"], rules: { ...restrict([ legacyBan, frameworkBan, noPipelines, - only(["runtime", "store", "llm", "evidence"], "@launchstack/search"), + // "indexing" is here for exactly one edge: the graph + // algorithm's Neo4j backend reuses indexing's graph + // client (isNeo4jConfigured/getNeo4jSession). Indexing + // sits below retrieval in the DAG, so the edge is legal. + only( + ["runtime", "store", "llm", "evidence", "indexing"], + "@launchstack/retrieval" + ), ]), ...noEnv, }, @@ -485,7 +495,7 @@ const eslintConfig = [ }, }, // @launchstack/tools — shared, contract-typed capabilities the - // verticals compose. A brick above search, below pipelines; may + // verticals compose. A brick above retrieval, below pipelines; may // read process.env (social/web-research provider keys, inherited // from the features tier where these capabilities were born). { @@ -495,7 +505,7 @@ const eslintConfig = [ frameworkBan, noPipelines, only( - ["runtime", "evidence", "store", "llm", "search", "conversion"], + ["runtime", "evidence", "store", "llm", "retrieval", "conversion"], "@launchstack/tools" ), ]), diff --git a/packages/collab/scripts/fix-esm-specifiers.mjs b/packages/collab/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/collab/scripts/fix-esm-specifiers.mjs +++ b/packages/collab/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/conversion/scripts/fix-esm-specifiers.mjs b/packages/conversion/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/conversion/scripts/fix-esm-specifiers.mjs +++ b/packages/conversion/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/editing/scripts/fix-esm-specifiers.mjs b/packages/editing/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/editing/scripts/fix-esm-specifiers.mjs +++ b/packages/editing/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/engine/package.json b/packages/engine/package.json index 73ad012c3..11a2e08af 100644 --- a/packages/engine/package.json +++ b/packages/engine/package.json @@ -45,7 +45,7 @@ "@launchstack/llm": "workspace:^", "@launchstack/conversion": "workspace:^", "@launchstack/indexing": "workspace:^", - "@launchstack/search": "workspace:^", + "@launchstack/retrieval": "workspace:^", "@launchstack/orchestration": "workspace:^", "zod": "^3.23.8" }, diff --git a/packages/engine/scripts/fix-esm-specifiers.mjs b/packages/engine/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/engine/scripts/fix-esm-specifiers.mjs +++ b/packages/engine/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/engine/src/config/types.ts b/packages/engine/src/config/types.ts index 40f099931..6176f1911 100644 --- a/packages/engine/src/config/types.ts +++ b/packages/engine/src/config/types.ts @@ -9,7 +9,7 @@ export type { LoggerPort } from "@launchstack/runtime"; import type { LoggerPort } from "@launchstack/runtime"; import type { JobDispatcherPort } from "@launchstack/runtime"; import type { CreditsPort, MeteringMode } from "@launchstack/store/credits"; -import type { RagPort } from "@launchstack/search"; +import type { RagPort } from "@launchstack/retrieval"; import type { ChatModelsConfig } from "@launchstack/llm"; import type { AuxiliaryOpenAIConfig } from "@launchstack/llm"; diff --git a/packages/engine/src/engine.ts b/packages/engine/src/engine.ts index 8142132ee..f3583c8b9 100644 --- a/packages/engine/src/engine.ts +++ b/packages/engine/src/engine.ts @@ -16,7 +16,7 @@ import { import { configureStorage } from "@launchstack/runtime"; import { configureJobDispatcher } from "@launchstack/runtime"; import { configureCredits, configureMetering } from "@launchstack/store/credits"; -import { configureRag } from "@launchstack/search"; +import { configureRag } from "@launchstack/retrieval"; import { configureChatModels } from "@launchstack/llm"; import { configureAuxiliaryOpenAI } from "@launchstack/llm"; import { configureVlmEnrichment } from "@launchstack/conversion/ocr"; diff --git a/packages/evidence/scripts/fix-esm-specifiers.mjs b/packages/evidence/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/evidence/scripts/fix-esm-specifiers.mjs +++ b/packages/evidence/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/indexing/scripts/fix-esm-specifiers.mjs b/packages/indexing/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/indexing/scripts/fix-esm-specifiers.mjs +++ b/packages/indexing/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/llm/scripts/fix-esm-specifiers.mjs b/packages/llm/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/llm/scripts/fix-esm-specifiers.mjs +++ b/packages/llm/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/orchestration/scripts/fix-esm-specifiers.mjs b/packages/orchestration/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/orchestration/scripts/fix-esm-specifiers.mjs +++ b/packages/orchestration/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/search/LICENSE b/packages/retrieval/LICENSE similarity index 100% rename from packages/search/LICENSE rename to packages/retrieval/LICENSE diff --git a/packages/retrieval/README.md b/packages/retrieval/README.md new file mode 100644 index 000000000..f5de9b18a --- /dev/null +++ b/packages/retrieval/README.md @@ -0,0 +1,78 @@ +# @launchstack/retrieval + +Question in, cited answer out. Every retrieval algorithm as a documented +folder behind a replaceable port, plus the retrieval-facing tools built on +them. It deliberately does not contain index-time work — embeddings are +generated by llm and persisted by indexing; this package only reads. + +## Layout + +Two sections, one folder per algorithm or tool. Each folder carries its own +README (what it is, how it works, when it wins, tuning knobs), an `index.ts` +as its only public surface, and its tests beside the implementation — open +the folder before tuning the algorithm. + +``` +src/ +├─ index.ts — RagPort, slot, citation builder +├─ algorithms/ — given a query, which rows come back, in what order +│ ├─ bm25/ — lexical ranking (+ SQL-side FTS variant) +│ ├─ vector/ — semantic search; strategies/ holds the named ANN variants +│ ├─ fusion/ — Reciprocal Rank Fusion, page-level hybrid search +│ ├─ ensemble/ — leg orchestration, weights, injected config +│ ├─ rlm/ — hierarchical, token-budgeted navigation +│ ├─ graph/ — knowledge-graph traversal (pg + neo4j backends) +│ └─ reranking/ — second-pass reordering +└─ tools/ — how an agent, pipeline, or route consumes retrieval + ├─ citation-builder/ + ├─ grounded-retrieval/ + ├─ rag-search-tool/ + └─ rlm-search/ +``` + +Algorithms never import tools; tools compose algorithms and the port. + +## Install + +```bash +pnpm add @launchstack/retrieval +``` + +## Use + +```ts +import { buildCitations, configureRag, getRag } from "@launchstack/retrieval"; +import { documentEnsembleSearch } from "@launchstack/retrieval/algorithms/ensemble"; +import { retrieveCompanySnippets } from "@launchstack/retrieval/tools/grounded-retrieval"; +``` + +## API + +| Subpath | What it is | +| --- | --- | +| `.` | the rag port, slot, and citation builder | +| `./algorithms` | everything below, one barrel | +| `./algorithms/` | bm25 · vector · fusion · ensemble · rlm · graph · reranking | +| `./tools` | everything below, one barrel | +| `./tools/` | citation-builder · grounded-retrieval · rag-search-tool · rlm-search | +| `./search-types` | the ensemble's result vocabulary | + +`./retrievers`, `./reranking`, and `./citation-builder` survive one release +as aliases for the pre-consolidation layout. + +## Configuration + +Nothing here reads `process.env`. Configuration is injected by the +composition root — `createEngine(config)` in `@launchstack/engine`, or the +package's own `configure*` hooks when used standalone: `configureRag(port)` +for the port, `configureEnsemble({ graphRetrieval, notesLegs })` for the +ensemble's optional legs. + +## Stability + +0.x. `relevance` is deliberately not called confidence — extraction +confidence belongs to evidence, retrieval relevance to the query. + +## License + +Apache-2.0 — see [LICENSE](LICENSE). diff --git a/packages/retrieval/package.json b/packages/retrieval/package.json new file mode 100644 index 000000000..d2ff19b29 --- /dev/null +++ b/packages/retrieval/package.json @@ -0,0 +1,151 @@ +{ + "name": "@launchstack/retrieval", + "version": "0.1.0", + "description": "Question in, cited answer out: hybrid retrieval (BM25 + vector ensemble behind a replaceable port), second-pass reranking, and the citation builder that turns permission-scoped retrieval rows into stable anchored citations.", + "license": "Apache-2.0", + "type": "module", + "sideEffects": false, + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts", + "./algorithms": "./src/algorithms/index.ts", + "./algorithms/bm25": "./src/algorithms/bm25/index.ts", + "./algorithms/vector": "./src/algorithms/vector/index.ts", + "./algorithms/fusion": "./src/algorithms/fusion/index.ts", + "./algorithms/ensemble": "./src/algorithms/ensemble/index.ts", + "./algorithms/rlm": "./src/algorithms/rlm/index.ts", + "./algorithms/graph": "./src/algorithms/graph/index.ts", + "./algorithms/reranking": "./src/algorithms/reranking/index.ts", + "./tools": "./src/tools/index.ts", + "./tools/citation-builder": "./src/tools/citation-builder/index.ts", + "./tools/rag-search-tool": "./src/tools/rag-search-tool/index.ts", + "./tools/grounded-retrieval": "./src/tools/grounded-retrieval/index.ts", + "./tools/rlm-search": "./src/tools/rlm-search/index.ts", + "./search-types": "./src/search-types.ts", + "./types": "./src/types.ts", + "./retrievers": "./src/algorithms/index.ts", + "./reranking": "./src/algorithms/reranking/index.ts", + "./citation-builder": "./src/tools/citation-builder/index.ts", + "./package.json": "./package.json" + }, + "files": [ + "dist", + "README.md", + "LICENSE" + ], + "publishConfig": { + "access": "public", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./algorithms": { + "types": "./dist/algorithms/index.d.ts", + "default": "./dist/algorithms/index.js" + }, + "./algorithms/bm25": { + "types": "./dist/algorithms/bm25/index.d.ts", + "default": "./dist/algorithms/bm25/index.js" + }, + "./algorithms/vector": { + "types": "./dist/algorithms/vector/index.d.ts", + "default": "./dist/algorithms/vector/index.js" + }, + "./algorithms/fusion": { + "types": "./dist/algorithms/fusion/index.d.ts", + "default": "./dist/algorithms/fusion/index.js" + }, + "./algorithms/ensemble": { + "types": "./dist/algorithms/ensemble/index.d.ts", + "default": "./dist/algorithms/ensemble/index.js" + }, + "./algorithms/rlm": { + "types": "./dist/algorithms/rlm/index.d.ts", + "default": "./dist/algorithms/rlm/index.js" + }, + "./algorithms/graph": { + "types": "./dist/algorithms/graph/index.d.ts", + "default": "./dist/algorithms/graph/index.js" + }, + "./algorithms/reranking": { + "types": "./dist/algorithms/reranking/index.d.ts", + "default": "./dist/algorithms/reranking/index.js" + }, + "./tools": { + "types": "./dist/tools/index.d.ts", + "default": "./dist/tools/index.js" + }, + "./tools/citation-builder": { + "types": "./dist/tools/citation-builder/index.d.ts", + "default": "./dist/tools/citation-builder/index.js" + }, + "./tools/rag-search-tool": { + "types": "./dist/tools/rag-search-tool/index.d.ts", + "default": "./dist/tools/rag-search-tool/index.js" + }, + "./tools/grounded-retrieval": { + "types": "./dist/tools/grounded-retrieval/index.d.ts", + "default": "./dist/tools/grounded-retrieval/index.js" + }, + "./tools/rlm-search": { + "types": "./dist/tools/rlm-search/index.d.ts", + "default": "./dist/tools/rlm-search/index.js" + }, + "./search-types": { + "types": "./dist/search-types.d.ts", + "default": "./dist/search-types.js" + }, + "./types": { + "types": "./dist/types.d.ts", + "default": "./dist/types.js" + }, + "./retrievers": { + "types": "./dist/algorithms/index.d.ts", + "default": "./dist/algorithms/index.js" + }, + "./reranking": { + "types": "./dist/algorithms/reranking/index.d.ts", + "default": "./dist/algorithms/reranking/index.js" + }, + "./citation-builder": { + "types": "./dist/tools/citation-builder/index.d.ts", + "default": "./dist/tools/citation-builder/index.js" + }, + "./package.json": "./package.json" + } + }, + "scripts": { + "build": "tsc -p tsconfig.build.json && node ./scripts/fix-esm-specifiers.mjs", + "clean": "rm -rf dist .tsbuildinfo", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@launchstack/runtime": "workspace:^", + "@launchstack/store": "workspace:^", + "@launchstack/llm": "workspace:^", + "@launchstack/indexing": "workspace:^", + "@langchain/community": "^0.3.56", + "@langchain/core": "^0.3.74", + "langchain": "^0.3.33", + "drizzle-orm": "^0.45.1", + "zod": "^3.23.8", + "@launchstack/evidence": "workspace:^" + }, + "devDependencies": { + "typescript": "^5.9.2", + "vitest": "^3.0.5" + }, + "peerDependencies": { + "neo4j-driver": "^6.0.0" + }, + "peerDependenciesMeta": { + "neo4j-driver": { + "optional": true + } + } +} diff --git a/packages/search/scripts/fix-esm-specifiers.mjs b/packages/retrieval/scripts/fix-esm-specifiers.mjs similarity index 98% rename from packages/search/scripts/fix-esm-specifiers.mjs rename to packages/retrieval/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/search/scripts/fix-esm-specifiers.mjs +++ b/packages/retrieval/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/retrieval/src/algorithms/bm25/README.md b/packages/retrieval/src/algorithms/bm25/README.md new file mode 100644 index 000000000..74c8feb9b --- /dev/null +++ b/packages/retrieval/src/algorithms/bm25/README.md @@ -0,0 +1,31 @@ +# bm25 — lexical ranking + +**What it is.** Keyword relevance over document chunks. Two layers share this +folder: SQL-side candidate fetch (current-version chunks scoped to a +document, a company, or a document set) and in-memory BM25 ranking over +those candidates via LangChain's `BM25Retriever`. + +**How it works.** BM25 scores a chunk by how often the query terms appear in +it (term frequency, saturating), how rare those terms are across the corpus +(inverse document frequency), and how long the chunk is (length +normalization). It needs no embeddings, no model call, and no index beyond +the chunk rows themselves — the fetch joins through `document` so only the +current version's chunks are ever ranked. + +**When it wins.** Exact identifiers, names, codes, and rare terms — the +queries where semantic similarity is too fuzzy. It is also the ensemble's +fallback when vector search fails, because it cannot lose the query's own +words. It loses on paraphrase and synonymy; that is what the vector leg is +for. + +**Knobs.** `topK` per creator. The ensemble hands each leg +`topK × RERANK_CANDIDATE_MULTIPLIER` candidates and fuses ranks downstream. + +**Surface.** `create{Document,Company,MultiDoc}BM25Retriever`, plus the raw +chunk fetchers (`get*Chunks`, `chunksToDocuments`) the fallback path reuses. + +**FTS variant.** `fts.ts` is the SQL-side sibling: `to_tsquery` OR-matching +ranked by `ts_rank`, returning a plain ranked list instead of LangChain +Documents. The fusion folder's hybrid search uses it when it needs page-level +ranks without retriever machinery. Different scoring model than BM25 (no +term-frequency saturation), same job: literal-term relevance. diff --git a/packages/search/src/retrievers/bm25-retriever.ts b/packages/retrieval/src/algorithms/bm25/bm25.ts similarity index 98% rename from packages/search/src/retrievers/bm25-retriever.ts rename to packages/retrieval/src/algorithms/bm25/bm25.ts index 935c6a635..fcf2c8a2e 100644 --- a/packages/search/src/retrievers/bm25-retriever.ts +++ b/packages/retrieval/src/algorithms/bm25/bm25.ts @@ -4,7 +4,7 @@ import { Document } from "@langchain/core/documents"; import { getDb } from "@launchstack/store/client"; import { documentSections, document } from "@launchstack/store/schema"; -import type { ChunkRow, SearchScope } from "../search-types"; +import type { ChunkRow, SearchScope } from "../../search-types"; export async function getDocumentChunks(documentId: number): Promise { // Join to `document` so only chunks from its current version are returned. diff --git a/packages/retrieval/src/algorithms/bm25/fts.ts b/packages/retrieval/src/algorithms/bm25/fts.ts new file mode 100644 index 000000000..1c68eb2a3 --- /dev/null +++ b/packages/retrieval/src/algorithms/bm25/fts.ts @@ -0,0 +1,60 @@ +/** + * Postgres full-text variant of the lexical leg: `to_tsquery` OR-matching + * ranked by `ts_rank`, entirely SQL-side. Cheaper than fetching chunks for + * in-memory BM25 when the caller only needs a ranked list (page-level fusion + * in the hybrid search) rather than LangChain Documents. + */ + +import { and, eq, inArray, sql } from "drizzle-orm"; +import { getDb } from "@launchstack/store/client"; +import { document, documentSections } from "@launchstack/store/schema"; +import type { RankedResult } from "../fusion/rrf"; + +export async function ftsSearch( + query: string, + docIds: number[], + limit = 10 +): Promise { + if (docIds.length === 0) return []; + + const tsQuery = query + .split(/\s+/) + .filter(w => w.length > 1) + .map(w => w.replace(/[^a-zA-Z0-9]/g, "")) + .filter(Boolean) + .join(" | "); + + if (!tsQuery) return []; + + const results = await getDb() + .select({ + id: documentSections.id, + content: documentSections.content, + page: documentSections.pageNumber, + documentId: documentSections.documentId, + rank: sql`ts_rank(to_tsvector('english', ${documentSections.content}), to_tsquery('english', ${tsQuery}))`, + }) + .from(documentSections) + .innerJoin(document, eq(documentSections.documentId, document.id)) + .where( + and( + inArray( + documentSections.documentId, + docIds.map(id => BigInt(id)) + ), + eq(documentSections.versionId, document.currentVersionId), + sql`to_tsvector('english', ${documentSections.content}) @@ to_tsquery('english', ${tsQuery})` + ) + ) + .orderBy( + sql`ts_rank(to_tsvector('english', ${documentSections.content}), to_tsquery('english', ${tsQuery})) DESC` + ) + .limit(limit); + + return results.map((r, idx) => ({ + documentId: Number(r.documentId), + page: r.page ?? 1, + content: r.content, + rank: idx + 1, + })); +} diff --git a/apps/web/src/server/rag/retrievers/bm25-retriever.ts b/packages/retrieval/src/algorithms/bm25/index.ts similarity index 78% rename from apps/web/src/server/rag/retrievers/bm25-retriever.ts rename to packages/retrieval/src/algorithms/bm25/index.ts index dec588a40..cc6c04a6f 100644 --- a/apps/web/src/server/rag/retrievers/bm25-retriever.ts +++ b/packages/retrieval/src/algorithms/bm25/index.ts @@ -6,4 +6,6 @@ export { createDocumentBM25Retriever, createCompanyBM25Retriever, createMultiDocBM25Retriever, -} from "~/lib/tools/rag/retrievers/bm25-retriever"; +} from "./bm25"; + +export { ftsSearch } from "./fts"; diff --git a/packages/retrieval/src/algorithms/ensemble/README.md b/packages/retrieval/src/algorithms/ensemble/README.md new file mode 100644 index 000000000..a96b09b65 --- /dev/null +++ b/packages/retrieval/src/algorithms/ensemble/README.md @@ -0,0 +1,28 @@ +# ensemble — leg orchestration and fusion + +**What it is.** The composition layer: it assembles the retrieval legs +(BM25 + vector always; knowledge-graph and user-notes when configured), +fuses their rankings, and hands the fused list to the reranker. + +**How it works.** Each leg returns `topK × 4` candidates; LangChain's +`EnsembleRetriever` fuses them with weighted Reciprocal Rank Fusion — a +chunk's score is the weighted sum of `1/(rank + c)` across the legs that +returned it, so agreement between legs beats a high rank in any single leg. +Defaults: `[0.4, 0.6]` (bm25, vector), `[0.3, 0.5, 0.2]` with the graph leg, +`+0.15` for the notes leg. After fusion, the optional second-pass reranker +reorders the pool and the top `topK` survive. Every search logs a per-leg +breakdown (`Leg breakdown: chunk=…, note=…`) so a silently dead leg is +visible in the logs, not just a smaller total. + +**Configuration.** Nothing here reads `process.env`. The composition root +calls `configureEnsemble({ graphRetrieval, notesLegs })` (see `config.ts`): +the graph flag turns that leg on (backend picked per call — Neo4j when +configured, Postgres fallback otherwise), and `notesLegs` injects the +app-owned notes retriever, which lives in product schema this package +cannot see. + +**Failure.** A failing ensemble degrades to BM25-only over the same scope — +retrieval never throws to the caller; it narrows. + +**When to touch it.** Weight changes shift relevance for every consumer at +once. Change them against pinned golden queries, not by eye. diff --git a/packages/retrieval/src/algorithms/ensemble/config.ts b/packages/retrieval/src/algorithms/ensemble/config.ts new file mode 100644 index 000000000..32f283eb4 --- /dev/null +++ b/packages/retrieval/src/algorithms/ensemble/config.ts @@ -0,0 +1,55 @@ +/** + * Ensemble runtime configuration, injected by the composition root. + * + * Nothing in this package reads `process.env` — the app decides which legs + * run and hands the decision (and any app-owned legs) over before retrieval + * is used. Two things are injectable: + * + * - `graphRetrieval`: whether the knowledge-graph leg joins the ensemble. + * The backend (Neo4j vs the Postgres fallback) is chosen per call by + * `shouldUseNeo4jRetriever()`; this flag only turns the leg on. + * - `notesLegs`: a provider for the user-notes leg. Notes live in product + * schema owned by the app (not in @launchstack/store), so the retriever + * itself cannot live in this package — the app registers factories and the + * ensemble unions their results like any other leg. No provider, no leg. + */ + +import type { BaseRetriever } from "@langchain/core/retrievers"; +import type { EmbeddingsProvider } from "../../search-types"; + +export interface NotesLegProvider { + createDocumentLeg( + documentId: number, + embeddings: EmbeddingsProvider, + topK: number + ): BaseRetriever; + createCompanyLeg( + companyId: number | string, + embeddings: EmbeddingsProvider, + topK: number + ): BaseRetriever; + createMultiDocLeg( + documentIds: number[], + embeddings: EmbeddingsProvider, + topK: number + ): BaseRetriever; +} + +export interface EnsembleRuntimeConfig { + graphRetrieval: boolean; + notesLegs: NotesLegProvider | null; +} + +let config: EnsembleRuntimeConfig = { + graphRetrieval: false, + notesLegs: null, +}; + +/** Merge-configure; call from the composition root before retrieval runs. */ +export function configureEnsemble(partial: Partial): void { + config = { ...config, ...partial }; +} + +export function getEnsembleConfig(): EnsembleRuntimeConfig { + return config; +} diff --git a/apps/web/src/lib/tools/rag/search/ensemble-search.ts b/packages/retrieval/src/algorithms/ensemble/ensemble.ts similarity index 90% rename from apps/web/src/lib/tools/rag/search/ensemble-search.ts rename to packages/retrieval/src/algorithms/ensemble/ensemble.ts index 31acbf604..a7c9442da 100644 --- a/apps/web/src/lib/tools/rag/search/ensemble-search.ts +++ b/packages/retrieval/src/algorithms/ensemble/ensemble.ts @@ -3,13 +3,12 @@ import { BM25Retriever } from "@langchain/community/retrievers/bm25"; import type { BaseRetriever } from "@langchain/core/retrievers"; import { createEmbeddingModel } from "@launchstack/llm/embeddings"; import { resolveEmbeddingIndex } from "@launchstack/llm/embeddings"; -import { getRerankProvider, isRerankConfigured } from "@launchstack/search/reranking"; -import { env } from "~/env"; +import { getRerankProvider, isRerankConfigured } from "../reranking"; import { createDocumentVectorRetriever, createCompanyVectorRetriever, createMultiDocVectorRetriever, -} from "../retrievers/vector-retriever"; +} from "../vector"; import { createDocumentBM25Retriever, createCompanyBM25Retriever, @@ -18,17 +17,9 @@ import { getCompanyChunks, getMultiDocChunks, chunksToDocuments, -} from "../retrievers/bm25-retriever"; -import { - createNeo4jGraphRetriever, - shouldUseNeo4jRetriever, -} from "../retrievers/neo4j-graph-retriever"; -import { createGraphRetriever } from "../retrievers/graph-retriever"; -import { - createDocumentNotesRetriever, - createCompanyNotesRetriever, - createMultiDocNotesRetriever, -} from "../retrievers/notes-retriever"; +} from "../bm25"; +import { createNeo4jGraphRetriever, shouldUseNeo4jRetriever, createGraphRetriever } from "../graph"; +import { getEnsembleConfig } from "./config"; import type { SearchResult, DocumentSearchOptions, @@ -36,7 +27,7 @@ import type { MultiDocSearchOptions, EmbeddingsProvider, SearchScope, -} from "../types"; +} from "../../search-types"; const DEFAULT_WEIGHTS_2: number[] = [0.4, 0.6]; const DEFAULT_WEIGHTS_3: number[] = [0.3, 0.5, 0.2]; @@ -51,16 +42,23 @@ const NOTES_DEFAULT_WEIGHT = 0.15; const NOTES_MAX_CANDIDATES = 8; function isGraphRetrievalEnabled(): boolean { - return env.server.ENABLE_GRAPH_RETRIEVER === true; + return getEnsembleConfig().graphRetrieval; } /** - * Notes retrieval is opt-in per deployment. Until a workspace actually has - * user-authored notes worth retrieving over, this flag should stay off — - * empty-notes paths run the SQL query anyway and just add noise. + * Per-leg visibility: a silently dead leg (graph peer down, empty notes) is + * invisible in the total count, so log how many candidates each source + * contributed. `source` is stamped by each retriever's Document metadata. */ -function isNotesRetrievalEnabled(): boolean { - return env.server.ENABLE_NOTES_RETRIEVER === true; +function logLegBreakdown(scopeLabel: string, results: Array<{ metadata: object }>): void { + const bySource = new Map(); + for (const r of results) { + const raw = (r.metadata as { source?: unknown }).source; + const source = typeof raw === "string" ? raw : "chunk"; + bySource.set(source, (bySource.get(source) ?? 0) + 1); + } + const parts = [...bySource.entries()].map(([source, n]) => `${source}=${n}`); + console.log(`[EnsembleSearch] Leg breakdown (${scopeLabel}): ${parts.join(", ") || "none"}`); } export function createOpenAIEmbeddings(): EmbeddingsProvider { @@ -102,8 +100,9 @@ export async function createDocumentEnsembleRetriever( } } - if (isNotesRetrievalEnabled()) { - const notesRetriever = createDocumentNotesRetriever( + const notesLegs = getEnsembleConfig().notesLegs; + if (notesLegs) { + const notesRetriever = notesLegs.createDocumentLeg( documentId, emb, Math.min(candidateK, NOTES_MAX_CANDIDATES) @@ -145,8 +144,9 @@ export async function createCompanyEnsembleRetriever( } } - if (isNotesRetrievalEnabled()) { - const notesRetriever = createCompanyNotesRetriever( + const notesLegs = getEnsembleConfig().notesLegs; + if (notesLegs) { + const notesRetriever = notesLegs.createCompanyLeg( companyId, emb, Math.min(candidateK, NOTES_MAX_CANDIDATES) @@ -189,8 +189,9 @@ export async function createMultiDocEnsembleRetriever( } } - if (isNotesRetrievalEnabled()) { - const notesRetriever = createMultiDocNotesRetriever( + const notesLegs = getEnsembleConfig().notesLegs; + if (notesLegs) { + const notesRetriever = notesLegs.createMultiDocLeg( documentIds, emb, Math.min(candidateK, NOTES_MAX_CANDIDATES) @@ -254,6 +255,7 @@ export async function documentEnsembleSearch( }, })); + logLegBreakdown("document", results); const reranked = await rerankResults(query, mapped); return reranked.slice(0, topK); } catch (error) { @@ -299,6 +301,7 @@ export async function companyEnsembleSearch( }, })); + logLegBreakdown("company", results); const reranked = await rerankResults(query, mapped); return reranked.slice(0, topK); } catch (error) { @@ -343,6 +346,7 @@ export async function multiDocEnsembleSearch( }, })); + logLegBreakdown("multi-document", results); const reranked = await rerankResults(query, mapped); return reranked.slice(0, topK); } catch (error) { diff --git a/apps/web/src/server/rag/search/index.ts b/packages/retrieval/src/algorithms/ensemble/index.ts similarity index 55% rename from apps/web/src/server/rag/search/index.ts rename to packages/retrieval/src/algorithms/ensemble/index.ts index 90ce8ed1e..ebf0db4a4 100644 --- a/apps/web/src/server/rag/search/index.ts +++ b/packages/retrieval/src/algorithms/ensemble/index.ts @@ -1,9 +1,17 @@ export { createOpenAIEmbeddings, + createEmbeddingsForIndex, createDocumentEnsembleRetriever, createCompanyEnsembleRetriever, createMultiDocEnsembleRetriever, documentEnsembleSearch, companyEnsembleSearch, multiDocEnsembleSearch, -} from "~/lib/tools/rag/search"; +} from "./ensemble"; + +export { + configureEnsemble, + getEnsembleConfig, + type EnsembleRuntimeConfig, + type NotesLegProvider, +} from "./config"; diff --git a/packages/retrieval/src/algorithms/fusion/README.md b/packages/retrieval/src/algorithms/fusion/README.md new file mode 100644 index 000000000..37a66d4d2 --- /dev/null +++ b/packages/retrieval/src/algorithms/fusion/README.md @@ -0,0 +1,23 @@ +# fusion — rank merging + +**What it is.** How independent ranked lists become one: Reciprocal Rank +Fusion (`rrf.ts`) and the page-granular lexical+dense hybrid search built on +it (`hybrid-search.ts`). + +**How it works.** RRF scores each item `Σ 1/(k + rankᵢ)` across the lists +that contain it (k=60, Cormack et al.) — items several methods agree on +outrank items any single method loves. It needs only ranks, never raw +scores, so it fuses methods whose score scales are incomparable (ts_rank vs +cosine distance). `hybridSearchWithRRF` runs the FTS leg and a +cosine-distance scan, fuses at `document:page` granularity, and reports a +bounded pseudo-similarity (`score × 60`, capped at 0.95). The query embedder +is injected by the caller; an empty embedding degrades to lexical-only. + +**Relation to ensemble/.** Same fusion idea, different altitude: `ensemble/` +composes LangChain retrievers for the Q&A pipeline (chunk-granular, +config-driven legs, reranking); this folder is the standalone primitive for +callers that want a ranked page list without the retriever machinery — the +document matcher was its first consumer. + +**Knobs.** `k` (list-agreement bias) on RRF; `limit` and the vector distance +threshold (0.4) on the hybrid search. diff --git a/packages/retrieval/src/algorithms/fusion/hybrid-search.ts b/packages/retrieval/src/algorithms/fusion/hybrid-search.ts new file mode 100644 index 000000000..96e678ce1 --- /dev/null +++ b/packages/retrieval/src/algorithms/fusion/hybrid-search.ts @@ -0,0 +1,106 @@ +/** + * Lexical + dense hybrid search fused with RRF, at page granularity: the FTS + * leg and a raw cosine-distance scan each produce a ranked list, RRF merges + * them, and the top fused pages come back with a snippet and a bounded + * pseudo-similarity. The query embedder is injected — this module never + * chooses an embedding provider. + */ + +import { and, eq, inArray, sql } from "drizzle-orm"; +import { getDb } from "@launchstack/store/client"; +import { document, documentSections } from "@launchstack/store/schema"; +import { ftsSearch } from "../bm25/fts"; +import { reciprocalRankFusion, type RankedResult } from "./rrf"; + +export interface HybridSearchMatch { + documentId: number; + page: number; + snippet: string; + similarity: number; + content: string; +} + +function truncate(text: string, maxLength: number): string { + if (text.length <= maxLength) return text; + return text.substring(0, maxLength - 3) + "..."; +} + +/** + * Dense vector leg: cosine distance over `documentSections.embedding`, + * returned as a ranked list for fusion. + */ +async function vectorRankedSearch( + queryEmbedding: number[], + docIds: number[], + limit = 10, + threshold = 0.4 +): Promise { + if (docIds.length === 0 || queryEmbedding.length === 0) return []; + + const embeddingStr = `[${queryEmbedding.join(",")}]`; + + const results = await getDb() + .select({ + id: documentSections.id, + content: documentSections.content, + page: documentSections.pageNumber, + documentId: documentSections.documentId, + distance: sql`${documentSections.embedding} <=> ${embeddingStr}::vector`, + }) + .from(documentSections) + .innerJoin(document, eq(documentSections.documentId, document.id)) + .where( + and( + inArray( + documentSections.documentId, + docIds.map(id => BigInt(id)) + ), + eq(documentSections.versionId, document.currentVersionId), + sql`${documentSections.embedding} <=> ${embeddingStr}::vector < ${threshold}` + ) + ) + .orderBy(sql`${documentSections.embedding} <=> ${embeddingStr}::vector`) + .limit(limit); + + return results.map((r, idx) => ({ + documentId: Number(r.documentId), + page: r.page ?? 1, + content: r.content, + rank: idx + 1, + })); +} + +/** + * Hybrid search combining full-text and vector similarity with RRF. + * `embedQuery` supplies the query vector (empty vector ⇒ lexical-only). + */ +export async function hybridSearchWithRRF( + query: string, + docIds: number[], + limit: number, + embedQuery: (query: string) => Promise +): Promise { + if (docIds.length === 0) return []; + + const [ftsResults, vecResults] = await Promise.all([ + ftsSearch(query, docIds, limit * 2).catch(() => [] as RankedResult[]), + embedQuery(query) + .then(embedding => vectorRankedSearch(embedding, docIds, limit * 2)) + .catch(() => [] as RankedResult[]), + ]); + + if (ftsResults.length === 0 && vecResults.length === 0) return []; + + const fused = reciprocalRankFusion([ftsResults, vecResults]); + + return Array.from(fused.values()) + .sort((a, b) => b.score - a.score) + .slice(0, limit) + .map(r => ({ + documentId: r.documentId, + page: r.page, + snippet: truncate(r.content, 150), + similarity: Math.min(r.score * 60, 0.95), + content: r.content, + })); +} diff --git a/packages/retrieval/src/algorithms/fusion/index.ts b/packages/retrieval/src/algorithms/fusion/index.ts new file mode 100644 index 000000000..b38a65ee7 --- /dev/null +++ b/packages/retrieval/src/algorithms/fusion/index.ts @@ -0,0 +1,2 @@ +export { reciprocalRankFusion, type RankedResult, type FusedResult } from "./rrf"; +export { hybridSearchWithRRF, type HybridSearchMatch } from "./hybrid-search"; diff --git a/packages/retrieval/src/algorithms/fusion/rrf.ts b/packages/retrieval/src/algorithms/fusion/rrf.ts new file mode 100644 index 000000000..009406ce4 --- /dev/null +++ b/packages/retrieval/src/algorithms/fusion/rrf.ts @@ -0,0 +1,49 @@ +/** + * Reciprocal Rank Fusion: merges ranked lists from different retrieval + * methods. RRF(d) = sum( 1 / (k + rank_i(d)) ) for each list i containing d. + * k=60 is standard (from the original Cormack et al. paper). Agreement + * between lists beats a high rank in any single list. + */ + +export interface RankedResult { + documentId: number; + page: number; + content: string; + /** 1-based rank within its own list. */ + rank: number; +} + +export interface FusedResult { + score: number; + documentId: number; + page: number; + content: string; +} + +export function reciprocalRankFusion(lists: RankedResult[][], k = 60): Map { + const fused = new Map(); + + for (const list of lists) { + for (const item of list) { + const key = `${item.documentId}:${item.page}`; + const existing = fused.get(key); + const rrfScore = 1 / (k + item.rank); + + if (existing) { + existing.score += rrfScore; + if (item.content.length > existing.content.length) { + existing.content = item.content; + } + } else { + fused.set(key, { + score: rrfScore, + documentId: item.documentId, + page: item.page, + content: item.content, + }); + } + } + } + + return fused; +} diff --git a/packages/retrieval/src/algorithms/graph/README.md b/packages/retrieval/src/algorithms/graph/README.md new file mode 100644 index 000000000..2090b4462 --- /dev/null +++ b/packages/retrieval/src/algorithms/graph/README.md @@ -0,0 +1,29 @@ +# graph — knowledge-graph traversal + +**What it is.** One algorithm family, two backends. Query terms are matched +against extracted entities, the entity graph is traversed to pull in +co-occurring neighbours, and the sections those entities are mentioned in +become the leg's candidates. It finds connective context — sections that +never share the query's words but share its entities. + +**Backends.** + +- `neo4j.ts` — Cypher over the synced graph (`Entity`, `CO_OCCURS`, + `MENTIONED_IN`): fuzzy entity match, 0–`maxHops` co-occurrence expansion, + section IDs back to Postgres for the content. Requires the optional + `neo4j-driver` peer and a configured connection + (`@launchstack/indexing/knowledge-graph`, set up by the composition root). +- `pg.ts` — the same shape over the relational mirror (`kgEntities`, + `kgEntityMentions`) when Neo4j is absent. + +`shouldUseNeo4jRetriever()` picks the backend; whether the graph leg runs at +all is the ensemble config's `graphRetrieval` flag. + +**Failure.** Both backends degrade to zero candidates — a down peer never +fails a search, it just thins it. The ensemble's leg-breakdown log is where +a permanently silent graph leg shows up. + +**When it wins.** Entity-bridging questions ("what connects X and Y", +policies referenced across documents). It contributes noise on purely +lexical or purely semantic queries, which is why its default weight is the +smallest of the three legs. diff --git a/packages/retrieval/src/algorithms/graph/index.ts b/packages/retrieval/src/algorithms/graph/index.ts new file mode 100644 index 000000000..82ae750fe --- /dev/null +++ b/packages/retrieval/src/algorithms/graph/index.ts @@ -0,0 +1,2 @@ +export { GraphRetriever, createGraphRetriever } from "./pg"; +export { Neo4jGraphRetriever, createNeo4jGraphRetriever, shouldUseNeo4jRetriever } from "./neo4j"; diff --git a/apps/web/src/lib/tools/rag/retrievers/neo4j-graph-retriever.ts b/packages/retrieval/src/algorithms/graph/neo4j.ts similarity index 94% rename from apps/web/src/lib/tools/rag/retrievers/neo4j-graph-retriever.ts rename to packages/retrieval/src/algorithms/graph/neo4j.ts index 9b9887b65..491012db2 100644 --- a/apps/web/src/lib/tools/rag/retrievers/neo4j-graph-retriever.ts +++ b/packages/retrieval/src/algorithms/graph/neo4j.ts @@ -17,12 +17,11 @@ import { BaseRetriever, type BaseRetrieverInput } from "@langchain/core/retrievers"; import { Document } from "@langchain/core/documents"; import type { CallbackManagerForRetrieverRun } from "@langchain/core/callbacks/manager"; -import { db } from "~/server/db/index"; +import { getDb } from "@launchstack/store/client"; import { document, documentSections } from "@launchstack/store/schema"; import { inArray, and, eq, sql, type SQLWrapper } from "drizzle-orm"; import { isNeo4jConfigured, getNeo4jSession } from "@launchstack/indexing/knowledge-graph"; import neo4j, { type Session } from "neo4j-driver"; -import { getEngine } from "~/server/engine"; const currentVersionPredicate = ( versionColumn: SQLWrapper, @@ -74,7 +73,6 @@ export class Neo4jGraphRetriever extends BaseRetriever { let session: Session | null = null; try { - getEngine(); // ensures configureNeo4j has run session = getNeo4jSession(); const sectionIds = await this.findSectionsViaCypher(session, queryTerms); @@ -173,7 +171,7 @@ export class Neo4jGraphRetriever extends BaseRetriever { whereClause = and(whereClause, inArray(documentSections.documentId, docBigInts))!; } - const rows = await db + const rows = await getDb() .select({ id: documentSections.id, content: documentSections.content, @@ -225,13 +223,11 @@ export function createNeo4jGraphRetriever( } /** - * Returns true if Neo4j graph retrieval should be used. + * Returns true if the Neo4j backend should serve graph retrieval. The + * enable/disable decision for graph retrieval as a whole lives in the + * ensemble config (set by the composition root, which also configures + * Neo4j itself before any retrieval runs) — this only picks the backend. */ export function shouldUseNeo4jRetriever(): boolean { - getEngine(); // ensures configureNeo4j has run - return ( - isNeo4jConfigured() && - (process.env.ENABLE_GRAPH_RETRIEVER === "true" || - process.env.ENABLE_GRAPH_RETRIEVER === "1") - ); + return isNeo4jConfigured(); } diff --git a/apps/web/src/lib/tools/rag/retrievers/graph-retriever.ts b/packages/retrieval/src/algorithms/graph/pg.ts similarity index 97% rename from apps/web/src/lib/tools/rag/retrievers/graph-retriever.ts rename to packages/retrieval/src/algorithms/graph/pg.ts index 418c50377..4bd716aa1 100644 --- a/apps/web/src/lib/tools/rag/retrievers/graph-retriever.ts +++ b/packages/retrieval/src/algorithms/graph/pg.ts @@ -13,7 +13,7 @@ * 5. Return as LangChain Documents */ -import { db } from "~/server/db/index"; +import { getDb } from "@launchstack/store/client"; import { eq, inArray, and, or, ilike, sql, type SQLWrapper } from "drizzle-orm"; import { BaseRetriever, type BaseRetrieverInput } from "@langchain/core/retrievers"; import { Document } from "@langchain/core/documents"; @@ -166,7 +166,7 @@ export class GraphRetriever extends BaseRetriever { // Build OR conditions for fuzzy name matching const conditions = terms.map(term => ilike(kgEntities.name, `%${term}%`)); - const results = await db + const results = await getDb() .select({ id: kgEntities.id }) .from(kgEntities) .where(and(eq(kgEntities.companyId, BigInt(this.companyId)), or(...conditions))) @@ -181,12 +181,12 @@ export class GraphRetriever extends BaseRetriever { private async getNeighborEntities(entityIds: number[]): Promise { if (entityIds.length === 0) return []; - const outgoing = await db + const outgoing = await getDb() .select({ id: kgRelationships.targetEntityId }) .from(kgRelationships) .where(inArray(kgRelationships.sourceEntityId, entityIds)); - const incoming = await db + const incoming = await getDb() .select({ id: kgRelationships.sourceEntityId }) .from(kgRelationships) .where(inArray(kgRelationships.targetEntityId, entityIds)); @@ -204,7 +204,7 @@ export class GraphRetriever extends BaseRetriever { private async getSectionIdsForEntities(entityIds: number[]): Promise { if (entityIds.length === 0) return []; - const query = db + const query = getDb() .select({ sectionId: kgEntityMentions.sectionId }) .from(kgEntityMentions) .where(inArray(kgEntityMentions.entityId, entityIds)); @@ -230,7 +230,7 @@ export class GraphRetriever extends BaseRetriever { whereClause = and(whereClause, inArray(documentSections.documentId, docBigInts))!; } - const rows = await db + const rows = await getDb() .select({ id: documentSections.id, content: documentSections.content, diff --git a/packages/retrieval/src/algorithms/index.ts b/packages/retrieval/src/algorithms/index.ts new file mode 100644 index 000000000..dd4eeaa93 --- /dev/null +++ b/packages/retrieval/src/algorithms/index.ts @@ -0,0 +1,13 @@ +/** + * The algorithms section: given a query, which rows come back and in what + * order. One folder per algorithm — open the folder's README before tuning + * it. Algorithms never import from ../tools. + */ + +export * from "./bm25"; +export * from "./vector"; +export * from "./fusion"; +export * from "./ensemble"; +export * from "./rlm"; +export * from "./graph"; +export * from "./reranking"; diff --git a/packages/retrieval/src/algorithms/reranking/README.md b/packages/retrieval/src/algorithms/reranking/README.md new file mode 100644 index 000000000..06b66234c --- /dev/null +++ b/packages/retrieval/src/algorithms/reranking/README.md @@ -0,0 +1,20 @@ +# reranking — second-pass reordering + +**What it is.** An optional precision pass over the fused candidate pool: +`(query, candidates[]) → reordered candidates`, scored by a model that sees +the query and each candidate together — signal rank fusion cannot have, +since fusion only sees per-leg ranks. + +**How it works.** `getRerankProvider()` resolves the configured provider: +the dedicated `/v1/rerank` client when `RERANK_API_BASE_URL` names one +(resolved by the composition root, not here), otherwise a chat-model scorer +on the deployment's endpoint (`gemini.ts`). The ensemble retrieves +`topK × 4` candidates, reranks the pool, and keeps the top `topK`. + +**Failure.** Unconfigured or failing reranking is not an error: candidates +pass through in RRF order. `isRerankConfigured()` is the cheap gate the +ensemble checks before spending a call. + +**When it wins.** Question-shaped queries over large candidate pools, where +near-duplicates crowd out the one chunk that actually answers. Skip it for +latency-critical paths — it adds a model call per search. diff --git a/packages/search/src/reranking/gemini.ts b/packages/retrieval/src/algorithms/reranking/gemini.ts similarity index 100% rename from packages/search/src/reranking/gemini.ts rename to packages/retrieval/src/algorithms/reranking/gemini.ts diff --git a/packages/search/src/reranking/index.ts b/packages/retrieval/src/algorithms/reranking/index.ts similarity index 100% rename from packages/search/src/reranking/index.ts rename to packages/retrieval/src/algorithms/reranking/index.ts diff --git a/packages/search/src/reranking/rerank-api.ts b/packages/retrieval/src/algorithms/reranking/rerank-api.ts similarity index 100% rename from packages/search/src/reranking/rerank-api.ts rename to packages/retrieval/src/algorithms/reranking/rerank-api.ts diff --git a/packages/retrieval/src/algorithms/rlm/README.md b/packages/retrieval/src/algorithms/rlm/README.md new file mode 100644 index 000000000..4e3535cc6 --- /dev/null +++ b/packages/retrieval/src/algorithms/rlm/README.md @@ -0,0 +1,28 @@ +# rlm — hierarchical, token-budgeted navigation + +**What it is.** Retrieval for Recursive-Language-Model-style inference: +instead of stuffing top-k chunks into context, the model navigates the +document's structure programmatically and pays for exactly the sections it +reads. + +**How it works.** Access patterns, cheapest first: + +1. `getDocumentOverview` — metadata and outline shape, for planning. +2. `getDocumentTree` — the hierarchical structure without bodies. +3. `probeSection` — a preview of a section before committing to it. +4. `getSectionsWithinBudget` — full sections selected under an explicit + token budget, with per-section cost accounting. +5. Workspace operations — store and re-read intermediate results across + steps of a multi-call analysis. + +Semantic filtering rides on the section metadata (semantic type, page), and +vector search over retrieval chunks is available inside a scope when the +structure alone isn't enough. + +**When it wins.** Large documents and multi-step analyses where context is +the scarce resource — course packs, contracts with schedules, anything where +"read the right 5%" beats "embed similarity over everything". For one-shot +questions, the plain ensemble is cheaper and simpler. + +**Knobs.** `TokenBudgetOptions` (budget, per-section caps), +`WorkspaceStoreOptions` for intermediate results. diff --git a/packages/retrieval/src/algorithms/rlm/index.ts b/packages/retrieval/src/algorithms/rlm/index.ts new file mode 100644 index 000000000..aceee42db --- /dev/null +++ b/packages/retrieval/src/algorithms/rlm/index.ts @@ -0,0 +1,11 @@ +export { RLMRetriever, createRLMRetriever, getDocumentSummary, getStructureContent } from "./rlm"; + +export type { + DocumentOverview, + StructureNode, + SectionWithCost, + SectionPreview, + WorkspaceEntry, + TokenBudgetOptions, + WorkspaceStoreOptions, +} from "./rlm"; diff --git a/apps/web/src/lib/tools/rag/retrievers/rlm-retriever.ts b/packages/retrieval/src/algorithms/rlm/rlm.ts similarity index 97% rename from apps/web/src/lib/tools/rag/retrievers/rlm-retriever.ts rename to packages/retrieval/src/algorithms/rlm/rlm.ts index 84ba225ee..8b941a318 100644 --- a/apps/web/src/lib/tools/rag/retrievers/rlm-retriever.ts +++ b/packages/retrieval/src/algorithms/rlm/rlm.ts @@ -16,8 +16,8 @@ * 5. Workspace operations - Store/retrieve intermediate results */ -import { db, toRows } from "~/server/db/index"; -import { T } from "~/server/db/tables"; +import { getDb, toRows } from "@launchstack/store/client"; +import { T } from "@launchstack/store/tables"; import { eq, and, sql, asc, desc, lte, inArray, isNull, type SQLWrapper } from "drizzle-orm"; import { documentStructure, @@ -34,7 +34,7 @@ import { type ResultType, } from "@launchstack/store/schema"; import { isLegacyEmbeddingIndex, type EmbeddingIndexConfig } from "@launchstack/llm/embeddings"; -import type { EmbeddingsProvider } from "../types"; +import type { EmbeddingsProvider } from "../../search-types"; const currentVersionPredicate = ( versionColumn: SQLWrapper, @@ -148,7 +148,7 @@ export class RLMRetriever { * This is the first call an RLM should make - cheap, informative. */ async getDocumentOverview(documentId: number): Promise { - const [meta] = await db + const [meta] = await getDb() .select({ documentId: documentMetadata.documentId, title: document.title, @@ -195,7 +195,7 @@ export class RLMRetriever { async getDocumentOverviews(documentIds: number[]): Promise { if (documentIds.length === 0) return []; - const metas = await db + const metas = await getDb() .select({ documentId: documentMetadata.documentId, title: document.title, @@ -245,7 +245,7 @@ export class RLMRetriever { * Enables recursive decomposition of document. */ async getDocumentTree(documentId: number, maxDepth = 2): Promise { - const nodes = await db + const nodes = await getDb() .select({ id: documentStructure.id, parentId: documentStructure.parentId, @@ -312,7 +312,7 @@ export class RLMRetriever { * Get children of a specific structure node. */ async getStructureChildren(structureId: number): Promise { - const nodes = await db + const nodes = await getDb() .select({ id: documentStructure.id, parentId: documentStructure.parentId, @@ -355,7 +355,7 @@ export class RLMRetriever { * Get structure node by path (e.g., "1.2.3"). */ async getStructureByPath(documentId: number, path: string): Promise { - const [node] = await db + const [node] = await getDb() .select({ id: documentStructure.id, parentId: documentStructure.parentId, @@ -432,7 +432,7 @@ export class RLMRetriever { orderClause = [asc(documentSections.id)]; } - const allSections = await db + const allSections = await getDb() .select({ id: documentSections.id, content: documentSections.content, @@ -472,7 +472,7 @@ export class RLMRetriever { // Get structure path let structurePath: string | null = null; if (section.structureId) { - const [struct] = await db + const [struct] = await getDb() .select({ path: documentStructure.path }) .from(documentStructure) .innerJoin(document, eq(documentStructure.documentId, document.id)) @@ -504,7 +504,7 @@ export class RLMRetriever { * Get sections by structure node (all content under a tree node). */ async getSectionsByStructure(structureId: number): Promise { - const sections = await db + const sections = await getDb() .select({ id: documentSections.id, content: documentSections.content, @@ -554,7 +554,7 @@ export class RLMRetriever { startPage: number, endPage: number ): Promise { - const sections = await db + const sections = await getDb() .select({ id: documentSections.id, content: documentSections.content, @@ -618,7 +618,7 @@ export class RLMRetriever { conditions.push(inArray(documentPreviews.previewType, previewTypes)); } - const previews = await db + const previews = await getDb() .select({ id: documentPreviews.id, previewType: documentPreviews.previewType, @@ -645,7 +645,7 @@ export class RLMRetriever { * Get preview for a specific section (before reading full content). */ async getSectionPreview(sectionId: number): Promise { - const [preview] = await db + const [preview] = await getDb() .select({ id: documentPreviews.id, previewType: documentPreviews.previewType, @@ -697,7 +697,7 @@ export class RLMRetriever { > { if (sectionIds.length === 0) return []; - const sections = await db + const sections = await getDb() .select({ id: documentSections.id, tokenCount: documentSections.tokenCount, @@ -729,7 +729,7 @@ export class RLMRetriever { ? new Date(Date.now() + options.ttlHours * 60 * 60 * 1000) : new Date(Date.now() + 24 * 60 * 60 * 1000); // Default 24h TTL - const [result] = await db + const [result] = await getDb() .insert(workspaceResults) .values({ sessionId: options.sessionId, @@ -763,7 +763,7 @@ export class RLMRetriever { conditions.push(inArray(workspaceResults.resultType, resultTypes)); } - const results = await db + const results = await getDb() .select() .from(workspaceResults) .where(and(...conditions)) @@ -786,7 +786,7 @@ export class RLMRetriever { * Get a specific result by ID. */ async getResultById(resultId: number): Promise { - const [result] = await db + const [result] = await getDb() .select() .from(workspaceResults) .where(eq(workspaceResults.id, resultId)) @@ -811,7 +811,7 @@ export class RLMRetriever { * Get child results (for tracking recursion chains). */ async getChildResults(parentResultId: number): Promise { - const results = await db + const results = await getDb() .select() .from(workspaceResults) .where(eq(workspaceResults.parentResultId, BigInt(parentResultId))) @@ -834,7 +834,7 @@ export class RLMRetriever { * Clean up expired workspace results. */ async cleanupExpiredResults(): Promise { - const result = await db + const result = await getDb() .delete(workspaceResults) .where(sql`${workspaceResults.expiresAt} < NOW()`) .returning({ id: workspaceResults.id }); @@ -869,7 +869,7 @@ export class RLMRetriever { : null; const results = useLegacyPath - ? await db + ? await getDb() .select({ id: documentSections.id, content: documentSections.content, @@ -905,7 +905,7 @@ export class RLMRetriever { structurePath: string | null; distance: number; }>( - await db.execute(sql` + await getDb().execute(sql` SELECT cc.id, cc.content, diff --git a/packages/retrieval/src/algorithms/vector/README.md b/packages/retrieval/src/algorithms/vector/README.md new file mode 100644 index 000000000..b8013220e --- /dev/null +++ b/packages/retrieval/src/algorithms/vector/README.md @@ -0,0 +1,26 @@ +# vector — semantic similarity search + +**What it is.** Nearest-neighbour search over pgvector embeddings. +`retriever.ts` is the LangChain adapter, not the algorithm: it scopes the +query (document / company / multi-document), picks the embedding table for +the active index dimension, and delegates the actual ranking to a search +strategy. + +**How it works.** The query is embedded with the same index configuration +the chunks were embedded under (`EmbeddingIndexConfig` from +`@launchstack/llm/embeddings` — dimension, table, legacy flags), then chunks +are ordered by cosine distance (`<=>`) with a short-vector prefilter where +the index supports it. Only current-version chunks are searched — every +query joins through `document.currentVersionId`. + +**When it wins.** Paraphrase, synonymy, "what does this mean"-shaped +questions — anywhere the reader's words differ from the document's. It loses +on exact identifiers and rare literal terms, which is the BM25 leg's job. + +**Strategies.** `strategies/` holds the named ANN variants (exact scan, +HNSW, IVF, prefiltered, matryoshka short-vector two-pass) — see its README +for when each applies. `similarity.ts` has the in-memory measures +(cosine, euclidean) for scoring embeddings after they've been fetched. + +**Knobs.** `topK`, `SearchFilters` (semantic type, page range), and the +embedding index key per call. diff --git a/packages/retrieval/src/algorithms/vector/index.ts b/packages/retrieval/src/algorithms/vector/index.ts new file mode 100644 index 000000000..127648a38 --- /dev/null +++ b/packages/retrieval/src/algorithms/vector/index.ts @@ -0,0 +1,19 @@ +export { + VectorRetriever, + createDocumentVectorRetriever, + createCompanyVectorRetriever, + createMultiDocVectorRetriever, +} from "./retriever"; + +export { cosineSimilarity, euclideanDistance } from "./similarity"; + +export { + ANNOptimizer, + exactScanSearch, + ivfSearch, + prefilteredSearch, + matryoshkaSearch, + buildDocumentCluster, + calculateDocumentRelevanceScores, + findRelevantDocumentClusters, +} from "./strategies"; diff --git a/packages/search/src/retrievers/vector-retriever.ts b/packages/retrieval/src/algorithms/vector/retriever.ts similarity index 99% rename from packages/search/src/retrievers/vector-retriever.ts rename to packages/retrieval/src/algorithms/vector/retriever.ts index c4922a988..ce68822b6 100644 --- a/packages/search/src/retrievers/vector-retriever.ts +++ b/packages/retrieval/src/algorithms/vector/retriever.ts @@ -10,7 +10,7 @@ import { supportsShortVectorSearch, type EmbeddingIndexConfig, } from "@launchstack/llm/embeddings"; -import type { EmbeddingsProvider, SearchScope, SearchFilters } from "../search-types"; +import type { EmbeddingsProvider, SearchScope, SearchFilters } from "../../search-types"; interface VectorRetrieverConfig extends BaseRetrieverInput { embeddings: EmbeddingsProvider; diff --git a/packages/retrieval/src/algorithms/vector/similarity.ts b/packages/retrieval/src/algorithms/vector/similarity.ts new file mode 100644 index 000000000..2633623fd --- /dev/null +++ b/packages/retrieval/src/algorithms/vector/similarity.ts @@ -0,0 +1,33 @@ +/** + * Plain vector-distance measures for in-memory scoring. Database-side + * similarity goes through pgvector operators in the retriever; these exist + * for the few places that compare embeddings after they have been fetched. + */ + +export function cosineSimilarity(a: number[], b: number[]): number { + if (a.length !== b.length) return 0; + + let dotProduct = 0; + let normA = 0; + let normB = 0; + + for (let i = 0; i < a.length; i++) { + dotProduct += a[i]! * b[i]!; + normA += a[i]! * a[i]!; + normB += b[i]! * b[i]!; + } + + if (normA === 0 || normB === 0) return 0; + return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); +} + +export function euclideanDistance(a: number[], b: number[]): number { + if (a.length !== b.length) return Infinity; + + let sum = 0; + for (let i = 0; i < a.length; i++) { + const diff = a[i]! - b[i]!; + sum += diff * diff; + } + return Math.sqrt(sum); +} diff --git a/packages/retrieval/src/algorithms/vector/strategies/README.md b/packages/retrieval/src/algorithms/vector/strategies/README.md new file mode 100644 index 000000000..79b84991d --- /dev/null +++ b/packages/retrieval/src/algorithms/vector/strategies/README.md @@ -0,0 +1,22 @@ +# strategies — the named ANN variants + +**What they are.** The actual nearest-neighbour algorithms behind the vector +retriever, each an individually documented module. `ANNOptimizer` is the +config-driven dispatcher consumers hold; strategy functions are also +callable directly. + +| Strategy | Module | Mechanism | Reaches for | +| --- | --- | --- | --- | +| `hnsw` | `exact.ts` | Ordered cosine-distance scan, 5× over-sample then in-memory refine; rides the column's HNSW index when one exists | Small scopes; the universal fallback | +| `ivf` | `ivf.ts` | Rank per-document centroid clusters, scan only the top `probeCount` clusters' chunks | Many documents, few relevant | +| `prefiltered` | `prefiltered.ts` | Score whole documents via centroids, scan qualifying documents best-first until `limit` fills | Medium scopes with skewed relevance | +| `matryoshka` | `matryoshka.ts` | 512-dim short-vector coarse pass (HNSW-indexed) → full-dim re-rank of the survivors | Large scopes where full-dim scans are too slow | +| `hybrid` | `index.ts` | Adaptive: ≤5 docs → exact, ≤20 → prefiltered, else matryoshka | The default when the scope size varies | + +**Shared machinery.** `clusters.ts` builds and caches the per-document +centroid clusters (1h in-process TTL; `ANNOptimizer.clearCache()` resets). +`sanitize.ts` strips query vectors out of error messages before logging. + +**Contract.** Every strategy returns `ANNResult[]` filtered to +`distance ≤ threshold`, sorted ascending by distance, and degrades to `[]` +on error — a failing strategy thins results, never throws. diff --git a/packages/retrieval/src/algorithms/vector/strategies/clusters.ts b/packages/retrieval/src/algorithms/vector/strategies/clusters.ts new file mode 100644 index 000000000..44acf49fd --- /dev/null +++ b/packages/retrieval/src/algorithms/vector/strategies/clusters.ts @@ -0,0 +1,142 @@ +/** + * Per-document centroid clusters backing the IVF and prefiltered strategies. + * A cluster summarizes one document's chunk embeddings (centroid + member + * chunk ids); comparing the query against centroids is how those strategies + * decide which documents deserve a full scan. Cached in-process for an hour. + */ + +import { and, eq } from "drizzle-orm"; +import { getDb } from "@launchstack/store/client"; +import { document, documentSections } from "@launchstack/store/schema"; +import type { DocumentCluster } from "../../../search-types"; +import { cosineSimilarity, euclideanDistance } from "../similarity"; + +const CLUSTER_TTL_MS = 3600000; + +const documentClustersCache = new Map(); + +export async function buildDocumentCluster(documentId: number): Promise { + const chunks = await getDb() + .select({ + id: documentSections.id, + embedding: documentSections.embedding, + }) + .from(documentSections) + .innerJoin(document, eq(documentSections.documentId, document.id)) + .where( + and( + eq(documentSections.documentId, BigInt(documentId)), + eq(documentSections.versionId, document.currentVersionId) + ) + ); + + if (chunks.length === 0) { + return { + documentId, + centroid: [], + chunkIds: [], + avgDistance: 1, + lastUpdated: new Date(), + }; + } + + const dimension = chunks[0]?.embedding?.length ?? 1536; + const centroid = new Array(dimension).fill(0); + + for (const chunk of chunks) { + if (chunk.embedding) { + for (let i = 0; i < dimension; i++) { + centroid[i] = (centroid[i] ?? 0) + (chunk.embedding[i] ?? 0); + } + } + } + + for (let i = 0; i < dimension; i++) { + centroid[i] = (centroid[i] ?? 0) / chunks.length; + } + + let totalDistance = 0; + let comparisons = 0; + + for (let i = 0; i < chunks.length && comparisons < 100; i++) { + for (let j = i + 1; j < chunks.length && comparisons < 100; j++) { + if (chunks[i]?.embedding && chunks[j]?.embedding) { + totalDistance += euclideanDistance(chunks[i]!.embedding!, chunks[j]!.embedding!); + comparisons++; + } + } + } + + const avgDistance = comparisons > 0 ? totalDistance / comparisons : 1; + + return { + documentId, + centroid, + chunkIds: chunks.map(c => c.id), + avgDistance, + lastUpdated: new Date(), + }; +} + +export async function calculateDocumentRelevanceScores( + queryEmbedding: number[], + documentIds: number[] +): Promise<{ documentId: number; score: number }[]> { + const scores: { documentId: number; score: number }[] = []; + + for (const docId of documentIds) { + let cluster = documentClustersCache.get(docId); + + if (!cluster || Date.now() - cluster.lastUpdated.getTime() > CLUSTER_TTL_MS) { + cluster = await buildDocumentCluster(docId); + documentClustersCache.set(docId, cluster); + } + + const similarity = cosineSimilarity(queryEmbedding, cluster.centroid); + scores.push({ documentId: docId, score: similarity }); + } + + return scores; +} + +export async function findRelevantDocumentClusters( + queryEmbedding: number[], + documentIds: number[], + topK = 3 +): Promise { + const clusters: Array<{ cluster: DocumentCluster; similarity: number }> = []; + + for (const docId of documentIds) { + let cluster = documentClustersCache.get(docId); + + if (!cluster) { + cluster = await buildDocumentCluster(docId); + documentClustersCache.set(docId, cluster); + } + + if (cluster.centroid.length > 0) { + const similarity = cosineSimilarity(queryEmbedding, cluster.centroid); + clusters.push({ cluster, similarity }); + } + } + + return clusters + .sort((a, b) => b.similarity - a.similarity) + .slice(0, topK) + .map(c => c.cluster); +} + +export function clearClusterCache(): void { + documentClustersCache.clear(); +} + +export function getClusterCacheStats(): { size: number; oldestEntry: Date | null } { + const entries = Array.from(documentClustersCache.values()); + return { + size: entries.length, + oldestEntry: + entries.length > 0 + ? new Date(Math.min(...entries.map(e => e.lastUpdated.getTime()))) + : null, + }; +} diff --git a/packages/retrieval/src/algorithms/vector/strategies/exact.ts b/packages/retrieval/src/algorithms/vector/strategies/exact.ts new file mode 100644 index 000000000..4dc27fd51 --- /dev/null +++ b/packages/retrieval/src/algorithms/vector/strategies/exact.ts @@ -0,0 +1,73 @@ +/** + * Exact ordered scan over `documentSections.embedding` — the baseline every + * other strategy falls back to. Fetches an over-sampled candidate window + * (5×limit, capped at 100) ordered by cosine distance, then refines in + * memory. When the column carries an HNSW index this is also the "hnsw" + * strategy: the planner uses the index for the ORDER BY, and the code path + * is identical. + */ + +import { and, eq, inArray, sql } from "drizzle-orm"; +import { getDb } from "@launchstack/store/client"; +import { document, documentSections } from "@launchstack/store/schema"; +import type { ANNResult } from "../../../search-types"; +import { sanitizeErrorMessage } from "./sanitize"; + +type ANNRow = { id: number; content: string; page: number; documentId: number; distance: number }; + +export async function exactScanSearch( + queryEmbedding: number[], + documentIds: number[], + limit: number, + threshold: number +): Promise { + try { + const embeddingStr = `[${queryEmbedding.join(",")}]`; + + const approximateLimit = Math.min(limit * 5, 100); + + const results = await getDb() + .select({ + id: documentSections.id, + content: documentSections.content, + page: documentSections.pageNumber, + documentId: documentSections.documentId, + distance: sql`${documentSections.embedding} <=> ${embeddingStr}::vector`, + }) + .from(documentSections) + .innerJoin(document, eq(documentSections.documentId, document.id)) + .where( + and( + inArray( + documentSections.documentId, + documentIds.map(id => BigInt(id)) + ), + eq(documentSections.versionId, document.currentVersionId) + ) + ) + .orderBy(sql`${documentSections.embedding} <=> ${embeddingStr}::vector`) + .limit(approximateLimit); + + const rows: ANNRow[] = results.map(r => ({ + id: r.id, + content: r.content, + page: r.page ?? 0, + documentId: Number(r.documentId), + distance: Number(r.distance ?? 1), + })); + + const refinedResults = rows + .map(row => ({ + ...row, + confidence: Math.max(0, 1 - row.distance), + })) + .filter(r => r.distance <= threshold) + .sort((a, b) => a.distance - b.distance) + .slice(0, limit); + + return refinedResults; + } catch (error) { + console.warn("Exact-scan (hnsw) search failed:", sanitizeErrorMessage(error)); + return []; + } +} diff --git a/packages/retrieval/src/algorithms/vector/strategies/index.ts b/packages/retrieval/src/algorithms/vector/strategies/index.ts new file mode 100644 index 000000000..3a936f1cb --- /dev/null +++ b/packages/retrieval/src/algorithms/vector/strategies/index.ts @@ -0,0 +1,106 @@ +/** + * Named ANN strategies behind the vector retriever, plus ANNOptimizer — the + * config-driven dispatcher consumers hold on to. "hybrid" is the adaptive + * strategy: exact scan for small scopes (≤5 docs), prefiltered for medium + * (≤20), matryoshka coarse-to-fine for large ones. + */ + +import type { ANNConfig, ANNResult } from "../../../search-types"; +import { exactScanSearch } from "./exact"; +import { ivfSearch } from "./ivf"; +import { prefilteredSearch } from "./prefiltered"; +import { matryoshkaSearch } from "./matryoshka"; +import { clearClusterCache, getClusterCacheStats } from "./clusters"; + +export { exactScanSearch } from "./exact"; +export { ivfSearch } from "./ivf"; +export { prefilteredSearch } from "./prefiltered"; +export { matryoshkaSearch } from "./matryoshka"; +export { + buildDocumentCluster, + calculateDocumentRelevanceScores, + findRelevantDocumentClusters, +} from "./clusters"; + +export class ANNOptimizer { + private config: ANNConfig; + + constructor(config: ANNConfig = { strategy: "hybrid" }) { + this.config = config; + } + + async searchSimilarChunks( + queryEmbedding: number[], + documentIds: number[], + limit = 10, + distanceThreshold = 0.7 + ): Promise { + if (!documentIds || documentIds.length === 0) { + return []; + } + + switch (this.config.strategy) { + case "hnsw": + return exactScanSearch(queryEmbedding, documentIds, limit, distanceThreshold); + + case "ivf": + return ivfSearch( + queryEmbedding, + documentIds, + limit, + distanceThreshold, + this.config.probeCount ?? 3 + ); + + case "prefiltered": + return prefilteredSearch( + queryEmbedding, + documentIds, + limit, + distanceThreshold, + this.config.prefilterThreshold ?? 0.3 + ); + + case "matryoshka": + return matryoshkaSearch(queryEmbedding, documentIds, limit, distanceThreshold); + + case "hybrid": + default: + return this.adaptiveSearch(queryEmbedding, documentIds, limit, distanceThreshold); + } + } + + private async adaptiveSearch( + queryEmbedding: number[], + documentIds: number[], + limit: number, + threshold: number + ): Promise { + if (documentIds.length <= 5) { + return exactScanSearch(queryEmbedding, documentIds, limit, threshold); + } + + if (documentIds.length <= 20) { + return prefilteredSearch( + queryEmbedding, + documentIds, + limit, + threshold, + this.config.prefilterThreshold ?? 0.3 + ); + } + + // For large document sets, use Matryoshka coarse-to-fine + return matryoshkaSearch(queryEmbedding, documentIds, limit, threshold); + } + + static clearCache(): void { + clearClusterCache(); + } + + static getCacheStats(): { size: number; oldestEntry: Date | null } { + return getClusterCacheStats(); + } +} + +export default ANNOptimizer; diff --git a/packages/retrieval/src/algorithms/vector/strategies/ivf.ts b/packages/retrieval/src/algorithms/vector/strategies/ivf.ts new file mode 100644 index 000000000..14a5ac581 --- /dev/null +++ b/packages/retrieval/src/algorithms/vector/strategies/ivf.ts @@ -0,0 +1,74 @@ +/** + * IVF-style cluster probing: rank per-document centroid clusters against the + * query, scan only the chunks belonging to the top `probeCount` clusters. + * Trades recall for latency on multi-document scopes; falls back to the + * exact scan when no cluster matches. + */ + +import { and, eq, inArray, sql } from "drizzle-orm"; +import { getDb } from "@launchstack/store/client"; +import { document, documentSections } from "@launchstack/store/schema"; +import type { ANNResult } from "../../../search-types"; +import { findRelevantDocumentClusters } from "./clusters"; +import { exactScanSearch } from "./exact"; +import { sanitizeErrorMessage } from "./sanitize"; + +export async function ivfSearch( + queryEmbedding: number[], + documentIds: number[], + limit: number, + threshold: number, + probeCount = 3 +): Promise { + try { + const relevantClusters = await findRelevantDocumentClusters( + queryEmbedding, + documentIds, + probeCount + ); + + if (relevantClusters.length === 0) { + return exactScanSearch(queryEmbedding, documentIds, limit, threshold); + } + + const clusterChunkIds = relevantClusters.flatMap(c => c.chunkIds); + + if (clusterChunkIds.length === 0) { + return []; + } + + const embeddingStr = `[${queryEmbedding.join(",")}]`; + + const results = await getDb() + .select({ + id: documentSections.id, + content: documentSections.content, + page: documentSections.pageNumber, + documentId: documentSections.documentId, + distance: sql`${documentSections.embedding} <=> ${embeddingStr}::vector`, + }) + .from(documentSections) + .innerJoin(document, eq(documentSections.documentId, document.id)) + .where( + and( + inArray(documentSections.id, clusterChunkIds), + eq(documentSections.versionId, document.currentVersionId), + sql`${documentSections.embedding} <=> ${embeddingStr}::vector <= ${threshold}` + ) + ) + .orderBy(sql`${documentSections.embedding} <=> ${embeddingStr}::vector`) + .limit(limit); + + return results.map(row => ({ + id: row.id, + content: row.content, + page: row.page ?? 0, + documentId: Number(row.documentId), + distance: Number(row.distance ?? 1), + confidence: Math.max(0, 1 - Number(row.distance ?? 1)), + })); + } catch (error) { + console.warn("IVF search failed:", sanitizeErrorMessage(error)); + return []; + } +} diff --git a/packages/retrieval/src/algorithms/vector/strategies/matryoshka.ts b/packages/retrieval/src/algorithms/vector/strategies/matryoshka.ts new file mode 100644 index 000000000..c4a26a9e8 --- /dev/null +++ b/packages/retrieval/src/algorithms/vector/strategies/matryoshka.ts @@ -0,0 +1,121 @@ +/** + * Matryoshka coarse-to-fine: use 512-dim short embeddings from + * `document_retrieval_chunks` (HNSW-indexed) for fast candidate filtering, + * then re-rank the top candidates with full-dimension embeddings. Page + * numbers resolve through the candidates' context chunks. Falls back to the + * exact scan when the short-vector pass returns nothing. + */ + +import { and, eq, inArray, sql } from "drizzle-orm"; +import { getDb } from "@launchstack/store/client"; +import { document, documentRetrievalChunks, documentSections } from "@launchstack/store/schema"; +import type { ANNResult } from "../../../search-types"; +import { exactScanSearch } from "./exact"; +import { sanitizeErrorMessage } from "./sanitize"; + +export async function matryoshkaSearch( + queryEmbedding: number[], + documentIds: number[], + limit: number, + threshold: number +): Promise { + try { + const shortDim = 512; + const queryShort = queryEmbedding.slice(0, shortDim); + const shortStr = `[${queryShort.join(",")}]`; + + const coarseCandidateCount = Math.min(limit * 6, 120); + + const coarseResults = await getDb() + .select({ + id: documentRetrievalChunks.id, + content: documentRetrievalChunks.content, + documentId: documentRetrievalChunks.documentId, + contextChunkId: documentRetrievalChunks.contextChunkId, + shortDistance: sql`${documentRetrievalChunks.embeddingShort} <=> ${shortStr}::vector`, + }) + .from(documentRetrievalChunks) + .innerJoin(document, eq(documentRetrievalChunks.documentId, document.id)) + .where( + and( + inArray( + documentRetrievalChunks.documentId, + documentIds.map(id => BigInt(id)) + ), + eq(documentRetrievalChunks.versionId, document.currentVersionId) + ) + ) + .orderBy(sql`${documentRetrievalChunks.embeddingShort} <=> ${shortStr}::vector`) + .limit(coarseCandidateCount); + + if (coarseResults.length === 0) { + return exactScanSearch(queryEmbedding, documentIds, limit, threshold); + } + + const candidateIds = coarseResults.map(r => r.id); + const fullStr = `[${queryEmbedding.join(",")}]`; + + const refinedResults = await getDb() + .select({ + id: documentRetrievalChunks.id, + content: documentRetrievalChunks.content, + documentId: documentRetrievalChunks.documentId, + distance: sql`${documentRetrievalChunks.embedding} <=> ${fullStr}::vector`, + }) + .from(documentRetrievalChunks) + .innerJoin(document, eq(documentRetrievalChunks.documentId, document.id)) + .where( + and( + inArray(documentRetrievalChunks.id, candidateIds), + eq(documentRetrievalChunks.versionId, document.currentVersionId) + ) + ) + .orderBy(sql`${documentRetrievalChunks.embedding} <=> ${fullStr}::vector`) + .limit(limit); + + const contextChunkIds = coarseResults + .map(r => Number(r.contextChunkId)) + .filter(id => !isNaN(id)); + + const pageMap = new Map(); + if (contextChunkIds.length > 0) { + const pages = await getDb() + .select({ + id: documentSections.id, + page: documentSections.pageNumber, + }) + .from(documentSections) + .innerJoin(document, eq(documentSections.documentId, document.id)) + .where( + and( + inArray(documentSections.id, contextChunkIds), + eq(documentSections.versionId, document.currentVersionId) + ) + ); + + for (const p of pages) { + pageMap.set(p.id, p.page ?? 1); + } + } + + const contextIdMap = new Map(coarseResults.map(r => [r.id, Number(r.contextChunkId)])); + + return refinedResults + .map(row => { + const dist = Number(row.distance ?? 1); + const ctxId = contextIdMap.get(row.id); + return { + id: row.id, + content: row.content, + page: ctxId ? (pageMap.get(ctxId) ?? 1) : 1, + documentId: Number(row.documentId), + distance: dist, + confidence: Math.max(0, 1 - dist), + }; + }) + .filter(r => r.distance <= threshold); + } catch (error) { + console.warn("Matryoshka search failed:", sanitizeErrorMessage(error)); + return []; + } +} diff --git a/packages/retrieval/src/algorithms/vector/strategies/prefiltered.ts b/packages/retrieval/src/algorithms/vector/strategies/prefiltered.ts new file mode 100644 index 000000000..0f03d16e0 --- /dev/null +++ b/packages/retrieval/src/algorithms/vector/strategies/prefiltered.ts @@ -0,0 +1,80 @@ +/** + * Prefiltered search: score whole documents against the query first (via + * their centroid clusters), then scan only documents above the prefilter + * threshold, most relevant first, until `limit` is filled. Cheaper than a + * flat scan when most documents in scope are irrelevant; falls back to the + * exact scan when nothing clears the threshold. + */ + +import { and, eq, sql } from "drizzle-orm"; +import { getDb } from "@launchstack/store/client"; +import { document, documentSections } from "@launchstack/store/schema"; +import type { ANNResult } from "../../../search-types"; +import { calculateDocumentRelevanceScores } from "./clusters"; +import { exactScanSearch } from "./exact"; +import { sanitizeErrorMessage } from "./sanitize"; + +export async function prefilteredSearch( + queryEmbedding: number[], + documentIds: number[], + limit: number, + threshold: number, + prefilterThreshold = 0.3 +): Promise { + try { + const docScores = await calculateDocumentRelevanceScores(queryEmbedding, documentIds); + + const sortedDocIds = docScores + .filter(d => d.score > prefilterThreshold) + .sort((a, b) => b.score - a.score) + .map(d => d.documentId); + + if (sortedDocIds.length === 0) { + return exactScanSearch(queryEmbedding, documentIds, limit, threshold); + } + + const results: ANNResult[] = []; + const embeddingStr = `[${queryEmbedding.join(",")}]`; + + for (const docId of sortedDocIds) { + if (results.length >= limit) break; + + const remaining = limit - results.length; + const docResults = await getDb() + .select({ + id: documentSections.id, + content: documentSections.content, + page: documentSections.pageNumber, + documentId: documentSections.documentId, + distance: sql`${documentSections.embedding} <=> ${embeddingStr}::vector`, + }) + .from(documentSections) + .innerJoin(document, eq(documentSections.documentId, document.id)) + .where( + and( + eq(documentSections.documentId, BigInt(docId)), + eq(documentSections.versionId, document.currentVersionId), + sql`${documentSections.embedding} <=> ${embeddingStr}::vector <= ${threshold}` + ) + ) + .orderBy(sql`${documentSections.embedding} <=> ${embeddingStr}::vector`) + .limit(remaining * 2); + + const mappedResults: ANNResult[] = docResults.map(row => ({ + id: row.id, + content: row.content, + page: row.page ?? 0, + documentId: Number(row.documentId), + distance: Number(row.distance ?? 1), + confidence: Math.max(0, 1 - Number(row.distance ?? 1)), + })); + + results.push(...mappedResults.slice(0, remaining)); + } + + return results.sort((a, b) => a.distance - b.distance); + } catch (error) { + console.warn("Prefiltered search failed:", sanitizeErrorMessage(error)); + return []; + } +} diff --git a/packages/retrieval/src/algorithms/vector/strategies/sanitize.ts b/packages/retrieval/src/algorithms/vector/strategies/sanitize.ts new file mode 100644 index 000000000..8d606e502 --- /dev/null +++ b/packages/retrieval/src/algorithms/vector/strategies/sanitize.ts @@ -0,0 +1,12 @@ +const VECTOR_PATTERN = /\[[-\d.,eE+\s]{200,}\]/g; + +/** + * Error messages from failed vector queries can embed the full query vector + * (thousands of characters of floats). Strip it before logging. + */ +export function sanitizeErrorMessage(error: unknown): string { + if (!(error instanceof Error)) return "Unknown error"; + const msg = error.message; + if (msg.length < 300) return msg; + return msg.replace(VECTOR_PATTERN, "[]").slice(0, 500); +} diff --git a/packages/retrieval/src/index.ts b/packages/retrieval/src/index.ts new file mode 100644 index 000000000..a99e7c434 --- /dev/null +++ b/packages/retrieval/src/index.ts @@ -0,0 +1,21 @@ +/** + * @launchstack/retrieval — question in, cited answer out. + * + * The root exports the RagPort, its slot, and the citation builder. The + * algorithms live under ./algorithms (one folder per algorithm, each with + * its own README), the agent- and pipeline-facing tools under ./tools. + */ +export type { + RagPort, + CompanySearchOptions, + RagSearchFilters, + RagSearchResult, + RagSearchMetadata, +} from "./types"; +export { configureRag, getRag, getRagOrNull, ragCompanySearchSafe } from "./slot"; +export { + buildCitations, + type RetrievedEvidence, + type SourceVersionInfo, + type Citation, +} from "./tools/citation-builder"; diff --git a/packages/search/src/search-types.ts b/packages/retrieval/src/search-types.ts similarity index 99% rename from packages/search/src/search-types.ts rename to packages/retrieval/src/search-types.ts index 9d9909d5f..ad191cb5a 100644 --- a/packages/search/src/search-types.ts +++ b/packages/retrieval/src/search-types.ts @@ -117,7 +117,7 @@ export interface ANNResult { confidence: number; } -export type ANNStrategy = "hnsw" | "ivf" | "hybrid" | "prefiltered"; +export type ANNStrategy = "hnsw" | "ivf" | "hybrid" | "prefiltered" | "matryoshka"; export interface ANNConfig { strategy: ANNStrategy; diff --git a/packages/search/src/slot.ts b/packages/retrieval/src/slot.ts similarity index 84% rename from packages/search/src/slot.ts rename to packages/retrieval/src/slot.ts index 3d618a3ac..e828586e1 100644 --- a/packages/search/src/slot.ts +++ b/packages/retrieval/src/slot.ts @@ -21,7 +21,7 @@ export function getRag(): RagPort { const port = portSlot.get(); if (!port) { throw new Error( - "[@launchstack/adapters/rag] No RagPort registered. Pass `rag.port` to createEngine, or call configureRag(port) directly." + "[@launchstack/retrieval] No RagPort registered. Pass `rag.port` to createEngine, or call configureRag(port) directly." ); } return port; @@ -40,7 +40,7 @@ export async function ragCompanySearchSafe( try { return await port.companyEnsembleSearch(query, options); } catch (err) { - console.warn("[@launchstack/adapters/rag] companyEnsembleSearch failed:", err); + console.warn("[@launchstack/retrieval] companyEnsembleSearch failed:", err); return []; } } diff --git a/packages/retrieval/src/tools/citation-builder/README.md b/packages/retrieval/src/tools/citation-builder/README.md new file mode 100644 index 000000000..c4a67c56c --- /dev/null +++ b/packages/retrieval/src/tools/citation-builder/README.md @@ -0,0 +1,15 @@ +# citation-builder — retrieval hits to anchored citations + +**What it is.** The query-path half of citations (ADR-005 §3–4): it turns +already-permission-scoped retrieval rows into stable, anchored `Citation`s a +UI can render and a reader can trust. + +**How it works.** Each hit carries its source, version, and anchor; the +builder keys anchors (`anchorKey`), computes freshness against the source's +current version (`computeFreshness`, `DEFAULT_FRESHNESS_POLICY` from +`@launchstack/evidence`), and emits citations whose `relevance` is the +retrieval score. `relevance` is deliberately not called confidence — +extraction confidence belongs to evidence, retrieval relevance to the query. + +**When to use it.** Any consumer that surfaces retrieval results to a person. +If the answer shows text from a document, it should have gone through here. diff --git a/packages/search/src/citation-builder.test.ts b/packages/retrieval/src/tools/citation-builder/citation-builder.test.ts similarity index 100% rename from packages/search/src/citation-builder.test.ts rename to packages/retrieval/src/tools/citation-builder/citation-builder.test.ts diff --git a/packages/search/src/citation-builder.ts b/packages/retrieval/src/tools/citation-builder/citation-builder.ts similarity index 100% rename from packages/search/src/citation-builder.ts rename to packages/retrieval/src/tools/citation-builder/citation-builder.ts diff --git a/packages/retrieval/src/tools/citation-builder/index.ts b/packages/retrieval/src/tools/citation-builder/index.ts new file mode 100644 index 000000000..3e53a79d5 --- /dev/null +++ b/packages/retrieval/src/tools/citation-builder/index.ts @@ -0,0 +1,6 @@ +export { + buildCitations, + type RetrievedEvidence, + type SourceVersionInfo, + type Citation, +} from "./citation-builder"; diff --git a/packages/retrieval/src/tools/grounded-retrieval/README.md b/packages/retrieval/src/tools/grounded-retrieval/README.md new file mode 100644 index 000000000..265d186ae --- /dev/null +++ b/packages/retrieval/src/tools/grounded-retrieval/README.md @@ -0,0 +1,23 @@ +# grounded-retrieval — company-scoped retrieval with named policies + +**What it is.** One implementation of the retrieve → clean → cap pipeline +that marketing's stage modules each hand-rolled (weights `[0.4, 0.6]` was +previously repeated verbatim at six call sites with three different failure +behaviors). Pipelines and tools call this instead of the RagPort directly — +an architecture test enforces it. + +**How it works.** `retrieveCompanySnippets` runs the port's company ensemble +search under a named `SnippetPolicy` (topK, weights, snippet length), cleans +and caps each hit, and returns snippets ready for `formatSnippetBlock`. + +**Failure policy is declared per call, never implicit:** + +- `"throw"` — retrieval errors (including an unregistered RAG port) + propagate to the caller, which owns what a failure means. +- `"empty"` — errors degrade to zero snippets; the swallowed error is + logged so operators can still see it. Use only where the caller has + decided thin context beats no result. + +**When to use it.** Any brick or pipeline pulling company knowledge into a +prompt. Direct `getRag()` calls outside this folder are a boundary +violation, not a shortcut. diff --git a/packages/retrieval/src/tools/grounded-retrieval/grounded-retrieval.ts b/packages/retrieval/src/tools/grounded-retrieval/grounded-retrieval.ts new file mode 100644 index 000000000..849328179 --- /dev/null +++ b/packages/retrieval/src/tools/grounded-retrieval/grounded-retrieval.ts @@ -0,0 +1,91 @@ +/** + * grounded-retrieval — company-scoped RAG retrieval with named policies. + * + * One implementation of the retrieve → clean → cap pipeline that marketing's + * stage modules each hand-rolled (weights [0.4, 0.6] was previously repeated + * verbatim at six call sites with three different failure behaviors). + * + * Failure policy is declared per call, never implicit: + * - "throw": retrieval errors (including an unregistered RAG port) + * propagate to the caller, which owns what a failure means. + * - "empty": errors degrade to zero snippets; the swallowed error is logged + * so operators can still see it. Use only where the caller has decided + * that thin context is better than no result. + */ + +import { getRag } from "../../slot"; +import type { CompanySearchOptions, RagSearchResult } from "../../types"; + +export interface SnippetPolicy { + topK: number; + weights: [number, number]; + maxSnippets: number; + maxSnippetChars: number; +} + +/** + * Named policies freeze the constants the marketing pipeline used per call + * site at extraction time (design tenet: consolidation never changes values). + */ +export const SNIPPET_POLICIES = { + /** KB context, brand voice, persona (topK 6, 400-char snippets). */ + standard: { topK: 6, weights: [0.4, 0.6], maxSnippets: 6, maxSnippetChars: 400 }, + /** CompanyDNA RAG fallback (topK 4, 320-char snippets). */ + compact: { topK: 4, weights: [0.4, 0.6], maxSnippets: 4, maxSnippetChars: 320 }, + /** Per-claim source lookup (topK 2, 200-char snippets). */ + pinpoint: { topK: 2, weights: [0.4, 0.6], maxSnippets: 2, maxSnippetChars: 200 }, +} satisfies Record; + +export type RetrievalErrorPolicy = "throw" | "empty"; + +export interface RetrieveCompanySnippetsArgs { + companyId: number; + query: string; + policy: SnippetPolicy; + /** What a retrieval error means here. Required thinking, defaulted to "throw". */ + onError?: RetrievalErrorPolicy; +} + +export interface RetrievedSnippets { + /** Cleaned snippet texts (trimmed, whitespace-collapsed, char-capped). */ + snippets: string[]; + /** The raw results, for callers that need scores or metadata. */ + results: RagSearchResult[]; +} + +export function cleanSnippet(text: string, maxChars: number): string { + return text.trim().replace(/\s+/g, " ").slice(0, maxChars); +} + +export async function retrieveCompanySnippets( + args: RetrieveCompanySnippetsArgs +): Promise { + const { companyId, query, policy, onError = "throw" } = args; + const options: CompanySearchOptions = { + companyId, + topK: policy.topK, + weights: policy.weights, + }; + + let results: RagSearchResult[]; + try { + results = await getRag().companyEnsembleSearch(query, options); + } catch (error) { + if (onError === "throw") throw error; + console.warn("[tools/grounded-retrieval] retrieval failed (policy: empty):", error); + return { snippets: [], results: [] }; + } + + const snippets = results + .slice(0, policy.maxSnippets) + .map(r => cleanSnippet(r.pageContent, policy.maxSnippetChars)) + .filter(Boolean); + + return { snippets, results }; +} + +/** Number snippets into a prompt block: "1. …\n\n2. …", or the empty text. */ +export function formatSnippetBlock(snippets: string[], emptyText: string): string { + if (snippets.length === 0) return emptyText; + return snippets.map((s, i) => `${i + 1}. ${s}`).join("\n\n"); +} diff --git a/packages/retrieval/src/tools/grounded-retrieval/index.ts b/packages/retrieval/src/tools/grounded-retrieval/index.ts new file mode 100644 index 000000000..fd4d6ae83 --- /dev/null +++ b/packages/retrieval/src/tools/grounded-retrieval/index.ts @@ -0,0 +1,10 @@ +export { + SNIPPET_POLICIES, + cleanSnippet, + retrieveCompanySnippets, + formatSnippetBlock, + type SnippetPolicy, + type RetrievalErrorPolicy, + type RetrieveCompanySnippetsArgs, + type RetrievedSnippets, +} from "./grounded-retrieval"; diff --git a/packages/retrieval/src/tools/index.ts b/packages/retrieval/src/tools/index.ts new file mode 100644 index 000000000..4053876c4 --- /dev/null +++ b/packages/retrieval/src/tools/index.ts @@ -0,0 +1,9 @@ +/** + * The tools section: how an agent, pipeline, or route consumes retrieval. + * Tools compose the algorithms and the port; algorithms never import tools. + */ + +export * from "./citation-builder"; +export * from "./rag-search-tool"; +export * from "./grounded-retrieval"; +export * from "./rlm-search"; diff --git a/packages/retrieval/src/tools/rag-search-tool/README.md b/packages/retrieval/src/tools/rag-search-tool/README.md new file mode 100644 index 000000000..c52f3fba8 --- /dev/null +++ b/packages/retrieval/src/tools/rag-search-tool/README.md @@ -0,0 +1,17 @@ +# rag-search-tool — retrieval as a LangChain tool + +**What it is.** The agent-facing wrapper: a `rag_search` tool an agent can +call to search a user's documents, plus `formatResultsForPrompt` to render +hits as prompt context grouped by document with page markers. + +**How it works.** `createRagSearchTool(validateAccess)` builds the tool +around an app-supplied `AccessValidator` — which documents a user may search +is product-schema knowledge this package cannot hold, so the check is +injected and enforced before any retrieval runs. Validated IDs go through +the multi-document ensemble (`[0.4, 0.6]` bm25/vector), results come back +as JSON the agent can cite from, capped for context size. The userId rides +in the LangChain run's `configurable` bag. + +**When to use it.** Agent loops that decide for themselves when to consult +the corpus. Routes that always retrieve should call the ensemble directly — +the tool wrapper only adds value when a model chooses to invoke it. diff --git a/packages/retrieval/src/tools/rag-search-tool/format.ts b/packages/retrieval/src/tools/rag-search-tool/format.ts new file mode 100644 index 000000000..106d7a9b3 --- /dev/null +++ b/packages/retrieval/src/tools/rag-search-tool/format.ts @@ -0,0 +1,48 @@ +import type { SearchResult } from "../../search-types"; + +/** + * Group retrieval hits by document and render them as prompt context, page + * markers included. Titles resolve from the caller's map first, then hit + * metadata, then a plain "Document N" fallback. + */ +export function formatResultsForPrompt( + results: SearchResult[], + documentTitles?: Map +): string { + if (results.length === 0) { + return ""; + } + + const byDocument = new Map(); + for (const result of results) { + const docId = result.metadata.documentId; + if (docId !== undefined) { + if (!byDocument.has(docId)) { + byDocument.set(docId, []); + } + byDocument.get(docId)!.push(result); + } + } + + const sections: string[] = []; + + for (const [docId, docResults] of byDocument.entries()) { + const title = + documentTitles?.get(docId) ?? + docResults[0]?.metadata.documentTitle ?? + `Document ${docId}`; + + docResults.sort((a, b) => (a.metadata.page ?? 0) - (b.metadata.page ?? 0)); + + const content = docResults + .map(r => { + const pageInfo = r.metadata.page ? `[Page ${r.metadata.page}]` : ""; + return `${pageInfo}\n${r.pageContent}`; + }) + .join("\n\n"); + + sections.push(`--- ${title} ---\n${content}`); + } + + return sections.join("\n\n"); +} diff --git a/packages/retrieval/src/tools/rag-search-tool/index.ts b/packages/retrieval/src/tools/rag-search-tool/index.ts new file mode 100644 index 000000000..27b6ade71 --- /dev/null +++ b/packages/retrieval/src/tools/rag-search-tool/index.ts @@ -0,0 +1,2 @@ +export { createRagSearchTool, executeRAGSearch, type AccessValidator } from "./rag-search-tool"; +export { formatResultsForPrompt } from "./format"; diff --git a/apps/web/src/lib/tools/rag/agentic/rag-search-tool.ts b/packages/retrieval/src/tools/rag-search-tool/rag-search-tool.ts similarity index 50% rename from apps/web/src/lib/tools/rag/agentic/rag-search-tool.ts rename to packages/retrieval/src/tools/rag-search-tool/rag-search-tool.ts index 5fe0da62b..52f33ff16 100644 --- a/apps/web/src/lib/tools/rag/agentic/rag-search-tool.ts +++ b/packages/retrieval/src/tools/rag-search-tool/rag-search-tool.ts @@ -2,12 +2,30 @@ * RAG Search Tool * Role: LangChain tool that runs BM25+vector ensemble search on user documents. * Purpose: validate access, fetch relevant chunks, and format context for prompts. + * + * Access validation is injected: which documents a user may read is an + * app-side question (it lives in product schema this package cannot see), so + * the composition root supplies an AccessValidator and this tool enforces its + * answer before any retrieval runs. */ import { tool } from "@langchain/core/tools"; import { z } from "zod"; -import { multiDocEnsembleSearch, validateDocumentAccess, formatResultsForPrompt } from "../index"; -import type { RAGSearchResult, RAGSearchInput } from "../types"; +import { multiDocEnsembleSearch } from "../../algorithms/ensemble"; +import { formatResultsForPrompt } from "./format"; +import type { RAGSearchResult, RAGSearchInput } from "../../search-types"; + +/** + * Resolves which of the requested documents the user may search, plus their + * titles. Implemented by the app (workspace membership is product schema). + */ +export type AccessValidator = ( + userId: string, + requestedDocIds: (string | number)[] +) => Promise<{ + validDocIds: number[]; + documentTitles: Map; +}>; const RAGSearchSchema = z.object({ query: z.string().describe("The search query to find relevant document content"), @@ -20,7 +38,8 @@ const RAGSearchSchema = z.object({ */ export async function executeRAGSearch( input: RAGSearchInput, - userId: string + userId: string, + validateAccess: AccessValidator ): Promise<{ results: RAGSearchResult[]; formattedContext: string; @@ -30,10 +49,7 @@ export async function executeRAGSearch( try { // Validate document access - const { validDocIds, documentTitles } = await validateDocumentAccess( - userId, - input.documentIds - ); + const { validDocIds, documentTitles } = await validateAccess(userId, input.documentIds); if (validDocIds.length === 0) { return { @@ -91,53 +107,57 @@ export async function executeRAGSearch( } /** - * RAG Search Tool for LangChain + * Build the LangChain rag_search tool around an app-supplied access + * validator. The userId travels in the run's `configurable` bag, as before. */ -export const ragSearchTool = tool( - async (input, config): Promise => { - const userId = (config?.configurable as { userId?: string } | undefined)?.userId; - - if (!userId) { - return JSON.stringify({ - success: false, - error: "User ID not provided", - results: [], - }); - } +export function createRagSearchTool(validateAccess: AccessValidator) { + return tool( + async (input, config): Promise => { + const userId = (config?.configurable as { userId?: string } | undefined)?.userId; - try { - const { results, formattedContext } = await executeRAGSearch( - { - query: input.query, - documentIds: input.documentIds, - topK: input.topK, - }, - userId - ); - - return JSON.stringify({ - success: true, - resultCount: results.length, - results: results.slice(0, 5).map(r => ({ - content: r.content.substring(0, 500), - page: r.page, - documentTitle: r.documentTitle, - relevanceScore: r.relevanceScore, - })), - formattedContext: formattedContext.substring(0, 4000), - }); - } catch (error) { - return JSON.stringify({ - success: false, - error: error instanceof Error ? error.message : "Unknown error", - results: [], - }); + if (!userId) { + return JSON.stringify({ + success: false, + error: "User ID not provided", + results: [], + }); + } + + try { + const { results, formattedContext } = await executeRAGSearch( + { + query: input.query, + documentIds: input.documentIds, + topK: input.topK, + }, + userId, + validateAccess + ); + + return JSON.stringify({ + success: true, + resultCount: results.length, + results: results.slice(0, 5).map(r => ({ + content: r.content.substring(0, 500), + page: r.page, + documentTitle: r.documentTitle, + relevanceScore: r.relevanceScore, + })), + formattedContext: formattedContext.substring(0, 4000), + }); + } catch (error) { + return JSON.stringify({ + success: false, + error: error instanceof Error ? error.message : "Unknown error", + results: [], + }); + } + }, + { + name: "rag_search", + description: + "Search through uploaded study documents to find relevant content for answering questions or generating study materials. Use this when you need information from the user's documents.", + schema: RAGSearchSchema, } - }, - { - name: "rag_search", - description: - "Search through uploaded study documents to find relevant content for answering questions or generating study materials. Use this when you need information from the user's documents.", - schema: RAGSearchSchema, - } -); + ); +} diff --git a/packages/retrieval/src/tools/rlm-search/README.md b/packages/retrieval/src/tools/rlm-search/README.md new file mode 100644 index 000000000..20438a3b3 --- /dev/null +++ b/packages/retrieval/src/tools/rlm-search/README.md @@ -0,0 +1,23 @@ +# rlm-search — cost-aware retrieval as a service + +**What it is.** The consumable face of the RLM algorithm +(`algorithms/rlm/`): one call that plans, retrieves under a token budget, +and returns LLM-ready combined content — for routes and agents that want +cost-aware retrieval without driving the retriever's access patterns +themselves. + +**How it works.** `performRLMSearch(documentId, query, options)` resolves +the document's company embedding config, then either runs semantic search +(`prioritize: "relevance"`, the default) or budget-ordered section retrieval +(`"start"` / `"end"`, with semantic-type and page-range filters). The result +carries the sections with cumulative token costs, the overview, optional +previews, and a `combinedContent` string formatted for direct prompt +injection. Companion helpers expose the cheap planning calls: overviews +(single and batch), the structure tree, and drill-down by structure path. + +**When to use it.** Large documents where "retrieve the right 5% under a +budget" beats top-k chunks. For one-shot questions the ensemble is simpler +and cheaper. + +**Knobs.** `maxTokens` (default 4000), `prioritize`, `semanticTypes`, +`pageRange`, `includeOverview` / `includePreviews`, `embeddingIndexKey`. diff --git a/packages/retrieval/src/tools/rlm-search/index.ts b/packages/retrieval/src/tools/rlm-search/index.ts new file mode 100644 index 000000000..1f37adab2 --- /dev/null +++ b/packages/retrieval/src/tools/rlm-search/index.ts @@ -0,0 +1,9 @@ +export { + performRLMSearch, + getDocumentOverviewForPlanning, + getDocumentOverviewsBatch, + getDocumentStructureTree, + getSectionsByPath, + type RLMSearchOptions, + type RLMSearchResult, +} from "./rlm-search"; diff --git a/apps/web/src/app/api/agents/documentQ&A/services/rlmSearch.ts b/packages/retrieval/src/tools/rlm-search/rlm-search.ts similarity index 97% rename from apps/web/src/app/api/agents/documentQ&A/services/rlmSearch.ts rename to packages/retrieval/src/tools/rlm-search/rlm-search.ts index fb3c89130..3dd5e846e 100644 --- a/apps/web/src/app/api/agents/documentQ&A/services/rlmSearch.ts +++ b/packages/retrieval/src/tools/rlm-search/rlm-search.ts @@ -20,11 +20,10 @@ import { type SectionWithCost, type SectionPreview, type TokenBudgetOptions, -} from "~/lib/tools/rag/retrievers"; -import { getEmbeddings } from "./models"; -import { resolveEmbeddingIndex } from "@launchstack/llm/embeddings"; +} from "../../algorithms/rlm"; +import { createEmbeddingModel, resolveEmbeddingIndex } from "@launchstack/llm/embeddings"; import { getCompanyEmbeddingConfig } from "@launchstack/llm/embeddings"; -import { db } from "~/server/db"; +import { getDb } from "@launchstack/store/client"; import { document, type SemanticType, type PreviewType } from "@launchstack/store/schema"; // ============================================================================ @@ -110,7 +109,7 @@ export async function performRLMSearch( console.log(`🔍 [RLM Search] Starting search for document ${documentId}`); console.log(` Token budget: ${maxTokens}, Prioritize: ${prioritize}`); - const [documentRecord] = await db + const [documentRecord] = await getDb() .select({ companyId: document.companyId, }) @@ -125,7 +124,7 @@ export async function performRLMSearch( const needsEmbeddings = prioritize === "relevance"; const embeddingIndex = resolveEmbeddingIndex(embeddingIndexKey, companyConfig ?? undefined); const embeddings = needsEmbeddings - ? getEmbeddings(embeddingIndex.indexKey, companyConfig ?? undefined) + ? createEmbeddingModel(embeddingIndex, companyConfig ?? undefined) : undefined; const retriever = createRLMRetriever(embeddings, embeddingIndex); diff --git a/packages/search/src/types.ts b/packages/retrieval/src/types.ts similarity index 100% rename from packages/search/src/types.ts rename to packages/retrieval/src/types.ts diff --git a/packages/search/tsconfig.build.json b/packages/retrieval/tsconfig.build.json similarity index 100% rename from packages/search/tsconfig.build.json rename to packages/retrieval/tsconfig.build.json diff --git a/packages/search/tsconfig.json b/packages/retrieval/tsconfig.json similarity index 100% rename from packages/search/tsconfig.json rename to packages/retrieval/tsconfig.json diff --git a/packages/runtime/scripts/fix-esm-specifiers.mjs b/packages/runtime/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/runtime/scripts/fix-esm-specifiers.mjs +++ b/packages/runtime/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/schema-generator/scripts/fix-esm-specifiers.mjs b/packages/schema-generator/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/schema-generator/scripts/fix-esm-specifiers.mjs +++ b/packages/schema-generator/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/search/README.md b/packages/search/README.md deleted file mode 100644 index 14bd20ccf..000000000 --- a/packages/search/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# @launchstack/search - -Question in, cited answer out: hybrid retrieval (BM25 + vector ensemble behind a replaceable port), second-pass reranking, and the citation builder that turns permission-scoped retrieval rows into stable anchored citations. It deliberately does not contain index-time work — embeddings are generated by llm and persisted by indexing; this package only reads. - -## Install - -```bash -pnpm add @launchstack/search -``` - -## Use - -```ts -import { buildCitations } from "@launchstack/search"; -import { configureRag, getRag } from "@launchstack/search"; -``` - -## API - -| Subpath | What it is | -| --- | --- | -| `.` | the rag port, slot, and citation builder | -| `./retrievers` | bm25-retriever · vector-retriever | -| `./search-types` | the ensemble's result vocabulary | -| `./reranking` | (query, candidates[]) → reordered candidates | -| `./citation-builder` | (retrieval hits, version info) → Citation[] | - -## Configuration - -Nothing here reads `process.env`. Configuration is injected by the -composition root — `createEngine(config)` in `@launchstack/engine`, or the -package's own `configure*` hooks when used standalone. - - - -## Stability - -0.x. `relevance` is deliberately not called confidence — extraction confidence belongs to evidence, retrieval relevance to the query. - -## License - -Apache-2.0 — see [LICENSE](LICENSE). diff --git a/packages/search/package.json b/packages/search/package.json deleted file mode 100644 index 7a40c412c..000000000 --- a/packages/search/package.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "name": "@launchstack/search", - "version": "0.1.0", - "description": "Question in, cited answer out: hybrid retrieval (BM25 + vector ensemble behind a replaceable port), second-pass reranking, and the citation builder that turns permission-scoped retrieval rows into stable anchored citations.", - "license": "Apache-2.0", - "type": "module", - "sideEffects": false, - "main": "./src/index.ts", - "types": "./src/index.ts", - "exports": { - ".": "./src/index.ts", - "./retrievers": "./src/retrievers/index.ts", - "./search-types": "./src/search-types.ts", - "./types": "./src/types.ts", - "./reranking": "./src/reranking/index.ts", - "./citation-builder": "./src/citation-builder.ts", - "./package.json": "./package.json" - }, - "files": [ - "dist", - "README.md", - "LICENSE" - ], - "publishConfig": { - "access": "public", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "default": "./dist/index.js" - }, - "./retrievers": { - "types": "./dist/retrievers/index.d.ts", - "default": "./dist/retrievers/index.js" - }, - "./search-types": { - "types": "./dist/search-types.d.ts", - "default": "./dist/search-types.js" - }, - "./types": { - "types": "./dist/types.d.ts", - "default": "./dist/types.js" - }, - "./reranking": { - "types": "./dist/reranking/index.d.ts", - "default": "./dist/reranking/index.js" - }, - "./citation-builder": { - "types": "./dist/citation-builder.d.ts", - "default": "./dist/citation-builder.js" - }, - "./package.json": "./package.json" - } - }, - "scripts": { - "build": "tsc -p tsconfig.build.json && node ./scripts/fix-esm-specifiers.mjs", - "clean": "rm -rf dist .tsbuildinfo", - "typecheck": "tsc -p tsconfig.json --noEmit", - "test": "vitest run" - }, - "dependencies": { - "@launchstack/runtime": "workspace:^", - "@launchstack/store": "workspace:^", - "@launchstack/llm": "workspace:^", - "@langchain/community": "^0.3.56", - "@langchain/core": "^0.3.74", - "drizzle-orm": "^0.45.1", - "zod": "^3.23.8", - "@launchstack/evidence": "workspace:^" - }, - "devDependencies": { - "typescript": "^5.9.2", - "vitest": "^3.0.5" - } -} diff --git a/packages/search/src/hybrid-search.ts b/packages/search/src/hybrid-search.ts deleted file mode 100644 index d73ba7844..000000000 --- a/packages/search/src/hybrid-search.ts +++ /dev/null @@ -1,8 +0,0 @@ -export type { - RagPort, - CompanySearchOptions, - RagSearchFilters, - RagSearchResult, - RagSearchMetadata, -} from "./types"; -export { configureRag, getRag, getRagOrNull, ragCompanySearchSafe } from "./slot"; diff --git a/packages/search/src/index.ts b/packages/search/src/index.ts deleted file mode 100644 index 1755f5c66..000000000 --- a/packages/search/src/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -/** - * @launchstack/search — question in, cited answer out. Hybrid retrieval - * behind a replaceable port, second-pass reranking, and the citation builder. - */ -export * from "./hybrid-search"; -export { - buildCitations, - type RetrievedEvidence, - type SourceVersionInfo, - type Citation, -} from "./citation-builder"; diff --git a/packages/search/src/retrievers/index.ts b/packages/search/src/retrievers/index.ts deleted file mode 100644 index 627a5c98f..000000000 --- a/packages/search/src/retrievers/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Engine retrievers. - * - * Deliberately NOT re-exported from ../index.ts (the published `rag` facade subpath): - * bm25-retriever imports `@langchain/community`, which is an *optional* peer - * dependency. A consumer who only wants the RagPort types must not be forced to - * install it, so the retrievers live on their own `./rag/retrievers` subpath and - * the peer is only resolved by importers that actually reach for them. - */ - -export { - VectorRetriever, - createDocumentVectorRetriever, - createCompanyVectorRetriever, - createMultiDocVectorRetriever, -} from "./vector-retriever"; - -export { - getDocumentChunks, - getCompanyChunks, - getMultiDocChunks, - chunksToDocuments, - createDocumentBM25Retriever, - createCompanyBM25Retriever, - createMultiDocBM25Retriever, -} from "./bm25-retriever"; diff --git a/packages/store/scripts/fix-esm-specifiers.mjs b/packages/store/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/store/scripts/fix-esm-specifiers.mjs +++ b/packages/store/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/tools/__tests__/claim-evidence.test.ts b/packages/tools/__tests__/claim-evidence.test.ts index 77e0d3535..d77802d8c 100644 --- a/packages/tools/__tests__/claim-evidence.test.ts +++ b/packages/tools/__tests__/claim-evidence.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { configureRag, type RagPort, type RagSearchResult } from "@launchstack/search"; +import { configureRag, type RagPort, type RagSearchResult } from "@launchstack/retrieval"; import { lookUpClaim } from "@launchstack/tools/claim-evidence"; function port(impl: RagPort["companyEnsembleSearch"]): RagPort { diff --git a/packages/tools/__tests__/grounded-retrieval.test.ts b/packages/tools/__tests__/grounded-retrieval.test.ts index b629665e3..55034c3c2 100644 --- a/packages/tools/__tests__/grounded-retrieval.test.ts +++ b/packages/tools/__tests__/grounded-retrieval.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { configureRag, type RagPort, type RagSearchResult } from "@launchstack/search"; +import { configureRag, type RagPort, type RagSearchResult } from "@launchstack/retrieval"; import { cleanSnippet, formatSnippetBlock, diff --git a/packages/tools/package.json b/packages/tools/package.json index cfcb9b6d0..6d7dbaac3 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -71,7 +71,7 @@ "dependencies": { "@langchain/core": "^0.3.74", "@launchstack/llm": "workspace:^", - "@launchstack/search": "workspace:^", + "@launchstack/retrieval": "workspace:^", "@launchstack/store": "workspace:^", "drizzle-orm": "^0.45.1", "zod": "^3.23.8" diff --git a/packages/tools/scripts/fix-esm-specifiers.mjs b/packages/tools/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/packages/tools/scripts/fix-esm-specifiers.mjs +++ b/packages/tools/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/packages/tools/src/grounded-retrieval/index.ts b/packages/tools/src/grounded-retrieval/index.ts index ce6e42eef..49965c34c 100644 --- a/packages/tools/src/grounded-retrieval/index.ts +++ b/packages/tools/src/grounded-retrieval/index.ts @@ -1,90 +1,9 @@ /** - * grounded-retrieval — company-scoped RAG retrieval with named policies. - * - * One implementation of the retrieve → clean → cap pipeline that marketing's - * stage modules each hand-rolled (weights [0.4, 0.6] was previously repeated - * verbatim at six call sites with three different failure behaviors). - * - * Failure policy is declared per call, never implicit: - * - "throw": retrieval errors (including an unregistered RAG port) - * propagate to the caller, which owns what a failure means. - * - "empty": errors degrade to zero snippets; the swallowed error is logged - * so operators can still see it. Use only where the caller has decided - * that thin context is better than no result. + * Moved to the retrieval brick: it is a retrieval policy, not a vertical + * capability, and holding it here forced a cross-brick import for every + * consumer. Kept as a re-export so packages/tools consumers (brand-voice, + * company-context, claim-evidence, persona) and the pipelines keep their + * import paths; new code should import + * @launchstack/retrieval/tools/grounded-retrieval directly. */ - -import { getRag, type CompanySearchOptions, type RagSearchResult } from "@launchstack/search"; - -export interface SnippetPolicy { - topK: number; - weights: [number, number]; - maxSnippets: number; - maxSnippetChars: number; -} - -/** - * Named policies freeze the constants the marketing pipeline used per call - * site at extraction time (design tenet: consolidation never changes values). - */ -export const SNIPPET_POLICIES = { - /** KB context, brand voice, persona (topK 6, 400-char snippets). */ - standard: { topK: 6, weights: [0.4, 0.6], maxSnippets: 6, maxSnippetChars: 400 }, - /** CompanyDNA RAG fallback (topK 4, 320-char snippets). */ - compact: { topK: 4, weights: [0.4, 0.6], maxSnippets: 4, maxSnippetChars: 320 }, - /** Per-claim source lookup (topK 2, 200-char snippets). */ - pinpoint: { topK: 2, weights: [0.4, 0.6], maxSnippets: 2, maxSnippetChars: 200 }, -} satisfies Record; - -export type RetrievalErrorPolicy = "throw" | "empty"; - -export interface RetrieveCompanySnippetsArgs { - companyId: number; - query: string; - policy: SnippetPolicy; - /** What a retrieval error means here. Required thinking, defaulted to "throw". */ - onError?: RetrievalErrorPolicy; -} - -export interface RetrievedSnippets { - /** Cleaned snippet texts (trimmed, whitespace-collapsed, char-capped). */ - snippets: string[]; - /** The raw results, for callers that need scores or metadata. */ - results: RagSearchResult[]; -} - -export function cleanSnippet(text: string, maxChars: number): string { - return text.trim().replace(/\s+/g, " ").slice(0, maxChars); -} - -export async function retrieveCompanySnippets( - args: RetrieveCompanySnippetsArgs -): Promise { - const { companyId, query, policy, onError = "throw" } = args; - const options: CompanySearchOptions = { - companyId, - topK: policy.topK, - weights: policy.weights, - }; - - let results: RagSearchResult[]; - try { - results = await getRag().companyEnsembleSearch(query, options); - } catch (error) { - if (onError === "throw") throw error; - console.warn("[tools/grounded-retrieval] retrieval failed (policy: empty):", error); - return { snippets: [], results: [] }; - } - - const snippets = results - .slice(0, policy.maxSnippets) - .map(r => cleanSnippet(r.pageContent, policy.maxSnippetChars)) - .filter(Boolean); - - return { snippets, results }; -} - -/** Number snippets into a prompt block: "1. …\n\n2. …", or the empty text. */ -export function formatSnippetBlock(snippets: string[], emptyText: string): string { - if (snippets.length === 0) return emptyText; - return snippets.map((s, i) => `${i + 1}. ${s}`).join("\n\n"); -} +export * from "@launchstack/retrieval/tools/grounded-retrieval"; diff --git a/pipelines/package.json b/pipelines/package.json index f7c5d4eb0..e58bb38a8 100644 --- a/pipelines/package.json +++ b/pipelines/package.json @@ -194,7 +194,7 @@ "@launchstack/llm": "workspace:^", "@launchstack/orchestration": "workspace:^", "@launchstack/runtime": "workspace:^", - "@launchstack/search": "workspace:^", + "@launchstack/retrieval": "workspace:^", "@launchstack/store": "workspace:^", "@launchstack/tools": "workspace:^", "dayjs": "^1.11.18", diff --git a/pipelines/scripts/fix-esm-specifiers.mjs b/pipelines/scripts/fix-esm-specifiers.mjs index 2f30524fe..2402c47a3 100644 --- a/pipelines/scripts/fix-esm-specifiers.mjs +++ b/pipelines/scripts/fix-esm-specifiers.mjs @@ -9,7 +9,7 @@ * files are real ESM and Node requires a full path with an extension. Without * this step: * - * import("@launchstack/search/retrievers") + * import("@launchstack/retrieval/retrievers") * → ERR_MODULE_NOT_FOUND: Cannot find module '.../dist/rag/retrievers/vector-retriever' * * `publint` does NOT catch this — it validates the exports map against the diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d57a7c17d..0b3215ce3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -183,15 +183,15 @@ importers: '@launchstack/pipelines': specifier: workspace:^ version: link:../../pipelines + '@launchstack/retrieval': + specifier: workspace:^ + version: link:../../packages/retrieval '@launchstack/runtime': specifier: workspace:^ version: link:../../packages/runtime '@launchstack/schema-generator': specifier: workspace:^ version: link:../../packages/schema-generator - '@launchstack/search': - specifier: workspace:^ - version: link:../../packages/search '@launchstack/store': specifier: workspace:^ version: link:../../packages/store @@ -568,15 +568,15 @@ importers: '@launchstack/pipelines': specifier: workspace:^ version: link:../../pipelines + '@launchstack/retrieval': + specifier: workspace:^ + version: link:../../packages/retrieval '@launchstack/runtime': specifier: workspace:^ version: link:../../packages/runtime '@launchstack/schema-generator': specifier: workspace:^ version: link:../../packages/schema-generator - '@launchstack/search': - specifier: workspace:^ - version: link:../../packages/search '@launchstack/store': specifier: workspace:^ version: link:../../packages/store @@ -717,12 +717,12 @@ importers: '@launchstack/orchestration': specifier: workspace:^ version: link:../orchestration + '@launchstack/retrieval': + specifier: workspace:^ + version: link:../retrieval '@launchstack/runtime': specifier: workspace:^ version: link:../runtime - '@launchstack/search': - specifier: workspace:^ - version: link:../search '@launchstack/store': specifier: workspace:^ version: link:../store @@ -858,6 +858,49 @@ importers: specifier: ^3.0.5 version: 3.2.7(@types/debug@4.1.13)(@types/node@26.1.1)(jiti@2.5.1)(jsdom@29.0.2(@noble/hashes@2.4.0))(tsx@4.21.0)(yaml@2.9.0) + packages/retrieval: + dependencies: + '@langchain/community': + specifier: ^0.3.56 + version: 0.3.59(d2cd0da04fa74a59c40878b8ffb084da) + '@langchain/core': + specifier: ^0.3.74 + version: 0.3.74(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.220.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.9.0(@opentelemetry/api@1.9.1))(openai@5.12.2(ws@8.21.1)(zod@3.25.76)) + '@launchstack/evidence': + specifier: workspace:^ + version: link:../evidence + '@launchstack/indexing': + specifier: workspace:^ + version: link:../indexing + '@launchstack/llm': + specifier: workspace:^ + version: link:../llm + '@launchstack/runtime': + specifier: workspace:^ + version: link:../runtime + '@launchstack/store': + specifier: workspace:^ + version: link:../store + drizzle-orm: + specifier: ^0.45.1 + version: 0.45.1(@neondatabase/serverless@1.0.1)(@opentelemetry/api@1.9.1)(@types/pg@8.15.6)(postgres@3.4.7) + langchain: + specifier: ^0.3.33 + version: 0.3.33(16022a5e08c627e041522716ef8432a7) + neo4j-driver: + specifier: ^6.0.0 + version: 6.0.1 + zod: + specifier: ^3.23.8 + version: 3.25.76 + devDependencies: + typescript: + specifier: ^5.9.2 + version: 5.9.3 + vitest: + specifier: ^3.0.5 + version: 3.2.7(@types/debug@4.1.13)(@types/node@26.1.1)(jiti@2.5.1)(jsdom@29.0.2(@noble/hashes@2.4.0))(tsx@4.21.0)(yaml@2.9.0) + packages/runtime: devDependencies: typescript: @@ -898,40 +941,6 @@ importers: specifier: ^3.0.5 version: 3.2.7(@types/debug@4.1.13)(@types/node@26.1.1)(jiti@2.5.1)(jsdom@29.0.2(@noble/hashes@2.4.0))(tsx@4.21.0)(yaml@2.9.0) - packages/search: - dependencies: - '@langchain/community': - specifier: ^0.3.56 - version: 0.3.59(d2cd0da04fa74a59c40878b8ffb084da) - '@langchain/core': - specifier: ^0.3.74 - version: 0.3.74(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.220.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.9.0(@opentelemetry/api@1.9.1))(openai@5.12.2(ws@8.21.1)(zod@3.25.76)) - '@launchstack/evidence': - specifier: workspace:^ - version: link:../evidence - '@launchstack/llm': - specifier: workspace:^ - version: link:../llm - '@launchstack/runtime': - specifier: workspace:^ - version: link:../runtime - '@launchstack/store': - specifier: workspace:^ - version: link:../store - drizzle-orm: - specifier: ^0.45.1 - version: 0.45.1(@neondatabase/serverless@1.0.1)(@opentelemetry/api@1.9.1)(@types/pg@8.15.6)(postgres@3.4.7) - zod: - specifier: ^3.23.8 - version: 3.25.76 - devDependencies: - typescript: - specifier: ^5.9.2 - version: 5.9.3 - vitest: - specifier: ^3.0.5 - version: 3.2.7(@types/debug@4.1.13)(@types/node@26.1.1)(jiti@2.5.1)(jsdom@29.0.2(@noble/hashes@2.4.0))(tsx@4.21.0)(yaml@2.9.0) - packages/store: dependencies: '@launchstack/runtime': @@ -971,9 +980,9 @@ importers: '@launchstack/llm': specifier: workspace:^ version: link:../llm - '@launchstack/search': + '@launchstack/retrieval': specifier: workspace:^ - version: link:../search + version: link:../retrieval '@launchstack/store': specifier: workspace:^ version: link:../store @@ -1017,12 +1026,12 @@ importers: '@launchstack/orchestration': specifier: workspace:^ version: link:../packages/orchestration + '@launchstack/retrieval': + specifier: workspace:^ + version: link:../packages/retrieval '@launchstack/runtime': specifier: workspace:^ version: link:../packages/runtime - '@launchstack/search': - specifier: workspace:^ - version: link:../packages/search '@launchstack/store': specifier: workspace:^ version: link:../packages/store @@ -2700,19 +2709,10 @@ packages: peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - '@grpc/grpc-js@1.13.4': - resolution: {integrity: sha512-GsFaMXCkMqkKIvwCQjCrwH+GHbPKBjhwo/8ZuUkWHqbI73Kky9I+pQltrlT0+MWpedCoosda53lgjYfyEPgxBg==} - engines: {node: '>=12.10.0'} - '@grpc/grpc-js@1.14.4': resolution: {integrity: sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==} engines: {node: '>=12.10.0'} - '@grpc/proto-loader@0.7.15': - resolution: {integrity: sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==} - engines: {node: '>=6'} - hasBin: true - '@grpc/proto-loader@0.8.1': resolution: {integrity: sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==} engines: {node: '>=6'} @@ -13665,23 +13665,11 @@ snapshots: dependencies: graphql: 16.11.0 - '@grpc/grpc-js@1.13.4': - dependencies: - '@grpc/proto-loader': 0.7.15 - '@js-sdsl/ordered-map': 4.4.2 - '@grpc/grpc-js@1.14.4': dependencies: '@grpc/proto-loader': 0.8.1 '@js-sdsl/ordered-map': 4.4.2 - '@grpc/proto-loader@0.7.15': - dependencies: - lodash.camelcase: 4.3.0 - long: 5.3.2 - protobufjs: 7.5.4 - yargs: 17.7.2 - '@grpc/proto-loader@0.8.1': dependencies: lodash.camelcase: 4.3.0 @@ -19384,7 +19372,7 @@ snapshots: isstream: 0.1.2 jsonwebtoken: 9.0.3 mime-types: 2.1.35 - retry-axios: 2.6.0(axios@1.7.4(debug@4.4.3)) + retry-axios: 2.6.0(axios@1.7.4) tough-cookie: 4.1.4 transitivePeerDependencies: - supports-color @@ -21009,7 +20997,7 @@ snapshots: nice-grpc@2.1.12: dependencies: - '@grpc/grpc-js': 1.13.4 + '@grpc/grpc-js': 1.14.4 abort-controller-x: 0.4.3 nice-grpc-common: 2.0.2 @@ -22025,7 +22013,7 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 - retry-axios@2.6.0(axios@1.7.4(debug@4.4.3)): + retry-axios@2.6.0(axios@1.7.4): dependencies: axios: 1.7.4(debug@4.4.3) diff --git a/scripts/ci/e2e-ingest.mjs b/scripts/ci/e2e-ingest.mjs index eab277788..63dc825a0 100644 --- a/scripts/ci/e2e-ingest.mjs +++ b/scripts/ci/e2e-ingest.mjs @@ -27,10 +27,10 @@ const { createDocumentLifecycle } = await import( "../../packages/orchestration/src/source-lifecycle/lifecycle.ts" ); const { createCompanyBM25Retriever } = await import( - "../../packages/search/src/retrievers/bm25-retriever.ts" + "../../packages/retrieval/src/retrievers/bm25-retriever.ts" ); const { buildCitations } = await import( - "../../packages/search/src/citation-builder.ts" + "../../packages/retrieval/src/citation-builder.ts" ); const handle = createDb({ url: DATABASE_URL });