diff --git a/.changeset/odd-drinks-glow.md b/.changeset/odd-drinks-glow.md new file mode 100644 index 00000000..7594a192 --- /dev/null +++ b/.changeset/odd-drinks-glow.md @@ -0,0 +1,16 @@ +--- +'@rushdb/javascript-sdk': patch +'@rushdb/mcp-server': patch +'rushdb-dashboard': patch +'@rushdb/skills': patch +'rushdb-core': patch +'rushdb-docs': patch +--- + +Rework `$cycle` and stabilize relationship/query intelligence + +- **`$cycle` is now a record-level operator**: `where: { $cycle: { type, direction, hops } }` — the value IS the traversal spec. Replaces the previous `KEY: { $cycle: true, $relation: {...} }` block form entirely (breaking: the old form now throws). Compiles to an `EXISTS` subquery so the engine stops at the first cycle found per record instead of enumerating every path — avoids exponential blowup on densely connected graphs. Default traversal hop cap (`RUSHDB_MAX_TRAVERSAL_HOPS`) lowered from 25 to 10 to keep worst-case queries within the transaction budget. +- **Relationship pattern suggestions are now verified against live data**: candidates are proposed from schema names/types and LLM semantic judgment, then confirmed with a real graph probe before being surfaced — sampled schema values (which carry no signal on high-cardinality data) no longer gate or reject a suggestion. +- SmartSearch prompts and specs (core, MCP server, skills, docs) made more data-agnostic and kept in sync across all four DSL mirrors. +- Fix: CSV import no longer fails when a column parses to a JS `Date` (folded back to ISO string). +- Fix: dashboard record search box text no longer leaks between project switches. diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6e8afca7..bb00619f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -29,17 +29,14 @@ db.records.find({ labels: ['ACCOUNT'], where: { - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } }) ``` - A `$cycle` block requires `$relation` with `hops` (`min` ≥ 2, the default) and accepts no other keys; its label key is a display name. Combine with `$not` to select records not on a cycle. + `$cycle` is a record-level predicate: its value is the traversal spec itself (`type`, `direction`, `hops` — `hops` mandatory, `min` ≥ 2, defaulting to 2). A cycle has no separate endpoint, so it accepts no `$alias`, no property criteria, and no nested labels. Combine with `$not` to select records not on a cycle. - **Traversal depth policy** — `hops.max` is capped per deployment via `RUSHDB_MAX_TRAVERSAL_HOPS` (default 25) on the shared cloud connection. Self-hosted deployments (`RUSHDB_SELF_HOSTED=true`) and projects with a custom external Neo4j allow unbounded traversal (`max` omitted), guarded by the existing transaction timeout. + **Traversal depth policy** — `hops.max` is capped per deployment via `RUSHDB_MAX_TRAVERSAL_HOPS` (default 10) on the shared cloud connection. Self-hosted deployments (`RUSHDB_SELF_HOSTED=true`) and projects with a custom external Neo4j allow unbounded traversal (`max` omitted), guarded by the existing transaction timeout. Also in this release: diff --git a/docs/docs/connect/skills.mdx b/docs/docs/connect/skills.mdx index c1e603d3..8e8f5ff5 100644 --- a/docs/docs/connect/skills.mdx +++ b/docs/docs/connect/skills.mdx @@ -8,7 +8,7 @@ displayed_sidebar: docs Install RushDB Agent Skills to teach compatible AI assistants how to query RushDB correctly, model graph data, build faceted search, and use RushDB as persistent memory. -Skills complement the [MCP server](./mcp): MCP gives an agent runtime tools; Skills teach the agent when and how to use them. +Skills complement the [MCP server](/connect/mcp): MCP gives an agent runtime tools; Skills teach the agent when and how to use them. ## Install @@ -46,11 +46,11 @@ An empty schema is valid for a new project. The important result is that the age ## OpenClaw -OpenClaw uses workspace-level skill installation. Follow the [OpenClaw walkthrough](../learn/tutorials/agent-memory/agent-skills-with-openclaw#part-2--install-the-rushdb-skills-pack) for native `openclaw` and ClawHub commands. +OpenClaw uses workspace-level skill installation. Follow the [OpenClaw walkthrough](/learn/tutorials/agent-memory/agent-skills-with-openclaw#part-2--install-the-rushdb-skills-pack) for native `openclaw` and ClawHub commands. ## Next Steps -- [Connect the MCP server](./mcp) -- [Bootstrap persistent agent memory](../learn/agent-memory/quickstart) -- [Use RushDB with AI agents](./agents) +- [Connect the MCP server](/connect/mcp) +- [Bootstrap persistent agent memory](/learn/agent-memory/quickstart) +- [Use RushDB with AI agents](/connect/agents) - [Browse the skills source](https://github.com/rush-db/rushdb/tree/main/packages/skills) diff --git a/docs/docs/get-started/quick-tutorial.mdx b/docs/docs/get-started/quick-tutorial.mdx index c60f829d..7b16c0e8 100644 --- a/docs/docs/get-started/quick-tutorial.mdx +++ b/docs/docs/get-started/quick-tutorial.mdx @@ -197,20 +197,20 @@ curl -X POST "https://api.rushdb.com/api/v1/records/import/json" \ -→ [TypeScript SDK](/learn/reference/typescript) · [Python SDK](/learn/reference/python) · [REST API](../learn/reference/rest-api/introduction) +→ [TypeScript SDK](/learn/reference/typescript) · [Python SDK](/learn/reference/python) · [REST API](/learn/reference/rest-api/introduction) --- ## What to build -| Use case | Guide | -| ------------------------------------------- | ---------------------------------------------------------------------------------------- | -| Agent memory — sessions, decisions, context | [Episodic Memory for Multi-Step Agents](../learn/tutorials/agent-memory/episodic-memory) | -| GraphRAG — graph + vector search | [GraphRAG Tutorial](../learn/tutorials/ai-and-rag/graphrag) | -| Semantic search over your data | [Semantic Search in 5 Minutes](../learn/tutorials/ai-and-rag/ai-semantic-search) | -| Hybrid filter + semantic search | [Hybrid Retrieval](../learn/tutorials/ai-and-rag/hybrid-retrieval) | -| Explore an unknown dataset | [Discovery Queries](../learn/tutorials/search-and-queries/discovery-queries) | -| Safe agent query planning | [Agent-Safe Query Planning](../learn/tutorials/agent-memory/agent-safe-query-planning) | +| Use case | Guide | +| ------------------------------------------- | -------------------------------------------------------------------------------------- | +| Agent memory — sessions, decisions, context | [Episodic Memory for Multi-Step Agents](/learn/tutorials/agent-memory/episodic-memory) | +| GraphRAG — graph + vector search | [GraphRAG Tutorial](/learn/tutorials/ai-and-rag/graphrag) | +| Semantic search over your data | [Semantic Search in 5 Minutes](/learn/tutorials/ai-and-rag/ai-semantic-search) | +| Hybrid filter + semantic search | [Hybrid Retrieval](/learn/tutorials/ai-and-rag/hybrid-retrieval) | +| Explore an unknown dataset | [Discovery Queries](/learn/tutorials/search-and-queries/discovery-queries) | +| Safe agent query planning | [Agent-Safe Query Planning](/learn/tutorials/agent-memory/agent-safe-query-planning) | --- diff --git a/docs/docs/learn/records-and-queries/find-and-query.mdx b/docs/docs/learn/records-and-queries/find-and-query.mdx index 1cfb795a..4a17887f 100644 --- a/docs/docs/learn/records-and-queries/find-and-query.mdx +++ b/docs/docs/learn/records-and-queries/find-and-query.mdx @@ -322,7 +322,7 @@ curl -X POST https://api.rushdb.com/api/v1/records/search \ ### Multi-hop and cycles -Add `hops` to `$relation` to traverse a variable number of hops (`type`/`direction` apply to every hop; criteria filter only the final record), and `$cycle: true` to find records on a closed path back to themselves: +Add `hops` to `$relation` to traverse a variable number of hops (`type`/`direction` apply to every hop; criteria filter only the final record), and use the `$cycle` operator — its value is the traversal spec itself, with `hops` mandatory (`min` ≥ 2) — to find records on a closed path back to themselves: @@ -343,10 +343,7 @@ result = db.records.find({ result = db.records.find({ "labels": ["ACCOUNT"], "where": { - "RING": { # display name, not matched as a label - "$cycle": True, - "$relation": {"type": "TRANSFERRED_TO", "direction": "out", "hops": {"min": 2, "max": 6}} - } + "$cycle": {"type": "TRANSFERRED_TO", "direction": "out", "hops": {"min": 2, "max": 6}} } }) ``` @@ -370,11 +367,7 @@ const { data } = await db.records.find({ const { data: rings } = await db.records.find({ labels: ['ACCOUNT'], where: { - RING: { - // display name, not matched as a label - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } }) ``` diff --git a/docs/docs/learn/reference/python/SearchQuery.md b/docs/docs/learn/reference/python/SearchQuery.md index 344a67ce..31d64d61 100644 --- a/docs/docs/learn/reference/python/SearchQuery.md +++ b/docs/docs/learn/reference/python/SearchQuery.md @@ -238,7 +238,7 @@ result = db.records.find({ ##### Variable-length traversal (`hops`) -Add `hops` to `$relation` to match records up to N hops away. A number means exactly N hops; a `{"min": ..., "max": ...}` dict is a range (`min` defaults to `1`). `type` and `direction` apply to every hop; property criteria and the block's label constrain only the final record — intermediates are anonymous. `max` is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 25); omitting it requests unbounded traversal, allowed only on self-hosted deployments or projects with a custom Neo4j instance: +Add `hops` to `$relation` to match records up to N hops away. A number means exactly N hops; a `{"min": ..., "max": ...}` dict is a range (`min` defaults to `1`). `type` and `direction` apply to every hop; property criteria and the block's label constrain only the final record — intermediates are anonymous. `max` is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 10); omitting it requests unbounded traversal, allowed only on self-hosted deployments or projects with a custom Neo4j instance: ```python # Employees whose management chain (up to 4 hops) contains Alice @@ -259,20 +259,17 @@ result = db.records.find({ ##### Cycle detection (`$cycle`) -`"$cycle": True` matches records on a closed path back to themselves (rings, circular ownership). It requires a dict `$relation` with `hops` (`min` ≥ 2, defaults to 2) and accepts nothing else — no `$alias`, no property criteria, no nested labels. The block's key is a display name, not matched as a label: +`$cycle` is a record-level predicate that matches records on a closed path back to themselves (rings, circular ownership). Its value is the traversal spec itself — `type`, `direction`, `hops` — with `hops` mandatory (`min` ≥ 2, defaults to 2). It accepts nothing else — no `$alias`, no property criteria, no nested labels; intermediate node labels are unconstrained. It composes under `$not`/`$or`/`$and`, and can be placed inside a label block to anchor the cycle at a related record: ```python # Accounts on a directed transfer ring of 2–6 hops result = db.records.find({ "labels": ["ACCOUNT"], "where": { - "RING": { - "$cycle": True, - "$relation": { - "type": "TRANSFERRED_TO", - "direction": "out", - "hops": { "min": 2, "max": 6 } - } + "$cycle": { + "type": "TRANSFERRED_TO", + "direction": "out", + "hops": { "min": 2, "max": 6 } } } }) diff --git a/docs/docs/learn/reference/typescript/SearchQuery.md b/docs/docs/learn/reference/typescript/SearchQuery.md index 02581514..5dfe544d 100644 --- a/docs/docs/learn/reference/typescript/SearchQuery.md +++ b/docs/docs/learn/reference/typescript/SearchQuery.md @@ -386,7 +386,6 @@ export type Related = Models> = [Key in keyof M]?: { $alias?: string $relation?: TraversalRelationOptions | string - $cycle?: boolean } & Where } @@ -395,6 +394,10 @@ export type TraversalHops = number | { min?: number; max?: number } export type TraversalRelationOptions = RelationOptions & { hops?: TraversalHops } + +export type CycleExpression = RelationOptions & { + hops: TraversalHops +} ``` Defines conditions on related records. The key of the nested object **is** the label name (case-sensitive). Use `$alias` to name the traversal for later use in `select`/`groupBy`, and `$relation` to constrain the relationship type or direction: @@ -421,7 +424,7 @@ Defines conditions on related records. The key of the nested object **is** the l } ``` -`hops` turns the traversal into a variable-length one: a number means exactly N hops, `{ min?, max? }` a range (`min` defaults to `1`). `type` and `direction` apply to every hop; property criteria and the nested label constrain only the final record — intermediates are anonymous. `max` is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 25); omitting it requests unbounded traversal, allowed only on self-hosted deployments or projects with a custom Neo4j instance: +`hops` turns the traversal into a variable-length one: a number means exactly N hops, `{ min?, max? }` a range (`min` defaults to `1`). `type` and `direction` apply to every hop; property criteria and the nested label constrain only the final record — intermediates are anonymous. `max` is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 10); omitting it requests unbounded traversal, allowed only on self-hosted deployments or projects with a custom Neo4j instance: ```typescript // Employees whose management chain (1–4 hops) contains Alice @@ -436,17 +439,14 @@ Defines conditions on related records. The key of the nested object **is** the l } ``` -`$cycle: true` matches records that sit on a closed path back to themselves. It requires an object `$relation` with `hops` (`min` ≥ 2, defaults to 2) and accepts nothing else — no `$alias`, no property criteria, no nested labels. The block's key is a display name, not matched as a label: +`$cycle` is a record-level predicate that matches records sitting on a closed path back to themselves. Its value is the traversal spec itself — `type`, `direction`, `hops` — with `hops` mandatory (`min` ≥ 2, defaults to 2). It accepts nothing else — no `$alias`, no property criteria, no nested labels; intermediate node labels are unconstrained. It composes under `$not`/`$or`/`$and`, and can be placed inside a label block to anchor the cycle at a related record: ```typescript // Accounts on a directed transfer ring of 2–6 hops { labels: ['ACCOUNT'], where: { - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } } ``` diff --git a/docs/docs/learn/search-query/search-query.md b/docs/docs/learn/search-query/search-query.md index 61bf88b6..ce199897 100644 --- a/docs/docs/learn/search-query/search-query.md +++ b/docs/docs/learn/search-query/search-query.md @@ -126,7 +126,7 @@ For optimal performance when using the Search API: 1. **Be Specific**: Filter by labels when possible to narrow the search scope 2. **Use Indexed Properties**: Prioritize filtering on properties that have indexes 3. **Limit Results**: Use pagination to retrieve only the records you need -4. **Bound Your Traversals**: Variable-length traversal (`$relation.hops`) is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 25) — keep `max` as small as your use case allows, and constrain `type` and `direction` to prune the search +4. **Bound Your Traversals**: Variable-length traversal (`$relation.hops`) is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 10) — keep `max` as small as your use case allows, and constrain `type` and `direction` to prune the search 5. **Use Aliases Efficiently**: Define aliases only for records you need to reference in aggregations ## Next Steps diff --git a/docs/docs/learn/search-query/where-operators.md b/docs/docs/learn/search-query/where-operators.md index d9b9c114..db3a466a 100644 --- a/docs/docs/learn/search-query/where-operators.md +++ b/docs/docs/learn/search-query/where-operators.md @@ -710,7 +710,7 @@ Key semantics: - `type` and `direction` constrain **every hop**; the nested label constrains only the **endpoint** record. Intermediate records are anonymous and unconstrained. - Omitting `type` means "any relationship between records" — each hop may even use a different relationship type. RushDB automatically excludes its internal property metadata edges, so untyped traversal never leaves your data model. -- `hops.max` is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 25). Omitting `max` requests **unbounded** traversal (`*min..`), which is only allowed on self-hosted deployments and projects with a custom Neo4j instance — bounded there by the transaction timeout. +- `hops.max` is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 10). Omitting `max` requests **unbounded** traversal (`*min..`), which is only allowed on self-hosted deployments and projects with a custom Neo4j instance — bounded there by the transaction timeout. - Deeper nesting composes naturally: a nested label block under a `hops` endpoint continues from that endpoint. > **Performance:** variable-length traversal explores every matching path. Keep `max` as small as your use case allows, and prefer setting `direction` — an undirected, untyped `hops` query is the most expensive shape. @@ -719,19 +719,16 @@ Key semantics: ### Cycle Detection (`$cycle`) -Use `$cycle: true` to find records that sit on a **closed path back to themselves** — fraud rings, circular ownership, dependency cycles: +`$cycle` is a record-level predicate that finds records sitting on a **closed path back to themselves** — fraud rings, circular ownership, dependency cycles. Its value is the traversal spec itself: ```typescript { labels: ["ACCOUNT"], where: { - RING: { // Key is a display name — NOT matched as a label - $cycle: true, - $relation: { - type: "TRANSFERRED_TO", - direction: "out", - hops: { min: 2, max: 6 } // Ring length; min defaults to 2 for cycles - } + $cycle: { + type: "TRANSFERRED_TO", + direction: "out", + hops: { min: 2, max: 6 } // Ring length; min defaults to 2 for cycles } } } @@ -741,10 +738,10 @@ This returns every ACCOUNT participating in a directed transfer ring of 2–6 ho Rules and semantics: -- A `$cycle` block requires `$relation` with `hops` (`min` ≥ 2 — a 1-hop "cycle" would be a self-loop) and accepts **nothing else**: no `$alias`, no property criteria, no nested labels. A cycle has no separate endpoint to filter or alias — filter the root record instead. -- The block's key is ignored as a label (both ends of the pattern are the root record); it just needs to be unique among its siblings. +- `$cycle`'s value is the traversal spec — `type`, `direction`, `hops` — with `hops` mandatory (`min` ≥ 2, defaults to 2 — a 1-hop "cycle" would be a self-loop). It accepts **nothing else**: no `$alias`, no property criteria, no nested labels. A cycle has no separate endpoint to filter or alias — filter the root record instead. Intermediate node labels are unconstrained. - Set `direction` for flow-like semantics (money, ownership, dependencies). An undirected cycle also matches innocent back-and-forth pairs like mutual transfers. -- Combine with logical operators: `$not: { RING: { $cycle: true, ... } }` finds records **not** on a cycle. +- Combine with logical operators: `$not: { $cycle: { ... } }` finds records **not** on a cycle. +- `$cycle` can also be placed inside a label block to anchor the cycle at a **related** record: `ACCOUNT: { country: "US", $cycle: { ... } }`. - Cypher trail semantics apply: each _relationship_ is used once per path, but a path may revisit a record through a different relationship — figure-eight shapes count as cycles. ### Aliasing for Aggregations diff --git a/docs/docs/learn/tutorials/graph-modeling/fraud-rings.mdx b/docs/docs/learn/tutorials/graph-modeling/fraud-rings.mdx index 12d3e5f5..c456c9bf 100644 --- a/docs/docs/learn/tutorials/graph-modeling/fraud-rings.mdx +++ b/docs/docs/learn/tutorials/graph-modeling/fraud-rings.mdx @@ -157,14 +157,10 @@ For Model B you would create a `TRANSACTION` record per movement (`{ amount: 950 const ringParticipants = await db.records.find({ labels: ['ACCOUNT'], where: { - RING: { - // Display name — NOT matched as a label - $cycle: true, - $relation: { - type: 'TRANSFERRED_TO', // Every hop is a transfer - direction: 'out', // Money always flows forward - hops: { min: 2, max: 6 } // Ring length in transfers - } + $cycle: { + type: 'TRANSFERRED_TO', // Every hop is a transfer + direction: 'out', // Money always flows forward + hops: { min: 2, max: 6 } // Ring length in transfers } } }) @@ -178,13 +174,10 @@ const ringParticipants = await db.records.find({ ring_participants = db.records.find({ "labels": ["ACCOUNT"], "where": { - "RING": { - "$cycle": True, - "$relation": { - "type": "TRANSFERRED_TO", - "direction": "out", - "hops": {"min": 2, "max": 6} - } + "$cycle": { + "type": "TRANSFERRED_TO", + "direction": "out", + "hops": {"min": 2, "max": 6} } } }) @@ -199,10 +192,7 @@ curl -s -X POST "$BASE/records/search" \ -d '{ "labels": ["ACCOUNT"], "where": { - "RING": { - "$cycle": true, - "$relation": {"type": "TRANSFERRED_TO", "direction": "out", "hops": {"min": 2, "max": 6}} - } + "$cycle": {"type": "TRANSFERRED_TO", "direction": "out", "hops": {"min": 2, "max": 6}} } }' ``` @@ -210,15 +200,14 @@ curl -s -X POST "$BASE/records/search" \ -The rules of a `$cycle` block, briefly: +The rules of the `$cycle` operator, briefly: -- It requires `$relation` with `hops` (`min` ≥ 2, defaulting to 2 — a 1-hop "cycle" would be a self-loop) and accepts **nothing else**: no `$alias`, no property criteria, no nested labels. Both ends of the path are the root record, so filter the root instead. -- The key (`RING`) is a display name, not a label; it just has to be unique among its siblings. -- `type` and `direction` apply to every hop; `hops` bounds the ring length. +- It is a record-level predicate: its value is the traversal spec itself — `type`, `direction`, `hops` — with `hops` mandatory (`min` ≥ 2, defaulting to 2 — a 1-hop "cycle" would be a self-loop). It accepts **nothing else**: no `$alias`, no property criteria, no nested labels. Both ends of the path are the root record, so filter the root instead. +- `type` and `direction` apply to every hop; `hops` bounds the ring length. Intermediate node labels are unconstrained. ### Model B: untyped, directed hops — with doubled counts -In Model B the path alternates `ACCOUNT -[SENT]-> TRANSACTION -[CREDITS]-> ACCOUNT`, so each money movement costs **two hops**. A ring of 2–6 transfers is a closed path of 4–12 hops. And because the hops alternate between two relationship types, omit `type` — untyped traversal allows a different type on each hop (RushDB automatically excludes its internal property metadata edges, so this never leaves your data model): +In Model B the path alternates `ACCOUNT -[SENT]-> TRANSACTION -[CREDITS]-> ACCOUNT`, so each money movement costs **two hops**. A ring of 2–5 transfers is a closed path of 4–10 hops (the default cloud hop cap is 10, so in Model B that budget covers rings of up to 5 transfers). And because the hops alternate between two relationship types, omit `type` — untyped traversal allows a different type on each hop (RushDB automatically excludes its internal property metadata edges, so this never leaves your data model): @@ -227,12 +216,9 @@ In Model B the path alternates `ACCOUNT -[SENT]-> TRANSACTION -[CREDITS]-> ACCOU const ringParticipants = await db.records.find({ labels: ['ACCOUNT'], where: { - RING: { - $cycle: true, - $relation: { - direction: 'out', // SENT and CREDITS both point "downstream" - hops: { min: 4, max: 12 } // 2–6 transfers × 2 hops each - } + $cycle: { + direction: 'out', // SENT and CREDITS both point "downstream" + hops: { min: 4, max: 10 } // 2–5 transfers × 2 hops each } } }) @@ -245,10 +231,7 @@ const ringParticipants = await db.records.find({ ring_participants = db.records.find({ "labels": ["ACCOUNT"], "where": { - "RING": { - "$cycle": True, - "$relation": {"direction": "out", "hops": {"min": 4, "max": 12}} - } + "$cycle": {"direction": "out", "hops": {"min": 4, "max": 10}} } }) ``` @@ -262,10 +245,7 @@ curl -s -X POST "$BASE/records/search" \ -d '{ "labels": ["ACCOUNT"], "where": { - "RING": { - "$cycle": true, - "$relation": {"direction": "out", "hops": {"min": 4, "max": 12}} - } + "$cycle": {"direction": "out", "hops": {"min": 4, "max": 10}} } }' ``` @@ -296,7 +276,7 @@ hops: { min: 3, max: 6 } // rings of 3+ accounts only ## Step 3: Combine with root filters -A `$cycle` block cannot carry criteria — but the **root** record can. Scope candidate detection to the accounts your analysts actually care about: +A `$cycle` spec cannot carry criteria — but the **root** record can. Scope candidate detection to the accounts your analysts actually care about: @@ -308,10 +288,7 @@ const suspects = await db.records.find({ where: { country: { $in: ['CY', 'LV', 'MT'] }, riskTier: 'high', - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } }) @@ -320,10 +297,7 @@ const cleanAccounts = await db.records.find({ labels: ['ACCOUNT'], where: { $not: { - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } } }) @@ -338,10 +312,7 @@ suspects = db.records.find({ "where": { "country": {"$in": ["CY", "LV", "MT"]}, "riskTier": "high", - "RING": { - "$cycle": True, - "$relation": {"type": "TRANSFERRED_TO", "direction": "out", "hops": {"min": 2, "max": 6}} - } + "$cycle": {"type": "TRANSFERRED_TO", "direction": "out", "hops": {"min": 2, "max": 6}} } }) @@ -349,10 +320,7 @@ clean_accounts = db.records.find({ "labels": ["ACCOUNT"], "where": { "$not": { - "RING": { - "$cycle": True, - "$relation": {"type": "TRANSFERRED_TO", "direction": "out", "hops": {"min": 2, "max": 6}} - } + "$cycle": {"type": "TRANSFERRED_TO", "direction": "out", "hops": {"min": 2, "max": 6}} } } }) @@ -369,10 +337,7 @@ curl -s -X POST "$BASE/records/search" \ "where": { "country": {"$in": ["CY", "LV", "MT"]}, "riskTier": "high", - "RING": { - "$cycle": true, - "$relation": {"type": "TRANSFERRED_TO", "direction": "out", "hops": {"min": 2, "max": 6}} - } + "$cycle": {"type": "TRANSFERRED_TO", "direction": "out", "hops": {"min": 2, "max": 6}} } }' ``` @@ -465,16 +430,16 @@ Each step is a one-hop, ID-anchored query — cheap and index-friendly. In pract ### Honest limitations -- **No per-hop predicates.** You cannot express "every transfer along the path exceeds $10,000". In Model A the amounts don't exist at all; in Model B the intermediate `TRANSACTION` records inside a `hops` traversal are anonymous and unconstrained — only the endpoint can be filtered, and a `$cycle`block can't even do that. Amount-aware ring scoring belongs in the reconstruction step, where each`TRANSACTION` is a normal record you can filter and aggregate. +- **No per-hop predicates.** You cannot express "every transfer along the path exceeds $10,000". In Model A the amounts don't exist at all; in Model B the intermediate `TRANSACTION` records inside a `hops` traversal are anonymous and unconstrained — only the endpoint can be filtered, and a `$cycle` spec can't even do that. Amount-aware ring scoring belongs in the reconstruction step, where each`TRANSACTION` is a normal record you can filter and aggregate. - **Trail semantics.** Each _relationship_ is used at most once per path, but records may repeat — a figure-eight through a shared mule account counts as one cycle. Treat `$cycle` as a candidate generator, not a verdict. --- ## Performance notes -- **Keep `max` small.** Variable-length traversal explores every matching path, and each extra hop multiplies the path count. Real laundering rings are short (3–6 parties); start there and widen only with evidence. Remember that in Model B the hop budget spends twice as fast — 12 hops is only 6 transfers. +- **Keep `max` small.** Variable-length traversal explores every matching path, and each extra hop multiplies the path count. Real laundering rings are short (3–6 parties); start there and widen only with evidence. Remember that in Model B the hop budget spends twice as fast — 10 hops is only 5 transfers. - **Always set `direction`.** It halves the expansion at every hop _and_ it is what makes the result mean "circular flow". An undirected, untyped `hops` query is the most expensive shape SearchQuery can produce. -- **Mind the hop cap.** `hops.max` is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 25) on shared cloud. Unbounded traversal (omitting `max`) is only available on self-hosted deployments and projects with a custom Neo4j instance, bounded there by the transaction timeout. +- **Mind the hop cap.** `hops.max` is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 10) on shared cloud. Unbounded traversal (omitting `max`) is only available on self-hosted deployments and projects with a custom Neo4j instance, bounded there by the transaction timeout. - **Filter the root first.** Property criteria on the root (`country`, `riskTier`, activity flags) run before traversal and shrink the candidate set the cycle check has to cover. --- diff --git a/docs/docs/learn/tutorials/graph-modeling/modeling-hierarchies.mdx b/docs/docs/learn/tutorials/graph-modeling/modeling-hierarchies.mdx index 090f8182..513be5c8 100644 --- a/docs/docs/learn/tutorials/graph-modeling/modeling-hierarchies.mdx +++ b/docs/docs/learn/tutorials/graph-modeling/modeling-hierarchies.mdx @@ -316,7 +316,7 @@ How `hops` behaves: - `hops: 3` matches exactly 3 hops; `hops: { min, max }` matches a range (`min` defaults to `1`). - `type` and `direction` constrain **every hop**; the nested label and its criteria constrain only the **endpoint** record. Intermediate records are anonymous and unconstrained. -- `hops.max` is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 25). Omitting `max` requests unbounded traversal, which is only allowed on self-hosted deployments and projects with a custom Neo4j instance. +- `hops.max` is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 10). Omitting `max` requests unbounded traversal, which is only allowed on self-hosted deployments and projects with a custom Neo4j instance. Keep manual nesting when levels carry different labels, or when you need to alias or filter intermediate levels — `hops` cannot express per-level criteria. @@ -484,7 +484,7 @@ graph LR PKG_C -->|DEPENDS_ON| PKG_D ``` -SearchQuery supports variable-length traversal natively: add [`hops` to `$relation`](/learn/search-query/where-operators#variable-length-traversal-hops) to follow a relationship pattern up to N hops in a single block. Traversal depth is bounded by a per-deployment cap (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 25); truly unbounded traversal (omitting `max`) is available only on self-hosted deployments and projects with a custom Neo4j instance. Explicitly scoping traversal to the depth your product requires remains good practice — for blast-radius analysis (which packages are affected if `crypto-utils` has a CVE?), traverse up to a known safe depth. +SearchQuery supports variable-length traversal natively: add [`hops` to `$relation`](/learn/search-query/where-operators#variable-length-traversal-hops) to follow a relationship pattern up to N hops in a single block. Traversal depth is bounded by a per-deployment cap (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 10); truly unbounded traversal (omitting `max`) is available only on self-hosted deployments and projects with a custom Neo4j instance. Explicitly scoping traversal to the depth your product requires remains good practice — for blast-radius analysis (which packages are affected if `crypto-utils` has a CVE?), traverse up to a known safe depth. ### Ingesting the dependency graph @@ -726,7 +726,7 @@ One caveat when aggregating over multihop matches: the traversal produces one ro ### Cyclic query: detecting circular dependencies with `$cycle` -Cyclic graphs raise a question tree-shaped data never does: does anything depend on itself, directly or transitively? [`$cycle: true`](/learn/search-query/where-operators#cycle-detection-cycle) finds records that sit on a closed path back to themselves: +Cyclic graphs raise a question tree-shaped data never does: does anything depend on itself, directly or transitively? [`$cycle`](/learn/search-query/where-operators#cycle-detection-cycle) finds records that sit on a closed path back to themselves: @@ -735,11 +735,7 @@ Cyclic graphs raise a question tree-shaped data never does: does anything depend const cyclicPackages = await db.records.find({ labels: ['PACKAGE'], where: { - DEPENDENCY_CYCLE: { - // Display name — NOT matched as a label - $cycle: true, - $relation: { type: 'DEPENDS_ON', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'DEPENDS_ON', direction: 'out', hops: { min: 2, max: 6 } } } }) @@ -748,10 +744,7 @@ const acyclicPackages = await db.records.find({ labels: ['PACKAGE'], where: { $not: { - DEPENDENCY_CYCLE: { - $cycle: true, - $relation: { type: 'DEPENDS_ON', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'DEPENDS_ON', direction: 'out', hops: { min: 2, max: 6 } } } } }) @@ -764,10 +757,7 @@ const acyclicPackages = await db.records.find({ cyclic_packages = db.records.find({ "labels": ["PACKAGE"], "where": { - "DEPENDENCY_CYCLE": { - "$cycle": True, - "$relation": {"type": "DEPENDS_ON", "direction": "out", "hops": {"min": 2, "max": 6}} - } + "$cycle": {"type": "DEPENDS_ON", "direction": "out", "hops": {"min": 2, "max": 6}} } }) ``` @@ -781,10 +771,7 @@ curl -s -X POST "$BASE/records/search" \ -d '{ "labels": ["PACKAGE"], "where": { - "DEPENDENCY_CYCLE": { - "$cycle": true, - "$relation": {"type": "DEPENDS_ON", "direction": "out", "hops": {"min": 2, "max": 6}} - } + "$cycle": {"type": "DEPENDS_ON", "direction": "out", "hops": {"min": 2, "max": 6}} } }' ``` @@ -794,8 +781,7 @@ curl -s -X POST "$BASE/records/search" \ `$cycle` rules: -- The block requires `$relation` with `hops` (`min` ≥ 2, and it defaults to 2 — a 1-hop "cycle" would be a self-loop) and accepts **nothing else**: no `$alias`, no property criteria, no nested labels. Both ends of the path are the root record, so filter the root instead. -- The block's key (`DEPENDENCY_CYCLE` here) is a display name, not a label — it just needs to be unique among its siblings. +- `$cycle` is a record-level predicate: its value is the traversal spec itself — `type`, `direction`, `hops` — with `hops` mandatory (`min` ≥ 2, and it defaults to 2 — a 1-hop "cycle" would be a self-loop). It accepts **nothing else**: no `$alias`, no property criteria, no nested labels. Both ends of the path are the root record, so filter the root instead; intermediate node labels are unconstrained. - `direction: 'out'` matters: it makes the query mean "A depends on B depends on … depends on A". An undirected cycle would also match harmless mutual pairs. - Trail semantics apply: each relationship is used once per path, but records may repeat — figure-eight shapes count as cycles. - The query returns cycle **participants**, not the rings themselves. To reconstruct which packages form a specific cycle, walk outward from a participant with bounded one-hop queries. @@ -808,7 +794,7 @@ curl -s -X POST "$BASE/records/search" \ | ------------ | ------------------------------------------ | ---------------------------------------------------------------- | ----------------------------------------------------------------------- | | Tree | Single parent per node | Nested blocks for mixed-label paths; `hops` for same-label depth | Per-level filters/aliases need manual nesting — `hops` skips the middle | | Many-to-many | Nodes can appear in multiple relationships | Aggregation by relationship type | Fan-out can be large without limit | -| Cyclic | Loops are possible | `hops` for blast radius, `$cycle` for loop detection | `hops.max` is capped per deployment (default 25); keep it small anyway | +| Cyclic | Loops are possible | `hops` for blast radius, `$cycle` for loop detection | `hops.max` is capped per deployment (default 10); keep it small anyway | --- @@ -816,7 +802,7 @@ curl -s -X POST "$BASE/records/search" \ Each shape has a fan-out risk. In trees, deep hierarchies multiply candidates at every hop. In networks, highly-connected hubs (an author with 200 papers) inflate traversal cost. In cyclic graphs, even a two-hop traversal can cover thousands of paths in large dependency graphs. -Variable-length traversal explores every matching path, so treat `hops.max` as a budget: keep it as small as your use case allows, and always set `type` and `direction` — an undirected, untyped `hops` query is the most expensive shape. `hops.max` is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 25); unbounded traversal is only available on self-hosted deployments and custom Neo4j instances, bounded there by the transaction timeout. +Variable-length traversal explores every matching path, so treat `hops.max` as a budget: keep it as small as your use case allows, and always set `type` and `direction` — an undirected, untyped `hops` query is the most expensive shape. `hops.max` is capped per deployment (`RUSHDB_MAX_TRAVERSAL_HOPS`, default 10); unbounded traversal is only available on self-hosted deployments and custom Neo4j instances, bounded there by the transaction timeout. Apply `limit` conservatively and filter early on high-selectivity properties (e.g. `name`, `status`, `version`). Measure response times before and after adding hops to your query. diff --git a/docs/docs/learn/tutorials/search-and-queries/searchquery-advanced-patterns.mdx b/docs/docs/learn/tutorials/search-and-queries/searchquery-advanced-patterns.mdx index ea4ebe64..2a36c2c6 100644 --- a/docs/docs/learn/tutorials/search-and-queries/searchquery-advanced-patterns.mdx +++ b/docs/docs/learn/tutorials/search-and-queries/searchquery-advanced-patterns.mdx @@ -558,7 +558,7 @@ chain = db.records.find({ ### Cycle detection -Use `$cycle: true` to find records on a closed path back to themselves. The block requires an object `$relation` with `hops` (`min` ≥ 2, defaults to 2) and accepts nothing else — no `$alias`, property criteria, or nested labels. Its key is a display name, not matched as a label. +Use `$cycle` to find records on a closed path back to themselves. It is a record-level predicate whose value is the traversal spec itself — `type`, `direction`, `hops` — with `hops` mandatory (`min` ≥ 2, defaults to 2). It accepts nothing else — no `$alias`, property criteria, or nested labels. @@ -567,10 +567,7 @@ Use `$cycle: true` to find records on a closed path back to themselves. The bloc const rings = await db.records.find({ labels: ['ACCOUNT'], where: { - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } }, limit: 50 }) @@ -582,10 +579,7 @@ const rings = await db.records.find({ rings = db.records.find({ "labels": ["ACCOUNT"], "where": { - "RING": { - "$cycle": True, - "$relation": {"type": "TRANSFERRED_TO", "direction": "out", "hops": {"min": 2, "max": 6}} - } + "$cycle": {"type": "TRANSFERRED_TO", "direction": "out", "hops": {"min": 2, "max": 6}} }, "limit": 50 }) diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index d69ac000..b34648f3 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -169,6 +169,7 @@ const config: Config = { // Set the // pathname under which your site is served // For GitHub pages deployment, it is often '//' baseUrl: '/', + trailingSlash: true, // Serve raw .mdx/.md source files at their natural paths (e.g. /deploy/infra-neo4j.mdx) // so CopyPageButton can fetch content locally without depending on GitHub raw. diff --git a/e2e/sdk/ai.byov-project-isolation.e2e.test.ts b/e2e/sdk/ai.byov-project-isolation.e2e.test.ts new file mode 100644 index 00000000..b9c82d78 --- /dev/null +++ b/e2e/sdk/ai.byov-project-isolation.e2e.test.ts @@ -0,0 +1,291 @@ +/** + * BYOV project-isolation e2e coverage. + * + * This suite uses two real projects with two API tokens and the same label/property + * pair in both projects. It verifies that external vector indexes and semantic + * search candidates are scoped by project, so a `description` index in project A + * never leaks into project B when project B searches its own `description` index. + */ + +import RushDB from '../../packages/javascript-sdk/src/index.node' +import type { EmbeddingIndex } from '../../packages/javascript-sdk/src/api/types' +import { ADMIN_LOGIN, ADMIN_PASSWORD, BASE_URL } from '../setup/env' + +jest.setTimeout(120_000) + +const API = `${BASE_URL}/api/v1` +const RUN = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}` + +type RawResponse = { status: number; body: any } + +const apiRaw = async ( + path: string, + { + method = 'GET', + token, + headers = {}, + body + }: { method?: string; token?: string; headers?: Record; body?: unknown } = {} +): Promise => { + const response = await fetch(`${API}${path}`, { + method, + headers: { + 'Content-Type': 'application/json', + ...(token ? { Authorization: `Bearer ${token}` } : {}), + ...headers + }, + ...(body !== undefined ? { body: JSON.stringify(body) } : {}) + }) + const json: any = await response.json().catch(() => ({})) + return { status: response.status, body: json } +} + +const api = async (path: string, options: Parameters[1] = {}) => { + const { status, body } = await apiRaw(path, options) + if (status < 200 || status >= 300) { + throw new Error(`${options.method ?? 'GET'} ${path} -> ${status}: ${JSON.stringify(body)}`) + } + return body.data ?? body +} + +const unitVec = (i: number, dims = 4): number[] => Array.from({ length: dims }, (_, k) => (k === i ? 1 : 0)) + +async function waitForIndexReady( + db: RushDB, + indexId: string, + timeoutMs = 60_000, + interval = 2_000 +): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + const list = await db.ai.indexes.find() + const idx = (list.data as EmbeddingIndex[]).find((i) => i.id === indexId) + if (idx?.status === 'ready') return + if (idx?.status === 'error') throw new Error(`Embedding index ${indexId} entered error state`) + await new Promise((r) => setTimeout(r, interval)) + } + throw new Error(`Embedding index ${indexId} did not become ready within ${timeoutMs}ms`) +} + +describe('BYOV semantic indexes are project isolated (e2e)', () => { + const apiUrl = process.env.RUSHDB_API_URL || BASE_URL + + const LABEL = `ByovSharedDoc_${RUN.replace(/-/g, '_')}` + const PROPERTY = 'description' + const DIMS = 4 + + let tokenA: string + let tokenB: string + let dbA: RushDB + let dbB: RushDB + let indexAId: string | undefined + let indexBId: string | undefined + + const createProjectWithToken = async (jwt: string, workspaceId: string, name: string): Promise => { + const project = await api('/projects', { + method: 'POST', + token: jwt, + headers: { 'x-workspace-id': workspaceId }, + body: { name, description: 'BYOV project isolation e2e' } + }) + const token = await api('/tokens', { + method: 'POST', + token: jwt, + headers: { 'x-project-id': project.id, 'x-workspace-id': workspaceId }, + body: { name: 'e2e-byov', expiration: '1d' } + }) + return token.value + } + + beforeAll(async () => { + const user = await api('/auth/login', { + method: 'POST', + body: { login: ADMIN_LOGIN, password: ADMIN_PASSWORD } + }) + const workspaces = await api('/workspaces', { token: user.token }) + const workspaceId: string = workspaces[0]?.id + expect(workspaceId).toBeTruthy() + + tokenA = await createProjectWithToken(user.token, workspaceId, `e2e-byov-a-${RUN}`) + tokenB = await createProjectWithToken(user.token, workspaceId, `e2e-byov-b-${RUN}`) + dbA = new RushDB(tokenA, { url: apiUrl }) + dbB = new RushDB(tokenB, { url: apiUrl }) + }) + + afterAll(async () => { + if (dbA) { + await dbA.records.delete({ labels: [LABEL] }).catch(() => {}) + if (indexAId) await dbA.ai.indexes.delete(indexAId).catch(() => {}) + } + if (dbB) { + await dbB.records.delete({ labels: [LABEL] }).catch(() => {}) + if (indexBId) await dbB.ai.indexes.delete(indexBId).catch(() => {}) + } + }) + + it('supports end-to-end BYOV writes and vector search in two projects with the same label/property', async () => { + // Seed the same label/property in both projects so each project can create + // an external index with the exact same semantic key. + await Promise.all([ + dbA.records.create({ label: LABEL, data: { description: 'project-a schema seed', owner: 'userA-seed' } }), + dbB.records.create({ label: LABEL, data: { description: 'project-b schema seed', owner: 'userB-seed' } }) + ]) + + const [{ data: indexA }, { data: indexB }] = await Promise.all([ + dbA.ai.indexes.create({ + label: LABEL, + propertyName: PROPERTY, + external: true, + dimensions: DIMS, + similarityFunction: 'cosine' + }), + dbB.ai.indexes.create({ + label: LABEL, + propertyName: PROPERTY, + external: true, + dimensions: DIMS, + similarityFunction: 'cosine' + }) + ]) + indexAId = indexA.id + indexBId = indexB.id + + const projectARecord = await dbA.records.create({ + label: LABEL, + data: { description: 'shared property text that belongs only to project A', owner: 'userA' }, + vectors: [{ propertyName: PROPERTY, vector: unitVec(0), dimensions: DIMS, similarityFunction: 'cosine' }] + }) + + await dbB.records.createMany({ + label: LABEL, + data: [ + { description: 'shared property text that belongs only to project B', owner: 'userB-primary' }, + { description: 'secondary project B document', owner: 'userB-secondary' } + ], + vectors: [ + [{ propertyName: PROPERTY, vector: unitVec(1), dimensions: DIMS, similarityFunction: 'cosine' }], + [{ propertyName: PROPERTY, vector: unitVec(2), dimensions: DIMS, similarityFunction: 'cosine' }] + ] + }) + + // Back-fill the schema seed records so both external indexes become ready. + const [seedA, seedB] = await Promise.all([ + dbA.records.find({ labels: [LABEL], where: { description: 'project-a schema seed' } }), + dbB.records.find({ labels: [LABEL], where: { description: 'project-b schema seed' } }) + ]) + await Promise.all([ + dbA.ai.indexes.upsertVectors(indexAId, { + items: [{ recordId: seedA.data[0].id, vector: unitVec(3) }] + }), + dbB.ai.indexes.upsertVectors(indexBId, { + items: [{ recordId: seedB.data[0].id, vector: unitVec(3) }] + }) + ]) + + await Promise.all([waitForIndexReady(dbA, indexAId), waitForIndexReady(dbB, indexBId)]) + + const projectAResults = await dbA.ai.search({ + propertyName: PROPERTY, + labels: [LABEL], + sourceType: 'external', + similarityFunction: 'cosine', + dimensions: DIMS, + queryVector: unitVec(0), + limit: 5 + }) + + expect(projectAResults.data[0]?.data.__id).toBe(projectARecord.id) + expect(projectAResults.data.map((record) => record.data.owner)).not.toContain('userB-primary') + + const projectBResultsForAQuery = await dbB.ai.search({ + propertyName: PROPERTY, + labels: [LABEL], + sourceType: 'external', + similarityFunction: 'cosine', + dimensions: DIMS, + queryVector: unitVec(0), + limit: 5 + }) + + // Project B is allowed to search its own same-named `description` index, but + // project A's perfect vector match must never appear in project B results. + expect(projectBResultsForAQuery.data.map((record) => record.data.__id)).not.toContain(projectARecord.id) + expect(projectBResultsForAQuery.data.every((record) => String(record.data.owner ?? '').startsWith('userB'))).toBe( + true + ) + + const projectBResults = await dbB.ai.search({ + propertyName: PROPERTY, + labels: [LABEL], + sourceType: 'external', + similarityFunction: 'cosine', + dimensions: DIMS, + queryVector: unitVec(1), + limit: 5 + }) + + expect(projectBResults.data[0]?.data.owner).toBe('userB-primary') + expect(projectBResults.data[0]?.data.__score).toBeCloseTo(1, 2) + }) + + it('does not let project B use project A as the matching index when project B has no index', async () => { + const noIndexLabel = `${LABEL}_NoIndex` + + await dbA.records.create({ label: noIndexLabel, data: { description: 'project A only seed' } }) + const { data: projectAOnlyIndex } = await dbA.ai.indexes.create({ + label: noIndexLabel, + propertyName: PROPERTY, + external: true, + dimensions: DIMS, + similarityFunction: 'cosine' + }) + + try { + const aRecord = await dbA.records.create({ + label: noIndexLabel, + data: { description: 'project A only indexed document' }, + vectors: [{ propertyName: PROPERTY, vector: unitVec(0), dimensions: DIMS, similarityFunction: 'cosine' }] + }) + const seedA = await dbA.records.find({ + labels: [noIndexLabel], + where: { description: 'project A only seed' } + }) + await dbA.ai.indexes.upsertVectors(projectAOnlyIndex.id, { + items: [{ recordId: seedA.data[0].id, vector: unitVec(1) }] + }) + await waitForIndexReady(dbA, projectAOnlyIndex.id) + + await dbB.records.create({ + label: noIndexLabel, + data: { description: 'project B same label and property but no index' } + }) + + await expect( + dbB.ai.search({ + propertyName: PROPERTY, + labels: [noIndexLabel], + sourceType: 'external', + similarityFunction: 'cosine', + dimensions: DIMS, + queryVector: unitVec(0), + limit: 5 + }) + ).rejects.toBeTruthy() + + const ownerResults = await dbA.ai.search({ + propertyName: PROPERTY, + labels: [noIndexLabel], + sourceType: 'external', + similarityFunction: 'cosine', + dimensions: DIMS, + queryVector: unitVec(0), + limit: 5 + }) + expect(ownerResults.data[0]?.data.__id).toBe(aRecord.id) + } finally { + await dbA.records.delete({ labels: [noIndexLabel] }).catch(() => {}) + await dbB.records.delete({ labels: [noIndexLabel] }).catch(() => {}) + await dbA.ai.indexes.delete(projectAOnlyIndex.id).catch(() => {}) + } + }) +}) diff --git a/e2e/sdk/ai.search-query.generation.e2e.test.ts b/e2e/sdk/ai.search-query.generation.e2e.test.ts new file mode 100644 index 00000000..a40f4f09 --- /dev/null +++ b/e2e/sdk/ai.search-query.generation.e2e.test.ts @@ -0,0 +1,356 @@ +/** + * E2E tests for the AI SearchQuery generation endpoint (`POST /api/v1/ai/search-query`) + * and for the engine semantics its validator exists to guard. + * + * WHY THIS SUITE EXISTS + * ───────────────────── + * SmartSearch regressions are prompt regressions: the LLM prompts + * (platform/core/src/core/ai/search-query-{builder,spec}.prompt.md) encode query-shape + * rules through worked examples, and models generalize from those examples' surface + * tokens more than from their prose. A prompt that was only ever exercised against the + * dataset its examples were written from cannot show that gap. This suite therefore + * seeds a deliberately HOSTILE dataset whose naming avoids every token used in the + * prompts and in typical demo data: + * + * - display properties are `title` / `fullName` — NOT `name` + * - the scalar reference is `winningDriverRef` — camelCase, no `_id` suffix + * - labels are MixedCase (`RaceEvent`) — not UPPER_CASE like the prompt examples + * + * Dataset (all records carry `seedMark` for cross-run cleanup): + * + * RaceDriver (4): Ayla, Bruno, Kenji, Marta — display `fullName`, join key `driverCode` + * RaceEvent (6): display `title`; scalar `winningDriverRef` records who WON + * (wins: Ayla 3, Bruno 2, Kenji 1 — no "won" relationship exists!) + * PARTICIPATED_IN edges record who RACED + * (participation: Bruno 5, Ayla 4, Kenji 2, Marta 1) + * -> "who won the most" and "who participated in the most" have DIFFERENT answers, + * so a semantically wrong query shape produces a visibly wrong result. + * SupplyDepot (5): SHIPS_TO ring North -> South -> East -> North, plus a linear + * chain West -> Hub (cycle tests must return only the ring). + * + * TWO TEST LAYERS + * ─────────────── + * 1. Engine guardrails (always run): pin the /records/search semantics the generator's + * validator protects against — correlated `where` references and `$ref`-in-where + * match NOTHING silently (they are matched as literal strings), which is exactly + * why the generator must reject them instead of returning a "valid" empty query. + * 2. LLM generation (gated): requires the server to run with RUSHDB_LLM_API_KEY and + * RUSHDB_LLM_MODEL (export them before `pnpm test:e2e`; global-setup passes host + * env through to platform/core). Skipped gracefully when the endpoint returns 503. + * These are eval-style tests: they assert structural invariants of the generated + * queries and EXECUTE them against the seeded data. A failure here means a prompt + * or validator change regressed a scenario, not that the code is broken. + * + * Prerequisites: RUSHDB_API_KEY / RUSHDB_API_URL (harness-provisioned or from + * packages/javascript-sdk/.env). Whole suite skips without an API key. + */ + +import RushDB from '../../packages/javascript-sdk/src/index.node' + +jest.setTimeout(180_000) + +const apiKey = process.env.RUSHDB_API_KEY +const apiUrl = process.env.RUSHDB_API_URL || 'http://localhost:3000' +const apiBase = `${apiUrl.replace(/\/$/, '')}/api/v1` + +if (!apiKey) { + describe('ai.search-query generation (e2e)', () => { + it('skips because RUSHDB_API_KEY is not set', () => expect(true).toBe(true)) + }) +} else { + const db = new RushDB(apiKey, { url: apiUrl }) + + const seedMark = 'ai-sqgen-suite' + const L_DRIVER = 'RaceDriver' + const L_EVENT = 'RaceEvent' + const L_DEPOT = 'SupplyDepot' + + const WINS = { driver_ayla: 3, driver_bruno: 2, driver_kenji: 1 } as Record + const PARTICIPATION = { driver_bruno: 5, driver_ayla: 4, driver_kenji: 2, driver_marta: 1 } + const CYCLE_MEMBER_TITLES = ['Depot East', 'Depot North', 'Depot South'] + + // ── Raw HTTP helpers (endpoints the SDK does not expose) ─────────────────── + const post = async (path: string, body: unknown): Promise<{ status: number; json: any }> => { + const res = await fetch(`${apiBase}${path}`, { + method: 'POST', + headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }) + return { status: res.status, json: await res.json().catch(() => ({})) } + } + + const searchRecords = (searchQuery: unknown) => post('/records/search', searchQuery) + + /** + * Calls the generation endpoint. Retries only transient LLM transport failures + * ("AI returned invalid JSON." / network hiccups) — a "Generated query is not valid" + * 400 is a real prompt/validator regression and must surface as a test failure. + * Success responses are wrapped by the platform as { data: { searchQuery, warnings } }. + */ + const generate = async ( + prompt: string, + retries = 2 + ): Promise<{ status: number; json: any; searchQuery: any }> => { + for (let attempt = 0; ; attempt++) { + const result = await post('/ai/search-query', { prompt }) + const transient = + result.status >= 500 || + (result.status === 400 && String(result.json?.message ?? '').includes('AI returned invalid JSON')) + if (!transient || attempt >= retries) { + const payload = result.json?.data ?? result.json + return { ...result, searchQuery: payload?.searchQuery } + } + } + } + + /** No correlated references anywhere inside where — the invariant this suite guards. */ + const expectNoCorrelatedRefs = (where: unknown) => { + const text = JSON.stringify(where ?? {}) + expect(text).not.toContain('"$ref"') + expect(text).not.toContain('$record.') + } + + /** Finds the traversal/cycle blocks of a where object as [key, value] entries. */ + const whereEntries = (where: unknown): Array<[string, any]> => + where && typeof where === 'object' ? Object.entries(where as Record) : [] + + // ── Seed ──────────────────────────────────────────────────────────────────── + beforeAll(async () => { + // Cross-run cleanup: seedMark is constant, so leftovers from any previous run go too. + await db.records.delete({ where: { seedMark } }).catch(() => {}) + + await db.records.createMany({ + label: L_DRIVER, + data: [ + { fullName: 'Ayla Ferreira', driverCode: 'driver_ayla', careerPoints: 480, seedMark }, + { fullName: 'Bruno Costa', driverCode: 'driver_bruno', careerPoints: 310, seedMark }, + { fullName: 'Kenji Watanabe', driverCode: 'driver_kenji', careerPoints: 150, seedMark }, + { fullName: 'Marta Vidal', driverCode: 'driver_marta', careerPoints: 12, seedMark } + ], + options: { returnResult: false } + }) + + // participantA..C are join columns for edge creation only; winningDriverRef is the + // scalar condition property that intentionally has NO relationship counterpart. + await db.records.createMany({ + label: L_EVENT, + data: [ + // prettier-ignore + { title: 'Monaco Grand Prix', seasonYear: 2023, winningDriverRef: 'driver_ayla', participantA: 'driver_ayla', participantB: 'driver_bruno', seedMark }, + // prettier-ignore + { title: 'Silverstone Sprint', seasonYear: 2023, winningDriverRef: 'driver_ayla', participantA: 'driver_ayla', participantB: 'driver_bruno', participantC: 'driver_kenji', seedMark }, + // prettier-ignore + { title: 'Suzuka Endurance', seasonYear: 2024, winningDriverRef: 'driver_bruno', participantA: 'driver_ayla', participantB: 'driver_bruno', seedMark }, + // prettier-ignore + { title: 'Monza Classic', seasonYear: 2024, winningDriverRef: 'driver_bruno', participantA: 'driver_bruno', participantB: 'driver_marta', seedMark }, + // prettier-ignore + { title: 'Spa Rain Cup', seasonYear: 2025, winningDriverRef: 'driver_ayla', participantA: 'driver_ayla', participantB: 'driver_bruno', seedMark }, + // prettier-ignore + { title: 'Desert Night 500', seasonYear: 2025, winningDriverRef: 'driver_kenji', participantA: 'driver_kenji', seedMark } + ], + options: { returnResult: false } + }) + + for (const key of ['participantA', 'participantB', 'participantC']) { + await db.relationships.createMany({ + source: { label: L_EVENT, key, where: { seedMark } }, + target: { label: L_DRIVER, key: 'driverCode', where: { seedMark } }, + type: 'PARTICIPATED_IN', + direction: 'in' + }) + } + + // Ring North -> South -> East -> North; linear West -> Hub. + await db.records.createMany({ + label: L_DEPOT, + data: [ + { title: 'Depot North', regionCode: 'N', shipsToTitle: 'Depot South', seedMark }, + { title: 'Depot South', regionCode: 'S', shipsToTitle: 'Depot East', seedMark }, + { title: 'Depot East', regionCode: 'E', shipsToTitle: 'Depot North', seedMark }, + { title: 'Depot West', regionCode: 'W', shipsToTitle: 'Depot Hub', seedMark }, + { title: 'Depot Hub', regionCode: 'H', seedMark } + ], + options: { returnResult: false } + }) + await db.relationships.createMany({ + source: { label: L_DEPOT, key: 'shipsToTitle', where: { seedMark } }, + target: { label: L_DEPOT, key: 'title', where: { seedMark } }, + type: 'SHIPS_TO', + direction: 'out' + }) + }) + + afterAll(async () => { + await db.records.delete({ where: { seedMark } }).catch(() => {}) + }) + + // ── 1. Engine guardrails — the semantics the generator validator protects ── + describe('engine guardrails on hostile-named data (no LLM required)', () => { + it('grouped ranking over a scalar reference property executes and orders correctly', async () => { + const { status, json } = await searchRecords({ + labels: [L_EVENT], + where: { seedMark }, + select: { driver: '$record.winningDriverRef', wins: { $count: '*' } }, + groupBy: ['$record.winningDriverRef'], + orderBy: { wins: 'desc' } + }) + + expect(status).toBe(200) + const rows = json.data as Array<{ winningDriverRef: string; wins: number }> + expect(rows[0]).toMatchObject({ winningDriverRef: 'driver_ayla', wins: WINS.driver_ayla }) + const byDriver = Object.fromEntries(rows.map((row) => [row.winningDriverRef, row.wins])) + expect(byDriver).toMatchObject(WINS) + }) + + it('a correlated where value ("$record.id") is matched as a literal and returns zero rows', async () => { + const { status, json } = await searchRecords({ + labels: [L_DRIVER], + where: { seedMark, [L_EVENT]: { winningDriverRef: '$record.id' } }, + limit: 10 + }) + + expect(status).toBe(200) + expect(json.total).toBe(0) + }) + + it('"$ref" inside where matches nothing silently — why the generator must reject it upstream', async () => { + const { status, json } = await searchRecords({ + labels: [L_EVENT], + where: { seedMark, winningDriverRef: { $ref: '$record.id' } }, + limit: 10 + }) + + expect(status).toBe(200) + expect(json.total).toBe(0) + }) + + it('a $cycle query returns exactly the ring members, not the linear chain', async () => { + const { status, json } = await searchRecords({ + labels: [L_DEPOT], + where: { + seedMark, + $cycle: { type: 'SHIPS_TO', direction: 'out', hops: { min: 2, max: 6 } } + } + }) + + expect(status).toBe(200) + const titles = (json.data as any[]).map((record) => record.title).sort() + expect(titles).toEqual(CYCLE_MEMBER_TITLES) + }) + + it('a bounded hops traversal reaches multi-hop neighbours', async () => { + // Two directed hops from Depot North: South (1) and East (2). West/Hub unreachable. + const { status, json } = await searchRecords({ + labels: [L_DEPOT], + where: { + seedMark, + [L_DEPOT]: { + $relation: { type: 'SHIPS_TO', direction: 'in', hops: { min: 1, max: 2 } }, + title: 'Depot North' + } + } + }) + + expect(status).toBe(200) + const titles = (json.data as any[]).map((record) => record.title).sort() + expect(titles).toEqual(['Depot East', 'Depot South']) + }) + }) + + // ── 2. LLM generation — gated on the server having an LLM configured ─────── + describe('LLM generation against hostile naming (requires RUSHDB_LLM_* on the server)', () => { + let aiEnabled = false + + beforeAll(async () => { + // Make the freshly seeded labels visible to the generator (its schema is cached). + const schema = await post('/ai/schema/md', { force: true }) + expect(schema.status).toBe(200) + expect(String(schema.json?.data ?? '')).toContain(L_EVENT) + + const probe = await generate('List all records') + aiEnabled = probe.status !== 503 + if (!aiEnabled) { + console.warn('[ai.search-query e2e] RUSHDB_LLM_* not configured on server — skipping LLM tests') + } + }) + + const llmIt = (title: string, fn: () => Promise) => + it(title, async () => { + if (!aiEnabled) return + await fn() + }) + + llmIt('condition recorded only as a scalar property: roots on the owning label and groups by it', async () => { + const { status, searchQuery: query } = await generate('Which driver won the most races?') + + expect(status).toBe(200) + expectNoCorrelatedRefs(query.where) + // "won" exists only as RaceEvent.winningDriverRef — the exception shape is the + // ONLY valid query. Rooting on RaceDriver would count participation, not wins. + expect(query.labels).toEqual([L_EVENT]) + expect(JSON.stringify(query.groupBy)).toContain('winningDriverRef') + + const executed = await searchRecords(query) + expect(executed.status).toBe(200) + const top = (executed.json.data as any[])[0] + expect(top.winningDriverRef).toBe('driver_ayla') + expect(Object.values(top)).toContain(WINS.driver_ayla) + }) + + llmIt('condition backed by a real relationship: keeps the asked-for entity as root and traverses', async () => { + const { status, searchQuery: query } = await generate('Which driver participated in the most races?') + + expect(status).toBe(200) + expectNoCorrelatedRefs(query.where) + expect(query.labels).toEqual([L_DRIVER]) + // Must traverse RaceEvent (participation IS a relationship) and rank by count. + const traversal = whereEntries(query.where).find(([key]) => key === L_EVENT) + expect(traversal).toBeDefined() + expect(JSON.stringify(query.select)).toContain('$count') + + const executed = await searchRecords(query) + expect(executed.status).toBe(200) + const top = (executed.json.data as any[])[0] + // Bruno leads participation (5) while Ayla leads wins — a wins-shaped query here + // would surface Ayla and fail this assertion. + expect(JSON.stringify(top)).toContain('Bruno') + expect(Object.values(top)).toContain(PARTICIPATION.driver_bruno) + }) + + llmIt('cycle intent produces a valid $cycle operator that finds exactly the ring', async () => { + const { status, searchQuery: query } = await generate('Which supply depots ship to each other in a circular loop?') + + expect(status).toBe(200) + expectNoCorrelatedRefs(query.where) + expect(query.labels).toEqual([L_DEPOT]) + // The spec prompt teaches the operator form: $cycle's value IS the traversal spec. + const cycleOperator = whereEntries(query.where).find(([key]) => key === '$cycle') + expect(cycleOperator).toBeDefined() + expect(cycleOperator![1]?.hops).toBeDefined() + + const executed = await searchRecords(query) + expect(executed.status).toBe(200) + const titles = (executed.json.data as any[]) + .filter((record) => record.seedMark === seedMark) + .map((record) => record.title) + .sort() + expect(titles).toEqual(CYCLE_MEMBER_TITLES) + }) + + llmIt('display-property filters land on the schema-listed property, not an assumed "name"', async () => { + const { status, searchQuery: query } = await generate('Find the Monaco race') + + expect(status).toBe(200) + expectNoCorrelatedRefs(query.where) + expect(query.labels).toEqual([L_EVENT]) + const whereText = JSON.stringify(query.where) + expect(whereText).toContain('"title"') + expect(whereText).not.toContain('"name"') + + const executed = await searchRecords(query) + expect(executed.status).toBe(200) + const titles = (executed.json.data as any[]).map((record) => record.title) + expect(titles).toContain('Monaco Grand Prix') + }) + }) +} diff --git a/e2e/sdk/records.find.multihop-cycles.e2e.test.ts b/e2e/sdk/records.find.multihop-cycles.e2e.test.ts index f774a3ca..687cc152 100644 --- a/e2e/sdk/records.find.multihop-cycles.e2e.test.ts +++ b/e2e/sdk/records.find.multihop-cycles.e2e.test.ts @@ -227,10 +227,7 @@ if (!apiKey) { labels: [L_ACCOUNT], where: { tenantId, - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } }) expect(names(res)).toEqual(['A', 'B', 'C']) @@ -243,15 +240,13 @@ if (!apiKey) { where: { tenantId, $not: { - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } } }) expect(names(res)).toEqual(['X', 'Y', 'Z']) }) + }) // ══════════════════════════════════════════════════════════════════════ @@ -279,27 +274,35 @@ if (!apiKey) { ).rejects.toThrow() }) - it('rejects $cycle with $alias', async () => { + it('rejects the removed $cycle block form', async () => { await expect( db.records.find({ labels: [L_ACCOUNT], where: { tenantId, - RING: { + SOME_KEY: { $cycle: true, - $alias: '$ring', - $relation: { type: 'TRANSFERRED_TO', hops: { min: 2, max: 4 } } + $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 4 } } } - } + } as never + }) + ).rejects.toThrow() + }) + + it('rejects the $cycle operator without hops', async () => { + await expect( + db.records.find({ + labels: [L_ACCOUNT], + where: { tenantId, $cycle: { type: 'TRANSFERRED_TO' } as never } }) ).rejects.toThrow() }) - it('rejects $cycle without hops', async () => { + it('rejects $cycle operator hops below the cycle floor of 2', async () => { await expect( db.records.find({ labels: [L_ACCOUNT], - where: { tenantId, RING: { $cycle: true, $relation: { type: 'TRANSFERRED_TO' } } } + where: { tenantId, $cycle: { type: 'TRANSFERRED_TO', hops: { min: 1, max: 4 } } } }) ).rejects.toThrow() }) diff --git a/packages/javascript-sdk/CHANGELOG.md b/packages/javascript-sdk/CHANGELOG.md index 5bd5fc6f..2f5d9f6b 100644 --- a/packages/javascript-sdk/CHANGELOG.md +++ b/packages/javascript-sdk/CHANGELOG.md @@ -29,17 +29,14 @@ db.records.find({ labels: ['ACCOUNT'], where: { - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } }) ``` - A `$cycle` block requires `$relation` with `hops` (`min` ≥ 2, the default) and accepts no other keys; its label key is a display name. Combine with `$not` to select records not on a cycle. + `$cycle` is a record-level predicate: its value is the traversal spec itself (`type`, `direction`, `hops` — `hops` mandatory, `min` ≥ 2, defaulting to 2). A cycle has no separate endpoint, so it accepts no `$alias`, no property criteria, and no nested labels. Combine with `$not` to select records not on a cycle. - **Traversal depth policy** — `hops.max` is capped per deployment via `RUSHDB_MAX_TRAVERSAL_HOPS` (default 25) on the shared cloud connection. Self-hosted deployments (`RUSHDB_SELF_HOSTED=true`) and projects with a custom external Neo4j allow unbounded traversal (`max` omitted), guarded by the existing transaction timeout. + **Traversal depth policy** — `hops.max` is capped per deployment via `RUSHDB_MAX_TRAVERSAL_HOPS` (default 10) on the shared cloud connection. Self-hosted deployments (`RUSHDB_SELF_HOSTED=true`) and projects with a custom external Neo4j allow unbounded traversal (`max` omitted), guarded by the existing transaction timeout. Also in this release: diff --git a/packages/javascript-sdk/src/types/query.ts b/packages/javascript-sdk/src/types/query.ts index f64500cd..e412b42c 100644 --- a/packages/javascript-sdk/src/types/query.ts +++ b/packages/javascript-sdk/src/types/query.ts @@ -13,9 +13,9 @@ import type { Schema } from './schema.js' import type { AnyObject, MaybeArray, RequireAtLeastOne } from './utils.js' /** - * Variable-length traversal depth for `$relation.hops`. + * Variable-length traversal depth for `$relation.hops` and `$cycle.hops`. * A number matches exactly that many hops; `{ min?, max? }` matches a range - * (`min` defaults to 1, or 2 inside a `$cycle` block). Omitting `max` requests + * (`min` defaults to 1, or 2 for `$cycle`). Omitting `max` requests * unbounded traversal, which is only allowed on self-hosted deployments and * projects with a custom Neo4j instance. */ @@ -29,18 +29,23 @@ export type TraversalRelationOptions = RelationOptions & { hops?: TraversalHops } +/** + * Record-level cycle predicate: matches records sitting on a closed path + * (cycle/ring) back to themselves over the described edges. The value is the + * traversal spec itself — `hops` is mandatory (`min` ≥ 2, defaults to 2); + * intermediate node labels are unconstrained. A cycle has no separate endpoint + * to filter or alias. + */ +export type CycleExpression = RelationOptions & { + hops: TraversalHops +} + export type Related = Models> = keyof M extends never ? AnyObject : { [Key in keyof M]?: { $alias?: string $relation?: TraversalRelationOptions | string - /** - * Matches records sitting on a closed path (cycle/ring) back to the - * parent record. Requires `$relation` with `hops` (`min` ≥ 2); accepts - * no other keys — a cycle has no separate endpoint to filter or alias. - */ - $cycle?: boolean } & Where } @@ -100,8 +105,9 @@ export type Expression = }) export type Where = - | ((Expression & Related) & LogicalGrouping & Related>) - | LogicalGrouping & Related> + | ((Expression & Related & { $cycle?: CycleExpression }) & + LogicalGrouping & Related & { $cycle?: CycleExpression }>) + | LogicalGrouping & Related & { $cycle?: CycleExpression }> export type OrderDirection = 'asc' | 'desc' export type Order = OrderDirection | Partial> diff --git a/packages/mcp-server/CHANGELOG.md b/packages/mcp-server/CHANGELOG.md index 682832ee..d64c5a21 100644 --- a/packages/mcp-server/CHANGELOG.md +++ b/packages/mcp-server/CHANGELOG.md @@ -29,17 +29,14 @@ db.records.find({ labels: ['ACCOUNT'], where: { - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } }) ``` - A `$cycle` block requires `$relation` with `hops` (`min` ≥ 2, the default) and accepts no other keys; its label key is a display name. Combine with `$not` to select records not on a cycle. + `$cycle` is a record-level predicate: its value is the traversal spec itself (`type`, `direction`, `hops` — `hops` mandatory, `min` ≥ 2, defaulting to 2). A cycle has no separate endpoint, so it accepts no `$alias`, no property criteria, and no nested labels. Combine with `$not` to select records not on a cycle. - **Traversal depth policy** — `hops.max` is capped per deployment via `RUSHDB_MAX_TRAVERSAL_HOPS` (default 25) on the shared cloud connection. Self-hosted deployments (`RUSHDB_SELF_HOSTED=true`) and projects with a custom external Neo4j allow unbounded traversal (`max` omitted), guarded by the existing transaction timeout. + **Traversal depth policy** — `hops.max` is capped per deployment via `RUSHDB_MAX_TRAVERSAL_HOPS` (default 10) on the shared cloud connection. Self-hosted deployments (`RUSHDB_SELF_HOSTED=true`) and projects with a custom external Neo4j allow unbounded traversal (`max` omitted), guarded by the existing transaction timeout. Also in this release: diff --git a/packages/mcp-server/searchQuerySpec.ts b/packages/mcp-server/searchQuerySpec.ts index 95c7ae30..25ec5757 100644 --- a/packages/mcp-server/searchQuerySpec.ts +++ b/packages/mcp-server/searchQuerySpec.ts @@ -48,8 +48,9 @@ record to traverse. field or alias reference: fieldA: "$record.id" // WRONG — matched as the literal text "$record.id", returns nothing fieldA: { $eq: "$alias.id" } // WRONG — same problem inside an operator - RushDB has no correlated where predicate (no "join on fieldA = fieldB"). $record.*, $alias.*, - and $relation references are valid ONLY in select / groupBy / aggregate. + fieldA: { $ref: "$record.id" } // WRONG — $ref exists only in select expressions + RushDB has no correlated where predicate (no "join on fieldA = fieldB") in ANY syntax. + $record.*, $alias.*, and $relation references are valid ONLY in select / groupBy / aggregate. To rank or group by a scalar field that is NOT a relationship in the schema, root on the label that owns the field and groupBy that field — do not fabricate a related-label traversal or a correlated filter. @@ -200,14 +201,15 @@ $relation — constrain relationship type and/or direction: type is OPTIONAL — omitting it traverses any relationship, which is valid (including with hops). ⚠ TYPE COMES FROM THE SCHEMA, VERBATIM: copy $relation.type exactly from a schema - Relationships row — never invent a semantic type like the AUTHORED/REPORTS_TO examples here. - Data imported as nested JSON has only __RUSHDB__RELATION__DEFAULT__ edges; when the schema - shows that, either use that exact string or omit type entirely. + Relationships row — never invent a semantic type like the AUTHORED/REPORTS_TO examples here, + and never write __RUSHDB__RELATION__DEFAULT__ unless a schema row shows that exact type. + When it does (data imported as nested JSON often has only such edges), either use that + exact string or omit type entirely. READING THE SCHEMA: each Relationships row is a directed pattern rooted at that label. (SELF)-[:TYPE]->(OTHER) → from SELF, traverse OTHER with $relation { type:'TYPE', direction:'out' } (SELF)<-[:TYPE]-(OTHER) → from SELF, traverse OTHER with $relation { type:'TYPE', direction:'in' } - Only patterns listed in the schema are traversable. A scalar *_id property (e.g. winner_faction_id) + Only patterns listed in the schema are traversable. A scalar reference property (e.g. author_id) is a plain value, NOT an edge — do not nest a label to "join" on it; root on the label that owns it and groupBy that field instead. @@ -225,25 +227,27 @@ $relation.hops — variable-length traversal (multihop over ONE pattern): record; intermediates are unconstrained. • Keep max as small as the request allows; deep undirected traversal is expensive. Omit type only when the user genuinely means "any relationship"; keep direction when the schema gives one. - • hops.max is capped per deployment (default 25); omitting max (unbounded) is only accepted - on self-hosted / dedicated-database setups. + • Always set hops.max (an integer as small as the request allows). Omitting max means + unbounded traversal, which is rejected on shared cloud connections and only accepted on + self-hosted / dedicated-database setups; hops.max is capped per deployment (default 10). • Endpoint aggregations apply per PATH: $count/$collect deduplicate, but $sum/$avg over a multihop alias count one row per path — prefer counting to summing across multihop endpoints. • One hop stays the default: never add hops for a plain related-record condition. -$cycle — cycle/ring detection (endpoint binds back to the parent record): +$cycle — cycle/ring detection (a record-level predicate; the path binds back to the record): where: { - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } Matches roots sitting on a closed path (fraud rings, circular ownership, dependency cycles). - A $cycle block accepts ONLY $relation (with hops, min ≥ 2 — defaults to 2): no $alias, no - property criteria, no nested labels — a cycle has no separate endpoint. The block's key is a - display name, NOT matched as a label. Set direction for flow-like semantics (money, ownership); - undirected cycles also match innocent back-and-forth pairs. Wrap in $not for "NOT on a cycle". - Paths may revisit a record via different relationships (only relationships are unique per path). + The operator's value IS the traversal spec (type, direction, hops — hops mandatory, min ≥ 2, + defaults to 2). No $alias, no property criteria, no nested labels — a cycle has no separate + endpoint. Intermediate node labels are unconstrained. Place inside a label block + (ACCOUNT: { country: 'US', $cycle: {...} }) to anchor the cycle at the related record. + Set direction for flow-like semantics (money, ownership); undirected cycles also match + innocent back-and-forth pairs. Wrap in $not for "NOT on a cycle". Paths may revisit a record + via different relationships (only relationships are unique per path). + Never wrap $cycle in a named block and never nest $relation inside it — the value IS the + relation spec, and it sits directly at a where level like any other operator. $id — filter by record ID (supports $in, $nin, string operators): where: { $id: { $in: ['id1','id2'] } } @@ -551,9 +555,9 @@ Avoid over-nesting: CYCLE / RING DETECTION "Records on a loop back to themselves" (fraud rings, circular ownership, dependency cycles) -→ $cycle block (see §1). Root label = the entity type; the $cycle block holds only $relation: +→ $cycle operator (see §1). Root label = the entity type; the operator's value is the traversal spec: findRecords({ labels:['ACCOUNT'], - where:{ RING:{ $cycle:true, $relation:{ type:'TRANSFERRED_TO', direction:'out', hops:{ min:2, max:6 } } } } }) + where:{ $cycle:{ type:'TRANSFERRED_TO', direction:'out', hops:{ min:2, max:6 } } } }) Returns ring PARTICIPANTS. To reconstruct a specific ring's composition, follow up with bounded one-hop queries from each flagged record. @@ -577,7 +581,7 @@ bounded one-hop queries from each flagged record. → $relation: { type?, direction, hops:{ max:N } } on ONE traversal block (type verbatim from the schema, or omitted). • "ring", "loop", "circular …", "cycles back to itself" - → { $cycle:true, $relation:{ type?, direction, hops:{ min:2, max:N } } }. + → { $cycle: { type?, direction, hops:{ min:2, max:N } } }. • Field names are case-sensitive. String comparisons are case-insensitive by default. ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ @@ -599,7 +603,7 @@ Before submitting a findRecords call, verify: ($relation.hops and $cycle are VALID operators — do not "correct" them away.) □ $relation.type copied verbatim from a schema Relationships row, or omitted; never invented. □ hops only for repeated-pattern traversal (hierarchy/degrees); bounded max, as small as possible. -□ $cycle block contains ONLY $relation (hops min ≥ 2); no $alias/criteria/nested labels inside. +□ $cycle value is the traversal spec itself (hops mandatory, min ≥ 2); never a named block, never $relation nested inside. □ Vector threshold semantics: euclidean → $lte; others → $gte. □ Month+day without year → ask for year. □ Aggregation intent? → query MUST include select + groupBy. Raw records ≠ aggregation. @@ -707,7 +711,7 @@ Multihop hierarchy ("everyone in Alice's reporting chain, up to 4 levels"): Cycle detection ("accounts on a circular transfer ring"): findRecords({ labels:['ACCOUNT'], - where:{ RING:{ $cycle:true, $relation:{ type:'TRANSFERRED_TO', direction:'out', hops:{ min:2, max:6 } } } } }) + where:{ $cycle:{ type:'TRANSFERRED_TO', direction:'out', hops:{ min:2, max:6 } } } }) Filter by ID: where:{ $id:{ $in:['id1','id2'] } } diff --git a/packages/mcp-server/systemPrompt.ts b/packages/mcp-server/systemPrompt.ts index c666b076..2d04a153 100644 --- a/packages/mcp-server/systemPrompt.ts +++ b/packages/mcp-server/systemPrompt.ts @@ -68,15 +68,16 @@ STEP 3 — BUILD For "top N records by scalar field on the same label", use orderBy + limit, not select. For "which parent has most/more/least/less/fewer/fewest related children", root the parent label, traverse child in where with $alias, count the child alias, group by the parent's display property from the schema, and order desc for most/more or asc for least/less/fewer. Do not let a related/filter label become root for related-count rankings when a valid parent→related traversal path exists. + Exception — condition verbs ("won", "wrote", "approved", …): a traversal satisfies the verb only if the relationship type expresses it. When the condition exists only as a scalar reference property on the related label (author_id / winner_* / approvedBy style — discover with findProperties), root on the label that OWNS that property and groupBy it instead; where cannot correlate two records (no $record.*/$alias.* values, no $ref — $ref is select-only). Keep ambiguous named-reference filters loose with $contains on the display property discovery shows for that label. Resource-local where rule: - findRecords.where filters Records. - findRelationships.where filters relationship edges; use source/target for endpoint Records. - findLabels.where and findProperties.where filter Records before returning labels/properties. The traversal key in where IS a label copied exactly as spelled in the schema (labels are case-sensitive and may be any case — never change a label's case). Alias is $alias only. - Use $relation on a related-label block to constrain edge type/direction (for example: POST: { $relation: { type: 'AUTHORED', direction: 'in' } }). Copy type verbatim from a schema Relationships row or omit it — data imported as nested JSON has only __RUSHDB__RELATION__DEFAULT__ edges, and untyped traversal is valid. + Use $relation on a related-label block to constrain edge type/direction (for example: POST: { $relation: { type: 'AUTHORED', direction: 'in' } }). Copy type verbatim from a schema Relationships row or omit it (untyped traversal is valid) — never write __RUSHDB__RELATION__DEFAULT__ unless a schema row shows that exact type. For multihop over one pattern (hierarchies, "within N degrees") add hops inside $relation (for example: EMPLOYEE: { $relation: { type: 'REPORTS_TO', direction: 'out', hops: { max: 4 } } }). - For rings/loops/circular flows use a $cycle block holding only $relation with hops (min 2+) (for example: RING: { $cycle: true, $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } }). + For rings/loops/circular flows use the record-level $cycle operator whose value is the traversal spec itself, hops mandatory with min 2+ (for example: where: { $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } }). Never wrap $cycle in a named block and never nest $relation inside it. Operators $label / $direction / $as / $of / $through / $hops do not exist — never use them; multihop depth is always $relation.hops. ` as const diff --git a/packages/mcp-server/tests/spec-invariants.test.ts b/packages/mcp-server/tests/spec-invariants.test.ts index 907ad59e..b897479c 100644 --- a/packages/mcp-server/tests/spec-invariants.test.ts +++ b/packages/mcp-server/tests/spec-invariants.test.ts @@ -42,6 +42,10 @@ const BANNED: Array<{ pattern: RegExp; reason: string }> = [ { pattern: /even if (the related record doesn'?t exist|no related record exists)/i, reason: 'traversal blocks require existence (recordN IS NOT NULL)' + }, + { + pattern: /\$cycle["']?\s*:\s*true/, + reason: 'the removed $cycle block form must never be shown, even as a counter-example' } ] @@ -51,7 +55,11 @@ const REQUIRED: Array<{ pattern: RegExp; reason: string }> = [ { pattern: /__RUSHDB__RELATION__DEFAULT__/, reason: 'default-relation reality for imported data' }, { pattern: /exactly as spelled in the schema/, reason: 'case-agnostic, schema-verbatim traversal keys' }, { pattern: /hops/, reason: 'variable-length traversal documented' }, - { pattern: /\$cycle/, reason: 'cycle detection documented' } + { pattern: /\$cycle/, reason: 'cycle detection documented' }, + { + pattern: /\$cycle["']?\s*:\s*\{\s*["']?(type|direction|hops)/, + reason: 'cycle operator form documented (the value IS the traversal spec)' + } ] describe('SearchQuery spec invariants', () => { diff --git a/packages/skills/CHANGELOG.md b/packages/skills/CHANGELOG.md index 62d72522..9225b504 100644 --- a/packages/skills/CHANGELOG.md +++ b/packages/skills/CHANGELOG.md @@ -29,17 +29,14 @@ db.records.find({ labels: ['ACCOUNT'], where: { - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } }) ``` - A `$cycle` block requires `$relation` with `hops` (`min` ≥ 2, the default) and accepts no other keys; its label key is a display name. Combine with `$not` to select records not on a cycle. + `$cycle` is a record-level predicate: its value is the traversal spec itself (`type`, `direction`, `hops` — `hops` mandatory, `min` ≥ 2, defaulting to 2). A cycle has no separate endpoint, so it accepts no `$alias`, no property criteria, and no nested labels. Combine with `$not` to select records not on a cycle. - **Traversal depth policy** — `hops.max` is capped per deployment via `RUSHDB_MAX_TRAVERSAL_HOPS` (default 25) on the shared cloud connection. Self-hosted deployments (`RUSHDB_SELF_HOSTED=true`) and projects with a custom external Neo4j allow unbounded traversal (`max` omitted), guarded by the existing transaction timeout. + **Traversal depth policy** — `hops.max` is capped per deployment via `RUSHDB_MAX_TRAVERSAL_HOPS` (default 10) on the shared cloud connection. Self-hosted deployments (`RUSHDB_SELF_HOSTED=true`) and projects with a custom external Neo4j allow unbounded traversal (`max` omitted), guarded by the existing transaction timeout. Also in this release: diff --git a/packages/skills/rushdb-data-modeling/SKILL.md b/packages/skills/rushdb-data-modeling/SKILL.md index f3788f84..5200744f 100644 --- a/packages/skills/rushdb-data-modeling/SKILL.md +++ b/packages/skills/rushdb-data-modeling/SKILL.md @@ -155,7 +155,7 @@ Relationships are traversed in `where` by using the target label as the key: Omit `$relation` when direction and type are not important — RushDB traverses any edge. -For self-referencing hierarchies (org charts, category trees, dependency graphs), one nested block traverses multiple levels via `hops`: `"$relation": { "type": "REPORTS_TO", "direction": "out", "hops": { "max": 4 } }`. Cycles in such graphs are queryable with a `$cycle` block — see the query-builder skill for both. +For self-referencing hierarchies (org charts, category trees, dependency graphs), one nested block traverses multiple levels via `hops`: `"$relation": { "type": "REPORTS_TO", "direction": "out", "hops": { "max": 4 } }`. Cycles in such graphs are queryable with the `$cycle` operator — see the query-builder skill for both. --- diff --git a/packages/skills/rushdb-query-builder/SKILL.md b/packages/skills/rushdb-query-builder/SKILL.md index 63069765..111abc30 100644 --- a/packages/skills/rushdb-query-builder/SKILL.md +++ b/packages/skills/rushdb-query-builder/SKILL.md @@ -79,6 +79,7 @@ Top-N rule: - "Top N records by a scalar field on the same label" is a listing: use `orderBy` + `limit`, no `select`. - "Which parent has most/more/least/less/fewer/fewest related children" is an aggregation: root the parent label, traverse the child label with `$alias`, count the child alias, group by the parent's display property from the schema. - Related-count direction: most/more/highest/largest/greatest => `desc`; least/less/fewer/fewest/lowest/smallest => `asc`. +- Condition-verb exception: when the ranking condition ("won", "wrote", "approved", …) is expressed only by a scalar reference property on the related label (`author_id` / `winner_*` / `approvedBy` style — confirm via discovery) and no relationship type expresses it, root on the label that owns that property and `groupBy` it instead. `where` cannot correlate two records — no `$record.*`/`$alias.*` values and no `$ref` (`$ref` is select-only). - Do not let the related/filter label become the root just because it owns the filtered field. If the requested parent-to-related traversal path is absent, do not silently switch roots; state the path is unavailable or use a closest valid fallback with an explicit assumption. Named-reference rule: @@ -228,8 +229,8 @@ where: { POST: { $relation: { type: "AUTHORED", direction: "in" } } } // Multihop over one pattern (hierarchies, "within N degrees") — hops inside $relation: where: { EMPLOYEE: { $relation: { type: "REPORTS_TO", direction: "out", hops: { max: 4 } } } } -// Cycle/ring detection (fraud rings, circular ownership) — $cycle block holds only $relation: -where: { RING: { $cycle: true, $relation: { type: "TRANSFERRED_TO", direction: "out", hops: { min: 2, max: 6 } } } } +// Cycle/ring detection (fraud rings, circular ownership) — the $cycle operator's value IS the traversal spec: +where: { $cycle: { type: "TRANSFERRED_TO", direction: "out", hops: { min: 2, max: 6 } } } ``` Each Relationships row in the schema is a directed pattern rooted at that label: `(SELF)-[:TYPE]->(OTHER)` is outgoing, `(SELF)<-[:TYPE]-(OTHER)` is incoming. To pin the edge, set `$relation: { type: "TYPE", direction }` — `"out"` for `->`, `"in"` for `<-`. Only patterns shown in the schema are traversable; a scalar `*_id` property is a plain value, not an edge — never nest a label to "join" on it (root on the owning label and `groupBy` it instead). diff --git a/packages/skills/rushdb-query-builder/references/search-query-spec.md b/packages/skills/rushdb-query-builder/references/search-query-spec.md index 70983a17..ef6ceadc 100644 --- a/packages/skills/rushdb-query-builder/references/search-query-spec.md +++ b/packages/skills/rushdb-query-builder/references/search-query-spec.md @@ -53,6 +53,8 @@ All label, field, and relationship-type names in this document are placeholders The where clause mechanism: when a nested object key is NOT a criteria operator (like `$gt`, `$contains`, etc.) and NOT a flat value, RushDB interprets that key as the **LABEL** of a related record to traverse. +Predicate values must be **literals** (strings, numbers, booleans, dates). A where value can never be a field or alias reference: `{ field: "$record.id" }`, `{ field: { $eq: "$alias.id" } }`, and `{ field: { $ref: "$record.id" } }` are all invalid — `$ref` exists only in `select`, and a plain reference is matched as a literal string and returns nothing. There is no correlated where predicate (no "join on fieldA = fieldB") in any syntax. To rank or group by a scalar field that is not a relationship in the schema, root on the label that owns the field and `groupBy` that field instead of building a traversal. + ### Primitive Value Matching ```js @@ -282,7 +284,7 @@ where: { // type is OPTIONAL — omitting it traverses any relationship, which is valid (including with hops) ``` -⚠ **Type comes from the schema, verbatim**: copy `$relation.type` exactly from a schema Relationships row — never invent a semantic type like the `AUTHORED`/`REPORTS_TO` examples here. Data imported as nested JSON has only `__RUSHDB__RELATION__DEFAULT__` edges; when the schema shows that, either use that exact string or omit `type` entirely. +⚠ **Type comes from the schema, verbatim**: copy `$relation.type` exactly from a schema Relationships row — never invent a semantic type like the `AUTHORED`/`REPORTS_TO` examples here, and never write `__RUSHDB__RELATION__DEFAULT__` unless a schema row shows that exact type. When it does (data imported as nested JSON often has only such edges), either use that exact string or omit `type` entirely. Each Relationships row in the schema is a directed pattern rooted at that label: `(SELF)-[:TYPE]->(OTHER)` is outgoing, `(SELF)<-[:TYPE]-(OTHER)` is incoming. Map straight to `$relation: { type: "TYPE", direction }` — `"out"` for `->`, `"in"` for `<-`. Only patterns shown in the schema are traversable; a scalar `*_id` property is a plain value, not an edge. @@ -302,22 +304,19 @@ where: { Use for hierarchies ("any ancestor/descendant", "whole reporting chain"), "within N degrees", or transitive closure over one relationship type. The endpoint label constrains only the **final** record; intermediates are unconstrained. - Keep `max` as small as the request allows; deep undirected traversal is expensive. Omit `type` only when the user genuinely means "any relationship"; keep `direction` when the schema gives one. -- `hops.max` is capped per deployment (default 25); omitting `max` (unbounded) is only accepted on self-hosted / dedicated-database setups. +- Always set `hops.max` (an integer as small as the request allows). Omitting `max` means unbounded traversal, which is rejected on shared cloud connections and only accepted on self-hosted / dedicated-database setups; `hops.max` is capped per deployment (default 10). - Endpoint aggregations apply per **path**: `$count`/`$collect` deduplicate, but `$sum`/`$avg` over a multihop alias count one row per path — prefer counting to summing across multihop endpoints. - One hop stays the default: never add `hops` for a plain related-record condition. -**`$cycle`** — cycle/ring detection (endpoint binds back to the parent record): +**`$cycle`** — cycle/ring detection (a record-level predicate; the path binds back to the record): ```js where: { - RING: { - $cycle: true, - $relation: { type: "TRANSFERRED_TO", direction: "out", hops: { min: 2, max: 6 } } - } + $cycle: { type: "TRANSFERRED_TO", direction: "out", hops: { min: 2, max: 6 } } } ``` -Matches roots sitting on a closed path (fraud rings, circular ownership, dependency cycles). A `$cycle` block accepts **only** `$relation` (with `hops`, `min` ≥ 2 — defaults to 2): no `$alias`, no property criteria, no nested labels — a cycle has no separate endpoint. The block's key is a display name, not matched as a label. Set `direction` for flow-like semantics (money, ownership); undirected cycles also match innocent back-and-forth pairs. Wrap in `$not` for "NOT on a cycle". Paths may revisit a record via different relationships (only relationships are unique per path). +Matches roots sitting on a closed path (fraud rings, circular ownership, dependency cycles). The operator's value IS the traversal spec (`type`, `direction`, `hops` — `hops` mandatory, `min` ≥ 2, defaults to 2). No `$alias`, no property criteria, no nested labels — a cycle has no separate endpoint. Intermediate node labels are unconstrained. Place inside a label block (`ACCOUNT: { country: "US", $cycle: {...} }`) to anchor the cycle at the related record. Set `direction` for flow-like semantics (money, ownership); undirected cycles also match innocent back-and-forth pairs. Wrap in `$not` for "NOT on a cycle". Paths may revisit a record via different relationships (only relationships are unique per path). Never wrap `$cycle` in a named block and never nest `$relation` inside it — the value IS the relation spec, and it sits directly at a `where` level like any other operator. **`$id`** — filter by record ID: @@ -714,13 +713,13 @@ where: { EMPLOYEE: { $relation: { type: 'REPORTS_TO', direction: 'out', hops: { ### Cycle / Ring Detection -"Records on a loop back to themselves" (fraud rings, circular ownership, dependency cycles) → `$cycle` block (see §1). Root label = the entity type; the `$cycle` block holds only `$relation`: +"Records on a loop back to themselves" (fraud rings, circular ownership, dependency cycles) → `$cycle` operator (see §1). Root label = the entity type; the operator's value is the traversal spec: ```js findRecords({ labels: ['ACCOUNT'], where: { - RING: { $cycle: true, $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } }) ``` @@ -748,7 +747,7 @@ Returns ring **participants**. To reconstruct a specific ring's composition, fol | NOT condition | `$not: { ... }` | | related to X | `LABEL_NAME: { ...filters... }` | | within N hops / any ancestor | `LABEL: { $relation: { type?, direction, hops: { max: N } } }` — type verbatim from schema, or omitted | -| ring / loop / circular flow | `{ $cycle: true, $relation: { type?, direction, hops: { min: 2, max: N } } }` | +| ring / loop / circular flow | `{ $cycle: { type?, direction, hops: { min: 2, max: N } } }` | | last 7 days | compute ISO boundary → `createdAt: { $gte: "ISO-UTC" }` | | in year 1994 | `date: { $gte: { $year:1994 }, $lt: { $year:1995 } }` | @@ -774,7 +773,7 @@ Before submitting any `findRecords` call, verify: - `$relation.hops` and `$cycle` are VALID operators — do not "correct" them away - [ ] `$relation.type` copied verbatim from a schema Relationships row, or omitted; never invented - [ ] `hops` only for repeated-pattern traversal (hierarchy/degrees); bounded `max`, as small as possible -- [ ] `$cycle` block contains ONLY `$relation` (`hops` min ≥ 2); no `$alias`/criteria/nested labels inside +- [ ] `$cycle` value is the traversal spec itself (`hops` mandatory, min ≥ 2); never a named block, never `$relation` nested inside - [ ] Vector similarity: still uses legacy `aggregate` (not `select`) - [ ] Vector threshold semantics: euclidean → `$lte`; others → `$gte` - [ ] Month+day without year → ask for year @@ -923,7 +922,7 @@ where: { EMPLOYEE: { $relation: { type: 'REPORTS_TO', direction: 'out', hops: { findRecords({ labels: ['ACCOUNT'], where: { - RING: { $cycle: true, $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } }) ``` diff --git a/platform/core/.env.example b/platform/core/.env.example index f1a75d60..5e5b28ce 100644 --- a/platform/core/.env.example +++ b/platform/core/.env.example @@ -108,7 +108,7 @@ RUSHDB_PASSWORD=password # back when exceeded and the API returns 408. Defaults to 55000 — just under the 60s gateway # limit on managed deployments. Self-hosted setups without a gateway may raise this. -# RUSHDB_MAX_TRAVERSAL_HOPS=25 +# RUSHDB_MAX_TRAVERSAL_HOPS=10 # Ceiling for variable-length traversal depth (`$relation.hops` in SearchQuery). Defaults to 25. # Only enforced on the shared default Neo4j connection: self-hosted deployments # (RUSHDB_SELF_HOSTED=true) and projects with a custom external Neo4j allow unbounded hops, diff --git a/platform/core/CHANGELOG.md b/platform/core/CHANGELOG.md index 1f8e14fa..13f896c6 100644 --- a/platform/core/CHANGELOG.md +++ b/platform/core/CHANGELOG.md @@ -29,17 +29,14 @@ db.records.find({ labels: ['ACCOUNT'], where: { - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } }) ``` - A `$cycle` block requires `$relation` with `hops` (`min` ≥ 2, the default) and accepts no other keys; its label key is a display name. Combine with `$not` to select records not on a cycle. + `$cycle` is a record-level predicate: its value is the traversal spec itself (`type`, `direction`, `hops` — `hops` mandatory, `min` ≥ 2, defaulting to 2). A cycle has no separate endpoint, so it accepts no `$alias`, no property criteria, and no nested labels. Combine with `$not` to select records not on a cycle. - **Traversal depth policy** — `hops.max` is capped per deployment via `RUSHDB_MAX_TRAVERSAL_HOPS` (default 25) on the shared cloud connection. Self-hosted deployments (`RUSHDB_SELF_HOSTED=true`) and projects with a custom external Neo4j allow unbounded traversal (`max` omitted), guarded by the existing transaction timeout. + **Traversal depth policy** — `hops.max` is capped per deployment via `RUSHDB_MAX_TRAVERSAL_HOPS` (default 10) on the shared cloud connection. Self-hosted deployments (`RUSHDB_SELF_HOSTED=true`) and projects with a custom external Neo4j allow unbounded traversal (`max` omitted), guarded by the existing transaction timeout. Also in this release: diff --git a/platform/core/src/core/ai/ai-query.service.ts b/platform/core/src/core/ai/ai-query.service.ts index 18c70e9e..6d7724bf 100644 --- a/platform/core/src/core/ai/ai-query.service.ts +++ b/platform/core/src/core/ai/ai-query.service.ts @@ -372,7 +372,9 @@ export class AiQueryService { qb.append(clause) } // Enforce that traversed nodes were actually found (converts OPTIONAL to required). - if (extraMatchClauses.length > 0 && requiredAliasCheck) { + // Gated on the check itself, not extraMatchClauses: a $cycle-only where compiles to + // an alias-free EXISTS predicate with no OPTIONAL MATCH clauses to inject. + if (requiredAliasCheck) { qb.append(`WITH ${nodeAliases.join(', ')} WHERE ${requiredAliasCheck}`) } qb.append( diff --git a/platform/core/src/core/ai/search-query-builder.prompt.md b/platform/core/src/core/ai/search-query-builder.prompt.md index c5c9cd2c..5883e6cf 100644 --- a/platform/core/src/core/ai/search-query-builder.prompt.md +++ b/platform/core/src/core/ai/search-query-builder.prompt.md @@ -33,6 +33,8 @@ The user message is a JSON object with: ## Schema Rules +All label, property, and relationship-type names in this document's rules and examples are placeholders — real names come exclusively from the provided `schemaMarkdown`, never from these instructions. + - Use only labels that exist in the provided `schemaMarkdown`. - Use only property names that exist in the provided `schemaMarkdown`. - Use only relationship paths shown by the provided `schemaMarkdown`. @@ -42,7 +44,7 @@ The user message is a JSON object with: - If a user asks for a concept whose exact label/property is absent, choose the closest schema-backed field only when it is obvious. Add a warning for the assumption. - If the requested metric field is absent on the target label, inspect related labels from the schema before giving up. - When filtering a display property with free text the user typed, first confirm the property exists in the schema (often `name` or `title`, but never assume — pick from the label's listed properties), then resolve the text against the sample values the schema lists for that property. The schema shows only a few sample values per property and often truncates them with `(+N more)`, so the list is not exhaustive. - - If the user's term exactly equals, or unambiguously maps to, one of the listed sample values, filter by that full canonical value (e.g. user "Falcon" + listed value `Millennium Falcon` => `"name": "Millennium Falcon"`). + - If the user's term exactly equals, or unambiguously maps to, one of the listed sample values, filter by that full canonical value (e.g. user "Gatsby" + listed value `The Great Gatsby` => `"title": "The Great Gatsby"`). - Otherwise — no listed value matches, the values are truncated behind `(+N more)`, or none are shown — you cannot confirm the canonical value, so use `{ "$contains": "" }`. Never exact-match the raw user text against a property whose matching value you have not seen; that silently returns zero rows when the stored value is longer. - Use exact equality only for IDs, a value you confirmed from the sample list, or an explicit exact-match request. This applies to filters on any label, including related labels traversed inside `where`. @@ -53,10 +55,12 @@ Build query shape from intent: - Listing/search requests: use `labels`, `where`, `orderBy`, `skip`, and `limit`. - "Top N by " requests are listings, not aggregations. Use `labels`, `orderBy`, and `limit`; do not add `select` just to choose display columns. - Sum, avg, min, max, count, total, per-X, breakdown, distribution, grouped requests, and rankings by computed/related aggregate metrics: use `select` and `groupBy`. -- Relationship requests: use label-key traversal inside `where`. Each Relationships row in `schemaMarkdown` is a directed pattern rooted at that label — `(SELF)-[:TYPE]->(OTHER)` is outgoing, `(SELF)<-[:TYPE]-(OTHER)` is incoming. To pin the edge, set `$relation: { "type": "TYPE", "direction": "out" | "in" }` ("out" for `->`, "in" for `<-`); copy the type verbatim from the schema row or omit it — imported data often has only `__RUSHDB__RELATION__DEFAULT__` edges, and untyped traversal is valid. Only patterns shown in the schema are traversable; a scalar `*_id` property is a plain value, not an edge — never nest a label to "join" on it. +- Relationship requests: use label-key traversal inside `where`. Each Relationships row in `schemaMarkdown` is a directed pattern rooted at that label — `(SELF)-[:TYPE]->(OTHER)` is outgoing, `(SELF)<-[:TYPE]-(OTHER)` is incoming. To pin the edge, set `$relation: { "type": "TYPE", "direction": "out" | "in" }` ("out" for `->`, "in" for `<-`); copy the type verbatim from the schema row it appears in, or omit `type` entirely (untyped traversal is valid). Never write `__RUSHDB__RELATION__DEFAULT__` unless a schema Relationships row shows that exact type. Only patterns shown in the schema are traversable; a scalar `*_id` property is a plain value, not an edge — never nest a label to "join" on it. - Nested object requests: use `select` with `$collect` when a nested response is appropriate. - "Top N related records per parent/root entity" requests are nested listings. Use root `labels` and a `$collect` with `orderBy` and `limit` inside the collect. Do not add root `groupBy`; do not order the root query by the collected array field. -- "Which/what has most/more/least/less/fewer/fewest " requests are grouped related-count queries. Use the parent label as the only root `labels` entry, traverse the related label in `where` with `$alias`, put filters for the related records inside that related-label block, count that alias in `select`, group by the parent's display property from the schema (e.g. `$record.name` when the schema shows a `name` property), and order by the count. +- "Which/what has most/more/least/less/fewer/fewest " requests are grouped related-count queries. Use the parent label as the only root `labels` entry, traverse the related label in `where` with `$alias`, put filters for the related records inside that related-label block, count that alias in `select`, group by the parent's display property from the schema (`$record.name` when the schema shows `name`, `$record.title` when it shows `title`, or whatever string property the schema actually lists), and order by the count. +- Condition-verb check for rankings: when the comparative phrase carries a condition verb beyond mere relatedness ("won", "wrote", "approved", "caused", …), a traversal satisfies it only if the relationship type itself expresses that verb. Otherwise scan the related label's property list for a property whose name matches the verb ("won" → `winner_*`/`wonBy`-style, "approved" → `approvedBy`/`approver_*`-style, …); if one exists, apply the related-count exception below instead. Counting merely related records for a condition-verb request silently answers a different question — never do it without a warning. +- Related-count exception — the condition is a property, not a relationship: when the comparative condition itself ("won", "authored", "assigned", …) is recorded only as a scalar property on the related label — typically a reference-style `*_id` name such as `author_id`, but the naming is only a hint (it may equally be `authorRef` or `writtenBy`); the test is that no schema Relationships row expresses the condition — a traversal cannot state it and `where` cannot correlate two records. Root on the label that owns that property instead, `groupBy` that property, and count records (e.g. "which author wrote the most books" with only `BOOK.author_id` => root `BOOK`, group by `$record.author_id`, order `$count` desc). Never attempt the correlation in `where` — not as a `$record.*`/`$alias.*` value and not with `$ref`. Add a warning that rows are keyed by the raw property value. - Related-count direction: most/more/highest/largest/greatest => `desc`; least/less/fewer/fewest/lowest/smallest => `asc`. - Semantic/vector requests: use `aggregate` only when the schema shows a suitable vector-indexed property and the SearchQuery reference supports the requested operation. @@ -68,7 +72,7 @@ Build query shape from intent: - If the user asks "departments by project budget", root should usually be `DEPARTMENT` and the related `PROJECT` label should be traversed and aliased. - If the user asks "projects by budget", root should usually be `PROJECT`. - When a metric lives on a related label, keep the requested row entity as root, traverse to the metric label through `where`, set `$alias`, and reference that alias in `select`. -- In "which/what has " requests, do not let the related/filter label become the root just because it owns the filtered field. Keep the requested parent/entity as root if the schema shows a traversal path. +- In "which/what has " requests, do not let the related/filter label become the root just because it owns the filtered field. Keep the requested parent/entity as root if the schema shows a traversal path — unless the comparative condition exists only as a scalar property on the related label (a `*_id`-style reference or any other property, with no schema relationship expressing it — see the related-count exception above), in which case root on the label that owns that property and `groupBy` it. - If the requested parent-to-related traversal path is absent from the schema, do not silently switch root to the related label. Return the closest valid query only with a warning that the requested relationship path is unavailable. - Put each filter on the label that actually owns the filtered property. - For any named reference the user typed as free text, keep string filters loose with `$contains` on the display property the schema shows for that label, whether it lives on the root label or on a related label traversed with `$alias`. Do not use exact equality unless the value is an ID or the user explicitly requests an exact match. @@ -148,6 +152,38 @@ Example pattern for "which parent has the most/fewest related records": Use `"desc"` for most/more/highest/largest/greatest and `"asc"` for least/less/fewer/fewest/lowest/smallest. Only use this pattern when the schema actually shows `DEPARTMENT`, related `PROJECT`, `DEPARTMENT.name`, and a `DEPARTMENT` to `PROJECT` traversal. +Example pattern for a ranking whose condition exists only as a scalar reference property (the related-count exception). Prompt: "which author wrote the most books", where the schema shows `BOOK.author_id` but no `AUTHOR`-to-`BOOK` relationship expressing "wrote": + +WRONG — roots on the parent and attempts a correlated join; a `where` value can never reference the root record: + +```json +{ + "labels": ["AUTHOR"], + "where": { + "BOOK": { + "$alias": "$book", + "author_id": "$record.id" + } + } +} +``` + +RIGHT — roots on the label that owns the reference property and groups by it: + +```json +{ + "labels": ["BOOK"], + "select": { + "author": "$record.author_id", + "books_written": { "$count": "*" } + }, + "groupBy": ["$record.author_id"], + "orderBy": { "books_written": "desc" } +} +``` + +Apply this whatever the condition wording and property naming: "which team won the most matches" with only `MATCH.winner_team_id` => root `MATCH`, group by `$record.winner_team_id`; "which reviewer approved the most changes" with only `CHANGE.approvedBy` => root `CHANGE`, group by `$record.approvedBy`. Whenever a traversal exists but the condition itself lives in a scalar property, this grouped shape on the owning label is the only valid query. + Example pattern for simple top-N entity listing: ```json @@ -238,10 +274,11 @@ If `validationErrors` and `previousQuery` are provided: - Never output `orderBy` as `{"property":"budget","direction":"desc"}`. Use `{"budget":"desc"}`. - Never use unsupported traversal operators such as `$label`, `$direction`, `$as`, `$of`, or `$through`. - Never put related labels in root `labels`. Use `where` traversal with `$alias` for related labels. -- Never switch the root label from the user-requested parent/entity to the related/filter label for related-count rankings when a valid traversal path exists. +- Never switch the root label from the user-requested parent/entity to the related/filter label for related-count rankings when a valid traversal path exists — unless the ranking condition is recorded only as a scalar property on the related label (`*_id`-style or otherwise) with no schema relationship expressing it, in which case rooting on the owning label with `groupBy` on that property is the only valid shape. - Never use alias-only values in `groupBy`, such as `"$record"` or `"$related"`. Use a property reference such as `"$record.name"` or a select key such as `"total"`. - Never use a field or alias reference as a `where` predicate value, e.g. `{ "fieldA": "$record.id" }` or `{ "fieldA": { "$eq": "$alias.id" } }`. `where` values must be literals (or sample values from the schema); `$record.*`, `$alias.*`, and `$relation` references are valid only in `select`, `groupBy`, and `aggregate`. A reference used as a `where` value is matched as a literal string and silently returns nothing. - Never simulate a join on a scalar field that is not a relationship in the schema. If the user wants records ranked or grouped by such a field, root on the label that owns the field and `groupBy` that field — do not fabricate a `where` traversal or a correlated predicate. +- Never use `$ref` anywhere inside `where`. `$ref` exists only in `select`, for referencing another select output key. If a previous query attempted a correlated join — via `$ref` or a `$record.*`/`$alias.*` value — do not rephrase the join; restructure the query to root on the label that owns the field and `groupBy` it. - Never exact-match a user-typed named reference on a display property, including name/title filters inside related-label `where` blocks. Prefer `$contains` on the appropriate string property from the schema. Exact equality is only for IDs or explicit exact-match requests. - Never use `select` just to project fields for a simple listing. Prefer full record results with `labels`, `where`, `orderBy`, and `limit`. - Never include `limit` or `skip` for scan-level aggregate `select` queries unless the SearchQuery reference explicitly allows that specific shape. diff --git a/platform/core/src/core/ai/search-query-generator.service.spec.ts b/platform/core/src/core/ai/search-query-generator.service.spec.ts new file mode 100644 index 00000000..be28d5e4 --- /dev/null +++ b/platform/core/src/core/ai/search-query-generator.service.spec.ts @@ -0,0 +1,178 @@ +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-nocheck +import { SchemaItem } from '@/core/ai/ai.types' +import { SearchQueryGeneratorService } from '@/core/ai/search-query-generator.service' + +describe('SearchQueryGeneratorService.validateAndNormalize', () => { + const service = new SearchQueryGeneratorService(undefined, undefined) + + const schema: SchemaItem[] = [ + { + label: 'ACCOUNT', + count: 10, + properties: [ + { id: 'p1', name: 'name', type: 'string' }, + { id: 'p2', name: 'balance', type: 'number' } + ], + relationships: [{ label: 'ACCOUNT', type: 'TRANSFERRED_TO', direction: 'out' }] + }, + { + label: 'BATTLE', + count: 5, + properties: [ + { id: 'p3', name: 'name', type: 'string' }, + { id: 'p4', name: 'winner_faction_id', type: 'string' } + ], + relationships: [] + } + ] + + const validate = (query: unknown) => service['validateAndNormalize'](query, schema) + + let selfHosted: string | undefined + beforeAll(() => { + selfHosted = process.env.RUSHDB_SELF_HOSTED + delete process.env.RUSHDB_SELF_HOSTED + }) + afterAll(() => { + if (selfHosted !== undefined) { + process.env.RUSHDB_SELF_HOSTED = selfHosted + } + }) + + it('accepts the $cycle operator exactly as the spec prompt teaches it', () => { + const { errors } = validate({ + labels: ['ACCOUNT'], + where: { + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } + } + }) + expect(errors).toEqual([]) + }) + + it('accepts the $cycle operator nested inside a label block', () => { + const { errors } = validate({ + labels: ['ACCOUNT'], + where: { + ACCOUNT: { + name: 'x', + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } + } + } + }) + expect(errors).toEqual([]) + }) + + it('rejects a bare $cycle without a traversal spec', () => { + const { errors } = validate({ + labels: ['ACCOUNT'], + where: { $cycle: true } + }) + expect(errors.some((error) => error.includes("requires a traversal spec with 'hops'"))).toBe(true) + }) + + it('rejects the $cycle operator without hops', () => { + const { errors } = validate({ + labels: ['ACCOUNT'], + where: { $cycle: { type: 'TRANSFERRED_TO', direction: 'out' } } + }) + expect(errors.some((error) => error.includes("requires a traversal spec with 'hops'"))).toBe(true) + }) + + it('rejects $cycle operator hops below the cycle floor of 2', () => { + const { errors } = validate({ + labels: ['ACCOUNT'], + where: { $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 1, max: 6 } } } + }) + expect(errors.some((error) => error.includes("'hops.min'"))).toBe(true) + }) + + it('defaults missing hops.max on the $cycle operator to the deployment cap', () => { + const { query, errors } = validate({ + labels: ['ACCOUNT'], + where: { $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2 } } } + }) + expect(errors).toEqual([]) + expect((query.where as any).$cycle.hops.max).toBe(10) + }) + + it('rejects the removed block form', () => { + const { errors } = validate({ + labels: ['ACCOUNT'], + where: { + SOME_KEY: { + $cycle: true, + $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } + } + } + }) + expect(errors.length).toBeGreaterThan(0) + }) + + it('accepts $relation.hops on a regular traversal block', () => { + const { errors } = validate({ + labels: ['ACCOUNT'], + where: { + ACCOUNT: { + $alias: '$other', + $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 1, max: 4 } } + } + } + }) + expect(errors).toEqual([]) + }) + + it('defaults missing hops.max to the deployment cap instead of erroring', () => { + const { query, warnings, errors } = validate({ + labels: ['ACCOUNT'], + where: { + ACCOUNT: { $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 1 } } } + } + }) + expect(errors).toEqual([]) + expect((query.where as any).ACCOUNT.$relation.hops.max).toBe(10) + expect(warnings.some((warning) => warning.includes('hops.max'))).toBe(true) + }) + + it('rejects hops without an in/out direction', () => { + const { errors } = validate({ + labels: ['ACCOUNT'], + where: { + ACCOUNT: { $relation: { type: 'TRANSFERRED_TO', direction: 'both', hops: { min: 1, max: 4 } } } + } + }) + expect(errors.some((error) => error.includes('must be "in" or "out"'))).toBe(true) + }) + + it('explains how to restructure when $ref is used inside where', () => { + const { errors } = validate({ + labels: ['ACCOUNT'], + where: { BATTLE: { winner_faction_id: { $ref: '$record.id' } } } + }) + expect( + errors.some( + (error) => + error.includes('Unsupported where operator "$ref"') && + error.includes('root on the label that owns that field and use groupBy') + ) + ).toBe(true) + }) + + it('still flags field references used as plain where values', () => { + const { errors } = validate({ + labels: ['ACCOUNT'], + where: { BATTLE: { winner_faction_id: '$record.id' } } + }) + expect(errors.some((error) => error.includes('field/alias reference'))).toBe(true) + }) + + it('keeps validating an ordinary grouped query without errors', () => { + const { errors } = validate({ + labels: ['BATTLE'], + select: { faction: '$record.winner_faction_id', wins: { $count: '*' } }, + groupBy: ['$record.winner_faction_id'], + orderBy: { wins: 'desc' } + }) + expect(errors).toEqual([]) + }) +}) diff --git a/platform/core/src/core/ai/search-query-generator.service.ts b/platform/core/src/core/ai/search-query-generator.service.ts index 80783f7b..8fb00a87 100644 --- a/platform/core/src/core/ai/search-query-generator.service.ts +++ b/platform/core/src/core/ai/search-query-generator.service.ts @@ -6,6 +6,7 @@ import { Transaction } from 'neo4j-driver' import { AiService } from '@/core/ai/ai.service' import { SchemaItem } from '@/core/ai/ai.types' import { SearchDto } from '@/core/search/dto/search.dto' +import { resolveMaxTraversalHops } from '@/core/search/search.constants' import { DEFAULT_TRANSACTION_TIMEOUT_MS } from '@/database/transaction.constants' import { existsSync, readFileSync } from 'fs' @@ -185,8 +186,6 @@ export class SearchQueryGeneratorService { const queryBuilderPrompt = this.readPromptFile(QUERY_BUILDER_PROMPT_FILE) const searchQuerySpecPrompt = this.readPromptFile(SEARCH_QUERY_SPEC_PROMPT_FILE) - this.logger.log(`[AI schemaMarkdown]: ${schemaMarkdown}`) - const response = await axios.post( `${baseUrl.replace(/\/$/, '')}/chat/completions`, { @@ -324,6 +323,7 @@ export class SearchQueryGeneratorService { const query = JSON.parse(JSON.stringify(searchQuery)) as SearchDto this.normalizeOrderBy(query, warnings) + this.normalizeTraversalHops(query.where, warnings) for (const key of Object.keys(query)) { if (!TOP_LEVEL_KEYS.has(key)) { errors.push(`Unsupported top-level key "${key}"`) @@ -424,6 +424,42 @@ export class SearchQueryGeneratorService { } } + // Unbounded hops are rejected by the parser on capped connections; defaulting max to the + // deployment cap (like the limit/skip normalization above) saves a repair round. + private normalizeTraversalHops(value: unknown, warnings: string[]) { + if (!value || typeof value !== 'object') { + return + } + if (Array.isArray(value)) { + value.forEach((item) => this.normalizeTraversalHops(item, warnings)) + return + } + + for (const [key, next] of Object.entries(value)) { + // The $cycle operator's value is a relation spec itself — its hops live directly + // under the operator, not under a $relation wrapper. + if ( + (key === '$relation' || key === '$cycle') && + next && + typeof next === 'object' && + !Array.isArray(next) + ) { + const hops = (next as any).hops + if (hops && typeof hops === 'object' && !Array.isArray(hops) && hops.max === undefined) { + const maxHops = resolveMaxTraversalHops() + if (maxHops !== Infinity) { + hops.max = maxHops + warnings.push( + `Set hops.max to ${maxHops} because unbounded traversal is not allowed on this connection.` + ) + } + } + continue + } + this.normalizeTraversalHops(next, warnings) + } + } + private validateSelectExpressions(select: unknown, errors: string[]) { if (!select || typeof select !== 'object' || Array.isArray(select)) { return @@ -503,11 +539,27 @@ export class SearchQueryGeneratorService { this.validateWhere(next, schemaByLabel, propertiesByLabel, allProperties, errors, currentLabels) continue } + if (key === '$relation') { + this.validateRelation(next, 1, `label "${currentLabels.join('|') || ''}"`, errors) + continue + } + if (key === '$cycle') { + this.validateCycleOperator(next, errors) + continue + } if (COMPARISON_KEYS.has(key) || RELATION_META_KEYS.has(key)) { continue } if (key.startsWith('$')) { - errors.push(`Unsupported where operator "${key}"`) + if (key === '$ref') { + errors.push( + `Unsupported where operator "$ref": "$ref" is only valid inside select expressions. ` + + `where values must be literals — correlated joins between records are not supported. ` + + `To rank or relate records by a scalar field, root on the label that owns that field and use groupBy.` + ) + } else { + errors.push(`Unsupported where operator "${key}"`) + } continue } if (schemaByLabel.has(key)) { @@ -527,6 +579,70 @@ export class SearchQueryGeneratorService { } } + // Canonical operator form: { "$cycle": { "type"?, "direction", "hops" } } — the value + // IS the traversal spec. Mirrors the parser's normalizeHops rules (hops mandatory, + // floor 2) so a query the spec prompt teaches is not rejected here or at execution. + private validateCycleOperator(value: unknown, errors: string[]) { + if (!value || typeof value !== 'object' || Array.isArray(value) || !('hops' in value)) { + errors.push( + `'$cycle' requires a traversal spec with 'hops', e.g. ` + + `{ "$cycle": { "type": "FLOWS_TO", "direction": "out", "hops": { "min": 2, "max": 6 } } }` + ) + return + } + this.validateRelation(value, 2, `'$cycle'`, errors) + } + + private validateRelation(relation: unknown, hopsFloor: number, context: string, errors: string[]) { + if (!relation || typeof relation !== 'object' || Array.isArray(relation)) { + return + } + + const direction = (relation as any).direction + const hops = (relation as any).hops + if (hops === undefined) { + return + } + if (direction !== undefined && direction !== 'in' && direction !== 'out') { + errors.push( + `'$relation.direction' must be "in" or "out" when 'hops' is set in ${context}, got ${JSON.stringify(direction)}` + ) + } + + const maxHops = resolveMaxTraversalHops() + const maxHopsLabel = maxHops === Infinity ? 'unbounded' : maxHops + if (typeof hops === 'number') { + if (!Number.isInteger(hops) || hops < hopsFloor || hops > maxHops) { + errors.push( + `'hops' in ${context} must be an integer between ${hopsFloor} and ${maxHopsLabel}, got ${JSON.stringify(hops)}` + ) + } + return + } + if (hops && typeof hops === 'object' && !Array.isArray(hops)) { + const max = (hops as any).max + const min = (hops as any).min ?? hopsFloor + if (max === undefined) { + if (maxHops !== Infinity) { + errors.push( + `'hops.max' is required in ${context}: unbounded traversal is not allowed on this connection (max allowed hops: ${maxHops})` + ) + } + } else if (!Number.isInteger(max) || max < hopsFloor || max > maxHops) { + errors.push( + `'hops.max' in ${context} must be an integer between ${hopsFloor} and ${maxHopsLabel}, got ${JSON.stringify(max)}` + ) + } + if (!Number.isInteger(min) || min < hopsFloor || (Number.isInteger(max) && min > max)) { + errors.push( + `'hops.min' in ${context} must be an integer between ${hopsFloor} and 'hops.max', got ${JSON.stringify(min)}` + ) + } + return + } + errors.push(`'hops' in ${context} must be a number or { min?, max? } object, got ${JSON.stringify(hops)}`) + } + private validateSelectRefs( value: unknown, schemaByLabel: Map, diff --git a/platform/core/src/core/ai/search-query-spec.prompt.md b/platform/core/src/core/ai/search-query-spec.prompt.md index 5941959a..d5fdae05 100644 --- a/platform/core/src/core/ai/search-query-spec.prompt.md +++ b/platform/core/src/core/ai/search-query-spec.prompt.md @@ -70,7 +70,7 @@ This returns full records and lets the dashboard table render the normal record All label, field, and relationship-type names in this document are placeholders — real names come exclusively from the schema, never from these examples. Labels are case-sensitive: copy them exactly as spelled in the schema and never change their case. -Predicate values must be **literals** (strings, numbers, booleans, dates) or sample values from the schema. A `where` value can never be a field or alias reference: `{ "field": "$record.id" }` and `{ "field": { "$eq": "$alias.id" } }` are invalid — the reference is matched as a literal string and returns nothing. Correlated comparisons between two records (a "join on" a field) are not supported in `where`; `$record.*`, `$alias.*`, and `$relation` references belong in `select`, `groupBy`, and `aggregate` only. To rank or group by a scalar field that is not a relationship in the schema, root on the label that owns the field and `groupBy` that field instead of building a traversal. +Predicate values must be **literals** (strings, numbers, booleans, dates) or sample values from the schema. A `where` value can never be a field or alias reference: `{ "field": "$record.id" }`, `{ "field": { "$eq": "$alias.id" } }`, and `{ "field": { "$ref": "$record.id" } }` are all invalid — `$ref` exists only in `select` expressions, and a plain reference is matched as a literal string and returns nothing. Correlated comparisons between two records (a "join on" a field) are not supported in `where` in any syntax; `$record.*`, `$alias.*`, and `$relation` references belong in `select`, `groupBy`, and `aggregate` only. To rank or group by a scalar field that is not a relationship in the schema, root on the label that owns the field and `groupBy` that field instead of building a traversal. ### Primitive Matching @@ -225,7 +225,7 @@ Use `$relation` only to constrain relationship type or direction when the user e `$relation` can also be a relationship type string. -`type` must be copied **verbatim** from a schema Relationships row — never invent a semantic type like the examples here. Data imported as nested JSON has only `__RUSHDB__RELATION__DEFAULT__` edges; when the schema shows that, either use that exact string or omit `type` entirely (untyped traversal is valid, including with `hops`). +`type` must be copied **verbatim** from a schema Relationships row — never invent a semantic type like the examples here, and never write `__RUSHDB__RELATION__DEFAULT__` unless a schema row shows that exact type. When it does — data imported as nested JSON often has only such edges — either use that exact string or omit `type` entirely (untyped traversal is valid, including with `hops`). Each Relationships row in `schemaMarkdown` is a directed pattern rooted at that label: `(SELF)-[:TYPE]->(OTHER)` is outgoing and `(SELF)<-[:TYPE]-(OTHER)` is incoming. To traverse, nest `OTHER` as a label key under `where` (add `$alias` if it is referenced in `select`/`groupBy`); to pin the edge set `$relation: { "type": "TYPE", "direction": ... }` with `"out"` for `->` and `"in"` for `<-`. Only patterns shown in the schema are traversable — a scalar `*_id` property is a plain value, not an edge, so never nest a label to "join" on it. @@ -249,27 +249,26 @@ Add `hops` to `$relation` when the user asks about reachability across more than Rules: - Keep `max` as small as the request allows; deep undirected traversal is expensive. Omit `type` only when the user genuinely means "any relationship" — and keep `direction` if the schema gives one, since it prunes the search. -- `hops.max` is capped per deployment (default 25); omitting `max` (unbounded) is only accepted on self-hosted/dedicated database setups. +- Always set `hops.max` (an integer as small as the request allows). Omitting `max` means unbounded traversal, which is rejected on shared cloud connections and only accepted on self-hosted/dedicated database setups; `hops.max` is capped per deployment (default 10). - Filters and aggregations on the endpoint (`$alias` + `select`) apply per **path**: `$count`/`$collect` deduplicate, but `$sum`/`$avg` over a multihop alias count one row per path — prefer counting to summing across multihop endpoints. - One hop is still the default: never add `hops` for a plain related-record condition. ### Cycle Detection (`$cycle`) -Use `$cycle: true` when the user asks for rings, loops, circular flows, or records that "come back to themselves" (fraud rings, circular ownership, dependency cycles). The traversal block's endpoint binds back to the parent record itself, so the block accepts **only** `$relation` (with `hops`, `min` ≥ 2 — defaults to 2). No `$alias`, no property criteria, no nested labels inside a `$cycle` block; the block's key is a display name and is not matched as a label. +Use the `$cycle` operator when the user asks for rings, loops, circular flows, or records that "come back to themselves" (fraud rings, circular ownership, dependency cycles). `$cycle` is a record-level predicate — its value is the traversal spec itself (`type`, `direction`, `hops`; `hops` is mandatory, `min` ≥ 2, defaulting to 2). A cycle has no separate endpoint: no `$alias`, no property criteria, no nested labels. ```json { "labels": ["ACCOUNT"], "where": { - "RING": { - "$cycle": true, - "$relation": { "type": "TRANSFERRED_TO", "direction": "out", "hops": { "min": 2, "max": 6 } } - } + "$cycle": { "type": "TRANSFERRED_TO", "direction": "out", "hops": { "min": 2, "max": 6 } } } } ``` -This returns every ACCOUNT sitting on a directed transfer ring of 2–6 hops. Set `direction` for flow-like semantics (money, ownership); an undirected cycle also matches innocent back-and-forth pairs. Wrap in `$not` to express "records NOT on a cycle". Paths may revisit a record through different relationships (only relationships are unique per path). +This returns every ACCOUNT sitting on a directed transfer ring of 2–6 hops. Set `direction` for flow-like semantics (money, ownership); an undirected cycle also matches innocent back-and-forth pairs. Wrap in `$not` to express "records NOT on a cycle"; place inside a label block (`"ACCOUNT": { "country": "US", "$cycle": {...} }`) to anchor the cycle at the related record. Intermediate node labels are unconstrained. Paths may revisit a record through different relationships (only relationships are unique per path). + +Never wrap `$cycle` in a named block and never nest `$relation` inside it — the operator's value IS the relation spec, and it sits directly at a `where` level like any other operator. Never use `$label`, `$direction`, `$as`, `$of`, `$through`, or `$hops` (hops lives inside `$relation`). @@ -303,6 +302,20 @@ Direction mapping: most/more/highest/largest/greatest => `"desc"`; least/less/fe Only use this shape when the schema shows the parent label, related label, display field, and traversal path. If the traversal path is absent, do not silently root on the related label; return the closest valid query with a warning. +Exception: this shape requires the traversal itself to express the requested condition. If the condition ("won", "authored", "assigned", …) is recorded only as a scalar property on the related label — typically a reference-style `*_id` name such as `author_id`, but any property counts, whatever its naming (`authorRef`, `writtenBy`, …); the test is that no schema Relationships row expresses the condition — do not use this shape. A traversal cannot state it and `where` cannot correlate the property with the parent (no `$record.*`/`$alias.*` values, no `$ref`). Root on the label that owns the property, `groupBy` that property, count records, and add a warning that rows are keyed by the raw property value: + +```json +{ + "labels": [""], + "select": { + "parent": "$record.", + "count": { "$count": "*" } + }, + "groupBy": ["$record."], + "orderBy": { "count": "desc" } +} +``` + ### ID Filter ```json @@ -413,6 +426,8 @@ Use field references to produce one row per distinct value. } ``` +The group key is any property the schema lists — a display property works the same way whatever it is called (`$record.title` on a label whose display property is `title`, `$record.name` when it is `name`). + The returned group key appears without the alias prefix. ### Self Grouping diff --git a/platform/core/src/core/common/types.ts b/platform/core/src/core/common/types.ts index b1481c64..b0c84527 100644 --- a/platform/core/src/core/common/types.ts +++ b/platform/core/src/core/common/types.ts @@ -86,11 +86,14 @@ export type TraversalHops = number | { min?: number; max?: number } export type Relation = string | { type?: string; direction?: TRelationDirection; hops?: TraversalHops } +/** Record-level cycle predicate: the record lies on a closed path over these edges. + * `hops` is mandatory (floor 2); intermediate node labels are unconstrained. */ +export type CycleExpression = { type?: string; direction?: TRelationDirection; hops: TraversalHops } + export type Related = { [K in keyof Models]?: { $relation?: Relation $alias?: string - $cycle?: boolean } & Where } @@ -144,8 +147,8 @@ export type Condition = } | { $id?: MaybeLogicalExpression } -export type Where = (Condition & Related) & - Partial & Related>> +export type Where = (Condition & Related & { $cycle?: CycleExpression }) & + Partial & Related & { $cycle?: CycleExpression }>> /** @deprecated Use SelectExprMap / select expressions instead. To be removed in a future major version. */ export type AggregateCollectFn = { diff --git a/platform/core/src/core/entity/entity-query.service.ts b/platform/core/src/core/entity/entity-query.service.ts index bd22a659..c9b19d9a 100755 --- a/platform/core/src/core/entity/entity-query.service.ts +++ b/platform/core/src/core/entity/entity-query.service.ts @@ -308,7 +308,7 @@ export class EntityQueryService { .append(`MATCH ${relatedQueryPart}(record:${RUSHDB_LABEL_RECORD}${labelPart} { ${projectIdInline()} })`) .append(normalizedQueryClauses) - if (parsedWhere.nodeAliases.filter(toBoolean).length > 1) { + if (this.needsWhereBarrier(parsedWhere)) { const wherePart = parsedWhere.where ? `WHERE ${parsedWhere.where}` : '' queryBuilder.append(`WITH ${parsedWhere.nodeAliases.join(', ')} ${wherePart}`.trim()) @@ -329,6 +329,16 @@ export class EntityQueryService { return queryBuilder.getQuery() } + /** + * The WITH…WHERE barrier is needed when the where tree references traversal aliases + * (nodeAliases beyond the root) or carries alias-free predicates — a $cycle-only + * query compiles to an EXISTS subquery with nodeAliases === ['record'] but a + * non-empty where, which the alias-count check alone would silently drop. + */ + private needsWhereBarrier(parsedWhere: { nodeAliases: string[]; where: string }): boolean { + return parsedWhere.nodeAliases.filter(toBoolean).length > 1 || toBoolean(parsedWhere.where) + } + getRecordsCount({ id, searchQuery }: { id?: string; searchQuery?: SearchDto }) { const relatedQueryPart = buildRelatedQueryPart(id) const labelPart = singleLabelPart(searchQuery?.labels) @@ -346,7 +356,7 @@ export class EntityQueryService { .append(`MATCH ${relatedQueryPart}(record:${RUSHDB_LABEL_RECORD}${labelPart} { ${projectIdInline()} })`) .append(queryClauses.join(`\n`)) - if (parsedWhere.nodeAliases.filter(toBoolean).length > 1) { + if (this.needsWhereBarrier(parsedWhere)) { const wherePart = parsedWhere.where ? `WHERE ${parsedWhere.where}` : '' queryBuilder.append(`WITH ${parsedWhere.nodeAliases.join(', ')} ${wherePart}`.trim()) @@ -378,7 +388,7 @@ export class EntityQueryService { .append(`MATCH (record:${RUSHDB_LABEL_RECORD}${labelPart} { ${projectIdInline()} })`) .append(queryClauses.join(`\n`)) - if (parsedWhere.nodeAliases.filter(toBoolean).length > 1) { + if (this.needsWhereBarrier(parsedWhere)) { const wherePart = parsedWhere.where ? `WHERE ${parsedWhere.where}` : '' queryBuilder.append(`WITH ${parsedWhere.nodeAliases.join(', ')} ${wherePart}`.trim()) @@ -1166,6 +1176,34 @@ END` return queryBuilder.getQuery() } + /** + * Counts (up to `limit`) the distinct record pairs a join pattern would connect, + * via the same pair statement createRelationsByKeys executes — so a probe result + * is exact evidence of what applying the pattern would do. LIMIT caps the pairs + * pulled from the lazy pair pipeline, bounding probe cost on large graphs. + */ + countRelationCandidatesByKeys(payload: { + sourceLabel: string + sourceKey?: string + targetLabel: string + targetKey?: string + sourceWhere?: Where + targetWhere?: Where + limit: number + }) { + const { pairStatement } = this.constructRelationshipQueryArguments(payload) + const limit = Math.max(1, Math.floor(payload.limit)) + + const queryBuilder = new QueryBuilder() + + queryBuilder + .append(`CALL { ${pairStatement} }`) + .append(`WITH s, t LIMIT ${limit}`) + .append('RETURN count(*) AS matchCount') + + return queryBuilder.getQuery() + } + retypeRelationsByLabels({ sourceLabel, targetLabel, diff --git a/platform/core/src/core/entity/entity.service.ts b/platform/core/src/core/entity/entity.service.ts index 429fdd4f..88c2eb75 100755 --- a/platform/core/src/core/entity/entity.service.ts +++ b/platform/core/src/core/entity/entity.service.ts @@ -451,6 +451,38 @@ export class EntityService { : Number(committedOperations ?? 0) } + /** + * Probes how many record pairs a join pattern would actually connect, capped at + * `limit`. Read-only; runs the same pair-matching statement as createRelationsByKeys. + */ + async countRelationCandidatesByKeys({ + source, + target, + projectId, + transaction, + limit + }: { + source: { label: string; key?: string; where?: Where } + target: { label: string; key?: string; where?: Where } + projectId: string + transaction: Transaction + limit: number + }): Promise { + const query = this.entityQueryService.countRelationCandidatesByKeys({ + sourceLabel: source.label, + sourceKey: source.key, + targetLabel: target.label, + targetKey: target.key, + sourceWhere: source.where, + targetWhere: target.where, + limit + }) + + const result = await transaction.run(query, { projectId }) + const matchCount = result.records[0]?.get('matchCount') + return typeof matchCount?.toNumber === 'function' ? matchCount.toNumber() : Number(matchCount ?? 0) + } + async retypeRelationsByLabels({ source, target, diff --git a/platform/core/src/core/entity/import-export/import.service.spec.ts b/platform/core/src/core/entity/import-export/import.service.spec.ts index 873703fd..9c1f1d47 100644 --- a/platform/core/src/core/entity/import-export/import.service.spec.ts +++ b/platform/core/src/core/entity/import-export/import.service.spec.ts @@ -71,4 +71,51 @@ describe('ImportService', () => { expect(props.find((p) => p.name === 'empties')).toBeUndefined() }) }) + + // PapaParse dynamicTyping (the CSV import default when suggestTypes is on) parses + // ISO-8601 cells into JS Date instances. A Date must never reach the driver as a + // property value — it serializes as an empty map and Neo4j rejects the whole chunk + // ("Property values can only be of primitive types... Encountered: Map{}"). + describe('Date instances from CSV dynamicTyping', () => { + const propsFor = (data: Record, suggestTypes: boolean) => { + const [records] = service.serializeBFS([data] as any, 'TRANSACTION', { suggestTypes }) + return Object.fromEntries((records[0].properties ?? []).map((p) => [p.name, p])) + } + + it('folds a Date value back to its ISO string and types it as datetime', () => { + const byName = propsFor({ id: 't1', timestamp: new Date('2026-06-02T08:14:00Z') }, true) + expect(byName.timestamp.value).toBe('2026-06-02T08:14:00.000Z') + expect(byName.timestamp.type).toBe('datetime') + }) + + it('stores the ISO string (not a locale string) when suggestTypes is off', () => { + const byName = propsFor({ id: 't1', timestamp: new Date('2026-06-02T08:14:00Z') }, false) + expect(byName.timestamp.value).toBe('2026-06-02T08:14:00.000Z') + expect(byName.timestamp.type).toBe('string') + }) + + it('leaves no object-typed property values anywhere in a CSV-shaped row', () => { + const [records] = service.serializeBFS( + [ + { + transaction_id: 't2001', + sender_account: 'acc_017', + receiver_account: 'acc_042', + amount: 9800, + timestamp: new Date('2026-06-14T09:12:00Z'), + memo: 'consulting fee' + } + ] as any, + 'TRANSACTION', + { suggestTypes: true } + ) + for (const property of records[0].properties ?? []) { + const value = (property as any).value + const values = Array.isArray(value) ? value : [value] + for (const item of values) { + expect(typeof item === 'object' && item !== null).toBe(false) + } + } + }) + }) }) diff --git a/platform/core/src/core/entity/import-export/import.service.ts b/platform/core/src/core/entity/import-export/import.service.ts index c1802885..bf508c2f 100644 --- a/platform/core/src/core/entity/import-export/import.service.ts +++ b/platform/core/src/core/entity/import-export/import.service.ts @@ -104,6 +104,17 @@ export class ImportService { // skipEmptyValues: an empty string is treated as unset — no property created. // (0 and false are real values and are handled in the branch below.) } else { + // PapaParse dynamicTyping (CSV import) turns ISO-8601 cells into JS Date instances. + // A Date is not a driver-serializable property value — it reaches Neo4j as an empty + // map and fails the whole write — so fold it back to the ISO string it came from + // (which suggestPropertyType then types as datetime). + if (value instanceof Date) { + value = value.toISOString() + } else if (isArray(value)) { + value = value.map((item) => + (item as unknown) instanceof Date ? (item as any as Date).toISOString() : item + ) + } const property = { id: uuidv7(), name: key, diff --git a/platform/core/src/core/relationship-patterns/relationship-patterns.service.spec.ts b/platform/core/src/core/relationship-patterns/relationship-patterns.service.spec.ts index 3186a7cf..e980ac2e 100644 --- a/platform/core/src/core/relationship-patterns/relationship-patterns.service.spec.ts +++ b/platform/core/src/core/relationship-patterns/relationship-patterns.service.spec.ts @@ -9,7 +9,11 @@ type TestableRelationshipPatternsService = { candidate: RelationshipPatternCandidate, schema: SchemaItem[] ): RelationshipPatternCandidate | undefined - suggestDeterministicCandidates(schema: SchemaItem[]): RelationshipPatternCandidate[] + buildCandidateHints(schema: SchemaItem[]): RelationshipPatternCandidate[] + probeJoinCandidates( + projectId: string, + candidates: RelationshipPatternCandidate[] + ): Promise } const folderSchema: SchemaItem[] = [ @@ -123,36 +127,29 @@ const starWarsSchema: SchemaItem[] = [ } ], relationships: [] - }, - { - label: 'BATCH_CHARACTER', - count: 3, - properties: [ - { - id: 'p7', - name: 'id', - type: 'string', - values: ['character_padme_amidala', 'character_boss_nass', 'character_luke_skywalker'] - }, - { id: 'p8', name: 'name', type: 'string', values: ['Padme Amidala', 'Boss Nass'] } - ], - relationships: [] } ] -describe('RelationshipPatternsService', () => { - const service = new RelationshipPatternsService( +const makeService = (overrides: Partial> = {}) => + new RelationshipPatternsService( {} as never, {} as never, {} as never, + (overrides.neogmaService ?? {}) as never, {} as never, {} as never, - {} as never, - {} as never + (overrides.entityService ?? {}) as never ) as unknown as TestableRelationshipPatternsService +describe('RelationshipPatternsService', () => { + const service = makeService() + + // Validation is structural plausibility only: semantics are the LLM's judgment and + // evidence comes from the live-graph probe — sampled values never gate a candidate, + // because tiny samples of high-cardinality data (thousands of account ids) carry no + // overlap signal even for perfectly joinable columns. describe('validateCandidate', () => { - it('accepts same-label joins with sampled value overlap', () => { + it('accepts same-label self-reference joins', () => { const candidate = service.validateCandidate( { source: { label: 'Folder', key: 'id' }, @@ -210,7 +207,7 @@ describe('RelationshipPatternsService', () => { }) }) - it('rejects cross-label joins without sampled value overlap', () => { + it('passes cross-label joins through to graph probing instead of judging sampled overlap', () => { const candidate = service.validateCandidate( { source: { label: 'CHARACTER', key: 'name' }, @@ -223,10 +220,11 @@ describe('RelationshipPatternsService', () => { starWarsSchema ) - expect(candidate).toBeUndefined() + // Structurally plausible — whether any records actually match is the probe's call. + expect(candidate).toBeDefined() }) - it('rejects boolean and categorical enum overlap as relationship evidence', () => { + it('rejects joins on non-reference value types', () => { expect( service.validateCandidate( { @@ -258,7 +256,7 @@ describe('RelationshipPatternsService', () => { ).toBeUndefined() }) - it('accepts reference fields with sampled overlap', () => { + it('accepts comma-separated reference fields joining scalar fields', () => { const candidate = service.validateCandidate( { source: { label: 'STARSHIP', key: 'pilots' }, @@ -279,28 +277,7 @@ describe('RelationshipPatternsService', () => { }) }) - it('accepts array reference fields with sampled overlap', () => { - const candidate = service.validateCandidate( - { - source: { label: 'BATTLE', key: 'commander_character_ids' }, - target: { label: 'BATCH_CHARACTER', key: 'id' }, - direction: 'out', - type: 'HAS_COMMANDER_CHARACTER', - mode: 'join_pattern', - confidence: 0.9 - }, - starWarsSchema - ) - - expect(candidate).toMatchObject({ - source: { label: 'BATTLE', key: 'commander_character_ids' }, - target: { label: 'BATCH_CHARACTER', key: 'id' }, - type: 'HAS_COMMANDER_CHARACTER', - mode: 'join_pattern' - }) - }) - - it('accepts name-backed reference fields even without sampled overlap', () => { + it('accepts array reference fields joining scalar fields', () => { const candidate = service.validateCandidate( { source: { label: 'BATTLE', key: 'commander_character_ids' }, @@ -322,9 +299,11 @@ describe('RelationshipPatternsService', () => { }) }) - describe('suggestDeterministicCandidates', () => { - it('suggests joins from sampled overlap', () => { - expect(service.suggestDeterministicCandidates(starWarsSchema)).toContainEqual( + // Hints are derived from names and types only and handed to the LLM for semantic + // vetting — they are hypotheses, not suggestions. + describe('buildCandidateHints', () => { + it('hints name-backed reference fields regardless of sampled values', () => { + expect(service.buildCandidateHints(starWarsSchema)).toContainEqual( expect.objectContaining({ source: { label: 'BATTLE', key: 'commander_character_ids' }, target: { label: 'CHARACTER', key: 'id' }, @@ -332,31 +311,264 @@ describe('RelationshipPatternsService', () => { mode: 'join_pattern' }) ) - expect(service.suggestDeterministicCandidates(starWarsSchema)).toContainEqual( + }) + + it('does not hint boolean columns or list-to-list pairs', () => { + const hints = service.buildCandidateHints(starWarsSchema) + + expect(hints).not.toContainEqual( expect.objectContaining({ - source: { label: 'STARSHIP', key: 'pilots' }, - target: { label: 'CHARACTER', key: 'name' }, - type: 'HAS_PILOT', + source: expect.objectContaining({ key: 'force_sensitive' }) + }) + ) + expect(hints).not.toContainEqual( + expect.objectContaining({ + source: expect.objectContaining({ key: 'mentor_character_ids' }), + target: expect.objectContaining({ key: 'apprentice_character_ids' }) + }) + ) + }) + }) + + // Same-label chain/flow patterns: two reference-like columns on ONE label identifying + // the same kind of entity (receiver_account / sender_account). Detected purely from + // names — shared entity token + opposing direction words. Sampled values are ignored: + // on real data (thousands of accounts) 10-value samples of the two columns virtually + // never overlap even though the graph joins on them everywhere. + describe('same-label chain hints', () => { + const transactionsSchema: SchemaItem[] = [ + { + label: 'TRANSACTION', + count: 25, + properties: [ + { id: 'p1', name: 'transaction_id', type: 'string', values: ['t1001', 't1002', 't2001'] }, + { + id: 'p2', + name: 'sender_account', + type: 'string', + values: ['acc_003', 'acc_006', 'acc_017'] + }, + { + id: 'p3', + name: 'receiver_account', + type: 'string', + // Deliberately no overlap with sender_account samples — must not matter. + values: ['acc_201', 'acc_205', 'acc_311'] + }, + { id: 'p4', name: 'amount', type: 'number', values: [54.2, 9800, 42000] as any }, + { id: 'p5', name: 'currency', type: 'string', values: ['USD'] }, + { id: 'p6', name: 'status', type: 'string', values: ['posted', 'pending'] }, + { id: 'p7', name: 'previous_status', type: 'string', values: ['pending', 'created'] } + ], + relationships: [] + } + ] + + it('hints chaining records through destination/origin reference columns without sampled overlap', () => { + expect(service.buildCandidateHints(transactionsSchema)).toContainEqual( + expect.objectContaining({ + source: { label: 'TRANSACTION', key: 'receiver_account' }, + target: { label: 'TRANSACTION', key: 'sender_account' }, + direction: 'out', + type: 'FLOWS_TO_ACCOUNT', mode: 'join_pattern' }) ) }) - it('does not suggest enum or list-to-list joins', () => { - const suggestions = service.suggestDeterministicCandidates(starWarsSchema) + it('passes end-to-end candidate validation', () => { + const validated = service.validateCandidate( + { + source: { label: 'TRANSACTION', key: 'receiver_account' }, + target: { label: 'TRANSACTION', key: 'sender_account' }, + direction: 'out', + type: 'FLOWS_TO_ACCOUNT', + mode: 'join_pattern', + confidence: 0.82 + }, + transactionsSchema + ) + expect(validated).toBeDefined() + expect(validated?.type).toBe('FLOWS_TO_ACCOUNT') + }) - expect(suggestions).not.toContainEqual( + it('does not hint columns whose names carry no flow direction (status, previous_status)', () => { + const hints = service.buildCandidateHints(transactionsSchema) + expect(hints).not.toContainEqual( expect.objectContaining({ - source: expect.objectContaining({ key: 'force_sensitive' }) + source: expect.objectContaining({ key: expect.stringContaining('status') }) }) ) - expect(suggestions).not.toContainEqual( + expect(hints).not.toContainEqual( expect.objectContaining({ - source: expect.objectContaining({ key: 'mentor_character_ids' }), - target: expect.objectContaining({ key: 'apprentice_character_ids' }) + target: expect.objectContaining({ key: expect.stringContaining('status') }) }) ) }) + + it('infers flow direction from from/to naming', () => { + const legsSchema: SchemaItem[] = [ + { + label: 'RouteLeg', + count: 10, + properties: [ + { + id: 'p1', + name: 'from_station', + type: 'string', + values: ['st_alpha', 'st_beta'] + }, + { + id: 'p2', + name: 'to_station', + type: 'string', + values: ['st_gamma', 'st_omega'] + } + ], + relationships: [] + } + ] + + expect(service.buildCandidateHints(legsSchema)).toContainEqual( + expect.objectContaining({ + source: { label: 'RouteLeg', key: 'to_station' }, + target: { label: 'RouteLeg', key: 'from_station' }, + type: 'FLOWS_TO_STATION' + }) + ) + }) + + it('leaves direction-ambiguous pairs to the LLM but validates them as plausible joins', () => { + const pairSchema: SchemaItem[] = [ + { + label: 'Merger', + count: 8, + properties: [ + { + id: 'p1', + name: 'primary_company', + type: 'string', + values: ['co_ax', 'co_bx'] + }, + { + id: 'p2', + name: 'secondary_company', + type: 'string', + values: ['co_cx', 'co_ex'] + } + ], + relationships: [] + } + ] + + // No deterministic hint — the column names carry no flow direction… + expect(service.buildCandidateHints(pairSchema)).toEqual([]) + // …but an LLM-proposed candidate for the pair survives validation. + const validated = service.validateCandidate( + { + source: { label: 'Merger', key: 'primary_company' }, + target: { label: 'Merger', key: 'secondary_company' }, + direction: 'out', + type: 'MERGED_WITH', + mode: 'join_pattern', + confidence: 0.7 + }, + pairSchema + ) + expect(validated).toBeDefined() + }) + + it('does not pair columns whose only shared name token is a generic reference suffix', () => { + const refSchema: SchemaItem[] = [ + { + label: 'Asset', + count: 8, + properties: [ + { id: 'p1', name: 'owner_ref', type: 'string', values: ['x_1', 'x_2', 'x_3'] }, + { id: 'p2', name: 'region_ref', type: 'string', values: ['x_2', 'x_3', 'x_4'] } + ], + relationships: [] + } + ] + expect(service.buildCandidateHints(refSchema)).toEqual([]) + }) + }) + + // The probe is the evidence gate: a join candidate survives only when the live graph + // actually contains at least one matching record pair. + describe('probeJoinCandidates', () => { + const makeProbeService = (matchCounts: number[]) => { + const transaction = { + commit: jest.fn().mockResolvedValue(undefined), + rollback: jest.fn().mockResolvedValue(undefined), + isOpen: jest.fn().mockReturnValue(false) + } + const countRelationCandidatesByKeys = jest.fn() + for (const count of matchCounts) { + countRelationCandidatesByKeys.mockResolvedValueOnce(count) + } + const probeService = makeService({ + neogmaService: { + createSession: jest.fn().mockReturnValue({ beginTransaction: () => transaction }), + closeSession: jest.fn().mockResolvedValue(undefined) + }, + entityService: { countRelationCandidatesByKeys } + }) + return { probeService, countRelationCandidatesByKeys } + } + + const joinCandidate = (key: string): RelationshipPatternCandidate => ({ + source: { label: 'TRANSACTION', key: 'receiver_account' }, + target: { label: 'TRANSACTION', key }, + direction: 'out', + type: 'FLOWS_TO_ACCOUNT', + mode: 'join_pattern', + confidence: 0.82 + }) + + it('keeps candidates with matching pairs and records the probed count as evidence', async () => { + const { probeService } = makeProbeService([42]) + + const evidenced = await probeService.probeJoinCandidates('project-1', [joinCandidate('sender_account')]) + + expect(evidenced).toEqual([expect.objectContaining({ sampleMatchCount: 42 })]) + }) + + it('drops join candidates the live graph shows no matching pairs for', async () => { + const { probeService } = makeProbeService([0, 7]) + + const evidenced = await probeService.probeJoinCandidates('project-1', [ + joinCandidate('sender_account'), + joinCandidate('sender_name') + ]) + + expect(evidenced).toEqual([ + expect.objectContaining({ + target: expect.objectContaining({ key: 'sender_name' }), + sampleMatchCount: 7 + }) + ]) + }) + + it('passes retype candidates through without probing', async () => { + const { probeService, countRelationCandidatesByKeys } = makeProbeService([1]) + + const retype: RelationshipPatternCandidate = { + source: { label: 'ORDER' }, + target: { label: 'CUSTOMER' }, + direction: 'out', + type: 'PLACED_BY', + mode: 'retype_existing_relationship', + confidence: 0.9 + } + const evidenced = await probeService.probeJoinCandidates('project-1', [ + retype, + joinCandidate('sender_account') + ]) + + expect(evidenced).toHaveLength(2) + expect(countRelationCandidatesByKeys).toHaveBeenCalledTimes(1) + }) }) }) @@ -466,4 +678,22 @@ describe('EntityQueryService relationship creation', () => { ) expect(query).toContain('batchMode: "BATCH_SINGLE"') }) + + it('probes join candidates through the same pair statement with a bounded count', () => { + const query = new EntityQueryService().countRelationCandidatesByKeys({ + sourceLabel: 'TRANSACTION', + sourceKey: 'receiver_account', + targetLabel: 'TRANSACTION', + targetKey: 'sender_account', + limit: 100 + }) + + // Same join semantics as apply: key map, type-strict pair filter, self-exclusion… + expect(query).toContain('apoc.map.fromPairs(collect([joinKey, targets])) AS targetsByKey') + expect(query).toContain('id(s) <> id(t)') + // …but read-only, with cost bounded by the probe limit. + expect(query).toContain('WITH s, t LIMIT 100') + expect(query).toContain('RETURN count(*) AS matchCount') + expect(query).not.toContain('MERGE') + }) }) diff --git a/platform/core/src/core/relationship-patterns/relationship-patterns.service.ts b/platform/core/src/core/relationship-patterns/relationship-patterns.service.ts index f41f2265..02b2f678 100644 --- a/platform/core/src/core/relationship-patterns/relationship-patterns.service.ts +++ b/platform/core/src/core/relationship-patterns/relationship-patterns.service.ts @@ -31,12 +31,11 @@ import type { RelationshipPatternRow } from '@/database/sql/schema/types' const ANALYSIS_DEBOUNCE_MS = 60_000 const MAX_LLM_CANDIDATES = 20 - -type ReferenceValueProfile = { - tokens: Set - maxTokensPerValue: number - hasListValue: boolean -} +// Reference-suffix tokens that never identify the entity a chain flows through. +const CHAIN_GENERIC_NAME_TOKENS = new Set(['id', 'ref', 'key', 'code']) +// Pairs to count per candidate when probing the live graph; one matched pair is +// already real evidence, the cap only bounds probe cost on large graphs. +const PROBE_MATCH_LIMIT = 100 @Injectable() export class RelationshipPatternsService { @@ -365,7 +364,7 @@ export class RelationshipPatternsService { const resolvedWorkspaceId = workspaceId ?? (await this.projectRepository.findById(projectId))?.workspaceId - const existingPatterns = await this.pruneIncoherentJoinSuggestions( + const existingPatterns = await this.pruneOrphanedJoinSuggestions( projectId, await this.repository.findByProjectId(projectId), schema @@ -390,17 +389,22 @@ export class RelationshipPatternsService { return } + // Names propose, the LLM judges, the graph verifies: deterministic name analysis + // only produces *hints* the LLM evaluates alongside the schema — sampled values + // never gate anything (tiny samples of high-cardinality data carry no evidence) — + // and every surviving join is probed against the live graph before it is stored. + const candidateHints = this.buildCandidateHints(schema) const tLlm = Date.now() const { candidates, promptTokens, completionTokens, totalTokens } = await this.suggestCandidates( schema, - existingPatterns + existingPatterns, + candidateHints ) this.logger.log( `[RelationshipAnalysis] project ${projectId} LLM returned ${candidates.length} candidates in ${Date.now() - tLlm}ms` ) - const deterministicCandidates = this.suggestDeterministicCandidates(schema) const validCandidates = this.dedupeInverseCandidates( - [...deterministicCandidates, ...candidates] + candidates .map((candidate) => this.validateCandidate(candidate, schema)) .filter((candidate): candidate is RelationshipPatternCandidate => Boolean(candidate)) ) @@ -408,7 +412,9 @@ export class RelationshipPatternsService { .sort((a, b) => this.scoreCandidate(b) - this.scoreCandidate(a)) .slice(0, MAX_LLM_CANDIDATES) - for (const candidate of validCandidates) { + const evidencedCandidates = await this.probeJoinCandidates(projectId, validCandidates) + + for (const candidate of evidencedCandidates) { const row = this.candidateToInsert(projectId, candidate) await this.repository.upsertCandidate(row) } @@ -419,7 +425,7 @@ export class RelationshipPatternsService { promptTokens, completionTokens, totalTokens, - candidateCount: validCandidates.length, + candidateCount: evidencedCandidates.length, trigger: isManual ? 'manual' : 'scheduler' }) } @@ -456,21 +462,22 @@ export class RelationshipPatternsService { } /** - * Deletes previously stored auto-suggestions that no longer pass the join-coherence - * gate in isSafeJoinCandidate. Only 'suggested' rows are touched; approved/ignored rows - * reflect an explicit user decision and are kept. + * Deletes previously stored auto-suggestions whose endpoints no longer make sense + * against the current schema (label or key gone, or the property type became + * non-joinable). Only 'suggested' rows are touched; approved/ignored rows reflect + * an explicit user decision and are kept. */ - private async pruneIncoherentJoinSuggestions( + private async pruneOrphanedJoinSuggestions( projectId: string, patterns: RelationshipPatternRow[], schema: SchemaItem[] ): Promise { const kept: RelationshipPatternRow[] = [] for (const pattern of patterns) { - const incoherent = + const orphaned = pattern.status === 'suggested' && this.normalizePatternMode(pattern.mode) === 'join_pattern' && - !this.isSafeJoinCandidate( + !this.isPlausibleJoinCandidate( { source: { label: pattern.sourceLabel, key: pattern.sourceKey ?? undefined }, target: { label: pattern.targetLabel, key: pattern.targetKey ?? undefined } @@ -478,10 +485,10 @@ export class RelationshipPatternsService { schema ) - if (incoherent) { + if (orphaned) { await this.repository.deletePattern(pattern.id, projectId) this.logger.log( - `[RelationshipAnalysis] pruned incoherent join suggestion ${pattern.id}: ` + + `[RelationshipAnalysis] pruned orphaned join suggestion ${pattern.id}: ` + `${pattern.sourceLabel}.${pattern.sourceKey} -> ${pattern.targetLabel}.${pattern.targetKey} (${pattern.type})` ) } else { @@ -491,10 +498,17 @@ export class RelationshipPatternsService { return kept } - private suggestDeterministicCandidates(schema: SchemaItem[]): RelationshipPatternCandidate[] { + /** + * Name-derived join hypotheses handed to the LLM for semantic vetting. Built from + * the schema's structure only (labels, property names, types) — never from sampled + * values, whose overlap carries no signal on high-cardinality real data. The LLM + * decides which hints reference an entity (vs. attribute coincidences like paired + * country columns), and the graph probe then supplies the actual evidence. + */ + private buildCandidateHints(schema: SchemaItem[]): RelationshipPatternCandidate[] { return [ ...this.suggestNameBackedReferenceCandidates(schema), - ...this.suggestSampleBackedReferenceCandidates(schema) + ...this.suggestSameLabelChainCandidates(schema) ] } @@ -516,10 +530,6 @@ export class RelationshipPatternsService { continue } - const sourceValues = this.referenceValueProfile(sourceProperty) - const targetValues = this.referenceValueProfile(targetProperty) - const overlap = this.countTokenOverlap(sourceValues.tokens, targetValues.tokens) - candidates.push({ source: { label: source.label, key: sourceProperty.name }, target: { label: target.label, key: targetProperty.name }, @@ -527,7 +537,6 @@ export class RelationshipPatternsService { type: this.referencePropertyRelationshipType(sourceProperty.name), mode: 'join_pattern', confidence: sourceProperty.isArray ? 0.9 : 0.86, - sampleMatchCount: overlap, rationale: `${source.label}.${sourceProperty.name} names ${target.label}.${targetProperty.name} as a reference field.` }) } @@ -538,24 +547,24 @@ export class RelationshipPatternsService { return candidates } - private suggestSampleBackedReferenceCandidates(schema: SchemaItem[]): RelationshipPatternCandidate[] { + // Same-label chain/flow detection: two reference-like properties on ONE label that + // identify the same kind of entity (receiver_account / sender_account, to_station / + // from_station) chain records into a sequence. Detected purely from property names — + // shared entity token plus opposing direction words — since sampled values from + // high-cardinality columns (thousands of accounts) practically never overlap. + private suggestSameLabelChainCandidates(schema: SchemaItem[]): RelationshipPatternCandidate[] { const candidates: RelationshipPatternCandidate[] = [] - for (let sourceIndex = 0; sourceIndex < schema.length; sourceIndex += 1) { - const left = schema[sourceIndex] - for (let targetIndex = sourceIndex; targetIndex < schema.length; targetIndex += 1) { - const right = schema[targetIndex] - - for (const leftProperty of left.properties) { - for (const rightProperty of right.properties) { - if (left.label === right.label && leftProperty.name === rightProperty.name) { - continue - } - - const candidate = this.buildSampleBackedCandidate(left, leftProperty, right, rightProperty) - if (candidate) { - candidates.push(candidate) - } + for (const item of schema) { + for (let leftIndex = 0; leftIndex < item.properties.length; leftIndex += 1) { + for (let rightIndex = leftIndex + 1; rightIndex < item.properties.length; rightIndex += 1) { + const candidate = this.buildSameLabelChainCandidate( + item, + item.properties[leftIndex], + item.properties[rightIndex] + ) + if (candidate) { + candidates.push(candidate) } } } @@ -564,45 +573,119 @@ export class RelationshipPatternsService { return candidates } - private buildSampleBackedCandidate( - left: SchemaItem, + private buildSameLabelChainCandidate( + item: SchemaItem, leftProperty: SchemaItem['properties'][number], - right: SchemaItem, rightProperty: SchemaItem['properties'][number] ): RelationshipPatternCandidate | undefined { - const leftValues = this.referenceValueProfile(leftProperty) - const rightValues = this.referenceValueProfile(rightProperty) - const overlap = this.countTokenOverlap(leftValues.tokens, rightValues.tokens) - - if ( - overlap === 0 || - !this.hasReferenceLikeOverlap(leftProperty, leftValues, rightProperty, rightValues, overlap) - ) { + if (!this.isSameLabelChainPair(leftProperty, rightProperty)) { return undefined } - const leftIsBetterSource = - leftValues.maxTokensPerValue > rightValues.maxTokensPerValue || - (leftValues.maxTokensPerValue === rightValues.maxTokensPerValue && - leftValues.tokens.size > rightValues.tokens.size) || - (leftValues.maxTokensPerValue === rightValues.maxTokensPerValue && - leftValues.tokens.size === rightValues.tokens.size && - `${left.label}.${leftProperty.name}` <= `${right.label}.${rightProperty.name}`) - const source = leftIsBetterSource ? left : right - const sourceProperty = leftIsBetterSource ? leftProperty : rightProperty - const target = leftIsBetterSource ? right : left - const targetProperty = leftIsBetterSource ? rightProperty : leftProperty + // A chain hint is only emitted when the column names disambiguate flow direction; + // ambiguous pairs are left to the LLM pass (isPlausibleJoinCandidate still admits + // them there). Edge semantics: the record whose destination-side value is X points + // at the record whose origin-side value is X — (hop N)-[:FLOW]->(hop N+1). + const leftRole = this.chainDirectionRole(leftProperty.name) + const rightRole = this.chainDirectionRole(rightProperty.name) + if (!leftRole || !rightRole || leftRole === rightRole) { + return undefined + } + + const sourceProperty = leftRole === 'destination' ? leftProperty : rightProperty + const targetProperty = leftRole === 'destination' ? rightProperty : leftProperty + const sharedTokens = [...this.nameTokens(sourceProperty.name)] + .filter((token) => this.nameTokens(targetProperty.name).has(token)) + .sort() return { - source: { label: source.label, key: sourceProperty.name }, - target: { label: target.label, key: targetProperty.name }, + source: { label: item.label, key: sourceProperty.name }, + target: { label: item.label, key: targetProperty.name }, direction: 'out', - type: this.referencePropertyRelationshipType(sourceProperty.name), + type: this.chainRelationshipType(sharedTokens), mode: 'join_pattern', - confidence: Math.min(0.95, 0.75 + overlap / 100), - sampleMatchCount: overlap, - rationale: `${source.label}.${sourceProperty.name} and ${target.label}.${targetProperty.name} share ${overlap} sampled value${overlap === 1 ? '' : 's'}.` + confidence: 0.82, + rationale: + `${item.label}.${sourceProperty.name} and ${item.label}.${targetProperty.name} appear to identify ` + + `the same kind of entity — records chain into a flow where one record's ` + + `${sourceProperty.name} is the next record's ${targetProperty.name}.` + } + } + + private isSameLabelChainPair( + leftProperty: SchemaItem['properties'][number], + rightProperty: SchemaItem['properties'][number] + ): boolean { + if (leftProperty.name === rightProperty.name) { + return false + } + if (this.isNonReferenceProperty(leftProperty) || this.isNonReferenceProperty(rightProperty)) { + return false + } + if (leftProperty.isArray || rightProperty.isArray) { + return false + } + + // The two columns must talk about the same entity: their names share a token + // (receiver_ACCOUNT / sender_ACCOUNT) beyond the direction word — generic reference + // suffixes (id/ref/key/code) don't count, or ownerRef/regionRef would pair up. + const leftTokens = this.nameTokens(leftProperty.name) + return [...this.nameTokens(rightProperty.name)].some( + (token) => leftTokens.has(token) && !CHAIN_GENERIC_NAME_TOKENS.has(token) + ) + } + + private chainDirectionRole(propertyName: string): 'destination' | 'origin' | undefined { + // Tokens as emitted by nameTokens (lowercased, crudely singularized). + const DESTINATION_TOKENS = new Set([ + 'to', + 'receiver', + 'recipient', + 'dest', + 'destination', + 'next', + 'child', + 'successor', + 'output', + 'sink', + 'head' + ]) + const ORIGIN_TOKENS = new Set([ + 'from', + 'sender', + 'origin', + 'prev', + 'previou', + 'parent', + 'predecessor', + 'input', + 'tail' + ]) + + let role: 'destination' | 'origin' | undefined + for (const token of this.nameTokens(propertyName)) { + const tokenRole = + DESTINATION_TOKENS.has(token) ? ('destination' as const) + : ORIGIN_TOKENS.has(token) ? ('origin' as const) + : undefined + if (!tokenRole) { + continue + } + if (role && role !== tokenRole) { + return undefined + } + role = tokenRole } + return role + } + + private chainRelationshipType(sharedTokens: string[]): string { + const entity = sharedTokens + .filter((token) => !CHAIN_GENERIC_NAME_TOKENS.has(token)) + .map((token) => token.toUpperCase()) + .join('_') + + return entity ? `FLOWS_TO_${entity}` : 'FLOWS_TO' } private referencePropertyRelationshipType(propertyName: string): string { @@ -622,7 +705,8 @@ export class RelationshipPatternsService { private async suggestCandidates( schema: SchemaItem[], - existingPatterns: RelationshipPatternRow[] + existingPatterns: RelationshipPatternRow[], + candidateHints: RelationshipPatternCandidate[] = [] ): Promise<{ candidates: RelationshipPatternCandidate[] promptTokens: number @@ -638,6 +722,40 @@ export class RelationshipPatternsService { return empty } + const systemContent = + 'You infer graph relationship patterns for RushDB. Return only JSON: {"candidates":[...]}. ' + + 'Every candidate must include mode: "join_pattern" or "retype_existing_relationship". ' + + 'Judge joinability from label semantics, property names, and property types — the schema is your substrate. ' + + 'Sampled property values, when present, are illustrative only: they are tiny samples of potentially huge datasets, so a lack of overlap between samples is NEVER evidence against a join. ' + + 'Every join_pattern you return is verified against the live graph before it is suggested to the user, so propose every join you judge semantically correct. ' + + 'Only propose joins between properties that reference an entity (identifiers, names of things, reference fields). Never join mere attribute columns (statuses, countries, currencies, categories) whose values may coincide without referencing an entity. ' + + 'candidateHints in the user message are join hypotheses derived mechanically from property names. Evaluate each hint: include it (refining type and direction if needed) when it references an entity; omit it when the shared naming is an attribute coincidence rather than an entity reference. ' + + 'Array/list fields and comma-separated reference fields may join scalar fields. ' + + 'Return at most ONE join_pattern per label pair; if several key pairs could join the same two labels, pick the single most semantically correct one. ' + + 'Use "retype_existing_relationship" when schema already shows a RUSHDB_DEFAULT_RELATION between labels; in that case source.key and target.key are optional and the task is to rename existing structure semantically. ' + + 'For retype_existing_relationship, infer the semantic relationship from the existing graph structure and label meanings. ' + + 'If a default relationship already connects two labels, prefer retype_existing_relationship over join_pattern. ' + + 'Return ONE canonical candidate per semantic relationship; never return both A->B and B->A for the same relationship. ' + + 'Do not suggest a relationship when the same mode and label/key pair already appears in existingPatterns, even if you would use a different synonym or inverse type. ' + + 'Choose source as the natural actor/owner/parent and target as the natural object/action/child. ' + + 'For same-label relationships, never join a property to itself. Choose exactly one canonical orientation from the schema context. ' + + 'Same-label chain/flow patterns are valid join_patterns: when two different reference-like properties on ONE label identify the same kind of entity (e.g. receiver_account of one record equals sender_account of the next; to_station / from_station), suggest source.label === target.label with source.key = the destination-side property and target.key = the origin-side property, chaining records into a sequence. ' + + 'Relationship type must read naturally from source to target. Do not choose a direction where the target would appear to act on or create the source. ' + + 'Each candidate must include source.label, target.label, mode, direction "out", type, confidence 0..1, and rationale. join_pattern must also include source.key and target.key. Return at most 12 candidates.' + const userContent = JSON.stringify({ + schema, + existingPatterns: this.compactExistingPatternsForRelationshipAnalysis(existingPatterns), + candidateHints: candidateHints.map((hint) => ({ + source: hint.source, + target: hint.target, + direction: hint.direction, + type: hint.type, + rationale: hint.rationale + })), + relationshipTypeRules: + 'Use uppercase Neo4j-safe verb phrases from source to target. Do not use inverse duplicate types for the same relationship.' + }) + const response = await axios.post( `${baseUrl.replace(/\/$/, '')}/chat/completions`, { @@ -645,33 +763,8 @@ export class RelationshipPatternsService { temperature: 0, response_format: { type: 'json_object' }, messages: [ - { - role: 'system', - content: - 'You infer graph relationship patterns for RushDB. Return only JSON: {"candidates":[...]}. ' + - 'Every candidate must include mode: "join_pattern" or "retype_existing_relationship". ' + - 'Use "join_pattern" only when sampled property values show that records can actually be matched. Shared property names or semantic-looking key names alone are not enough. ' + - 'Array/list fields and comma-separated reference fields may join scalar fields when their sampled items overlap. ' + - 'Return at most ONE join_pattern per label pair; if several key pairs could join the same two labels, pick the single most semantically correct one. ' + - 'Use "retype_existing_relationship" when schema already shows a RUSHDB_DEFAULT_RELATION between labels; in that case source.key and target.key are optional and the task is to rename existing structure semantically. ' + - 'For retype_existing_relationship, infer the semantic relationship from the existing graph structure and label meanings. ' + - 'If a default relationship already connects two labels, prefer retype_existing_relationship over join_pattern. ' + - 'Return ONE canonical candidate per semantic relationship; never return both A->B and B->A for the same relationship. ' + - 'Do not suggest a relationship when the same mode and label/key pair already appears in existingPatterns, even if you would use a different synonym or inverse type. ' + - 'Choose source as the natural actor/owner/parent and target as the natural object/action/child. ' + - 'For same-label relationships, never join a property to itself. Choose exactly one canonical orientation from sampled values and schema context. ' + - 'Relationship type must read naturally from source to target. Do not choose a direction where the target would appear to act on or create the source. ' + - 'Each candidate must include source.label, target.label, mode, direction "out", type, confidence 0..1, and rationale. join_pattern must also include source.key and target.key. Return at most 8 candidates.' - }, - { - role: 'user', - content: JSON.stringify({ - schema, - existingPatterns: this.compactExistingPatternsForRelationshipAnalysis(existingPatterns), - relationshipTypeRules: - 'Use uppercase Neo4j-safe verb phrases from source to target. Do not use inverse duplicate types for the same relationship.' - }) - } + { role: 'system', content: systemContent }, + { role: 'user', content: userContent } ] }, { @@ -689,12 +782,6 @@ export class RelationshipPatternsService { } const usage = response.data?.usage - const systemContent = - 'You infer graph relationship patterns for RushDB. Return only JSON: {"candidates":[...]}. ' - const userContent = JSON.stringify({ - schema, - existingPatterns: this.compactExistingPatternsForRelationshipAnalysis(existingPatterns) - }) const promptTokens = usage?.prompt_tokens ?? estimateTokens(systemContent + userContent) const completionTokens = usage?.completion_tokens ?? estimateTokens(content) const totalTokens = usage?.total_tokens ?? promptTokens + completionTokens @@ -838,7 +925,7 @@ export class RelationshipPatternsService { const sourceHasKey = source.properties.some((property) => property.name === candidate.source.key) const targetHasKey = target.properties.some((property) => property.name === candidate.target.key) - if (!sourceHasKey || !targetHasKey || !this.isSafeJoinCandidate(candidate, schema)) { + if (!sourceHasKey || !targetHasKey || !this.isPlausibleJoinCandidate(candidate, schema)) { return undefined } } @@ -888,29 +975,94 @@ export class RelationshipPatternsService { ) } - private isSafeJoinCandidate(candidate: RelationshipPatternCandidate, schema: SchemaItem[] = []): boolean { + /** + * Structural plausibility only — the semantic judgment belongs to the LLM and the + * factual evidence to the graph probe. Rejects joins that can never be meaningful + * regardless of data: a property joined to itself, missing properties, non-reference + * value types, and list-to-list joins. + */ + private isPlausibleJoinCandidate( + candidate: RelationshipPatternCandidate, + schema: SchemaItem[] = [] + ): boolean { if (candidate.source.label === candidate.target.label && candidate.source.key === candidate.target.key) { return false } - const sourceKey = candidate.source.key ?? '' - const targetKey = candidate.target.key ?? '' - - const sourceProperty = this.findProperty(schema, candidate.source.label, sourceKey) - const targetProperty = this.findProperty(schema, candidate.target.label, targetKey) + const sourceProperty = this.findProperty(schema, candidate.source.label, candidate.source.key ?? '') + const targetProperty = this.findProperty(schema, candidate.target.label, candidate.target.key ?? '') if (!sourceProperty || !targetProperty) { return false } - const sourceTokens = this.referenceValueProfile(sourceProperty) - const targetTokens = this.referenceValueProfile(targetProperty) - const overlap = this.countTokenOverlap(sourceTokens.tokens, targetTokens.tokens) + if (this.isNonReferenceProperty(sourceProperty) || this.isNonReferenceProperty(targetProperty)) { + return false + } - return ( - this.hasNameBackedReference(sourceProperty, candidate.target.label, targetProperty) || - (overlap > 0 && - this.hasReferenceLikeOverlap(sourceProperty, sourceTokens, targetProperty, targetTokens, overlap)) - ) + return !(sourceProperty.isArray && targetProperty.isArray) + } + + /** + * Verifies join candidates against the live graph: each join_pattern is kept only + * when at least one real record pair matches its join, and sampleMatchCount is set + * to the probed pair count (capped at PROBE_MATCH_LIMIT). This is the evidence + * gate — schema samples never decide, the actual data does. Retype candidates pass + * through untouched: their evidence is the existing relationship in the schema. + */ + private async probeJoinCandidates( + projectId: string, + candidates: RelationshipPatternCandidate[] + ): Promise { + if (!candidates.some((candidate) => this.normalizePatternMode(candidate.mode) === 'join_pattern')) { + return candidates + } + + const evidenced: RelationshipPatternCandidate[] = [] + const session = this.neogmaService.createSession('relationship-pattern-probe') + const transaction = session.beginTransaction({ timeout: 60_000 }) + try { + for (const candidate of candidates) { + if (this.normalizePatternMode(candidate.mode) !== 'join_pattern') { + evidenced.push(candidate) + continue + } + + try { + const matchCount = await this.entityService.countRelationCandidatesByKeys({ + source: candidate.source, + target: candidate.target, + projectId, + transaction, + limit: PROBE_MATCH_LIMIT + }) + + if (matchCount > 0) { + evidenced.push({ ...candidate, sampleMatchCount: matchCount }) + } else { + this.logger.log( + `[RelationshipAnalysis] probe found no matching pairs for ` + + `${candidate.source.label}.${candidate.source.key} -> ` + + `${candidate.target.label}.${candidate.target.key} (${candidate.type}) — dropped` + ) + } + } catch (error) { + // No evidence gathered — drop rather than suggest unverified structure. + this.logger.error( + `[RelationshipAnalysis] probe failed for ${candidate.source.label}.${candidate.source.key} -> ` + + `${candidate.target.label}.${candidate.target.key}`, + error + ) + } + } + await transaction.commit() + } finally { + if (transaction.isOpen()) { + await transaction.rollback() + } + await this.neogmaService.closeSession(session, 'relationship-pattern-probe') + } + + return evidenced } private calibrateConfidence( @@ -929,20 +1081,6 @@ export class RelationshipPatternsService { return schema.find((item) => item.label === label)?.properties.find((property) => property.name === key) } - private hasNameBackedReference( - referenceProperty: SchemaItem['properties'][number], - targetLabel: string, - targetProperty: SchemaItem['properties'][number] - ): boolean { - if (this.isNonReferenceProperty(referenceProperty) || this.isNonReferenceProperty(targetProperty)) { - return false - } - if (targetProperty.isArray) { - return false - } - return this.propertyNameReferencesLabelAndKey(referenceProperty.name, targetLabel, targetProperty.name) - } - private propertyNameReferencesLabelAndKey( referencePropertyName: string, targetLabel: string, @@ -977,109 +1115,10 @@ export class RelationshipPatternsService { ) } - private hasReferenceLikeOverlap( - leftProperty: SchemaItem['properties'][number], - left: ReferenceValueProfile, - rightProperty: SchemaItem['properties'][number], - right: ReferenceValueProfile, - overlap: number - ): boolean { - if (this.isNonReferenceProperty(leftProperty) || this.isNonReferenceProperty(rightProperty)) { - return false - } - - const leftIsCollection = this.isCollectionReferenceProfile(left) - const rightIsCollection = this.isCollectionReferenceProfile(right) - - if (leftIsCollection && rightIsCollection) { - return false - } - - if (leftIsCollection || rightIsCollection) { - return true - } - - const hasStrictSubsetOverlap = this.isStrictSubsetOverlap(left, right, overlap) - - if ((this.isLowSignalEnum(left) || this.isLowSignalEnum(right)) && !hasStrictSubsetOverlap) { - return false - } - - return hasStrictSubsetOverlap - } - private isNonReferenceProperty(property: SchemaItem['properties'][number]): boolean { return property.type === 'boolean' || property.type === 'datetime' } - private isLowSignalEnum(profile: ReferenceValueProfile): boolean { - return profile.maxTokensPerValue <= 1 && profile.tokens.size <= 3 - } - - private isCollectionReferenceProfile(profile: ReferenceValueProfile): boolean { - return profile.hasListValue || profile.maxTokensPerValue > 1 - } - - private isStrictSubsetOverlap( - left: ReferenceValueProfile, - right: ReferenceValueProfile, - overlap: number - ): boolean { - return overlap === Math.min(left.tokens.size, right.tokens.size) && left.tokens.size !== right.tokens.size - } - - private referenceValueProfile(property: SchemaItem['properties'][number]): ReferenceValueProfile { - const profile = this.referenceValueTokens(property.values) - return { - ...profile, - hasListValue: profile.hasListValue || property.isArray === true - } - } - - private referenceValueTokens(values: unknown): ReferenceValueProfile { - const tokens = new Set() - let maxTokensPerValue = 0 - let hasListValue = false - if (!Array.isArray(values)) { - return { tokens, maxTokensPerValue, hasListValue } - } - - for (const rawValue of values) { - const rawValueIsArray = Array.isArray(rawValue) - const rawParts = rawValueIsArray ? rawValue : String(rawValue ?? '').split(',') - const parts = rawParts - .map((part) => - String(part ?? '') - .trim() - .toLowerCase() - ) - .filter(Boolean) - - if (!parts.length) { - continue - } - hasListValue = hasListValue || rawValueIsArray - let tokensInValue = 0 - for (const token of parts) { - tokens.add(token) - tokensInValue += 1 - } - maxTokensPerValue = Math.max(maxTokensPerValue, tokensInValue) - } - - return { tokens, maxTokensPerValue, hasListValue } - } - - private countTokenOverlap(left: Set, right: Set): number { - let overlaps = 0 - for (const value of left) { - if (right.has(value)) { - overlaps += 1 - } - } - return overlaps - } - private normalizeEndpoint(endpoint: RelationshipPatternEndpoint): RelationshipPatternEndpoint { return { label: String(endpoint.label).trim(), diff --git a/platform/core/src/core/search/parser/buildQuery.ts b/platform/core/src/core/search/parser/buildQuery.ts index 38769185..5ae0cf93 100644 --- a/platform/core/src/core/search/parser/buildQuery.ts +++ b/platform/core/src/core/search/parser/buildQuery.ts @@ -85,7 +85,8 @@ export const parseWhereClause = ( aliasesMap, level: 0, result: { [options.nodeAlias]: '' }, - withQueryQueue: { [options.nodeAlias]: [] } + withQueryQueue: { [options.nodeAlias]: [] }, + cycleExistsByLevel: {} } parseLevel('', normalizedInput, options, ctx) diff --git a/platform/core/src/core/search/parser/buildWhereClause.ts b/platform/core/src/core/search/parser/buildWhereClause.ts index 0313f451..fc9e7ef4 100644 --- a/platform/core/src/core/search/parser/buildWhereClause.ts +++ b/platform/core/src/core/search/parser/buildWhereClause.ts @@ -4,7 +4,7 @@ import { toBoolean } from '@/common/utils/toBolean' import { ROOT_RECORD_ALIAS } from '@/core/common/constants' import { Where } from '@/core/common/types' import { ParseContext } from '@/core/search/parser/types' -import { splitCriteria, traversalRelsVar, wrapInParentheses } from '@/core/search/parser/utils' +import { isCycleOperatorKey, splitCriteria, wrapInParentheses } from '@/core/search/parser/utils' import { TSearchQueryBuilderOptions } from '@/core/search/search.types' const parseCurrentLevel = (input: Where, options?: TSearchQueryBuilderOptions, ctx?: ParseContext) => { @@ -64,7 +64,7 @@ const parseCurrentLevel = (input: Where, options?: TSearchQueryBuilderOptions, c const { currentLevel, subQueries } = splitCriteria(input) // SUB QUERY PROCESSING if (toBoolean(subQueries)) { - return Object.entries(subQueries).map(([k, value]) => parseSubQuery(value, options, ctx)) + return Object.entries(subQueries).map(([k, value]) => parseSubQuery(k, value, options, ctx)) } else { return Object.entries(currentLevel).map(([k, v]) => { switch (k) { @@ -134,9 +134,12 @@ const parseCurrentLevel = (input: Where, options?: TSearchQueryBuilderOptions, c ctx ) - const condition = conditionRaw?.filter?.(toBoolean) + // Array input (e.g. $and: [{...}, {...}]) makes parseCurrentLevel return a + // pre-joined string — pass it through like the other operators do, instead + // of dropping it via a filter() that only exists on arrays. + const condition = isArray(conditionRaw) ? conditionRaw.filter(toBoolean) : conditionRaw - if (condition) { + if (isArray(condition) ? condition.length : toBoolean(condition)) { return wrapInParentheses(isArray(condition) ? condition.join(' AND ') : condition) } break @@ -150,17 +153,18 @@ const parseCurrentLevel = (input: Where, options?: TSearchQueryBuilderOptions, c } } -const parseSubQuery = (input: any, options?: TSearchQueryBuilderOptions, ctx?: ParseContext) => { - const { $relation, $alias, $cycle, ...other } = input as any - +const parseSubQuery = (key: string, input: any, options?: TSearchQueryBuilderOptions, ctx?: ParseContext) => { ctx.level += 1 - if (toBoolean($cycle)) { - // Cycle blocks bind no endpoint node — existence is carried by the rel-list - // variable. No recursion: Pass 1 rejects children, so level lockstep holds. - return `${traversalRelsVar(ctx.level)} IS NOT NULL` + if (isCycleOperatorKey(key)) { + // $cycle binds no endpoint node — Pass 1 compiled it to an EXISTS subquery + // predicate (short-circuits at the first cycle found per record) stored by level. + // No recursion: the operator's value is a traversal spec, so level lockstep holds. + return ctx.cycleExistsByLevel[ctx.level] } + const { $relation, $alias, ...other } = input as any + const nodeAlias = ROOT_RECORD_ALIAS + ctx.level const result = parseCurrentLevel(other, options, ctx) diff --git a/platform/core/src/core/search/parser/parseSubQuery.ts b/platform/core/src/core/search/parser/parseSubQuery.ts index 923f2d2c..8e4daea3 100644 --- a/platform/core/src/core/search/parser/parseSubQuery.ts +++ b/platform/core/src/core/search/parser/parseSubQuery.ts @@ -6,7 +6,7 @@ import { Relation, TraversalHops, Where } from '@/core/common/types' import { parseLevel } from '@/core/search/parser/buildQuery' import { TraversalQueryError } from '@/core/search/parser/errors' import { ParseContext } from '@/core/search/parser/types' -import { traversalRelsVar } from '@/core/search/parser/utils' +import { isCycleOperatorKey, traversalRelsVar } from '@/core/search/parser/utils' import { resolveMaxTraversalHops } from '@/core/search/search.constants' import { TSearchQueryBuilderOptions } from '@/core/search/search.types' @@ -23,7 +23,7 @@ const normalizeHops = ( if (hops === undefined) { if (isCycle) { throw new TraversalQueryError( - `'$cycle' requires '$relation' with 'hops', e.g. { "$relation": { "hops": { "min": 2, "max": 6 } } }.` + `'$cycle' requires 'hops', e.g. { "$cycle": { "type": "FLOWS_TO", "direction": "out", "hops": { "min": 2, "max": 6 } } }.` ) } return null @@ -100,7 +100,7 @@ const buildRelationPart = ( if (isCycle && !isObject(relation)) { throw new TraversalQueryError( - `'$cycle' requires '$relation' with 'hops', e.g. { "$relation": { "hops": { "min": 2, "max": 6 } } }.` + `'$cycle' requires a traversal spec with 'hops', e.g. { "$cycle": { "type": "FLOWS_TO", "direction": "out", "hops": { "min": 2, "max": 6 } } }.` ) } @@ -117,9 +117,10 @@ const buildRelationPart = ( } const hops = normalizeHops(relation, isCycle, maxHops) - // Variable-length patterns bind the relationship list when the untyped filter or - // the cycle existence check (`relsN IS NOT NULL`) needs to reference it. - const bindVar = hops !== null && (isCycle || !relation.type) + // Variable-length patterns bind the relationship list only when the untyped + // VALUE-edge filter needs to reference it; cycles compile to EXISTS subqueries + // and carry no variable out of the pattern. + const bindVar = hops !== null && !relation.type const needsValueEdgeFilter = hops !== null && !relation.type const inner = `${bindVar ? relVar : ''}${relation.type ? `:${relation.type}` : ''}${renderHops(hops)}` @@ -153,40 +154,33 @@ export const parseSubQuery = ( options?: TSearchQueryBuilderOptions, ctx?: ParseContext ) => { - const { $relation, $alias, $cycle, ...other } = input as any - ctx.level += 1 // Capture the level before parseLevel recursion mutates ctx.level. const level = ctx.level const maxHops = options?.maxHops ?? resolveMaxTraversalHops() - if (toBoolean($cycle)) { - if ($alias) { - throw new TraversalQueryError(`'$cycle' does not accept '$alias': a cycle has no separate endpoint.`) - } - if (Object.keys(other).length) { - throw new TraversalQueryError( - `'$cycle' accepts only '$relation': criteria and nested labels have no endpoint to anchor to, got ${JSON.stringify(Object.keys(other))}.` - ) - } - + if (isCycleOperatorKey(key)) { + // { $cycle: { type?, direction, hops } } — `input` IS the traversal spec; a cycle + // has no endpoint, so there is no $alias, no criteria, no nested labels. const relVar = traversalRelsVar(level) - const { pattern, needsValueEdgeFilter } = buildRelationPart($relation, relVar, true, maxHops) + const { pattern, needsValueEdgeFilter } = buildRelationPart(input as Relation, relVar, true, maxHops) const filter = needsValueEdgeFilter ? valueEdgeFilter(relVar) : '' - // The rel var takes the place of a node alias: the existence check in the final - // `WITH ... WHERE` is `relsN IS NOT NULL`, and WITH is a projection barrier. - ctx.nodeAliases.push(relVar) - - // Keyed by the level's record slot purely for sortQueryParts ordering; both - // pattern endpoints are the parent alias — the block's label key is ignored. - ctx.result[ROOT_RECORD_ALIAS + level] = - `OPTIONAL MATCH (${options.nodeAlias})${pattern}(${options.nodeAlias})${ - filter ? ` WHERE ${filter}` : '' - }` + // $cycle is a pure existence predicate: it binds no endpoint and nothing downstream + // ever references the path. Compiling it to an EXISTS subquery lets Neo4j stop at + // the FIRST cycle found per record — an OPTIONAL MATCH would enumerate every + // relationship-unique path (branching^hops rows on dense graphs) only to null-check + // the list afterwards. The predicate is stored for buildWhereClause (Pass 2), which + // walks levels in lockstep and inlines it into the boolean tree, so $or/$not + // composition keeps working. + ctx.cycleExistsByLevel[level] = `EXISTS { MATCH (${options.nodeAlias})${pattern}(${options.nodeAlias})${ + filter ? ` WHERE ${filter}` : '' + } }` return } + const { $relation, $alias, ...other } = input as any + const nodeAlias = ROOT_RECORD_ALIAS + level ctx.nodeAliases.push(nodeAlias) diff --git a/platform/core/src/core/search/parser/tests/buildCompleteQuery.spec.ts b/platform/core/src/core/search/parser/tests/buildCompleteQuery.spec.ts index 30555120..31a87dbd 100644 --- a/platform/core/src/core/search/parser/tests/buildCompleteQuery.spec.ts +++ b/platform/core/src/core/search/parser/tests/buildCompleteQuery.spec.ts @@ -1621,7 +1621,7 @@ RETURN DISTINCT record {.*, __RUSHDB__KEY__LABEL__: [label IN labels(record) WHE searchQuery: { labels: ['ACCOUNT'], where: { - RING: { $cycle: true, $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { max: 6 } } } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { max: 6 } } } } }) @@ -1629,8 +1629,7 @@ RETURN DISTINCT record {.*, __RUSHDB__KEY__LABEL__: [label IN labels(record) WHE expect(result) .toEqual(`MATCH (record:__RUSHDB__LABEL__RECORD__:\`ACCOUNT\` { __RUSHDB__KEY__PROJECT__ID__: $projectId }) ORDER BY record.\`__RUSHDB__KEY__ID__\` DESC SKIP 0 LIMIT 100 -OPTIONAL MATCH (record)-[rels1:TRANSFERRED_TO*2..6]->(record) -WITH record, rels1 WHERE record IS NOT NULL AND rels1 IS NOT NULL +WITH record WHERE record IS NOT NULL AND EXISTS { MATCH (record)-[:TRANSFERRED_TO*2..6]->(record) } RETURN DISTINCT record {.*, __RUSHDB__KEY__LABEL__: [label IN labels(record) WHERE label <> "__RUSHDB__LABEL__RECORD__"][0]} AS records`) }) @@ -1639,16 +1638,14 @@ RETURN DISTINCT record {.*, __RUSHDB__KEY__LABEL__: [label IN labels(record) WHE searchQuery: { labels: ['ACCOUNT'], where: { - RING: { $cycle: true, $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { max: 6 } } } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { max: 6 } } } } }) expect(result) .toEqual(`MATCH (record:__RUSHDB__LABEL__RECORD__:\`ACCOUNT\` { __RUSHDB__KEY__PROJECT__ID__: $projectId }) - -OPTIONAL MATCH (record)-[rels1:TRANSFERRED_TO*2..6]->(record) -WITH record, rels1 WHERE record IS NOT NULL AND rels1 IS NOT NULL +WITH record WHERE record IS NOT NULL AND EXISTS { MATCH (record)-[:TRANSFERRED_TO*2..6]->(record) } RETURN count(DISTINCT record) as total`) }) }) diff --git a/platform/core/src/core/search/parser/tests/parseQuery.spec.ts b/platform/core/src/core/search/parser/tests/parseQuery.spec.ts index afe078fc..00961660 100644 --- a/platform/core/src/core/search/parser/tests/parseQuery.spec.ts +++ b/platform/core/src/core/search/parser/tests/parseQuery.spec.ts @@ -689,56 +689,45 @@ describe('variable-length traversal ($relation.hops)', () => { }) }) +// { $cycle: { type?, direction, hops } } at record level — the value IS the traversal +// spec. A cycle has no endpoint, so there is no key, no $relation wrapper, no $alias. describe('cycle detection ($cycle)', () => { const options = (): TSearchQueryBuilderOptions => ({ nodeAlias: 'record' }) - it('binds both pattern endpoints to the parent for a typed cycle', () => { + it('compiles a typed cycle to an EXISTS predicate anchored to the parent', () => { const result = parseWhereClause( - { - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { max: 6 } } - } - }, + { $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { max: 6 } } }, options() ) + // No OPTIONAL MATCH part and no bound rel var: EXISTS short-circuits at the first + // cycle found per record instead of enumerating every path and null-checking after. expect(result).toEqual({ aliasesMap: { $record: 'record' }, - nodeAliases: ['record', 'rels1'], + nodeAliases: ['record'], queryParts: { - record: '', - record1: 'OPTIONAL MATCH (record)-[rels1:TRANSFERRED_TO*2..6]->(record)' + record: '' }, - where: 'record IS NOT NULL AND rels1 IS NOT NULL' + where: 'record IS NOT NULL AND EXISTS { MATCH (record)-[:TRANSFERRED_TO*2..6]->(record) }' }) }) - it('adds the VALUE-edge filter for an untyped cycle', () => { - const result = parseWhereClause( - { - RING: { - $cycle: true, - $relation: { direction: 'out', hops: { max: 6 } } - } - }, - options() - ) + it('adds the VALUE-edge filter inside EXISTS for an untyped cycle', () => { + const result = parseWhereClause({ $cycle: { direction: 'out', hops: { max: 6 } } }, options()) - expect(result.queryParts.record1).toEqual( - "OPTIONAL MATCH (record)-[rels1*2..6]->(record) WHERE all(r IN rels1 WHERE type(r) <> '__RUSHDB__RELATION__VALUE__')" + expect(result.where).toEqual( + 'record IS NOT NULL AND ' + + "EXISTS { MATCH (record)-[rels1*2..6]->(record) WHERE all(r IN rels1 WHERE type(r) <> '__RUSHDB__RELATION__VALUE__') }" ) + expect(result.queryParts).toEqual({ record: '' }) }) - it('anchors a cycle to a nested parent and keeps its criteria', () => { + it('keeps level numbering for siblings after a cycle predicate', () => { const result = parseWhereClause( { - ACCOUNT: { - country: 'US', - RING: { - $cycle: true, - $relation: { direction: 'out', hops: { max: 4 } } - } + $cycle: { type: 'T', direction: 'out', hops: { max: 3 } }, + FRIEND: { + name: 'Bob' } }, options() @@ -746,100 +735,95 @@ describe('cycle detection ($cycle)', () => { expect(result.queryParts).toEqual({ record: '', - record1: - 'OPTIONAL MATCH (record)--(record1:__RUSHDB__LABEL__RECORD__:`ACCOUNT`) WHERE (any(value IN record1.`country` WHERE value = "US"))', record2: - "OPTIONAL MATCH (record1)-[rels2*2..4]->(record1) WHERE all(r IN rels2 WHERE type(r) <> '__RUSHDB__RELATION__VALUE__')" + 'OPTIONAL MATCH (record)--(record2:__RUSHDB__LABEL__RECORD__:`FRIEND`) WHERE (any(value IN record2.`name` WHERE value = "Bob"))' }) - expect(result.nodeAliases).toEqual(['record', 'record1', 'rels2']) - expect(result.where).toEqual('record IS NOT NULL AND (record1 IS NOT NULL AND rels2 IS NOT NULL)') + expect(result.nodeAliases).toEqual(['record', 'record2']) + expect(result.where).toEqual( + 'record IS NOT NULL AND EXISTS { MATCH (record)-[:T*2..3]->(record) } AND record2 IS NOT NULL' + ) + }) + + it('defaults cycle min to 2 and rejects hops below it', () => { + expect(() => parseWhereClause({ $cycle: { type: 'T', hops: 1 } }, options())).toThrow() + expect(() => parseWhereClause({ $cycle: { type: 'T', hops: { min: 1, max: 3 } } }, options())).toThrow( + "'hops.min'" + ) }) - it('keeps level numbering for siblings after a cycle block', () => { + it('anchors to a related record when nested in a label block', () => { const result = parseWhereClause( { - RING: { - $cycle: true, - $relation: { type: 'T', direction: 'out', hops: { max: 3 } } - }, - FRIEND: { - name: 'Bob' + ACCOUNT: { + country: 'US', + $cycle: { direction: 'out', hops: { max: 4 } } } }, options() ) - expect(result.queryParts).toEqual({ - record: '', - record1: 'OPTIONAL MATCH (record)-[rels1:T*2..3]->(record)', - record2: - 'OPTIONAL MATCH (record)--(record2:__RUSHDB__LABEL__RECORD__:`FRIEND`) WHERE (any(value IN record2.`name` WHERE value = "Bob"))' - }) - expect(result.nodeAliases).toEqual(['record', 'rels1', 'record2']) - expect(result.where).toEqual('record IS NOT NULL AND rels1 IS NOT NULL AND record2 IS NOT NULL') + expect(result.where).toEqual( + 'record IS NOT NULL AND (record1 IS NOT NULL AND ' + + "EXISTS { MATCH (record1)-[rels2*2..4]->(record1) WHERE all(r IN rels2 WHERE type(r) <> '__RUSHDB__RELATION__VALUE__') })" + ) }) - it('combines a cycle with a normal block under $or', () => { + it('composes under $or with other blocks', () => { const result = parseWhereClause( { $or: { - RING: { - $cycle: true, - $relation: { type: 'T', direction: 'out', hops: { max: 3 } } - }, - EMPLOYEE: { - name: 'X' - } + $cycle: { type: 'T', direction: 'out', hops: { max: 3 } }, + EMPLOYEE: { name: 'X' } } }, options() ) - expect(result.where).toEqual('record IS NOT NULL AND (rels1 IS NOT NULL OR record2 IS NOT NULL)') + expect(result.where).toEqual( + 'record IS NOT NULL AND (EXISTS { MATCH (record)-[:T*2..3]->(record) } OR record2 IS NOT NULL)' + ) + }) + + it('negates with $not (acyclic check)', () => { + const result = parseWhereClause( + { $not: { $cycle: { type: 'T', direction: 'out', hops: { max: 3 } } } }, + options() + ) + + expect(result.where).toEqual( + 'record IS NOT NULL AND (NOT(EXISTS { MATCH (record)-[:T*2..3]->(record) }))' + ) }) - it('negates a cycle with $not (acyclic check)', () => { + it('supports several cycle predicates at one level via an $and array', () => { const result = parseWhereClause( { - $not: { - RING: { - $cycle: true, - $relation: { type: 'T', direction: 'out', hops: { max: 3 } } - } - } + $and: [ + { $cycle: { type: 'A', direction: 'out', hops: { max: 3 } } }, + { $cycle: { type: 'B', direction: 'out', hops: { max: 4 } } } + ] }, options() ) - expect(result.where).toEqual('record IS NOT NULL AND (NOT(rels1 IS NOT NULL))') + expect(result.where).toContain('EXISTS { MATCH (record)-[:A*2..3]->(record) }') + expect(result.where).toContain('EXISTS { MATCH (record)-[:B*2..4]->(record) }') }) - it('defaults cycle min to 2 and rejects hops below it', () => { - expect(() => - parseWhereClause({ RING: { $cycle: true, $relation: { type: 'T', hops: 1 } } }, options()) - ).toThrow() + it.each([ + ['a bare boolean', { $cycle: true }], + ['a string value', { $cycle: 'T' }], + ['a spec without hops', { $cycle: { type: 'T' } }] + ])('rejects $cycle with %s', (_, where) => { + expect(() => parseWhereClause(where as never, options())).toThrow("'$cycle' requires") + }) + + it('rejects the removed block form with operator-form guidance', () => { expect(() => parseWhereClause( - { RING: { $cycle: true, $relation: { type: 'T', hops: { min: 1, max: 3 } } } }, + { SOME_KEY: { $cycle: true, $relation: { type: 'T', direction: 'out', hops: { max: 3 } } } }, options() ) - ).toThrow("'hops.min'") - }) - - it.each([ - ['without $relation', { RING: { $cycle: true } }], - ['with a string $relation', { RING: { $cycle: true, $relation: 'T' } }], - ['without hops', { RING: { $cycle: true, $relation: { type: 'T' } } }], - ['with $alias', { RING: { $cycle: true, $alias: '$ring', $relation: { type: 'T', hops: { max: 3 } } } }], - [ - 'with property criteria', - { RING: { $cycle: true, $relation: { type: 'T', hops: { max: 3 } }, name: 'X' } } - ], - [ - 'with a nested label block', - { RING: { $cycle: true, $relation: { type: 'T', hops: { max: 3 } }, POST: { title: 'X' } } } - ] - ])('rejects $cycle %s', (_, where) => { - expect(() => parseWhereClause(where, options())).toThrow() + ).toThrow("'$cycle' requires") }) }) diff --git a/platform/core/src/core/search/parser/types.ts b/platform/core/src/core/search/parser/types.ts index c8adede5..b1974b7f 100644 --- a/platform/core/src/core/search/parser/types.ts +++ b/platform/core/src/core/search/parser/types.ts @@ -6,6 +6,9 @@ export type ParseContext = { result: Record aliasesMap: AliasesMap withQueryQueue?: Record + /** EXISTS predicates compiled from $cycle blocks in Pass 1, keyed by traversal level + * so Pass 2 (buildWhereClause), which walks levels in lockstep, can inline them. */ + cycleExistsByLevel: Record } export type AggregateContext = { diff --git a/platform/core/src/core/search/parser/utils.ts b/platform/core/src/core/search/parser/utils.ts index 8ec9817d..403832e9 100644 --- a/platform/core/src/core/search/parser/utils.ts +++ b/platform/core/src/core/search/parser/utils.ts @@ -23,6 +23,12 @@ export const wrapInParentheses = (input: string) => `(${input})` // the tree independently but increment ctx.level in lockstep). export const traversalRelsVar = (level: number) => `rels${level}` +// The $cycle operator's subQueries key: `$cycle` itself, or the `$cycle#N` suffix +// splitCriteria mints when an $or/$and array carries several cycle predicates at +// one level ('$' keys can never be user labels, so the namespace is safe). +export const isCycleOperatorKey = (key: string) => + key === CYCLE_CLAUSE_OPERATOR || key.startsWith(`${CYCLE_CLAUSE_OPERATOR}#`) + export const wrapInCurlyBraces = (input: string) => `{${input}}` export const isSubQuery = (input: Where) => { @@ -59,7 +65,20 @@ export function splitCriteria(input: Where) { // !containsAllowedKeys(value, allowedKeys) const split = (v: Where) => Object.entries(v).forEach(([key, value]) => { - if (isObject(value) && isSubQuery(value as Where)) { + if (key === CYCLE_CLAUSE_OPERATOR) { + // { $cycle: { type?, direction, hops } } — the value IS the traversal spec and + // is stored raw; both passes recognize the key via isCycleOperatorKey. Suffix + // duplicates when an $or/$and array carries several cycle predicates at one + // level — deterministic across both passes, which walk the same input in the + // same order. Storing the raw value keeps re-splitting idempotent (the + // logical-operator path splits twice); suffixed keys re-enter through the + // isSubQuery branch below and are still parsed by key prefix. + let uniqueKey: string = key + for (let n = 2; uniqueKey in subQueries; n += 1) { + uniqueKey = `${key}#${n}` + } + subQueries[uniqueKey] = value + } else if (isObject(value) && isSubQuery(value as Where)) { subQueries[key] = value } else { currentLevel[key] = value diff --git a/platform/core/src/core/search/search.constants.ts b/platform/core/src/core/search/search.constants.ts index 690fb4fd..94dcb41f 100644 --- a/platform/core/src/core/search/search.constants.ts +++ b/platform/core/src/core/search/search.constants.ts @@ -48,7 +48,7 @@ export const ALIAS_CLAUSE_OPERATOR = '$alias' export const ID_CLAUSE_OPERATOR = '$id' export const CYCLE_CLAUSE_OPERATOR = '$cycle' -export const DEFAULT_MAX_TRAVERSAL_HOPS = 25 +export const DEFAULT_MAX_TRAVERSAL_HOPS = 10 /** * Effective hop ceiling for variable-length traversal (`$relation.hops`). diff --git a/platform/dashboard/CHANGELOG.md b/platform/dashboard/CHANGELOG.md index ef910b1d..d809d786 100644 --- a/platform/dashboard/CHANGELOG.md +++ b/platform/dashboard/CHANGELOG.md @@ -29,17 +29,14 @@ db.records.find({ labels: ['ACCOUNT'], where: { - RING: { - $cycle: true, - $relation: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } - } + $cycle: { type: 'TRANSFERRED_TO', direction: 'out', hops: { min: 2, max: 6 } } } }) ``` - A `$cycle` block requires `$relation` with `hops` (`min` ≥ 2, the default) and accepts no other keys; its label key is a display name. Combine with `$not` to select records not on a cycle. + `$cycle` is a record-level predicate: its value is the traversal spec itself (`type`, `direction`, `hops` — `hops` mandatory, `min` ≥ 2, defaulting to 2). A cycle has no separate endpoint, so it accepts no `$alias`, no property criteria, and no nested labels. Combine with `$not` to select records not on a cycle. - **Traversal depth policy** — `hops.max` is capped per deployment via `RUSHDB_MAX_TRAVERSAL_HOPS` (default 25) on the shared cloud connection. Self-hosted deployments (`RUSHDB_SELF_HOSTED=true`) and projects with a custom external Neo4j allow unbounded traversal (`max` omitted), guarded by the existing transaction timeout. + **Traversal depth policy** — `hops.max` is capped per deployment via `RUSHDB_MAX_TRAVERSAL_HOPS` (default 10) on the shared cloud connection. Self-hosted deployments (`RUSHDB_SELF_HOSTED=true`) and projects with a custom external Neo4j allow unbounded traversal (`max` omitted), guarded by the existing transaction timeout. Also in this release: diff --git a/platform/dashboard/src/features/projects/components/SearchBox.tsx b/platform/dashboard/src/features/projects/components/SearchBox.tsx index 7d58d1a6..bb2b3d74 100644 --- a/platform/dashboard/src/features/projects/components/SearchBox.tsx +++ b/platform/dashboard/src/features/projects/components/SearchBox.tsx @@ -35,6 +35,7 @@ import { api } from '~/lib/api.ts' import { isInViewport, normalizeString } from '~/lib/utils.ts' import { $activeLabels, $currentProjectFilters, addFilter } from '../stores/current-project.ts' +import { $recordQuery } from '../stores/records-search.ts' import { useProjectFieldsQuery } from '../hooks/useProjectQueries' import { convertToSearchQuery, filterToSearchOperation } from '~/features/projects/utils.ts' @@ -47,8 +48,6 @@ const MAX_SEARCH_CHARS = 200 export const $open = atom(false) -const $recordQuery = atom('') - // pages enum BoxPages { // ChooseMinMax = 'ChooseMinMax', diff --git a/platform/dashboard/src/features/projects/stores/records-search.ts b/platform/dashboard/src/features/projects/stores/records-search.ts index 299dfff2..6bed5b8f 100644 --- a/platform/dashboard/src/features/projects/stores/records-search.ts +++ b/platform/dashboard/src/features/projects/stores/records-search.ts @@ -18,6 +18,10 @@ export const $semanticSearchPrompt = atom('') export const $semanticSearchIndexId = atom() export const $searchQueryModalOpen = atom(false) +// Builder search-box input text; lives here (not in SearchBox.tsx) so it can be +// reset alongside the rest of the search state on project switch. +export const $recordQuery = atom('') + export const setAiSearchQuery = (query: SearchQuery | undefined) => { $aiSearchQuery.set(query) if (query) { @@ -56,6 +60,19 @@ export const resetRecordsView = () => { $semanticSearchIndexId.set(undefined) } +// Search state lives in module-level atoms, so it survives client-side navigation +// between projects. Clear it whenever the current project changes; mirrors the +// filters/pagination reset in current-project.ts. Same-project navigation (e.g. +// applySavedQuery) is unaffected: the atom doesn't notify when the id is unchanged. +$currentProjectId.subscribe(() => { + $recordsSearchMode.set('ai') + resetAiSearch() + $semanticSearchPrompt.set('') + $semanticSearchIndexId.set(undefined) + $recordQuery.set('') + $searchQueryModalOpen.set(false) +}) + // Restores a saved query onto the Records page and navigates there. Manual and // AI queries are re-run through the AI path (which executes the stored // SearchQuery verbatim); semantic queries restore the index + prompt so the