From 7c95ebdcaaa0b1fff7e4523cbeb80a41ac1a837d Mon Sep 17 00:00:00 2001 From: David de Boer Date: Wed, 2 Sep 2026 19:38:27 +0200 Subject: [PATCH 1/8] fix(search)!: fan out a qualified edge into one entry per tuple MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A weld asks whether ONE entry satisfies every condition. An entry whose leaves hold arrays stands for every combination at once, so it answers the weld with none of them – and Typesense 30.2, the current stable release, does not answer at all: the search hangs indefinitely, which reaches a consumer as an unexplained error. - refuse a nested field declaring both filterable and array, so a leaf a weld can name states one value per entry - fan an edge out into one entry per combination of its weldable leaves, on the framed node before projection, so each leaf still passes through transform, folding and the facet companion unchanged - write a nested identity companion under the arity of its reference, since that companion is the leaf a weld actually names; a top-level companion stands for the whole document and is unchanged - bound the fan-out with maxEntries (default 100), per ADR 12: a cartesian product over an edge's values is a bound in the data's own units - pin the engine guarantee against a real container, the array-shaped hang included, so a future engine bump reports when it is gone Measured at 1,000,000 documents: the welded filter answers in 14–19 ms where it previously never returned, and the fanned-out shape is no slower than the array shape on 31.0, so the eventual upgrade unwinds nothing. Indexing is ~15 % slower for ~50 % more entries. BREAKING CHANGE: a nested field may no longer declare both “filterable” and “array”; searchSchema refuses it. Declare the leaf single-valued – the projection now emits one entry per combination. --- ...qualified-edge-into-one-entry-per-tuple.md | 180 +++++++++++ docs/reference/search.md | 35 +++ .../extraction-roundtrip.integration.test.ts | 3 +- .../search-pipeline/test/extraction.test.ts | 3 +- .../test/nested-fanout.integration.test.ts | 3 +- .../src/collection-definition.ts | 34 +- .../test/qualified-relation.test.ts | 77 +++-- .../test/welded-filter.integration.test.ts | 129 ++++++++ packages/search-typesense/vite.config.ts | 2 +- packages/search/src/project.ts | 153 ++++++++- packages/search/src/schema.ts | 41 ++- .../search/test/qualified-relation.test.ts | 297 ++++++++++++++++-- packages/search/test/schema.test.ts | 24 +- 13 files changed, 895 insertions(+), 86 deletions(-) create mode 100644 docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md create mode 100644 packages/search-typesense/test/welded-filter.integration.test.ts diff --git a/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md b/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md new file mode 100644 index 00000000..f6fa3352 --- /dev/null +++ b/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md @@ -0,0 +1,180 @@ +# 26. Fan out a qualified edge into one entry per tuple + +Date: 2026-09-02 + +## Status + +Proposed + +Amends [ADR 24 (Carry data on a reference edge)](./0024-carry-data-on-a-reference-edge.md), +which introduced the welded co-element filter but left the shape of an entry +under-specified. Relates to +[ADR 18 (Filter across several fields with one clause)](./0018-filter-across-several-fields-with-one-clause.md) +and [ADR 12 (Bound memory by the unit of work, not the input)](./0012-bound-memory-by-the-unit-of-work-not-the-input.md). + +## Context + +[ADR 24](./0024-carry-data-on-a-reference-edge.md) says the shape is **one entry +per edge**, and gives the welded filter that only an edge can answer: _this +agent in this role_, rather than “this role appears, and this agent appears, +somewhere in this document”. Unwelded, a work where X is the publisher and +somebody else is the photographer matches `role: fotograaf && creator: X`. That +false positive is the entire reason the nested entry exists. + +What the ADR did not say is what an entry holds when the graph gives an edge +more than one value. Nothing stopped a nested leaf declaring `array: true`, and +two ordinary modelling facts push straight at it: + +- a role may be stated once per language, so `role` arrives as two literals; +- the edge’s endpoint may be multi-valued, so `creator` arrives as two IRIs. + +So the projection wrote entries like +`{ "role": ["etser", "etcher"], "creator_id": ["p1", "p3"] }`, and every +qualified relation in production carried them. + +**Two things are wrong with that, and only one of them is a bug in somebody +else’s code.** + +### The engine hangs + +`typesense/typesense:30.2` – the current stable release – **hangs indefinitely** +on a welded filter whenever the entry holds arrays. Measured against a live +container, one document, no LDE code in the path: + +``` +c.{role:=etser && aid:=p1} with role:["etser"] aid:["p1"] → no response, ever +c.{role:=etser && aid:=p1} with role:"etser" aid:"p1" → 1 hit, 5 ms +``` + +The trigger is narrower than “the fields hold arrays”: the hang needs both +conditions to find a match somewhere in the document **and** at least one matched +value to be array-valued. A single array-valued leaf on either side is enough – +`role:"etser" aid:["p1"]` hangs exactly as hard as both-arrays. A document that +matches neither condition is never slow, however array-shaped. At 1 000 000 +documents the welded filter never returns, while either condition alone answers +in 16 ms. + +Through the GraphQL surface that reaches a consumer as `Unexpected error`, the +5 s engine timeout retried once. + +Typesense fixed it in the 31.0 release candidates – `31.0.rc1` hangs, `rc5` +through `rc14` answer correctly – but 31.0 has no stable release, and the +neighbouring upstream reports +([typesense#2469](https://github.com/typesense/typesense/issues/2469), +[typesense#2964](https://github.com/typesense/typesense/issues/2964)) are both +still open. + +### The entry has no tuple to test + +The engine bug is the reason this got noticed. It is not the reason to change +the shape. + +A weld asks a question about **one element**: _is there an entry whose role is +`etser` and whose agent is `p1`_. An entry holding +`{ role: ["etser", "etcher"], creator_id: ["p1", "p3"] }` has no single answer – +it stands for four (role, agent) pairs at once, and the weld silently degenerates +into the cross-product it existed to exclude. Welding inside such an entry is the +same mistake as not welding at all, one level down. + +So the array-valued entry is not a valid input the engine mishandles. It is a +shape that never had a meaning, which no engine could have answered, and which +[ADR 24](./0024-carry-data-on-a-reference-edge.md)’s own words – _one entry per +edge_ – already exclude. + +## Decision + +**A weldable nested leaf is single-valued, and multiplicity moves to the entry +list.** + +Three parts. + +### 1. A `filterable` nested field may not declare `array: true` + +Refused by `searchSchema`, beside the Roles nesting already cannot serve. A +nested leaf that a weld can name states one value per entry, and a declaration +saying otherwise is refused at startup rather than producing entries no filter +can read. + +`output`-only nested leaves are untouched: nothing welds them, so an entry may +carry a list for display. + +### 2. The projection fans out one entry per tuple + +Where the graph gives an edge several values for a weldable leaf, the projection +emits **one entry per combination**, each leaf single-valued: + +```jsonc +// the graph +{ "role": ["etser", "etcher"], "creator": ["p1", "p3"] } + +// the entries +[ { "role": "etser", "creator_id": "p1" }, + { "role": "etser", "creator_id": "p3" }, + { "role": "etcher", "creator_id": "p1" }, + { "role": "etcher", "creator_id": "p3" } ] +``` + +Fan-out happens on the **framed node**, before the entry is projected, so each +leaf passes through `transform`, folding and the facet companion exactly as a +single-valued field always has. Nothing downstream learns a new shape. + +Nothing is dropped: the four entries carry what the two arrays carried. What +changes is that each one now answers the weld. + +### 3. Language variants are labels, not values + +A role stated once per language is **one** role. Declaring `role` multi-valued to +hold `"etser"@nl` and `"etcher"@en` models a labelling accident as data, and +fan-out would then emit two entries for one relation and facet them into two +buckets. + +Index the role’s canonical IRI, single-valued, and resolve labels at the surface +like every other reference. Measured on 1 000 000 documents this halves the +entries per document (3.74 against 7.49) and gives 15 facet buckets rather than +30 that split one role across languages. + +### What bounds it + +A cartesian product over an edge’s own values is a bound stated in the data’s own +units, which [ADR 12](./0012-bound-memory-by-the-unit-of-work-not-the-input.md) +says is not a bound. An inline reference therefore declares `maxEntries` +(default 100): entries past it are dropped and reported, rather than a +pathological edge multiplying a document until the run dies. + +## Consequences + +**The weld works on the engine we run.** No release to wait for, no release +candidate in production. + +**It costs nothing to upgrade later.** Measured at 1 000 000 documents, both end +states are the same speed, and the fanned-out shape is no slower on 31.0 than the +array shape it replaces: + +| | welded filter, p50 | role alone | facet | +| ------------------------ | ------------------ | ---------- | ----- | +| 30.2 + fan-out | 14–19 ms | 15.9 ms | 30 ms | +| 31.0 + arrays | 13–16 ms | 15.8 ms | 64 ms | +| 30.2 + arrays _(before)_ | **hangs** | 15.8 ms | 30 ms | +| 31.0 + fan-out | 13–16 ms | 16.1 ms | 28 ms | + +So this is not a trade the 31.0 upgrade unwinds. When 31.0 goes stable it is +routine maintenance, not a migration back. + +**Indexing is ~15 % slower** – 54 s against 47 s for 1 000 000 documents, +consistent across both engine versions – because a document carries about 50 % +more entries. A batch cost, not a request cost. + +**Filters and facets stay on the real fields.** The alternative that also works on +30.2 is to weld at index time into a composite `role|agent` key and filter it with +one condition. It is faster (3 ms against 17 ms) and worse: at 1 000 000 documents +faceting that key yields 2 142 125 buckets in 1.4 s, against 15 buckets in 48 ms +for the role itself. It also needs a separator that can never occur in an IRI, and +forces the schema to declare which pairs are weldable, narrowing the query surface +from _any two edge fields_ to _the declared pairs_. Rejected. + +**A schema that declared a weldable leaf `array: true` now fails at startup**, with +a message saying to fan out instead. We are pre-release; there is no migration. + +**One array-valued weldable leaf re-arms the hang**, which is why part 1 is a +refusal rather than a convention. A deployment cannot opt out of it by accident, +and the failure is at startup rather than a query that never returns. diff --git a/docs/reference/search.md b/docs/reference/search.md index fc14a4e1..bbfe5770 100644 --- a/docs/reference/search.md +++ b/docs/reference/search.md @@ -536,6 +536,41 @@ Welding on identity needs the endpoint's `filterable`, which fans out its id as a leaf beside the stored object – an engine welds conditions on an entry's own leaf fields only. That is a physical detail: you write the logical field. +**A weldable leaf is single-valued.** A nested field declaring `filterable` may +not also declare `array`; `searchSchema` refuses it. A weld asks whether _one_ +entry satisfies every condition, and an entry holding a list stands for each +combination at once – so it answers the weld with none of them, and the weld +degenerates into the cross-product it exists to exclude. + +Multiplicity belongs to the entry list instead. Where the graph gives an edge +several roles or several endpoints, the projection emits **one entry per +combination**: + +```jsonc +// the graph +{ "role": ["etser", "etcher"], "creator": ["p1", "p3"] } + +// the entries +[ { "role": "etser", "creator_id": "p1" }, + { "role": "etser", "creator_id": "p3" }, + { "role": "etcher", "creator_id": "p1" }, + { "role": "etcher", "creator_id": "p3" } ] +``` + +Nothing is dropped, and each entry now answers the weld. Two consequences worth +knowing when you declare an edge: + +- A role stated once per language is **one** role, not two. Index its canonical + IRI single-valued and resolve labels at the surface; declaring the label + multi-valued makes fan-out emit an entry per language and splits one role + across two facet buckets. +- `maxEntries` on the inline reference caps the entries one document stores + (default 100), so a pathological edge cannot multiply a document without + bound. An `output`-only nested leaf is untouched by all of this: nothing welds + it, so it may carry a list for display. + +See [ADR 26](../decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple). + Out of scope for now: faceting an edge's own values, which the current engine cannot serve correctly. diff --git a/packages/search-pipeline/test/extraction-roundtrip.integration.test.ts b/packages/search-pipeline/test/extraction-roundtrip.integration.test.ts index 938c5faf..e1bf58d7 100644 --- a/packages/search-pipeline/test/extraction-roundtrip.integration.test.ts +++ b/packages/search-pipeline/test/extraction-roundtrip.integration.test.ts @@ -89,9 +89,10 @@ const creatorRole = defineSearchType({ name: 'CreatorRole', fields: [ { + // Single-valued: a leaf a weld can name states one value per entry, and + // an edge the graph gave several roles fans out (ADR 26). name: 'role', kind: 'keyword', - array: true, output: true, filterable: true, path: `<${SCHEMA}roleName>`, diff --git a/packages/search-pipeline/test/extraction.test.ts b/packages/search-pipeline/test/extraction.test.ts index 334f9b72..0ed63c05 100644 --- a/packages/search-pipeline/test/extraction.test.ts +++ b/packages/search-pipeline/test/extraction.test.ts @@ -102,9 +102,10 @@ const creatorRole = defineSearchType({ name: 'CreatorRole', fields: [ { + // Single-valued: a leaf a weld can name states one value per entry, and + // an edge the graph gave several roles fans out (ADR 26). name: 'role', kind: 'keyword', - array: true, output: true, filterable: true, path: `<${SCHEMA}roleName>`, diff --git a/packages/search-pipeline/test/nested-fanout.integration.test.ts b/packages/search-pipeline/test/nested-fanout.integration.test.ts index 32ea4a46..ef62cec4 100644 --- a/packages/search-pipeline/test/nested-fanout.integration.test.ts +++ b/packages/search-pipeline/test/nested-fanout.integration.test.ts @@ -46,9 +46,10 @@ const creatorRole = defineSearchType({ name: 'CreatorRole', fields: [ { + // Single-valued: a leaf a weld can name states one value per entry, and + // an edge the graph gave several roles fans out (ADR 26). name: 'role', kind: 'keyword', - array: true, output: true, filterable: true, path: `<${SCHEMA}roleName>`, diff --git a/packages/search-typesense/src/collection-definition.ts b/packages/search-typesense/src/collection-definition.ts index a014abf5..5275c91d 100644 --- a/packages/search-typesense/src/collection-definition.ts +++ b/packages/search-typesense/src/collection-definition.ts @@ -538,10 +538,12 @@ function nestedIdentityFields( return [ { name: nestedFieldName(prefix, names.identity), - // Always a list, whatever the enclosing reference's arity: the projection - // writes it with `setArray`, and an indexed field's declared type is - // checked against what is stored – a `string` here rejects every - // document carrying such an edge, at import. + // Always a list, whatever the enclosing reference's arity, and whatever + // ONE entry holds: a nested field's declared type describes the whole + // path across the document, and a document holds many entries – so + // `string` here rejects every document carrying such an edge, at import. + // The entries themselves store a single id each, which the engine accepts + // under this type and which is what makes the weld answerable (ADR 26). type: 'string[]', index: true, optional: true, @@ -613,14 +615,18 @@ function nestedLeafFields( // itself – its search companion below indexes on its own account. const indexed = field.filterable === true; const valueType = typesenseValueType(field); - // Two ways a list arrives: the field declares one, or an ancestor's - // `object[]` flattens it into one. `typesenseValueType` honours `array` only - // for the string-shaped kinds, so a multi-valued nested `integer` would - // otherwise be declared scalar and rejected at import. - const storesAList = flattensToArray || field.array === true; + // An ancestor's `object[]` flattens this value into a list, so an indexed + // leaf must be declared as one or the import rejects every document carrying + // the edge. The field's OWN `array` cannot also be widening it here: + // `searchSchema` refuses `filterable` alongside `array` on a nested field, so + // an indexed leaf is single-valued (ADR 26) – which is what makes the cast + // below sound, `typesenseValueType` returning `string[]` only for `array`. fields.push({ name: nestedFieldName(prefix, field.name), - type: indexed && storesAList ? arrayValueType(valueType) : valueType, + type: + indexed && flattensToArray + ? arrayValueType(valueType as Exclude) + : valueType, index: indexed, // Always optional: `required` is a promise about the *referent* (this value // is on every referent), while Typesense’s flag is about the document. @@ -648,7 +654,9 @@ function nestedLeafFields( * declaration is. Enumerated rather than string-appended so an invalid * combination cannot be built. */ -function arrayValueType(type: ValueType): CollectionFieldSchema['type'] { +function arrayValueType( + type: Exclude, +): CollectionFieldSchema['type'] { switch (type) { case 'string': return 'string[]'; @@ -658,10 +666,6 @@ function arrayValueType(type: ValueType): CollectionFieldSchema['type'] { return 'float[]'; case 'bool': return 'bool[]'; - // Already a list: a multi-valued declaration under a multi-valued ancestor - // flattens no further. - case 'string[]': - return 'string[]'; } } diff --git a/packages/search-typesense/test/qualified-relation.test.ts b/packages/search-typesense/test/qualified-relation.test.ts index 038d7c7b..183d5851 100644 --- a/packages/search-typesense/test/qualified-relation.test.ts +++ b/packages/search-typesense/test/qualified-relation.test.ts @@ -337,12 +337,23 @@ describe('nested fields of other kinds', () => { filterable: true, }, { + // Output-only, so it may stay a list: nothing welds it, and a weldable + // leaf is single-valued (ADR 26). name: 'source', kind: 'keyword', path: `${SCHEMA_ORG}isBasedOn`, array: true, output: true, - filterable: true, + }, + { + // Searchable rather than filterable, so it may stay a list too: free + // text is not a weld, and its folded companion is what gets indexed. + name: 'attribution', + kind: 'keyword', + path: `${SCHEMA_ORG}creditText`, + array: true, + output: true, + searchable: { weight: 1 }, }, ], }); @@ -415,30 +426,23 @@ describe('nested fields of other kinds', () => { ); }); - it('gives a searchable nested keyword its folded companion', () => { - expect(richField('credit.note_search')).toMatchObject({ - type: 'string[]', - }); - }); - - it('widens a multi-valued nested numeric under a single-valued edge', () => { - // Two ways a list arrives – the field declares one, or an ancestor - // flattens it – and `typesenseValueType` honours only the string-shaped - // kinds' own `array`, so this one needs widening on its own account. - const countEdge = defineSearchType({ - name: 'CountEdge', + it('stems a language-tagged nested text field in its own locale', () => { + // The counterpart of the `und` case above: a declared locale stems in + // itself, never in `defaultLocale`, so a Dutch note is not stemmed as if + // it were English. + const taggedEdge = defineSearchType({ + name: 'TaggedEdge', fields: [ { - name: 'position', - kind: 'integer', - path: `${SCHEMA_ORG}position`, - array: true, - output: true, - filterable: true, + name: 'note', + kind: 'text', + path: `${SCHEMA_ORG}description`, + locales: ['nl'], + searchable: { weight: 1 }, }, ], }); - const singleEdgeWork = defineSearchType({ + const taggedWork = defineSearchType({ name: 'Work', class: `${SCHEMA_ORG}CreativeWork`, fields: [ @@ -446,32 +450,51 @@ describe('nested fields of other kinds', () => { name: 'credit', kind: 'reference', path: `${SCHEMA_ORG}creator`, + array: true, output: true, - ref: { strategy: 'inline', typeName: 'CountEdge' }, + ref: { strategy: 'inline', typeName: 'TaggedEdge' }, }, ], }); const fields = - buildCollectionDefinition(singleEdgeWork, { - schema: searchSchema(singleEdgeWork, countEdge), + buildCollectionDefinition(taggedWork, { + schema: searchSchema(taggedWork, taggedEdge), + defaultLocale: 'en', }).fields ?? []; expect( - fields.find((field) => field.name === 'credit.position'), - ).toMatchObject({ type: 'int64[]', index: true }); + fields.find((field) => field.name === 'credit.note_search_nl'), + ).toMatchObject({ type: 'string[]', stem: true, locale: 'nl' }); + }); + + it('gives a searchable nested keyword its folded companion', () => { + expect(richField('credit.note_search')).toMatchObject({ + type: 'string[]', + }); }); it.each([ ['credit.position', 'int64[]'], ['credit.certainty', 'float[]'], ['credit.disputed', 'bool[]'], - // Already a list on its own: flattening does not double it. - ['credit.source', 'string[]'], ])('widens indexed nested %s to %s', (name, type) => { // An engine checks an indexed field's declared type against what is // stored, and the `object[]` above these flattens each value into a list. expect(richField(name)).toMatchObject({ type, index: true }); }); + + it('does not double a nested list that is already one', () => { + // A leaf a weld can name is single-valued (ADR 26), so a nested list is + // either output-only or searchable. Flattening one under the `object[]` + // widens it once, not twice – `string[]`, never `string[][]`. + expect(richField('credit.source')).toMatchObject({ + type: 'string[]', + index: false, + }); + expect(richField('credit.attribution_search')).toMatchObject({ + type: 'string[]', + }); + }); }); describe('a facet policy over the companion', () => { diff --git a/packages/search-typesense/test/welded-filter.integration.test.ts b/packages/search-typesense/test/welded-filter.integration.test.ts new file mode 100644 index 00000000..58bf7f79 --- /dev/null +++ b/packages/search-typesense/test/welded-filter.integration.test.ts @@ -0,0 +1,129 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import type { Client } from 'typesense'; +import { TypesenseContainer } from './typesense-container.js'; + +/** + * The engine guarantee the welded co-element filter rests on, pinned against a + * real Typesense rather than reasoned about – *this agent in this role*, the + * one question a qualified edge buys over two flat fields. + * + * **Why an integration test and not a compiler assertion.** The filter this + * compiles to is valid, documented syntax, and the compiler emitted it + * correctly all along. What no unit test could see is that the engine’s answer + * depends on the *shape of the stored values*: on `typesense/typesense:30.2` a + * weld over entries whose leaves hold arrays never returns at all – no error, + * no timeout, no response – while every unwelded variant answers in + * milliseconds. Only a live engine says so, which is why the entries below are + * written in both shapes and both are asserted. + * + * The trap, asserted here because it is silent in the other direction: + * `role:=X && agent:=Y` written OUTSIDE the braces matches a work where X and Y + * occur in *different* entries. That false positive is the whole reason the + * weld exists, so a weld that merely returns is not enough – it has to return + * strictly less. + * + * See [ADR 26](../../../docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md) + * and [#798](https://github.com/ldelements/lde/issues/798). + */ +describe('a welded co-element filter', () => { + const container = new TypesenseContainer(); + let client: Client; + + const collection = 'works'; + const etser = 'http://vocab.example/role/etser'; + const drukker = 'http://vocab.example/role/drukker'; + const rembrandt = 'http://data.example/agent/rembrandt'; + const other = 'http://data.example/agent/other'; + + /** The filter a compiled {@link WeldedCriterion} produces. */ + const welded = (role: string, agent: string) => + `credit.{role:=\`${role}\` && agent_id:=\`${agent}\`}`; + + const found = async (filterBy: string) => { + const result = await client + .collections(collection) + .documents() + .search({ q: '*', query_by: '', filter_by: filterBy }, {}); + return (result.hits ?? []) + .map((hit) => (hit.document as { id: string }).id) + .sort(); + }; + + beforeAll(async () => { + client = await container.start(); + await client.collections().create({ + name: collection, + enable_nested_fields: true, + fields: [ + { name: 'credit', type: 'object[]' }, + { name: 'credit.role', type: 'string[]' }, + { name: 'credit.agent_id', type: 'string[]' }, + ], + }); + await client + .collections(collection) + .documents() + .import( + [ + // Fanned out: one entry per (role, agent) tuple, every leaf a single + // value – what the projection now writes. + { + id: 'fanned-match', + credit: [{ role: etser, agent_id: rembrandt }], + }, + // The false positive the weld exists to exclude: both values are + // present in the document, in different entries. + { + id: 'fanned-cross', + credit: [ + { role: etser, agent_id: other }, + { role: drukker, agent_id: rembrandt }, + ], + }, + ], + { action: 'create' }, + ); + }, 120_000); + + afterAll(async () => { + await container.stop(); + }); + + it('matches only the work whose ONE entry satisfies both conditions', async () => { + expect(await found(welded(etser, rembrandt))).toEqual(['fanned-match']); + }); + + it('returns strictly less than the same conditions unwelded', async () => { + // Unwelded, the cross-matched work comes back too – Rembrandt is on it, and + // so is the etser role, just never together. This is the assertion that + // makes the weld worth its cost. + expect( + await found( + `credit.role:=\`${etser}\` && credit.agent_id:=\`${rembrandt}\``, + ), + ).toEqual(['fanned-cross', 'fanned-match']); + }); + + it('answers a weld naming no matching tuple', async () => { + expect(await found(welded(drukker, other))).toEqual([]); + }); + + it('hangs on 30.2 where an entry holds arrays, which is why we fan out', async () => { + // The defect this shape exists to avoid, pinned so a future engine bump + // tells us when it is gone. Typesense answers every OTHER form of this + // query in milliseconds; welded over array-valued leaves it never responds, + // so the client's own 5 s timeout is what ends the call. + await client + .collections(collection) + .documents() + .import( + [{ id: 'arrayed', credit: [{ role: [etser], agent_id: [rembrandt] }] }], + { action: 'create' }, + ); + + // Either condition alone still answers instantly over the same document. + expect(await found(`credit.role:=\`${etser}\``)).toContain('arrayed'); + + await expect(found(welded(etser, rembrandt))).rejects.toThrow(); + }, 60_000); +}); diff --git a/packages/search-typesense/vite.config.ts b/packages/search-typesense/vite.config.ts index 6ff57161..3a732f06 100644 --- a/packages/search-typesense/vite.config.ts +++ b/packages/search-typesense/vite.config.ts @@ -28,7 +28,7 @@ export default mergeConfig( // projection naming what no lookup reaches are unreachable through // the port, since `assertValidQuery` rejects such a query first. // They hold for a direct caller, and are exercised as one. - branches: 95.7, + branches: 95.82, statements: 99.48, }, }, diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index 4ba1dd3a..4fb5ee78 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -7,6 +7,7 @@ import { } from './frame-by-type.js'; import { assertTypeInSchema, + DEFAULT_MAX_ENTRIES, displayFieldName, documentKeyOf, fieldNamed, @@ -319,7 +320,15 @@ function applyField( // project nothing rather than fall through and emit the referent IRIs under // the field name (the wrong shape). if (schema !== undefined) { - applyInlineReference(document, node, alias, field, schema, context); + applyInlineReference( + document, + node, + alias, + field, + schema, + context, + searchType.class === undefined, + ); } return; } @@ -342,7 +351,13 @@ function applyField( // a filter can reach – an engine welds conditions on an entry's LEAF // fields only. `filterable` therefore fans out the id beside the object, // exactly as an inline reference's identity companion does. - applyLocalIdentity(document, endpoints, field, schema); + applyLocalIdentity( + document, + endpoints, + field, + schema, + searchType.class === undefined, + ); return; } } @@ -660,6 +675,7 @@ function applyInlineReference( field: ReferenceField & { readonly ref: { readonly typeName: string } }, schema: SearchSchema, context: ProjectionContext, + nested: boolean, ): void { // Resolves for a schema that declares the referent (always so for the schema a // type is projected through); a type framed against a foreign schema that @@ -676,7 +692,7 @@ function applyInlineReference( schema, context, ); - applyIdentityCompanion(document, referents, field, schema); + applyIdentityCompanion(document, referents, field, schema, nested); } /** @@ -700,6 +716,7 @@ function applyIdentityCompanion( referents: readonly ProjectedNode[], field: ReferenceField, schema: SearchSchema, + nested: boolean, ): void { const names = physicalFields(field, schema); if (names.identity === undefined) { @@ -723,14 +740,51 @@ function applyIdentityCompanion( if (ids.length === 0) { return; } - setArray(document, names.identity, ids); + setIdentity(document, names.identity, ids, field, nested); // Same rule as every other facetable reference: where the target declares a // facet policy, the facet reads a narrowed companion of its own, so an // excluded id is never seen by the engine rather than merely unlabelled. const policy = inheritedFacetKeys(field, schema); if (policy !== undefined) { - setArray(document, names.facet as string, ids.filter(policy.only)); + setIdentity( + document, + names.facet as string, + ids.filter(policy.only), + field, + nested, + ); + } +} + +/** + * Write an identity companion under the arity of the reference it belongs to – + * `array` decides the shape here exactly as it does for every other kind + * ({@link applyFacet}). + * + * It matters most where the companion is a **nested** leaf. That is the field a + * weld actually names – the endpoint's own id is a level deeper than a weld can + * reach – and a weld asks whether ONE entry satisfies every condition. A + * companion holding a list inside an entry stands for each of its ids at once, + * so the entry answers the weld with none of them; a Typesense 30.2 engine does + * not answer at all, and hangs (ADR 26). The entry fans out instead + * ({@link tuplesOf}), which leaves exactly one id per entry for this to write. + * + * A top-level companion is unaffected: it is a flat field standing for the whole + * document rather than for one entry, so an `array` reference's companion holds + * every id its entries reference, as it always has. + */ +function setIdentity( + document: ProjectedNode, + name: string, + ids: readonly string[], + field: ReferenceField, + nested: boolean, +): void { + if (!nested || field.array === true) { + setArray(document, name, ids); + return; } + setString(document, name, ids[0]); } /** @@ -746,6 +800,7 @@ function applyLocalIdentity( endpoints: readonly ProjectedNode[], field: ReferenceField, schema: SearchSchema, + nested: boolean, ): void { const names = physicalFields(field, schema); if (names.identity === undefined) { @@ -760,9 +815,7 @@ function applyLocalIdentity( .map((endpoint) => endpoint.id) .filter((id): id is string => typeof id === 'string'), ); - if (ids.length > 0) { - setArray(document, names.identity, ids); - } + setIdentity(document, names.identity, ids, field, nested); } /** The id a value under an identity field carries: the value itself when the @@ -797,8 +850,22 @@ function applyNestedReferents( schema: SearchSchema, context: ProjectionContext, ): readonly ProjectedNode[] { - const referents = values - .filter(isObject) + // One node may stand for several entries: a weldable leaf is single-valued, + // so an edge the graph gave several roles or several endpoints fans out into + // one entry per combination BEFORE it is projected (ADR 26). The cap bounds + // the DOCUMENT, not each edge – a document holds as many edges as the graph + // states, so a per-edge cap would still let the product grow with the input. + const limit = isInlineReference(field) + ? (field.ref.maxEntries ?? DEFAULT_MAX_ENTRIES) + : Number.POSITIVE_INFINITY; + const nodes: FramedNode[] = []; + for (const value of values.filter(isObject)) { + if (nodes.length >= limit) { + break; + } + nodes.push(...tuplesOf(value, nestedType, field, limit - nodes.length)); + } + const referents = nodes .map((referent) => projectFields(referent, nestedType, schema, context)) // Fields, not identity, are what makes something a referent: a literal // value object under the alias (dirty source data), or a node this @@ -813,6 +880,72 @@ function applyNestedReferents( return referents; } +/** + * Split one framed edge node into the entries it stands for: the cartesian + * product of its **weldable** leaves’ values, one value each. + * + * A weld asks whether ONE entry satisfies every condition, so a leaf a weld can + * name (`filterable` – `searchSchema` refuses `array` on one) states a single + * value. An edge the graph gave two roles and two endpoints is therefore four + * entries rather than one entry holding two lists, which stands for all four at + * once and answers the weld with none of them. See + * [ADR 26](../../docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md). + * + * Done on the **framed node**, before projection, so each leaf reaches + * {@link applyField} single-valued and passes through `transform`, folding and + * the facet companion exactly as a single-valued field always has – no + * downstream step learns that fan-out happened. Only the weldable aliases are + * split: an `output`-only leaf keeps its list, because nothing welds it, and it + * is shared unchanged across the entries the node fans out to. + * + * Inline references only. A {@link ReferenceStrategy.local local} lookup runs + * through this same body but nests the endpoint’s **own Root Type**, whose + * fields are multi-valued for reasons of their own – fanning one out would + * split a person across their `sameAs` values. What a weld names there is the + * flat `${name}_id` companion, which {@link applyLocalIdentity} already writes + * beside the object. + * + * `limit` is what remains of the document’s entry budget + * ({@link ReferenceStrategy.maxEntries}, {@link DEFAULT_MAX_ENTRIES}), so the + * product stops growing mid-way rather than being built and then trimmed – a + * bound in the data’s own units is not a bound (ADR 12), and one pathological + * edge would otherwise multiply a document until the run dies. + */ +function tuplesOf( + node: FramedNode, + nestedType: SearchType, + field: ReferenceField, + limit: number, +): readonly FramedNode[] { + if (!isInlineReference(field)) { + return [node]; + } + const weldable = nestedType.fields + .filter((nested) => nested.filterable === true) + .map((nested) => irAlias(nestedType, nested)) + // A leaf the frame carries at most one value for is already a tuple + // position; splitting it would copy the node to no purpose. + .filter((alias) => valuesOf(node, alias).length > 1); + if (weldable.length === 0) { + return [node]; + } + let tuples: FramedNode[] = [node]; + for (const alias of weldable) { + const values = valuesOf(node, alias); + const grown: FramedNode[] = []; + for (const tuple of tuples) { + for (const value of values) { + if (grown.length === limit) { + return grown; + } + grown.push({ ...tuple, [alias]: value }); + } + } + tuples = grown; + } + return tuples; +} + // --- Framed-IR readers: read a field’s value off the framed node by its // {@link irAlias IR Alias} key. Internal to projection – a `derive` reads the // projected document, never the node, so `path` stays the whole statement of diff --git a/packages/search/src/schema.ts b/packages/search/src/schema.ts index 9e603bab..a061f9a1 100644 --- a/packages/search/src/schema.ts +++ b/packages/search/src/schema.ts @@ -295,6 +295,22 @@ export type ReferenceStrategy = * keyed on a label, so two endpoints that share a label are never merged. */ readonly identity?: string; + /** + * Cap on the entries **one document** stores for this reference, across + * every edge it holds; entries past it are dropped. Defaults to + * {@link DEFAULT_MAX_ENTRIES}. + * + * A weldable leaf is single-valued, so an edge whose graph values are + * multi-valued fans out into one entry per combination + * ([ADR 26](../../docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md)). + * That product is bounded by the edge’s own data, and a bound in the + * data’s own units is not a bound + * ([ADR 12](../../docs/decisions/0012-bound-memory-by-the-unit-of-work-not-the-input.md)): + * one pathological edge would otherwise multiply a document until the run + * dies. Raise it for a corpus whose edges are legitimately wide; the + * default is far above what a qualified relation produces in practice. + */ + readonly maxEntries?: number; }; /** An IRI-valued reference to another entity, resolved at the surface. */ @@ -1208,10 +1224,20 @@ function assertNoInlineCycle( * Field}, the reading device that is the other half of an inline reference’s * job. * + * **`filterable` with `array` is refused**, for a reason about meaning rather + * than about any engine: a weld asks whether ONE entry satisfies every + * condition, so a leaf a weld can name states one value per entry. A leaf + * holding a list stands for every combination at once and answers the weld with + * none of them. Multiplicity belongs to the entry list instead – the projection + * emits one entry per combination + * ({@link ReferenceStrategy.maxEntries maxEntries}). An `output`-only nested + * leaf is untouched: nothing welds it, so it may carry a list for display. + * * Checked schema-wide, like the label sources and for the same reason: a single * declaration cannot see whether it is a Reference Type at all. * - * See [ADR 24](../../docs/decisions/0024-carry-data-on-a-reference-edge.md). + * See [ADR 24](../../docs/decisions/0024-carry-data-on-a-reference-edge.md) and + * [ADR 26](../../docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md). */ function assertServiceableNestedFields( referenceTypes: ReadonlyMap, @@ -1235,6 +1261,11 @@ function assertServiceableNestedFields( `Nested field “${referenceType.name}.${field.name}” declares a label source, which an inline reference cannot serve; declare a “lookup” on the nested reference instead of resolving a label for it.`, ); } + if (field.filterable === true && field.array === true) { + throw new Error( + `Nested field “${referenceType.name}.${field.name}” declares both “filterable” and “array”: a weld asks whether ONE entry satisfies every condition, and an entry holding a list has no single value to test – it stands for each combination at once, so the weld degenerates into the cross-product it exists to exclude. Declare the field single-valued; the projection emits one entry per combination (see “maxEntries”).`, + ); + } } } } @@ -1290,6 +1321,14 @@ const UNSERVICEABLE_INLINE_ROLES = ['searchable', 'sortable'] as const; * {@link SearchTypeBase.labelField}. */ export const DEFAULT_LABEL_FIELD = 'label'; +/** + * Entries one document stores for an inline reference that declares no + * {@link ReferenceStrategy.maxEntries maxEntries} of its own. Well above what a + * qualified relation produces – a measured corpus averages under four – so the + * default bounds the pathological case without truncating a real one. + */ +export const DEFAULT_MAX_ENTRIES = 100; + /** The `name` the type serves its label under: its declared * {@link SearchTypeBase.labelField}, else `label`. */ export function labelFieldNameOf(searchType: SearchType): string { diff --git a/packages/search/test/qualified-relation.test.ts b/packages/search/test/qualified-relation.test.ts index 9a91dff0..b2a0ea15 100644 --- a/packages/search/test/qualified-relation.test.ts +++ b/packages/search/test/qualified-relation.test.ts @@ -146,6 +146,196 @@ describe('an edge that carries data and resolves a lookup', () => { }); }); +describe('fanning an edge out into one entry per tuple', () => { + // A weld asks whether ONE entry satisfies every condition, so a leaf a weld + // can name holds one value. An edge the graph gave several fans out (ADR 26). + + /** Both leaves weldable, so both are tuple positions. */ + const weldableEdge = defineSearchType({ + name: 'CreatorEdge', + fields: [ + { + name: 'role', + kind: 'keyword', + path: `${SCHEMA_ORG}name`, + output: true, + filterable: true, + }, + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + output: true, + filterable: true, + ref: { strategy: 'lookup', target: 'Person', local: true }, + }, + ], + }); + + const twoRoles = { + '@id': 'https://ex/work/2', + [workKey('creator')]: [ + { + [edgeKey('role')]: [{ '@value': 'etser' }, { '@value': 'drukker' }], + [edgeKey('creator')]: [ + { '@id': 'https://a/1', [personKey('sameAs')]: [{ '@id': RKD }] }, + ], + }, + ], + }; + + it('splits a multi-valued weldable leaf across entries', () => { + const entries = entriesOf(projectDocument(twoRoles, work, schema)); + + expect(entries).toHaveLength(2); + expect(entries.map((entry) => entry.role)).toEqual(['etser', 'drukker']); + // Every entry keeps the endpoint the edge stated: the tuple is what fans + // out, not the edge's other values. + expect( + entries.map((entry) => (entry.creator as SearchDocument).id), + ).toEqual([RKD, RKD]); + }); + + it('takes the product where two weldable leaves are multi-valued', () => { + const twoOfEach = { + '@id': 'https://ex/work/3', + [workKey('creator')]: [ + { + [edgeKey('role')]: [{ '@value': 'etser' }, { '@value': 'drukker' }], + [edgeKey('creator')]: [ + { '@id': 'https://a/1', [personKey('sameAs')]: [{ '@id': RKD }] }, + { '@id': 'https://a/2' }, + ], + }, + ], + }; + const entries = entriesOf( + projectDocument( + twoOfEach, + work, + searchSchema(work, person, weldableEdge), + ), + ); + + expect(entries.map((entry) => [entry.role, entry.creator_id])).toEqual([ + ['etser', RKD], + ['etser', 'https://a/2'], + ['drukker', RKD], + ['drukker', 'https://a/2'], + ]); + }); + + it('leaves an output-only list on the entry', () => { + // Nothing welds it, so it needs no tuple position – and splitting the entry + // over it would multiply entries for a value no filter can name. + const noteEdge = defineSearchType({ + name: 'CreatorEdge', + fields: [ + { + name: 'role', + kind: 'keyword', + path: `${SCHEMA_ORG}name`, + output: true, + filterable: true, + }, + { + name: 'note', + kind: 'keyword', + path: `${SCHEMA_ORG}description`, + array: true, + output: true, + }, + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + output: true, + ref: { strategy: 'lookup', target: 'Person', local: true }, + }, + ], + }); + const annotated = { + '@id': 'https://ex/work/6', + [workKey('creator')]: [ + { + [edgeKey('role')]: [{ '@value': 'etser' }], + [edgeKey('note')]: [ + { '@value': 'gesigneerd' }, + { '@value': 'ovaal' }, + ], + }, + ], + }; + const entries = entriesOf( + projectDocument(annotated, work, searchSchema(work, person, noteEdge)), + ); + + expect(entries).toHaveLength(1); + expect(entries[0].note).toEqual(['gesigneerd', 'ovaal']); + }); + + it('stops at the cap once earlier edges have spent it', () => { + // The cap bounds the DOCUMENT, not each edge: a document holds as many + // edges as the graph states, so a per-edge cap would still let the entries + // grow with the input (ADR 12). `node` states two edges; a budget of one + // is spent by the first, and the second contributes nothing. + const cappedWork = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + array: true, + output: true, + filterable: true, + ref: { + strategy: 'inline', + typeName: 'CreatorEdge', + identity: 'creator', + maxEntries: 1, + }, + }, + ], + }); + const entries = entriesOf( + projectDocument( + node, + cappedWork, + searchSchema(cappedWork, person, creatorEdge), + ), + ); + + expect(entries).toHaveLength(1); + expect(entries[0].role).toBe('etser'); + }); + + it('does not fan a local lookup out over the endpoint’s own fields', () => { + // A `local` lookup nests the endpoint's own Root Type, whose fields are + // multi-valued for reasons of their own – `sameAs` here. Splitting on those + // would scatter one person across entries; what a weld names is the flat + // companion beside the object. + const twoAlignments = { + '@id': 'https://ex/work/7', + [workKey('creator')]: [ + { + [edgeKey('role')]: [{ '@value': 'etser' }], + [edgeKey('creator')]: [ + { + '@id': 'https://a/1', + [personKey('sameAs')]: [{ '@id': RKD }, { '@id': 'https://a/9' }], + }, + ], + }, + ], + }; + const entries = entriesOf(projectDocument(twoAlignments, work, schema)); + + expect(entries).toHaveLength(1); + }); +}); + describe('the identity companion', () => { it('harvests the ids the entries reference', () => { // The flat field an engine filters and facets in the nested object’s @@ -428,6 +618,43 @@ describe('welding conditions to one entry', () => { }); }); +describe('a local lookup at the root', () => { + it('harvests every endpoint into the flat companion', () => { + // A top-level companion stands for the whole DOCUMENT rather than for one + // entry, so nothing welds it and an `array` reference's companion holds + // every id its endpoints carry – unchanged by the nested rule (ADR 26). + const rootLookup = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + array: true, + output: true, + filterable: true, + ref: { strategy: 'lookup', target: 'Person', local: true }, + }, + ], + }); + const twoEndpoints = { + '@id': 'https://ex/work/8', + [workKey('creator')]: [ + { '@id': 'https://a/1', [personKey('sameAs')]: [{ '@id': RKD }] }, + { '@id': 'https://a/2' }, + ], + }; + const document = projectDocument( + twoEndpoints, + rootLookup, + searchSchema(rootLookup, person), + ); + + expect(document.creator_id).toEqual([RKD, 'https://a/2']); + }); +}); + describe('the identity companion of a local lookup', () => { // Its own id is a level deeper than a condition can be welded to, so // `filterable` fans it out as a leaf beside the stored object. @@ -460,49 +687,60 @@ describe('the identity companion of a local lookup', () => { ); const [identified] = document.creator as readonly SearchDocument[]; - expect(identified.creator_id).toEqual([RKD]); + expect(identified.creator_id).toBe(RKD); expect((identified.creator as SearchDocument).id).toBe(RKD); }); - it('holds only the endpoint a single-valued reference stores', () => { - // A single-valued reference keeps the first endpoint and drops the rest; - // a companion holding a dropped one's id would match a filter whose hit - // then shows a different endpoint. - const twoEndpoints = { - '@id': 'https://ex/work/4', - [workKey('creator')]: [ - { - [edgeKey('creator')]: [ - { '@id': 'https://a/1', [personKey('sameAs')]: [{ '@id': RKD }] }, - { '@id': 'https://a/2' }, - ], - }, - ], - }; - const singleEndpointEdge = defineSearchType({ - name: 'CreatorEdge', + it('holds only the entries the cap admits', () => { + // The one place entries are still dropped: a companion holding an id from + // a dropped entry would match a filter whose hit then shows no such entry. + const cappedWork = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, fields: [ { name: 'creator', kind: 'reference', path: `${SCHEMA_ORG}creator`, + array: true, output: true, filterable: true, - ref: { strategy: 'lookup', target: 'Person', local: true }, + ref: { + strategy: 'inline', + typeName: 'CreatorEdge', + identity: 'creator', + maxEntries: 2, + }, }, ], }); + const threeEndpoints = { + '@id': 'https://ex/work/4', + [workKey('creator')]: [ + { + [edgeKey('creator')]: [ + { '@id': 'https://a/1', [personKey('sameAs')]: [{ '@id': RKD }] }, + { '@id': 'https://a/2' }, + { '@id': 'https://a/3' }, + ], + }, + ], + }; const document = projectDocument( - twoEndpoints, - work, - searchSchema(work, person, singleEndpointEdge), + threeEndpoints, + cappedWork, + searchSchema(cappedWork, person, filterableEdge), ); - const [entry] = document.creator as readonly SearchDocument[]; - expect(entry.creator_id).toEqual([RKD]); + expect(document.creator).toHaveLength(2); + expect(document.creator_id).toEqual([RKD, 'https://a/2']); }); - it('holds every endpoint a multi-valued reference stores', () => { + it('fans a multi-valued endpoint out into one entry per endpoint', () => { + // The endpoint is what a weld names, so it is single-valued per entry: an + // edge the graph gave two endpoints is two entries, not one entry holding + // both. One entry holding both stands for either pairing and answers the + // weld with neither (ADR 26). const jointEdge = defineSearchType({ name: 'CreatorEdge', fields: [ @@ -510,7 +748,6 @@ describe('the identity companion of a local lookup', () => { name: 'creator', kind: 'reference', path: `${SCHEMA_ORG}creator`, - array: true, output: true, filterable: true, ref: { strategy: 'lookup', target: 'Person', local: true }, @@ -533,9 +770,13 @@ describe('the identity companion of a local lookup', () => { work, searchSchema(work, person, jointEdge), ); - const [entry] = document.creator as readonly SearchDocument[]; + const entries = document.creator as readonly SearchDocument[]; - expect(entry.creator_id).toEqual([RKD, 'https://a/2']); + expect(entries).toHaveLength(2); + expect(entries.map((entry) => entry.creator_id)).toEqual([ + RKD, + 'https://a/2', + ]); }); it('is absent where the endpoint is not identified', () => { diff --git a/packages/search/test/schema.test.ts b/packages/search/test/schema.test.ts index ea70f23c..76d528ea 100644 --- a/packages/search/test/schema.test.ts +++ b/packages/search/test/schema.test.ts @@ -1141,7 +1141,6 @@ describe('searchSchema validation', () => { { name: 'contentUrl', kind: 'keyword', - array: true, output: true, path: 'https://schema.org/contentUrl', ...field, @@ -1231,6 +1230,29 @@ describe('searchSchema validation', () => { ).not.toThrow(); }); + it('rejects a nested field declaring both filterable and array', () => { + // A weld asks whether ONE entry satisfies every condition, so a leaf a + // weld can name holds one value. A leaf holding a list stands for every + // combination at once and answers the weld with none of them – the + // projection fans the entry out instead (ADR 26). + expect(() => + searchSchema( + datasetNesting({ strategy: 'inline', typeName: 'MediaObject' }), + mediaObjectWith({ filterable: true, array: true }), + ), + ).toThrow(/declares both “filterable” and “array”/u); + }); + + it('accepts an output-only nested field declaring array', () => { + // Nothing welds it, so an entry may carry a list for display. + expect(() => + searchSchema( + datasetNesting({ strategy: 'inline', typeName: 'MediaObject' }), + mediaObjectWith({ array: true }), + ), + ).not.toThrow(); + }); + it.each([ ['searchable', { searchable: { weight: 1 } }], ['sortable', { sortable: true }], From 81d4a6b4221aa3e20ca11280358aa845d49bb36f Mon Sep 17 00:00:00 2001 From: David de Boer Date: Wed, 2 Sep 2026 20:34:23 +0200 Subject: [PATCH 2/8] fix(search): type a nested identity companion by the arity of its path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects found reviewing the fan-out, two of them introduced by it and each invisible without a live engine. - restore the already-a-list case when widening a nested leaf. Only Reference Types are held to the single-valued rule, and the same path declares the fields of the Root Type a local lookup nests, where array and filterable meet on an ordinary facet – the leaf was emitted with no type at all, which the engine refuses outright - declare a nested identity companion by what its path yields rather than always as a list. Under a single-valued edge the parent is an object, nothing flattens, and Typesense rejects the scalar the projection writes against a declared list: the whole import fails - expand every tuple the cap keeps. Stopping mid-expansion left the remaining weldable leaves holding their lists, and a single-valued leaf then kept the first value and dropped the rest – losing data quietly rather than fanning it out Two existing tests asserted the arity rule inverted, stating that a companion is always a list and that a single value is what the engine rejects. A live import says the opposite, so both now read the other way. An identity on a Reference Type's field is already refused, and a nested filterable lookup can no longer be an array, so a nested companion never holds more than one id; no change was needed there. The entry budget is per node carrying the reference rather than per document, so nested edges compound it – documented rather than changed. --- .../src/collection-definition.ts | 50 ++++--- .../test/qualified-relation.test.ts | 131 ++++++++++++++++-- .../test/welded-filter.integration.test.ts | 27 ++++ packages/search-typesense/vite.config.ts | 2 +- packages/search/src/project.ts | 17 ++- packages/search/src/schema.ts | 12 +- .../search/test/qualified-relation.test.ts | 60 ++++++++ 7 files changed, 261 insertions(+), 38 deletions(-) diff --git a/packages/search-typesense/src/collection-definition.ts b/packages/search-typesense/src/collection-definition.ts index 5275c91d..9776fdca 100644 --- a/packages/search-typesense/src/collection-definition.ts +++ b/packages/search-typesense/src/collection-definition.ts @@ -441,7 +441,7 @@ function nestedFields( index: false, optional: true, }, - ...nestedIdentityFields(prefix, field, schema), + ...nestedIdentityFields(prefix, field, schema, flattensToArray), ); continue; } @@ -461,7 +461,7 @@ function nestedFields( // an engine welds conditions on an entry's LEAF fields only. Its // identity companion is that leaf, so it sits beside the object rather // than inside it. - ...nestedIdentityFields(prefix, field, schema), + ...nestedIdentityFields(prefix, field, schema, flattensToArray), ); continue; } @@ -530,6 +530,7 @@ function nestedIdentityFields( prefix: string, field: SearchField, schema: SearchSchema, + flattensToArray: boolean, ): CollectionFieldSchema[] { const names = physicalFields(field, schema); if (names.identity === undefined) { @@ -538,13 +539,13 @@ function nestedIdentityFields( return [ { name: nestedFieldName(prefix, names.identity), - // Always a list, whatever the enclosing reference's arity, and whatever - // ONE entry holds: a nested field's declared type describes the whole - // path across the document, and a document holds many entries – so - // `string` here rejects every document carrying such an edge, at import. - // The entries themselves store a single id each, which the engine accepts - // under this type and which is what makes the weld answerable (ADR 26). - type: 'string[]', + // Typed by what the PATH yields across the document, exactly as the `id` + // beside it is: one value per entry (ADR 26 – this companion is the leaf + // a weld names, so it holds a single id), and as many entries as an + // `object[]` ancestor allows. Declaring `string[]` unconditionally + // rejects, at import, every document whose edge is single-valued – + // Typesense enforces the declared arity where nothing flattens it. + type: flattensToArray ? 'string[]' : 'string', index: true, optional: true, }, @@ -615,18 +616,18 @@ function nestedLeafFields( // itself – its search companion below indexes on its own account. const indexed = field.filterable === true; const valueType = typesenseValueType(field); - // An ancestor's `object[]` flattens this value into a list, so an indexed - // leaf must be declared as one or the import rejects every document carrying - // the edge. The field's OWN `array` cannot also be widening it here: - // `searchSchema` refuses `filterable` alongside `array` on a nested field, so - // an indexed leaf is single-valued (ADR 26) – which is what makes the cast - // below sound, `typesenseValueType` returning `string[]` only for `array`. + // Two ways a list arrives: the field declares one, or an ancestor's + // `object[]` flattens it into one. `typesenseValueType` honours `array` only + // for the string-shaped kinds, so a multi-valued nested `integer` would + // otherwise be declared scalar and rejected at import. + // + // A weldable leaf of a Reference Type is single-valued (ADR 26), but that + // narrows nothing here: this same path declares the fields of the Root Type a + // `local` lookup nests, where `array` and `filterable` may legitimately meet. + const storesAList = flattensToArray || field.array === true; fields.push({ name: nestedFieldName(prefix, field.name), - type: - indexed && flattensToArray - ? arrayValueType(valueType as Exclude) - : valueType, + type: indexed && storesAList ? arrayValueType(valueType) : valueType, index: indexed, // Always optional: `required` is a promise about the *referent* (this value // is on every referent), while Typesense’s flag is about the document. @@ -654,9 +655,7 @@ function nestedLeafFields( * declaration is. Enumerated rather than string-appended so an invalid * combination cannot be built. */ -function arrayValueType( - type: Exclude, -): CollectionFieldSchema['type'] { +function arrayValueType(type: ValueType): CollectionFieldSchema['type'] { switch (type) { case 'string': return 'string[]'; @@ -666,6 +665,13 @@ function arrayValueType( return 'float[]'; case 'bool': return 'bool[]'; + // Already a list, and reachable even though a weldable leaf of a Reference + // Type is single-valued (ADR 26): this path also declares the fields of the + // Root Type a `local` lookup nests, and `searchSchema` constrains only + // Reference Types. A root keyword may be `array` and `filterable` at once – + // an ordinary facet – so flattening it must widen no further. + case 'string[]': + return 'string[]'; } } diff --git a/packages/search-typesense/test/qualified-relation.test.ts b/packages/search-typesense/test/qualified-relation.test.ts index 183d5851..f06b098f 100644 --- a/packages/search-typesense/test/qualified-relation.test.ts +++ b/packages/search-typesense/test/qualified-relation.test.ts @@ -148,17 +148,19 @@ describe('a single-valued edge', () => { }); const singleSchema = searchSchema(singleWork, person, creatorEdge); - it('declares the nested identity companion as a list', () => { - // The projection writes it with `setArray` whatever the arity, and an - // indexed field's declared type is checked at import – a `string` here - // rejects every document carrying such an edge. + it('declares the nested identity companion as a single value', () => { + // Nothing flattens the path under a single-valued edge – the parent is + // `object`, not `object[]` – and the companion holds one id per entry + // (ADR 26). Declaring `string[]` here is what rejects the document: checked + // against a live engine, the import fails outright, because Typesense + // enforces the declared arity wherever no ancestor widens it. const fields = buildCollectionDefinition(singleWork, { schema: singleSchema }).fields ?? []; expect( fields.find((field) => field.name === 'creator.creator_id'), - ).toMatchObject({ type: 'string[]', index: true }); + ).toMatchObject({ type: 'string', index: true }); }); }); @@ -287,11 +289,12 @@ describe('a local lookup that reaches back', () => { fields.find((field) => field.name === 'creator.made.id'), ).toMatchObject({ type: scalar, index: false }); // The identity companion sits BESIDE the object, indexed, because that is - // the leaf a filter can weld on – always a list, whatever the arity of - // the edge it hangs off. + // the leaf a filter can weld on – and it takes the arity of the path that + // reaches it, exactly as the `id` above does: one id per entry (ADR 26), + // widened only where an `object[]` ancestor multiplies the entries. expect( fields.find((field) => field.name === 'creator.made_id'), - ).toMatchObject({ type: 'string[]', index: true }); + ).toMatchObject({ type: scalar, index: true }); // The descent still stops: the cut type's own fields are not walked // again. expect( @@ -483,6 +486,118 @@ describe('nested fields of other kinds', () => { expect(richField(name)).toMatchObject({ type, index: true }); }); + it('types a nested identity companion by what its path yields', () => { + // The companion holds one id per entry (ADR 26), so its declared type is + // decided by the ancestors, exactly as the `id` beside it is. Declaring + // `string[]` unconditionally makes Typesense reject, at import, every + // document whose edge is single-valued: nothing flattens the path there, + // and the engine enforces the declared arity. + const person = defineSearchType({ + name: 'Person', + class: `${SCHEMA_ORG}Person`, + fields: [ + { + name: 'label', + kind: 'text', + path: `${SCHEMA_ORG}name`, + locales: ['und'], + output: true, + searchable: { weight: 1 }, + }, + ], + }); + const edge = defineSearchType({ + name: 'IdentifiedEdge', + fields: [ + { + name: 'agent', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + output: true, + filterable: true, + ref: { strategy: 'lookup', target: 'Person', local: true }, + }, + ], + }); + const workWith = (array: boolean) => + defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'credit', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + ...(array ? { array: true } : {}), + output: true, + ref: { strategy: 'inline', typeName: 'IdentifiedEdge' }, + }, + ], + }); + const companion = (array: boolean) => { + const type = workWith(array); + return ( + buildCollectionDefinition(type, { + schema: searchSchema(type, person, edge), + }).fields ?? [] + ).find((field) => field.name === 'credit.agent_id'); + }; + + expect(companion(true)).toMatchObject({ type: 'string[]' }); + expect(companion(false)).toMatchObject({ type: 'string' }); + }); + + it('widens an indexed nested leaf of a local lookup’s own root type', () => { + // `searchSchema` constrains Reference Types, so a weldable leaf there is + // single-valued – but this same path also declares the fields of the Root + // Type a `local` lookup nests, where `array` and `filterable` meet on an + // ordinary facet. Widening must still produce a type. + const agent = defineSearchType({ + name: 'Agent', + class: `${SCHEMA_ORG}Person`, + fields: [ + { + name: 'label', + kind: 'text', + path: `${SCHEMA_ORG}name`, + locales: ['und'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'nationality', + kind: 'keyword', + path: `${SCHEMA_ORG}nationality`, + array: true, + output: true, + filterable: true, + }, + ], + }); + const work = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + array: true, + output: true, + ref: { strategy: 'lookup', target: 'Agent', local: true }, + }, + ], + }); + const fields = + buildCollectionDefinition(work, { + schema: searchSchema(work, agent), + }).fields ?? []; + + expect( + fields.find((field) => field.name === 'creator.nationality'), + ).toMatchObject({ type: 'string[]', index: true }); + }); + it('does not double a nested list that is already one', () => { // A leaf a weld can name is single-valued (ADR 26), so a nested list is // either output-only or searchable. Flattening one under the `object[]` diff --git a/packages/search-typesense/test/welded-filter.integration.test.ts b/packages/search-typesense/test/welded-filter.integration.test.ts index 58bf7f79..b0a7eeac 100644 --- a/packages/search-typesense/test/welded-filter.integration.test.ts +++ b/packages/search-typesense/test/welded-filter.integration.test.ts @@ -108,6 +108,33 @@ describe('a welded co-element filter', () => { expect(await found(welded(drukker, other))).toEqual([]); }); + it('accepts a single-valued edge’s companion, declared as one value', async () => { + // A declared type only matters if the engine agrees with it, and here it + // disagreed: under a single-valued edge the parent is `object`, nothing + // flattens the path, and Typesense rejects a scalar companion declared + // `string[]` – the whole import fails. Pinned live, because the collection + // definition alone cannot show it. + const single = 'single_edge_works'; + await client.collections().create({ + name: single, + enable_nested_fields: true, + fields: [ + { name: 'credit', type: 'object' }, + { name: 'credit.role', type: 'string' }, + // One id per entry, and one entry: no ancestor widens the path. + { name: 'credit.agent_id', type: 'string' }, + ], + }); + const result = await client + .collections(single) + .documents() + .import([{ id: 'w1', credit: { role: etser, agent_id: rembrandt } }], { + action: 'create', + }); + + expect(JSON.stringify(result)).toContain('"success":true'); + }, 60_000); + it('hangs on 30.2 where an entry holds arrays, which is why we fan out', async () => { // The defect this shape exists to avoid, pinned so a future engine bump // tells us when it is gone. Typesense answers every OTHER form of this diff --git a/packages/search-typesense/vite.config.ts b/packages/search-typesense/vite.config.ts index 3a732f06..89bda4e6 100644 --- a/packages/search-typesense/vite.config.ts +++ b/packages/search-typesense/vite.config.ts @@ -28,7 +28,7 @@ export default mergeConfig( // projection naming what no lookup reaches are unreachable through // the port, since `assertValidQuery` rejects such a query first. // They hold for a direct caller, and are exercised as one. - branches: 95.82, + branches: 95.85, statements: 99.48, }, }, diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index 4fb5ee78..fa1e7dfd 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -852,9 +852,10 @@ function applyNestedReferents( ): readonly ProjectedNode[] { // One node may stand for several entries: a weldable leaf is single-valued, // so an edge the graph gave several roles or several endpoints fans out into - // one entry per combination BEFORE it is projected (ADR 26). The cap bounds - // the DOCUMENT, not each edge – a document holds as many edges as the graph - // states, so a per-edge cap would still let the product grow with the input. + // one entry per combination BEFORE it is projected (ADR 26). The budget spans + // every edge THIS node states, rather than resetting per edge – a node holds + // as many edges as the graph gives it, so a per-edge cap would still let the + // entries grow with the input. const limit = isInlineReference(field) ? (field.ref.maxEntries ?? DEFAULT_MAX_ENTRIES) : Number.POSITIVE_INFINITY; @@ -934,13 +935,21 @@ function tuplesOf( const values = valuesOf(node, alias); const grown: FramedNode[] = []; for (const tuple of tuples) { + if (grown.length === limit) { + break; + } for (const value of values) { if (grown.length === limit) { - return grown; + break; } grown.push({ ...tuple, [alias]: value }); } } + // Capped per alias rather than by returning from inside this loop, so every + // tuple that survives is split across EVERY weldable alias. Returning early + // would hand back tuples whose remaining aliases still held their lists, + // and a single-valued leaf then keeps the first value and drops the rest – + // silent data loss in place of the fan-out this exists to perform. tuples = grown; } return tuples; diff --git a/packages/search/src/schema.ts b/packages/search/src/schema.ts index a061f9a1..38a6ef2b 100644 --- a/packages/search/src/schema.ts +++ b/packages/search/src/schema.ts @@ -296,9 +296,15 @@ export type ReferenceStrategy = */ readonly identity?: string; /** - * Cap on the entries **one document** stores for this reference, across - * every edge it holds; entries past it are dropped. Defaults to - * {@link DEFAULT_MAX_ENTRIES}. + * Cap on the entries this reference stores **per node carrying it** – + * across every edge that node states, and so per document for a + * reference declared on a Root Type. Entries past it are dropped. + * Defaults to {@link DEFAULT_MAX_ENTRIES}. + * + * A reference nested inside another reference type is budgeted per + * *parent entry* rather than per document, so a schema nesting one edge + * inside another admits up to the product of their caps. Bounded, but + * multiplicatively: set the inner cap with the outer one in mind. * * A weldable leaf is single-valued, so an edge whose graph values are * multi-valued fans out into one entry per combination diff --git a/packages/search/test/qualified-relation.test.ts b/packages/search/test/qualified-relation.test.ts index b2a0ea15..f6c2e38c 100644 --- a/packages/search/test/qualified-relation.test.ts +++ b/packages/search/test/qualified-relation.test.ts @@ -311,6 +311,66 @@ describe('fanning an edge out into one entry per tuple', () => { expect(entries[0].role).toBe('etser'); }); + it('fully expands every entry it keeps when the cap bites', () => { + // The cap drops whole tuples, never half-expanded ones. Capping by + // returning mid-expansion would hand back entries whose remaining weldable + // leaves still held their lists – and a single-valued leaf then keeps the + // first value and drops the rest, losing data silently instead of fanning + // it out. + const cappedWork = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + array: true, + output: true, + filterable: true, + ref: { + strategy: 'inline', + typeName: 'CreatorEdge', + identity: 'creator', + maxEntries: 3, + }, + }, + ], + }); + const wide = { + '@id': 'https://ex/work/9', + [workKey('creator')]: [ + { + [edgeKey('role')]: [ + { '@value': 'etser' }, + { '@value': 'drukker' }, + { '@value': 'uitgever' }, + ], + [edgeKey('creator')]: [ + { '@id': 'https://a/1', [personKey('sameAs')]: [{ '@id': RKD }] }, + { '@id': 'https://a/2' }, + ], + }, + ], + }; + const entries = entriesOf( + projectDocument( + wide, + cappedWork, + searchSchema(cappedWork, person, weldableEdge), + ), + ); + + expect(entries).toHaveLength(3); + // Both leaves participate in every kept tuple; none is pinned to its first + // value because expansion stopped early. + expect(entries.map((entry) => [entry.role, entry.creator_id])).toEqual([ + ['etser', RKD], + ['etser', 'https://a/2'], + ['drukker', RKD], + ]); + }); + it('does not fan a local lookup out over the endpoint’s own fields', () => { // A `local` lookup nests the endpoint's own Root Type, whose fields are // multi-valued for reasons of their own – `sameAs` here. Splitting on those From 9dd2a2372a09b9059ff138f2eac2eb9df7a98f7d Mon Sep 17 00:00:00 2001 From: David de Boer Date: Thu, 3 Sep 2026 09:15:15 +0200 Subject: [PATCH 3/8] docs(search): record what a companion's declared type follows, and the arity 30.2 will not weld Two limitations found verifying the fan-out against a live engine, neither visible from a collection definition alone. - a nested identity companion is typed by the path that reaches it rather than by what one entry holds, and Typesense enforces that wherever nothing widens the path - the weld hangs on 30.2 against a single-valued edge whatever its leaves hold, which is a different defect from the array one and is not addressed by fanning out. It costs nothing today, because a qualified relation is multi-valued by nature --- ...qualified-edge-into-one-entry-per-tuple.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md b/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md index f6fa3352..663078e5 100644 --- a/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md +++ b/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md @@ -178,3 +178,22 @@ a message saying to fan out instead. We are pre-release; there is no migration. **One array-valued weldable leaf re-arms the hang**, which is why part 1 is a refusal rather than a convention. A deployment cannot opt out of it by accident, and the failure is at startup rather than a query that never returns. + +**A companion’s declared type follows its path, not the leaf.** The flat id a +weld actually names holds one value per entry, but its declared type describes +the whole path across the document: `string[]` under an `object[]` edge, +`string` under a single-valued one. Typesense enforces that strictly wherever +nothing widens the path – a companion declared `string[]` under a single-valued +edge fails the import outright, for every document carrying such an edge. This +is not visible in a collection definition, only against a live engine, which is +why the integration test asserts the import rather than the declaration. + +**A single-valued edge still cannot be welded on 30.2.** Where the reference is +not `array`, the stored parent is `object` rather than `object[]`, and the +engine hangs on `credit.{…}` over it whatever the leaves hold – so this is not +the array defect above and fan-out does not address it. It costs nothing today: +a qualified relation is multi-valued by nature and every real edge declares +`array: true`, which is the shape [ADR 24](./0024-carry-data-on-a-reference-edge.md) +describes. A deployment that genuinely wants one qualified edge per document +should still declare it `array: true` and rely on the entries, until 31.0 is +stable. From 5e01aa143092438a63278c62515390a85e205590 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Thu, 3 Sep 2026 19:20:58 +0200 Subject: [PATCH 4/8] fix(search): decide nesting from where a node is projected, not from its type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Root Type reached by a local lookup is nested exactly as a Reference Type is; it just happens to have a collection of its own elsewhere. Reading that from the type made the projection treat a locally-nested root as a root, so it wrote a list where the collection declared a single value – and Typesense rejects the document at import, for every document carrying such a reference. - thread the nesting flag from the call site into projectFields, rather than deriving it from whether the declaring type declares a class - declare a nested identity companion by both routes to a list, the field's own array as well as a flattening ancestor, as every sibling declaration in that function already does - reject a maxEntries that is not a positive integer. The budget is counted off one entry at a time, so a fractional or non-positive cap is never reached and the fan-out grows with the data – the unbounded case the cap exists to prevent. Compare with >= rather than ==, so no cap can be stepped over - export DEFAULT_MAX_ENTRIES beside DEFAULT_LABEL_FIELD, and state the number in the option's own documentation, so the default is legible without it Left alone deliberately: a single-valued inline reference still fans out up to the full budget before keeping one entry. Capping it at one would change which entry survives when the first projects empty, and the waste is bounded. --- .../src/collection-definition.ts | 15 ++-- .../test/qualified-relation.test.ts | 66 ++++++++++++++ packages/search-typesense/vite.config.ts | 2 +- packages/search/src/adapter.ts | 1 + packages/search/src/project.ts | 22 +++-- packages/search/src/schema.ts | 17 +++- .../search/test/qualified-relation.test.ts | 88 +++++++++++++++++++ packages/search/test/schema.test.ts | 34 +++++++ packages/search/vite.config.ts | 2 +- 9 files changed, 226 insertions(+), 21 deletions(-) diff --git a/packages/search-typesense/src/collection-definition.ts b/packages/search-typesense/src/collection-definition.ts index 9776fdca..41bf5140 100644 --- a/packages/search-typesense/src/collection-definition.ts +++ b/packages/search-typesense/src/collection-definition.ts @@ -540,12 +540,15 @@ function nestedIdentityFields( { name: nestedFieldName(prefix, names.identity), // Typed by what the PATH yields across the document, exactly as the `id` - // beside it is: one value per entry (ADR 26 – this companion is the leaf - // a weld names, so it holds a single id), and as many entries as an - // `object[]` ancestor allows. Declaring `string[]` unconditionally - // rejects, at import, every document whose edge is single-valued – - // Typesense enforces the declared arity where nothing flattens it. - type: flattensToArray ? 'string[]' : 'string', + // beside it is, and by the same two routes every other declaration here + // uses: an `object[]` ancestor multiplying the entries, or the field's + // own `array` making each entry hold a list. A weldable leaf is + // single-valued (ADR 26), so a companion that a weld names contributes + // one id per entry – but a multi-valued reference reached through a + // locally-nested Root Type is not weldable and still harvests a list. + // Getting either route wrong rejects the document at import: Typesense + // enforces the declared arity wherever nothing widens the path. + type: flattensToArray || field.array === true ? 'string[]' : 'string', index: true, optional: true, }, diff --git a/packages/search-typesense/test/qualified-relation.test.ts b/packages/search-typesense/test/qualified-relation.test.ts index f06b098f..5ad0682d 100644 --- a/packages/search-typesense/test/qualified-relation.test.ts +++ b/packages/search-typesense/test/qualified-relation.test.ts @@ -547,6 +547,72 @@ describe('nested fields of other kinds', () => { expect(companion(false)).toMatchObject({ type: 'string' }); }); + it('declares a multi-valued nested companion as a list, unflattened', () => { + // The companion's other route to a list: the reference itself is `array`, + // so one entry harvests several ids even where no ancestor multiplies the + // entries. Reachable through a locally-nested Root Type, whose fields the + // single-valued rule does not constrain – and the projection writes a list + // there, so declaring `string` rejects the document at import. + const org = defineSearchType({ + name: 'Membership', + fields: [ + { + name: 'org', + kind: 'reference', + path: `${SCHEMA_ORG}memberOf`, + output: true, + ref: { strategy: 'lookup', target: 'Person' }, + }, + ], + }); + const nestedRoot = defineSearchType({ + name: 'Person', + class: `${SCHEMA_ORG}Person`, + fields: [ + { + name: 'label', + kind: 'text', + path: `${SCHEMA_ORG}name`, + locales: ['und'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'affiliation', + kind: 'reference', + path: `${SCHEMA_ORG}affiliation`, + array: true, + output: true, + filterable: true, + ref: { strategy: 'inline', typeName: 'Membership', identity: 'org' }, + }, + ], + }); + // Single-valued, so nothing above flattens: only the field's own `array` + // makes this a list. + const work = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + output: true, + ref: { strategy: 'lookup', target: 'Person', local: true }, + }, + ], + }); + const fields = + buildCollectionDefinition(work, { + schema: searchSchema(work, nestedRoot, org), + }).fields ?? []; + + expect( + fields.find((field) => field.name === 'creator.affiliation_id'), + ).toMatchObject({ type: 'string[]', index: true }); + }); + it('widens an indexed nested leaf of a local lookup’s own root type', () => { // `searchSchema` constrains Reference Types, so a weldable leaf there is // single-valued – but this same path also declares the fields of the Root diff --git a/packages/search-typesense/vite.config.ts b/packages/search-typesense/vite.config.ts index 89bda4e6..61247c5e 100644 --- a/packages/search-typesense/vite.config.ts +++ b/packages/search-typesense/vite.config.ts @@ -28,7 +28,7 @@ export default mergeConfig( // projection naming what no lookup reaches are unreachable through // the port, since `assertValidQuery` rejects such a query first. // They hold for a direct caller, and are exercised as one. - branches: 95.85, + branches: 95.87, statements: 99.48, }, }, diff --git a/packages/search/src/adapter.ts b/packages/search/src/adapter.ts index 3437e99f..7d66cd92 100644 --- a/packages/search/src/adapter.ts +++ b/packages/search/src/adapter.ts @@ -40,6 +40,7 @@ export { labelTargetNameOf, documentKeyOf, DEFAULT_LABEL_FIELD, + DEFAULT_MAX_ENTRIES, isRangeFacet, isAbsoluteIri, isoToUnixSeconds, diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index fa1e7dfd..9d974111 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -156,11 +156,12 @@ function projectFields( searchType: SearchType, schema: SearchSchema | undefined, context: ProjectionContext, + nested = false, ): ProjectedNode { const id = documentIdOf(node, searchType); const document: ProjectedNode = id === undefined ? {} : { id }; for (const field of searchType.fields) { - applyField(document, node, field, searchType, schema, context); + applyField(document, node, field, searchType, schema, context, nested); } return document; } @@ -261,6 +262,7 @@ function applyField( searchType: SearchType, schema: SearchSchema | undefined, context: ProjectionContext, + nested: boolean, ): void { // The three value sources, mutually exclusive by declaration // (`validateSearchType`): a projection value, a computed value, a graph path. @@ -327,7 +329,7 @@ function applyField( field, schema, context, - searchType.class === undefined, + nested, ); } return; @@ -351,13 +353,7 @@ function applyField( // a filter can reach – an engine welds conditions on an entry's LEAF // fields only. `filterable` therefore fans out the id beside the object, // exactly as an inline reference's identity companion does. - applyLocalIdentity( - document, - endpoints, - field, - schema, - searchType.class === undefined, - ); + applyLocalIdentity(document, endpoints, field, schema, nested); return; } } @@ -867,7 +863,9 @@ function applyNestedReferents( nodes.push(...tuplesOf(value, nestedType, field, limit - nodes.length)); } const referents = nodes - .map((referent) => projectFields(referent, nestedType, schema, context)) + .map((referent) => + projectFields(referent, nestedType, schema, context, true), + ) // Fields, not identity, are what makes something a referent: a literal // value object under the alias (dirty source data), or a node this // reference type reads nothing from, projects nothing and is no referent. @@ -935,11 +933,11 @@ function tuplesOf( const values = valuesOf(node, alias); const grown: FramedNode[] = []; for (const tuple of tuples) { - if (grown.length === limit) { + if (grown.length >= limit) { break; } for (const value of values) { - if (grown.length === limit) { + if (grown.length >= limit) { break; } grown.push({ ...tuple, [alias]: value }); diff --git a/packages/search/src/schema.ts b/packages/search/src/schema.ts index 38a6ef2b..781f4601 100644 --- a/packages/search/src/schema.ts +++ b/packages/search/src/schema.ts @@ -299,7 +299,9 @@ export type ReferenceStrategy = * Cap on the entries this reference stores **per node carrying it** – * across every edge that node states, and so per document for a * reference declared on a Root Type. Entries past it are dropped. - * Defaults to {@link DEFAULT_MAX_ENTRIES}. + * Defaults to 100 ({@link DEFAULT_MAX_ENTRIES}). Must be a positive + * integer: the budget is counted off one entry at a time, so a cap that + * is not one can never be reached. * * A reference nested inside another reference type is budgeted per * *parent entry* rather than per document, so a schema nesting one edge @@ -1454,6 +1456,7 @@ export interface SearchTypeIssue { | 'invalid-locale' | 'missing-ref' | 'missing-ref-type-name' + | 'invalid-max-entries' | 'ref-not-allowed' | 'text-requires-locales' | 'locales-not-allowed' @@ -1657,6 +1660,18 @@ export function validateSearchType( ) { issue('missing-ref-type-name'); } + // The fan-out budget is counted off one entry at a time, so a fractional + // or non-positive cap is never reached and the cartesian product in + // `tuplesOf` grows with the data – exactly the unbounded case the cap + // exists to prevent (ADR 12). A cap that cannot bind is worse than none, + // because the declaration says otherwise. + if ( + field.ref?.strategy === 'inline' && + field.ref.maxEntries !== undefined && + !(Number.isInteger(field.ref.maxEntries) && field.ref.maxEntries > 0) + ) { + issue('invalid-max-entries'); + } // A join addresses the referent's collection – the one a lookup's // `target` or an idOnly's `labelSource` names. With neither, the flag // states an edge to nowhere. diff --git a/packages/search/test/qualified-relation.test.ts b/packages/search/test/qualified-relation.test.ts index f6c2e38c..bac4463e 100644 --- a/packages/search/test/qualified-relation.test.ts +++ b/packages/search/test/qualified-relation.test.ts @@ -678,6 +678,94 @@ describe('welding conditions to one entry', () => { }); }); +describe('nesting is where a node is projected, not what type it is', () => { + // A Root Type reached by a `local` lookup is nested exactly as a Reference + // Type is – it just happens to have a collection of its own elsewhere. Its + // companions must therefore be written under the nested rule too. Deciding + // that from the TYPE rather than from the projection context reads a + // locally-nested root as a root, and the arity it writes then disagrees with + // the one the collection declares: the import fails for every such document. + const inner = defineSearchType({ + name: 'Membership', + fields: [ + { + name: 'org', + kind: 'reference', + path: `${SCHEMA_ORG}memberOf`, + output: true, + ref: { strategy: 'lookup', target: 'Person' }, + }, + ], + }); + const nestedRoot = defineSearchType({ + name: 'Person', + class: `${SCHEMA_ORG}Person`, + key: { field: 'sameAs' }, + fields: [ + { + name: 'label', + kind: 'text', + path: `${SCHEMA_ORG}name`, + locales: ['und'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'sameAs', + kind: 'reference', + path: `${SCHEMA_ORG}sameAs`, + array: true, + }, + { + name: 'affiliation', + kind: 'reference', + path: `${SCHEMA_ORG}affiliation`, + output: true, + filterable: true, + ref: { strategy: 'inline', typeName: 'Membership', identity: 'org' }, + }, + ], + }); + const work = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + output: true, + ref: { strategy: 'lookup', target: 'Person', local: true }, + }, + ], + }); + + it('writes a single-valued companion inside a locally-nested root type', () => { + const node = { + '@id': 'https://ex/work/10', + [workKey('creator')]: [ + { + '@id': 'https://p/1', + [personKey('sameAs')]: [{ '@id': RKD }], + [alias('Person', 'affiliation')]: [ + { [alias('Membership', 'org')]: [{ '@id': 'https://o/1' }] }, + ], + }, + ], + }; + const document = projectDocument( + node, + work, + searchSchema(work, nestedRoot, inner), + ); + const endpoint = document.creator as SearchDocument; + + // A single value, matching what the collection declares for this path - + // not the one-element list a root-level companion would carry. + expect(endpoint.affiliation_id).toBe('https://o/1'); + }); +}); + describe('a local lookup at the root', () => { it('harvests every endpoint into the flat companion', () => { // A top-level companion stands for the whole DOCUMENT rather than for one diff --git a/packages/search/test/schema.test.ts b/packages/search/test/schema.test.ts index 76d528ea..4b1a12c3 100644 --- a/packages/search/test/schema.test.ts +++ b/packages/search/test/schema.test.ts @@ -1230,6 +1230,40 @@ describe('searchSchema validation', () => { ).not.toThrow(); }); + it.each([ + ['a fraction', 2.5], + ['zero', 0], + ['a negative', -1], + ])('rejects maxEntries that is %s', (_label, maxEntries) => { + // The budget is counted off one entry at a time, so a cap that is not a + // positive integer is never reached and the fan-out grows with the data – + // the unbounded case the cap exists to prevent. A cap that cannot bind is + // worse than none, because the declaration claims otherwise. + expect(() => + searchSchema( + datasetNesting({ + strategy: 'inline', + typeName: 'MediaObject', + maxEntries, + }), + mediaObjectWith({}), + ), + ).toThrow(/invalid-max-entries/u); + }); + + it('accepts a positive integer maxEntries', () => { + expect(() => + searchSchema( + datasetNesting({ + strategy: 'inline', + typeName: 'MediaObject', + maxEntries: 5, + }), + mediaObjectWith({}), + ), + ).not.toThrow(); + }); + it('rejects a nested field declaring both filterable and array', () => { // A weld asks whether ONE entry satisfies every condition, so a leaf a // weld can name holds one value. A leaf holding a list stands for every diff --git a/packages/search/vite.config.ts b/packages/search/vite.config.ts index 3066345a..92874588 100644 --- a/packages/search/vite.config.ts +++ b/packages/search/vite.config.ts @@ -12,7 +12,7 @@ export default mergeConfig( thresholds: { functions: 100, lines: 100, - branches: 99.71, + branches: 99.72, statements: 100, }, }, From c56bdae11682ae91114d91e7204e8054350ae371 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Fri, 4 Sep 2026 11:06:41 +0200 Subject: [PATCH 5/8] fix(search): spend the fan-out budget only on what fans out, and on what survives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three ways the entry budget took data it was never meant to touch, each verified against the projection before and after. - cap only a reference that CAN fan out. The budget bounds the multiplication this decision introduces, but it was applied to every inline reference: a display-only nesting of 300 entries silently became 100, shortening a list that has always been stored whole - charge it against the entries that SURVIVE, not the framed values considered. A run of values the reference type reads nothing from – dirty source data, which the emptiness filter exists for – spent the budget and left the real referents behind them with nowhere to go, dropping the reference entirely - narrow a nested companion only where fan-out underwrites it. An identity is earned by filterable OR facetable, and only a weldable leaf is ever split, so a facetable-only companion kept its first id and dropped the rest – silently, because the collection declared it a single value too The declared type follows the same rule, so the two agree again. Also drop an unreachable guard: tuplesOf is handed what remains of the budget and never returns more, so the inner loop cannot overshoot it. The ADR claimed entries past the cap are “dropped and reported”. There is no reporting channel and inventing one for data that should never arise is not worth the seam, so it now says the drop is silent, that truncation follows declaration order rather than being representative, and why a local lookup is uncapped. --- ...qualified-edge-into-one-entry-per-tuple.md | 21 ++- .../src/collection-definition.ts | 8 +- .../test/qualified-relation.test.ts | 61 +++++++ packages/search/src/project.ts | 66 +++++--- packages/search/src/schema.ts | 15 +- .../search/test/qualified-relation.test.ts | 154 ++++++++++++++++++ 6 files changed, 297 insertions(+), 28 deletions(-) diff --git a/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md b/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md index 663078e5..6ace74be 100644 --- a/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md +++ b/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md @@ -138,8 +138,25 @@ entries per document (3.74 against 7.49) and gives 15 facet buckets rather than A cartesian product over an edge’s own values is a bound stated in the data’s own units, which [ADR 12](./0012-bound-memory-by-the-unit-of-work-not-the-input.md) says is not a bound. An inline reference therefore declares `maxEntries` -(default 100): entries past it are dropped and reported, rather than a -pathological edge multiplying a document until the run dies. +(default 100), so a pathological edge cannot multiply a document until the run +dies. + +**It bounds the multiplication and nothing else.** A reference whose type +declares no weldable leaf can never fan out, and is not capped: its entry list +is as long as the graph makes it, exactly as it was before this decision. +Capping it would silently shorten data no one asked us to shorten. By the same +reading a `local` lookup is uncapped – it nests the endpoint’s own document +rather than a product, so nothing here multiplies. Whether _that_ list wants a +bound of its own is an ADR 12 question this decision does not answer. + +**The drop is silent, and truncation is not representative.** Entries past the +cap are discarded with no diagnostic: the projection has no reporting channel, +and inventing one for a case that should never arise in real data is not worth +the seam. Truncation also fills the product in declaration order, so a bound +that binds part-way through keeps every combination of the earlier leaves and +few of the later ones. Both are acceptable because the cap is a guard against +data that is already wrong, not a sampling policy – but a deployment that sees +it bite is looking at a modelling error, not at a tuning knob. ## Consequences diff --git a/packages/search-typesense/src/collection-definition.ts b/packages/search-typesense/src/collection-definition.ts index 41bf5140..07002914 100644 --- a/packages/search-typesense/src/collection-definition.ts +++ b/packages/search-typesense/src/collection-definition.ts @@ -548,7 +548,13 @@ function nestedIdentityFields( // locally-nested Root Type is not weldable and still harvests a list. // Getting either route wrong rejects the document at import: Typesense // enforces the declared arity wherever nothing widens the path. - type: flattensToArray || field.array === true ? 'string[]' : 'string', + // `filterable` is what makes an entry hold exactly one id: fan-out splits + // weldable leaves only. An identity earned by `facetable` alone is never + // split, so its entries still harvest a list. + type: + flattensToArray || field.array === true || field.filterable !== true + ? 'string[]' + : 'string', index: true, optional: true, }, diff --git a/packages/search-typesense/test/qualified-relation.test.ts b/packages/search-typesense/test/qualified-relation.test.ts index 5ad0682d..5cec28ee 100644 --- a/packages/search-typesense/test/qualified-relation.test.ts +++ b/packages/search-typesense/test/qualified-relation.test.ts @@ -613,6 +613,67 @@ describe('nested fields of other kinds', () => { ).toMatchObject({ type: 'string[]', index: true }); }); + it('declares a facetable-only companion as a list', () => { + // An identity is earned by `filterable` OR `facetable`, and only a weldable + // leaf fans out – so a facetable-only companion still harvests every id its + // entry references, and must be declared as the list the projection writes. + const org = defineSearchType({ + name: 'Membership', + fields: [ + { + name: 'org', + kind: 'reference', + path: `${SCHEMA_ORG}memberOf`, + output: true, + ref: { strategy: 'lookup', target: 'Person' }, + }, + ], + }); + const facetedRoot = defineSearchType({ + name: 'Person', + class: `${SCHEMA_ORG}Person`, + fields: [ + { + name: 'label', + kind: 'text', + path: `${SCHEMA_ORG}name`, + locales: ['und'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'affiliation', + kind: 'reference', + path: `${SCHEMA_ORG}affiliation`, + output: true, + facetable: true, + ref: { strategy: 'inline', typeName: 'Membership', identity: 'org' }, + }, + ], + }); + const work = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + output: true, + ref: { strategy: 'lookup', target: 'Person', local: true }, + }, + ], + }); + const fields = + buildCollectionDefinition(work, { + schema: searchSchema(work, facetedRoot, org), + }).fields ?? []; + + expect( + fields.find((field) => field.name === 'creator.affiliation_id'), + ).toMatchObject({ type: 'string[]' }); + }); + it('widens an indexed nested leaf of a local lookup’s own root type', () => { // `searchSchema` constrains Reference Types, so a weldable leaf there is // single-valued – but this same path also declares the fields of the Root diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index 9d974111..5fea084e 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -776,7 +776,11 @@ function setIdentity( field: ReferenceField, nested: boolean, ): void { - if (!nested || field.array === true) { + // `filterable` is the whole condition: fan-out splits weldable leaves only + // (`tuplesOf`), so that is the only case where an entry is guaranteed to hold + // one id. An identity may also be earned by `facetable` alone, which nothing + // splits – narrowing there would silently keep the first id and drop the rest. + if (!nested || field.array === true || field.filterable !== true) { setArray(document, name, ids); return; } @@ -848,30 +852,50 @@ function applyNestedReferents( ): readonly ProjectedNode[] { // One node may stand for several entries: a weldable leaf is single-valued, // so an edge the graph gave several roles or several endpoints fans out into - // one entry per combination BEFORE it is projected (ADR 26). The budget spans - // every edge THIS node states, rather than resetting per edge – a node holds - // as many edges as the graph gives it, so a per-edge cap would still let the - // entries grow with the input. - const limit = isInlineReference(field) - ? (field.ref.maxEntries ?? DEFAULT_MAX_ENTRIES) - : Number.POSITIVE_INFINITY; - const nodes: FramedNode[] = []; + // one entry per combination BEFORE it is projected (ADR 26). + // + // The budget bounds THAT MULTIPLICATION, and nothing else. A reference whose + // type declares no weldable leaf can never fan out, so capping it would + // silently truncate a plain list of entries that has always been stored whole + // – a data change this decision never asked for. Its length is the graph's + // business, as it was before fan-out existed. + // + // Where it does apply, it spans every edge THIS node states rather than + // resetting per edge: a node holds as many edges as the graph gives it, so a + // per-edge cap would still let the entries grow with the input (ADR 12). + const limit = + isInlineReference(field) && + nestedType.fields.some((nested) => nested.filterable === true) + ? (field.ref.maxEntries ?? DEFAULT_MAX_ENTRIES) + : Number.POSITIVE_INFINITY; + // Charged against the entries that SURVIVE, not the framed values considered. + // Fields, not identity, are what makes something a referent: a literal value + // object under the alias (dirty source data), or a node this reference type + // reads nothing from, projects nothing and is no referent. Nesting it would + // hand the writer a content-free document – and, for a single-valued + // reference, let it win the slot over a real referent. Charging the budget + // before that filter lets a run of dirty values spend it all and leave the + // real referents behind them with nowhere to go. + const referents: ProjectedNode[] = []; for (const value of values.filter(isObject)) { - if (nodes.length >= limit) { + if (referents.length >= limit) { break; } - nodes.push(...tuplesOf(value, nestedType, field, limit - nodes.length)); + // `tuplesOf` is handed what remains of the budget and never returns more, + // so this inner loop cannot overshoot it – the guard belongs on the values, + // above, where a fresh edge would otherwise start spending afresh. + for (const tuple of tuplesOf( + value, + nestedType, + field, + limit - referents.length, + )) { + const referent = projectFields(tuple, nestedType, schema, context, true); + if (Object.keys(referent).length > 0) { + referents.push(referent); + } + } } - const referents = nodes - .map((referent) => - projectFields(referent, nestedType, schema, context, true), - ) - // Fields, not identity, are what makes something a referent: a literal - // value object under the alias (dirty source data), or a node this - // reference type reads nothing from, projects nothing and is no referent. - // Nesting it would hand the writer a content-free document – and, for a - // single-valued reference, let it win the slot over a real referent. - .filter((referent) => Object.keys(referent).length > 0); if (referents.length === 0) { return referents; } diff --git a/packages/search/src/schema.ts b/packages/search/src/schema.ts index 781f4601..0bea7a10 100644 --- a/packages/search/src/schema.ts +++ b/packages/search/src/schema.ts @@ -298,10 +298,17 @@ export type ReferenceStrategy = /** * Cap on the entries this reference stores **per node carrying it** – * across every edge that node states, and so per document for a - * reference declared on a Root Type. Entries past it are dropped. - * Defaults to 100 ({@link DEFAULT_MAX_ENTRIES}). Must be a positive - * integer: the budget is counted off one entry at a time, so a cap that - * is not one can never be reached. + * reference declared on a Root Type. Defaults to 100 + * ({@link DEFAULT_MAX_ENTRIES}). Must be a positive integer: the budget + * is counted off one entry at a time, so a cap that is not one can never + * be reached. + * + * It bounds the **fan-out** ([ADR 26](../../docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md)), + * so it applies only where the reference type declares a weldable leaf to + * fan out on; a display-only nesting stores every entry the graph states, + * as it always has. Entries past the cap are dropped silently, and in + * declaration order rather than representatively – it guards against data + * that is already wrong, and is not a sampling policy. * * A reference nested inside another reference type is budgeted per * *parent entry* rather than per document, so a schema nesting one edge diff --git a/packages/search/test/qualified-relation.test.ts b/packages/search/test/qualified-relation.test.ts index bac4463e..eb26c83f 100644 --- a/packages/search/test/qualified-relation.test.ts +++ b/packages/search/test/qualified-relation.test.ts @@ -371,6 +371,97 @@ describe('fanning an edge out into one entry per tuple', () => { ]); }); + it('does not cap a reference that cannot fan out', () => { + // The budget bounds the multiplication, and a type with no weldable leaf + // never multiplies. Capping it would silently shorten a plain list of + // entries that has always been stored whole – a data change this decision + // never asked for. + const displayOnly = defineSearchType({ + name: 'CreatorEdge', + fields: [ + { + name: 'role', + kind: 'keyword', + path: `${SCHEMA_ORG}name`, + output: true, + }, + ], + }); + const displayWork = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + array: true, + output: true, + ref: { strategy: 'inline', typeName: 'CreatorEdge' }, + }, + ], + }); + const wide = { + '@id': 'https://ex/work/11', + [workKey('creator')]: Array.from({ length: 300 }, (_, index) => ({ + [edgeKey('role')]: [{ '@value': `role-${index}` }], + })), + }; + const entries = entriesOf( + projectDocument( + wide, + displayWork, + searchSchema(displayWork, displayOnly), + ), + ); + + expect(entries).toHaveLength(300); + }); + + it('spends the budget on entries that survive, not values considered', () => { + // A run of values the reference type reads nothing from – dirty source + // data – must not consume the budget and leave the real referents behind + // them with nowhere to go. + const cappedWork = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + array: true, + output: true, + filterable: true, + ref: { + strategy: 'inline', + typeName: 'CreatorEdge', + identity: 'creator', + maxEntries: 3, + }, + }, + ], + }); + const dirtyThenReal = { + '@id': 'https://ex/work/12', + [workKey('creator')]: [ + { [`${SCHEMA_ORG}unrelated`]: [{ '@value': 'x' }] }, + { [`${SCHEMA_ORG}unrelated`]: [{ '@value': 'y' }] }, + { [`${SCHEMA_ORG}unrelated`]: [{ '@value': 'z' }] }, + { [edgeKey('role')]: [{ '@value': 'etser' }] }, + ], + }; + const entries = entriesOf( + projectDocument( + dirtyThenReal, + cappedWork, + searchSchema(cappedWork, person, weldableEdge), + ), + ); + + expect(entries.map((entry) => entry.role)).toEqual(['etser']); + }); + it('does not fan a local lookup out over the endpoint’s own fields', () => { // A `local` lookup nests the endpoint's own Root Type, whose fields are // multi-valued for reasons of their own – `sameAs` here. Splitting on those @@ -740,6 +831,69 @@ describe('nesting is where a node is projected, not what type it is', () => { ], }); + it('keeps every id where only facetable earned the companion', () => { + // An identity is earned by `filterable` OR `facetable`, and fan-out splits + // weldable leaves only. A facetable-only companion is therefore never + // split, so narrowing it to one id would drop the rest – silently, since + // the collection declares it a list. + const facetedRoot = defineSearchType({ + name: 'Person', + class: `${SCHEMA_ORG}Person`, + fields: [ + { + name: 'label', + kind: 'text', + path: `${SCHEMA_ORG}name`, + locales: ['und'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'affiliation', + kind: 'reference', + path: `${SCHEMA_ORG}affiliation`, + array: true, + output: true, + facetable: true, + ref: { strategy: 'inline', typeName: 'Membership', identity: 'org' }, + }, + ], + }); + const work = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + output: true, + ref: { strategy: 'lookup', target: 'Person', local: true }, + }, + ], + }); + const node = { + '@id': 'https://ex/work/13', + [workKey('creator')]: [ + { + '@id': 'https://p/1', + [alias('Person', 'affiliation')]: [ + { [alias('Membership', 'org')]: [{ '@id': 'https://o/1' }] }, + { [alias('Membership', 'org')]: [{ '@id': 'https://o/2' }] }, + ], + }, + ], + }; + const document = projectDocument( + node, + work, + searchSchema(work, facetedRoot, inner), + ); + const endpoint = document.creator as SearchDocument; + + expect(endpoint.affiliation_id).toEqual(['https://o/1', 'https://o/2']); + }); + it('writes a single-valued companion inside a locally-nested root type', () => { const node = { '@id': 'https://ex/work/10', From 2edf409e2971ede7687801b86e54712518168fad Mon Sep 17 00:00:00 2001 From: David de Boer Date: Fri, 4 Sep 2026 14:26:17 +0200 Subject: [PATCH 6/8] refactor(search): drop the entry cap, which guarded the wrong end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fan-out needs a bound – a cartesian product over an edge's own values is no bound at all – but capping the entries a document stores is not it. By the time the projection runs every cost has been paid: the CONSTRUCT matched those values and the endpoint paid for it, they crossed the wire, the subject index holds them, and framing has materialised them into one node. Capping the product declines the last and cheapest step while keeping all the expensive ones, and bounds nothing framing did not already hold. It also missed the case that predates fan-out entirely: a wide edge, or a display-only nesting of ten thousand entries, is linear rather than multiplicative and was never bounded either. - remove maxEntries, its default, its validation and its export - remove the budget from the projection, and with it the survivor-charging and part-way expansion the cap alone made necessary - record in ADR 26 why the bound belongs at the framing seam instead, and that the fan-out is unbounded until it lands The bound is tracked separately, where it can be decided on its own evidence: capping values per leaf where they enter memory makes the product k^(weldable leaves), and leaf count is a schema constant – a bound in the schema's own units rather than a number written against the data's. Lower the branch threshold by 0.01 to match: the tests that raised it existed only to exercise the cap, and the code they covered has gone with it. --- ...qualified-edge-into-one-entry-per-tuple.md | 48 ++-- docs/reference/search.md | 10 +- packages/search-typesense/vite.config.ts | 2 +- packages/search/src/adapter.ts | 1 - packages/search/src/project.ts | 87 ++----- packages/search/src/schema.ts | 57 +---- .../search/test/qualified-relation.test.ts | 233 ------------------ packages/search/test/schema.test.ts | 34 --- packages/search/vite.config.ts | 2 +- 9 files changed, 56 insertions(+), 418 deletions(-) diff --git a/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md b/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md index 6ace74be..2475a5ea 100644 --- a/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md +++ b/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md @@ -133,30 +133,36 @@ like every other reference. Measured on 1 000 000 documents this halves the entries per document (3.74 against 7.49) and gives 15 facet buckets rather than 30 that split one role across languages. -### What bounds it +### What bounds it – nothing yet, and deliberately so A cartesian product over an edge’s own values is a bound stated in the data’s own units, which [ADR 12](./0012-bound-memory-by-the-unit-of-work-not-the-input.md) -says is not a bound. An inline reference therefore declares `maxEntries` -(default 100), so a pathological edge cannot multiply a document until the run -dies. - -**It bounds the multiplication and nothing else.** A reference whose type -declares no weldable leaf can never fan out, and is not capped: its entry list -is as long as the graph makes it, exactly as it was before this decision. -Capping it would silently shorten data no one asked us to shorten. By the same -reading a `local` lookup is uncapped – it nests the endpoint’s own document -rather than a product, so nothing here multiplies. Whether _that_ list wants a -bound of its own is an ADR 12 question this decision does not answer. - -**The drop is silent, and truncation is not representative.** Entries past the -cap are discarded with no diagnostic: the projection has no reporting channel, -and inventing one for a case that should never arise in real data is not worth -the seam. Truncation also fills the product in declaration order, so a bound -that binds part-way through keeps every combination of the earlier leaves and -few of the later ones. Both are acceptable because the cap is a guard against -data that is already wrong, not a sampling policy – but a deployment that sees -it bite is looking at a modelling error, not at a tuning knob. +says is not a bound. Fan-out therefore needs one, and **this decision does not +supply it**. + +An earlier draft capped the entries a document may store (`maxEntries`, default +100). That was the wrong place, for a reason worth recording: by the time the +projection runs, every cost has already been paid. The CONSTRUCT matched those +values and the endpoint paid for it, they crossed the wire, the subject index +holds them, and framing has materialised them into one node. Capping the +_product_ declines the last and cheapest step while keeping all the expensive +ones – and it bounds nothing that framing did not already hold. + +The bound belongs where the data enters memory. Capping values **per leaf** at +the framing seam bounds the framed node itself, and makes the product +`k^(weldable leaves)` – leaf count is a schema constant, so that is a bound in +the schema’s own units rather than a number written against the data’s. It also +bounds the linear case an entry cap never addressed: a wide edge, or a +display-only nesting of ten thousand entries, both of which predate fan-out and +are equally unbounded today. + +That belongs to the framing seam, which serves every consumer rather than this +one, and wants its own evidence and its own record. Tracked in +[#826](https://github.com/ldelements/lde/issues/826). + +**Until then the fan-out is unbounded**, exactly as the entry list it replaces +always was. What changed is that the growth can now be multiplicative rather +than linear, which is why the bound is worth doing rather than assuming. ## Consequences diff --git a/docs/reference/search.md b/docs/reference/search.md index bbfe5770..ed8d5ac5 100644 --- a/docs/reference/search.md +++ b/docs/reference/search.md @@ -564,10 +564,12 @@ knowing when you declare an edge: IRI single-valued and resolve labels at the surface; declaring the label multi-valued makes fan-out emit an entry per language and splits one role across two facet buckets. -- `maxEntries` on the inline reference caps the entries one document stores - (default 100), so a pathological edge cannot multiply a document without - bound. An `output`-only nested leaf is untouched by all of this: nothing welds - it, so it may carry a list for display. +- An `output`-only nested leaf is untouched by all of this: nothing welds it, so + it may carry a list for display. +- Fan-out is **not yet bounded**. An edge with pathologically many values + multiplies into as many entries; the bound belongs at the framing seam, where + the values enter memory, rather than on the product + ([#826](https://github.com/ldelements/lde/issues/826)). See [ADR 26](../decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple). diff --git a/packages/search-typesense/vite.config.ts b/packages/search-typesense/vite.config.ts index 61247c5e..15ebf0ea 100644 --- a/packages/search-typesense/vite.config.ts +++ b/packages/search-typesense/vite.config.ts @@ -28,7 +28,7 @@ export default mergeConfig( // projection naming what no lookup reaches are unreachable through // the port, since `assertValidQuery` rejects such a query first. // They hold for a direct caller, and are exercised as one. - branches: 95.87, + branches: 95.89, statements: 99.48, }, }, diff --git a/packages/search/src/adapter.ts b/packages/search/src/adapter.ts index 7d66cd92..3437e99f 100644 --- a/packages/search/src/adapter.ts +++ b/packages/search/src/adapter.ts @@ -40,7 +40,6 @@ export { labelTargetNameOf, documentKeyOf, DEFAULT_LABEL_FIELD, - DEFAULT_MAX_ENTRIES, isRangeFacet, isAbsoluteIri, isoToUnixSeconds, diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index 5fea084e..422a568e 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -7,7 +7,6 @@ import { } from './frame-by-type.js'; import { assertTypeInSchema, - DEFAULT_MAX_ENTRIES, displayFieldName, documentKeyOf, fieldNamed, @@ -853,49 +852,16 @@ function applyNestedReferents( // One node may stand for several entries: a weldable leaf is single-valued, // so an edge the graph gave several roles or several endpoints fans out into // one entry per combination BEFORE it is projected (ADR 26). - // - // The budget bounds THAT MULTIPLICATION, and nothing else. A reference whose - // type declares no weldable leaf can never fan out, so capping it would - // silently truncate a plain list of entries that has always been stored whole - // – a data change this decision never asked for. Its length is the graph's - // business, as it was before fan-out existed. - // - // Where it does apply, it spans every edge THIS node states rather than - // resetting per edge: a node holds as many edges as the graph gives it, so a - // per-edge cap would still let the entries grow with the input (ADR 12). - const limit = - isInlineReference(field) && - nestedType.fields.some((nested) => nested.filterable === true) - ? (field.ref.maxEntries ?? DEFAULT_MAX_ENTRIES) - : Number.POSITIVE_INFINITY; - // Charged against the entries that SURVIVE, not the framed values considered. - // Fields, not identity, are what makes something a referent: a literal value - // object under the alias (dirty source data), or a node this reference type - // reads nothing from, projects nothing and is no referent. Nesting it would - // hand the writer a content-free document – and, for a single-valued - // reference, let it win the slot over a real referent. Charging the budget - // before that filter lets a run of dirty values spend it all and leave the - // real referents behind them with nowhere to go. - const referents: ProjectedNode[] = []; - for (const value of values.filter(isObject)) { - if (referents.length >= limit) { - break; - } - // `tuplesOf` is handed what remains of the budget and never returns more, - // so this inner loop cannot overshoot it – the guard belongs on the values, - // above, where a fresh edge would otherwise start spending afresh. - for (const tuple of tuplesOf( - value, - nestedType, - field, - limit - referents.length, - )) { - const referent = projectFields(tuple, nestedType, schema, context, true); - if (Object.keys(referent).length > 0) { - referents.push(referent); - } - } - } + const referents = values + .filter(isObject) + .flatMap((value) => tuplesOf(value, nestedType, field)) + .map((tuple) => projectFields(tuple, nestedType, schema, context, true)) + // Fields, not identity, are what makes something a referent: a literal + // value object under the alias (dirty source data), or a node this + // reference type reads nothing from, projects nothing and is no referent. + // Nesting it would hand the writer a content-free document – and, for a + // single-valued reference, let it win the slot over a real referent. + .filter((referent) => Object.keys(referent).length > 0); if (referents.length === 0) { return referents; } @@ -928,17 +894,17 @@ function applyNestedReferents( * flat `${name}_id` companion, which {@link applyLocalIdentity} already writes * beside the object. * - * `limit` is what remains of the document’s entry budget - * ({@link ReferenceStrategy.maxEntries}, {@link DEFAULT_MAX_ENTRIES}), so the - * product stops growing mid-way rather than being built and then trimmed – a - * bound in the data’s own units is not a bound (ADR 12), and one pathological - * edge would otherwise multiply a document until the run dies. + * **Unbounded, deliberately and only for now.** The product grows with the + * edge's own values, which ADR 12 says is no bound at all. The bound belongs + * where the data enters memory – the CONSTRUCT already paid for those values, + * the subject index already holds them, and the framed node already + * materialised them – not here, at the last and cheapest step to skip. See + * [#826](https://github.com/ldelements/lde/issues/826). */ function tuplesOf( node: FramedNode, nestedType: SearchType, field: ReferenceField, - limit: number, ): readonly FramedNode[] { if (!isInlineReference(field)) { return [node]; @@ -955,24 +921,9 @@ function tuplesOf( let tuples: FramedNode[] = [node]; for (const alias of weldable) { const values = valuesOf(node, alias); - const grown: FramedNode[] = []; - for (const tuple of tuples) { - if (grown.length >= limit) { - break; - } - for (const value of values) { - if (grown.length >= limit) { - break; - } - grown.push({ ...tuple, [alias]: value }); - } - } - // Capped per alias rather than by returning from inside this loop, so every - // tuple that survives is split across EVERY weldable alias. Returning early - // would hand back tuples whose remaining aliases still held their lists, - // and a single-valued leaf then keeps the first value and drops the rest – - // silent data loss in place of the fan-out this exists to perform. - tuples = grown; + tuples = tuples.flatMap((tuple) => + values.map((value) => ({ ...tuple, [alias]: value })), + ); } return tuples; } diff --git a/packages/search/src/schema.ts b/packages/search/src/schema.ts index 0bea7a10..bf184da9 100644 --- a/packages/search/src/schema.ts +++ b/packages/search/src/schema.ts @@ -295,37 +295,6 @@ export type ReferenceStrategy = * keyed on a label, so two endpoints that share a label are never merged. */ readonly identity?: string; - /** - * Cap on the entries this reference stores **per node carrying it** – - * across every edge that node states, and so per document for a - * reference declared on a Root Type. Defaults to 100 - * ({@link DEFAULT_MAX_ENTRIES}). Must be a positive integer: the budget - * is counted off one entry at a time, so a cap that is not one can never - * be reached. - * - * It bounds the **fan-out** ([ADR 26](../../docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md)), - * so it applies only where the reference type declares a weldable leaf to - * fan out on; a display-only nesting stores every entry the graph states, - * as it always has. Entries past the cap are dropped silently, and in - * declaration order rather than representatively – it guards against data - * that is already wrong, and is not a sampling policy. - * - * A reference nested inside another reference type is budgeted per - * *parent entry* rather than per document, so a schema nesting one edge - * inside another admits up to the product of their caps. Bounded, but - * multiplicatively: set the inner cap with the outer one in mind. - * - * A weldable leaf is single-valued, so an edge whose graph values are - * multi-valued fans out into one entry per combination - * ([ADR 26](../../docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md)). - * That product is bounded by the edge’s own data, and a bound in the - * data’s own units is not a bound - * ([ADR 12](../../docs/decisions/0012-bound-memory-by-the-unit-of-work-not-the-input.md)): - * one pathological edge would otherwise multiply a document until the run - * dies. Raise it for a corpus whose edges are legitimately wide; the - * default is far above what a qualified relation produces in practice. - */ - readonly maxEntries?: number; }; /** An IRI-valued reference to another entity, resolved at the surface. */ @@ -1244,8 +1213,7 @@ function assertNoInlineCycle( * condition, so a leaf a weld can name states one value per entry. A leaf * holding a list stands for every combination at once and answers the weld with * none of them. Multiplicity belongs to the entry list instead – the projection - * emits one entry per combination - * ({@link ReferenceStrategy.maxEntries maxEntries}). An `output`-only nested + * emits one entry per combination. An `output`-only nested * leaf is untouched: nothing welds it, so it may carry a list for display. * * Checked schema-wide, like the label sources and for the same reason: a single @@ -1278,7 +1246,7 @@ function assertServiceableNestedFields( } if (field.filterable === true && field.array === true) { throw new Error( - `Nested field “${referenceType.name}.${field.name}” declares both “filterable” and “array”: a weld asks whether ONE entry satisfies every condition, and an entry holding a list has no single value to test – it stands for each combination at once, so the weld degenerates into the cross-product it exists to exclude. Declare the field single-valued; the projection emits one entry per combination (see “maxEntries”).`, + `Nested field “${referenceType.name}.${field.name}” declares both “filterable” and “array”: a weld asks whether ONE entry satisfies every condition, and an entry holding a list has no single value to test – it stands for each combination at once, so the weld degenerates into the cross-product it exists to exclude. Declare the field single-valued; the projection emits one entry per combination.`, ); } } @@ -1336,14 +1304,6 @@ const UNSERVICEABLE_INLINE_ROLES = ['searchable', 'sortable'] as const; * {@link SearchTypeBase.labelField}. */ export const DEFAULT_LABEL_FIELD = 'label'; -/** - * Entries one document stores for an inline reference that declares no - * {@link ReferenceStrategy.maxEntries maxEntries} of its own. Well above what a - * qualified relation produces – a measured corpus averages under four – so the - * default bounds the pathological case without truncating a real one. - */ -export const DEFAULT_MAX_ENTRIES = 100; - /** The `name` the type serves its label under: its declared * {@link SearchTypeBase.labelField}, else `label`. */ export function labelFieldNameOf(searchType: SearchType): string { @@ -1463,7 +1423,6 @@ export interface SearchTypeIssue { | 'invalid-locale' | 'missing-ref' | 'missing-ref-type-name' - | 'invalid-max-entries' | 'ref-not-allowed' | 'text-requires-locales' | 'locales-not-allowed' @@ -1667,18 +1626,6 @@ export function validateSearchType( ) { issue('missing-ref-type-name'); } - // The fan-out budget is counted off one entry at a time, so a fractional - // or non-positive cap is never reached and the cartesian product in - // `tuplesOf` grows with the data – exactly the unbounded case the cap - // exists to prevent (ADR 12). A cap that cannot bind is worse than none, - // because the declaration says otherwise. - if ( - field.ref?.strategy === 'inline' && - field.ref.maxEntries !== undefined && - !(Number.isInteger(field.ref.maxEntries) && field.ref.maxEntries > 0) - ) { - issue('invalid-max-entries'); - } // A join addresses the referent's collection – the one a lookup's // `target` or an idOnly's `labelSource` names. With neither, the flag // states an edge to nowhere. diff --git a/packages/search/test/qualified-relation.test.ts b/packages/search/test/qualified-relation.test.ts index eb26c83f..98f6b383 100644 --- a/packages/search/test/qualified-relation.test.ts +++ b/packages/search/test/qualified-relation.test.ts @@ -274,194 +274,6 @@ describe('fanning an edge out into one entry per tuple', () => { expect(entries[0].note).toEqual(['gesigneerd', 'ovaal']); }); - it('stops at the cap once earlier edges have spent it', () => { - // The cap bounds the DOCUMENT, not each edge: a document holds as many - // edges as the graph states, so a per-edge cap would still let the entries - // grow with the input (ADR 12). `node` states two edges; a budget of one - // is spent by the first, and the second contributes nothing. - const cappedWork = defineSearchType({ - name: 'Work', - class: `${SCHEMA_ORG}CreativeWork`, - fields: [ - { - name: 'creator', - kind: 'reference', - path: `${SCHEMA_ORG}creator`, - array: true, - output: true, - filterable: true, - ref: { - strategy: 'inline', - typeName: 'CreatorEdge', - identity: 'creator', - maxEntries: 1, - }, - }, - ], - }); - const entries = entriesOf( - projectDocument( - node, - cappedWork, - searchSchema(cappedWork, person, creatorEdge), - ), - ); - - expect(entries).toHaveLength(1); - expect(entries[0].role).toBe('etser'); - }); - - it('fully expands every entry it keeps when the cap bites', () => { - // The cap drops whole tuples, never half-expanded ones. Capping by - // returning mid-expansion would hand back entries whose remaining weldable - // leaves still held their lists – and a single-valued leaf then keeps the - // first value and drops the rest, losing data silently instead of fanning - // it out. - const cappedWork = defineSearchType({ - name: 'Work', - class: `${SCHEMA_ORG}CreativeWork`, - fields: [ - { - name: 'creator', - kind: 'reference', - path: `${SCHEMA_ORG}creator`, - array: true, - output: true, - filterable: true, - ref: { - strategy: 'inline', - typeName: 'CreatorEdge', - identity: 'creator', - maxEntries: 3, - }, - }, - ], - }); - const wide = { - '@id': 'https://ex/work/9', - [workKey('creator')]: [ - { - [edgeKey('role')]: [ - { '@value': 'etser' }, - { '@value': 'drukker' }, - { '@value': 'uitgever' }, - ], - [edgeKey('creator')]: [ - { '@id': 'https://a/1', [personKey('sameAs')]: [{ '@id': RKD }] }, - { '@id': 'https://a/2' }, - ], - }, - ], - }; - const entries = entriesOf( - projectDocument( - wide, - cappedWork, - searchSchema(cappedWork, person, weldableEdge), - ), - ); - - expect(entries).toHaveLength(3); - // Both leaves participate in every kept tuple; none is pinned to its first - // value because expansion stopped early. - expect(entries.map((entry) => [entry.role, entry.creator_id])).toEqual([ - ['etser', RKD], - ['etser', 'https://a/2'], - ['drukker', RKD], - ]); - }); - - it('does not cap a reference that cannot fan out', () => { - // The budget bounds the multiplication, and a type with no weldable leaf - // never multiplies. Capping it would silently shorten a plain list of - // entries that has always been stored whole – a data change this decision - // never asked for. - const displayOnly = defineSearchType({ - name: 'CreatorEdge', - fields: [ - { - name: 'role', - kind: 'keyword', - path: `${SCHEMA_ORG}name`, - output: true, - }, - ], - }); - const displayWork = defineSearchType({ - name: 'Work', - class: `${SCHEMA_ORG}CreativeWork`, - fields: [ - { - name: 'creator', - kind: 'reference', - path: `${SCHEMA_ORG}creator`, - array: true, - output: true, - ref: { strategy: 'inline', typeName: 'CreatorEdge' }, - }, - ], - }); - const wide = { - '@id': 'https://ex/work/11', - [workKey('creator')]: Array.from({ length: 300 }, (_, index) => ({ - [edgeKey('role')]: [{ '@value': `role-${index}` }], - })), - }; - const entries = entriesOf( - projectDocument( - wide, - displayWork, - searchSchema(displayWork, displayOnly), - ), - ); - - expect(entries).toHaveLength(300); - }); - - it('spends the budget on entries that survive, not values considered', () => { - // A run of values the reference type reads nothing from – dirty source - // data – must not consume the budget and leave the real referents behind - // them with nowhere to go. - const cappedWork = defineSearchType({ - name: 'Work', - class: `${SCHEMA_ORG}CreativeWork`, - fields: [ - { - name: 'creator', - kind: 'reference', - path: `${SCHEMA_ORG}creator`, - array: true, - output: true, - filterable: true, - ref: { - strategy: 'inline', - typeName: 'CreatorEdge', - identity: 'creator', - maxEntries: 3, - }, - }, - ], - }); - const dirtyThenReal = { - '@id': 'https://ex/work/12', - [workKey('creator')]: [ - { [`${SCHEMA_ORG}unrelated`]: [{ '@value': 'x' }] }, - { [`${SCHEMA_ORG}unrelated`]: [{ '@value': 'y' }] }, - { [`${SCHEMA_ORG}unrelated`]: [{ '@value': 'z' }] }, - { [edgeKey('role')]: [{ '@value': 'etser' }] }, - ], - }; - const entries = entriesOf( - projectDocument( - dirtyThenReal, - cappedWork, - searchSchema(cappedWork, person, weldableEdge), - ), - ); - - expect(entries.map((entry) => entry.role)).toEqual(['etser']); - }); - it('does not fan a local lookup out over the endpoint’s own fields', () => { // A `local` lookup nests the endpoint's own Root Type, whose fields are // multi-valued for reasons of their own – `sameAs` here. Splitting on those @@ -993,51 +805,6 @@ describe('the identity companion of a local lookup', () => { expect((identified.creator as SearchDocument).id).toBe(RKD); }); - it('holds only the entries the cap admits', () => { - // The one place entries are still dropped: a companion holding an id from - // a dropped entry would match a filter whose hit then shows no such entry. - const cappedWork = defineSearchType({ - name: 'Work', - class: `${SCHEMA_ORG}CreativeWork`, - fields: [ - { - name: 'creator', - kind: 'reference', - path: `${SCHEMA_ORG}creator`, - array: true, - output: true, - filterable: true, - ref: { - strategy: 'inline', - typeName: 'CreatorEdge', - identity: 'creator', - maxEntries: 2, - }, - }, - ], - }); - const threeEndpoints = { - '@id': 'https://ex/work/4', - [workKey('creator')]: [ - { - [edgeKey('creator')]: [ - { '@id': 'https://a/1', [personKey('sameAs')]: [{ '@id': RKD }] }, - { '@id': 'https://a/2' }, - { '@id': 'https://a/3' }, - ], - }, - ], - }; - const document = projectDocument( - threeEndpoints, - cappedWork, - searchSchema(cappedWork, person, filterableEdge), - ); - - expect(document.creator).toHaveLength(2); - expect(document.creator_id).toEqual([RKD, 'https://a/2']); - }); - it('fans a multi-valued endpoint out into one entry per endpoint', () => { // The endpoint is what a weld names, so it is single-valued per entry: an // edge the graph gave two endpoints is two entries, not one entry holding diff --git a/packages/search/test/schema.test.ts b/packages/search/test/schema.test.ts index 4b1a12c3..76d528ea 100644 --- a/packages/search/test/schema.test.ts +++ b/packages/search/test/schema.test.ts @@ -1230,40 +1230,6 @@ describe('searchSchema validation', () => { ).not.toThrow(); }); - it.each([ - ['a fraction', 2.5], - ['zero', 0], - ['a negative', -1], - ])('rejects maxEntries that is %s', (_label, maxEntries) => { - // The budget is counted off one entry at a time, so a cap that is not a - // positive integer is never reached and the fan-out grows with the data – - // the unbounded case the cap exists to prevent. A cap that cannot bind is - // worse than none, because the declaration claims otherwise. - expect(() => - searchSchema( - datasetNesting({ - strategy: 'inline', - typeName: 'MediaObject', - maxEntries, - }), - mediaObjectWith({}), - ), - ).toThrow(/invalid-max-entries/u); - }); - - it('accepts a positive integer maxEntries', () => { - expect(() => - searchSchema( - datasetNesting({ - strategy: 'inline', - typeName: 'MediaObject', - maxEntries: 5, - }), - mediaObjectWith({}), - ), - ).not.toThrow(); - }); - it('rejects a nested field declaring both filterable and array', () => { // A weld asks whether ONE entry satisfies every condition, so a leaf a // weld can name holds one value. A leaf holding a list stands for every diff --git a/packages/search/vite.config.ts b/packages/search/vite.config.ts index 92874588..3066345a 100644 --- a/packages/search/vite.config.ts +++ b/packages/search/vite.config.ts @@ -12,7 +12,7 @@ export default mergeConfig( thresholds: { functions: 100, lines: 100, - branches: 99.72, + branches: 99.71, statements: 100, }, }, From a98a3457c1e0cee1e608ffa55b7350d5b6492c4f Mon Sep 17 00:00:00 2001 From: David de Boer Date: Fri, 4 Sep 2026 14:57:48 +0200 Subject: [PATCH 7/8] fix(search): treat an inline reference's identity field as a tuple position MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing welds the identity field by name, so fan-out passed over it – but the flat companion harvested FROM it is the leaf a weld uses to name the endpoint. An entry whose identity field held three ids therefore stood for three endpoints at once, and the companion kept the first and dropped the rest: a filter on either of the others missed a document that genuinely carried it, with the entry beside it still listing all three. The same tuple problem the fan-out exists to remove, one field further in, so it takes the same answer: the identity field is a tuple position like any weldable leaf, and multiplicity moves to the entry list. Three endpoints now make three entries holding one endpoint each, and every id stays reachable. This is what the narrowing in setIdentity always assumed. It justified itself with "fan-out splits weldable leaves only", which was true of `filterable` fields and not of the one the companion actually reads. --- packages/search/src/project.ts | 9 ++- .../search/test/qualified-relation.test.ts | 73 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index 422a568e..bab5ada8 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -909,8 +909,15 @@ function tuplesOf( if (!isInlineReference(field)) { return [node]; } + // `filterable` names the leaves a weld can condition on directly. The + // reference's `identity` field joins them: nothing welds it by name, but the + // flat companion harvested FROM it is the leaf a weld uses to name the + // endpoint ({@link applyIdentityCompanion}). An entry whose identity field + // holds three ids stands for three endpoints at once – the same tuple + // problem, one field further in – so it is a tuple position like any other. + const identity = field.ref.identity; const weldable = nestedType.fields - .filter((nested) => nested.filterable === true) + .filter((nested) => nested.filterable === true || nested.name === identity) .map((nested) => irAlias(nestedType, nested)) // A leaf the frame carries at most one value for is already a tuple // position; splitting it would copy the node to no purpose. diff --git a/packages/search/test/qualified-relation.test.ts b/packages/search/test/qualified-relation.test.ts index 98f6b383..c416b558 100644 --- a/packages/search/test/qualified-relation.test.ts +++ b/packages/search/test/qualified-relation.test.ts @@ -274,6 +274,79 @@ describe('fanning an edge out into one entry per tuple', () => { expect(entries[0].note).toEqual(['gesigneerd', 'ovaal']); }); + it('treats the identity field as a tuple position', () => { + // Nothing welds the identity field by name, but the flat companion + // harvested from it is the leaf a weld uses to name the endpoint. An entry + // whose identity field holds three ids stands for three endpoints at once – + // the same tuple problem, one field further in – and the companion would + // otherwise keep the first and drop the rest, silently. + const endpointEdge = defineSearchType({ + name: 'CreatorEdge', + fields: [ + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + // Multi-valued, and NOT filterable – so only its role as the + // reference's identity makes it a tuple position. + array: true, + output: true, + ref: { strategy: 'lookup', target: 'Person' }, + }, + ], + }); + const identityWork = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + array: true, + output: true, + filterable: true, + ref: { + strategy: 'inline', + typeName: 'CreatorEdge', + identity: 'creator', + }, + }, + ], + }); + const threeEndpoints = { + '@id': 'https://ex/work/14', + [workKey('creator')]: [ + { + [edgeKey('creator')]: [ + { '@id': 'https://a/1' }, + { '@id': 'https://a/2' }, + { '@id': 'https://a/3' }, + ], + }, + ], + }; + const document = projectDocument( + threeEndpoints, + identityWork, + searchSchema(identityWork, person, endpointEdge), + ); + const entries = entriesOf(document); + + // One endpoint per entry, and every id still reachable. + expect(entries).toHaveLength(3); + expect(entries.map((entry) => entry.creator)).toEqual([ + ['https://a/1'], + ['https://a/2'], + ['https://a/3'], + ]); + expect(document.creator_id).toEqual([ + 'https://a/1', + 'https://a/2', + 'https://a/3', + ]); + }); + it('does not fan a local lookup out over the endpoint’s own fields', () => { // A `local` lookup nests the endpoint's own Root Type, whose fields are // multi-valued for reasons of their own – `sameAs` here. Splitting on those From dd6329769cde8fdbfe61b6c536a21b4bd7871089 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Fri, 4 Sep 2026 15:48:47 +0200 Subject: [PATCH 8/8] fix(search): dedupe fanned entries, and say what a single-valued edge drops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fan-out splits RAW framed values, and two distinct ones can still collapse downstream – a transform mapping two spellings onto one canonical value, or a keyed target re-keying two referent IRIs to the same document key. Before fan-out those values met inside one entry and were deduped there; split across entries they reached the index and the API as byte-identical duplicates. - dedupe the projected entries by content, restoring the guarantee the pre-fan-out shape had - document what a single-valued edge now drops. It stores one entry, so an edge the graph gave two endpoints indexes the first where the old single entry listed both. The old shape was never answerable by a weld, so this is the ordinary single-valued rule meeting data that outgrew the declaration – but it changes what a filter matches, and nothing can refuse it at startup because the declaration is only wrong once the data has more than one value - correct the justification shared by setIdentity and nestedIdentityFields. Both claimed a facetable-only identity is never split, which stopped being true when the identity field became a tuple position: it is always split, and `filterable` decides only whether the companion is worth the narrower declared type. The arity was right; the reason given for it was not Also fixes the reference docs promising that nothing is dropped, which held only for an array edge, and counting two consequences before listing three. --- ...qualified-edge-into-one-entry-per-tuple.md | 13 +++++- docs/reference/search.md | 15 +++++-- .../src/collection-definition.ts | 8 ++-- packages/search/src/project.ts | 38 ++++++++++++---- .../search/test/qualified-relation.test.ts | 44 +++++++++++++++++++ 5 files changed, 102 insertions(+), 16 deletions(-) diff --git a/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md b/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md index 2475a5ea..2bfba4da 100644 --- a/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md +++ b/docs/decisions/0026-fan-out-a-qualified-edge-into-one-entry-per-tuple.md @@ -118,8 +118,17 @@ Fan-out happens on the **framed node**, before the entry is projected, so each leaf passes through `transform`, folding and the facet companion exactly as a single-valued field always has. Nothing downstream learns a new shape. -Nothing is dropped: the four entries carry what the two arrays carried. What -changes is that each one now answers the weld. +Nothing is dropped **where the reference is `array`**: the four entries carry +what the two arrays carried, and each one now answers the weld. + +A single-valued reference is the exception, and worth stating plainly. It stores +one entry, so an edge the graph gave two endpoints indexes the first and drops +the second – where before fan-out the single entry listed both. The old shape +was not answerable by a weld (that is the whole tuple problem), so this is the +ordinary single-valued rule meeting data that outgrew the declaration rather +than a new kind of loss. It cannot be refused at startup, because the +declaration is only wrong once the data has more than one value. A qualified +relation is multi-valued by nature, so declare the edge `array: true`. ### 3. Language variants are labels, not values diff --git a/docs/reference/search.md b/docs/reference/search.md index ed8d5ac5..9efc1eca 100644 --- a/docs/reference/search.md +++ b/docs/reference/search.md @@ -557,9 +557,18 @@ combination**: { "role": "etcher", "creator_id": "p3" } ] ``` -Nothing is dropped, and each entry now answers the weld. Two consequences worth -knowing when you declare an edge: - +Each entry now answers the weld, and on an `array` edge nothing is dropped – the +four entries carry what the two lists carried. Four consequences worth knowing +when you declare an edge: + +- **A single-valued edge keeps one entry, so fan-out narrows it.** Declared + without `array`, the reference stores the first entry and its companion + matches: an edge whose graph gives two endpoints now indexes one, where before + fan-out the single entry listed both. That is the ordinary single-valued rule + meeting a tuple – the two-endpoint entry was never answerable by a weld – but + it is a silent change of what a filter matches. Declare the edge `array: true` + wherever the graph may give it more than one value, which for a qualified + relation is nearly always. - A role stated once per language is **one** role, not two. Index its canonical IRI single-valued and resolve labels at the surface; declaring the label multi-valued makes fan-out emit an entry per language and splits one role diff --git a/packages/search-typesense/src/collection-definition.ts b/packages/search-typesense/src/collection-definition.ts index 07002914..c794d385 100644 --- a/packages/search-typesense/src/collection-definition.ts +++ b/packages/search-typesense/src/collection-definition.ts @@ -548,9 +548,11 @@ function nestedIdentityFields( // locally-nested Root Type is not weldable and still harvests a list. // Getting either route wrong rejects the document at import: Typesense // enforces the declared arity wherever nothing widens the path. - // `filterable` is what makes an entry hold exactly one id: fan-out splits - // weldable leaves only. An identity earned by `facetable` alone is never - // split, so its entries still harvest a list. + // `filterable` is the condition because it is what the projection writes + // a single id under, not because it decides the split: `tuplesOf` splits + // the identity field whatever Role earned it. A `facetable`-only + // companion is declared a list, which is what a facet reads and what the + // projection writes for it. type: flattensToArray || field.array === true || field.filterable !== true ? 'string[]' diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index bab5ada8..3e47efdf 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -775,10 +775,12 @@ function setIdentity( field: ReferenceField, nested: boolean, ): void { - // `filterable` is the whole condition: fan-out splits weldable leaves only - // (`tuplesOf`), so that is the only case where an entry is guaranteed to hold - // one id. An identity may also be earned by `facetable` alone, which nothing - // splits – narrowing there would silently keep the first id and drop the rest. + // `filterable` is the whole condition, and it is about the COMPANION rather + // than about the split. `tuplesOf` splits the identity field whatever Role + // earned it, so an entry holds one id either way – but only a `filterable` + // reference has a weld to serve, and only there is the single value worth the + // narrower declared type the collection then matches. A `facetable`-only + // companion stays a list, which is what a facet reads. if (!nested || field.array === true || field.filterable !== true) { setArray(document, name, ids); return; @@ -862,11 +864,18 @@ function applyNestedReferents( // Nesting it would hand the writer a content-free document – and, for a // single-valued reference, let it win the slot over a real referent. .filter((referent) => Object.keys(referent).length > 0); - if (referents.length === 0) { - return referents; + // Fan-out splits RAW framed values, and two distinct ones can still collapse + // downstream – a `transform` mapping two spellings onto one canonical value, + // or a keyed target re-keying two referent IRIs to the same document key. The + // pre-fan-out shape deduped, because those values met inside one entry and + // `applyFacet` deduped them there; split across entries they would reach the + // index and the API as byte-identical duplicates instead. + const distinct = dedupeBy(referents, (referent) => JSON.stringify(referent)); + if (distinct.length === 0) { + return distinct; } - document[field.name] = field.array === true ? referents : referents[0]; - return referents; + document[field.name] = field.array === true ? distinct : distinct[0]; + return distinct; } /** @@ -1075,6 +1084,19 @@ function dedupe(values: readonly string[]): string[] { return [...new Set(values)]; } +/** Keep the first of each distinct `key`, in order. */ +function dedupeBy(values: readonly T[], key: (value: T) => string): T[] { + const seen = new Set(); + return values.filter((value) => { + const identity = key(value); + if (seen.has(identity)) { + return false; + } + seen.add(identity); + return true; + }); +} + function setString( document: ProjectedNode, field: string, diff --git a/packages/search/test/qualified-relation.test.ts b/packages/search/test/qualified-relation.test.ts index c416b558..e0516b30 100644 --- a/packages/search/test/qualified-relation.test.ts +++ b/packages/search/test/qualified-relation.test.ts @@ -274,6 +274,50 @@ describe('fanning an edge out into one entry per tuple', () => { expect(entries[0].note).toEqual(['gesigneerd', 'ovaal']); }); + it('does not emit two entries for values that collapse to one', () => { + // Fan-out splits RAW framed values, but a `transform` can map two of them + // onto the same stored value. Before fan-out they met inside one entry and + // were deduped there; split across entries they would reach the index and + // the API as byte-identical duplicates. + const canonicalising = defineSearchType({ + name: 'CreatorEdge', + fields: [ + { + name: 'role', + kind: 'keyword', + path: `${SCHEMA_ORG}name`, + output: true, + filterable: true, + transform: (value: string) => value.toLowerCase(), + }, + { + name: 'creator', + kind: 'reference', + path: `${SCHEMA_ORG}creator`, + output: true, + ref: { strategy: 'lookup', target: 'Person', local: true }, + }, + ], + }); + const spelled = { + '@id': 'https://ex/work/15', + [workKey('creator')]: [ + { + [edgeKey('role')]: [{ '@value': 'Etser' }, { '@value': 'etser' }], + }, + ], + }; + const entries = entriesOf( + projectDocument( + spelled, + work, + searchSchema(work, person, canonicalising), + ), + ); + + expect(entries).toEqual([{ role: 'etser' }]); + }); + it('treats the identity field as a tuple position', () => { // Nothing welds the identity field by name, but the flat companion // harvested from it is the leaf a weld uses to name the endpoint. An entry