diff --git a/docs/reference/search-typesense.md b/docs/reference/search-typesense.md index 70a62b98..1dc62891 100644 --- a/docs/reference/search-typesense.md +++ b/docs/reference/search-typesense.md @@ -184,7 +184,8 @@ document on disk), so RAM tracks the _indexed_ surface – roughly 2–3× the s of the fields you search, facet or sort on – not the full document. `buildCollectionDefinition` keeps that surface minimal: the `output` display labels land in one `index: false` regex field (`${name}_`, one value per -present language), stored on disk and fetched only for a hit, so they cost no RAM +present language, or every value of it as a `string[]` for an `array` field), +stored on disk and fetched only for a hit, so they cost no RAM and preserve every language; only the folded `*_search_${locale}`, facet/reference and `*_sort_${locale}` companions are indexed. Keeping retrieval-only fields un-indexed is the lever for holding a large index’s RAM diff --git a/docs/reference/search.md b/docs/reference/search.md index 0e5e22bd..fc14a4e1 100644 --- a/docs/reference/search.md +++ b/docs/reference/search.md @@ -309,8 +309,11 @@ outside it leaves the field absent, as any unparseable value does. `array` field stores a list, and a single-valued one stores the first value – for every kind alike, so the projection, the engine collection definition (`string` vs `string[]`) and the API output type never describe one declaration -differently. Declare `array: true` wherever the source may carry several values -you want to keep, including on an internal field a `derive` counts. +differently. For localized `text` the list is per language: an `array` field +keeps every value of each present language, a single-valued one the first of +each, while search folds every value either way. Declare `array: true` wherever +the source may carry several values you want to keep, including on an internal +field a `derive` counts. A `reference` carries one of three strategies, which decide how much of the referent it carries and therefore what it surfaces as: diff --git a/packages/search-typesense/src/collection-definition.ts b/packages/search-typesense/src/collection-definition.ts index acd99b1e..a014abf5 100644 --- a/packages/search-typesense/src/collection-definition.ts +++ b/packages/search-typesense/src/collection-definition.ts @@ -92,7 +92,8 @@ export interface CollectionDefinitionOptions { * size of the fields you search, facet or sort on – not the whole document. * This builder keeps that surface minimal: the `output` display labels land in * a single `index: false` regex field (`${name}_`, one value per present - * language), kept on disk and read back only for a hit, so they cost no RAM; + * language, or every value of it as a `string[]` for an `array` field), kept + * on disk and read back only for a hit, so they cost no RAM; * only the folded `*_search_${locale}`, facet/reference and `*_sort_${locale}` * companions are indexed. Keeping retrieval-only fields un-indexed is the lever * for holding a large index’s RAM down. @@ -273,12 +274,14 @@ function typesenseFields( // the folded `*_search` companions, so the display copies stay on disk and // off RAM (fetched only for a hit), accents preserved, and a language // outside `locales` still renders. This is the memory lever: RAM tracks - // the search surface, not the display text. Absent for a non-output field. + // the search surface, not the display text. Absent for a non-output + // field. An `array` field stores every value of a language, so its + // pattern is typed a list – `array` decides the shape for every kind. ...(displayPattern !== undefined ? [ { name: displayPattern, - type: 'string', + type: typesenseValueType(field), index: false, optional: true, } satisfies CollectionFieldSchema, @@ -582,10 +585,11 @@ function nestedLeafFields( if (pattern !== undefined) { // Display values are never indexed, whatever Roles the field declares: // search hits the folded companions below, so the display copies stay on - // disk with every language they carry. + // disk with every language they carry – every value of it for an + // `array` field. fields.push({ name: nestedFieldName(prefix, pattern), - type: 'string', + type: typesenseValueType(field), index: false, optional: true, }); diff --git a/packages/search-typesense/src/search.ts b/packages/search-typesense/src/search.ts index bdd551d4..163f1d37 100644 --- a/packages/search-typesense/src/search.ts +++ b/packages/search-typesense/src/search.ts @@ -1024,6 +1024,9 @@ function logicalValue( * recovers languages outside the declared `locales` too – a value tagged in an * undeclared language, or untagged (`und`), still reconstructs rather than being * dropped. + * + * A display field holds one string for a plain field, or a list of them for an + * `array` field. */ function localizedValue( flat: Record, @@ -1031,12 +1034,15 @@ function localizedValue( ): LocalizedValue | undefined { const map: Record = {}; for (const [key, value] of Object.entries(flat)) { - if (typeof value !== 'string') { + const lang = displayLangOf(field, key); + if (lang === undefined) { continue; } - const lang = displayLangOf(field, key); - if (lang !== undefined) { - map[lang] = [value]; + const values = (Array.isArray(value) ? value : [value]).filter( + (entry): entry is string => typeof entry === 'string', + ); + if (values.length > 0) { + map[lang] = values; } } return Object.keys(map).length > 0 ? map : undefined; diff --git a/packages/search-typesense/test/collection-definition.test.ts b/packages/search-typesense/test/collection-definition.test.ts index 67e8d781..72e3209c 100644 --- a/packages/search-typesense/test/collection-definition.test.ts +++ b/packages/search-typesense/test/collection-definition.test.ts @@ -237,6 +237,38 @@ describe('und-locale text', () => { ]); }); + it('types the display field of an array text field as a list', () => { + const schema = buildCollectionDefinition( + { + name: 'Doc', + class: 'urn:example:Doc', + fields: [ + { + name: 'alternateName', + kind: 'text', + locales: ['und'], + array: true, + output: true, + searchable: { weight: 1 }, + }, + ], + }, + { collectionNameFor: () => 'docs' }, + ); + // `array` decides the shape for text as for every other kind: the display + // pattern holds every value of a language. The folded search companion is + // one string either way. + expect(schema.fields).toEqual([ + { + name: 'alternateName_[^_]+', + type: 'string[]', + index: false, + optional: true, + }, + { name: 'alternateName_search_und', type: 'string', optional: true }, + ]); + }); + it('emits no display field for a search-only (non-output) text field', () => { const schema = buildCollectionDefinition( { @@ -373,6 +405,45 @@ describe('surfaced inline references', () => { }); }; + it('types a nested array text field’s display as a list', () => { + const person = defineSearchType({ + name: 'Person', + fields: [ + { + name: 'alternateName', + kind: 'text', + locales: ['und'], + array: true, + output: true, + path: 'https://schema.org/alternateName', + }, + ], + }); + const creativeWork = defineSearchType({ + name: 'CreativeWork', + class: 'https://schema.org/CreativeWork', + fields: [ + { + name: 'creator', + kind: 'reference', + output: true, + path: 'https://schema.org/creator', + ref: { typeName: 'Person', strategy: 'inline' }, + }, + ], + }); + const collection = buildCollectionDefinition(creativeWork, { + collectionNameFor: () => 'works', + schema: searchSchema(creativeWork, person), + }); + expect(collection.fields).toContainEqual({ + name: 'creator.alternateName_[^_]+', + type: 'string[]', + index: false, + optional: true, + }); + }); + it('stores a multi-valued inline reference as one nested object per referent', () => { const collection = definitionFor({ array: true }); // `object[]` – each referent keeps its own values grouped, so a consumer diff --git a/packages/search-typesense/test/parse-response.test.ts b/packages/search-typesense/test/parse-response.test.ts index 4d65a77a..cc815c54 100644 --- a/packages/search-typesense/test/parse-response.test.ts +++ b/packages/search-typesense/test/parse-response.test.ts @@ -184,6 +184,43 @@ describe('parseSearchResponse', () => { expect(result.hits[1].document.title).toEqual({ nl: ['Andere'] }); }); + it('reconstructs every value of an array text field per language', () => { + const person: SearchType = { + name: 'Person', + class: 'https://example.org/Person', + fields: [ + { + name: 'alternateName', + kind: 'text', + locales: ['nl', 'und'], + array: true, + output: true, + }, + ], + }; + const parsed = parseSearchResponse( + { + found: 1, + hits: [ + { + document: { + id: 'https://p/1', + alternateName_und: ['Clermont en Chainaye', 'Lambert et Cie'], + alternateName_nl: ['Keramische Industrie'], + }, + }, + ], + }, + person, + new Map(), + searchSchema(person), + ); + expect(parsed.hits[0].document.alternateName).toEqual({ + und: ['Clermont en Chainaye', 'Lambert et Cie'], + nl: ['Keramische Industrie'], + }); + }); + it('resolves reference IRIs to labelled references, id-only when unlabelled', () => { expect(result.hits[0].document.publisher).toEqual([ { id: 'https://org/1', label: { nl: ['Het Utrechts Archief'] } }, diff --git a/packages/search-typesense/vite.config.ts b/packages/search-typesense/vite.config.ts index 5015fb5f..6ff57161 100644 --- a/packages/search-typesense/vite.config.ts +++ b/packages/search-typesense/vite.config.ts @@ -28,7 +28,7 @@ export default mergeConfig( // projection naming what no lookup reaches are unreachable through // the port, since `assertValidQuery` rejects such a query first. // They hold for a direct caller, and are exercised as one. - branches: 95.69, + branches: 95.7, statements: 99.48, }, }, diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index 1a4e32ee..4ba1dd3a 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -448,9 +448,10 @@ function referenceValues( /** * Project a text field. **Display** (when `output`) preserves *every* language - * present – one label per language (accents preserved, untagged under `und`), - * stored `index: false` so extra languages cost nothing – so a value in an - * undeclared language still renders rather than collapsing to a bare IRI. + * present – one label per language, or every label of it for an `array` field + * (accents preserved, untagged under `und`), stored `index: false` so extra + * languages cost nothing – so a value in an undeclared language still renders + * rather than collapsing to a bare IRI. * **Search** (folded, when `searchable`) and **sort** (folded primary, when * `sortable`) stay on the declared `locales`, which drive the indexed, stemmed, * weighted fanout; a value in an undeclared language is not indexed. Absent @@ -473,13 +474,28 @@ function applyText( field: TextField, ): void { if (field.output) { - // First value of each present language wins; a language absent from - // `locales` still lands as a display field (kept off the search index). - const seenLangs = new Set(); + // Every present language lands as a display field (kept off the search + // index), a language absent from `locales` included. `array` decides the + // shape, as it does for every other kind: a declared list keeps every + // value of a language, deduped; a single-valued field keeps the first. + const valuesPerLang = new Map(); for (const { lang, value } of values) { - if (!seenLangs.has(lang)) { - seenLangs.add(lang); - setString(document, displayFieldName(field, lang), value); + if (value === '') { + continue; + } + const langValues = valuesPerLang.get(lang); + if (langValues === undefined) { + valuesPerLang.set(lang, [value]); + } else { + langValues.push(value); + } + } + for (const [lang, langValues] of valuesPerLang) { + const name = displayFieldName(field, lang); + if (field.array === true) { + setArray(document, name, dedupe(langValues)); + } else { + setString(document, name, langValues[0]); } } } diff --git a/packages/search/src/schema.ts b/packages/search/src/schema.ts index b51a96ed..9e603bab 100644 --- a/packages/search/src/schema.ts +++ b/packages/search/src/schema.ts @@ -99,7 +99,9 @@ export interface SearchFieldBase { /** Multi-valued: the field stores a list. Single-valued (the default) stores * the FIRST value the graph carries, whatever their number – for every kind * alike – so one declaration cannot mean a list to the projection and a - * scalar to the collection definition and the API. */ + * scalar to the collection definition and the API. For localized `text` + * the list is per language: an `array` field keeps every value of each + * present language, a single-valued one the first of each. */ readonly array?: boolean; /** Always present: a non-null scalar in the API output and * a non-optional field in the engine index. Moot for arrays/booleans/`id`, diff --git a/packages/search/test/project.test.ts b/packages/search/test/project.test.ts index 5f4f50de..7dc62486 100644 --- a/packages/search/test/project.test.ts +++ b/packages/search/test/project.test.ts @@ -496,6 +496,47 @@ describe('projectDocument', () => { expect(document.title_search_nl).toBe('titel ondertitel'); }); + it('displays every value of a locale for an array text field, deduped', () => { + const document = projectDocument( + { + '@id': 'https://ex/d/8', + [dsKey('alternateName')]: [ + { '@value': 'Clermont en Chainaye' }, + { '@value': 'Guillaume Lambert et Compagnie' }, + { '@value': 'Clermont en Chainaye' }, + { '@value': '' }, + { '@language': 'nl', '@value': 'Keramische Industrie' }, + ], + }, + { + name: 'Dataset', + class: DATASET, + fields: [ + { + name: 'alternateName', + path: dcterms.alternative.value, + kind: 'text', + locales: ['nl', 'und'], + array: true, + output: true, + searchable: { weight: 1 }, + }, + ], + }, + ); + // `array` decides the shape for text as for every other kind: every value + // of a language lands, not only the first, so what search finds displays. + // An empty literal is dropped, as it is for a single-valued field. + expect(document.alternateName_und).toEqual([ + 'Clermont en Chainaye', + 'Guillaume Lambert et Compagnie', + ]); + expect(document.alternateName_nl).toEqual(['Keramische Industrie']); + expect(document.alternateName_search_und).toBe( + 'clermont en chainaye guillaume lambert et compagnie clermont en chainaye', + ); + }); + it('computes a derived field via derive, which may read earlier fields', () => { const document = projectDocument( {