fix(search): display every value of an array text field per language

docs/reference/search-typesense.md

@@ -184,7 +184,8 @@ document on disk), so RAM tracks the _indexed_ surface – roughly 2–3× the s
 of the fields you search, facet or sort on – not the full document.
 `buildCollectionDefinition` keeps that surface minimal: the `output` display
 labels land in one `index: false` regex field (`${name}_<lang>`, one value per
-present language), stored on disk and fetched only for a hit, so they cost no RAM
+present language, or every value of it as a `string[]` for an `array` field),
+stored on disk and fetched only for a hit, so they cost no RAM
 and preserve every language; only the folded `*_search_${locale}`,
 facet/reference and `*_sort_${locale}` companions are indexed. Keeping
 retrieval-only fields un-indexed is the lever for holding a large index’s RAM

docs/reference/search.md

@@ -309,8 +309,11 @@ outside it leaves the field absent, as any unparseable value does.
 `array` field stores a list, and a single-valued one stores the first value –
 for every kind alike, so the projection, the engine collection definition
 (`string` vs `string[]`) and the API output type never describe one declaration
-differently. Declare `array: true` wherever the source may carry several values
-you want to keep, including on an internal field a `derive` counts.
+differently. For localized `text` the list is per language: an `array` field
+keeps every value of each present language, a single-valued one the first of
+each, while search folds every value either way. Declare `array: true` wherever
+the source may carry several values you want to keep, including on an internal
+field a `derive` counts.
 
 A `reference` carries one of three strategies, which decide how much of the
 referent it carries and therefore what it surfaces as:

packages/search-typesense/src/collection-definition.ts

@@ -92,7 +92,8 @@ export interface CollectionDefinitionOptions {
  * size of the fields you search, facet or sort on – not the whole document.
  * This builder keeps that surface minimal: the `output` display labels land in
  * a single `index: false` regex field (`${name}_<lang>`, one value per present
- * language), kept on disk and read back only for a hit, so they cost no RAM;
+ * language, or every value of it as a `string[]` for an `array` field), kept
+ * on disk and read back only for a hit, so they cost no RAM;
  * only the folded `*_search_${locale}`, facet/reference and `*_sort_${locale}`
  * companions are indexed. Keeping retrieval-only fields un-indexed is the lever
  * for holding a large index’s RAM down.
@@ -273,12 +274,14 @@ function typesenseFields(
       // the folded `*_search` companions, so the display copies stay on disk and
       // off RAM (fetched only for a hit), accents preserved, and a language
       // outside `locales` still renders. This is the memory lever: RAM tracks
-      // the search surface, not the display text. Absent for a non-output field.
+      // the search surface, not the display text. Absent for a non-output
+      // field. An `array` field stores every value of a language, so its
+      // pattern is typed a list – `array` decides the shape for every kind.
       ...(displayPattern !== undefined
         ? [
             {
               name: displayPattern,
-              type: 'string',
+              type: typesenseValueType(field),
               index: false,
               optional: true,
             } satisfies CollectionFieldSchema,
@@ -582,10 +585,11 @@ function nestedLeafFields(
     if (pattern !== undefined) {
       // Display values are never indexed, whatever Roles the field declares:
       // search hits the folded companions below, so the display copies stay on
-      // disk with every language they carry.
+      // disk with every language they carry – every value of it for an
+      // `array` field.
       fields.push({
         name: nestedFieldName(prefix, pattern),
-        type: 'string',
+        type: typesenseValueType(field),
         index: false,
         optional: true,
       });

packages/search-typesense/src/search.ts

@@ -1024,19 +1024,25 @@ function logicalValue(
  * recovers languages outside the declared `locales` too – a value tagged in an
  * undeclared language, or untagged (`und`), still reconstructs rather than being
  * dropped.
+ *
+ * A display field holds one string for a plain field, or a list of them for an
+ * `array` field.
  */
 function localizedValue(
   flat: Record<string, unknown>,
   field: TextField,
 ): LocalizedValue | undefined {
   const map: Record<string, readonly string[]> = {};
   for (const [key, value] of Object.entries(flat)) {
-    if (typeof value !== 'string') {
+    const lang = displayLangOf(field, key);
+    if (lang === undefined) {
       continue;
     }
-    const lang = displayLangOf(field, key);
-    if (lang !== undefined) {
-      map[lang] = [value];
+    const values = (Array.isArray(value) ? value : [value]).filter(
+      (entry): entry is string => typeof entry === 'string',
+    );
+    if (values.length > 0) {
+      map[lang] = values;
     }
   }
   return Object.keys(map).length > 0 ? map : undefined;

packages/search-typesense/test/collection-definition.test.ts

@@ -237,6 +237,38 @@ describe('und-locale text', () => {
     ]);
   });
 
+  it('types the display field of an array text field as a list', () => {
+    const schema = buildCollectionDefinition(
+      {
+        name: 'Doc',
+        class: 'urn:example:Doc',
+        fields: [
+          {
+            name: 'alternateName',
+            kind: 'text',
+            locales: ['und'],
+            array: true,
+            output: true,
+            searchable: { weight: 1 },
+          },
+        ],
+      },
+      { collectionNameFor: () => 'docs' },
+    );
+    // `array` decides the shape for text as for every other kind: the display
+    // pattern holds every value of a language. The folded search companion is
+    // one string either way.
+    expect(schema.fields).toEqual([
+      {
+        name: 'alternateName_[^_]+',
+        type: 'string[]',
+        index: false,
+        optional: true,
+      },
+      { name: 'alternateName_search_und', type: 'string', optional: true },
+    ]);
+  });
+
   it('emits no display field for a search-only (non-output) text field', () => {
     const schema = buildCollectionDefinition(
       {
@@ -373,6 +405,45 @@ describe('surfaced inline references', () => {
     });
   };
 
+  it('types a nested array text field’s display as a list', () => {
+    const person = defineSearchType({
+      name: 'Person',
+      fields: [
+        {
+          name: 'alternateName',
+          kind: 'text',
+          locales: ['und'],
+          array: true,
+          output: true,
+          path: 'https://schema.org/alternateName',
+        },
+      ],
+    });
+    const creativeWork = defineSearchType({
+      name: 'CreativeWork',
+      class: 'https://schema.org/CreativeWork',
+      fields: [
+        {
+          name: 'creator',
+          kind: 'reference',
+          output: true,
+          path: 'https://schema.org/creator',
+          ref: { typeName: 'Person', strategy: 'inline' },
+        },
+      ],
+    });
+    const collection = buildCollectionDefinition(creativeWork, {
+      collectionNameFor: () => 'works',
+      schema: searchSchema(creativeWork, person),
+    });
+    expect(collection.fields).toContainEqual({
+      name: 'creator.alternateName_[^_]+',
+      type: 'string[]',
+      index: false,
+      optional: true,
+    });
+  });
+
   it('stores a multi-valued inline reference as one nested object per referent', () => {
     const collection = definitionFor({ array: true });
     // `object[]` – each referent keeps its own values grouped, so a consumer

packages/search-typesense/test/parse-response.test.ts

@@ -184,6 +184,43 @@ describe('parseSearchResponse', () => {
     expect(result.hits[1].document.title).toEqual({ nl: ['Andere'] });
   });
 
+  it('reconstructs every value of an array text field per language', () => {
+    const person: SearchType = {
+      name: 'Person',
+      class: 'https://example.org/Person',
+      fields: [
+        {
+          name: 'alternateName',
+          kind: 'text',
+          locales: ['nl', 'und'],
+          array: true,
+          output: true,
+        },
+      ],
+    };
+    const parsed = parseSearchResponse(
+      {
+        found: 1,
+        hits: [
+          {
+            document: {
+              id: 'https://p/1',
+              alternateName_und: ['Clermont en Chainaye', 'Lambert et Cie'],
+              alternateName_nl: ['Keramische Industrie'],
+            },
+          },
+        ],
+      },
+      person,
+      new Map(),
+      searchSchema(person),
+    );
+    expect(parsed.hits[0].document.alternateName).toEqual({
+      und: ['Clermont en Chainaye', 'Lambert et Cie'],
+      nl: ['Keramische Industrie'],
+    });
+  });
+
   it('resolves reference IRIs to labelled references, id-only when unlabelled', () => {
     expect(result.hits[0].document.publisher).toEqual([
       { id: 'https://org/1', label: { nl: ['Het Utrechts Archief'] } },

packages/search-typesense/vite.config.ts

@@ -28,7 +28,7 @@ export default mergeConfig(
           // projection naming what no lookup reaches are unreachable through
           // the port, since `assertValidQuery` rejects such a query first.
           // They hold for a direct caller, and are exercised as one.
-          branches: 95.69,
+          branches: 95.7,
           statements: 99.48,
         },
       },

packages/search/src/project.ts

@@ -448,9 +448,10 @@ function referenceValues(
 
 /**
  * Project a text field. **Display** (when `output`) preserves *every* language
- * present – one label per language (accents preserved, untagged under `und`),
- * stored `index: false` so extra languages cost nothing – so a value in an
- * undeclared language still renders rather than collapsing to a bare IRI.
+ * present – one label per language, or every label of it for an `array` field
+ * (accents preserved, untagged under `und`), stored `index: false` so extra
+ * languages cost nothing – so a value in an undeclared language still renders
+ * rather than collapsing to a bare IRI.
  * **Search** (folded, when `searchable`) and **sort** (folded primary, when
  * `sortable`) stay on the declared `locales`, which drive the indexed, stemmed,
  * weighted fanout; a value in an undeclared language is not indexed. Absent
@@ -473,13 +474,28 @@ function applyText(
   field: TextField,
 ): void {
   if (field.output) {
-    // First value of each present language wins; a language absent from
-    // `locales` still lands as a display field (kept off the search index).
-    const seenLangs = new Set<string>();
+    // Every present language lands as a display field (kept off the search
+    // index), a language absent from `locales` included. `array` decides the
+    // shape, as it does for every other kind: a declared list keeps every
+    // value of a language, deduped; a single-valued field keeps the first.
+    const valuesPerLang = new Map<string, string[]>();
     for (const { lang, value } of values) {
-      if (!seenLangs.has(lang)) {
-        seenLangs.add(lang);
-        setString(document, displayFieldName(field, lang), value);
+      if (value === '') {
+        continue;
+      }
+      const langValues = valuesPerLang.get(lang);
+      if (langValues === undefined) {
+        valuesPerLang.set(lang, [value]);
+      } else {
+        langValues.push(value);
+      }
+    }
+    for (const [lang, langValues] of valuesPerLang) {
+      const name = displayFieldName(field, lang);
+      if (field.array === true) {
+        setArray(document, name, dedupe(langValues));
+      } else {
+        setString(document, name, langValues[0]);
       }
     }
   }

packages/search/src/schema.ts

@@ -99,7 +99,9 @@ export interface SearchFieldBase {
   /** Multi-valued: the field stores a list. Single-valued (the default) stores
    *  the FIRST value the graph carries, whatever their number – for every kind
    *  alike – so one declaration cannot mean a list to the projection and a
-   *  scalar to the collection definition and the API. */
+   *  scalar to the collection definition and the API. For localized `text`
+   *  the list is per language: an `array` field keeps every value of each
+   *  present language, a single-valued one the first of each. */
   readonly array?: boolean;
   /** Always present: a non-null scalar in the API output and
    *  a non-optional field in the engine index. Moot for arrays/booleans/`id`,

packages/search/test/project.test.ts

@@ -496,6 +496,47 @@ describe('projectDocument', () => {
     expect(document.title_search_nl).toBe('titel ondertitel');
   });
 
+  it('displays every value of a locale for an array text field, deduped', () => {
+    const document = projectDocument(
+      {
+        '@id': 'https://ex/d/8',
+        [dsKey('alternateName')]: [
+          { '@value': 'Clermont en Chainaye' },
+          { '@value': 'Guillaume Lambert et Compagnie' },
+          { '@value': 'Clermont en Chainaye' },
+          { '@value': '' },
+          { '@language': 'nl', '@value': 'Keramische Industrie' },
+        ],
+      },
+      {
+        name: 'Dataset',
+        class: DATASET,
+        fields: [
+          {
+            name: 'alternateName',
+            path: dcterms.alternative.value,
+            kind: 'text',
+            locales: ['nl', 'und'],
+            array: true,
+            output: true,
+            searchable: { weight: 1 },
+          },
+        ],
+      },
+    );
+    // `array` decides the shape for text as for every other kind: every value
+    // of a language lands, not only the first, so what search finds displays.
+    // An empty literal is dropped, as it is for a single-valued field.
+    expect(document.alternateName_und).toEqual([
+      'Clermont en Chainaye',
+      'Guillaume Lambert et Compagnie',
+    ]);
+    expect(document.alternateName_nl).toEqual(['Keramische Industrie']);
+    expect(document.alternateName_search_und).toBe(
+      'clermont en chainaye guillaume lambert et compagnie clermont en chainaye',
+    );
+  });
+
   it('computes a derived field via derive, which may read earlier fields', () => {
     const document = projectDocument(
       {