refactor: migrate boxel-cli and vscode-boxel-tools search to /_federated-search-v2 (data-only)

packages/boxel-cli/src/commands/search.ts

@@ -5,6 +5,7 @@ import {
   type ProfileManager,
 } from '../lib/profile-manager.ts';
 import { ensureTrailingSlash } from '@cardstack/runtime-common/paths';
+import { resourceIdentity } from '@cardstack/runtime-common/resource-identity';
 import { FG_RED, DIM, RESET } from '../lib/colors.ts';
 import { cliLog } from '../lib/cli-log.ts';
 
@@ -19,12 +20,190 @@ export interface SearchCommandOptions {
   profileManager?: ProfileManager;
 }
 
+// `_federated-search-v2` speaks the search-entry wire grammar: one query
+// rooted on `search-entry`, where entry membership is addressed through
+// `item.` (the card/file serialization). The type anchor is `item.on` and the
+// field paths inside the filter operators carry the `item.` prefix. Callers
+// here author ordinary card-rooted queries, so these helpers rewrite a query
+// into the `item.`-addressed form the endpoint expects.
+//
+// This mirrors runtime-common's `searchEntryWireQueryFromQuery`, kept local
+// because that module pulls the whole runtime-common index — and its
+// `https://cardstack.com/base/*` imports — into boxel-cli's deliberately
+// dependency-light graph.
+const ITEM_PREFIX = 'item.';
+const ITEM_ANCHOR = 'item.on';
+
+// The filter operators whose value is an object keyed by field paths; their
+// keys are the ones that take the `item.` prefix.
+const FIELD_KEYED_OPERATORS = ['eq', 'contains', 'in', 'range'];
+
+function toItemFilter(
+  filter: Record<string, unknown>,
+): Record<string, unknown> {
+  let out: Record<string, unknown> = {};
+  for (let [key, value] of Object.entries(filter)) {
+    if (key === 'type' || key === 'on') {
+      // both legacy spellings of the type anchor map to item.on
+      out[ITEM_ANCHOR] = value;
+    } else if (key === 'any' || key === 'every') {
+      if (!Array.isArray(value)) {
+        throw new Error(`filter.${key} must be an array`);
+      }
+      out[key] = value.map((node) =>
+        toItemFilter(node as Record<string, unknown>),
+      );
+    } else if (key === 'not') {
+      out.not = toItemFilter(value as Record<string, unknown>);
+    } else if (FIELD_KEYED_OPERATORS.includes(key)) {
+      if (typeof value !== 'object' || value == null || Array.isArray(value)) {
+        throw new Error(`filter.${key} must be an object`);
+      }
+      out[key] = Object.fromEntries(
+        Object.entries(value as Record<string, unknown>).map(
+          ([fieldPath, fieldValue]) => [
+            `${ITEM_PREFIX}${fieldPath}`,
+            fieldValue,
+          ],
+        ),
+      );
+    } else if (key === 'matches') {
+      // full-text match over the whole document — no field path to address
+      out.matches = value;
+    } else {
+      throw new Error(
+        `cannot translate filter member "${key}" to a search-entry query — the type anchor is "on"/"type" and field paths live under the ${FIELD_KEYED_OPERATORS.join('/')} operators`,
+      );
+    }
+  }
+  return out;
+}
+
+function toItemSort(entry: Record<string, unknown>): Record<string, unknown> {
+  let out: Record<string, unknown> = {};
+  for (let [key, value] of Object.entries(entry)) {
+    if (key === 'by') {
+      if (typeof value !== 'string') {
+        throw new Error('sort entry "by" must be a string');
+      }
+      out.by = `${ITEM_PREFIX}${value}`;
+    } else if (key === 'on') {
+      out[ITEM_ANCHOR] = value;
+    } else if (key === 'direction') {
+      out.direction = value;
+    } else {
+      throw new Error(`unknown sort member "${key}"`);
+    }
+  }
+  return out;
+}
+
+interface SearchEntryRequestBody {
+  realms: string[];
+  // boxel-cli never renders HTML, so it requests the data-only fieldset: each
+  // entry carries only its full `item` serialization (no prerendered `html`).
+  fields: { 'search-entry': ['item'] };
+  filter?: Record<string, unknown>;
+  sort?: Record<string, unknown>[];
+  page?: unknown;
+  cardUrls?: unknown;
+}
+
+/**
+ * Build the `_federated-search-v2` request body from a card-rooted query: the
+ * `item.`-addressed filter/sort, the realms to federate across, and the
+ * data-only fieldset.
+ */
+export function searchEntryRequestBody(
+  query: Record<string, unknown>,
+  realms: string[],
+): SearchEntryRequestBody {
+  let body: SearchEntryRequestBody = {
+    realms,
+    fields: { 'search-entry': ['item'] },
+  };
+  if (query.filter !== undefined) {
+    if (
+      typeof query.filter !== 'object' ||
+      query.filter == null ||
+      Array.isArray(query.filter)
+    ) {
+      throw new Error('filter must be an object');
+    }
+    body.filter = toItemFilter(query.filter as Record<string, unknown>);
+  }
+  if (query.sort !== undefined) {
+    if (!Array.isArray(query.sort)) {
+      throw new Error('sort must be an array');
+    }
+    body.sort = query.sort.map((entry) =>
+      toItemSort(entry as Record<string, unknown>),
+    );
+  }
+  if (query.page !== undefined) {
+    body.page = query.page;
+  }
+  if (query.cardUrls !== undefined) {
+    body.cardUrls = query.cardUrls;
+  }
+  return body;
+}
+
+// A data-only search-entry document, narrowed to the shape this client reads:
+// each entry links its serialization through `item`, and the `card`/`file-meta`
+// resource itself travels in `included`. A structural local type rather than
+// runtime-common's `SearchEntryCollectionDocument` — that one transitively
+// pulls the index's `https://cardstack.com/base/*` imports, which don't resolve
+// in a plain Node CLI (the same boundary the query helpers above note).
+interface SearchEntryDoc {
+  data?: {
+    relationships?: {
+      item?: { data?: { type: string; id: string } };
+    };
+  }[];
+  included?: { type: string; id: string }[];
+}
+
+/**
+ * Flatten a data-only search-entry document into the `item` serializations, in
+ * result order — the same `card`/`file-meta` resources the legacy endpoint
+ * returned as its top-level `data`. Each entry points at its serialization in
+ * `included`; resolve and collect them.
+ */
+export function itemsFromSearchEntryDoc(
+  doc: SearchEntryDoc,
+): Record<string, unknown>[] {
+  let byIdentity = new Map<string, Record<string, unknown>>();
+  for (let resource of doc.included ?? []) {
+    if (resource.type === 'card' || resource.type === 'file-meta') {
+      byIdentity.set(
+        resourceIdentity(resource.type, resource.id),
+        resource as Record<string, unknown>,
+      );
+    }
+  }
+  let items: Record<string, unknown>[] = [];
+  for (let entry of doc.data ?? []) {
+    let ref = entry.relationships?.item?.data;
+    if (!ref) {
+      continue;
+    }
+    let item = byIdentity.get(resourceIdentity(ref.type, ref.id));
+    if (item) {
+      items.push(item);
+    }
+  }
+  return items;
+}
+
 /**
- * Federated search across one or more realms via the `_federated-search`
+ * Federated search across one or more realms via the `_federated-search-v2`
  * server endpoint.
  *
- * Sends a QUERY request with the provided query object and a `realms` array
- * merged into the request body. Uses the server JWT via
+ * Sends the search-entry-rooted query as a QUERY request requesting the
+ * data-only fieldset (`fields[search-entry]=item`), and returns the `item`
+ * serializations the endpoint links in `included` — the `card`/`file-meta`
+ * resources callers consume. Uses the server JWT via
  * `ProfileManager.authedRealmServerFetch`.
  */
 export async function search(
@@ -42,35 +221,48 @@ export async function search(
   }
 
   let realmServerUrl = active.profile.realmServerUrl.replace(/\/$/, '');
-  let searchUrl = `${realmServerUrl}/_federated-search`;
+  let searchUrl = `${realmServerUrl}/_federated-search-v2`;
 
   let realms = (Array.isArray(realmUrls) ? realmUrls : [realmUrls]).map(
     ensureTrailingSlash,
   );
 
+  let body: SearchEntryRequestBody;
+  try {
+    body = searchEntryRequestBody(query, realms);
+  } catch (err) {
+    return {
+      ok: false,
+      status: 0,
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+
   try {
     let response = await pm.authedRealmServerFetch(searchUrl, {
       method: 'QUERY',
       headers: {
         Accept: 'application/vnd.card+json',
         'Content-Type': 'application/json',
       },
-      body: JSON.stringify({ realms, ...query }),
+      body: JSON.stringify(body),
     });
 
     if (!response.ok) {
-      let body = await response.text();
+      let responseBody = await response.text();
       return {
         ok: false,
         status: response.status,
-        error: `HTTP ${response.status}: ${body.slice(0, 300)}`,
+        error: `HTTP ${response.status}: ${responseBody.slice(0, 300)}`,
       };
     }
 
-    let result = (await response.json()) as {
-      data?: Record<string, unknown>[];
+    let result = (await response.json()) as SearchEntryDoc;
+    return {
+      ok: true,
+      status: response.status,
+      data: itemsFromSearchEntryDoc(result),
     };
-    return { ok: true, status: response.status, data: result.data };
   } catch (err) {
     return {
       ok: false,

packages/boxel-cli/src/lib/boxel-cli-client.ts

@@ -256,8 +256,9 @@ export class BoxelCLIClient {
   }
 
   /**
-   * Federated search across one or more realms via `_federated-search`.
-   * Delegates to the standalone `search()` in `commands/search.ts`.
+   * Federated search across one or more realms via `_federated-search-v2`.
+   * Delegates to the standalone `search()` in `commands/search.ts`, which
+   * returns the `item` serializations (the `card`/`file-meta` resources).
    */
   async search(
     realmUrls: string | string[],

packages/boxel-cli/tests/commands/ingest-card-graph.test.ts

@@ -148,10 +148,23 @@ function makeFakeAuthenticator(fetchedUrls: string[]): RealmAuthenticator {
   };
 }
 
-// Answers the two search shapes the ingester issues: instances of the entry
-// card's exported classes, and all base-realm Spec cards (filtered by
-// specType + ref in the ingester itself).
+// Answers the two search shapes the ingester issues — instances of the entry
+// card's exported classes, and all base-realm Spec cards — as v2 data-only
+// `search-entry` documents: each entry links its `item` serialization, which
+// rides in `included`. The ingester filters Specs (specType + ref) itself. The
+// type anchor arrives `item.`-addressed (`filter['item.on']`), the search-entry
+// query grammar `_federated-search-v2` speaks.
 function makeFakeProfileManager(): ProfileManager {
+  // Wrap card resources as a data-only search-entry document: one entry per
+  // card linking its `item`, with the card resources themselves in `included`.
+  let searchEntryDoc = (cards: { id: string; attributes?: unknown }[]) => ({
+    data: cards.map((card) => ({
+      id: card.id,
+      relationships: { item: { data: { type: 'card', id: card.id } } },
+    })),
+    included: cards.map((card) => ({ type: 'card', ...card })),
+    meta: { page: { total: cards.length } },
+  });
   return {
     getActiveProfile() {
       return {
@@ -160,12 +173,16 @@ function makeFakeProfileManager(): ProfileManager {
     },
     async authedRealmServerFetch(_url: string, init?: RequestInit) {
       let body = JSON.parse(String(init?.body ?? '{}')) as {
-        filter?: { type?: { module?: string; name?: string } };
+        filter?: { 'item.on'?: { module?: string; name?: string } };
+        fields?: unknown;
       };
-      let type = body.filter?.type;
-      let data: unknown[] = [];
-      if (type?.module === 'https://cardstack.com/base/spec') {
-        data = [
+      // Data-only consumers must request the item fieldset — that's what keeps
+      // prerendered html/css off the wire.
+      expect(body.fields).toEqual({ 'search-entry': ['item'] });
+      let on = body.filter?.['item.on'];
+      let cards: { id: string; attributes?: unknown }[] = [];
+      if (on?.module === 'https://cardstack.com/base/spec') {
+        cards = [
           {
             id: `${ROOT}Spec/gadget`,
             attributes: {
@@ -191,13 +208,12 @@ function makeFakeProfileManager(): ProfileManager {
             },
           },
         ];
-      } else if (
-        type?.module === GADGET_MODULE_ABS &&
-        type?.name === 'Gadget'
-      ) {
-        data = [{ id: `${ROOT}Gadget/g1` }];
+      } else if (on?.module === GADGET_MODULE_ABS && on?.name === 'Gadget') {
+        cards = [{ id: `${ROOT}Gadget/g1` }];
       }
-      return new Response(JSON.stringify({ data }), { status: 200 });
+      return new Response(JSON.stringify(searchEntryDoc(cards)), {
+        status: 200,
+      });
     },
   } as unknown as ProfileManager;
 }