From 38d9d3260e542921c610af4cfee10f61cfa5b290 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Thu, 18 Jun 2026 15:08:40 +0200 Subject: [PATCH 1/3] fix: ingest-card copies Spec + instances from shared/published source realms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ingest-card discovered instances and Specs via the profile-scoped `_federated-search`, which only covers realms in the active profile's set. A shared/published source realm (e.g. the catalog) isn't in that set, so every discovery query returned nothing and ingest copied modules only — forcing the software factory to reconstruct Specs/instances by hand on every adjust run. (The module crawl was unaffected because it uses direct file fetches.) Query the source realm's own `_search` endpoint directly via authedRealmFetch instead. A published realm returns matches in `included` with an empty `data`, so `selectSearchResults` prefers `data` and falls back to `included` (never merges, so a normal realm's linked-dep `included` isn't mistaken for matches). Verified end-to-end against the live catalog: ingesting the WineCellar card now copies both Specs + 10 sample instances alongside the 2 modules (was 2 files). CS-11652. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/commands/realm/ingest-card.ts | 49 ++++++++++++++----- .../tests/commands/ingest-card.test.ts | 25 ++++++++++ 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/packages/boxel-cli/src/commands/realm/ingest-card.ts b/packages/boxel-cli/src/commands/realm/ingest-card.ts index 50f75aeb13..117282e74b 100644 --- a/packages/boxel-cli/src/commands/realm/ingest-card.ts +++ b/packages/boxel-cli/src/commands/realm/ingest-card.ts @@ -17,7 +17,6 @@ import { type ProfileManager, } from '../../lib/profile-manager.ts'; import type { RealmAuthenticator } from '../../lib/realm-authenticator.ts'; -import { search } from '../search.ts'; const CARD_JSON = 'application/vnd.card+json'; const MODULE_EXTENSIONS = ['.gts', '.gjs', '.ts', '.js']; @@ -104,17 +103,14 @@ export function resolveSameRealmFile( class RealmCardIngester extends RealmSyncBase { hasError = false; copiedFiles: string[] = []; - private profileManager: ProfileManager; private cardUrl: string; private sourceCache = new Map(); constructor( options: SyncOptions & { cardUrl: string }, authenticator: RealmAuthenticator, - profileManager: ProfileManager, ) { super(options, authenticator); - this.profileManager = profileManager; this.cardUrl = options.cardUrl; } @@ -342,15 +338,30 @@ class RealmCardIngester extends RealmSyncBase { private async searchCards( query: Record, ): Promise { - let result = await search([this.realmRoot], query, { - profileManager: this.profileManager, - }); - if (!result.ok) { + // Query the SOURCE realm's own `_search` directly rather than the + // profile-scoped `_federated-search`. A shared/published source realm + // (e.g. the catalog) isn't in the active profile's federated set, so + // federated search returns nothing for it — which is why instances and + // Specs went uncopied (the module crawl survives because it uses direct + // file fetches). The realm's own `_search` sees its full index. + let res = await this.authenticator.authedRealmFetch( + `${this.realmRoot}_search`, + { + method: 'QUERY', + headers: { + Accept: CARD_JSON, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(query), + }, + ); + if (!res.ok) { this.hasError = true; - console.warn(` search failed: ${result.error ?? 'unknown'}`); + console.warn(` search failed: HTTP ${res.status}`); return []; } - return (result.data ?? []) as CardResource[]; + let json = (await res.json()) as SearchResponse; + return selectSearchResults(json); } private cardIdToInstanceRel(id: string | undefined): string | null { @@ -418,6 +429,23 @@ interface CardResource { attributes?: { specType?: string; ref?: unknown; [k: string]: unknown }; } +interface SearchResponse { + data?: CardResource[]; + included?: CardResource[]; +} + +/** + * Pick the matched cards out of a realm `_search` response. A normal realm + * returns matches in `data`; a published realm (e.g. the catalog) returns them + * in `included` with an empty `data`. Prefer `data`, fall back to `included` — + * never merge, so a normal realm's `included` (linked deps of the matches) is + * not mistaken for matches. Exported for unit testing. + */ +export function selectSearchResults(json: SearchResponse): CardResource[] { + let data = json.data ?? []; + return data.length > 0 ? data : (json.included ?? []); +} + function tryParseCardDoc( source: string, ): { data?: { meta?: { adoptsFrom?: { module?: string } } } } | null { @@ -490,7 +518,6 @@ export async function ingestCard( cardUrl, }, authenticator, - pm, ); console.log( `Ingesting ${cardUrl}\n from realm ${realmRoot}\n into ${localDir}`, diff --git a/packages/boxel-cli/tests/commands/ingest-card.test.ts b/packages/boxel-cli/tests/commands/ingest-card.test.ts index 0b7932462a..16a22b5834 100644 --- a/packages/boxel-cli/tests/commands/ingest-card.test.ts +++ b/packages/boxel-cli/tests/commands/ingest-card.test.ts @@ -3,9 +3,34 @@ import { extractImportSpecifiers, extractExportedClassNames, resolveSameRealmFile, + selectSearchResults, } from '../../src/commands/realm/ingest-card.js'; describe('ingest-card helpers', () => { + describe('selectSearchResults', () => { + it('uses `data` when present (normal realm)', () => { + let json = { + data: [{ id: 'a' }, { id: 'b' }], + included: [{ id: 'dep' }], + }; + expect(selectSearchResults(json)).toEqual([{ id: 'a' }, { id: 'b' }]); + }); + + it('falls back to `included` when `data` is empty (published realm)', () => { + // The catalog returns matches in `included` with `data: []`. + let json = { data: [], included: [{ id: 'spec-1' }, { id: 'spec-2' }] }; + expect(selectSearchResults(json)).toEqual([ + { id: 'spec-1' }, + { id: 'spec-2' }, + ]); + }); + + it('returns [] when both are empty/absent', () => { + expect(selectSearchResults({})).toEqual([]); + expect(selectSearchResults({ data: [], included: [] })).toEqual([]); + }); + }); + describe('extractImportSpecifiers', () => { it('captures value, type, namespace, re-export, and side-effect imports', () => { let src = ` From 7190f753a9ef26d54632a0775b7aa2ad2e16f9b8 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 19 Jun 2026 11:19:37 +0200 Subject: [PATCH 2/3] test: point ingest-card fake realm at per-realm _search The fake-realm graph test stubbed the old profile-scoped federated search (`profileManager.authedRealmServerFetch`); ingest now queries the source realm's own `_search` via `authedRealmFetch`, so move the instance/Spec stub there (keyed on `/_search`) and drop the now-dead profile-manager search mock. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/commands/ingest-card-graph.test.ts | 100 ++++++++++-------- 1 file changed, 55 insertions(+), 45 deletions(-) diff --git a/packages/boxel-cli/tests/commands/ingest-card-graph.test.ts b/packages/boxel-cli/tests/commands/ingest-card-graph.test.ts index 009b9b9190..47b15ba732 100644 --- a/packages/boxel-cli/tests/commands/ingest-card-graph.test.ts +++ b/packages/boxel-cli/tests/commands/ingest-card-graph.test.ts @@ -128,7 +128,7 @@ function makeFakeAuthenticator(fetchedUrls: string[]): RealmAuthenticator { Object.keys(REALM_FILES).map((p, i) => [`${ROOT}${p}`, 1000 + i]), ); return { - async authedRealmFetch(input: string | URL | Request) { + async authedRealmFetch(input: string | URL | Request, init?: RequestInit) { let url = String(input); fetchedUrls.push(url); if (url === `${ROOT}_mtimes`) { @@ -139,6 +139,15 @@ function makeFakeAuthenticator(fetchedUrls: string[]): RealmAuthenticator { }, ); } + // The ingester discovers instances + Specs via the source realm's own + // `_search` endpoint (a QUERY request), not the profile-scoped + // federated search — so a shared/published source realm is reachable. + if (url === `${ROOT}_search`) { + return new Response( + JSON.stringify({ data: fakeSearchData(String(init?.body ?? '{}')) }), + { status: 200 }, + ); + } let rel = url.startsWith(ROOT) ? url.slice(ROOT.length) : null; if (rel != null && REALM_FILES[rel] != null) { return new Response(REALM_FILES[rel], { status: 200 }); @@ -148,9 +157,51 @@ function makeFakeAuthenticator(fetchedUrls: string[]): RealmAuthenticator { }; } -// Answers the two search shapes the ingester issues: instances of the entry -// card's exported classes, and all base-realm Spec cards (filtered by -// specType + ref in the ingester itself). +// The data the source realm's `_search` returns for the two shapes the +// ingester issues: instances of the entry card's exported classes, and all +// base-realm Spec cards (filtered by specType + ref in the ingester itself). +function fakeSearchData(bodyStr: string): unknown[] { + let body = JSON.parse(bodyStr) as { + filter?: { type?: { module?: string; name?: string } }; + }; + let type = body.filter?.type; + if (type?.module === 'https://cardstack.com/base/spec') { + return [ + { + id: `${ROOT}Spec/gadget`, + attributes: { + specType: 'card', + ref: { module: '../widgets/gadget/gadget', name: 'Gadget' }, + }, + }, + { + id: `${ROOT}Spec/clock`, + attributes: { + specType: 'card', + ref: { module: '../standalone/clock', name: 'Clock' }, + }, + }, + { + id: `${ROOT}Spec/widget-part-component`, + attributes: { + specType: 'component', + ref: { + module: '../widgets/gadget/parts/widget-part', + name: 'WidgetPart', + }, + }, + }, + ]; + } + if (type?.module === GADGET_MODULE_ABS && type?.name === 'Gadget') { + return [{ id: `${ROOT}Gadget/g1` }]; + } + return []; +} + +// Auth is supplied directly via `authenticator`, so the profile manager is +// only here to satisfy ingestCard's option plumbing — search no longer goes +// through it. function makeFakeProfileManager(): ProfileManager { return { getActiveProfile() { @@ -158,47 +209,6 @@ function makeFakeProfileManager(): ProfileManager { profile: { realmServerUrl: 'https://realm-server.example.test' }, }; }, - async authedRealmServerFetch(_url: string, init?: RequestInit) { - let body = JSON.parse(String(init?.body ?? '{}')) as { - filter?: { type?: { module?: string; name?: string } }; - }; - let type = body.filter?.type; - let data: unknown[] = []; - if (type?.module === 'https://cardstack.com/base/spec') { - data = [ - { - id: `${ROOT}Spec/gadget`, - attributes: { - specType: 'card', - ref: { module: '../widgets/gadget/gadget', name: 'Gadget' }, - }, - }, - { - id: `${ROOT}Spec/clock`, - attributes: { - specType: 'card', - ref: { module: '../standalone/clock', name: 'Clock' }, - }, - }, - { - id: `${ROOT}Spec/widget-part-component`, - attributes: { - specType: 'component', - ref: { - module: '../widgets/gadget/parts/widget-part', - name: 'WidgetPart', - }, - }, - }, - ]; - } else if ( - type?.module === GADGET_MODULE_ABS && - type?.name === 'Gadget' - ) { - data = [{ id: `${ROOT}Gadget/g1` }]; - } - return new Response(JSON.stringify({ data }), { status: 200 }); - }, } as unknown as ProfileManager; } From 814f807c04a9319ce2fa41d4befbf69946d475a3 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 19 Jun 2026 14:02:20 +0200 Subject: [PATCH 3/3] boxel-cli: include statusText + body snippet on ingest-card search failure The realm `_search` failure path logged only the bare HTTP status; surface statusText and a truncated response body too, so auth errors / malformed queries are diagnosable from the CLI output (parity with the previous federated-search path). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/boxel-cli/src/commands/realm/ingest-card.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/boxel-cli/src/commands/realm/ingest-card.ts b/packages/boxel-cli/src/commands/realm/ingest-card.ts index 117282e74b..150a6145dd 100644 --- a/packages/boxel-cli/src/commands/realm/ingest-card.ts +++ b/packages/boxel-cli/src/commands/realm/ingest-card.ts @@ -357,7 +357,13 @@ class RealmCardIngester extends RealmSyncBase { ); if (!res.ok) { this.hasError = true; - console.warn(` search failed: HTTP ${res.status}`); + // Include statusText + a body snippet so auth errors, malformed queries, + // etc. are diagnosable from the CLI output, not just a bare status code. + let body = await res.text().catch(() => ''); + console.warn( + ` search failed: HTTP ${res.status} ${res.statusText}`.trimEnd() + + (body ? ` — ${body.slice(0, 300)}` : ''), + ); return []; } let json = (await res.json()) as SearchResponse;