From 910ee273edb07629d014e203422ea32fafb975b4 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Wed, 17 Jun 2026 18:13:21 -0400 Subject: [PATCH 1/3] feat: surface frontmatter parse errors via indexing diagnostics Markdown frontmatter that failed to parse was swallowed with only a console.warn, silently dropping whatever the frontmatter declared (a skill's commands, its boxel.kind) while the file still indexed body-only. Capture the parse failure in MarkdownDef.extractAttributes and route it, mirroring the brokenLinks pattern, onto the index row's diagnostics.frontmatterParseError. The /_indexing-errors endpoint and the `boxel realm indexing-errors` CLI surface it as a new "frontmatter-error" finding class so authors can see and fix the YAML. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/base/markdown-file-def.gts | 50 ++++++++++- .../plugin/skills/indexing-errors/SKILL.md | 58 ++++++++++--- .../src/commands/realm/indexing-errors.ts | 53 +++++++++++- .../integration/realm-indexing-errors.test.ts | 50 +++++++++++ .../utils/file-def-attributes-extractor.ts | 18 ++++ .../markdown-skill-frontmatter-test.gts | 53 ++++++++++++ .../realm-endpoints/indexing-errors-test.ts | 83 +++++++++++++++++++ .../index-runner/file-indexer.ts | 14 +++- packages/runtime-common/index.ts | 30 +++++++ packages/runtime-common/realm.ts | 25 +++++- 10 files changed, 412 insertions(+), 22 deletions(-) diff --git a/packages/base/markdown-file-def.gts b/packages/base/markdown-file-def.gts index 9cd2fef48d..cd339bb440 100644 --- a/packages/base/markdown-file-def.gts +++ b/packages/base/markdown-file-def.gts @@ -36,6 +36,35 @@ import { parseFrontmatter } from './frontmatter-parse'; // the same global symbol. See `file-def-attributes-extractor.ts`. const fileFieldMetaSymbol = Symbol.for('boxel:file-field-meta'); +// Channel for routing a frontmatter YAML parse failure out of `extractAttributes` +// without it leaking into the flat `search_doc`. The host file extractor lifts +// this off the returned attributes into the extract response, and the indexer +// persists it onto `boxel_index.diagnostics.frontmatterParseError` so authors +// see the failure (CS-11548) instead of silently losing whatever the +// frontmatter declared. Shape matches `FrontmatterParseError` in runtime-common. +const frontmatterParseErrorSymbol = Symbol.for( + 'boxel:file-frontmatter-parse-error', +); + +// Best-effort structured view of a YAML parse failure. The `yaml` library +// throws a `YAMLParseError` carrying `linePos` (`[{ line, col }, …]`); read it +// defensively so a non-YAMLParseError still yields a usable message. +function toFrontmatterParseError(err: unknown): { + message: string; + line?: number; + column?: number; +} { + let message = + err instanceof Error ? err.message : `Frontmatter parse failed: ${err}`; + let pos = (err as { linePos?: Array<{ line?: number; col?: number }> }) + ?.linePos?.[0]; + return { + message, + ...(typeof pos?.line === 'number' ? { line: pos.line } : {}), + ...(typeof pos?.col === 'number' ? { column: pos.col } : {}), + }; +} + const MARKDOWN_EXTENSIONS = new Set(['.md', '.markdown']); const EXCERPT_MAX_LENGTH = 500; @@ -549,15 +578,24 @@ export class MarkdownDef extends FileDef { let frontmatterData: Record = {}; let body = markdown; + let frontmatterParseError: + | { message: string; line?: number; column?: number } + | undefined; try { let parsed = parseFrontmatter(normalizeMarkdown(markdown)); frontmatterData = parsed.data; body = parsed.body; } catch (err) { // Invalid YAML: index the markdown without frontmatter rather than fail - // the whole file. TODO: surface this via indexing diagnostics rather than - // only a console warning, so frontmatter parse errors stay visible. - console.warn(`[markdown-file-def] frontmatter parse failed for ${url}:`, err); + // the whole file, but capture the failure so it surfaces via indexing + // diagnostics (CS-11548) instead of silently dropping whatever the + // frontmatter declared (e.g. a skill's commands). Routed out-of-band via + // `frontmatterParseErrorSymbol`, picked up by the host file extractor. + frontmatterParseError = toFrontmatterParseError(err); + console.warn( + `[markdown-file-def] frontmatter parse failed for ${url}:`, + err, + ); } let attributes: SerializedFile<{ @@ -617,6 +655,12 @@ export class MarkdownDef extends FileDef { } } + if (frontmatterParseError) { + (attributes as Record)[ + frontmatterParseErrorSymbol + ] = frontmatterParseError; + } + return attributes; } } diff --git a/packages/boxel-cli/plugin/skills/indexing-errors/SKILL.md b/packages/boxel-cli/plugin/skills/indexing-errors/SKILL.md index e2fcb94323..36945cc7bf 100644 --- a/packages/boxel-cli/plugin/skills/indexing-errors/SKILL.md +++ b/packages/boxel-cli/plugin/skills/indexing-errors/SKILL.md @@ -1,24 +1,26 @@ --- -description: List every `boxel_index` row in a realm whose latest indexing attempt failed (`has_error = TRUE`) OR rendered cleanly but holds broken `linksTo` / `linksToMany` targets. Use to answer "why is this realm broken", "what's failing to index", or "which cards have dead references" without reading the database directly. +description: List every `boxel_index` row in a realm whose latest indexing attempt failed (`has_error = TRUE`), rendered cleanly but holds broken `linksTo` / `linksToMany` targets, or indexed but couldn't parse its frontmatter YAML. Use to answer "why is this realm broken", "what's failing to index", "which cards have dead references", or "why did my skill's commands disappear" without reading the database directly. --- # Realm indexing errors -`boxel realm indexing-errors` returns two classes of "indexing finding" for a realm: +`boxel realm indexing-errors` returns three classes of "indexing finding" for a realm: 1. **`indexing-error`** — `boxel_index` rows where `has_error = TRUE`. The render or file extract failed and the persisted `errorDoc` (a `SerializedError`: `message`, `status`, `title`, optional `stack`, optional `deps`) explains why. 2. **`broken-link`** — `boxel_index` rows where `has_error = FALSE` but the indexer's `render.meta` scan persisted a non-empty `diagnostics.brokenLinks` array. The card itself indexes fine; one or more `linksTo` / `linksToMany` targets are dead. Each entry exposes `attributes.brokenLinks: BrokenLinkSummary[]` — `{ fieldName, reference, kind: 'error' | 'not-found' }`. +3. **`frontmatter-error`** — `boxel_index` rows where `has_error = FALSE` but a markdown file's leading YAML frontmatter wouldn't parse, persisted as `diagnostics.frontmatterParseError`. The file indexes body-only, so anything the frontmatter declared (a skill's `commands`, its `boxel.kind`, etc.) was silently dropped — this finding is the only signal the author gets. Each entry exposes `attributes.frontmatterParseError` — `{ message, line?, column? }`. Use this skill to triage realm health in one call, without reading the database directly or scraping per-card failures. ## When the user asks to... -| Ask | Run | -|---|---| -| "what's failing to index in ?" | `boxel realm indexing-errors --realm ` | -| "which cards have broken links in ?" | `boxel realm indexing-errors --realm --json \| jq '.data[] \| select(.type == "broken-link")'` | -| "give me the full payload as JSON" | `boxel realm indexing-errors --realm --json` | -| "is anything broken in this realm right now?" | `boxel realm indexing-errors --realm ` — exit 0 with "No indexing errors." is the all-clear | +| Ask | Run | +| --------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | +| "what's failing to index in ?" | `boxel realm indexing-errors --realm ` | +| "which cards have broken links in ?" | `boxel realm indexing-errors --realm --json \| jq '.data[] \| select(.type == "broken-link")'` | +| "why did my skill's commands disappear / which files have bad frontmatter?" | `boxel realm indexing-errors --realm --json \| jq '.data[] \| select(.type == "frontmatter-error")'` | +| "give me the full payload as JSON" | `boxel realm indexing-errors --realm --json` | +| "is anything broken in this realm right now?" | `boxel realm indexing-errors --realm ` — exit 0 with "No indexing errors." is the all-clear | ## Typical sequencing @@ -57,13 +59,14 @@ List every card or module in a realm whose latest indexing attempt errored Default (human-readable): ``` -3 indexing findings for https://realm.example.com/: +4 indexing findings for https://realm.example.com/: [instance] https://realm.example.com/broken-card.json Cannot find module './missing' [file] https://realm.example.com/recipes.gts Unexpected token '<' [instance] https://realm.example.com/clean-but-linked.json 2 broken: author→https://…, tags→https://… +[file] https://realm.example.com/skills/sync/SKILL.md frontmatter parse error (line 4:3): Implicit map keys need to be on a single line ``` -Each line is `[] `. For `indexing-error` rows the summary is `errorDoc.title` (falling back to `errorDoc.message`). For `broken-link` rows it is ` broken: fieldName→reference, …` (up to three, then `+N more`). +Each line is `[] `. For `indexing-error` rows the summary is `errorDoc.title` (falling back to `errorDoc.message`). For `broken-link` rows it is ` broken: fieldName→reference, …` (up to three, then `+N more`). For `frontmatter-error` rows it is `frontmatter parse error (line L:C): `. `--json` emits a JSON-API document: @@ -88,13 +91,41 @@ Each line is `[] `. For `indexing-error` rows t "entryType": "instance", "diagnostics": { "brokenLinks": [ - { "fieldName": "author", "reference": "https://...", "kind": "not-found" } + { + "fieldName": "author", + "reference": "https://...", + "kind": "not-found" + } ] }, "brokenLinks": [ - { "fieldName": "author", "reference": "https://...", "kind": "not-found" } + { + "fieldName": "author", + "reference": "https://...", + "kind": "not-found" + } ] } + }, + { + "type": "frontmatter-error", + "id": "file::https://realm.example.com/skills/sync/SKILL.md", + "attributes": { + "url": "https://realm.example.com/skills/sync/SKILL.md", + "entryType": "file", + "diagnostics": { + "frontmatterParseError": { + "message": "Implicit map keys need to be on a single line", + "line": 4, + "column": 3 + } + }, + "frontmatterParseError": { + "message": "Implicit map keys need to be on a single line", + "line": 4, + "column": 3 + } + } } ] } @@ -105,7 +136,8 @@ Each line is `[] `. For `indexing-error` rows t - `--realm` is required and must be the realm base URL (with the realm path, not just the server origin). - The endpoint reports findings for **all** entry types (instances, modules, files) — not just card instances. A failing `.gts` shows up here even if it isn't published as a card. - The default human-readable line truncates the error message to 100 characters and prefers `errorDoc.title` over `errorDoc.message`. Use `--json` for the full payload. -- `broken-link` findings have **no** `errorDoc` — `has_error` is `FALSE`. Branch on `data[].type` (or check `attributes.brokenLinks`) before reading `errorDoc`. +- `broken-link` and `frontmatter-error` findings have **no** `errorDoc` — `has_error` is `FALSE`. Branch on `data[].type` (or check `attributes.brokenLinks` / `attributes.frontmatterParseError`) before reading `errorDoc`. +- A `frontmatter-error` means the file _did_ index — just without its frontmatter. The body is fine; whatever the YAML declared (skill `commands`, `boxel.kind`) is gone until the YAML is fixed and the realm re-indexes. - JSON-API resource `id` is `${entryType}::${url}`, not bare `${url}`. To get the URL back, read `attributes.url`. - On transport failure with `--json`, stdout receives `{"error":"..."}` (not the data envelope) and the process exits non-zero. Successful calls always emit `{ "data": [...] }`. A consumer reading only stdout can distinguish failure from "realm is healthy" by checking for the `error` key (or the exit code). - The list does not include queue depth, running-job progress, or a "last finished at" timestamp. Those are separate concerns and may be folded into a future `indexing-status` command. diff --git a/packages/boxel-cli/src/commands/realm/indexing-errors.ts b/packages/boxel-cli/src/commands/realm/indexing-errors.ts index 180d8a1170..db6be26c67 100644 --- a/packages/boxel-cli/src/commands/realm/indexing-errors.ts +++ b/packages/boxel-cli/src/commands/realm/indexing-errors.ts @@ -40,7 +40,25 @@ export interface BrokenLinkEntry { }; } -export type IndexingErrorsEntry = IndexingErrorEntry | BrokenLinkEntry; +// Resource for a row that indexed cleanly but whose YAML frontmatter wouldn't +// parse, surfaced via `diagnostics.frontmatterParseError`. The file indexes +// body-only; anything the frontmatter declared (e.g. a skill's commands) was +// dropped. +export interface FrontmatterErrorEntry { + type: 'frontmatter-error'; + id: string; + attributes: { + url: string; + entryType: string; + diagnostics: Record | null; + frontmatterParseError: FrontmatterParseErrorLike; + }; +} + +export type IndexingErrorsEntry = + | IndexingErrorEntry + | BrokenLinkEntry + | FrontmatterErrorEntry; export interface IndexingErrorsDocument { data: IndexingErrorsEntry[]; @@ -54,6 +72,14 @@ export interface BrokenLinkLike { kind: 'error' | 'not-found'; } +// Mirror of FrontmatterParseError from @cardstack/runtime-common, kept local +// for the same reason. +export interface FrontmatterParseErrorLike { + message: string; + line?: number; + column?: number; +} + export interface IndexingErrorsResult { ok: boolean; document?: IndexingErrorsDocument; @@ -179,9 +205,34 @@ export function formatEntry(entry: IndexingErrorsEntry): string { if (entry.type === 'indexing-error') { return `${prefix} ${url} ${shortErrorMessage(entry.attributes.errorDoc)}`; } + if (entry.type === 'frontmatter-error') { + return `${prefix} ${url} ${shortFrontmatterError( + entry.attributes.frontmatterParseError, + )}`; + } return `${prefix} ${url} ${shortBrokenLinks(entry.attributes.brokenLinks)}`; } +export function shortFrontmatterError( + parseError: FrontmatterParseErrorLike | null | undefined, +): string { + if (!parseError) { + return ''; + } + let where = + typeof parseError.line === 'number' + ? ` (line ${parseError.line}${ + typeof parseError.column === 'number' ? `:${parseError.column}` : '' + })` + : ''; + let raw = (parseError.message ?? '').replace(/\s+/g, ' ').trim(); + let message = + raw.length <= SHORT_MESSAGE_MAX + ? raw + : `${raw.slice(0, SHORT_MESSAGE_MAX - 1)}…`; + return `frontmatter parse error${where}: ${message}`; +} + const BROKEN_LINKS_MAX_LIST = 3; export function shortBrokenLinks( diff --git a/packages/boxel-cli/tests/integration/realm-indexing-errors.test.ts b/packages/boxel-cli/tests/integration/realm-indexing-errors.test.ts index 64d4ee40ce..73570910f9 100644 --- a/packages/boxel-cli/tests/integration/realm-indexing-errors.test.ts +++ b/packages/boxel-cli/tests/integration/realm-indexing-errors.test.ts @@ -7,9 +7,11 @@ import { indexingErrors, shortErrorMessage, shortBrokenLinks, + shortFrontmatterError, formatEntry, type IndexingErrorEntry, type BrokenLinkEntry, + type FrontmatterErrorEntry, } from '../../src/commands/realm/indexing-errors.ts'; import { ProfileManager } from '../../src/lib/profile-manager.ts'; import { @@ -192,6 +194,41 @@ describe('realm indexing-errors (integration)', () => { expect(formatEntry(entry!)).toContain('1 broken: author→'); }); + it('surfaces frontmatter-error rows with has_error = FALSE', async () => { + let dbAdapter = getTestDbAdapter(); + let fileURL = `${realmUrl}skills/bad/SKILL.md`; + let fileAlias = `${realmUrl}skills/bad/SKILL`; + let frontmatterParseError = { + message: 'Implicit map keys need to be on a single line', + line: 4, + column: 3, + }; + let diagnostics = { frontmatterParseError }; + + await dbAdapter!.execute( + `INSERT INTO boxel_index + (url, file_alias, type, realm_version, realm_url, + has_error, error_doc, diagnostics, is_deleted) + VALUES ($1, $2, 'file', 1, $3, + FALSE, NULL, $4::jsonb, FALSE)`, + { + bind: [fileURL, fileAlias, realmUrl, JSON.stringify(diagnostics)], + }, + ); + + let result = await indexingErrors(realmUrl, { profileManager }); + expect(result.ok).toBe(true); + let entry = result.document!.data.find( + (e) => e.attributes.url === fileURL, + ) as FrontmatterErrorEntry | undefined; + expect(entry).toBeDefined(); + expect(entry!.type).toBe('frontmatter-error'); + expect(entry!.attributes.frontmatterParseError).toEqual( + frontmatterParseError, + ); + expect(formatEntry(entry!)).toContain('frontmatter parse error (line 4:3)'); + }); + it('returns ok=false when the realm is unreachable', async () => { let result = await indexingErrors('http://127.0.0.1:1/fake/', { profileManager, @@ -240,4 +277,17 @@ describe('realm indexing-errors (integration)', () => { ]), ).toBe('4 broken: a→x, b→y, c→z, …+1 more'); }); + + it('shortFrontmatterError renders message with optional position', () => { + expect(shortFrontmatterError(null)).toBe(''); + expect(shortFrontmatterError({ message: 'bad yaml' })).toBe( + 'frontmatter parse error: bad yaml', + ); + expect( + shortFrontmatterError({ message: 'bad yaml', line: 4, column: 3 }), + ).toBe('frontmatter parse error (line 4:3): bad yaml'); + expect(shortFrontmatterError({ message: 'bad yaml', line: 4 })).toBe( + 'frontmatter parse error (line 4): bad yaml', + ); + }); }); diff --git a/packages/host/app/utils/file-def-attributes-extractor.ts b/packages/host/app/utils/file-def-attributes-extractor.ts index ef2daf7571..7b66f42c84 100644 --- a/packages/host/app/utils/file-def-attributes-extractor.ts +++ b/packages/host/app/utils/file-def-attributes-extractor.ts @@ -9,6 +9,7 @@ import { SupportedMimeType, type CodeRef, type FileMetaResource, + type FrontmatterParseError, type QueryFieldMeta, type RealmResourceIdentifier, type RenderError, @@ -44,6 +45,10 @@ export type FileDefExtractResult = { deps: string[]; error?: RenderError; mismatch?: true; + // The frontmatter YAML wouldn't parse. The extract still succeeds (the file + // indexes body-only); lifted out of the searchDoc here so the indexer can + // persist it onto `diagnostics.frontmatterParseError`. See markdown-file-def. + frontmatterParseError?: FrontmatterParseError; }; export class FileDefAttributesExtractor { @@ -213,6 +218,18 @@ export class FileDefAttributesExtractor { if (fieldsMeta) { delete cleanedBag[fieldMetaSymbol]; } + // Same out-of-band lift for a frontmatter parse failure: keep it off + // the flat `search_doc` and hand it back so the indexer can persist it + // onto `diagnostics.frontmatterParseError`. + let frontmatterParseErrorSymbol = Symbol.for( + 'boxel:file-frontmatter-parse-error', + ); + let frontmatterParseError = cleanedBag[frontmatterParseErrorSymbol] as + | FrontmatterParseError + | undefined; + if (frontmatterParseError) { + delete cleanedBag[frontmatterParseErrorSymbol]; + } return { status: 'ready', searchDoc: cleanedDoc, @@ -228,6 +245,7 @@ export class FileDefAttributesExtractor { deps, ...(error ? { error } : {}), ...(mismatch ? { mismatch: true } : {}), + ...(frontmatterParseError ? { frontmatterParseError } : {}), }; } } diff --git a/packages/host/tests/integration/components/markdown-skill-frontmatter-test.gts b/packages/host/tests/integration/components/markdown-skill-frontmatter-test.gts index 072db3727c..c4fdaea06d 100644 --- a/packages/host/tests/integration/components/markdown-skill-frontmatter-test.gts +++ b/packages/host/tests/integration/components/markdown-skill-frontmatter-test.gts @@ -26,6 +26,9 @@ import { setupMockMatrix } from '../../helpers/mock-matrix'; import { setupRenderingTest } from '../../helpers/setup'; const FILE_FIELD_META = Symbol.for('boxel:file-field-meta'); +const FRONTMATTER_PARSE_ERROR = Symbol.for( + 'boxel:file-frontmatter-parse-error', +); let loader: Loader; @@ -211,6 +214,56 @@ module('Integration | markdown skill frontmatter', function (hooks) { ); }); + test('invalid frontmatter YAML routes a parse-error marker and still indexes the body', async function (assert) { + let { MarkdownDef } = await loadBase(); + let url = `${testRealmURL}skills/bad/SKILL.md`; + // Unterminated flow mapping — valid `---` fences, invalid YAML inside. + let badMarkdown = `--- +name: Bad Skill +boxel: { kind: skill, commands: [ +--- +# Bad Skill + +Body paragraph. +`; + let attrs = await MarkdownDef.extractAttributes( + url, + streamOf(badMarkdown), + {}, + ); + + let routed = (attrs as Record)[FRONTMATTER_PARSE_ERROR]; + assert.strictEqual( + typeof routed?.message, + 'string', + 'routes a frontmatter parse error carrying a message string', + ); + assert.true( + (routed?.message?.length ?? 0) > 0, + 'the routed parse-error message is non-empty', + ); + + assert.strictEqual( + attrs.kind, + undefined, + 'no searchable kind — the frontmatter was dropped', + ); + assert.strictEqual( + attrs.frontmatter, + undefined, + 'no typed frontmatter value when the YAML failed to parse', + ); + assert.strictEqual( + (attrs as Record)[FILE_FIELD_META], + undefined, + 'no routed field meta when the frontmatter was dropped', + ); + assert.true( + attrs.content.includes('Body paragraph.'), + 'body is still indexed despite the parse failure', + ); + }); + test('plain markdown (no frontmatter) carries no kind and no frontmatter value', async function (assert) { let { MarkdownDef } = await loadBase(); let url = `${testRealmURL}notes/readme.md`; diff --git a/packages/realm-server/tests/realm-endpoints/indexing-errors-test.ts b/packages/realm-server/tests/realm-endpoints/indexing-errors-test.ts index 961bcd67e4..1e6eb1af06 100644 --- a/packages/realm-server/tests/realm-endpoints/indexing-errors-test.ts +++ b/packages/realm-server/tests/realm-endpoints/indexing-errors-test.ts @@ -353,5 +353,88 @@ module(`realm-endpoints/${basename(__filename)}`, function () { 'brokenLinks payload included', ); }); + + test('surfaces frontmatter-error rows even when has_error is FALSE', async function (assert) { + await sourceRealm.realmIndexUpdater.fullIndex(); + + let fileURL = `${sourceRealm.url}skills/bad/SKILL.md`; + let frontmatterParseError = { + message: 'Implicit map keys need to be on a single line', + line: 4, + column: 3, + }; + let diagnostics = { frontmatterParseError }; + + // A file row that indexed cleanly (has_error = FALSE) but whose YAML + // frontmatter wouldn't parse. Upsert keeps the test re-runnable against + // the cached realm. + await dbAdapter.execute( + `INSERT INTO boxel_index + (url, file_alias, type, realm_version, realm_url, + has_error, error_doc, diagnostics, is_deleted) + VALUES ($1, $2, 'file', 1, $3, FALSE, NULL, $4::jsonb, FALSE) + ON CONFLICT (url, realm_url, type) DO UPDATE + SET has_error = FALSE, + error_doc = NULL, + diagnostics = EXCLUDED.diagnostics, + is_deleted = FALSE`, + { + bind: [ + fileURL, + fileURL.replace(/\.md$/, ''), + sourceRealm.url, + JSON.stringify(diagnostics), + ], + }, + ); + + let response = await request + .get(`${new URL(sourceRealm.url).pathname}_indexing-errors`) + .set('Accept', SupportedMimeType.JSONAPI) + .set( + 'Authorization', + `Bearer ${createJWT(sourceRealm, ownerUserId, DEFAULT_PERMISSIONS)}`, + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + let entries = ( + response.body.data as Array<{ + type: string; + id: string; + attributes: { + url: string; + entryType: string; + errorDoc?: unknown; + frontmatterParseError?: { + message: string; + line?: number; + column?: number; + }; + }; + }> + ).filter((e) => e.attributes.url === fileURL); + assert.strictEqual( + entries.length, + 1, + 'one frontmatter-error row reported', + ); + let entry = entries[0]; + assert.strictEqual(entry.type, 'frontmatter-error', 'discriminator'); + assert.strictEqual( + entry.id, + `file::${fileURL}`, + 'id encodes both entry type and URL', + ); + assert.strictEqual( + entry.attributes.errorDoc, + undefined, + 'no errorDoc on a healthy-but-unparseable-frontmatter row', + ); + assert.deepEqual( + entry.attributes.frontmatterParseError, + frontmatterParseError, + 'frontmatterParseError payload included', + ); + }); }); }); diff --git a/packages/runtime-common/index-runner/file-indexer.ts b/packages/runtime-common/index-runner/file-indexer.ts index 077c5c8178..2d4a7ee7f5 100644 --- a/packages/runtime-common/index-runner/file-indexer.ts +++ b/packages/runtime-common/index-runner/file-indexer.ts @@ -205,6 +205,18 @@ export async function performFileIndexing({ // is no longer acted on here — the fused visit already gates fileRender. void hasModulePrerender; + // A frontmatter parse failure doesn't fail the file (it still indexes + // body-only), so it rides on the row's diagnostics — mirroring brokenLinks — + // where `/_indexing-errors` surfaces it for the author. Merge it onto + // whatever render-side diagnostics the visit already produced. + let fileDiagnostics: Diagnostics | undefined = + extractResult.frontmatterParseError + ? { + ...(diagnostics ?? {}), + frontmatterParseError: extractResult.frontmatterParseError, + } + : diagnostics; + await updateEntry(entryURL, { type: 'file', lastModified, @@ -227,7 +239,7 @@ export async function performFileIndexing({ fittedHtml: renderResult?.fittedHTML ?? undefined, iconHTML: renderResult?.iconHTML ?? undefined, markdown: renderResult?.markdown ?? undefined, - diagnostics, + diagnostics: fileDiagnostics, }); return 'indexed'; diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index cdd5e7a290..9a55a01db0 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -57,6 +57,25 @@ export interface BrokenLinkSummary { kind: 'error' | 'not-found'; } +// A failure to parse a markdown file's leading YAML frontmatter block, +// recorded as a finding on the (still successful) index entry. The file +// indexes fine — `extractAttributes` falls back to treating the whole file +// as body when the frontmatter won't parse — so without this the failure is +// invisible and any frontmatter-declared behavior (e.g. a skill's `commands`) +// silently disappears. Surfaced on `diagnostics.frontmatterParseError` so the +// `/_indexing-errors` surface can flag it the way it flags `brokenLinks`, +// letting authors see and fix the YAML rather than wonder where their +// commands went. +export interface FrontmatterParseError { + // The YAML parser's error message. + message: string; + // 1-based line within the frontmatter block where the parse failed, when + // the parser reports a position. Omitted otherwise. + line?: number; + // 1-based column within that line, when reported. + column?: number; +} + // Per-render computed-field counters captured by the host's render.meta // route. Emitted alongside PrerenderMeta so the Prerenderer can lift them // onto `response.meta.diagnostics` and the indexer can persist them onto @@ -336,6 +355,11 @@ export interface FileExtractResponse { deps: string[]; error?: RenderError; mismatch?: true; + // Set when the file's leading YAML frontmatter block was present but + // wouldn't parse. The extract still succeeds (`status: 'ready'`, body-only); + // the file indexer merges this onto `diagnostics.frontmatterParseError` so + // the failure surfaces via `/_indexing-errors` instead of vanishing. + frontmatterParseError?: FrontmatterParseError; } export interface FileRenderResponse { @@ -432,6 +456,12 @@ export interface Diagnostics extends RenderTimeoutDiagnostics, PrerenderMetaDiagnostics { invalidationId?: string; indexedAt?: number; + // Frontmatter YAML that wouldn't parse during file extraction. The row + // still indexes (body-only); this is the only indexed signal that the + // file's frontmatter — and anything it declared — was dropped. Merged in + // by the file indexer from the extract response. Absent when the + // frontmatter parsed (or there was none). + frontmatterParseError?: FrontmatterParseError; } // Flatten a prerender `response.meta` block into the shape persisted to diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 781114275a..63689d7a9b 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -6240,6 +6240,7 @@ export class Realm { ` jsonb_typeof(diagnostics->'brokenLinks') = 'array'`, ` AND jsonb_array_length(diagnostics->'brokenLinks') > 0`, ` )`, + ` OR jsonb_typeof(diagnostics->'frontmatterParseError') = 'object'`, `)`, `ORDER BY type, url`, ])) as { @@ -6255,15 +6256,28 @@ export class Realm { row.diagnostics && Array.isArray(row.diagnostics.brokenLinks) ? (row.diagnostics.brokenLinks as unknown[]) : null; + let frontmatterParseError = + row.diagnostics && + typeof row.diagnostics.frontmatterParseError === 'object' && + row.diagnostics.frontmatterParseError !== null + ? (row.diagnostics.frontmatterParseError as Record) + : null; let hasError = row.error_doc != null; // 'indexing-error' = row.has_error = TRUE (rendered/indexed badly). // 'broken-link' = the index row is healthy but the rendered card has - // dead linksTo/linksToMany targets surfaced by render.meta. Both - // classes share the (entryType, url) key; the discriminator lets + // dead linksTo/linksToMany targets surfaced by render.meta. + // 'frontmatter-error' = the index row is healthy but the file's YAML + // frontmatter wouldn't parse, so anything it declared was dropped. + // All classes share the (entryType, url) key; the discriminator lets // consumers branch on which attributes to read. - let resourceType: 'indexing-error' | 'broken-link' = hasError + let resourceType: + | 'indexing-error' + | 'broken-link' + | 'frontmatter-error' = hasError ? 'indexing-error' - : 'broken-link'; + : frontmatterParseError + ? 'frontmatter-error' + : 'broken-link'; let attributes: Record = { url: row.url, entryType: row.type, @@ -6275,6 +6289,9 @@ export class Realm { if (brokenLinks && brokenLinks.length > 0) { attributes.brokenLinks = brokenLinks; } + if (frontmatterParseError) { + attributes.frontmatterParseError = frontmatterParseError; + } return { type: resourceType, // `(type, url)` is the boxel_index PK partition; encoding both From cc8c56a7778bad90c4bab1cefa75c106147f1f1a Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Thu, 18 Jun 2026 15:00:50 -0400 Subject: [PATCH 2/3] fix: emit broken-link and frontmatter-error as separate findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A healthy boxel_index row (has_error = FALSE) can carry both diagnostics.brokenLinks and diagnostics.frontmatterParseError — e.g. a markdown skill with invalid frontmatter and a dead card reference in its body. The _indexing-errors endpoint classified such a row only as frontmatter-error, so the CLI printed just the frontmatter summary and JSON consumers filtering for type == "broken-link" lost the broken-link signal entirely. Emit one resource per finding class instead of one per row. A row that yields more than one finding appends the finding class to its id so the two resources stay unique; single-finding rows keep the existing `::` id. Error rows are unchanged — brokenLinks still ride along as an attribute on the indexing-error resource. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../integration/realm-indexing-errors.test.ts | 52 ++++++++++ .../realm-endpoints/indexing-errors-test.ts | 99 +++++++++++++++++++ packages/runtime-common/realm.ts | 74 +++++++++----- 3 files changed, 200 insertions(+), 25 deletions(-) diff --git a/packages/boxel-cli/tests/integration/realm-indexing-errors.test.ts b/packages/boxel-cli/tests/integration/realm-indexing-errors.test.ts index 73570910f9..268532ee4a 100644 --- a/packages/boxel-cli/tests/integration/realm-indexing-errors.test.ts +++ b/packages/boxel-cli/tests/integration/realm-indexing-errors.test.ts @@ -229,6 +229,58 @@ describe('realm indexing-errors (integration)', () => { expect(formatEntry(entry!)).toContain('frontmatter parse error (line 4:3)'); }); + it('surfaces both findings when a healthy row has broken links AND a frontmatter error', async () => { + let dbAdapter = getTestDbAdapter(); + let fileURL = `${realmUrl}skills/both/SKILL.md`; + let fileAlias = `${realmUrl}skills/both/SKILL`; + let frontmatterParseError = { + message: 'Implicit map keys need to be on a single line', + line: 4, + column: 3, + }; + let brokenLinks = [ + { + fieldName: 'related', + reference: 'https://example.com/missing', + kind: 'not-found', + }, + ]; + let diagnostics = { frontmatterParseError, brokenLinks }; + + await dbAdapter!.execute( + `INSERT INTO boxel_index + (url, file_alias, type, realm_version, realm_url, + has_error, error_doc, diagnostics, is_deleted) + VALUES ($1, $2, 'file', 1, $3, + FALSE, NULL, $4::jsonb, FALSE)`, + { + bind: [fileURL, fileAlias, realmUrl, JSON.stringify(diagnostics)], + }, + ); + + let result = await indexingErrors(realmUrl, { profileManager }); + expect(result.ok).toBe(true); + let forUrl = result.document!.data.filter( + (e) => e.attributes.url === fileURL, + ); + expect(forUrl.length).toBe(2); + let byType = Object.fromEntries(forUrl.map((e) => [e.type, e])); + + let frontmatterEntry = byType['frontmatter-error'] as FrontmatterErrorEntry; + expect(frontmatterEntry).toBeDefined(); + expect(frontmatterEntry.attributes.frontmatterParseError).toEqual( + frontmatterParseError, + ); + expect(formatEntry(frontmatterEntry)).toContain( + 'frontmatter parse error (line 4:3)', + ); + + let brokenLinkEntry = byType['broken-link'] as BrokenLinkEntry; + expect(brokenLinkEntry).toBeDefined(); + expect(brokenLinkEntry.attributes.brokenLinks).toEqual(brokenLinks); + expect(formatEntry(brokenLinkEntry)).toContain('1 broken: related→'); + }); + it('returns ok=false when the realm is unreachable', async () => { let result = await indexingErrors('http://127.0.0.1:1/fake/', { profileManager, diff --git a/packages/realm-server/tests/realm-endpoints/indexing-errors-test.ts b/packages/realm-server/tests/realm-endpoints/indexing-errors-test.ts index 1e6eb1af06..ef1a9fa226 100644 --- a/packages/realm-server/tests/realm-endpoints/indexing-errors-test.ts +++ b/packages/realm-server/tests/realm-endpoints/indexing-errors-test.ts @@ -436,5 +436,104 @@ module(`realm-endpoints/${basename(__filename)}`, function () { 'frontmatterParseError payload included', ); }); + + test('emits both findings when a healthy row has broken links AND a frontmatter parse error', async function (assert) { + await sourceRealm.realmIndexUpdater.fullIndex(); + + let fileURL = `${sourceRealm.url}skills/both/SKILL.md`; + let frontmatterParseError = { + message: 'Implicit map keys need to be on a single line', + line: 4, + column: 3, + }; + let brokenLinks = [ + { + fieldName: 'related', + reference: 'https://example.com/missing', + kind: 'not-found', + }, + ]; + let diagnostics = { frontmatterParseError, brokenLinks }; + + // A healthy file row (has_error = FALSE) that carries two independent + // findings at once. Neither should mask the other. + await dbAdapter.execute( + `INSERT INTO boxel_index + (url, file_alias, type, realm_version, realm_url, + has_error, error_doc, diagnostics, is_deleted) + VALUES ($1, $2, 'file', 1, $3, FALSE, NULL, $4::jsonb, FALSE) + ON CONFLICT (url, realm_url, type) DO UPDATE + SET has_error = FALSE, + error_doc = NULL, + diagnostics = EXCLUDED.diagnostics, + is_deleted = FALSE`, + { + bind: [ + fileURL, + fileURL.replace(/\.md$/, ''), + sourceRealm.url, + JSON.stringify(diagnostics), + ], + }, + ); + + let response = await request + .get(`${new URL(sourceRealm.url).pathname}_indexing-errors`) + .set('Accept', SupportedMimeType.JSONAPI) + .set( + 'Authorization', + `Bearer ${createJWT(sourceRealm, ownerUserId, DEFAULT_PERMISSIONS)}`, + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + let entries = ( + response.body.data as Array<{ + type: string; + id: string; + attributes: { + url: string; + entryType: string; + errorDoc?: unknown; + brokenLinks?: Array<{ fieldName: string }>; + frontmatterParseError?: { message: string }; + }; + }> + ).filter((e) => e.attributes.url === fileURL); + assert.strictEqual( + entries.length, + 2, + 'one finding per class for the same row', + ); + let byType = Object.fromEntries(entries.map((e) => [e.type, e])); + + let frontmatterEntry = byType['frontmatter-error']; + assert.ok(frontmatterEntry, 'frontmatter-error finding present'); + assert.strictEqual( + frontmatterEntry.id, + `file::${fileURL}::frontmatter-error`, + 'multi-finding ids append the finding class to stay unique', + ); + assert.deepEqual( + frontmatterEntry.attributes.frontmatterParseError, + frontmatterParseError, + 'frontmatterParseError payload on the frontmatter-error finding', + ); + + let brokenLinkEntry = byType['broken-link']; + assert.ok( + brokenLinkEntry, + 'broken-link finding is not hidden by the frontmatter error', + ); + assert.strictEqual( + brokenLinkEntry.id, + `file::${fileURL}::broken-link`, + 'broken-link finding gets its own unique id', + ); + assert.deepEqual( + brokenLinkEntry.attributes.brokenLinks?.map((l) => l.fieldName), + ['related'], + 'brokenLinks payload on the broken-link finding', + ); + }); }); }); diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 63689d7a9b..e160979ca0 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -6251,7 +6251,7 @@ export class Realm { }[]; let doc = { - data: rows.map((row) => { + data: rows.flatMap((row) => { let brokenLinks = row.diagnostics && Array.isArray(row.diagnostics.brokenLinks) ? (row.diagnostics.brokenLinks as unknown[]) @@ -6263,43 +6263,67 @@ export class Realm { ? (row.diagnostics.frontmatterParseError as Record) : null; let hasError = row.error_doc != null; + // A single boxel_index row can carry more than one independent + // finding — e.g. a markdown skill with both unparseable frontmatter + // and a broken card reference in its body. We emit one resource per + // finding so a consumer filtering by `type` (the JSON CLI, or anyone + // selecting only 'broken-link') never loses a signal just because it + // co-occurs with another. + // // 'indexing-error' = row.has_error = TRUE (rendered/indexed badly). + // Any brokenLinks ride along as an attribute since the row's + // headline is the render failure, not the dead targets. // 'broken-link' = the index row is healthy but the rendered card has - // dead linksTo/linksToMany targets surfaced by render.meta. + // dead linksTo/linksToMany targets surfaced by render.meta. // 'frontmatter-error' = the index row is healthy but the file's YAML - // frontmatter wouldn't parse, so anything it declared was dropped. + // frontmatter wouldn't parse, so anything it declared was dropped. // All classes share the (entryType, url) key; the discriminator lets // consumers branch on which attributes to read. - let resourceType: - | 'indexing-error' - | 'broken-link' - | 'frontmatter-error' = hasError - ? 'indexing-error' - : frontmatterParseError - ? 'frontmatter-error' - : 'broken-link'; - let attributes: Record = { + let baseAttributes = { url: row.url, entryType: row.type, diagnostics: row.diagnostics, }; + let findings: { + type: 'indexing-error' | 'broken-link' | 'frontmatter-error'; + attributes: Record; + }[] = []; if (hasError) { - attributes.errorDoc = row.error_doc; - } - if (brokenLinks && brokenLinks.length > 0) { - attributes.brokenLinks = brokenLinks; - } - if (frontmatterParseError) { - attributes.frontmatterParseError = frontmatterParseError; + let attributes: Record = { + ...baseAttributes, + errorDoc: row.error_doc, + }; + if (brokenLinks && brokenLinks.length > 0) { + attributes.brokenLinks = brokenLinks; + } + findings.push({ type: 'indexing-error', attributes }); + } else { + if (frontmatterParseError) { + findings.push({ + type: 'frontmatter-error', + attributes: { ...baseAttributes, frontmatterParseError }, + }); + } + if (brokenLinks && brokenLinks.length > 0) { + findings.push({ + type: 'broken-link', + attributes: { ...baseAttributes, brokenLinks }, + }); + } } - return { - type: resourceType, + return findings.map((finding) => ({ + type: finding.type, // `(type, url)` is the boxel_index PK partition; encoding both // keeps the JSON:API resource id unique when the same URL fails - // as both 'instance' and 'file'. - id: `${row.type}::${row.url}`, - attributes, - }; + // as both 'instance' and 'file'. When a single row yields more + // than one finding we append the finding class too, so the two + // resources don't collide on a shared id. + id: + findings.length > 1 + ? `${row.type}::${row.url}::${finding.type}` + : `${row.type}::${row.url}`, + attributes: finding.attributes, + })); }), }; From dfdcc963442ee7729e38e560cb41695ddbc4ef14 Mon Sep 17 00:00:00 2001 From: ylm Date: Sat, 20 Jun 2026 14:02:08 -0400 Subject: [PATCH 3/3] refactor: address Copilot review on indexing-errors discriminator and symbol drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three small follow-ups to the frontmatter-error indexing surface (PR #5272): 1) `/_indexing-errors` now uses `has_error` from the SQL row as the discriminator instead of `error_doc != null`. The query filters on `has_error = TRUE`, so the previous branch silently dropped any row with `has_error = TRUE` but a NULL `error_doc` — it satisfied the filter but produced zero findings. Added a regression test. 2) Hoisted `'boxel:file-frontmatter-parse-error'` to a single exported constant `FRONTMATTER_PARSE_ERROR_SYMBOL` in runtime-common; the base markdown file-def, host extractor, and host test now all share it. 3) Replaced the inline `{ message, line?, column? }` shapes in `markdown-file-def.gts` with `FrontmatterParseError` (already exported from runtime-common) so the producer and consumer agree on one type. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/base/markdown-file-def.gts | 26 +++------- .../utils/file-def-attributes-extractor.ts | 12 ++--- .../markdown-skill-frontmatter-test.gts | 13 +++-- .../realm-endpoints/indexing-errors-test.ts | 48 +++++++++++++++++++ packages/runtime-common/index.ts | 9 ++++ packages/runtime-common/realm.ts | 9 +++- 6 files changed, 83 insertions(+), 34 deletions(-) diff --git a/packages/base/markdown-file-def.gts b/packages/base/markdown-file-def.gts index cd339bb440..cd313f39f4 100644 --- a/packages/base/markdown-file-def.gts +++ b/packages/base/markdown-file-def.gts @@ -1,8 +1,10 @@ import { byteStreamToUint8Array, extractCardReferenceUrls, + FRONTMATTER_PARSE_ERROR_SYMBOL, identifyCard, VirtualNetwork, + type FrontmatterParseError, } from '@cardstack/runtime-common'; import MarkdownIcon from '@cardstack/boxel-icons/align-box-left-middle'; import { @@ -36,24 +38,10 @@ import { parseFrontmatter } from './frontmatter-parse'; // the same global symbol. See `file-def-attributes-extractor.ts`. const fileFieldMetaSymbol = Symbol.for('boxel:file-field-meta'); -// Channel for routing a frontmatter YAML parse failure out of `extractAttributes` -// without it leaking into the flat `search_doc`. The host file extractor lifts -// this off the returned attributes into the extract response, and the indexer -// persists it onto `boxel_index.diagnostics.frontmatterParseError` so authors -// see the failure (CS-11548) instead of silently losing whatever the -// frontmatter declared. Shape matches `FrontmatterParseError` in runtime-common. -const frontmatterParseErrorSymbol = Symbol.for( - 'boxel:file-frontmatter-parse-error', -); - // Best-effort structured view of a YAML parse failure. The `yaml` library // throws a `YAMLParseError` carrying `linePos` (`[{ line, col }, …]`); read it // defensively so a non-YAMLParseError still yields a usable message. -function toFrontmatterParseError(err: unknown): { - message: string; - line?: number; - column?: number; -} { +function toFrontmatterParseError(err: unknown): FrontmatterParseError { let message = err instanceof Error ? err.message : `Frontmatter parse failed: ${err}`; let pos = (err as { linePos?: Array<{ line?: number; col?: number }> }) @@ -578,9 +566,7 @@ export class MarkdownDef extends FileDef { let frontmatterData: Record = {}; let body = markdown; - let frontmatterParseError: - | { message: string; line?: number; column?: number } - | undefined; + let frontmatterParseError: FrontmatterParseError | undefined; try { let parsed = parseFrontmatter(normalizeMarkdown(markdown)); frontmatterData = parsed.data; @@ -590,7 +576,7 @@ export class MarkdownDef extends FileDef { // the whole file, but capture the failure so it surfaces via indexing // diagnostics (CS-11548) instead of silently dropping whatever the // frontmatter declared (e.g. a skill's commands). Routed out-of-band via - // `frontmatterParseErrorSymbol`, picked up by the host file extractor. + // `FRONTMATTER_PARSE_ERROR_SYMBOL`, picked up by the host file extractor. frontmatterParseError = toFrontmatterParseError(err); console.warn( `[markdown-file-def] frontmatter parse failed for ${url}:`, @@ -657,7 +643,7 @@ export class MarkdownDef extends FileDef { if (frontmatterParseError) { (attributes as Record)[ - frontmatterParseErrorSymbol + FRONTMATTER_PARSE_ERROR_SYMBOL ] = frontmatterParseError; } diff --git a/packages/host/app/utils/file-def-attributes-extractor.ts b/packages/host/app/utils/file-def-attributes-extractor.ts index 7b66f42c84..fc54ef07ae 100644 --- a/packages/host/app/utils/file-def-attributes-extractor.ts +++ b/packages/host/app/utils/file-def-attributes-extractor.ts @@ -3,6 +3,7 @@ import { isEqual } from 'lodash'; import { baseRef, CardError, + FRONTMATTER_PARSE_ERROR_SYMBOL, identifyCard, inferContentType, internalKeyFor, @@ -221,14 +222,11 @@ export class FileDefAttributesExtractor { // Same out-of-band lift for a frontmatter parse failure: keep it off // the flat `search_doc` and hand it back so the indexer can persist it // onto `diagnostics.frontmatterParseError`. - let frontmatterParseErrorSymbol = Symbol.for( - 'boxel:file-frontmatter-parse-error', - ); - let frontmatterParseError = cleanedBag[frontmatterParseErrorSymbol] as - | FrontmatterParseError - | undefined; + let frontmatterParseError = cleanedBag[ + FRONTMATTER_PARSE_ERROR_SYMBOL + ] as FrontmatterParseError | undefined; if (frontmatterParseError) { - delete cleanedBag[frontmatterParseErrorSymbol]; + delete cleanedBag[FRONTMATTER_PARSE_ERROR_SYMBOL]; } return { status: 'ready', diff --git a/packages/host/tests/integration/components/markdown-skill-frontmatter-test.gts b/packages/host/tests/integration/components/markdown-skill-frontmatter-test.gts index c4fdaea06d..3dec359e03 100644 --- a/packages/host/tests/integration/components/markdown-skill-frontmatter-test.gts +++ b/packages/host/tests/integration/components/markdown-skill-frontmatter-test.gts @@ -15,7 +15,11 @@ import { getService } from '@universal-ember/test-support'; import { module, test } from 'qunit'; -import { baseRealm, identifyCard } from '@cardstack/runtime-common'; +import { + baseRealm, + FRONTMATTER_PARSE_ERROR_SYMBOL, + identifyCard, +} from '@cardstack/runtime-common'; import type { Loader } from '@cardstack/runtime-common/loader'; import { buildFileResource } from '@cardstack/host/utils/file-def-attributes-extractor'; @@ -26,9 +30,6 @@ import { setupMockMatrix } from '../../helpers/mock-matrix'; import { setupRenderingTest } from '../../helpers/setup'; const FILE_FIELD_META = Symbol.for('boxel:file-field-meta'); -const FRONTMATTER_PARSE_ERROR = Symbol.for( - 'boxel:file-frontmatter-parse-error', -); let loader: Loader; @@ -232,7 +233,9 @@ Body paragraph. {}, ); - let routed = (attrs as Record)[FRONTMATTER_PARSE_ERROR]; + let routed = (attrs as Record)[ + FRONTMATTER_PARSE_ERROR_SYMBOL + ]; assert.strictEqual( typeof routed?.message, 'string', diff --git a/packages/realm-server/tests/realm-endpoints/indexing-errors-test.ts b/packages/realm-server/tests/realm-endpoints/indexing-errors-test.ts index ef1a9fa226..973042ab76 100644 --- a/packages/realm-server/tests/realm-endpoints/indexing-errors-test.ts +++ b/packages/realm-server/tests/realm-endpoints/indexing-errors-test.ts @@ -277,6 +277,54 @@ module(`realm-endpoints/${basename(__filename)}`, function () { ); }); + test('surfaces rows with has_error TRUE even when error_doc is NULL', async function (assert) { + // The query selects on `has_error = TRUE`, so a row with that flag set + // must surface even if its `error_doc` column was never populated. This + // locks in that the discriminator follows `has_error`, not `error_doc`. + await sourceRealm.realmIndexUpdater.fullIndex(); + + let cardURL = `${sourceRealm.url}broken-instance.json`; + await dbAdapter.execute( + `UPDATE boxel_index + SET has_error = TRUE, + error_doc = NULL, + diagnostics = NULL + WHERE url = $1 AND realm_url = $2 AND type = 'instance'`, + { bind: [cardURL, sourceRealm.url] }, + ); + + let response = await request + .get(`${new URL(sourceRealm.url).pathname}_indexing-errors`) + .set('Accept', SupportedMimeType.JSONAPI) + .set( + 'Authorization', + `Bearer ${createJWT(sourceRealm, ownerUserId, DEFAULT_PERMISSIONS)}`, + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + let entries = ( + response.body.data as Array<{ + type: string; + id: string; + attributes: { url: string; entryType: string; errorDoc: unknown }; + }> + ).filter( + (e) => + e.attributes.url === cardURL && e.attributes.entryType === 'instance', + ); + assert.strictEqual( + entries.length, + 1, + 'has_error=TRUE row is surfaced even with a NULL error_doc', + ); + assert.strictEqual(entries[0].type, 'indexing-error', 'discriminator'); + assert.strictEqual( + entries[0].attributes.errorDoc, + null, + 'errorDoc passes through as null', + ); + }); + test('surfaces broken-link rows even when has_error is FALSE', async function (assert) { await sourceRealm.realmIndexUpdater.fullIndex(); diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index 9a55a01db0..01530f25da 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -76,6 +76,15 @@ export interface FrontmatterParseError { column?: number; } +// Global symbol channel used by file-def `extractAttributes` implementations +// to route a `FrontmatterParseError` back to the host file extractor without +// it leaking into the flat `search_doc`. Producer and consumer must agree on +// the exact string key — exported here so callers share one source of truth +// and a typo can't silently break the handoff. +export const FRONTMATTER_PARSE_ERROR_SYMBOL = Symbol.for( + 'boxel:file-frontmatter-parse-error', +); + // Per-render computed-field counters captured by the host's render.meta // route. Emitted alongside PrerenderMeta so the Prerenderer can lift them // onto `response.meta.diagnostics` and the indexer can persist them onto diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index e160979ca0..3eea9ae9df 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -6231,7 +6231,7 @@ export class Realm { let sourceRealmURL = ensureTrailingSlash(this.url); let rows = (await query(this.#dbAdapter, [ - `SELECT url, type, error_doc, diagnostics FROM boxel_index WHERE realm_url =`, + `SELECT url, type, has_error, error_doc, diagnostics FROM boxel_index WHERE realm_url =`, param(sourceRealmURL), `AND (is_deleted IS NULL OR is_deleted = FALSE)`, `AND (`, @@ -6246,6 +6246,7 @@ export class Realm { ])) as { url: string; type: string; + has_error: boolean | null; error_doc: SerializedError | null; diagnostics: Record | null; }[]; @@ -6262,7 +6263,11 @@ export class Realm { row.diagnostics.frontmatterParseError !== null ? (row.diagnostics.frontmatterParseError as Record) : null; - let hasError = row.error_doc != null; + // Source of truth is the row's `has_error` column — the SQL above + // filters on it, so we mirror that filter when branching. Using + // `row.error_doc != null` here would silently drop any row where + // `has_error = TRUE` but `error_doc` is NULL. + let hasError = row.has_error === true; // A single boxel_index row can carry more than one independent // finding — e.g. a markdown skill with both unparseable frontmatter // and a broken card reference in its body. We emit one resource per