From e3907c5699aebce7513c4f1c381a1f98b0440743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 29 Jun 2026 18:33:18 +0200 Subject: [PATCH 1/6] feat: add replay test reporters --- src/__tests__/cli-network.test.ts | 20 ++ src/cli-test-reporters/default.ts | 112 ++++++++++ src/cli-test-reporters/format.ts | 137 ++++++++++++ src/cli-test-reporters/junit.ts | 121 ++++++++++ src/cli-test-reporters/registry.ts | 79 +++++++ src/cli-test-reporters/types.ts | 13 ++ src/cli-test.ts | 339 ++--------------------------- src/cli/commands/generic.ts | 5 +- src/client-normalizers.ts | 1 - src/client-types.ts | 2 +- src/commands/replay/index.test.ts | 10 +- src/commands/replay/index.ts | 12 +- src/utils/__tests__/args.test.ts | 8 + src/utils/cli-flags.ts | 12 +- 14 files changed, 536 insertions(+), 335 deletions(-) create mode 100644 src/cli-test-reporters/default.ts create mode 100644 src/cli-test-reporters/format.ts create mode 100644 src/cli-test-reporters/junit.ts create mode 100644 src/cli-test-reporters/registry.ts create mode 100644 src/cli-test-reporters/types.ts diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index 76b66f654..929a4a63e 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -664,6 +664,7 @@ test('test command writes JUnit report with failure metadata', async () => { ); assert.equal(result.code, 1); + assert.equal(result.calls[0]?.flags?.reportJunit, undefined); const xml = await fs.readFile(reportPath, 'utf8'); assert.match( xml, @@ -689,3 +690,22 @@ test('test command writes JUnit report with failure metadata', async () => { await fs.rm(tmpDir, { recursive: true, force: true }); } }); + +test('test command supports explicit reporter lists', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-reporter-test-')); + const reportPath = path.join(tmpDir, 'replays.junit.xml'); + + try { + const result = await runCliCapture( + ['test', './suite', '--reporter', `junit:${reportPath}`], + async () => makeReplaySuiteResponse(), + ); + + assert.equal(result.code, 1); + assert.doesNotMatch(result.stdout, /Test summary:/); + const xml = await fs.readFile(reportPath, 'utf8'); + assert.match(xml, / renderReplayTestSummary(suite, { debug: context.debug }), + getExitCode: getReplayTestExitCode, + }; +} + +function renderReplayTestSummary(data: ReplaySuiteResult, options: { debug?: boolean } = {}): void { + const flaky = data.tests.filter(isFlakyReplayTestResult); + process.stdout.write(`${formatReplayTestSummaryLine(data, flaky.length)}\n`); + renderFailureDetails(data.tests.filter(isFailedReplayTestResult), { debug: options.debug }); + renderFlakyTestSummary(flaky); +} + +function formatReplayTestSummaryLine(data: ReplaySuiteResult, flakyCount: number): string { + const durationMs = typeof data.durationMs === 'number' ? data.durationMs : undefined; + const flakySuffix = flakyCount > 0 ? `, ${flakyCount} flaky` : ''; + const durationSuffix = durationMs !== undefined ? ` in ${formatDurationSeconds(durationMs)}` : ''; + return `Test summary: ${data.passed} passed, ${data.failed} failed${flakySuffix}${durationSuffix}`; +} + +function replayFlakyStatusIcon(): string { + const useColor = supportsColor(); + return useColor ? colorize('✓', 'yellow') : '✓'; +} + +function replayFailureConsoleLines(result: FailedReplayTestResult): string[] { + return [ + replayErrorHintLine(result.error), + replayArtifactsLine(result, 'artifacts'), + replayErrorLogLine(result.error, 'log'), + replayErrorDiagnosticLine(result.error, 'diagnostic'), + ].filter(isDefinedString); +} + +function renderFlakyTestSummary(results: PassedReplayTestResult[]): void { + if (results.length === 0) return; + process.stdout.write('\n'); + process.stdout.write('Flaky tests:\n'); + for (const result of results) { + process.stdout.write( + ` ${replayFlakyStatusIcon()} ${replayTestDisplayNameWithFile(result)} after ${result.attempts} attempts${formatFlakyReplayDurationSuffix(result)}\n`, + ); + for (const failure of result.attemptFailures ?? []) { + const attemptDuration = + typeof failure.durationMs === 'number' + ? ` (${formatDurationSeconds(failure.durationMs)})` + : ''; + process.stdout.write( + ` attempt ${failure.attempt} failed${attemptDuration}: ${failure.message}\n`, + ); + } + } +} + +function renderFailureDetails( + results: FailedReplayTestResult[], + options: { debug?: boolean } = {}, +): void { + if (results.length === 0) return; + process.stdout.write('\n'); + process.stdout.write('Failures:\n'); + for (const result of results) { + process.stdout.write(` ${replayTestDisplayNameWithFile(result)}\n`); + renderReplayFailureBody(result, { debug: options.debug, indent: ' ' }); + } +} + +function renderReplayFailureBody( + result: FailedReplayTestResult, + options: { debug?: boolean; indent: string }, +): void { + const { debug, indent } = options; + process.stdout.write(`${indent}${result.error?.message ?? 'Unknown test failure'}\n`); + for (const line of replayFailureConsoleLines(result)) { + process.stdout.write(`${indent}${line}\n`); + } + if (!debug) return; + for (const line of replayTestFailureStepLines(result)) { + process.stdout.write(`${indent}${line}\n`); + } +} + +function formatFlakyReplayDurationSuffix(result: PassedReplayTestResult): string { + const timings = [ + typeof result.finalAttemptDurationMs === 'number' + ? `passed attempt ${formatDurationSeconds(result.finalAttemptDurationMs)}` + : '', + result.durationMs > 0 ? `total ${formatDurationSeconds(result.durationMs)}` : '', + ].filter(Boolean); + return timings.length > 0 ? ` (${timings.join(', ')})` : ''; +} diff --git a/src/cli-test-reporters/format.ts b/src/cli-test-reporters/format.ts new file mode 100644 index 000000000..9bf8cfc1a --- /dev/null +++ b/src/cli-test-reporters/format.ts @@ -0,0 +1,137 @@ +import path from 'node:path'; +import type { ReplaySuiteTestResult } from '../daemon/types.ts'; + +export type PassedReplayTestResult = Extract; +export type FailedReplayTestResult = Extract; +export type ReplayTestError = FailedReplayTestResult['error']; + +export function getReplayTestExitCode(data: { failed: number }): number { + return data.failed > 0 ? 1 : 0; +} + +export function isFailedReplayTestResult( + result: ReplaySuiteTestResult, +): result is FailedReplayTestResult { + return result.status === 'failed'; +} + +export function isFlakyReplayTestResult( + result: ReplaySuiteTestResult, +): result is PassedReplayTestResult { + return result.status === 'passed' && result.attempts > 1; +} + +export function replayTestDisplayNameWithFile(result: ReplaySuiteTestResult): string { + const title = replayTestTitle(result); + const filename = path.basename(result.file); + const base = title && title.length > 0 ? `${JSON.stringify(title)} in ${filename}` : filename; + return `${base}${formatReplayTestShardSuffix(result)}`; +} + +export function replayTestCaseName(result: ReplaySuiteTestResult): string { + return `${replayTestTitle(result) ?? path.basename(result.file)}${formatReplayTestShardSuffix(result)}`; +} + +export function replayArtifactsLine( + result: ReplaySuiteTestResult, + label: 'artifacts' | 'artifactsDir', +): string | undefined { + return 'artifactsDir' in result && result.artifactsDir + ? `${label}: ${result.artifactsDir}` + : undefined; +} + +export function replayErrorHintLine(error: ReplayTestError): string | undefined { + return error.hint ? `hint: ${error.hint}` : undefined; +} + +export function replayErrorDiagnosticLine( + error: ReplayTestError, + label: 'diagnostic' | 'diagnosticId', +): string | undefined { + return error.diagnosticId ? `${label}: ${error.diagnosticId}` : undefined; +} + +export function replayErrorLogLine( + error: ReplayTestError, + label: 'log' | 'logPath', +): string | undefined { + return error.logPath ? `${label}: ${error.logPath}` : undefined; +} + +export function appendReplayErrorMetadata( + lines: string[], + error: ReplayTestError, + options: { includeMessage?: boolean; includeDetails?: boolean; detailsIndent?: number } = {}, +): void { + if (options.includeMessage) lines.push(`errorMessage: ${error.message}`); + appendOptionalLine(lines, replayErrorHintLine(error)); + appendOptionalLine(lines, replayErrorDiagnosticLine(error, 'diagnosticId')); + appendOptionalLine(lines, replayErrorLogLine(error, 'logPath')); + if (options.includeDetails !== false) { + appendReplayErrorDetails(lines, error, options.detailsIndent); + } +} + +export function appendReplayErrorDetails( + lines: string[], + error: ReplayTestError, + detailsIndent?: number, +): void { + const details = error.details ? JSON.stringify(error.details, null, detailsIndent) : undefined; + if (details) lines.push(`details: ${details}`); +} + +export function appendReplayTestShardMetadata( + lines: string[], + result: ReplaySuiteTestResult, +): void { + if (!('shardIndex' in result) || typeof result.shardIndex !== 'number') return; + lines.push(`shardIndex: ${result.shardIndex}`); + appendOptionalLine( + lines, + typeof result.shardCount === 'number' ? `shardCount: ${result.shardCount}` : undefined, + ); + appendOptionalLine( + lines, + typeof result.deviceId === 'string' ? `deviceId: ${result.deviceId}` : undefined, + ); +} + +export function replayTestWarningLines(result: ReplaySuiteTestResult): string[] { + if (result.status !== 'passed') return []; + return (result.warnings ?? []).map((warning) => `warning: ${warning}`); +} + +export function appendOptionalLine(lines: string[], line: string | undefined): void { + if (line) lines.push(line); +} + +export function formatJUnitSeconds(durationMs: number): string { + return (Math.max(0, durationMs) / 1000).toFixed(3); +} + +export function xmlEscape(value: string): string { + return value + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + +export function isDefinedString(value: string | undefined): value is string { + return value !== undefined; +} + +function replayTestTitle(result: ReplaySuiteTestResult): string | undefined { + const title = result.title?.trim(); + return title && title.length > 0 ? title : undefined; +} + +export function formatReplayTestShardSuffix(result: ReplaySuiteTestResult): string { + if (!('shardIndex' in result) || typeof result.shardIndex !== 'number') return ''; + const shardCount = typeof result.shardCount === 'number' ? result.shardCount : '?'; + const device = typeof result.deviceId === 'string' ? ` ${result.deviceId}` : ''; + return ` [shard ${result.shardIndex + 1}/${shardCount}${device}]`; +} diff --git a/src/cli-test-reporters/junit.ts b/src/cli-test-reporters/junit.ts new file mode 100644 index 000000000..3340f7d7c --- /dev/null +++ b/src/cli-test-reporters/junit.ts @@ -0,0 +1,121 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import type { ReplaySuiteResult, ReplaySuiteTestResult } from '../daemon/types.ts'; +import { AppError } from '../utils/errors.ts'; +import type { ReplayTestReporter } from './types.ts'; +import { + appendOptionalLine, + appendReplayErrorDetails, + appendReplayErrorMetadata, + appendReplayTestShardMetadata, + formatJUnitSeconds, + formatReplayTestShardSuffix, + getReplayTestExitCode, + isFlakyReplayTestResult, + replayArtifactsLine, + replayTestCaseName, + replayTestWarningLines, + xmlEscape, + type FailedReplayTestResult, +} from './format.ts'; + +export function createJunitReplayTestReporter(reportPath: string): ReplayTestReporter { + return { + name: 'junit', + onSuiteEnd: (suite) => writeReplayJunitReport(reportPath, suite), + getExitCode: getReplayTestExitCode, + }; +} + +function writeReplayJunitReport(reportPath: string, suite: ReplaySuiteResult): void { + const directory = path.dirname(reportPath); + try { + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync(reportPath, buildReplayJunitXml(suite), 'utf8'); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new AppError( + 'COMMAND_FAILED', + `Failed to write JUnit report to ${reportPath}: ${message}`, + ); + } +} + +function buildReplayJunitXml(suite: ReplaySuiteResult): string { + const lines = [ + '', + ``, + ` `, + ]; + + for (const test of suite.tests) { + lines.push(...renderJUnitTestCase(test)); + } + + lines.push(' '); + lines.push(''); + return `${lines.join('\n')}\n`; +} + +function renderJUnitTestCase(test: ReplaySuiteTestResult): string[] { + const name = xmlEscape(replayTestCaseName(test)); + const className = xmlEscape( + `${path.dirname(test.file) === '.' ? test.file : path.dirname(test.file)}${formatReplayTestShardSuffix(test)}`, + ); + const file = xmlEscape(test.file); + const time = formatJUnitSeconds(test.durationMs); + const lines = [ + ` `, + ]; + + if (test.status === 'failed') { + lines.push( + ` ${xmlEscape(buildFailureDetails(test))}`, + ); + } else if (test.status === 'skipped') { + lines.push(` `); + } + + const systemOut = buildSystemOut(test); + if (systemOut) { + lines.push(` ${xmlEscape(systemOut)}`); + } + + lines.push(' '); + return lines; +} + +function buildFailureDetails(test: FailedReplayTestResult): string { + const lines = [test.error.message]; + appendReplayErrorMetadata(lines, test.error, { includeDetails: false }); + appendOptionalLine(lines, replayArtifactsLine(test, 'artifactsDir')); + appendReplayErrorDetails(lines, test.error, 2); + return lines.join('\n'); +} + +function buildSystemOut(test: ReplaySuiteTestResult): string { + const lines = [`status: ${test.status}`, `durationMs: ${test.durationMs}`]; + appendReplaySystemOutMetadata(lines, test); + return lines.join('\n'); +} + +function appendReplaySystemOutMetadata(lines: string[], test: ReplaySuiteTestResult): void { + appendOptionalLine(lines, 'attempts' in test ? `attempts: ${test.attempts}` : undefined); + appendOptionalLine(lines, 'session' in test ? `session: ${test.session}` : undefined); + appendOptionalLine(lines, 'replayed' in test ? `replayed: ${test.replayed}` : undefined); + appendOptionalLine(lines, 'healed' in test ? `healed: ${test.healed}` : undefined); + for (const warning of replayTestWarningLines(test)) { + lines.push(warning); + } + appendOptionalLine(lines, replayArtifactsLine(test, 'artifactsDir')); + appendReplayTestShardMetadata(lines, test); + if (test.status === 'failed') { + appendReplayFailureSystemOut(lines, test); + } + appendOptionalLine(lines, isFlakyReplayTestResult(test) ? 'flaky: true' : undefined); +} + +function appendReplayFailureSystemOut(lines: string[], test: FailedReplayTestResult): void { + lines.push(`errorCode: ${test.error.code}`); + appendReplayErrorMetadata(lines, test.error, { includeMessage: true }); +} diff --git a/src/cli-test-reporters/registry.ts b/src/cli-test-reporters/registry.ts new file mode 100644 index 000000000..786623f29 --- /dev/null +++ b/src/cli-test-reporters/registry.ts @@ -0,0 +1,79 @@ +import type { ReplaySuiteResult } from '../daemon/types.ts'; +import { AppError } from '../utils/errors.ts'; +import { createDefaultReplayTestReporter } from './default.ts'; +import { getReplayTestExitCode } from './format.ts'; +import { createJunitReplayTestReporter } from './junit.ts'; +import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; + +export function resolveReplayTestReporters(options: { + reporters?: string[]; + reportJunit?: string; + json?: boolean; +}): ReplayTestReporter[] { + const specs = + options.reporters && options.reporters.length > 0 + ? [...options.reporters] + : options.json + ? [] + : ['default']; + + if (options.reportJunit) { + specs.push(`junit:${options.reportJunit}`); + } + + return specs.map(resolveReplayTestReporter); +} + +export async function runReplayTestReporters( + reporters: ReplayTestReporter[], + suite: ReplaySuiteResult, + context: ReplayTestReporterContext, +): Promise { + for (const reporter of reporters) { + await reporter.onSuiteEnd?.(suite, context); + } +} + +export function getReplayTestReporterExitCode( + reporters: ReplayTestReporter[], + suite: ReplaySuiteResult, +): number { + for (const reporter of reporters) { + const exitCode = reporter.getExitCode?.(suite); + if (exitCode !== undefined) return exitCode; + } + return getReplayTestExitCode(suite); +} + +function resolveReplayTestReporter(spec: string): ReplayTestReporter { + const { name, value } = splitReplayTestReporterSpec(spec); + if (name === 'default') { + if (value) { + throw new AppError('INVALID_ARGS', 'The default test reporter does not accept options.'); + } + return createDefaultReplayTestReporter(); + } + if (name === 'junit') { + if (!value) { + throw new AppError( + 'INVALID_ARGS', + 'The junit test reporter requires an output path. Use --reporter junit:.', + ); + } + return createJunitReplayTestReporter(value); + } + + throw new AppError( + 'INVALID_ARGS', + `Unknown test reporter "${name}". Built-in reporters: default, junit:.`, + ); +} + +function splitReplayTestReporterSpec(spec: string): { name: string; value?: string } { + const separatorIndex = spec.indexOf(':'); + if (separatorIndex < 0) return { name: spec.trim() }; + return { + name: spec.slice(0, separatorIndex).trim(), + value: spec.slice(separatorIndex + 1), + }; +} diff --git a/src/cli-test-reporters/types.ts b/src/cli-test-reporters/types.ts new file mode 100644 index 000000000..7cff2b542 --- /dev/null +++ b/src/cli-test-reporters/types.ts @@ -0,0 +1,13 @@ +import type { RequestProgressEvent } from '../daemon/request-progress.ts'; +import type { ReplaySuiteResult } from '../daemon/types.ts'; + +export type ReplayTestReporterContext = { + debug?: boolean; +}; + +export type ReplayTestReporter = { + name: string; + onProgress?(event: RequestProgressEvent, context: ReplayTestReporterContext): void; + onSuiteEnd?(suite: ReplaySuiteResult, context: ReplayTestReporterContext): Promise | void; + getExitCode?(suite: ReplaySuiteResult): number | undefined; +}; diff --git a/src/cli-test.ts b/src/cli-test.ts index c5eaf39d1..e64d23dcc 100644 --- a/src/cli-test.ts +++ b/src/cli-test.ts @@ -1,332 +1,23 @@ -import fs from 'node:fs'; -import path from 'node:path'; -import type { ReplaySuiteResult, ReplaySuiteTestResult } from './daemon/types.ts'; -import { replayTestFailureStepLines } from './cli-test-trace.ts'; -import { formatDurationSeconds } from './utils/duration-format.ts'; -import { AppError } from './utils/errors.ts'; -import { colorize, printJson, supportsColor } from './utils/output.ts'; - -type PassedReplayTestResult = Extract; -type FailedReplayTestResult = Extract; -type ReplayTestError = FailedReplayTestResult['error']; - -export function renderReplayTestResponse(options: { +import type { ReplaySuiteResult } from './daemon/types.ts'; +import { + getReplayTestReporterExitCode, + resolveReplayTestReporters, + runReplayTestReporters, +} from './cli-test-reporters/registry.ts'; +import { printJson } from './utils/output.ts'; + +export async function renderReplayTestResponse(options: { suite: ReplaySuiteResult; json?: boolean; debug?: boolean; + reporter?: string[]; reportJunit?: string; -}): number { - const { suite, json, debug, reportJunit } = options; - if (reportJunit) { - writeReplayJunitReport(reportJunit, suite); - } +}): Promise { + const { suite, json, debug, reporter, reportJunit } = options; + const reporters = resolveReplayTestReporters({ reporters: reporter, reportJunit, json }); + await runReplayTestReporters(reporters, suite, { debug }); if (json) { printJson({ success: true, data: suite }); - return getReplayTestExitCode(suite); - } - return renderReplayTestSummary(suite, { debug }); -} - -function renderReplayTestSummary( - data: ReplaySuiteResult, - options: { debug?: boolean } = {}, -): number { - const flaky = data.tests.filter(isFlakyReplayTestResult); - process.stdout.write(`${formatReplayTestSummaryLine(data, flaky.length)}\n`); - renderFailureDetails(data.tests.filter(isFailedReplayTestResult), { debug: options.debug }); - renderFlakyTestSummary(flaky); - return getReplayTestExitCode(data); -} - -function formatReplayTestSummaryLine(data: ReplaySuiteResult, flakyCount: number): string { - const durationMs = typeof data.durationMs === 'number' ? data.durationMs : undefined; - const flakySuffix = flakyCount > 0 ? `, ${flakyCount} flaky` : ''; - const durationSuffix = durationMs !== undefined ? ` in ${formatDurationSeconds(durationMs)}` : ''; - return `Test summary: ${data.passed} passed, ${data.failed} failed${flakySuffix}${durationSuffix}`; -} - -function replayFlakyStatusIcon(): string { - const useColor = supportsColor(); - return useColor ? colorize('✓', 'yellow') : '✓'; -} - -function isFailedReplayTestResult(result: ReplaySuiteTestResult): result is FailedReplayTestResult { - return result.status === 'failed'; -} - -function replayFailureConsoleLines(result: FailedReplayTestResult): string[] { - return [ - replayErrorHintLine(result.error), - replayArtifactsLine(result, 'artifacts'), - replayErrorLogLine(result.error, 'log'), - replayErrorDiagnosticLine(result.error, 'diagnostic'), - ].filter(isDefinedString); -} - -function isFlakyReplayTestResult(result: ReplaySuiteTestResult): result is PassedReplayTestResult { - return result.status === 'passed' && result.attempts > 1; -} - -function renderFlakyTestSummary(results: PassedReplayTestResult[]): void { - if (results.length === 0) return; - process.stdout.write('\n'); - process.stdout.write('Flaky tests:\n'); - for (const result of results) { - process.stdout.write( - ` ${replayFlakyStatusIcon()} ${replayTestDisplayNameWithFile(result)} after ${result.attempts} attempts${formatFlakyReplayDurationSuffix(result)}\n`, - ); - for (const failure of result.attemptFailures ?? []) { - const attemptDuration = - typeof failure.durationMs === 'number' - ? ` (${formatDurationSeconds(failure.durationMs)})` - : ''; - process.stdout.write( - ` attempt ${failure.attempt} failed${attemptDuration}: ${failure.message}\n`, - ); - } - } -} - -function renderFailureDetails( - results: FailedReplayTestResult[], - options: { debug?: boolean } = {}, -): void { - if (results.length === 0) return; - process.stdout.write('\n'); - process.stdout.write('Failures:\n'); - for (const result of results) { - process.stdout.write(` ${replayTestDisplayNameWithFile(result)}\n`); - renderReplayFailureBody(result, { debug: options.debug, indent: ' ' }); - } -} - -function renderReplayFailureBody( - result: FailedReplayTestResult, - options: { debug?: boolean; indent: string }, -): void { - const { debug, indent } = options; - process.stdout.write(`${indent}${result.error?.message ?? 'Unknown test failure'}\n`); - for (const line of replayFailureConsoleLines(result)) { - process.stdout.write(`${indent}${line}\n`); - } - if (!debug) return; - for (const line of replayTestFailureStepLines(result)) { - process.stdout.write(`${indent}${line}\n`); - } -} - -function replayTestDisplayNameWithFile(result: ReplaySuiteTestResult): string { - const title = replayTestTitle(result); - const filename = path.basename(result.file); - const base = title && title.length > 0 ? `${JSON.stringify(title)} in ${filename}` : filename; - return `${base}${formatReplayTestShardSuffix(result)}`; -} - -function replayTestCaseName(result: ReplaySuiteTestResult): string { - return `${replayTestTitle(result) ?? path.basename(result.file)}${formatReplayTestShardSuffix(result)}`; -} - -function replayTestTitle(result: ReplaySuiteTestResult): string | undefined { - const title = result.title?.trim(); - return title && title.length > 0 ? title : undefined; -} - -function formatFlakyReplayDurationSuffix(result: PassedReplayTestResult): string { - const timings = [ - typeof result.finalAttemptDurationMs === 'number' - ? `passed attempt ${formatDurationSeconds(result.finalAttemptDurationMs)}` - : '', - result.durationMs > 0 ? `total ${formatDurationSeconds(result.durationMs)}` : '', - ].filter(Boolean); - return timings.length > 0 ? ` (${timings.join(', ')})` : ''; -} - -function getReplayTestExitCode(data: ReplaySuiteResult): number { - return data.failed > 0 ? 1 : 0; -} - -function writeReplayJunitReport(reportPath: string, suite: ReplaySuiteResult): void { - const directory = path.dirname(reportPath); - try { - fs.mkdirSync(directory, { recursive: true }); - fs.writeFileSync(reportPath, buildReplayJunitXml(suite), 'utf8'); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new AppError( - 'COMMAND_FAILED', - `Failed to write JUnit report to ${reportPath}: ${message}`, - ); } -} - -function buildReplayJunitXml(suite: ReplaySuiteResult): string { - const lines = [ - '', - ``, - ` `, - ]; - - for (const test of suite.tests) { - lines.push(...renderJUnitTestCase(test)); - } - - lines.push(' '); - lines.push(''); - return `${lines.join('\n')}\n`; -} - -function renderJUnitTestCase(test: ReplaySuiteTestResult): string[] { - const name = xmlEscape(replayTestCaseName(test)); - const className = xmlEscape( - `${path.dirname(test.file) === '.' ? test.file : path.dirname(test.file)}${formatReplayTestShardSuffix(test)}`, - ); - const file = xmlEscape(test.file); - const time = formatJUnitSeconds(test.durationMs); - const lines = [ - ` `, - ]; - - if (test.status === 'failed') { - lines.push( - ` ${xmlEscape(buildFailureDetails(test))}`, - ); - } else if (test.status === 'skipped') { - lines.push(` `); - } - - const systemOut = buildSystemOut(test); - if (systemOut) { - lines.push(` ${xmlEscape(systemOut)}`); - } - - lines.push(' '); - return lines; -} - -function buildFailureDetails(test: FailedReplayTestResult): string { - const lines = [test.error.message]; - appendReplayErrorMetadata(lines, test.error, { includeDetails: false }); - appendOptionalLine(lines, replayArtifactsLine(test, 'artifactsDir')); - appendReplayErrorDetails(lines, test.error, 2); - return lines.join('\n'); -} - -function buildSystemOut(test: ReplaySuiteTestResult): string { - const lines = [`status: ${test.status}`, `durationMs: ${test.durationMs}`]; - appendReplaySystemOutMetadata(lines, test); - return lines.join('\n'); -} - -function appendReplaySystemOutMetadata(lines: string[], test: ReplaySuiteTestResult): void { - appendOptionalLine(lines, 'attempts' in test ? `attempts: ${test.attempts}` : undefined); - appendOptionalLine(lines, 'session' in test ? `session: ${test.session}` : undefined); - appendOptionalLine(lines, 'replayed' in test ? `replayed: ${test.replayed}` : undefined); - appendOptionalLine(lines, 'healed' in test ? `healed: ${test.healed}` : undefined); - for (const warning of replayTestWarningLines(test)) { - lines.push(warning); - } - appendOptionalLine(lines, replayArtifactsLine(test, 'artifactsDir')); - appendReplayTestShardMetadata(lines, test); - if (test.status === 'failed') { - appendReplayFailureSystemOut(lines, test); - } - appendOptionalLine(lines, isFlakyReplayTestResult(test) ? 'flaky: true' : undefined); -} - -function formatReplayTestShardSuffix(result: ReplaySuiteTestResult): string { - if (!('shardIndex' in result) || typeof result.shardIndex !== 'number') return ''; - const shardCount = typeof result.shardCount === 'number' ? result.shardCount : '?'; - const device = typeof result.deviceId === 'string' ? ` ${result.deviceId}` : ''; - return ` [shard ${result.shardIndex + 1}/${shardCount}${device}]`; -} - -function appendReplayTestShardMetadata(lines: string[], result: ReplaySuiteTestResult): void { - if (!('shardIndex' in result) || typeof result.shardIndex !== 'number') return; - lines.push(`shardIndex: ${result.shardIndex}`); - appendOptionalLine( - lines, - typeof result.shardCount === 'number' ? `shardCount: ${result.shardCount}` : undefined, - ); - appendOptionalLine( - lines, - typeof result.deviceId === 'string' ? `deviceId: ${result.deviceId}` : undefined, - ); -} - -function appendReplayFailureSystemOut(lines: string[], test: FailedReplayTestResult): void { - lines.push(`errorCode: ${test.error.code}`); - appendReplayErrorMetadata(lines, test.error, { includeMessage: true }); -} - -function appendReplayErrorMetadata( - lines: string[], - error: ReplayTestError, - options: { includeMessage?: boolean; includeDetails?: boolean; detailsIndent?: number } = {}, -): void { - if (options.includeMessage) lines.push(`errorMessage: ${error.message}`); - appendOptionalLine(lines, replayErrorHintLine(error)); - appendOptionalLine(lines, replayErrorDiagnosticLine(error, 'diagnosticId')); - appendOptionalLine(lines, replayErrorLogLine(error, 'logPath')); - if (options.includeDetails !== false) { - appendReplayErrorDetails(lines, error, options.detailsIndent); - } -} - -function appendReplayErrorDetails( - lines: string[], - error: ReplayTestError, - detailsIndent?: number, -): void { - const details = error.details ? JSON.stringify(error.details, null, detailsIndent) : undefined; - if (details) lines.push(`details: ${details}`); -} - -function replayArtifactsLine( - result: ReplaySuiteTestResult, - label: 'artifacts' | 'artifactsDir', -): string | undefined { - return 'artifactsDir' in result && result.artifactsDir - ? `${label}: ${result.artifactsDir}` - : undefined; -} - -function replayErrorHintLine(error: ReplayTestError): string | undefined { - return error.hint ? `hint: ${error.hint}` : undefined; -} - -function replayErrorDiagnosticLine( - error: ReplayTestError, - label: 'diagnostic' | 'diagnosticId', -): string | undefined { - return error.diagnosticId ? `${label}: ${error.diagnosticId}` : undefined; -} - -function replayErrorLogLine(error: ReplayTestError, label: 'log' | 'logPath'): string | undefined { - return error.logPath ? `${label}: ${error.logPath}` : undefined; -} - -function appendOptionalLine(lines: string[], line: string | undefined): void { - if (line) lines.push(line); -} - -function isDefinedString(value: string | undefined): value is string { - return value !== undefined; -} - -function replayTestWarningLines(result: ReplaySuiteTestResult): string[] { - if (result.status !== 'passed') return []; - return (result.warnings ?? []).map((warning) => `warning: ${warning}`); -} - -function formatJUnitSeconds(durationMs: number): string { - return (Math.max(0, durationMs) / 1000).toFixed(3); -} - -function xmlEscape(value: string): string { - return value - .replaceAll('&', '&') - .replaceAll('<', '<') - .replaceAll('>', '>') - .replaceAll('"', '"') - .replaceAll("'", '''); + return getReplayTestReporterExitCode(reporters, suite); } diff --git a/src/cli/commands/generic.ts b/src/cli/commands/generic.ts index c1c573896..82665dd1d 100644 --- a/src/cli/commands/generic.ts +++ b/src/cli/commands/generic.ts @@ -26,7 +26,7 @@ export async function runGenericClientBackedCommand({ if (cliOutput) { writeCliOutput(flags, cliOutput); } else { - const exitCode = writeGenericCliOutput(command, flags, result, { debug }); + const exitCode = await writeGenericCliOutput(command, flags, result, { debug }); if (exitCode !== 0) { process.exit(exitCode); } @@ -39,12 +39,13 @@ function writeGenericCliOutput( flags: CliFlags, data: CommandRequestResult, options: { debug?: boolean } = {}, -): number { +): Promise | number { if (command === 'test') { return renderReplayTestResponse({ suite: data as ReplaySuiteResult, debug: options.debug, json: flags.json, + reporter: flags.reporter, reportJunit: flags.reportJunit, }); } diff --git a/src/client-normalizers.ts b/src/client-normalizers.ts index b9254e971..2751a0455 100644 --- a/src/client-normalizers.ts +++ b/src/client-normalizers.ts @@ -332,7 +332,6 @@ export function buildFlags(options: InternalRequestOptions): CommandFlags { retries: options.retries, recordVideo: options.recordVideo, artifactsDir: options.artifactsDir, - reportJunit: options.reportJunit, shardAll: options.shardAll, shardSplit: options.shardSplit, findFirst: options.findFirst, diff --git a/src/client-types.ts b/src/client-types.ts index 051b66d9b..b4df07af7 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -754,6 +754,7 @@ export type ReplayTestOptions = AgentDeviceRequestOverrides & retries?: number; recordVideo?: boolean; artifactsDir?: string; + /** @deprecated Use the CLI --reporter junit: or --report-junit . */ reportJunit?: string; shardAll?: number; shardSplit?: number; @@ -903,7 +904,6 @@ type CommandExecutionOptions = Partial & { retries?: number; recordVideo?: boolean; artifactsDir?: string; - reportJunit?: string; shardAll?: number; shardSplit?: number; findFirst?: boolean; diff --git a/src/commands/replay/index.test.ts b/src/commands/replay/index.test.ts index bda3c1fda..92b98d9f2 100644 --- a/src/commands/replay/index.test.ts +++ b/src/commands/replay/index.test.ts @@ -77,7 +77,6 @@ describe('replay command interface', () => { retries: 2, recordVideo: true, artifactsDir: './artifacts', - reportJunit: './junit.xml', shardAll: 4, shardSplit: 2, }), @@ -92,7 +91,6 @@ describe('replay command interface', () => { retries: 2, recordVideo: true, artifactsDir: './artifacts', - reportJunit: './junit.xml', shardAll: 4, shardSplit: 2, }); @@ -118,12 +116,18 @@ describe('replay command interface', () => { }, }); - expect(testDaemonWriter({ paths: ['./suite.ad'], maestro: true })).toMatchObject({ + const testRequest = testDaemonWriter({ + paths: ['./suite.ad'], + maestro: true, + reportJunit: './junit.xml', + }); + expect(testRequest).toMatchObject({ command: 'test', positionals: ['./suite.ad'], options: { replayBackend: 'maestro', }, }); + expect(testRequest.options).not.toHaveProperty('reportJunit'); }); }); diff --git a/src/commands/replay/index.ts b/src/commands/replay/index.ts index 09569ba95..bafdd4f0e 100644 --- a/src/commands/replay/index.ts +++ b/src/commands/replay/index.ts @@ -52,7 +52,6 @@ export const testCommandMetadata = defineFieldCommandMetadata( retries: integerField(), recordVideo: booleanField(), artifactsDir: stringField(), - reportJunit: stringField(), shardAll: integerField(), shardSplit: integerField(), }, @@ -92,6 +91,7 @@ const testCliSchema = { 'retries', 'recordVideo', 'artifactsDir', + 'reporter', 'reportJunit', 'shardAll', 'shardSplit', @@ -117,7 +117,6 @@ export const testCliReader: CliReader = (positionals, flags) => ({ retries: flags.retries, recordVideo: flags.recordVideo, artifactsDir: flags.artifactsDir, - reportJunit: flags.reportJunit, shardAll: flags.shardAll, shardSplit: flags.shardSplit, }); @@ -133,7 +132,7 @@ export const replayDaemonWriter: DaemonWriter = (input) => export const testDaemonWriter: DaemonWriter = (input) => request(TEST_COMMAND_NAME, input.paths ?? [], { - ...input, + ...stripReplayTestPresentationInput(input), replayUpdate: input.update, replayBackend: readReplayBackend(input), replayEnv: input.env, @@ -167,6 +166,13 @@ function readReplayBackend(input: CommandInput): string | undefined { return input.backend ?? (input.maestro === true ? 'maestro' : undefined); } +function stripReplayTestPresentationInput(input: CommandInput): CommandInput { + const daemonInput = { ...input }; + delete daemonInput.reporter; + delete daemonInput.reportJunit; + return daemonInput; +} + function collectReplayClientShellEnv(env: NodeJS.ProcessEnv): Record { const result: Record = {}; for (const [key, value] of Object.entries(env)) { diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index f75b36c61..fea7083d8 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -147,6 +147,10 @@ test('parseArgs recognizes command-specific flag combinations', async () => { '2', '--artifacts-dir', '.agent-device/test-artifacts', + '--reporter', + 'default', + '--reporter', + 'junit:.agent-device/test-artifacts/junit.xml', ], strictFlags: true, assertParsed: (parsed) => { @@ -158,6 +162,10 @@ test('parseArgs recognizes command-specific flag combinations', async () => { assert.equal(parsed.flags.timeoutMs, 60000); assert.equal(parsed.flags.retries, 2); assert.equal(parsed.flags.artifactsDir, '.agent-device/test-artifacts'); + assert.deepEqual(parsed.flags.reporter, [ + 'default', + 'junit:.agent-device/test-artifacts/junit.xml', + ]); }, }, { diff --git a/src/utils/cli-flags.ts b/src/utils/cli-flags.ts index a1389b74e..b161e87ad 100644 --- a/src/utils/cli-flags.ts +++ b/src/utils/cli-flags.ts @@ -119,6 +119,7 @@ export type CliFlags = RemoteConfigMetroOptions & retries?: number; recordVideo?: boolean; artifactsDir?: string; + reporter?: string[]; reportJunit?: string; shardAll?: number; shardSplit?: number; @@ -942,12 +943,21 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageLabel: '--artifacts-dir ', usageDescription: 'Test: root directory for suite artifacts', }, + { + key: 'reporter', + names: ['--reporter'], + type: 'string', + multiple: true, + usageLabel: '--reporter ', + usageDescription: + 'Test: add a replay suite reporter; built-ins are default and junit: (repeatable)', + }, { key: 'reportJunit', names: ['--report-junit'], type: 'string', usageLabel: '--report-junit ', - usageDescription: 'Test: write a JUnit XML report for the replay suite', + usageDescription: 'Test: compatibility alias for --reporter junit:', }, { key: 'shardAll', From 7acf07a52e8bca30de6c7e4386c01d742bb8720a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 29 Jun 2026 18:40:45 +0200 Subject: [PATCH 2/6] feat: support custom replay test reporters --- src/__tests__/cli-network.test.ts | 39 +++++++++ src/cli-test-reporters/custom.ts | 133 +++++++++++++++++++++++++++++ src/cli-test-reporters/registry.ts | 15 ++-- src/cli-test-reporters/types.ts | 10 +++ src/cli-test.ts | 2 +- src/index.ts | 7 ++ src/utils/__tests__/args.test.ts | 4 + src/utils/cli-flags.ts | 2 +- website/docs/docs/replay-e2e.md | 59 +++++++++++++ 9 files changed, 264 insertions(+), 7 deletions(-) create mode 100644 src/cli-test-reporters/custom.ts diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index 929a4a63e..a5a670a17 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -709,3 +709,42 @@ test('test command supports explicit reporter lists', async () => { await fs.rm(tmpDir, { recursive: true, force: true }); } }); + +test('test command loads custom reporter modules with JSON options', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-custom-reporter-test-')); + const reporterPath = path.join(tmpDir, 'custom-reporter.mjs'); + const outputPath = path.join(tmpDir, 'custom-report.json'); + + try { + await fs.writeFile( + reporterPath, + [ + "import fs from 'node:fs';", + 'export default function createReporter(options) {', + ' return {', + " name: 'custom-json',", + ' onSuiteEnd(suite) {', + ' fs.writeFileSync(options.output, JSON.stringify({ total: suite.total, failed: suite.failed }), "utf8");', + ' },', + ' getExitCode() { return 0; },', + ' };', + '}', + ].join('\n'), + 'utf8', + ); + + const reporterSpec = `${reporterPath}:${JSON.stringify({ output: outputPath })}`; + const result = await runCliCapture(['test', './suite', '--reporter', reporterSpec], async () => + makeReplaySuiteResponse(), + ); + + assert.equal(result.code, null); + assert.doesNotMatch(result.stdout, /Test summary:/); + assert.deepEqual(JSON.parse(await fs.readFile(outputPath, 'utf8')), { + total: 3, + failed: 1, + }); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } +}); diff --git a/src/cli-test-reporters/custom.ts b/src/cli-test-reporters/custom.ts new file mode 100644 index 000000000..9dd91712f --- /dev/null +++ b/src/cli-test-reporters/custom.ts @@ -0,0 +1,133 @@ +import os from 'node:os'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { AppError } from '../utils/errors.ts'; +import type { + ReplayTestReporter, + ReplayTestReporterFactory, + ReplayTestReporterLoadContext, +} from './types.ts'; + +type CustomReporterModule = { + default?: unknown; + createReporter?: unknown; + reporter?: unknown; +}; + +type CustomReporterSpec = { + modulePath: string; + options: unknown; +}; + +export function isCustomReplayTestReporterSpec(spec: string): boolean { + const modulePath = readCustomReporterModulePath(spec); + return ( + modulePath.startsWith('.') || + modulePath.startsWith('/') || + modulePath.startsWith('~') || + modulePath.startsWith('file:') + ); +} + +export async function createCustomReplayTestReporter(spec: string): Promise { + const customSpec = splitCustomReplayTestReporterSpec(spec); + const modulePath = resolveCustomReporterModulePath(customSpec.modulePath); + const module = await importCustomReporterModule(modulePath); + const exported = module.createReporter ?? module.default ?? module.reporter; + if (!exported) { + throw new AppError( + 'INVALID_ARGS', + `Custom test reporter ${customSpec.modulePath} must export default, createReporter, or reporter.`, + ); + } + + const reporter = + typeof exported === 'function' + ? await (exported as ReplayTestReporterFactory)(customSpec.options, { + spec, + modulePath, + } satisfies ReplayTestReporterLoadContext) + : exported; + + return validateCustomReplayTestReporter(reporter, customSpec.modulePath); +} + +function splitCustomReplayTestReporterSpec(spec: string): CustomReporterSpec { + const optionsSeparator = spec.indexOf(':{'); + if (optionsSeparator < 0) return { modulePath: spec.trim(), options: undefined }; + const modulePath = readCustomReporterModulePath(spec); + const rawOptions = spec.slice(optionsSeparator + 1); + if (!modulePath) { + throw new AppError('INVALID_ARGS', 'Custom test reporter path cannot be empty.'); + } + try { + return { modulePath, options: JSON.parse(rawOptions) }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new AppError( + 'INVALID_ARGS', + `Invalid JSON options for custom test reporter ${modulePath}: ${message}`, + ); + } +} + +function readCustomReporterModulePath(spec: string): string { + const optionsSeparator = spec.indexOf(':{'); + return (optionsSeparator < 0 ? spec : spec.slice(0, optionsSeparator)).trim(); +} + +function resolveCustomReporterModulePath(modulePath: string): string { + if (modulePath.startsWith('file:')) return modulePath; + const expandedPath = modulePath.startsWith('~/') + ? path.join(os.homedir(), modulePath.slice(2)) + : modulePath; + return path.resolve(process.cwd(), expandedPath); +} + +async function importCustomReporterModule(modulePath: string): Promise { + try { + const href = modulePath.startsWith('file:') ? modulePath : pathToFileURL(modulePath).href; + return (await import(href)) as CustomReporterModule; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new AppError( + 'INVALID_ARGS', + `Failed to load custom test reporter ${modulePath}: ${message}`, + ); + } +} + +function validateCustomReplayTestReporter( + reporter: unknown, + modulePath: string, +): ReplayTestReporter { + if (!reporter || typeof reporter !== 'object') { + throw new AppError( + 'INVALID_ARGS', + `Custom test reporter ${modulePath} must export a reporter object or factory.`, + ); + } + const candidate = reporter as Partial; + if (typeof candidate.name !== 'string' || candidate.name.trim().length === 0) { + throw new AppError('INVALID_ARGS', `Custom test reporter ${modulePath} must define name.`); + } + if (candidate.onProgress !== undefined && typeof candidate.onProgress !== 'function') { + throw new AppError( + 'INVALID_ARGS', + `Custom test reporter ${modulePath} onProgress must be a function.`, + ); + } + if (candidate.onSuiteEnd !== undefined && typeof candidate.onSuiteEnd !== 'function') { + throw new AppError( + 'INVALID_ARGS', + `Custom test reporter ${modulePath} onSuiteEnd must be a function.`, + ); + } + if (candidate.getExitCode !== undefined && typeof candidate.getExitCode !== 'function') { + throw new AppError( + 'INVALID_ARGS', + `Custom test reporter ${modulePath} getExitCode must be a function.`, + ); + } + return candidate as ReplayTestReporter; +} diff --git a/src/cli-test-reporters/registry.ts b/src/cli-test-reporters/registry.ts index 786623f29..cf67a8d0f 100644 --- a/src/cli-test-reporters/registry.ts +++ b/src/cli-test-reporters/registry.ts @@ -1,15 +1,16 @@ import type { ReplaySuiteResult } from '../daemon/types.ts'; import { AppError } from '../utils/errors.ts'; +import { createCustomReplayTestReporter, isCustomReplayTestReporterSpec } from './custom.ts'; import { createDefaultReplayTestReporter } from './default.ts'; import { getReplayTestExitCode } from './format.ts'; import { createJunitReplayTestReporter } from './junit.ts'; import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; -export function resolveReplayTestReporters(options: { +export async function resolveReplayTestReporters(options: { reporters?: string[]; reportJunit?: string; json?: boolean; -}): ReplayTestReporter[] { +}): Promise { const specs = options.reporters && options.reporters.length > 0 ? [...options.reporters] @@ -21,7 +22,7 @@ export function resolveReplayTestReporters(options: { specs.push(`junit:${options.reportJunit}`); } - return specs.map(resolveReplayTestReporter); + return await Promise.all(specs.map(resolveReplayTestReporter)); } export async function runReplayTestReporters( @@ -45,7 +46,11 @@ export function getReplayTestReporterExitCode( return getReplayTestExitCode(suite); } -function resolveReplayTestReporter(spec: string): ReplayTestReporter { +async function resolveReplayTestReporter(spec: string): Promise { + if (isCustomReplayTestReporterSpec(spec)) { + return await createCustomReplayTestReporter(spec); + } + const { name, value } = splitReplayTestReporterSpec(spec); if (name === 'default') { if (value) { @@ -65,7 +70,7 @@ function resolveReplayTestReporter(spec: string): ReplayTestReporter { throw new AppError( 'INVALID_ARGS', - `Unknown test reporter "${name}". Built-in reporters: default, junit:.`, + `Unknown test reporter "${name}". Built-in reporters: default, junit:. Custom reporters must be file paths.`, ); } diff --git a/src/cli-test-reporters/types.ts b/src/cli-test-reporters/types.ts index 7cff2b542..c670f3822 100644 --- a/src/cli-test-reporters/types.ts +++ b/src/cli-test-reporters/types.ts @@ -5,9 +5,19 @@ export type ReplayTestReporterContext = { debug?: boolean; }; +export type ReplayTestReporterLoadContext = { + spec: string; + modulePath: string; +}; + export type ReplayTestReporter = { name: string; onProgress?(event: RequestProgressEvent, context: ReplayTestReporterContext): void; onSuiteEnd?(suite: ReplaySuiteResult, context: ReplayTestReporterContext): Promise | void; getExitCode?(suite: ReplaySuiteResult): number | undefined; }; + +export type ReplayTestReporterFactory = ( + options: unknown, + context: ReplayTestReporterLoadContext, +) => ReplayTestReporter | Promise; diff --git a/src/cli-test.ts b/src/cli-test.ts index e64d23dcc..59f92081c 100644 --- a/src/cli-test.ts +++ b/src/cli-test.ts @@ -14,7 +14,7 @@ export async function renderReplayTestResponse(options: { reportJunit?: string; }): Promise { const { suite, json, debug, reporter, reportJunit } = options; - const reporters = resolveReplayTestReporters({ reporters: reporter, reportJunit, json }); + const reporters = await resolveReplayTestReporters({ reporters: reporter, reportJunit, json }); await runReplayTestReporters(reporters, suite, { debug }); if (json) { printJson({ success: true, data: suite }); diff --git a/src/index.ts b/src/index.ts index 9152c52f5..79ad42368 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,6 +20,13 @@ export type { export type { AppErrorCode, NormalizedError } from './utils/errors.ts'; +export type { + ReplayTestReporter, + ReplayTestReporterContext, + ReplayTestReporterFactory, + ReplayTestReporterLoadContext, +} from './cli-test-reporters/types.ts'; + export type { CommandResult } from './core/command-descriptor/command-result.ts'; export type { BootCommandResult, ShutdownCommandResult } from './contracts/device.ts'; export type { ViewportCommandResult } from './contracts/viewport.ts'; diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index fea7083d8..f8a745365 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -2071,6 +2071,10 @@ test('command usage describes test suite flags', () => { assert.match(help, /--retries /); assert.match(help, /--record-video/); assert.match(help, /--artifacts-dir /); + assert.match(help, /--reporter /); + assert.match(help, /custom reporters are file paths/); + assert.match(help, /--report-junit /); + assert.match(help, /compatibility alias for --reporter junit:/); assert.doesNotMatch(help, /test --verbose prints per-test step timings without debug logs/); }); diff --git a/src/utils/cli-flags.ts b/src/utils/cli-flags.ts index b161e87ad..9338f7904 100644 --- a/src/utils/cli-flags.ts +++ b/src/utils/cli-flags.ts @@ -950,7 +950,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ multiple: true, usageLabel: '--reporter ', usageDescription: - 'Test: add a replay suite reporter; built-ins are default and junit: (repeatable)', + 'Test: add a replay suite reporter; built-ins are default and junit:; custom reporters are file paths with optional : options (repeatable)', }, { key: 'reportJunit', diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index c9b62802e..16fd9aeb3 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -87,6 +87,7 @@ agent-device test ./workflows agent-device test "./workflows/**/*.ad" --platform android agent-device test ./workflows --timeout 60000 --retries 1 agent-device test ./workflows --artifacts-dir ./tmp/agent-device-artifacts +agent-device test ./workflows --reporter default --reporter junit:./tmp/junit.xml ``` - `test` discovers `.ad` files from files, directories, or globs and runs them serially. @@ -97,8 +98,66 @@ agent-device test ./workflows --artifacts-dir ./tmp/agent-device-artifacts - `replay-timing.ndjson` records attempt, cleanup, and per-step start/stop events with durations. Upload it from CI even for passing runs when comparing local and CI performance. - Timeouts are cooperative: the runner marks the attempt failed at the timeout boundary, then gives the underlying replay a short grace period to stop before session cleanup. - The default text reporter streams one-line `pass`, `fail`, or `skip` progress on stderr as each suite entry finishes or retries. Each line includes current/total suite position and elapsed seconds such as `pass 3/6 ... duration=12.34s`, then the final summary prints failed tests and passed-on-retry flaky tests; use `--verbose` to print every final result. +- `--reporter` is repeatable. Built-ins are `default` for the console summary and `junit:` for JUnit XML. Passing any explicit reporter list replaces the implicit default reporter, so include `--reporter default` when you also want terminal output. `--report-junit ` remains a compatibility alias for `--reporter junit:`. - When `--fail-fast` and retries are both set, the current test still consumes its retries before the suite stops. +### Custom test reporters + +Custom reporters are CLI-only presentation adapters. The daemon still returns the structured replay suite result; reporters run in the local CLI process after the suite finishes. + +```bash +agent-device test ./workflows --reporter ./scripts/replay-reporter.mjs +agent-device test ./workflows --reporter './scripts/replay-reporter.mjs:{"output":"./tmp/report.txt"}' +``` + +Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive parsed JSON options and load context: + +```js +// scripts/replay-reporter.mjs +import fs from 'node:fs'; + +export default function createReporter(options, context) { + return { + name: 'summary-file', + onSuiteEnd(suite) { + fs.writeFileSync( + options.output, + JSON.stringify( + { + total: suite.total, + passed: suite.passed, + failed: suite.failed, + modulePath: context.modulePath, + }, + null, + 2, + ), + ); + }, + getExitCode(suite) { + return suite.failed > 0 ? 1 : 0; + }, + }; +} +``` + +TypeScript reporters can import the public types from `agent-device`: + +```ts +import type { ReplayTestReporterFactory } from 'agent-device'; + +const createReporter: ReplayTestReporterFactory = (options) => ({ + name: 'typed-reporter', + onSuiteEnd(suite) { + // Write artifacts, annotations, or summaries from suite. + }, +}); + +export default createReporter; +``` + +The supported hook today is final-result reporting through `onSuiteEnd`. `getExitCode` can override whether the finished suite exits successfully; when no reporter supplies one, failed tests exit with `1`. The `onProgress` hook is part of the reporter interface for live reporters, but the CLI currently invokes reporters after the suite result is available. + ## Parametrise `.ad` scripts Substitute `${VAR}` tokens in `.ad` scripts using values from the CLI, shell env, script-local `env` directives, or built-ins. From f8bfb9b3075e7b7bf081d874e6690b649fe85b77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 29 Jun 2026 19:21:34 +0200 Subject: [PATCH 3/6] fix: satisfy reporter CI guards --- scripts/integration-progress-model.ts | 1 + src/cli-test-reporters/custom.ts | 37 +++++++++++++++------------ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index f0d0b26d0..b347e47bf 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -276,6 +276,7 @@ function summarizeProviderScenarioFlagExclusions() { 'snapshotForceFull', 'baseline', 'threshold', + 'reporter', 'reportJunit', 'replayMaestro', 'replayExportFormat', diff --git a/src/cli-test-reporters/custom.ts b/src/cli-test-reporters/custom.ts index 9dd91712f..aaef40f8c 100644 --- a/src/cli-test-reporters/custom.ts +++ b/src/cli-test-reporters/custom.ts @@ -19,6 +19,12 @@ type CustomReporterSpec = { options: unknown; }; +const OPTIONAL_REPORTER_HOOKS = [ + 'onProgress', + 'onSuiteEnd', + 'getExitCode', +] as const satisfies readonly (keyof ReplayTestReporter)[]; + export function isCustomReplayTestReporterSpec(spec: string): boolean { const modulePath = readCustomReporterModulePath(spec); return ( @@ -111,23 +117,20 @@ function validateCustomReplayTestReporter( if (typeof candidate.name !== 'string' || candidate.name.trim().length === 0) { throw new AppError('INVALID_ARGS', `Custom test reporter ${modulePath} must define name.`); } - if (candidate.onProgress !== undefined && typeof candidate.onProgress !== 'function') { - throw new AppError( - 'INVALID_ARGS', - `Custom test reporter ${modulePath} onProgress must be a function.`, - ); - } - if (candidate.onSuiteEnd !== undefined && typeof candidate.onSuiteEnd !== 'function') { - throw new AppError( - 'INVALID_ARGS', - `Custom test reporter ${modulePath} onSuiteEnd must be a function.`, - ); - } - if (candidate.getExitCode !== undefined && typeof candidate.getExitCode !== 'function') { - throw new AppError( - 'INVALID_ARGS', - `Custom test reporter ${modulePath} getExitCode must be a function.`, - ); + for (const hook of OPTIONAL_REPORTER_HOOKS) { + validateOptionalReporterHook(candidate, modulePath, hook); } return candidate as ReplayTestReporter; } + +function validateOptionalReporterHook( + reporter: Partial, + modulePath: string, + hook: (typeof OPTIONAL_REPORTER_HOOKS)[number], +): void { + if (reporter[hook] === undefined || typeof reporter[hook] === 'function') return; + throw new AppError( + 'INVALID_ARGS', + `Custom test reporter ${modulePath} ${hook} must be a function.`, + ); +} From 60e38f92d8eebbb465792974fe3a4aaaaf67a184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 29 Jun 2026 19:39:57 +0200 Subject: [PATCH 4/6] refactor: modularize replay test reporters --- src/__tests__/cli-network.test.ts | 4 +- src/__tests__/cli-test-reporters-spec.test.ts | 65 +++++++ src/cli-test-reporters/custom.ts | 75 ++------ src/cli-test-reporters/default.ts | 58 +++--- src/cli-test-reporters/junit.ts | 31 ++- src/cli-test-reporters/registry.ts | 69 +++---- src/cli-test-reporters/spec.ts | 179 ++++++++++++++++++ src/cli-test-reporters/types.ts | 9 + src/cli-test.ts | 15 +- src/utils/__tests__/args.test.ts | 4 +- src/utils/cli-flags.ts | 4 +- website/docs/docs/replay-e2e.md | 29 +-- 12 files changed, 390 insertions(+), 152 deletions(-) create mode 100644 src/__tests__/cli-test-reporters-spec.test.ts create mode 100644 src/cli-test-reporters/spec.ts diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index a5a670a17..472e48ed1 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -697,7 +697,7 @@ test('test command supports explicit reporter lists', async () => { try { const result = await runCliCapture( - ['test', './suite', '--reporter', `junit:${reportPath}`], + ['test', './suite', '--reporter', JSON.stringify(['junit', { output: reportPath }])], async () => makeReplaySuiteResponse(), ); @@ -733,7 +733,7 @@ test('test command loads custom reporter modules with JSON options', async () => 'utf8', ); - const reporterSpec = `${reporterPath}:${JSON.stringify({ output: outputPath })}`; + const reporterSpec = JSON.stringify([reporterPath, { output: outputPath }]); const result = await runCliCapture(['test', './suite', '--reporter', reporterSpec], async () => makeReplaySuiteResponse(), ); diff --git a/src/__tests__/cli-test-reporters-spec.test.ts b/src/__tests__/cli-test-reporters-spec.test.ts new file mode 100644 index 000000000..d85079318 --- /dev/null +++ b/src/__tests__/cli-test-reporters-spec.test.ts @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + buildReplayTestReporterSpecs, + parseReplayTestReporterSpec, +} from '../cli-test-reporters/spec.ts'; + +test('parses built-in reporter shorthand specs', () => { + assert.deepEqual(parseReplayTestReporterSpec('default'), { + kind: 'builtin', + name: 'default', + raw: 'default', + }); + assert.deepEqual(parseReplayTestReporterSpec('junit:./report.xml'), { + kind: 'builtin', + name: 'junit', + raw: 'junit:./report.xml', + options: { output: './report.xml' }, + }); +}); + +test('parses JSON tuple reporter specs', () => { + assert.deepEqual(parseReplayTestReporterSpec('["junit",{"output":"./report.xml"}]'), { + kind: 'builtin', + name: 'junit', + raw: '["junit",{"output":"./report.xml"}]', + options: { output: './report.xml' }, + }); + assert.deepEqual(parseReplayTestReporterSpec('["./reporter.mjs",{"output":"./out.txt"}]'), { + kind: 'custom', + modulePath: './reporter.mjs', + raw: '["./reporter.mjs",{"output":"./out.txt"}]', + options: { output: './out.txt' }, + }); +}); + +test('parses custom reporter shorthand options', () => { + assert.deepEqual(parseReplayTestReporterSpec('./reporter.mjs:{"output":"./out.txt"}'), { + kind: 'custom', + modulePath: './reporter.mjs', + raw: './reporter.mjs:{"output":"./out.txt"}', + options: { output: './out.txt' }, + }); +}); + +test('expands implicit and compatibility reporter defaults', () => { + assert.deepEqual(buildReplayTestReporterSpecs({}), [ + { kind: 'builtin', name: 'default', raw: 'default' }, + ]); + assert.deepEqual(buildReplayTestReporterSpecs({ json: true, reportJunit: './report.xml' }), [ + { + kind: 'builtin', + name: 'junit', + raw: '["junit",{"output":"./report.xml"}]', + options: { output: './report.xml' }, + }, + ]); +}); + +test('rejects invalid reporter specs', () => { + assert.throws(() => parseReplayTestReporterSpec('junit'), /requires an output path/); + assert.throws(() => parseReplayTestReporterSpec('["junit",{}]'), /requires an output path/); + assert.throws(() => parseReplayTestReporterSpec('["default",{}]'), /does not accept options/); + assert.throws(() => parseReplayTestReporterSpec('["default",{},{}]'), /must contain/); +}); diff --git a/src/cli-test-reporters/custom.ts b/src/cli-test-reporters/custom.ts index aaef40f8c..d6be11ff5 100644 --- a/src/cli-test-reporters/custom.ts +++ b/src/cli-test-reporters/custom.ts @@ -2,11 +2,8 @@ import os from 'node:os'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { AppError } from '../utils/errors.ts'; -import type { - ReplayTestReporter, - ReplayTestReporterFactory, - ReplayTestReporterLoadContext, -} from './types.ts'; +import type { ReplayTestReporter, ReplayTestReporterFactory } from './types.ts'; +import type { ReplayTestReporterSpec } from './spec.ts'; type CustomReporterModule = { default?: unknown; @@ -14,72 +11,36 @@ type CustomReporterModule = { reporter?: unknown; }; -type CustomReporterSpec = { - modulePath: string; - options: unknown; -}; - const OPTIONAL_REPORTER_HOOKS = [ 'onProgress', 'onSuiteEnd', 'getExitCode', ] as const satisfies readonly (keyof ReplayTestReporter)[]; -export function isCustomReplayTestReporterSpec(spec: string): boolean { - const modulePath = readCustomReporterModulePath(spec); - return ( - modulePath.startsWith('.') || - modulePath.startsWith('/') || - modulePath.startsWith('~') || - modulePath.startsWith('file:') - ); +export async function createCustomReplayTestReporter( + spec: Extract, +): Promise { + const modulePath = resolveCustomReporterModulePath(spec.modulePath); + const module = await importCustomReporterModule(modulePath); + const factory = readCustomReporterFactory(module, spec.modulePath); + const reporter = await factory(spec.options, { spec: spec.raw, modulePath }); + return validateCustomReplayTestReporter(reporter, spec.modulePath); } -export async function createCustomReplayTestReporter(spec: string): Promise { - const customSpec = splitCustomReplayTestReporterSpec(spec); - const modulePath = resolveCustomReporterModulePath(customSpec.modulePath); - const module = await importCustomReporterModule(modulePath); +function readCustomReporterFactory( + module: CustomReporterModule, + modulePath: string, +): ReplayTestReporterFactory { const exported = module.createReporter ?? module.default ?? module.reporter; if (!exported) { throw new AppError( 'INVALID_ARGS', - `Custom test reporter ${customSpec.modulePath} must export default, createReporter, or reporter.`, - ); - } - - const reporter = - typeof exported === 'function' - ? await (exported as ReplayTestReporterFactory)(customSpec.options, { - spec, - modulePath, - } satisfies ReplayTestReporterLoadContext) - : exported; - - return validateCustomReplayTestReporter(reporter, customSpec.modulePath); -} - -function splitCustomReplayTestReporterSpec(spec: string): CustomReporterSpec { - const optionsSeparator = spec.indexOf(':{'); - if (optionsSeparator < 0) return { modulePath: spec.trim(), options: undefined }; - const modulePath = readCustomReporterModulePath(spec); - const rawOptions = spec.slice(optionsSeparator + 1); - if (!modulePath) { - throw new AppError('INVALID_ARGS', 'Custom test reporter path cannot be empty.'); - } - try { - return { modulePath, options: JSON.parse(rawOptions) }; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new AppError( - 'INVALID_ARGS', - `Invalid JSON options for custom test reporter ${modulePath}: ${message}`, + `Custom test reporter ${modulePath} must export default, createReporter, or reporter.`, ); } -} - -function readCustomReporterModulePath(spec: string): string { - const optionsSeparator = spec.indexOf(':{'); - return (optionsSeparator < 0 ? spec : spec.slice(0, optionsSeparator)).trim(); + return typeof exported === 'function' + ? (exported as ReplayTestReporterFactory) + : () => exported as ReplayTestReporter; } function resolveCustomReporterModulePath(modulePath: string): string { diff --git a/src/cli-test-reporters/default.ts b/src/cli-test-reporters/default.ts index 7379343e0..726aa7534 100644 --- a/src/cli-test-reporters/default.ts +++ b/src/cli-test-reporters/default.ts @@ -2,7 +2,8 @@ import type { ReplaySuiteResult } from '../daemon/types.ts'; import { replayTestFailureStepLines } from '../cli-test-trace.ts'; import { formatDurationSeconds } from '../utils/duration-format.ts'; import { colorize, supportsColor } from '../utils/output.ts'; -import type { ReplayTestReporter } from './types.ts'; +import { AppError } from '../utils/errors.ts'; +import type { ReplayTestReporterContext, ReplayTestReporterFactory } from './types.ts'; import { getReplayTestExitCode, isDefinedString, @@ -17,19 +18,25 @@ import { type PassedReplayTestResult, } from './format.ts'; -export function createDefaultReplayTestReporter(): ReplayTestReporter { +export const createDefaultReplayTestReporter: ReplayTestReporterFactory = (options) => { + if (options !== undefined) { + throw new AppError('INVALID_ARGS', 'The default test reporter does not accept options.'); + } return { name: 'default', - onSuiteEnd: (suite, context) => renderReplayTestSummary(suite, { debug: context.debug }), + onSuiteEnd: (suite, context) => renderReplayTestSummary(suite, context), getExitCode: getReplayTestExitCode, }; -} +}; -function renderReplayTestSummary(data: ReplaySuiteResult, options: { debug?: boolean } = {}): void { +function renderReplayTestSummary( + data: ReplaySuiteResult, + context: ReplayTestReporterContext, +): void { const flaky = data.tests.filter(isFlakyReplayTestResult); - process.stdout.write(`${formatReplayTestSummaryLine(data, flaky.length)}\n`); - renderFailureDetails(data.tests.filter(isFailedReplayTestResult), { debug: options.debug }); - renderFlakyTestSummary(flaky); + context.writeStdout(`${formatReplayTestSummaryLine(data, flaky.length)}\n`); + renderFailureDetails(data.tests.filter(isFailedReplayTestResult), context); + renderFlakyTestSummary(flaky, context); } function formatReplayTestSummaryLine(data: ReplaySuiteResult, flakyCount: number): string { @@ -53,12 +60,15 @@ function replayFailureConsoleLines(result: FailedReplayTestResult): string[] { ].filter(isDefinedString); } -function renderFlakyTestSummary(results: PassedReplayTestResult[]): void { +function renderFlakyTestSummary( + results: PassedReplayTestResult[], + context: ReplayTestReporterContext, +): void { if (results.length === 0) return; - process.stdout.write('\n'); - process.stdout.write('Flaky tests:\n'); + context.writeStdout('\n'); + context.writeStdout('Flaky tests:\n'); for (const result of results) { - process.stdout.write( + context.writeStdout( ` ${replayFlakyStatusIcon()} ${replayTestDisplayNameWithFile(result)} after ${result.attempts} attempts${formatFlakyReplayDurationSuffix(result)}\n`, ); for (const failure of result.attemptFailures ?? []) { @@ -66,7 +76,7 @@ function renderFlakyTestSummary(results: PassedReplayTestResult[]): void { typeof failure.durationMs === 'number' ? ` (${formatDurationSeconds(failure.durationMs)})` : ''; - process.stdout.write( + context.writeStdout( ` attempt ${failure.attempt} failed${attemptDuration}: ${failure.message}\n`, ); } @@ -75,29 +85,29 @@ function renderFlakyTestSummary(results: PassedReplayTestResult[]): void { function renderFailureDetails( results: FailedReplayTestResult[], - options: { debug?: boolean } = {}, + context: ReplayTestReporterContext, ): void { if (results.length === 0) return; - process.stdout.write('\n'); - process.stdout.write('Failures:\n'); + context.writeStdout('\n'); + context.writeStdout('Failures:\n'); for (const result of results) { - process.stdout.write(` ${replayTestDisplayNameWithFile(result)}\n`); - renderReplayFailureBody(result, { debug: options.debug, indent: ' ' }); + context.writeStdout(` ${replayTestDisplayNameWithFile(result)}\n`); + renderReplayFailureBody(result, context, ' '); } } function renderReplayFailureBody( result: FailedReplayTestResult, - options: { debug?: boolean; indent: string }, + context: ReplayTestReporterContext, + indent: string, ): void { - const { debug, indent } = options; - process.stdout.write(`${indent}${result.error?.message ?? 'Unknown test failure'}\n`); + context.writeStdout(`${indent}${result.error?.message ?? 'Unknown test failure'}\n`); for (const line of replayFailureConsoleLines(result)) { - process.stdout.write(`${indent}${line}\n`); + context.writeStdout(`${indent}${line}\n`); } - if (!debug) return; + if (!context.debug) return; for (const line of replayTestFailureStepLines(result)) { - process.stdout.write(`${indent}${line}\n`); + context.writeStdout(`${indent}${line}\n`); } } diff --git a/src/cli-test-reporters/junit.ts b/src/cli-test-reporters/junit.ts index 3340f7d7c..0caf2e40e 100644 --- a/src/cli-test-reporters/junit.ts +++ b/src/cli-test-reporters/junit.ts @@ -1,8 +1,7 @@ -import fs from 'node:fs'; import path from 'node:path'; import type { ReplaySuiteResult, ReplaySuiteTestResult } from '../daemon/types.ts'; import { AppError } from '../utils/errors.ts'; -import type { ReplayTestReporter } from './types.ts'; +import type { ReplayTestReporterContext, ReplayTestReporterFactory } from './types.ts'; import { appendOptionalLine, appendReplayErrorDetails, @@ -19,19 +18,37 @@ import { type FailedReplayTestResult, } from './format.ts'; -export function createJunitReplayTestReporter(reportPath: string): ReplayTestReporter { +export const createJunitReplayTestReporter: ReplayTestReporterFactory = (options) => { + const reportPath = readJunitReportPath(options); return { name: 'junit', - onSuiteEnd: (suite) => writeReplayJunitReport(reportPath, suite), + onSuiteEnd: (suite, context) => writeReplayJunitReport(reportPath, suite, context), getExitCode: getReplayTestExitCode, }; +}; + +function readJunitReportPath(options: unknown): string { + if (options && typeof options === 'object' && !Array.isArray(options)) { + const output = (options as Record).output; + if (typeof output === 'string' && output.trim().length > 0) { + return output; + } + } + throw new AppError( + 'INVALID_ARGS', + 'The junit test reporter requires an output path. Use --reporter junit:.', + ); } -function writeReplayJunitReport(reportPath: string, suite: ReplaySuiteResult): void { +function writeReplayJunitReport( + reportPath: string, + suite: ReplaySuiteResult, + context: ReplayTestReporterContext, +): void { const directory = path.dirname(reportPath); try { - fs.mkdirSync(directory, { recursive: true }); - fs.writeFileSync(reportPath, buildReplayJunitXml(suite), 'utf8'); + context.mkdir(directory); + context.writeFile(reportPath, buildReplayJunitXml(suite)); } catch (error) { const message = error instanceof Error ? error.message : String(error); throw new AppError( diff --git a/src/cli-test-reporters/registry.ts b/src/cli-test-reporters/registry.ts index cf67a8d0f..18ce82067 100644 --- a/src/cli-test-reporters/registry.ts +++ b/src/cli-test-reporters/registry.ts @@ -1,27 +1,31 @@ import type { ReplaySuiteResult } from '../daemon/types.ts'; import { AppError } from '../utils/errors.ts'; -import { createCustomReplayTestReporter, isCustomReplayTestReporterSpec } from './custom.ts'; +import { createCustomReplayTestReporter } from './custom.ts'; import { createDefaultReplayTestReporter } from './default.ts'; import { getReplayTestExitCode } from './format.ts'; import { createJunitReplayTestReporter } from './junit.ts'; -import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; +import { + buildReplayTestReporterSpecs, + type BuiltInReplayTestReporterName, + type ReplayTestReporterSpec, +} from './spec.ts'; +import type { + ReplayTestReporter, + ReplayTestReporterContext, + ReplayTestReporterFactory, +} from './types.ts'; + +const BUILT_IN_REPLAY_TEST_REPORTERS = { + default: createDefaultReplayTestReporter, + junit: createJunitReplayTestReporter, +} satisfies Record; export async function resolveReplayTestReporters(options: { reporters?: string[]; reportJunit?: string; json?: boolean; }): Promise { - const specs = - options.reporters && options.reporters.length > 0 - ? [...options.reporters] - : options.json - ? [] - : ['default']; - - if (options.reportJunit) { - specs.push(`junit:${options.reportJunit}`); - } - + const specs = buildReplayTestReporterSpecs(options); return await Promise.all(specs.map(resolveReplayTestReporter)); } @@ -46,39 +50,16 @@ export function getReplayTestReporterExitCode( return getReplayTestExitCode(suite); } -async function resolveReplayTestReporter(spec: string): Promise { - if (isCustomReplayTestReporterSpec(spec)) { +async function resolveReplayTestReporter( + spec: ReplayTestReporterSpec, +): Promise { + if (spec.kind === 'custom') { return await createCustomReplayTestReporter(spec); } - const { name, value } = splitReplayTestReporterSpec(spec); - if (name === 'default') { - if (value) { - throw new AppError('INVALID_ARGS', 'The default test reporter does not accept options.'); - } - return createDefaultReplayTestReporter(); + const factory = BUILT_IN_REPLAY_TEST_REPORTERS[spec.name]; + if (!factory) { + throw new AppError('INVALID_ARGS', `Unknown built-in test reporter "${spec.name}".`); } - if (name === 'junit') { - if (!value) { - throw new AppError( - 'INVALID_ARGS', - 'The junit test reporter requires an output path. Use --reporter junit:.', - ); - } - return createJunitReplayTestReporter(value); - } - - throw new AppError( - 'INVALID_ARGS', - `Unknown test reporter "${name}". Built-in reporters: default, junit:. Custom reporters must be file paths.`, - ); -} - -function splitReplayTestReporterSpec(spec: string): { name: string; value?: string } { - const separatorIndex = spec.indexOf(':'); - if (separatorIndex < 0) return { name: spec.trim() }; - return { - name: spec.slice(0, separatorIndex).trim(), - value: spec.slice(separatorIndex + 1), - }; + return await factory(spec.options, { spec: spec.raw, modulePath: spec.name }); } diff --git a/src/cli-test-reporters/spec.ts b/src/cli-test-reporters/spec.ts new file mode 100644 index 000000000..260d112d4 --- /dev/null +++ b/src/cli-test-reporters/spec.ts @@ -0,0 +1,179 @@ +import { AppError } from '../utils/errors.ts'; + +export type BuiltInReplayTestReporterName = 'default' | 'junit'; + +export type ReplayTestReporterSpec = + | { + kind: 'builtin'; + name: 'default'; + raw: string; + options?: undefined; + } + | { + kind: 'builtin'; + name: 'junit'; + raw: string; + options: { output: string }; + } + | { + kind: 'custom'; + modulePath: string; + raw: string; + options: unknown; + }; + +export function buildReplayTestReporterSpecs(options: { + reporters?: string[]; + reportJunit?: string; + json?: boolean; +}): ReplayTestReporterSpec[] { + const specs = + options.reporters && options.reporters.length > 0 + ? options.reporters.map(parseReplayTestReporterSpec) + : options.json + ? [] + : [parseReplayTestReporterSpec('default')]; + + if (options.reportJunit) { + specs.push(parseReplayTestReporterSpec(['junit', { output: options.reportJunit }])); + } + + return specs; +} + +export function parseReplayTestReporterSpec( + spec: string | [string, unknown], +): ReplayTestReporterSpec { + if (Array.isArray(spec)) { + return parseTupleReplayTestReporterSpec(spec, JSON.stringify(spec)); + } + + const trimmed = spec.trim(); + if (!trimmed) { + throw new AppError('INVALID_ARGS', 'Test reporter spec cannot be empty.'); + } + + if (trimmed.startsWith('[')) { + return parseJsonTupleReplayTestReporterSpec(trimmed); + } + + const { name, value } = splitReplayTestReporterSpec(trimmed); + if (isCustomReplayTestReporterName(name)) { + return { + kind: 'custom', + modulePath: name, + raw: trimmed, + options: readCustomReporterOptions(name, value), + }; + } + + return parseBuiltInReplayTestReporterSpec(name, value, trimmed); +} + +function parseJsonTupleReplayTestReporterSpec(spec: string): ReplayTestReporterSpec { + let parsed: unknown; + try { + parsed = JSON.parse(spec); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new AppError('INVALID_ARGS', `Invalid JSON reporter tuple: ${message}`); + } + if (!Array.isArray(parsed)) { + throw new AppError('INVALID_ARGS', 'JSON reporter spec must be an array.'); + } + return parseTupleReplayTestReporterSpec(parsed, spec); +} + +function parseTupleReplayTestReporterSpec(tuple: unknown[], raw: string): ReplayTestReporterSpec { + const [name, options] = tuple; + if (typeof name !== 'string' || name.trim().length === 0) { + throw new AppError( + 'INVALID_ARGS', + 'Reporter tuple first entry must be a reporter name or path.', + ); + } + if (tuple.length > 2) { + throw new AppError('INVALID_ARGS', 'Reporter tuple must contain [nameOrPath, options].'); + } + const reporterName = name.trim(); + if (isCustomReplayTestReporterName(reporterName)) { + return { kind: 'custom', modulePath: reporterName, raw, options }; + } + return parseBuiltInReplayTestReporterSpec(reporterName, options, raw); +} + +function parseBuiltInReplayTestReporterSpec( + name: string, + value: unknown, + raw: string, +): ReplayTestReporterSpec { + if (name === 'default') { + if (value !== undefined) { + throw new AppError('INVALID_ARGS', 'The default test reporter does not accept options.'); + } + return { kind: 'builtin', name, raw }; + } + + if (name === 'junit') { + return { kind: 'builtin', name, raw, options: readJunitReporterOptions(value) }; + } + + throw new AppError( + 'INVALID_ARGS', + `Unknown test reporter "${name}". Built-in reporters: default, junit:. Custom reporters must be file paths.`, + ); +} + +function readJunitReporterOptions(value: unknown): { output: string } { + if (typeof value === 'string' && value.trim().length > 0) { + return { output: value }; + } + if (value && typeof value === 'object' && !Array.isArray(value)) { + const output = + (value as Record).output ?? (value as Record).path; + if (typeof output === 'string' && output.trim().length > 0) { + return { output }; + } + } + throw new AppError( + 'INVALID_ARGS', + 'The junit test reporter requires an output path. Use --reporter junit:.', + ); +} + +function readCustomReporterOptions(modulePath: string, value: string | undefined): unknown { + if (value === undefined) return undefined; + if (!value.startsWith('{')) return value; + try { + return JSON.parse(value); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new AppError( + 'INVALID_ARGS', + `Invalid JSON options for custom test reporter ${modulePath}: ${message}`, + ); + } +} + +function splitReplayTestReporterSpec(spec: string): { name: string; value?: string } { + const optionsSeparator = spec.indexOf(':{'); + if (optionsSeparator >= 0) { + return { + name: spec.slice(0, optionsSeparator).trim(), + value: spec.slice(optionsSeparator + 1), + }; + } + + const separatorIndex = spec.indexOf(':'); + if (separatorIndex < 0) return { name: spec.trim() }; + return { + name: spec.slice(0, separatorIndex).trim(), + value: spec.slice(separatorIndex + 1), + }; +} + +function isCustomReplayTestReporterName(name: string): boolean { + return ( + name.startsWith('.') || name.startsWith('/') || name.startsWith('~') || name.startsWith('file:') + ); +} diff --git a/src/cli-test-reporters/types.ts b/src/cli-test-reporters/types.ts index c670f3822..1718a1937 100644 --- a/src/cli-test-reporters/types.ts +++ b/src/cli-test-reporters/types.ts @@ -3,6 +3,11 @@ import type { ReplaySuiteResult } from '../daemon/types.ts'; export type ReplayTestReporterContext = { debug?: boolean; + cwd: string; + writeStdout(text: string): void; + writeStderr(text: string): void; + mkdir(path: string): void; + writeFile(path: string, contents: string): void; }; export type ReplayTestReporterLoadContext = { @@ -12,6 +17,10 @@ export type ReplayTestReporterLoadContext = { export type ReplayTestReporter = { name: string; + /** + * Reserved for live reporter support. The CLI currently invokes reporters after the final + * ReplaySuiteResult is available, so custom reporters should use onSuiteEnd for now. + */ onProgress?(event: RequestProgressEvent, context: ReplayTestReporterContext): void; onSuiteEnd?(suite: ReplaySuiteResult, context: ReplayTestReporterContext): Promise | void; getExitCode?(suite: ReplaySuiteResult): number | undefined; diff --git a/src/cli-test.ts b/src/cli-test.ts index 59f92081c..70891a788 100644 --- a/src/cli-test.ts +++ b/src/cli-test.ts @@ -1,9 +1,11 @@ +import fs from 'node:fs'; import type { ReplaySuiteResult } from './daemon/types.ts'; import { getReplayTestReporterExitCode, resolveReplayTestReporters, runReplayTestReporters, } from './cli-test-reporters/registry.ts'; +import type { ReplayTestReporterContext } from './cli-test-reporters/types.ts'; import { printJson } from './utils/output.ts'; export async function renderReplayTestResponse(options: { @@ -15,9 +17,20 @@ export async function renderReplayTestResponse(options: { }): Promise { const { suite, json, debug, reporter, reportJunit } = options; const reporters = await resolveReplayTestReporters({ reporters: reporter, reportJunit, json }); - await runReplayTestReporters(reporters, suite, { debug }); + await runReplayTestReporters(reporters, suite, createReplayTestReporterContext({ debug })); if (json) { printJson({ success: true, data: suite }); } return getReplayTestReporterExitCode(reporters, suite); } + +function createReplayTestReporterContext(options: { debug?: boolean }): ReplayTestReporterContext { + return { + debug: options.debug, + cwd: process.cwd(), + writeStdout: (text) => process.stdout.write(text), + writeStderr: (text) => process.stderr.write(text), + mkdir: (directory) => fs.mkdirSync(directory, { recursive: true }), + writeFile: (filePath, contents) => fs.writeFileSync(filePath, contents, 'utf8'), + }; +} diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index f8a745365..b7bd80d88 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -2071,8 +2071,8 @@ test('command usage describes test suite flags', () => { assert.match(help, /--retries /); assert.match(help, /--record-video/); assert.match(help, /--artifacts-dir /); - assert.match(help, /--reporter /); - assert.match(help, /custom reporters are file paths/); + assert.match(help, /--reporter /); + assert.match(help, /JSON \[nameOrPath, options\]/); assert.match(help, /--report-junit /); assert.match(help, /compatibility alias for --reporter junit:/); assert.doesNotMatch(help, /test --verbose prints per-test step timings without debug logs/); diff --git a/src/utils/cli-flags.ts b/src/utils/cli-flags.ts index 9338f7904..83fa18f13 100644 --- a/src/utils/cli-flags.ts +++ b/src/utils/cli-flags.ts @@ -948,9 +948,9 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ names: ['--reporter'], type: 'string', multiple: true, - usageLabel: '--reporter ', + usageLabel: '--reporter ', usageDescription: - 'Test: add a replay suite reporter; built-ins are default and junit:; custom reporters are file paths with optional : options (repeatable)', + 'Test: add a replay suite reporter; use default, junit:, a custom reporter path, or JSON [nameOrPath, options] (repeatable)', }, { key: 'reportJunit', diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 16fd9aeb3..79bb19b80 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -108,26 +108,27 @@ Custom reporters are CLI-only presentation adapters. The daemon still returns th ```bash agent-device test ./workflows --reporter ./scripts/replay-reporter.mjs agent-device test ./workflows --reporter './scripts/replay-reporter.mjs:{"output":"./tmp/report.txt"}' +agent-device test ./workflows --reporter '["./scripts/replay-reporter.mjs",{"output":"./tmp/report.txt"}]' ``` -Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive parsed JSON options and load context: +Reporter options can use either the compact `path:{"key":"value"}` form or the JSON tuple form `["path", options]`. The tuple form also works for built-ins, for example `--reporter '["junit",{"output":"./tmp/junit.xml"}]'`, and avoids ambiguous path parsing. + +Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive parsed JSON options and load context. Reporter hooks receive an IO context with `cwd`, `writeStdout`, `writeStderr`, `mkdir`, and `writeFile` helpers: ```js // scripts/replay-reporter.mjs -import fs from 'node:fs'; - -export default function createReporter(options, context) { +export default function createReporter(options, loadContext) { return { name: 'summary-file', - onSuiteEnd(suite) { - fs.writeFileSync( + onSuiteEnd(suite, context) { + context.writeFile( options.output, JSON.stringify( { total: suite.total, passed: suite.passed, failed: suite.failed, - modulePath: context.modulePath, + modulePath: loadContext.modulePath, }, null, 2, @@ -156,6 +157,8 @@ const createReporter: ReplayTestReporterFactory = (options) => ({ export default createReporter; ``` +The CLI loads reporter modules with Node dynamic `import()`. Use `.mjs` or `.js` files at runtime; for TypeScript, compile the reporter to JavaScript before passing it to `--reporter`. Loading `.ts` files directly depends on Node's type-stripping behavior and is not part of the supported reporter contract. + The supported hook today is final-result reporting through `onSuiteEnd`. `getExitCode` can override whether the finished suite exits successfully; when no reporter supplies one, failed tests exit with `1`. The `onProgress` hook is part of the reporter interface for live reporters, but the CLI currently invokes reporters after the suite result is available. ## Parametrise `.ad` scripts @@ -174,12 +177,12 @@ click "label=${APP_ID}" ### Precedence -| Source | Priority | Example | -|---|---|---| -| CLI `-e KEY=VALUE` | highest | `agent-device test flow.ad -e APP_ID=demo` | -| Shell env prefixed `AD_VAR_` | | `AD_VAR_APP_ID=demo agent-device test flow.ad` (imported as `APP_ID`) | -| Script `env KEY=VALUE` | | `env APP_ID=settings` in header | -| Built-ins | runtime | `AD_PLATFORM`, `AD_SESSION`, `AD_FILENAME`, `AD_DEVICE`, `AD_ARTIFACTS` | +| Source | Priority | Example | +| ---------------------------- | -------- | ----------------------------------------------------------------------- | +| CLI `-e KEY=VALUE` | highest | `agent-device test flow.ad -e APP_ID=demo` | +| Shell env prefixed `AD_VAR_` | | `AD_VAR_APP_ID=demo agent-device test flow.ad` (imported as `APP_ID`) | +| Script `env KEY=VALUE` | | `env APP_ID=settings` in header | +| Built-ins | runtime | `AD_PLATFORM`, `AD_SESSION`, `AD_FILENAME`, `AD_DEVICE`, `AD_ARTIFACTS` | ### Built-ins From 9c13804c783e09e3d0463a9ee50b3926836cab32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 29 Jun 2026 20:00:40 +0200 Subject: [PATCH 5/6] refactor: trim replay reporter parsing --- src/__tests__/cli-test-reporters-spec.test.ts | 10 +-- src/cli-test-reporters/custom.ts | 18 ++-- src/cli-test-reporters/junit.ts | 6 +- src/cli-test-reporters/registry.ts | 10 +-- src/cli-test-reporters/spec.ts | 89 ++++--------------- src/cli-test-reporters/types.ts | 2 - src/cli-test.ts | 2 - website/docs/docs/replay-e2e.md | 2 +- 8 files changed, 38 insertions(+), 101 deletions(-) diff --git a/src/__tests__/cli-test-reporters-spec.test.ts b/src/__tests__/cli-test-reporters-spec.test.ts index d85079318..a8535e6b4 100644 --- a/src/__tests__/cli-test-reporters-spec.test.ts +++ b/src/__tests__/cli-test-reporters-spec.test.ts @@ -15,7 +15,7 @@ test('parses built-in reporter shorthand specs', () => { kind: 'builtin', name: 'junit', raw: 'junit:./report.xml', - options: { output: './report.xml' }, + options: './report.xml', }); }); @@ -51,15 +51,13 @@ test('expands implicit and compatibility reporter defaults', () => { { kind: 'builtin', name: 'junit', - raw: '["junit",{"output":"./report.xml"}]', - options: { output: './report.xml' }, + raw: 'junit:./report.xml', + options: './report.xml', }, ]); }); test('rejects invalid reporter specs', () => { - assert.throws(() => parseReplayTestReporterSpec('junit'), /requires an output path/); - assert.throws(() => parseReplayTestReporterSpec('["junit",{}]'), /requires an output path/); - assert.throws(() => parseReplayTestReporterSpec('["default",{}]'), /does not accept options/); assert.throws(() => parseReplayTestReporterSpec('["default",{},{}]'), /must contain/); + assert.throws(() => parseReplayTestReporterSpec('unknown'), /Unknown test reporter/); }); diff --git a/src/cli-test-reporters/custom.ts b/src/cli-test-reporters/custom.ts index d6be11ff5..0c9296918 100644 --- a/src/cli-test-reporters/custom.ts +++ b/src/cli-test-reporters/custom.ts @@ -79,19 +79,11 @@ function validateCustomReplayTestReporter( throw new AppError('INVALID_ARGS', `Custom test reporter ${modulePath} must define name.`); } for (const hook of OPTIONAL_REPORTER_HOOKS) { - validateOptionalReporterHook(candidate, modulePath, hook); + if (candidate[hook] === undefined || typeof candidate[hook] === 'function') continue; + throw new AppError( + 'INVALID_ARGS', + `Custom test reporter ${modulePath} ${hook} must be a function.`, + ); } return candidate as ReplayTestReporter; } - -function validateOptionalReporterHook( - reporter: Partial, - modulePath: string, - hook: (typeof OPTIONAL_REPORTER_HOOKS)[number], -): void { - if (reporter[hook] === undefined || typeof reporter[hook] === 'function') return; - throw new AppError( - 'INVALID_ARGS', - `Custom test reporter ${modulePath} ${hook} must be a function.`, - ); -} diff --git a/src/cli-test-reporters/junit.ts b/src/cli-test-reporters/junit.ts index 0caf2e40e..d7fdb6413 100644 --- a/src/cli-test-reporters/junit.ts +++ b/src/cli-test-reporters/junit.ts @@ -28,8 +28,12 @@ export const createJunitReplayTestReporter: ReplayTestReporterFactory = (options }; function readJunitReportPath(options: unknown): string { + if (typeof options === 'string' && options.trim().length > 0) { + return options; + } if (options && typeof options === 'object' && !Array.isArray(options)) { - const output = (options as Record).output; + const output = + (options as Record).output ?? (options as Record).path; if (typeof output === 'string' && output.trim().length > 0) { return output; } diff --git a/src/cli-test-reporters/registry.ts b/src/cli-test-reporters/registry.ts index 18ce82067..d8f1032dd 100644 --- a/src/cli-test-reporters/registry.ts +++ b/src/cli-test-reporters/registry.ts @@ -1,5 +1,4 @@ import type { ReplaySuiteResult } from '../daemon/types.ts'; -import { AppError } from '../utils/errors.ts'; import { createCustomReplayTestReporter } from './custom.ts'; import { createDefaultReplayTestReporter } from './default.ts'; import { getReplayTestExitCode } from './format.ts'; @@ -57,9 +56,8 @@ async function resolveReplayTestReporter( return await createCustomReplayTestReporter(spec); } - const factory = BUILT_IN_REPLAY_TEST_REPORTERS[spec.name]; - if (!factory) { - throw new AppError('INVALID_ARGS', `Unknown built-in test reporter "${spec.name}".`); - } - return await factory(spec.options, { spec: spec.raw, modulePath: spec.name }); + return await BUILT_IN_REPLAY_TEST_REPORTERS[spec.name](spec.options, { + spec: spec.raw, + modulePath: spec.name, + }); } diff --git a/src/cli-test-reporters/spec.ts b/src/cli-test-reporters/spec.ts index 260d112d4..b5399a11a 100644 --- a/src/cli-test-reporters/spec.ts +++ b/src/cli-test-reporters/spec.ts @@ -5,15 +5,9 @@ export type BuiltInReplayTestReporterName = 'default' | 'junit'; export type ReplayTestReporterSpec = | { kind: 'builtin'; - name: 'default'; + name: BuiltInReplayTestReporterName; raw: string; - options?: undefined; - } - | { - kind: 'builtin'; - name: 'junit'; - raw: string; - options: { output: string }; + options?: unknown; } | { kind: 'custom'; @@ -35,42 +29,28 @@ export function buildReplayTestReporterSpecs(options: { : [parseReplayTestReporterSpec('default')]; if (options.reportJunit) { - specs.push(parseReplayTestReporterSpec(['junit', { output: options.reportJunit }])); + specs.push(parseReplayTestReporterSpec(`junit:${options.reportJunit}`)); } return specs; } -export function parseReplayTestReporterSpec( - spec: string | [string, unknown], -): ReplayTestReporterSpec { - if (Array.isArray(spec)) { - return parseTupleReplayTestReporterSpec(spec, JSON.stringify(spec)); - } - +export function parseReplayTestReporterSpec(spec: string): ReplayTestReporterSpec { const trimmed = spec.trim(); if (!trimmed) { throw new AppError('INVALID_ARGS', 'Test reporter spec cannot be empty.'); } if (trimmed.startsWith('[')) { - return parseJsonTupleReplayTestReporterSpec(trimmed); + const [name, options] = readReporterTuple(trimmed); + return createReporterSpec(name, options, trimmed); } const { name, value } = splitReplayTestReporterSpec(trimmed); - if (isCustomReplayTestReporterName(name)) { - return { - kind: 'custom', - modulePath: name, - raw: trimmed, - options: readCustomReporterOptions(name, value), - }; - } - - return parseBuiltInReplayTestReporterSpec(name, value, trimmed); + return createReporterSpec(name, readShorthandOptions(name, value), trimmed); } -function parseJsonTupleReplayTestReporterSpec(spec: string): ReplayTestReporterSpec { +function readReporterTuple(spec: string): [string, unknown] { let parsed: unknown; try { parsed = JSON.parse(spec); @@ -81,41 +61,27 @@ function parseJsonTupleReplayTestReporterSpec(spec: string): ReplayTestReporterS if (!Array.isArray(parsed)) { throw new AppError('INVALID_ARGS', 'JSON reporter spec must be an array.'); } - return parseTupleReplayTestReporterSpec(parsed, spec); -} - -function parseTupleReplayTestReporterSpec(tuple: unknown[], raw: string): ReplayTestReporterSpec { - const [name, options] = tuple; + const [name, options] = parsed; if (typeof name !== 'string' || name.trim().length === 0) { throw new AppError( 'INVALID_ARGS', 'Reporter tuple first entry must be a reporter name or path.', ); } - if (tuple.length > 2) { + if (parsed.length > 2) { throw new AppError('INVALID_ARGS', 'Reporter tuple must contain [nameOrPath, options].'); } - const reporterName = name.trim(); - if (isCustomReplayTestReporterName(reporterName)) { - return { kind: 'custom', modulePath: reporterName, raw, options }; - } - return parseBuiltInReplayTestReporterSpec(reporterName, options, raw); + return [name.trim(), options]; } -function parseBuiltInReplayTestReporterSpec( - name: string, - value: unknown, - raw: string, -): ReplayTestReporterSpec { - if (name === 'default') { - if (value !== undefined) { - throw new AppError('INVALID_ARGS', 'The default test reporter does not accept options.'); - } - return { kind: 'builtin', name, raw }; +function createReporterSpec(name: string, options: unknown, raw: string): ReplayTestReporterSpec { + if (isCustomReplayTestReporterName(name)) { + return { kind: 'custom', modulePath: name, raw, options }; } - - if (name === 'junit') { - return { kind: 'builtin', name, raw, options: readJunitReporterOptions(value) }; + if (name === 'default' || name === 'junit') { + return options === undefined + ? { kind: 'builtin', name, raw } + : { kind: 'builtin', name, raw, options }; } throw new AppError( @@ -124,24 +90,7 @@ function parseBuiltInReplayTestReporterSpec( ); } -function readJunitReporterOptions(value: unknown): { output: string } { - if (typeof value === 'string' && value.trim().length > 0) { - return { output: value }; - } - if (value && typeof value === 'object' && !Array.isArray(value)) { - const output = - (value as Record).output ?? (value as Record).path; - if (typeof output === 'string' && output.trim().length > 0) { - return { output }; - } - } - throw new AppError( - 'INVALID_ARGS', - 'The junit test reporter requires an output path. Use --reporter junit:.', - ); -} - -function readCustomReporterOptions(modulePath: string, value: string | undefined): unknown { +function readShorthandOptions(modulePath: string, value: string | undefined): unknown { if (value === undefined) return undefined; if (!value.startsWith('{')) return value; try { diff --git a/src/cli-test-reporters/types.ts b/src/cli-test-reporters/types.ts index 1718a1937..a580b3916 100644 --- a/src/cli-test-reporters/types.ts +++ b/src/cli-test-reporters/types.ts @@ -3,9 +3,7 @@ import type { ReplaySuiteResult } from '../daemon/types.ts'; export type ReplayTestReporterContext = { debug?: boolean; - cwd: string; writeStdout(text: string): void; - writeStderr(text: string): void; mkdir(path: string): void; writeFile(path: string, contents: string): void; }; diff --git a/src/cli-test.ts b/src/cli-test.ts index 70891a788..ef4200a14 100644 --- a/src/cli-test.ts +++ b/src/cli-test.ts @@ -27,9 +27,7 @@ export async function renderReplayTestResponse(options: { function createReplayTestReporterContext(options: { debug?: boolean }): ReplayTestReporterContext { return { debug: options.debug, - cwd: process.cwd(), writeStdout: (text) => process.stdout.write(text), - writeStderr: (text) => process.stderr.write(text), mkdir: (directory) => fs.mkdirSync(directory, { recursive: true }), writeFile: (filePath, contents) => fs.writeFileSync(filePath, contents, 'utf8'), }; diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 79bb19b80..f0a11c328 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -113,7 +113,7 @@ agent-device test ./workflows --reporter '["./scripts/replay-reporter.mjs",{"out Reporter options can use either the compact `path:{"key":"value"}` form or the JSON tuple form `["path", options]`. The tuple form also works for built-ins, for example `--reporter '["junit",{"output":"./tmp/junit.xml"}]'`, and avoids ambiguous path parsing. -Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive parsed JSON options and load context. Reporter hooks receive an IO context with `cwd`, `writeStdout`, `writeStderr`, `mkdir`, and `writeFile` helpers: +Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive parsed JSON options and load context. Reporter hooks receive an IO context with `writeStdout`, `mkdir`, and `writeFile` helpers: ```js // scripts/replay-reporter.mjs From d2c671878ac04308ec7f223c3ba0318858a5806d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 29 Jun 2026 20:14:30 +0200 Subject: [PATCH 6/6] refactor: simplify custom replay reporters --- src/__tests__/cli-network.test.ts | 13 ++- src/__tests__/cli-test-reporters-spec.test.ts | 27 ++---- src/cli-test-reporters/custom.ts | 2 +- src/cli-test-reporters/default.ts | 10 +-- src/cli-test-reporters/junit.ts | 23 ++---- src/cli-test-reporters/registry.ts | 24 +----- src/cli-test-reporters/spec.ts | 82 ++++--------------- src/cli-test-reporters/types.ts | 1 - src/utils/__tests__/args.test.ts | 4 +- src/utils/cli-flags.ts | 4 +- website/docs/docs/replay-e2e.md | 12 +-- 11 files changed, 52 insertions(+), 150 deletions(-) diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index 472e48ed1..36bbd6aa8 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -697,7 +697,7 @@ test('test command supports explicit reporter lists', async () => { try { const result = await runCliCapture( - ['test', './suite', '--reporter', JSON.stringify(['junit', { output: reportPath }])], + ['test', './suite', '--reporter', `junit:${reportPath}`], async () => makeReplaySuiteResponse(), ); @@ -710,7 +710,7 @@ test('test command supports explicit reporter lists', async () => { } }); -test('test command loads custom reporter modules with JSON options', async () => { +test('test command loads custom reporter modules', async () => { const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-custom-reporter-test-')); const reporterPath = path.join(tmpDir, 'custom-reporter.mjs'); const outputPath = path.join(tmpDir, 'custom-report.json'); @@ -720,11 +720,11 @@ test('test command loads custom reporter modules with JSON options', async () => reporterPath, [ "import fs from 'node:fs';", - 'export default function createReporter(options) {', + 'export default function createReporter() {', ' return {', - " name: 'custom-json',", + " name: 'custom-file',", ' onSuiteEnd(suite) {', - ' fs.writeFileSync(options.output, JSON.stringify({ total: suite.total, failed: suite.failed }), "utf8");', + ` fs.writeFileSync(${JSON.stringify(outputPath)}, JSON.stringify({ total: suite.total, failed: suite.failed }), "utf8");`, ' },', ' getExitCode() { return 0; },', ' };', @@ -733,8 +733,7 @@ test('test command loads custom reporter modules with JSON options', async () => 'utf8', ); - const reporterSpec = JSON.stringify([reporterPath, { output: outputPath }]); - const result = await runCliCapture(['test', './suite', '--reporter', reporterSpec], async () => + const result = await runCliCapture(['test', './suite', '--reporter', reporterPath], async () => makeReplaySuiteResponse(), ); diff --git a/src/__tests__/cli-test-reporters-spec.test.ts b/src/__tests__/cli-test-reporters-spec.test.ts index a8535e6b4..b0b4c4d3d 100644 --- a/src/__tests__/cli-test-reporters-spec.test.ts +++ b/src/__tests__/cli-test-reporters-spec.test.ts @@ -15,31 +15,15 @@ test('parses built-in reporter shorthand specs', () => { kind: 'builtin', name: 'junit', raw: 'junit:./report.xml', - options: './report.xml', + outputPath: './report.xml', }); }); -test('parses JSON tuple reporter specs', () => { - assert.deepEqual(parseReplayTestReporterSpec('["junit",{"output":"./report.xml"}]'), { - kind: 'builtin', - name: 'junit', - raw: '["junit",{"output":"./report.xml"}]', - options: { output: './report.xml' }, - }); - assert.deepEqual(parseReplayTestReporterSpec('["./reporter.mjs",{"output":"./out.txt"}]'), { - kind: 'custom', - modulePath: './reporter.mjs', - raw: '["./reporter.mjs",{"output":"./out.txt"}]', - options: { output: './out.txt' }, - }); -}); - -test('parses custom reporter shorthand options', () => { - assert.deepEqual(parseReplayTestReporterSpec('./reporter.mjs:{"output":"./out.txt"}'), { +test('parses custom reporter paths', () => { + assert.deepEqual(parseReplayTestReporterSpec('./reporter.mjs'), { kind: 'custom', modulePath: './reporter.mjs', - raw: './reporter.mjs:{"output":"./out.txt"}', - options: { output: './out.txt' }, + raw: './reporter.mjs', }); }); @@ -52,12 +36,11 @@ test('expands implicit and compatibility reporter defaults', () => { kind: 'builtin', name: 'junit', raw: 'junit:./report.xml', - options: './report.xml', + outputPath: './report.xml', }, ]); }); test('rejects invalid reporter specs', () => { - assert.throws(() => parseReplayTestReporterSpec('["default",{},{}]'), /must contain/); assert.throws(() => parseReplayTestReporterSpec('unknown'), /Unknown test reporter/); }); diff --git a/src/cli-test-reporters/custom.ts b/src/cli-test-reporters/custom.ts index 0c9296918..9304abce6 100644 --- a/src/cli-test-reporters/custom.ts +++ b/src/cli-test-reporters/custom.ts @@ -23,7 +23,7 @@ export async function createCustomReplayTestReporter( const modulePath = resolveCustomReporterModulePath(spec.modulePath); const module = await importCustomReporterModule(modulePath); const factory = readCustomReporterFactory(module, spec.modulePath); - const reporter = await factory(spec.options, { spec: spec.raw, modulePath }); + const reporter = await factory({ spec: spec.raw, modulePath }); return validateCustomReplayTestReporter(reporter, spec.modulePath); } diff --git a/src/cli-test-reporters/default.ts b/src/cli-test-reporters/default.ts index 726aa7534..c3cd8f1dc 100644 --- a/src/cli-test-reporters/default.ts +++ b/src/cli-test-reporters/default.ts @@ -2,8 +2,7 @@ import type { ReplaySuiteResult } from '../daemon/types.ts'; import { replayTestFailureStepLines } from '../cli-test-trace.ts'; import { formatDurationSeconds } from '../utils/duration-format.ts'; import { colorize, supportsColor } from '../utils/output.ts'; -import { AppError } from '../utils/errors.ts'; -import type { ReplayTestReporterContext, ReplayTestReporterFactory } from './types.ts'; +import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; import { getReplayTestExitCode, isDefinedString, @@ -18,16 +17,13 @@ import { type PassedReplayTestResult, } from './format.ts'; -export const createDefaultReplayTestReporter: ReplayTestReporterFactory = (options) => { - if (options !== undefined) { - throw new AppError('INVALID_ARGS', 'The default test reporter does not accept options.'); - } +export function createDefaultReplayTestReporter(): ReplayTestReporter { return { name: 'default', onSuiteEnd: (suite, context) => renderReplayTestSummary(suite, context), getExitCode: getReplayTestExitCode, }; -}; +} function renderReplayTestSummary( data: ReplaySuiteResult, diff --git a/src/cli-test-reporters/junit.ts b/src/cli-test-reporters/junit.ts index d7fdb6413..b86aba405 100644 --- a/src/cli-test-reporters/junit.ts +++ b/src/cli-test-reporters/junit.ts @@ -1,7 +1,7 @@ import path from 'node:path'; import type { ReplaySuiteResult, ReplaySuiteTestResult } from '../daemon/types.ts'; import { AppError } from '../utils/errors.ts'; -import type { ReplayTestReporterContext, ReplayTestReporterFactory } from './types.ts'; +import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; import { appendOptionalLine, appendReplayErrorDetails, @@ -18,26 +18,17 @@ import { type FailedReplayTestResult, } from './format.ts'; -export const createJunitReplayTestReporter: ReplayTestReporterFactory = (options) => { - const reportPath = readJunitReportPath(options); +export function createJunitReplayTestReporter(reportPath: string | undefined): ReplayTestReporter { + const outputPath = readJunitReportPath(reportPath); return { name: 'junit', - onSuiteEnd: (suite, context) => writeReplayJunitReport(reportPath, suite, context), + onSuiteEnd: (suite, context) => writeReplayJunitReport(outputPath, suite, context), getExitCode: getReplayTestExitCode, }; -}; +} -function readJunitReportPath(options: unknown): string { - if (typeof options === 'string' && options.trim().length > 0) { - return options; - } - if (options && typeof options === 'object' && !Array.isArray(options)) { - const output = - (options as Record).output ?? (options as Record).path; - if (typeof output === 'string' && output.trim().length > 0) { - return output; - } - } +function readJunitReportPath(reportPath: string | undefined): string { + if (reportPath && reportPath.trim().length > 0) return reportPath; throw new AppError( 'INVALID_ARGS', 'The junit test reporter requires an output path. Use --reporter junit:.', diff --git a/src/cli-test-reporters/registry.ts b/src/cli-test-reporters/registry.ts index d8f1032dd..5baf0e2c3 100644 --- a/src/cli-test-reporters/registry.ts +++ b/src/cli-test-reporters/registry.ts @@ -3,21 +3,8 @@ import { createCustomReplayTestReporter } from './custom.ts'; import { createDefaultReplayTestReporter } from './default.ts'; import { getReplayTestExitCode } from './format.ts'; import { createJunitReplayTestReporter } from './junit.ts'; -import { - buildReplayTestReporterSpecs, - type BuiltInReplayTestReporterName, - type ReplayTestReporterSpec, -} from './spec.ts'; -import type { - ReplayTestReporter, - ReplayTestReporterContext, - ReplayTestReporterFactory, -} from './types.ts'; - -const BUILT_IN_REPLAY_TEST_REPORTERS = { - default: createDefaultReplayTestReporter, - junit: createJunitReplayTestReporter, -} satisfies Record; +import { buildReplayTestReporterSpecs, type ReplayTestReporterSpec } from './spec.ts'; +import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; export async function resolveReplayTestReporters(options: { reporters?: string[]; @@ -55,9 +42,6 @@ async function resolveReplayTestReporter( if (spec.kind === 'custom') { return await createCustomReplayTestReporter(spec); } - - return await BUILT_IN_REPLAY_TEST_REPORTERS[spec.name](spec.options, { - spec: spec.raw, - modulePath: spec.name, - }); + if (spec.name === 'default') return createDefaultReplayTestReporter(); + return createJunitReplayTestReporter(spec.outputPath); } diff --git a/src/cli-test-reporters/spec.ts b/src/cli-test-reporters/spec.ts index b5399a11a..45c3949f4 100644 --- a/src/cli-test-reporters/spec.ts +++ b/src/cli-test-reporters/spec.ts @@ -1,19 +1,21 @@ import { AppError } from '../utils/errors.ts'; -export type BuiltInReplayTestReporterName = 'default' | 'junit'; - export type ReplayTestReporterSpec = | { kind: 'builtin'; - name: BuiltInReplayTestReporterName; + name: 'default'; + raw: string; + } + | { + kind: 'builtin'; + name: 'junit'; raw: string; - options?: unknown; + outputPath?: string; } | { kind: 'custom'; modulePath: string; raw: string; - options: unknown; }; export function buildReplayTestReporterSpecs(options: { @@ -41,47 +43,21 @@ export function parseReplayTestReporterSpec(spec: string): ReplayTestReporterSpe throw new AppError('INVALID_ARGS', 'Test reporter spec cannot be empty.'); } - if (trimmed.startsWith('[')) { - const [name, options] = readReporterTuple(trimmed); - return createReporterSpec(name, options, trimmed); + if (isCustomReplayTestReporterName(trimmed)) { + return { kind: 'custom', modulePath: trimmed, raw: trimmed }; } const { name, value } = splitReplayTestReporterSpec(trimmed); - return createReporterSpec(name, readShorthandOptions(name, value), trimmed); -} - -function readReporterTuple(spec: string): [string, unknown] { - let parsed: unknown; - try { - parsed = JSON.parse(spec); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new AppError('INVALID_ARGS', `Invalid JSON reporter tuple: ${message}`); - } - if (!Array.isArray(parsed)) { - throw new AppError('INVALID_ARGS', 'JSON reporter spec must be an array.'); - } - const [name, options] = parsed; - if (typeof name !== 'string' || name.trim().length === 0) { - throw new AppError( - 'INVALID_ARGS', - 'Reporter tuple first entry must be a reporter name or path.', - ); - } - if (parsed.length > 2) { - throw new AppError('INVALID_ARGS', 'Reporter tuple must contain [nameOrPath, options].'); - } - return [name.trim(), options]; -} - -function createReporterSpec(name: string, options: unknown, raw: string): ReplayTestReporterSpec { - if (isCustomReplayTestReporterName(name)) { - return { kind: 'custom', modulePath: name, raw, options }; + if (name === 'default') { + if (value !== undefined) { + throw new AppError('INVALID_ARGS', 'The default test reporter does not accept options.'); + } + return { kind: 'builtin', name, raw: trimmed }; } - if (name === 'default' || name === 'junit') { - return options === undefined - ? { kind: 'builtin', name, raw } - : { kind: 'builtin', name, raw, options }; + if (name === 'junit') { + return value === undefined + ? { kind: 'builtin', name, raw: trimmed } + : { kind: 'builtin', name, raw: trimmed, outputPath: value }; } throw new AppError( @@ -90,29 +66,7 @@ function createReporterSpec(name: string, options: unknown, raw: string): Replay ); } -function readShorthandOptions(modulePath: string, value: string | undefined): unknown { - if (value === undefined) return undefined; - if (!value.startsWith('{')) return value; - try { - return JSON.parse(value); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new AppError( - 'INVALID_ARGS', - `Invalid JSON options for custom test reporter ${modulePath}: ${message}`, - ); - } -} - function splitReplayTestReporterSpec(spec: string): { name: string; value?: string } { - const optionsSeparator = spec.indexOf(':{'); - if (optionsSeparator >= 0) { - return { - name: spec.slice(0, optionsSeparator).trim(), - value: spec.slice(optionsSeparator + 1), - }; - } - const separatorIndex = spec.indexOf(':'); if (separatorIndex < 0) return { name: spec.trim() }; return { diff --git a/src/cli-test-reporters/types.ts b/src/cli-test-reporters/types.ts index a580b3916..e6753c94e 100644 --- a/src/cli-test-reporters/types.ts +++ b/src/cli-test-reporters/types.ts @@ -25,6 +25,5 @@ export type ReplayTestReporter = { }; export type ReplayTestReporterFactory = ( - options: unknown, context: ReplayTestReporterLoadContext, ) => ReplayTestReporter | Promise; diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index b7bd80d88..1c92fa5bd 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -2071,8 +2071,8 @@ test('command usage describes test suite flags', () => { assert.match(help, /--retries /); assert.match(help, /--record-video/); assert.match(help, /--artifacts-dir /); - assert.match(help, /--reporter /); - assert.match(help, /JSON \[nameOrPath, options\]/); + assert.match(help, /--reporter /); + assert.match(help, /custom reporter path/); assert.match(help, /--report-junit /); assert.match(help, /compatibility alias for --reporter junit:/); assert.doesNotMatch(help, /test --verbose prints per-test step timings without debug logs/); diff --git a/src/utils/cli-flags.ts b/src/utils/cli-flags.ts index 83fa18f13..21e827477 100644 --- a/src/utils/cli-flags.ts +++ b/src/utils/cli-flags.ts @@ -948,9 +948,9 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ names: ['--reporter'], type: 'string', multiple: true, - usageLabel: '--reporter ', + usageLabel: '--reporter ', usageDescription: - 'Test: add a replay suite reporter; use default, junit:, a custom reporter path, or JSON [nameOrPath, options] (repeatable)', + 'Test: add a replay suite reporter; use default, junit:, or a custom reporter path (repeatable)', }, { key: 'reportJunit', diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index f0a11c328..859d8a0b1 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -107,22 +107,18 @@ Custom reporters are CLI-only presentation adapters. The daemon still returns th ```bash agent-device test ./workflows --reporter ./scripts/replay-reporter.mjs -agent-device test ./workflows --reporter './scripts/replay-reporter.mjs:{"output":"./tmp/report.txt"}' -agent-device test ./workflows --reporter '["./scripts/replay-reporter.mjs",{"output":"./tmp/report.txt"}]' ``` -Reporter options can use either the compact `path:{"key":"value"}` form or the JSON tuple form `["path", options]`. The tuple form also works for built-ins, for example `--reporter '["junit",{"output":"./tmp/junit.xml"}]'`, and avoids ambiguous path parsing. - -Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive parsed JSON options and load context. Reporter hooks receive an IO context with `writeStdout`, `mkdir`, and `writeFile` helpers: +Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive load context. Reporter hooks receive an IO context with `writeStdout`, `mkdir`, and `writeFile` helpers: ```js // scripts/replay-reporter.mjs -export default function createReporter(options, loadContext) { +export default function createReporter(loadContext) { return { name: 'summary-file', onSuiteEnd(suite, context) { context.writeFile( - options.output, + './tmp/report.txt', JSON.stringify( { total: suite.total, @@ -147,7 +143,7 @@ TypeScript reporters can import the public types from `agent-device`: ```ts import type { ReplayTestReporterFactory } from 'agent-device'; -const createReporter: ReplayTestReporterFactory = (options) => ({ +const createReporter: ReplayTestReporterFactory = () => ({ name: 'typed-reporter', onSuiteEnd(suite) { // Write artifacts, annotations, or summaries from suite.