diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index f0d0b26d0..b347e47bf 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -276,6 +276,7 @@ function summarizeProviderScenarioFlagExclusions() { 'snapshotForceFull', 'baseline', 'threshold', + 'reporter', 'reportJunit', 'replayMaestro', 'replayExportFormat', diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index 76b66f654..36bbd6aa8 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -664,6 +664,7 @@ test('test command writes JUnit report with failure metadata', async () => { ); assert.equal(result.code, 1); + assert.equal(result.calls[0]?.flags?.reportJunit, undefined); const xml = await fs.readFile(reportPath, 'utf8'); assert.match( xml, @@ -689,3 +690,60 @@ test('test command writes JUnit report with failure metadata', async () => { await fs.rm(tmpDir, { recursive: true, force: true }); } }); + +test('test command supports explicit reporter lists', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-reporter-test-')); + const reportPath = path.join(tmpDir, 'replays.junit.xml'); + + try { + const result = await runCliCapture( + ['test', './suite', '--reporter', `junit:${reportPath}`], + async () => makeReplaySuiteResponse(), + ); + + assert.equal(result.code, 1); + assert.doesNotMatch(result.stdout, /Test summary:/); + const xml = await fs.readFile(reportPath, 'utf8'); + assert.match(xml, / { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-custom-reporter-test-')); + const reporterPath = path.join(tmpDir, 'custom-reporter.mjs'); + const outputPath = path.join(tmpDir, 'custom-report.json'); + + try { + await fs.writeFile( + reporterPath, + [ + "import fs from 'node:fs';", + 'export default function createReporter() {', + ' return {', + " name: 'custom-file',", + ' onSuiteEnd(suite) {', + ` fs.writeFileSync(${JSON.stringify(outputPath)}, JSON.stringify({ total: suite.total, failed: suite.failed }), "utf8");`, + ' },', + ' getExitCode() { return 0; },', + ' };', + '}', + ].join('\n'), + 'utf8', + ); + + const result = await runCliCapture(['test', './suite', '--reporter', reporterPath], async () => + makeReplaySuiteResponse(), + ); + + assert.equal(result.code, null); + assert.doesNotMatch(result.stdout, /Test summary:/); + assert.deepEqual(JSON.parse(await fs.readFile(outputPath, 'utf8')), { + total: 3, + failed: 1, + }); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } +}); diff --git a/src/__tests__/cli-test-reporters-spec.test.ts b/src/__tests__/cli-test-reporters-spec.test.ts new file mode 100644 index 000000000..b0b4c4d3d --- /dev/null +++ b/src/__tests__/cli-test-reporters-spec.test.ts @@ -0,0 +1,46 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + buildReplayTestReporterSpecs, + parseReplayTestReporterSpec, +} from '../cli-test-reporters/spec.ts'; + +test('parses built-in reporter shorthand specs', () => { + assert.deepEqual(parseReplayTestReporterSpec('default'), { + kind: 'builtin', + name: 'default', + raw: 'default', + }); + assert.deepEqual(parseReplayTestReporterSpec('junit:./report.xml'), { + kind: 'builtin', + name: 'junit', + raw: 'junit:./report.xml', + outputPath: './report.xml', + }); +}); + +test('parses custom reporter paths', () => { + assert.deepEqual(parseReplayTestReporterSpec('./reporter.mjs'), { + kind: 'custom', + modulePath: './reporter.mjs', + raw: './reporter.mjs', + }); +}); + +test('expands implicit and compatibility reporter defaults', () => { + assert.deepEqual(buildReplayTestReporterSpecs({}), [ + { kind: 'builtin', name: 'default', raw: 'default' }, + ]); + assert.deepEqual(buildReplayTestReporterSpecs({ json: true, reportJunit: './report.xml' }), [ + { + kind: 'builtin', + name: 'junit', + raw: 'junit:./report.xml', + outputPath: './report.xml', + }, + ]); +}); + +test('rejects invalid reporter specs', () => { + assert.throws(() => parseReplayTestReporterSpec('unknown'), /Unknown test reporter/); +}); diff --git a/src/cli-test-reporters/custom.ts b/src/cli-test-reporters/custom.ts new file mode 100644 index 000000000..9304abce6 --- /dev/null +++ b/src/cli-test-reporters/custom.ts @@ -0,0 +1,89 @@ +import os from 'node:os'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { AppError } from '../utils/errors.ts'; +import type { ReplayTestReporter, ReplayTestReporterFactory } from './types.ts'; +import type { ReplayTestReporterSpec } from './spec.ts'; + +type CustomReporterModule = { + default?: unknown; + createReporter?: unknown; + reporter?: unknown; +}; + +const OPTIONAL_REPORTER_HOOKS = [ + 'onProgress', + 'onSuiteEnd', + 'getExitCode', +] as const satisfies readonly (keyof ReplayTestReporter)[]; + +export async function createCustomReplayTestReporter( + spec: Extract, +): Promise { + const modulePath = resolveCustomReporterModulePath(spec.modulePath); + const module = await importCustomReporterModule(modulePath); + const factory = readCustomReporterFactory(module, spec.modulePath); + const reporter = await factory({ spec: spec.raw, modulePath }); + return validateCustomReplayTestReporter(reporter, spec.modulePath); +} + +function readCustomReporterFactory( + module: CustomReporterModule, + modulePath: string, +): ReplayTestReporterFactory { + const exported = module.createReporter ?? module.default ?? module.reporter; + if (!exported) { + throw new AppError( + 'INVALID_ARGS', + `Custom test reporter ${modulePath} must export default, createReporter, or reporter.`, + ); + } + return typeof exported === 'function' + ? (exported as ReplayTestReporterFactory) + : () => exported as ReplayTestReporter; +} + +function resolveCustomReporterModulePath(modulePath: string): string { + if (modulePath.startsWith('file:')) return modulePath; + const expandedPath = modulePath.startsWith('~/') + ? path.join(os.homedir(), modulePath.slice(2)) + : modulePath; + return path.resolve(process.cwd(), expandedPath); +} + +async function importCustomReporterModule(modulePath: string): Promise { + try { + const href = modulePath.startsWith('file:') ? modulePath : pathToFileURL(modulePath).href; + return (await import(href)) as CustomReporterModule; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new AppError( + 'INVALID_ARGS', + `Failed to load custom test reporter ${modulePath}: ${message}`, + ); + } +} + +function validateCustomReplayTestReporter( + reporter: unknown, + modulePath: string, +): ReplayTestReporter { + if (!reporter || typeof reporter !== 'object') { + throw new AppError( + 'INVALID_ARGS', + `Custom test reporter ${modulePath} must export a reporter object or factory.`, + ); + } + const candidate = reporter as Partial; + if (typeof candidate.name !== 'string' || candidate.name.trim().length === 0) { + throw new AppError('INVALID_ARGS', `Custom test reporter ${modulePath} must define name.`); + } + for (const hook of OPTIONAL_REPORTER_HOOKS) { + if (candidate[hook] === undefined || typeof candidate[hook] === 'function') continue; + throw new AppError( + 'INVALID_ARGS', + `Custom test reporter ${modulePath} ${hook} must be a function.`, + ); + } + return candidate as ReplayTestReporter; +} diff --git a/src/cli-test-reporters/default.ts b/src/cli-test-reporters/default.ts new file mode 100644 index 000000000..c3cd8f1dc --- /dev/null +++ b/src/cli-test-reporters/default.ts @@ -0,0 +1,118 @@ +import type { ReplaySuiteResult } from '../daemon/types.ts'; +import { replayTestFailureStepLines } from '../cli-test-trace.ts'; +import { formatDurationSeconds } from '../utils/duration-format.ts'; +import { colorize, supportsColor } from '../utils/output.ts'; +import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; +import { + getReplayTestExitCode, + isDefinedString, + isFailedReplayTestResult, + isFlakyReplayTestResult, + replayArtifactsLine, + replayErrorDiagnosticLine, + replayErrorHintLine, + replayErrorLogLine, + replayTestDisplayNameWithFile, + type FailedReplayTestResult, + type PassedReplayTestResult, +} from './format.ts'; + +export function createDefaultReplayTestReporter(): ReplayTestReporter { + return { + name: 'default', + onSuiteEnd: (suite, context) => renderReplayTestSummary(suite, context), + getExitCode: getReplayTestExitCode, + }; +} + +function renderReplayTestSummary( + data: ReplaySuiteResult, + context: ReplayTestReporterContext, +): void { + const flaky = data.tests.filter(isFlakyReplayTestResult); + context.writeStdout(`${formatReplayTestSummaryLine(data, flaky.length)}\n`); + renderFailureDetails(data.tests.filter(isFailedReplayTestResult), context); + renderFlakyTestSummary(flaky, context); +} + +function formatReplayTestSummaryLine(data: ReplaySuiteResult, flakyCount: number): string { + const durationMs = typeof data.durationMs === 'number' ? data.durationMs : undefined; + const flakySuffix = flakyCount > 0 ? `, ${flakyCount} flaky` : ''; + const durationSuffix = durationMs !== undefined ? ` in ${formatDurationSeconds(durationMs)}` : ''; + return `Test summary: ${data.passed} passed, ${data.failed} failed${flakySuffix}${durationSuffix}`; +} + +function replayFlakyStatusIcon(): string { + const useColor = supportsColor(); + return useColor ? colorize('✓', 'yellow') : '✓'; +} + +function replayFailureConsoleLines(result: FailedReplayTestResult): string[] { + return [ + replayErrorHintLine(result.error), + replayArtifactsLine(result, 'artifacts'), + replayErrorLogLine(result.error, 'log'), + replayErrorDiagnosticLine(result.error, 'diagnostic'), + ].filter(isDefinedString); +} + +function renderFlakyTestSummary( + results: PassedReplayTestResult[], + context: ReplayTestReporterContext, +): void { + if (results.length === 0) return; + context.writeStdout('\n'); + context.writeStdout('Flaky tests:\n'); + for (const result of results) { + context.writeStdout( + ` ${replayFlakyStatusIcon()} ${replayTestDisplayNameWithFile(result)} after ${result.attempts} attempts${formatFlakyReplayDurationSuffix(result)}\n`, + ); + for (const failure of result.attemptFailures ?? []) { + const attemptDuration = + typeof failure.durationMs === 'number' + ? ` (${formatDurationSeconds(failure.durationMs)})` + : ''; + context.writeStdout( + ` attempt ${failure.attempt} failed${attemptDuration}: ${failure.message}\n`, + ); + } + } +} + +function renderFailureDetails( + results: FailedReplayTestResult[], + context: ReplayTestReporterContext, +): void { + if (results.length === 0) return; + context.writeStdout('\n'); + context.writeStdout('Failures:\n'); + for (const result of results) { + context.writeStdout(` ${replayTestDisplayNameWithFile(result)}\n`); + renderReplayFailureBody(result, context, ' '); + } +} + +function renderReplayFailureBody( + result: FailedReplayTestResult, + context: ReplayTestReporterContext, + indent: string, +): void { + context.writeStdout(`${indent}${result.error?.message ?? 'Unknown test failure'}\n`); + for (const line of replayFailureConsoleLines(result)) { + context.writeStdout(`${indent}${line}\n`); + } + if (!context.debug) return; + for (const line of replayTestFailureStepLines(result)) { + context.writeStdout(`${indent}${line}\n`); + } +} + +function formatFlakyReplayDurationSuffix(result: PassedReplayTestResult): string { + const timings = [ + typeof result.finalAttemptDurationMs === 'number' + ? `passed attempt ${formatDurationSeconds(result.finalAttemptDurationMs)}` + : '', + result.durationMs > 0 ? `total ${formatDurationSeconds(result.durationMs)}` : '', + ].filter(Boolean); + return timings.length > 0 ? ` (${timings.join(', ')})` : ''; +} diff --git a/src/cli-test-reporters/format.ts b/src/cli-test-reporters/format.ts new file mode 100644 index 000000000..9bf8cfc1a --- /dev/null +++ b/src/cli-test-reporters/format.ts @@ -0,0 +1,137 @@ +import path from 'node:path'; +import type { ReplaySuiteTestResult } from '../daemon/types.ts'; + +export type PassedReplayTestResult = Extract; +export type FailedReplayTestResult = Extract; +export type ReplayTestError = FailedReplayTestResult['error']; + +export function getReplayTestExitCode(data: { failed: number }): number { + return data.failed > 0 ? 1 : 0; +} + +export function isFailedReplayTestResult( + result: ReplaySuiteTestResult, +): result is FailedReplayTestResult { + return result.status === 'failed'; +} + +export function isFlakyReplayTestResult( + result: ReplaySuiteTestResult, +): result is PassedReplayTestResult { + return result.status === 'passed' && result.attempts > 1; +} + +export function replayTestDisplayNameWithFile(result: ReplaySuiteTestResult): string { + const title = replayTestTitle(result); + const filename = path.basename(result.file); + const base = title && title.length > 0 ? `${JSON.stringify(title)} in ${filename}` : filename; + return `${base}${formatReplayTestShardSuffix(result)}`; +} + +export function replayTestCaseName(result: ReplaySuiteTestResult): string { + return `${replayTestTitle(result) ?? path.basename(result.file)}${formatReplayTestShardSuffix(result)}`; +} + +export function replayArtifactsLine( + result: ReplaySuiteTestResult, + label: 'artifacts' | 'artifactsDir', +): string | undefined { + return 'artifactsDir' in result && result.artifactsDir + ? `${label}: ${result.artifactsDir}` + : undefined; +} + +export function replayErrorHintLine(error: ReplayTestError): string | undefined { + return error.hint ? `hint: ${error.hint}` : undefined; +} + +export function replayErrorDiagnosticLine( + error: ReplayTestError, + label: 'diagnostic' | 'diagnosticId', +): string | undefined { + return error.diagnosticId ? `${label}: ${error.diagnosticId}` : undefined; +} + +export function replayErrorLogLine( + error: ReplayTestError, + label: 'log' | 'logPath', +): string | undefined { + return error.logPath ? `${label}: ${error.logPath}` : undefined; +} + +export function appendReplayErrorMetadata( + lines: string[], + error: ReplayTestError, + options: { includeMessage?: boolean; includeDetails?: boolean; detailsIndent?: number } = {}, +): void { + if (options.includeMessage) lines.push(`errorMessage: ${error.message}`); + appendOptionalLine(lines, replayErrorHintLine(error)); + appendOptionalLine(lines, replayErrorDiagnosticLine(error, 'diagnosticId')); + appendOptionalLine(lines, replayErrorLogLine(error, 'logPath')); + if (options.includeDetails !== false) { + appendReplayErrorDetails(lines, error, options.detailsIndent); + } +} + +export function appendReplayErrorDetails( + lines: string[], + error: ReplayTestError, + detailsIndent?: number, +): void { + const details = error.details ? JSON.stringify(error.details, null, detailsIndent) : undefined; + if (details) lines.push(`details: ${details}`); +} + +export function appendReplayTestShardMetadata( + lines: string[], + result: ReplaySuiteTestResult, +): void { + if (!('shardIndex' in result) || typeof result.shardIndex !== 'number') return; + lines.push(`shardIndex: ${result.shardIndex}`); + appendOptionalLine( + lines, + typeof result.shardCount === 'number' ? `shardCount: ${result.shardCount}` : undefined, + ); + appendOptionalLine( + lines, + typeof result.deviceId === 'string' ? `deviceId: ${result.deviceId}` : undefined, + ); +} + +export function replayTestWarningLines(result: ReplaySuiteTestResult): string[] { + if (result.status !== 'passed') return []; + return (result.warnings ?? []).map((warning) => `warning: ${warning}`); +} + +export function appendOptionalLine(lines: string[], line: string | undefined): void { + if (line) lines.push(line); +} + +export function formatJUnitSeconds(durationMs: number): string { + return (Math.max(0, durationMs) / 1000).toFixed(3); +} + +export function xmlEscape(value: string): string { + return value + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + +export function isDefinedString(value: string | undefined): value is string { + return value !== undefined; +} + +function replayTestTitle(result: ReplaySuiteTestResult): string | undefined { + const title = result.title?.trim(); + return title && title.length > 0 ? title : undefined; +} + +export function formatReplayTestShardSuffix(result: ReplaySuiteTestResult): string { + if (!('shardIndex' in result) || typeof result.shardIndex !== 'number') return ''; + const shardCount = typeof result.shardCount === 'number' ? result.shardCount : '?'; + const device = typeof result.deviceId === 'string' ? ` ${result.deviceId}` : ''; + return ` [shard ${result.shardIndex + 1}/${shardCount}${device}]`; +} diff --git a/src/cli-test-reporters/junit.ts b/src/cli-test-reporters/junit.ts new file mode 100644 index 000000000..b86aba405 --- /dev/null +++ b/src/cli-test-reporters/junit.ts @@ -0,0 +1,133 @@ +import path from 'node:path'; +import type { ReplaySuiteResult, ReplaySuiteTestResult } from '../daemon/types.ts'; +import { AppError } from '../utils/errors.ts'; +import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; +import { + appendOptionalLine, + appendReplayErrorDetails, + appendReplayErrorMetadata, + appendReplayTestShardMetadata, + formatJUnitSeconds, + formatReplayTestShardSuffix, + getReplayTestExitCode, + isFlakyReplayTestResult, + replayArtifactsLine, + replayTestCaseName, + replayTestWarningLines, + xmlEscape, + type FailedReplayTestResult, +} from './format.ts'; + +export function createJunitReplayTestReporter(reportPath: string | undefined): ReplayTestReporter { + const outputPath = readJunitReportPath(reportPath); + return { + name: 'junit', + onSuiteEnd: (suite, context) => writeReplayJunitReport(outputPath, suite, context), + getExitCode: getReplayTestExitCode, + }; +} + +function readJunitReportPath(reportPath: string | undefined): string { + if (reportPath && reportPath.trim().length > 0) return reportPath; + throw new AppError( + 'INVALID_ARGS', + 'The junit test reporter requires an output path. Use --reporter junit:.', + ); +} + +function writeReplayJunitReport( + reportPath: string, + suite: ReplaySuiteResult, + context: ReplayTestReporterContext, +): void { + const directory = path.dirname(reportPath); + try { + context.mkdir(directory); + context.writeFile(reportPath, buildReplayJunitXml(suite)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new AppError( + 'COMMAND_FAILED', + `Failed to write JUnit report to ${reportPath}: ${message}`, + ); + } +} + +function buildReplayJunitXml(suite: ReplaySuiteResult): string { + const lines = [ + '', + ``, + ` `, + ]; + + for (const test of suite.tests) { + lines.push(...renderJUnitTestCase(test)); + } + + lines.push(' '); + lines.push(''); + return `${lines.join('\n')}\n`; +} + +function renderJUnitTestCase(test: ReplaySuiteTestResult): string[] { + const name = xmlEscape(replayTestCaseName(test)); + const className = xmlEscape( + `${path.dirname(test.file) === '.' ? test.file : path.dirname(test.file)}${formatReplayTestShardSuffix(test)}`, + ); + const file = xmlEscape(test.file); + const time = formatJUnitSeconds(test.durationMs); + const lines = [ + ` `, + ]; + + if (test.status === 'failed') { + lines.push( + ` ${xmlEscape(buildFailureDetails(test))}`, + ); + } else if (test.status === 'skipped') { + lines.push(` `); + } + + const systemOut = buildSystemOut(test); + if (systemOut) { + lines.push(` ${xmlEscape(systemOut)}`); + } + + lines.push(' '); + return lines; +} + +function buildFailureDetails(test: FailedReplayTestResult): string { + const lines = [test.error.message]; + appendReplayErrorMetadata(lines, test.error, { includeDetails: false }); + appendOptionalLine(lines, replayArtifactsLine(test, 'artifactsDir')); + appendReplayErrorDetails(lines, test.error, 2); + return lines.join('\n'); +} + +function buildSystemOut(test: ReplaySuiteTestResult): string { + const lines = [`status: ${test.status}`, `durationMs: ${test.durationMs}`]; + appendReplaySystemOutMetadata(lines, test); + return lines.join('\n'); +} + +function appendReplaySystemOutMetadata(lines: string[], test: ReplaySuiteTestResult): void { + appendOptionalLine(lines, 'attempts' in test ? `attempts: ${test.attempts}` : undefined); + appendOptionalLine(lines, 'session' in test ? `session: ${test.session}` : undefined); + appendOptionalLine(lines, 'replayed' in test ? `replayed: ${test.replayed}` : undefined); + appendOptionalLine(lines, 'healed' in test ? `healed: ${test.healed}` : undefined); + for (const warning of replayTestWarningLines(test)) { + lines.push(warning); + } + appendOptionalLine(lines, replayArtifactsLine(test, 'artifactsDir')); + appendReplayTestShardMetadata(lines, test); + if (test.status === 'failed') { + appendReplayFailureSystemOut(lines, test); + } + appendOptionalLine(lines, isFlakyReplayTestResult(test) ? 'flaky: true' : undefined); +} + +function appendReplayFailureSystemOut(lines: string[], test: FailedReplayTestResult): void { + lines.push(`errorCode: ${test.error.code}`); + appendReplayErrorMetadata(lines, test.error, { includeMessage: true }); +} diff --git a/src/cli-test-reporters/registry.ts b/src/cli-test-reporters/registry.ts new file mode 100644 index 000000000..5baf0e2c3 --- /dev/null +++ b/src/cli-test-reporters/registry.ts @@ -0,0 +1,47 @@ +import type { ReplaySuiteResult } from '../daemon/types.ts'; +import { createCustomReplayTestReporter } from './custom.ts'; +import { createDefaultReplayTestReporter } from './default.ts'; +import { getReplayTestExitCode } from './format.ts'; +import { createJunitReplayTestReporter } from './junit.ts'; +import { buildReplayTestReporterSpecs, type ReplayTestReporterSpec } from './spec.ts'; +import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; + +export async function resolveReplayTestReporters(options: { + reporters?: string[]; + reportJunit?: string; + json?: boolean; +}): Promise { + const specs = buildReplayTestReporterSpecs(options); + return await Promise.all(specs.map(resolveReplayTestReporter)); +} + +export async function runReplayTestReporters( + reporters: ReplayTestReporter[], + suite: ReplaySuiteResult, + context: ReplayTestReporterContext, +): Promise { + for (const reporter of reporters) { + await reporter.onSuiteEnd?.(suite, context); + } +} + +export function getReplayTestReporterExitCode( + reporters: ReplayTestReporter[], + suite: ReplaySuiteResult, +): number { + for (const reporter of reporters) { + const exitCode = reporter.getExitCode?.(suite); + if (exitCode !== undefined) return exitCode; + } + return getReplayTestExitCode(suite); +} + +async function resolveReplayTestReporter( + spec: ReplayTestReporterSpec, +): Promise { + if (spec.kind === 'custom') { + return await createCustomReplayTestReporter(spec); + } + if (spec.name === 'default') return createDefaultReplayTestReporter(); + return createJunitReplayTestReporter(spec.outputPath); +} diff --git a/src/cli-test-reporters/spec.ts b/src/cli-test-reporters/spec.ts new file mode 100644 index 000000000..45c3949f4 --- /dev/null +++ b/src/cli-test-reporters/spec.ts @@ -0,0 +1,82 @@ +import { AppError } from '../utils/errors.ts'; + +export type ReplayTestReporterSpec = + | { + kind: 'builtin'; + name: 'default'; + raw: string; + } + | { + kind: 'builtin'; + name: 'junit'; + raw: string; + outputPath?: string; + } + | { + kind: 'custom'; + modulePath: string; + raw: string; + }; + +export function buildReplayTestReporterSpecs(options: { + reporters?: string[]; + reportJunit?: string; + json?: boolean; +}): ReplayTestReporterSpec[] { + const specs = + options.reporters && options.reporters.length > 0 + ? options.reporters.map(parseReplayTestReporterSpec) + : options.json + ? [] + : [parseReplayTestReporterSpec('default')]; + + if (options.reportJunit) { + specs.push(parseReplayTestReporterSpec(`junit:${options.reportJunit}`)); + } + + return specs; +} + +export function parseReplayTestReporterSpec(spec: string): ReplayTestReporterSpec { + const trimmed = spec.trim(); + if (!trimmed) { + throw new AppError('INVALID_ARGS', 'Test reporter spec cannot be empty.'); + } + + if (isCustomReplayTestReporterName(trimmed)) { + return { kind: 'custom', modulePath: trimmed, raw: trimmed }; + } + + const { name, value } = splitReplayTestReporterSpec(trimmed); + if (name === 'default') { + if (value !== undefined) { + throw new AppError('INVALID_ARGS', 'The default test reporter does not accept options.'); + } + return { kind: 'builtin', name, raw: trimmed }; + } + if (name === 'junit') { + return value === undefined + ? { kind: 'builtin', name, raw: trimmed } + : { kind: 'builtin', name, raw: trimmed, outputPath: value }; + } + + throw new AppError( + 'INVALID_ARGS', + `Unknown test reporter "${name}". Built-in reporters: default, junit:. Custom reporters must be file paths.`, + ); +} + +function splitReplayTestReporterSpec(spec: string): { name: string; value?: string } { + const separatorIndex = spec.indexOf(':'); + if (separatorIndex < 0) return { name: spec.trim() }; + return { + name: spec.slice(0, separatorIndex).trim(), + value: spec.slice(separatorIndex + 1), + }; +} + +function isCustomReplayTestReporterName(name: string): boolean { + return ( + name.startsWith('.') || name.startsWith('/') || name.startsWith('~') || name.startsWith('file:') + ); +} diff --git a/src/cli-test-reporters/types.ts b/src/cli-test-reporters/types.ts new file mode 100644 index 000000000..e6753c94e --- /dev/null +++ b/src/cli-test-reporters/types.ts @@ -0,0 +1,29 @@ +import type { RequestProgressEvent } from '../daemon/request-progress.ts'; +import type { ReplaySuiteResult } from '../daemon/types.ts'; + +export type ReplayTestReporterContext = { + debug?: boolean; + writeStdout(text: string): void; + mkdir(path: string): void; + writeFile(path: string, contents: string): void; +}; + +export type ReplayTestReporterLoadContext = { + spec: string; + modulePath: string; +}; + +export type ReplayTestReporter = { + name: string; + /** + * Reserved for live reporter support. The CLI currently invokes reporters after the final + * ReplaySuiteResult is available, so custom reporters should use onSuiteEnd for now. + */ + onProgress?(event: RequestProgressEvent, context: ReplayTestReporterContext): void; + onSuiteEnd?(suite: ReplaySuiteResult, context: ReplayTestReporterContext): Promise | void; + getExitCode?(suite: ReplaySuiteResult): number | undefined; +}; + +export type ReplayTestReporterFactory = ( + context: ReplayTestReporterLoadContext, +) => ReplayTestReporter | Promise; diff --git a/src/cli-test.ts b/src/cli-test.ts index c5eaf39d1..ef4200a14 100644 --- a/src/cli-test.ts +++ b/src/cli-test.ts @@ -1,332 +1,34 @@ import fs from 'node:fs'; -import path from 'node:path'; -import type { ReplaySuiteResult, ReplaySuiteTestResult } from './daemon/types.ts'; -import { replayTestFailureStepLines } from './cli-test-trace.ts'; -import { formatDurationSeconds } from './utils/duration-format.ts'; -import { AppError } from './utils/errors.ts'; -import { colorize, printJson, supportsColor } from './utils/output.ts'; - -type PassedReplayTestResult = Extract; -type FailedReplayTestResult = Extract; -type ReplayTestError = FailedReplayTestResult['error']; - -export function renderReplayTestResponse(options: { +import type { ReplaySuiteResult } from './daemon/types.ts'; +import { + getReplayTestReporterExitCode, + resolveReplayTestReporters, + runReplayTestReporters, +} from './cli-test-reporters/registry.ts'; +import type { ReplayTestReporterContext } from './cli-test-reporters/types.ts'; +import { printJson } from './utils/output.ts'; + +export async function renderReplayTestResponse(options: { suite: ReplaySuiteResult; json?: boolean; debug?: boolean; + reporter?: string[]; reportJunit?: string; -}): number { - const { suite, json, debug, reportJunit } = options; - if (reportJunit) { - writeReplayJunitReport(reportJunit, suite); - } +}): Promise { + const { suite, json, debug, reporter, reportJunit } = options; + const reporters = await resolveReplayTestReporters({ reporters: reporter, reportJunit, json }); + await runReplayTestReporters(reporters, suite, createReplayTestReporterContext({ debug })); if (json) { printJson({ success: true, data: suite }); - return getReplayTestExitCode(suite); - } - return renderReplayTestSummary(suite, { debug }); -} - -function renderReplayTestSummary( - data: ReplaySuiteResult, - options: { debug?: boolean } = {}, -): number { - const flaky = data.tests.filter(isFlakyReplayTestResult); - process.stdout.write(`${formatReplayTestSummaryLine(data, flaky.length)}\n`); - renderFailureDetails(data.tests.filter(isFailedReplayTestResult), { debug: options.debug }); - renderFlakyTestSummary(flaky); - return getReplayTestExitCode(data); -} - -function formatReplayTestSummaryLine(data: ReplaySuiteResult, flakyCount: number): string { - const durationMs = typeof data.durationMs === 'number' ? data.durationMs : undefined; - const flakySuffix = flakyCount > 0 ? `, ${flakyCount} flaky` : ''; - const durationSuffix = durationMs !== undefined ? ` in ${formatDurationSeconds(durationMs)}` : ''; - return `Test summary: ${data.passed} passed, ${data.failed} failed${flakySuffix}${durationSuffix}`; -} - -function replayFlakyStatusIcon(): string { - const useColor = supportsColor(); - return useColor ? colorize('✓', 'yellow') : '✓'; -} - -function isFailedReplayTestResult(result: ReplaySuiteTestResult): result is FailedReplayTestResult { - return result.status === 'failed'; -} - -function replayFailureConsoleLines(result: FailedReplayTestResult): string[] { - return [ - replayErrorHintLine(result.error), - replayArtifactsLine(result, 'artifacts'), - replayErrorLogLine(result.error, 'log'), - replayErrorDiagnosticLine(result.error, 'diagnostic'), - ].filter(isDefinedString); -} - -function isFlakyReplayTestResult(result: ReplaySuiteTestResult): result is PassedReplayTestResult { - return result.status === 'passed' && result.attempts > 1; -} - -function renderFlakyTestSummary(results: PassedReplayTestResult[]): void { - if (results.length === 0) return; - process.stdout.write('\n'); - process.stdout.write('Flaky tests:\n'); - for (const result of results) { - process.stdout.write( - ` ${replayFlakyStatusIcon()} ${replayTestDisplayNameWithFile(result)} after ${result.attempts} attempts${formatFlakyReplayDurationSuffix(result)}\n`, - ); - for (const failure of result.attemptFailures ?? []) { - const attemptDuration = - typeof failure.durationMs === 'number' - ? ` (${formatDurationSeconds(failure.durationMs)})` - : ''; - process.stdout.write( - ` attempt ${failure.attempt} failed${attemptDuration}: ${failure.message}\n`, - ); - } } + return getReplayTestReporterExitCode(reporters, suite); } -function renderFailureDetails( - results: FailedReplayTestResult[], - options: { debug?: boolean } = {}, -): void { - if (results.length === 0) return; - process.stdout.write('\n'); - process.stdout.write('Failures:\n'); - for (const result of results) { - process.stdout.write(` ${replayTestDisplayNameWithFile(result)}\n`); - renderReplayFailureBody(result, { debug: options.debug, indent: ' ' }); - } -} - -function renderReplayFailureBody( - result: FailedReplayTestResult, - options: { debug?: boolean; indent: string }, -): void { - const { debug, indent } = options; - process.stdout.write(`${indent}${result.error?.message ?? 'Unknown test failure'}\n`); - for (const line of replayFailureConsoleLines(result)) { - process.stdout.write(`${indent}${line}\n`); - } - if (!debug) return; - for (const line of replayTestFailureStepLines(result)) { - process.stdout.write(`${indent}${line}\n`); - } -} - -function replayTestDisplayNameWithFile(result: ReplaySuiteTestResult): string { - const title = replayTestTitle(result); - const filename = path.basename(result.file); - const base = title && title.length > 0 ? `${JSON.stringify(title)} in ${filename}` : filename; - return `${base}${formatReplayTestShardSuffix(result)}`; -} - -function replayTestCaseName(result: ReplaySuiteTestResult): string { - return `${replayTestTitle(result) ?? path.basename(result.file)}${formatReplayTestShardSuffix(result)}`; -} - -function replayTestTitle(result: ReplaySuiteTestResult): string | undefined { - const title = result.title?.trim(); - return title && title.length > 0 ? title : undefined; -} - -function formatFlakyReplayDurationSuffix(result: PassedReplayTestResult): string { - const timings = [ - typeof result.finalAttemptDurationMs === 'number' - ? `passed attempt ${formatDurationSeconds(result.finalAttemptDurationMs)}` - : '', - result.durationMs > 0 ? `total ${formatDurationSeconds(result.durationMs)}` : '', - ].filter(Boolean); - return timings.length > 0 ? ` (${timings.join(', ')})` : ''; -} - -function getReplayTestExitCode(data: ReplaySuiteResult): number { - return data.failed > 0 ? 1 : 0; -} - -function writeReplayJunitReport(reportPath: string, suite: ReplaySuiteResult): void { - const directory = path.dirname(reportPath); - try { - fs.mkdirSync(directory, { recursive: true }); - fs.writeFileSync(reportPath, buildReplayJunitXml(suite), 'utf8'); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new AppError( - 'COMMAND_FAILED', - `Failed to write JUnit report to ${reportPath}: ${message}`, - ); - } -} - -function buildReplayJunitXml(suite: ReplaySuiteResult): string { - const lines = [ - '', - ``, - ` `, - ]; - - for (const test of suite.tests) { - lines.push(...renderJUnitTestCase(test)); - } - - lines.push(' '); - lines.push(''); - return `${lines.join('\n')}\n`; -} - -function renderJUnitTestCase(test: ReplaySuiteTestResult): string[] { - const name = xmlEscape(replayTestCaseName(test)); - const className = xmlEscape( - `${path.dirname(test.file) === '.' ? test.file : path.dirname(test.file)}${formatReplayTestShardSuffix(test)}`, - ); - const file = xmlEscape(test.file); - const time = formatJUnitSeconds(test.durationMs); - const lines = [ - ` `, - ]; - - if (test.status === 'failed') { - lines.push( - ` ${xmlEscape(buildFailureDetails(test))}`, - ); - } else if (test.status === 'skipped') { - lines.push(` `); - } - - const systemOut = buildSystemOut(test); - if (systemOut) { - lines.push(` ${xmlEscape(systemOut)}`); - } - - lines.push(' '); - return lines; -} - -function buildFailureDetails(test: FailedReplayTestResult): string { - const lines = [test.error.message]; - appendReplayErrorMetadata(lines, test.error, { includeDetails: false }); - appendOptionalLine(lines, replayArtifactsLine(test, 'artifactsDir')); - appendReplayErrorDetails(lines, test.error, 2); - return lines.join('\n'); -} - -function buildSystemOut(test: ReplaySuiteTestResult): string { - const lines = [`status: ${test.status}`, `durationMs: ${test.durationMs}`]; - appendReplaySystemOutMetadata(lines, test); - return lines.join('\n'); -} - -function appendReplaySystemOutMetadata(lines: string[], test: ReplaySuiteTestResult): void { - appendOptionalLine(lines, 'attempts' in test ? `attempts: ${test.attempts}` : undefined); - appendOptionalLine(lines, 'session' in test ? `session: ${test.session}` : undefined); - appendOptionalLine(lines, 'replayed' in test ? `replayed: ${test.replayed}` : undefined); - appendOptionalLine(lines, 'healed' in test ? `healed: ${test.healed}` : undefined); - for (const warning of replayTestWarningLines(test)) { - lines.push(warning); - } - appendOptionalLine(lines, replayArtifactsLine(test, 'artifactsDir')); - appendReplayTestShardMetadata(lines, test); - if (test.status === 'failed') { - appendReplayFailureSystemOut(lines, test); - } - appendOptionalLine(lines, isFlakyReplayTestResult(test) ? 'flaky: true' : undefined); -} - -function formatReplayTestShardSuffix(result: ReplaySuiteTestResult): string { - if (!('shardIndex' in result) || typeof result.shardIndex !== 'number') return ''; - const shardCount = typeof result.shardCount === 'number' ? result.shardCount : '?'; - const device = typeof result.deviceId === 'string' ? ` ${result.deviceId}` : ''; - return ` [shard ${result.shardIndex + 1}/${shardCount}${device}]`; -} - -function appendReplayTestShardMetadata(lines: string[], result: ReplaySuiteTestResult): void { - if (!('shardIndex' in result) || typeof result.shardIndex !== 'number') return; - lines.push(`shardIndex: ${result.shardIndex}`); - appendOptionalLine( - lines, - typeof result.shardCount === 'number' ? `shardCount: ${result.shardCount}` : undefined, - ); - appendOptionalLine( - lines, - typeof result.deviceId === 'string' ? `deviceId: ${result.deviceId}` : undefined, - ); -} - -function appendReplayFailureSystemOut(lines: string[], test: FailedReplayTestResult): void { - lines.push(`errorCode: ${test.error.code}`); - appendReplayErrorMetadata(lines, test.error, { includeMessage: true }); -} - -function appendReplayErrorMetadata( - lines: string[], - error: ReplayTestError, - options: { includeMessage?: boolean; includeDetails?: boolean; detailsIndent?: number } = {}, -): void { - if (options.includeMessage) lines.push(`errorMessage: ${error.message}`); - appendOptionalLine(lines, replayErrorHintLine(error)); - appendOptionalLine(lines, replayErrorDiagnosticLine(error, 'diagnosticId')); - appendOptionalLine(lines, replayErrorLogLine(error, 'logPath')); - if (options.includeDetails !== false) { - appendReplayErrorDetails(lines, error, options.detailsIndent); - } -} - -function appendReplayErrorDetails( - lines: string[], - error: ReplayTestError, - detailsIndent?: number, -): void { - const details = error.details ? JSON.stringify(error.details, null, detailsIndent) : undefined; - if (details) lines.push(`details: ${details}`); -} - -function replayArtifactsLine( - result: ReplaySuiteTestResult, - label: 'artifacts' | 'artifactsDir', -): string | undefined { - return 'artifactsDir' in result && result.artifactsDir - ? `${label}: ${result.artifactsDir}` - : undefined; -} - -function replayErrorHintLine(error: ReplayTestError): string | undefined { - return error.hint ? `hint: ${error.hint}` : undefined; -} - -function replayErrorDiagnosticLine( - error: ReplayTestError, - label: 'diagnostic' | 'diagnosticId', -): string | undefined { - return error.diagnosticId ? `${label}: ${error.diagnosticId}` : undefined; -} - -function replayErrorLogLine(error: ReplayTestError, label: 'log' | 'logPath'): string | undefined { - return error.logPath ? `${label}: ${error.logPath}` : undefined; -} - -function appendOptionalLine(lines: string[], line: string | undefined): void { - if (line) lines.push(line); -} - -function isDefinedString(value: string | undefined): value is string { - return value !== undefined; -} - -function replayTestWarningLines(result: ReplaySuiteTestResult): string[] { - if (result.status !== 'passed') return []; - return (result.warnings ?? []).map((warning) => `warning: ${warning}`); -} - -function formatJUnitSeconds(durationMs: number): string { - return (Math.max(0, durationMs) / 1000).toFixed(3); -} - -function xmlEscape(value: string): string { - return value - .replaceAll('&', '&') - .replaceAll('<', '<') - .replaceAll('>', '>') - .replaceAll('"', '"') - .replaceAll("'", '''); +function createReplayTestReporterContext(options: { debug?: boolean }): ReplayTestReporterContext { + return { + debug: options.debug, + writeStdout: (text) => process.stdout.write(text), + mkdir: (directory) => fs.mkdirSync(directory, { recursive: true }), + writeFile: (filePath, contents) => fs.writeFileSync(filePath, contents, 'utf8'), + }; } diff --git a/src/cli/commands/generic.ts b/src/cli/commands/generic.ts index c1c573896..82665dd1d 100644 --- a/src/cli/commands/generic.ts +++ b/src/cli/commands/generic.ts @@ -26,7 +26,7 @@ export async function runGenericClientBackedCommand({ if (cliOutput) { writeCliOutput(flags, cliOutput); } else { - const exitCode = writeGenericCliOutput(command, flags, result, { debug }); + const exitCode = await writeGenericCliOutput(command, flags, result, { debug }); if (exitCode !== 0) { process.exit(exitCode); } @@ -39,12 +39,13 @@ function writeGenericCliOutput( flags: CliFlags, data: CommandRequestResult, options: { debug?: boolean } = {}, -): number { +): Promise | number { if (command === 'test') { return renderReplayTestResponse({ suite: data as ReplaySuiteResult, debug: options.debug, json: flags.json, + reporter: flags.reporter, reportJunit: flags.reportJunit, }); } diff --git a/src/client-normalizers.ts b/src/client-normalizers.ts index b9254e971..2751a0455 100644 --- a/src/client-normalizers.ts +++ b/src/client-normalizers.ts @@ -332,7 +332,6 @@ export function buildFlags(options: InternalRequestOptions): CommandFlags { retries: options.retries, recordVideo: options.recordVideo, artifactsDir: options.artifactsDir, - reportJunit: options.reportJunit, shardAll: options.shardAll, shardSplit: options.shardSplit, findFirst: options.findFirst, diff --git a/src/client-types.ts b/src/client-types.ts index 051b66d9b..b4df07af7 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -754,6 +754,7 @@ export type ReplayTestOptions = AgentDeviceRequestOverrides & retries?: number; recordVideo?: boolean; artifactsDir?: string; + /** @deprecated Use the CLI --reporter junit: or --report-junit . */ reportJunit?: string; shardAll?: number; shardSplit?: number; @@ -903,7 +904,6 @@ type CommandExecutionOptions = Partial & { retries?: number; recordVideo?: boolean; artifactsDir?: string; - reportJunit?: string; shardAll?: number; shardSplit?: number; findFirst?: boolean; diff --git a/src/commands/replay/index.test.ts b/src/commands/replay/index.test.ts index bda3c1fda..92b98d9f2 100644 --- a/src/commands/replay/index.test.ts +++ b/src/commands/replay/index.test.ts @@ -77,7 +77,6 @@ describe('replay command interface', () => { retries: 2, recordVideo: true, artifactsDir: './artifacts', - reportJunit: './junit.xml', shardAll: 4, shardSplit: 2, }), @@ -92,7 +91,6 @@ describe('replay command interface', () => { retries: 2, recordVideo: true, artifactsDir: './artifacts', - reportJunit: './junit.xml', shardAll: 4, shardSplit: 2, }); @@ -118,12 +116,18 @@ describe('replay command interface', () => { }, }); - expect(testDaemonWriter({ paths: ['./suite.ad'], maestro: true })).toMatchObject({ + const testRequest = testDaemonWriter({ + paths: ['./suite.ad'], + maestro: true, + reportJunit: './junit.xml', + }); + expect(testRequest).toMatchObject({ command: 'test', positionals: ['./suite.ad'], options: { replayBackend: 'maestro', }, }); + expect(testRequest.options).not.toHaveProperty('reportJunit'); }); }); diff --git a/src/commands/replay/index.ts b/src/commands/replay/index.ts index 09569ba95..bafdd4f0e 100644 --- a/src/commands/replay/index.ts +++ b/src/commands/replay/index.ts @@ -52,7 +52,6 @@ export const testCommandMetadata = defineFieldCommandMetadata( retries: integerField(), recordVideo: booleanField(), artifactsDir: stringField(), - reportJunit: stringField(), shardAll: integerField(), shardSplit: integerField(), }, @@ -92,6 +91,7 @@ const testCliSchema = { 'retries', 'recordVideo', 'artifactsDir', + 'reporter', 'reportJunit', 'shardAll', 'shardSplit', @@ -117,7 +117,6 @@ export const testCliReader: CliReader = (positionals, flags) => ({ retries: flags.retries, recordVideo: flags.recordVideo, artifactsDir: flags.artifactsDir, - reportJunit: flags.reportJunit, shardAll: flags.shardAll, shardSplit: flags.shardSplit, }); @@ -133,7 +132,7 @@ export const replayDaemonWriter: DaemonWriter = (input) => export const testDaemonWriter: DaemonWriter = (input) => request(TEST_COMMAND_NAME, input.paths ?? [], { - ...input, + ...stripReplayTestPresentationInput(input), replayUpdate: input.update, replayBackend: readReplayBackend(input), replayEnv: input.env, @@ -167,6 +166,13 @@ function readReplayBackend(input: CommandInput): string | undefined { return input.backend ?? (input.maestro === true ? 'maestro' : undefined); } +function stripReplayTestPresentationInput(input: CommandInput): CommandInput { + const daemonInput = { ...input }; + delete daemonInput.reporter; + delete daemonInput.reportJunit; + return daemonInput; +} + function collectReplayClientShellEnv(env: NodeJS.ProcessEnv): Record { const result: Record = {}; for (const [key, value] of Object.entries(env)) { diff --git a/src/index.ts b/src/index.ts index 9152c52f5..79ad42368 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,6 +20,13 @@ export type { export type { AppErrorCode, NormalizedError } from './utils/errors.ts'; +export type { + ReplayTestReporter, + ReplayTestReporterContext, + ReplayTestReporterFactory, + ReplayTestReporterLoadContext, +} from './cli-test-reporters/types.ts'; + export type { CommandResult } from './core/command-descriptor/command-result.ts'; export type { BootCommandResult, ShutdownCommandResult } from './contracts/device.ts'; export type { ViewportCommandResult } from './contracts/viewport.ts'; diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index f75b36c61..1c92fa5bd 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -147,6 +147,10 @@ test('parseArgs recognizes command-specific flag combinations', async () => { '2', '--artifacts-dir', '.agent-device/test-artifacts', + '--reporter', + 'default', + '--reporter', + 'junit:.agent-device/test-artifacts/junit.xml', ], strictFlags: true, assertParsed: (parsed) => { @@ -158,6 +162,10 @@ test('parseArgs recognizes command-specific flag combinations', async () => { assert.equal(parsed.flags.timeoutMs, 60000); assert.equal(parsed.flags.retries, 2); assert.equal(parsed.flags.artifactsDir, '.agent-device/test-artifacts'); + assert.deepEqual(parsed.flags.reporter, [ + 'default', + 'junit:.agent-device/test-artifacts/junit.xml', + ]); }, }, { @@ -2063,6 +2071,10 @@ test('command usage describes test suite flags', () => { assert.match(help, /--retries /); assert.match(help, /--record-video/); assert.match(help, /--artifacts-dir /); + assert.match(help, /--reporter /); + assert.match(help, /custom reporter path/); + assert.match(help, /--report-junit /); + assert.match(help, /compatibility alias for --reporter junit:/); assert.doesNotMatch(help, /test --verbose prints per-test step timings without debug logs/); }); diff --git a/src/utils/cli-flags.ts b/src/utils/cli-flags.ts index a1389b74e..21e827477 100644 --- a/src/utils/cli-flags.ts +++ b/src/utils/cli-flags.ts @@ -119,6 +119,7 @@ export type CliFlags = RemoteConfigMetroOptions & retries?: number; recordVideo?: boolean; artifactsDir?: string; + reporter?: string[]; reportJunit?: string; shardAll?: number; shardSplit?: number; @@ -942,12 +943,21 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageLabel: '--artifacts-dir ', usageDescription: 'Test: root directory for suite artifacts', }, + { + key: 'reporter', + names: ['--reporter'], + type: 'string', + multiple: true, + usageLabel: '--reporter ', + usageDescription: + 'Test: add a replay suite reporter; use default, junit:, or a custom reporter path (repeatable)', + }, { key: 'reportJunit', names: ['--report-junit'], type: 'string', usageLabel: '--report-junit ', - usageDescription: 'Test: write a JUnit XML report for the replay suite', + usageDescription: 'Test: compatibility alias for --reporter junit:', }, { key: 'shardAll', diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index c9b62802e..859d8a0b1 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -87,6 +87,7 @@ agent-device test ./workflows agent-device test "./workflows/**/*.ad" --platform android agent-device test ./workflows --timeout 60000 --retries 1 agent-device test ./workflows --artifacts-dir ./tmp/agent-device-artifacts +agent-device test ./workflows --reporter default --reporter junit:./tmp/junit.xml ``` - `test` discovers `.ad` files from files, directories, or globs and runs them serially. @@ -97,8 +98,65 @@ agent-device test ./workflows --artifacts-dir ./tmp/agent-device-artifacts - `replay-timing.ndjson` records attempt, cleanup, and per-step start/stop events with durations. Upload it from CI even for passing runs when comparing local and CI performance. - Timeouts are cooperative: the runner marks the attempt failed at the timeout boundary, then gives the underlying replay a short grace period to stop before session cleanup. - The default text reporter streams one-line `pass`, `fail`, or `skip` progress on stderr as each suite entry finishes or retries. Each line includes current/total suite position and elapsed seconds such as `pass 3/6 ... duration=12.34s`, then the final summary prints failed tests and passed-on-retry flaky tests; use `--verbose` to print every final result. +- `--reporter` is repeatable. Built-ins are `default` for the console summary and `junit:` for JUnit XML. Passing any explicit reporter list replaces the implicit default reporter, so include `--reporter default` when you also want terminal output. `--report-junit ` remains a compatibility alias for `--reporter junit:`. - When `--fail-fast` and retries are both set, the current test still consumes its retries before the suite stops. +### Custom test reporters + +Custom reporters are CLI-only presentation adapters. The daemon still returns the structured replay suite result; reporters run in the local CLI process after the suite finishes. + +```bash +agent-device test ./workflows --reporter ./scripts/replay-reporter.mjs +``` + +Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive load context. Reporter hooks receive an IO context with `writeStdout`, `mkdir`, and `writeFile` helpers: + +```js +// scripts/replay-reporter.mjs +export default function createReporter(loadContext) { + return { + name: 'summary-file', + onSuiteEnd(suite, context) { + context.writeFile( + './tmp/report.txt', + JSON.stringify( + { + total: suite.total, + passed: suite.passed, + failed: suite.failed, + modulePath: loadContext.modulePath, + }, + null, + 2, + ), + ); + }, + getExitCode(suite) { + return suite.failed > 0 ? 1 : 0; + }, + }; +} +``` + +TypeScript reporters can import the public types from `agent-device`: + +```ts +import type { ReplayTestReporterFactory } from 'agent-device'; + +const createReporter: ReplayTestReporterFactory = () => ({ + name: 'typed-reporter', + onSuiteEnd(suite) { + // Write artifacts, annotations, or summaries from suite. + }, +}); + +export default createReporter; +``` + +The CLI loads reporter modules with Node dynamic `import()`. Use `.mjs` or `.js` files at runtime; for TypeScript, compile the reporter to JavaScript before passing it to `--reporter`. Loading `.ts` files directly depends on Node's type-stripping behavior and is not part of the supported reporter contract. + +The supported hook today is final-result reporting through `onSuiteEnd`. `getExitCode` can override whether the finished suite exits successfully; when no reporter supplies one, failed tests exit with `1`. The `onProgress` hook is part of the reporter interface for live reporters, but the CLI currently invokes reporters after the suite result is available. + ## Parametrise `.ad` scripts Substitute `${VAR}` tokens in `.ad` scripts using values from the CLI, shell env, script-local `env` directives, or built-ins. @@ -115,12 +173,12 @@ click "label=${APP_ID}" ### Precedence -| Source | Priority | Example | -|---|---|---| -| CLI `-e KEY=VALUE` | highest | `agent-device test flow.ad -e APP_ID=demo` | -| Shell env prefixed `AD_VAR_` | | `AD_VAR_APP_ID=demo agent-device test flow.ad` (imported as `APP_ID`) | -| Script `env KEY=VALUE` | | `env APP_ID=settings` in header | -| Built-ins | runtime | `AD_PLATFORM`, `AD_SESSION`, `AD_FILENAME`, `AD_DEVICE`, `AD_ARTIFACTS` | +| Source | Priority | Example | +| ---------------------------- | -------- | ----------------------------------------------------------------------- | +| CLI `-e KEY=VALUE` | highest | `agent-device test flow.ad -e APP_ID=demo` | +| Shell env prefixed `AD_VAR_` | | `AD_VAR_APP_ID=demo agent-device test flow.ad` (imported as `APP_ID`) | +| Script `env KEY=VALUE` | | `env APP_ID=settings` in header | +| Built-ins | runtime | `AD_PLATFORM`, `AD_SESSION`, `AD_FILENAME`, `AD_DEVICE`, `AD_ARTIFACTS` | ### Built-ins