feat: add replay test reporters

scripts/integration-progress-model.ts

@@ -276,6 +276,7 @@ function summarizeProviderScenarioFlagExclusions() {
         'snapshotForceFull',
         'baseline',
         'threshold',
+        'reporter',
         'reportJunit',
         'replayMaestro',
         'replayExportFormat',

src/__tests__/cli-network.test.ts

@@ -664,6 +664,7 @@ test('test command writes JUnit report with failure metadata', async () => {
     );
 
     assert.equal(result.code, 1);
+    assert.equal(result.calls[0]?.flags?.reportJunit, undefined);
     const xml = await fs.readFile(reportPath, 'utf8');
     assert.match(
       xml,
@@ -689,3 +690,60 @@ test('test command writes JUnit report with failure metadata', async () => {
     await fs.rm(tmpDir, { recursive: true, force: true });
   }
 });
+
+test('test command supports explicit reporter lists', async () => {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-reporter-test-'));
+  const reportPath = path.join(tmpDir, 'replays.junit.xml');
+
+  try {
+    const result = await runCliCapture(
+      ['test', './suite', '--reporter', `junit:${reportPath}`],
+      async () => makeReplaySuiteResponse(),
+    );
+
+    assert.equal(result.code, 1);
+    assert.doesNotMatch(result.stdout, /Test summary:/);
+    const xml = await fs.readFile(reportPath, 'utf8');
+    assert.match(xml, /<testsuite name="agent-device replay suite" tests="3" failures="1"/);
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
+});
+
+test('test command loads custom reporter modules', async () => {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-custom-reporter-test-'));
+  const reporterPath = path.join(tmpDir, 'custom-reporter.mjs');
+  const outputPath = path.join(tmpDir, 'custom-report.json');
+
+  try {
+    await fs.writeFile(
+      reporterPath,
+      [
+        "import fs from 'node:fs';",
+        'export default function createReporter() {',
+        '  return {',
+        "    name: 'custom-file',",
+        '    onSuiteEnd(suite) {',
+        `      fs.writeFileSync(${JSON.stringify(outputPath)}, JSON.stringify({ total: suite.total, failed: suite.failed }), "utf8");`,
+        '    },',
+        '    getExitCode() { return 0; },',
+        '  };',
+        '}',
+      ].join('\n'),
+      'utf8',
+    );
+
+    const result = await runCliCapture(['test', './suite', '--reporter', reporterPath], async () =>
+      makeReplaySuiteResponse(),
+    );
+
+    assert.equal(result.code, null);
+    assert.doesNotMatch(result.stdout, /Test summary:/);
+    assert.deepEqual(JSON.parse(await fs.readFile(outputPath, 'utf8')), {
+      total: 3,
+      failed: 1,
+    });
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
+});

src/__tests__/cli-test-reporters-spec.test.ts

@@ -0,0 +1,46 @@
+import assert from 'node:assert/strict';
+import { test } from 'vitest';
+import {
+  buildReplayTestReporterSpecs,
+  parseReplayTestReporterSpec,
+} from '../cli-test-reporters/spec.ts';
+
+test('parses built-in reporter shorthand specs', () => {
+  assert.deepEqual(parseReplayTestReporterSpec('default'), {
+    kind: 'builtin',
+    name: 'default',
+    raw: 'default',
+  });
+  assert.deepEqual(parseReplayTestReporterSpec('junit:./report.xml'), {
+    kind: 'builtin',
+    name: 'junit',
+    raw: 'junit:./report.xml',
+    outputPath: './report.xml',
+  });
+});
+
+test('parses custom reporter paths', () => {
+  assert.deepEqual(parseReplayTestReporterSpec('./reporter.mjs'), {
+    kind: 'custom',
+    modulePath: './reporter.mjs',
+    raw: './reporter.mjs',
+  });
+});
+
+test('expands implicit and compatibility reporter defaults', () => {
+  assert.deepEqual(buildReplayTestReporterSpecs({}), [
+    { kind: 'builtin', name: 'default', raw: 'default' },
+  ]);
+  assert.deepEqual(buildReplayTestReporterSpecs({ json: true, reportJunit: './report.xml' }), [
+    {
+      kind: 'builtin',
+      name: 'junit',
+      raw: 'junit:./report.xml',
+      outputPath: './report.xml',
+    },
+  ]);
+});
+
+test('rejects invalid reporter specs', () => {
+  assert.throws(() => parseReplayTestReporterSpec('unknown'), /Unknown test reporter/);
+});

src/cli-test-reporters/custom.ts

@@ -0,0 +1,89 @@
+import os from 'node:os';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { AppError } from '../utils/errors.ts';
+import type { ReplayTestReporter, ReplayTestReporterFactory } from './types.ts';
+import type { ReplayTestReporterSpec } from './spec.ts';
+
+type CustomReporterModule = {
+  default?: unknown;
+  createReporter?: unknown;
+  reporter?: unknown;
+};
+
+const OPTIONAL_REPORTER_HOOKS = [
+  'onProgress',
+  'onSuiteEnd',
+  'getExitCode',
+] as const satisfies readonly (keyof ReplayTestReporter)[];
+
+export async function createCustomReplayTestReporter(
+  spec: Extract<ReplayTestReporterSpec, { kind: 'custom' }>,
+): Promise<ReplayTestReporter> {
+  const modulePath = resolveCustomReporterModulePath(spec.modulePath);
+  const module = await importCustomReporterModule(modulePath);
+  const factory = readCustomReporterFactory(module, spec.modulePath);
+  const reporter = await factory({ spec: spec.raw, modulePath });
+  return validateCustomReplayTestReporter(reporter, spec.modulePath);
+}
+
+function readCustomReporterFactory(
+  module: CustomReporterModule,
+  modulePath: string,
+): ReplayTestReporterFactory {
+  const exported = module.createReporter ?? module.default ?? module.reporter;
+  if (!exported) {
+    throw new AppError(
+      'INVALID_ARGS',
+      `Custom test reporter ${modulePath} must export default, createReporter, or reporter.`,
+    );
+  }
+  return typeof exported === 'function'
+    ? (exported as ReplayTestReporterFactory)
+    : () => exported as ReplayTestReporter;
+}
+
+function resolveCustomReporterModulePath(modulePath: string): string {
+  if (modulePath.startsWith('file:')) return modulePath;
+  const expandedPath = modulePath.startsWith('~/')
+    ? path.join(os.homedir(), modulePath.slice(2))
+    : modulePath;
+  return path.resolve(process.cwd(), expandedPath);
+}
+
+async function importCustomReporterModule(modulePath: string): Promise<CustomReporterModule> {
+  try {
+    const href = modulePath.startsWith('file:') ? modulePath : pathToFileURL(modulePath).href;
+    return (await import(href)) as CustomReporterModule;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new AppError(
+      'INVALID_ARGS',
+      `Failed to load custom test reporter ${modulePath}: ${message}`,
+    );
+  }
+}
+
+function validateCustomReplayTestReporter(
+  reporter: unknown,
+  modulePath: string,
+): ReplayTestReporter {
+  if (!reporter || typeof reporter !== 'object') {
+    throw new AppError(
+      'INVALID_ARGS',
+      `Custom test reporter ${modulePath} must export a reporter object or factory.`,
+    );
+  }
+  const candidate = reporter as Partial<ReplayTestReporter>;
+  if (typeof candidate.name !== 'string' || candidate.name.trim().length === 0) {
+    throw new AppError('INVALID_ARGS', `Custom test reporter ${modulePath} must define name.`);
+  }
+  for (const hook of OPTIONAL_REPORTER_HOOKS) {
+    if (candidate[hook] === undefined || typeof candidate[hook] === 'function') continue;
+    throw new AppError(
+      'INVALID_ARGS',
+      `Custom test reporter ${modulePath} ${hook} must be a function.`,
+    );
+  }
+  return candidate as ReplayTestReporter;
+}

src/cli-test-reporters/default.ts

@@ -0,0 +1,118 @@
+import type { ReplaySuiteResult } from '../daemon/types.ts';
+import { replayTestFailureStepLines } from '../cli-test-trace.ts';
+import { formatDurationSeconds } from '../utils/duration-format.ts';
+import { colorize, supportsColor } from '../utils/output.ts';
+import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts';
+import {
+  getReplayTestExitCode,
+  isDefinedString,
+  isFailedReplayTestResult,
+  isFlakyReplayTestResult,
+  replayArtifactsLine,
+  replayErrorDiagnosticLine,
+  replayErrorHintLine,
+  replayErrorLogLine,
+  replayTestDisplayNameWithFile,
+  type FailedReplayTestResult,
+  type PassedReplayTestResult,
+} from './format.ts';
+
+export function createDefaultReplayTestReporter(): ReplayTestReporter {
+  return {
+    name: 'default',
+    onSuiteEnd: (suite, context) => renderReplayTestSummary(suite, context),
+    getExitCode: getReplayTestExitCode,
+  };
+}
+
+function renderReplayTestSummary(
+  data: ReplaySuiteResult,
+  context: ReplayTestReporterContext,
+): void {
+  const flaky = data.tests.filter(isFlakyReplayTestResult);
+  context.writeStdout(`${formatReplayTestSummaryLine(data, flaky.length)}\n`);
+  renderFailureDetails(data.tests.filter(isFailedReplayTestResult), context);
+  renderFlakyTestSummary(flaky, context);
+}
+
+function formatReplayTestSummaryLine(data: ReplaySuiteResult, flakyCount: number): string {
+  const durationMs = typeof data.durationMs === 'number' ? data.durationMs : undefined;
+  const flakySuffix = flakyCount > 0 ? `, ${flakyCount} flaky` : '';
+  const durationSuffix = durationMs !== undefined ? ` in ${formatDurationSeconds(durationMs)}` : '';
+  return `Test summary: ${data.passed} passed, ${data.failed} failed${flakySuffix}${durationSuffix}`;
+}
+
+function replayFlakyStatusIcon(): string {
+  const useColor = supportsColor();
+  return useColor ? colorize('✓', 'yellow') : '✓';
+}
+
+function replayFailureConsoleLines(result: FailedReplayTestResult): string[] {
+  return [
+    replayErrorHintLine(result.error),
+    replayArtifactsLine(result, 'artifacts'),
+    replayErrorLogLine(result.error, 'log'),
+    replayErrorDiagnosticLine(result.error, 'diagnostic'),
+  ].filter(isDefinedString);
+}
+
+function renderFlakyTestSummary(
+  results: PassedReplayTestResult[],
+  context: ReplayTestReporterContext,
+): void {
+  if (results.length === 0) return;
+  context.writeStdout('\n');
+  context.writeStdout('Flaky tests:\n');
+  for (const result of results) {
+    context.writeStdout(
+      `  ${replayFlakyStatusIcon()} ${replayTestDisplayNameWithFile(result)} after ${result.attempts} attempts${formatFlakyReplayDurationSuffix(result)}\n`,
+    );
+    for (const failure of result.attemptFailures ?? []) {
+      const attemptDuration =
+        typeof failure.durationMs === 'number'
+          ? ` (${formatDurationSeconds(failure.durationMs)})`
+          : '';
+      context.writeStdout(
+        `    attempt ${failure.attempt} failed${attemptDuration}: ${failure.message}\n`,
+      );
+    }
+  }
+}
+
+function renderFailureDetails(
+  results: FailedReplayTestResult[],
+  context: ReplayTestReporterContext,
+): void {
+  if (results.length === 0) return;
+  context.writeStdout('\n');
+  context.writeStdout('Failures:\n');
+  for (const result of results) {
+    context.writeStdout(`  ${replayTestDisplayNameWithFile(result)}\n`);
+    renderReplayFailureBody(result, context, '    ');
+  }
+}
+
+function renderReplayFailureBody(
+  result: FailedReplayTestResult,
+  context: ReplayTestReporterContext,
+  indent: string,
+): void {
+  context.writeStdout(`${indent}${result.error?.message ?? 'Unknown test failure'}\n`);
+  for (const line of replayFailureConsoleLines(result)) {
+    context.writeStdout(`${indent}${line}\n`);
+  }
+  if (!context.debug) return;
+  for (const line of replayTestFailureStepLines(result)) {
+    context.writeStdout(`${indent}${line}\n`);
+  }
+}
+
+function formatFlakyReplayDurationSuffix(result: PassedReplayTestResult): string {
+  const timings = [
+    typeof result.finalAttemptDurationMs === 'number'
+      ? `passed attempt ${formatDurationSeconds(result.finalAttemptDurationMs)}`
+      : '',
+    result.durationMs > 0 ? `total ${formatDurationSeconds(result.durationMs)}` : '',
+  ].filter(Boolean);
+  return timings.length > 0 ? ` (${timings.join(', ')})` : '';
+}