diff --git a/examples/test-app/README.md b/examples/test-app/README.md index c3e240fd8..b09c35ff2 100644 --- a/examples/test-app/README.md +++ b/examples/test-app/README.md @@ -127,18 +127,15 @@ server from its launcher. The Maestro prototype suite lives in `examples/test-app/maestro` and runs through `agent-device replay --maestro`: -```bash -pnpm test-app:maestro:ios -- --open "Agent Device Tester" -pnpm test-app:maestro:android -- --open "Agent Device Tester" -``` - -When the development build is already open and connected to Metro, omit -`--open` and run the suite against the existing session: - ```bash pnpm test-app:maestro:ios +pnpm test-app:maestro:android ``` +The Maestro flow includes `launchApp`, so the suite launches the app inside each +test attempt. Start Metro first when the installed development build needs the +local bundle. + The suite intentionally covers the compat layer syntax used by public Maestro suites: `runFlow` file/inline blocks, `when.platform`, config hooks, deterministic `repeat.times`, flow `env`, selectors, input, assertions, and swipe. diff --git a/examples/test-app/app.json b/examples/test-app/app.json index 2cef4eae3..743ad9951 100644 --- a/examples/test-app/app.json +++ b/examples/test-app/app.json @@ -5,6 +5,9 @@ "version": "1.0.0", "orientation": "default", "userInterfaceStyle": "automatic", + "buildCacheProvider": { + "plugin": "expo-build-disk-cache" + }, "plugins": ["expo-router"], "ios": { "supportsTablet": true, diff --git a/examples/test-app/maestro/checkout-form.yaml b/examples/test-app/maestro/checkout-form.yaml index efb813490..8a22a349b 100644 --- a/examples/test-app/maestro/checkout-form.yaml +++ b/examples/test-app/maestro/checkout-form.yaml @@ -2,8 +2,10 @@ appId: com.callstack.agentdevicelab env: CHECKOUT_NAME: Ada Lovelace CHECKOUT_EMAIL: ada@example.com - PICKUP_TAPS: "2" + PICKUP_TAPS: '2' onFlowStart: + - launchApp: + clearState: true - assertVisible: Agent Device Tester onFlowComplete: - assertVisible: Delivery choices diff --git a/examples/test-app/maestro/helpers/open-checkout-form.yaml b/examples/test-app/maestro/helpers/open-checkout-form.yaml index 58298297a..48427d7bc 100644 --- a/examples/test-app/maestro/helpers/open-checkout-form.yaml +++ b/examples/test-app/maestro/helpers/open-checkout-form.yaml @@ -1,4 +1,7 @@ --- +- scrollUntilVisible: + element: + id: home-open-form - tapOn: id: home-open-form - assertVisible: Checkout form diff --git a/examples/test-app/package.json b/examples/test-app/package.json index 97ca6028a..1d02a4e68 100644 --- a/examples/test-app/package.json +++ b/examples/test-app/package.json @@ -26,6 +26,7 @@ }, "devDependencies": { "@types/react": "~19.2.2", + "expo-build-disk-cache": "^0.7.4", "typescript": "~6.0.3" } } diff --git a/examples/test-app/pnpm-lock.yaml b/examples/test-app/pnpm-lock.yaml index e11a31e32..ef374bb25 100644 --- a/examples/test-app/pnpm-lock.yaml +++ b/examples/test-app/pnpm-lock.yaml @@ -59,6 +59,9 @@ importers: '@types/react': specifier: ~19.2.2 version: 19.2.14 + expo-build-disk-cache: + specifier: ^0.7.4 + version: 0.7.4(@expo/cli@56.1.16(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-constants@56.0.18)(expo-font@56.0.7)(expo-router@56.2.11)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3))(@expo/config@56.0.9(typescript@6.0.3))(typescript@6.0.3) typescript: specifier: ~6.0.3 version: 6.0.3 @@ -1216,6 +1219,10 @@ packages: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} + callsites@3.1.0: + resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} + engines: {node: '>=6'} + camelcase@6.3.0: resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} engines: {node: '>=10'} @@ -1314,6 +1321,15 @@ packages: core-js-compat@3.49.0: resolution: {integrity: sha512-VQXt1jr9cBz03b331DFDCCP90b3fanciLkgiOoy8SBHy06gNf+vQ1A3WFLqG7I8TipYIKeYK9wxd0tUrvHcOZA==} + cosmiconfig@9.0.2: + resolution: {integrity: sha512-gtTZxTDau1wL7Y7zifc2dd8jHSK/k6BTx/2Xp/BpdlAdnlYWFVt7qhJqgwi7637yRwRQ3qL4ZidbB4I8tA5VOg==} + engines: {node: '>=14'} + peerDependencies: + typescript: '>=4.9.5' + peerDependenciesMeta: + typescript: + optional: true + cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -1405,6 +1421,13 @@ packages: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} + env-paths@2.2.1: + resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} + engines: {node: '>=6'} + + error-ex@1.3.4: + resolution: {integrity: sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==} + error-stack-parser@2.1.4: resolution: {integrity: sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ==} @@ -1438,6 +1461,13 @@ packages: react: '*' react-native: '*' + expo-build-disk-cache@0.7.4: + resolution: {integrity: sha512-mU1NaPC2kAwUZ0bVjbuRHQ+FBKnlFvNwu6BR1ZKPRV9c7McvL7Ir1V5B08VvzTDtEMpzYjhyITPSU6loO9Bsew==} + engines: {node: '>=18.0.0'} + peerDependencies: + '@expo/cli': '*' + '@expo/config': '*' + expo-constants@56.0.18: resolution: {integrity: sha512-8AMtbDGl/WVPnWlmbpGmvcdnNCy9E4PFnwdVwj600vljkMDPSxcAcjw8GVXEPk3PpZ+ngTqsrkltWyj0UKYAxw==} peerDependencies: @@ -1735,6 +1765,10 @@ packages: engines: {node: '>=16.x'} hasBin: true + import-fresh@3.3.1: + resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} + engines: {node: '>=6'} + indent-string@4.0.0: resolution: {integrity: sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==} engines: {node: '>=8'} @@ -1745,6 +1779,9 @@ packages: invariant@2.2.4: resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} + is-arrayish@0.2.1: + resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} + is-arrayish@0.3.4: resolution: {integrity: sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==} @@ -1806,6 +1843,9 @@ packages: engines: {node: '>=6'} hasBin: true + json-parse-even-better-errors@2.3.1: + resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} + json5@2.2.3: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} engines: {node: '>=6'} @@ -1900,6 +1940,9 @@ packages: resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} engines: {node: '>= 12.0.0'} + lines-and-columns@1.2.4: + resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + lodash.debounce@4.0.8: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} @@ -2058,6 +2101,11 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + nanoid@3.3.15: + resolution: {integrity: sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + negotiator@0.6.3: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} @@ -2115,6 +2163,14 @@ packages: resolution: {integrity: sha512-eNwHudNbO1folBP3JsZ19v9azXWtQZjICdr3Q0TDPIaeBQ3mXLrh54wM+er0+hSp+dWKf+Z8KM58CYzEyIYxYg==} engines: {node: '>=6'} + parent-module@1.0.1: + resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} + engines: {node: '>=6'} + + parse-json@5.2.0: + resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} + engines: {node: '>=8'} + parse-png@2.1.0: resolution: {integrity: sha512-Nt/a5SfCLiTnQAjx3fHlqp8hRgTL3z7kTQZzvIMS9uCAepnCyjpdEc6M/sz69WqMBdaDBw9sF1F1UaHROYzGkQ==} engines: {node: '>=10'} @@ -2343,6 +2399,10 @@ packages: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} + resolve-from@4.0.0: + resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} + engines: {node: '>=4'} + resolve-from@5.0.0: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} @@ -2727,6 +2787,9 @@ packages: zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zod@4.4.3: + resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} + snapshots: '@adobe/css-tools@4.5.0': {} @@ -4233,6 +4296,8 @@ snapshots: bytes@3.1.2: {} + callsites@3.1.0: {} + camelcase@6.3.0: {} caniuse-lite@1.0.30001787: {} @@ -4346,6 +4411,15 @@ snapshots: dependencies: browserslist: 4.28.2 + cosmiconfig@9.0.2(typescript@6.0.3): + dependencies: + env-paths: 2.2.1 + import-fresh: 3.3.1 + js-yaml: 4.1.1 + parse-json: 5.2.0 + optionalDependencies: + typescript: 6.0.3 + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -4402,6 +4476,12 @@ snapshots: encodeurl@2.0.0: {} + env-paths@2.2.1: {} + + error-ex@1.3.4: + dependencies: + is-arrayish: 0.2.1 + error-stack-parser@2.1.4: dependencies: stackframe: 1.3.4 @@ -4429,6 +4509,16 @@ snapshots: - supports-color - typescript + expo-build-disk-cache@0.7.4(@expo/cli@56.1.16(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-constants@56.0.18)(expo-font@56.0.7)(expo-router@56.2.11)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3))(@expo/config@56.0.9(typescript@6.0.3))(typescript@6.0.3): + dependencies: + '@expo/cli': 56.1.16(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-constants@56.0.18)(expo-font@56.0.7)(expo-router@56.2.11)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + '@expo/config': 56.0.9(typescript@6.0.3) + cosmiconfig: 9.0.2(typescript@6.0.3) + env-paths: 2.2.1 + zod: 4.4.3 + transitivePeerDependencies: + - typescript + expo-constants@56.0.18(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)): dependencies: '@expo/env': 2.3.0 @@ -4755,6 +4845,11 @@ snapshots: dependencies: queue: 6.0.2 + import-fresh@3.3.1: + dependencies: + parent-module: 1.0.1 + resolve-from: 4.0.0 + indent-string@4.0.0: {} inherits@2.0.4: {} @@ -4763,6 +4858,8 @@ snapshots: dependencies: loose-envify: 1.4.0 + is-arrayish@0.2.1: {} + is-arrayish@0.3.4: {} is-core-module@2.16.1: @@ -4820,6 +4917,8 @@ snapshots: jsesc@3.1.0: {} + json-parse-even-better-errors@2.3.1: {} + json5@2.2.3: {} kleur@3.0.3: {} @@ -4884,6 +4983,8 @@ snapshots: lightningcss-win32-arm64-msvc: 1.32.0 lightningcss-win32-x64-msvc: 1.32.0 + lines-and-columns@1.2.4: {} + lodash.debounce@4.0.8: {} lodash.throttle@4.1.1: {} @@ -5129,6 +5230,8 @@ snapshots: nanoid@3.3.11: {} + nanoid@3.3.15: {} + negotiator@0.6.3: {} negotiator@0.6.4: {} @@ -5182,6 +5285,17 @@ snapshots: strip-ansi: 5.2.0 wcwidth: 1.0.1 + parent-module@1.0.1: + dependencies: + callsites: 3.1.0 + + parse-json@5.2.0: + dependencies: + '@babel/code-frame': 7.29.0 + error-ex: 1.3.4 + json-parse-even-better-errors: 2.3.1 + lines-and-columns: 1.2.4 + parse-png@2.1.0: dependencies: pngjs: 3.4.0 @@ -5213,7 +5327,7 @@ snapshots: postcss@8.5.12: dependencies: - nanoid: 3.3.11 + nanoid: 3.3.15 picocolors: 1.1.1 source-map-js: 1.2.1 @@ -5447,6 +5561,8 @@ snapshots: require-directory@2.1.1: {} + resolve-from@4.0.0: {} + resolve-from@5.0.0: {} resolve-workspace-root@2.0.1: {} @@ -5757,3 +5873,5 @@ snapshots: yargs-parser: 21.1.1 zod@3.25.76: {} + + zod@4.4.3: {} diff --git a/scripts/run-test-app-maestro-suite.mjs b/scripts/run-test-app-maestro-suite.mjs index f0268e00e..e2b479607 100644 --- a/scripts/run-test-app-maestro-suite.mjs +++ b/scripts/run-test-app-maestro-suite.mjs @@ -11,7 +11,6 @@ const options = { platform: 'ios', session: 'test-app-maestro', flowDir: path.join(repoRoot, 'examples', 'test-app', 'maestro'), - openTarget: '', close: false, passthrough: [], }; @@ -37,11 +36,6 @@ for (let index = 2; index < process.argv.length; index += 1) { index += 1; continue; } - if (arg === '--open' && process.argv[index + 1]) { - options.openTarget = process.argv[index + 1]; - index += 1; - continue; - } if (arg === '--close') { options.close = true; continue; @@ -67,14 +61,9 @@ function runAgentDevice(args) { }); } -if (options.openTarget) { - runAgentDevice(['open', options.openTarget, '--platform', options.platform, ...options.passthrough]); - runAgentDevice(['wait', 'Agent Device Tester', '30000', '--platform', options.platform, ...options.passthrough]); -} - runAgentDevice([ 'test', - options.flowDir, + ...flows, '--maestro', '--platform', options.platform, diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index 36bbd6aa8..f9515f5ea 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -142,10 +142,11 @@ test('test command prints suite summary and exits non-zero on failures', async ( assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./); assert.doesNotMatch(result.stdout, /✓ 01-pass\.ad \(0\.01s\)/); assert.doesNotMatch(result.stdout, /⨯ "Checkout failure" in 02-fail\.ad/); + assert.match(result.stdout, /Failures:\n Checkout failure\n file: 02-fail\.ad/); assert.match(result.stdout, /Replay failed at step 1 \(open Demo\): boom/); assert.match(result.stdout, /artifacts: \/tmp\/test-artifacts\/02-fail/); assert.doesNotMatch(result.stdout, /SKIP \/tmp\/03-skip\.ad/); - assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 0\.025s/); + assert.match(result.stdout, /Test summary: 1 passed \(3\), 1 failed in 0\.025s/); }); test('test command --verbose prints all test statuses', async () => { @@ -158,7 +159,44 @@ test('test command --verbose prints all test statuses', async () => { assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./); assert.doesNotMatch(result.stdout, /✓ 01-pass\.ad \(0\.01s\)/); assert.doesNotMatch(result.stdout, /SKIP 03-skip\.ad/); - assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 0\.025s/); + assert.match(result.stdout, /Test summary: 1 passed \(3\), 1 failed in 0\.025s/); +}); + +test('test command colors suite summary segments when color is enabled', async () => { + const result = await runCliCapture( + ['test', './suite'], + async () => ({ + ok: true, + data: { + total: 1, + executed: 1, + passed: 1, + failed: 0, + skipped: 0, + notRun: 0, + durationMs: 25, + failures: [], + tests: [ + { + file: '/tmp/01-pass.ad', + session: 'default:test:suite:1', + status: 'passed', + durationMs: 25, + attempts: 1, + }, + ], + }, + }), + { env: { FORCE_COLOR: '1', NO_COLOR: undefined } }, + ); + + assert.equal(result.code, null); + assert.ok( + result.stdout.includes( + 'Test summary: \u001B[32m1 passed\u001B[39m \u001B[2m(1)\u001B[22m in \u001B[33m0.025s\u001B[39m', + ), + ); + assert.doesNotMatch(result.stdout, /0 failed/); }); test('test command --verbose omits step telemetry for passing tests without debug mode', async () => { @@ -388,13 +426,13 @@ test('test command reports flaky passed-on-retry cases in the default summary', assert.doesNotMatch(result.stdout, /FLAKY/); assert.doesNotMatch( result.stdout, - /^✓ "Authentication flow" in auth-flow\.yml \(passed attempt 17\.5s, total 112\.2s\)$/m, + /^✓ Authentication flow \(passed attempt 17\.5s, total 112\.2s\)$/m, ); - assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 0\.025s/); + assert.match(result.stdout, /Test summary: 1 passed \(1\), 1 flaky in 0\.025s/); assert.match(result.stdout, /Flaky tests:/); assert.match( result.stdout, - /✓ "Authentication flow" in auth-flow\.yml after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/, + /✓ Authentication flow after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/, ); assert.match( result.stdout, diff --git a/src/__tests__/cli-test-progress.test.ts b/src/__tests__/cli-test-progress.test.ts index c28690867..143e11e9e 100644 --- a/src/__tests__/cli-test-progress.test.ts +++ b/src/__tests__/cli-test-progress.test.ts @@ -1,6 +1,9 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; -import { formatReplayTestProgressEvent } from '../cli-test-progress.ts'; +import { + createReplayTestProgressRenderer, + formatReplayTestProgressEvent, +} from '../cli-test-progress.ts'; import type { RequestProgressEvent } from '../daemon/request-progress.ts'; function withStreamTty(stream: NodeJS.WriteStream, isTTY: boolean, run: () => T): T { @@ -70,7 +73,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', maxAttempts: 2, durationMs: 12_345, }, - expected: /^✓ 01-login\.ad \(12\.3s\)$/, + expected: /^✓ 01-login\.ad 12\.3s$/, }, { event: { @@ -91,6 +94,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', event: { type: 'replay-test', file: '/tmp/03-payment.ad', + title: 'Payment flow', status: 'fail', index: 3, total: 3, @@ -98,11 +102,29 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', maxAttempts: 2, durationMs: 9_876, message: 'assertVisible failed', + hint: 'Stop the owning daemon and retry', session: 'maestro-test:test:suite:3:attempt-2', artifactsDir: '/tmp/replay-suite/payment', }, expected: - /^⨯ 03-payment\.ad \(9\.88s\)\n failed at: assertVisible failed\n session: maestro-test:test:suite:3:attempt-2\n artifacts: \/tmp\/replay-suite\/payment$/, + /^⨯ Payment flow 9\.88s\n file: 03-payment\.ad\n failed at: assertVisible failed\n hint: Stop the owning daemon and retry\n session: maestro-test:test:suite:3:attempt-2\n artifacts: \/tmp\/replay-suite\/payment$/, + }, + { + event: { + type: 'replay-test', + file: '/tmp/05-sharded.ad', + title: 'Sharded flow', + status: 'pass', + index: 5, + total: 6, + attempt: 1, + durationMs: 100, + shardIndex: 0, + shardCount: 2, + deviceId: 'emulator-5554', + deviceName: 'Pixel 8', + }, + expected: /^✓ Sharded flow \[1\/2 Pixel 8\] 0\.1s$/, }, { event: { @@ -142,7 +164,37 @@ test('formatReplayTestProgressEvent colors stderr progress rows when stdout is p ), ); - assert.equal(line, '\u001B[32m✓\u001B[39m 01-pass.ad (0.01s)'); + assert.equal(line, '\u001B[32m✓\u001B[39m 01-pass.ad \u001B[33m0.01s\u001B[39m'); + } finally { + if (typeof originalForceColor === 'string') process.env.FORCE_COLOR = originalForceColor; + else delete process.env.FORCE_COLOR; + if (typeof originalNoColor === 'string') process.env.NO_COLOR = originalNoColor; + else delete process.env.NO_COLOR; + } +}); + +test('createReplayTestProgressRenderer dims live step progress when color is enabled', () => { + const originalForceColor = process.env.FORCE_COLOR; + const originalNoColor = process.env.NO_COLOR; + process.env.FORCE_COLOR = '1'; + delete process.env.NO_COLOR; + try { + const renderer = createReplayTestProgressRenderer({ liveProgress: true }); + const rendered = renderer.render({ + type: 'replay-test', + file: '/tmp/checkout.yaml', + title: 'Checkout flow', + status: 'progress', + index: 1, + total: 1, + stepIndex: 3, + stepTotal: 20, + }); + + assert.deepEqual(rendered, { + text: '\r\u001B[2K⊙ Checkout flow\u001B[2m [3/20]\u001B[22m', + newline: false, + }); } finally { if (typeof originalForceColor === 'string') process.env.FORCE_COLOR = originalForceColor; else delete process.env.FORCE_COLOR; @@ -175,7 +227,7 @@ test('formatReplayTestProgressEvent colors completed result markers when color i attempt: 1, durationMs: 10, }), - '\u001B[32m✓\u001B[39m 01-pass.ad (0.01s)', + '\u001B[32m✓\u001B[39m 01-pass.ad \u001B[33m0.01s\u001B[39m', ); assert.equal( formatReplayTestProgressEvent({ @@ -188,7 +240,7 @@ test('formatReplayTestProgressEvent colors completed result markers when color i attempt: 2, durationMs: 30, }), - '\u001B[33m✓\u001B[39m "Retry flow" in 02-flaky.yml (0.03s)', + '\u001B[33m✓\u001B[39m Retry flow \u001B[33m0.03s\u001B[39m', ); const failedLine = formatReplayTestProgressEvent({ type: 'replay-test', @@ -201,7 +253,7 @@ test('formatReplayTestProgressEvent colors completed result markers when color i durationMs: 5, message: 'boom', }); - assert.ok(failedLine?.startsWith('\u001B[31m⨯\u001B[39m "Checkout failure" in 03-fail.ad')); + assert.ok(failedLine?.startsWith('\u001B[31m⨯\u001B[39m Checkout failure')); } finally { if (typeof originalForceColor === 'string') process.env.FORCE_COLOR = originalForceColor; else delete process.env.FORCE_COLOR; diff --git a/src/__tests__/daemon-client-progress.test.ts b/src/__tests__/daemon-client-progress.test.ts index bc5254570..cf729c419 100644 --- a/src/__tests__/daemon-client-progress.test.ts +++ b/src/__tests__/daemon-client-progress.test.ts @@ -123,7 +123,7 @@ test('readDaemonSocketProgressResponse parses split progress lines before respon assert.deepEqual(await responsePromise, { ok: true, data: { via: 'socket-progress' } }); assert.equal(socket.encoding, 'utf8'); assert.equal(socket.ended, true); - assert.match(stderr, /✓ "Login flow" in 01-login\.ad \(1\.23s\)/); + assert.match(stderr, /✓ Login flow 1\.23s/); } finally { process.stderr.write = originalStderrWrite; } @@ -237,11 +237,9 @@ test('readDaemonSocketProgressResponse rewrites live progress and clears it for socket.emit('data', `${progress(3)}\n${progress(4)}\n${pass}\n${responseLine}\n`); assert.deepEqual(await responsePromise, { ok: true, data: { via: 'socket-progress' } }); - assert.ok(stderr.includes('\r\u001B[2K⊙ "Tab View - Co..." in tab-view-coverflow.yml [3/10]')); - assert.ok(stderr.includes('\r\u001B[2K⊙ "Tab View - Co..." in tab-view-coverflow.yml [4/10]')); - assert.ok( - stderr.includes('\r\u001B[2K✓ "Tab View - Coverflow" in tab-view-coverflow.yml (17.8s)\n'), - ); + assert.ok(stderr.includes('\r\u001B[2K⊙ Tab View - Coverflow [3/10]')); + assert.ok(stderr.includes('\r\u001B[2K⊙ Tab View - Coverflow [4/10]')); + assert.ok(stderr.includes('\r\u001B[2K✓ Tab View - Coverflow 17.8s\n')); } finally { if (typeof originalCi === 'string') process.env.CI = originalCi; else delete process.env.CI; @@ -308,7 +306,7 @@ test('readDaemonSocketProgressResponse suppresses live progress outside interact socket.emit('data', `${progress}\n${pass}\n${responseLine}\n`); assert.deepEqual(await responsePromise, { ok: true, data: { via: 'socket-progress' } }); - assert.equal(stderr, '✓ "Tab View - Coverflow" in tab-view-coverflow.yml (17.8s)\n'); + assert.equal(stderr, '✓ Tab View - Coverflow 17.8s\n'); } finally { process.stderr.write = originalStderrWrite; } diff --git a/src/cli-test-progress.ts b/src/cli-test-progress.ts index 4a135613f..f34169d38 100644 --- a/src/cli-test-progress.ts +++ b/src/cli-test-progress.ts @@ -88,15 +88,15 @@ function formatReplayTestLiveProgressLine( ): string { const title = event.title?.trim(); const file = path.basename(event.file); - const shardSuffix = formatReplayTestProgressShardSuffix(event); - const stepIndex = event.stepIndex ?? 0; - const stepTotal = event.stepTotal ?? 0; - const suffix = `${shardSuffix} [${stepIndex}/${stepTotal}]`; + const useColor = supportsColor(process.stderr); + const shardSuffix = formatReplayTestProgressShardSuffix(event, { useColor }); + const stepSuffix = formatReplayTestLiveProgressStepSuffix(event, { useColor }); + const suffix = `${shardSuffix}${stepSuffix}`; const prefix = '⊙ '; if (!title) return trimToColumns(`${prefix}${file}${suffix}`, options.columns); - const titlePrefix = `${prefix}"`; - const titleSuffix = `" in ${file}${suffix}`; + const titlePrefix = prefix; + const titleSuffix = suffix; const availableTitleColumns = Math.max( 0, resolveColumns(options.columns) - titlePrefix.length - titleSuffix.length, @@ -105,35 +105,79 @@ function formatReplayTestLiveProgressLine( return trimToColumns(`${titlePrefix}${formattedTitle}${titleSuffix}`, options.columns); } +function formatReplayTestLiveProgressStepSuffix( + event: ReplayTestCaseProgressEvent, + options: { useColor?: boolean } = {}, +): string { + const stepIndex = event.stepIndex ?? 0; + const stepTotal = event.stepTotal ?? 0; + const suffix = ` [${stepIndex}/${stepTotal}]`; + return options.useColor ? colorizeProgressMarker(suffix, 'dim') : suffix; +} + function addReplayTestCaseDetailLines( lines: string[], event: ReplayTestCaseProgressEvent, options: ReplayTestProgressFormatOptions, ): void { - if (options.verbose && event.status === 'fail') return; + if (shouldSuppressReplayTestCaseDetailLines(event, options)) return; + const fileLine = replayTestProgressFailureFileLine(event); + const messageLine = replayTestProgressMessageLine(event); + if (fileLine) lines.push(fileLine); + if (messageLine) lines.push(messageLine); + appendReplayTestProgressHintLine(lines, event); + lines.push(...replayTestProgressFailureContextLines(event)); +} + +function shouldSuppressReplayTestCaseDetailLines( + event: ReplayTestCaseProgressEvent, + options: ReplayTestProgressFormatOptions, +): boolean { + return options.verbose === true && event.status === 'fail'; +} + +function replayTestProgressFailureFileLine(event: ReplayTestCaseProgressEvent): string | undefined { + return event.status === 'fail' && event.title?.trim() + ? ` file: ${path.basename(event.file)}` + : undefined; +} + +function replayTestProgressMessageLine(event: ReplayTestCaseProgressEvent): string | undefined { const message = event.message?.replace(/\s+/g, ' ').trim(); - if (message) { - lines.push(` ${event.status === 'fail' ? `failed at: ${message}` : message}`); - } - if (event.status === 'fail' && !event.retrying) { - if (event.session) lines.push(` session: ${event.session}`); - if (event.artifactsDir) lines.push(` artifacts: ${event.artifactsDir}`); - } + if (!message) return undefined; + return ` ${event.status === 'fail' ? `failed at: ${message}` : message}`; +} + +function appendReplayTestProgressHintLine( + lines: string[], + event: ReplayTestCaseProgressEvent, +): void { + const hint = event.hint?.replace(/\s+/g, ' ').trim(); + if (event.status === 'fail' && hint) lines.push(` hint: ${hint}`); +} + +function replayTestProgressFailureContextLines(event: ReplayTestCaseProgressEvent): string[] { + if (event.status !== 'fail' || event.retrying) return []; + const lines: string[] = []; + if (event.session) lines.push(` session: ${event.session}`); + if (event.artifactsDir) lines.push(` artifacts: ${event.artifactsDir}`); + return lines; } function formatReplayTestCaseSummaryLine(event: ReplayTestCaseProgressEvent): string { + const useColor = supportsColor(process.stderr); const statusLabel = formatReplayTestProgressStatusLabel(event); const name = formatReplayTestProgressName(event); - const shardSuffix = formatReplayTestProgressShardSuffix(event); + const shardSuffix = formatReplayTestProgressShardSuffix(event, { useColor }); const durationSuffix = - event.durationMs !== undefined ? ` (${formatReplayProgressDuration(event)})` : ''; + event.durationMs !== undefined ? ` ${formatReplayProgressDuration(event, { useColor })}` : ''; return `${statusLabel} ${name}${shardSuffix}${durationSuffix}`; } function formatReplayTestProgressName(event: ReplayTestCaseProgressEvent): string { const title = event.title?.trim(); const file = path.basename(event.file); - return title ? `${JSON.stringify(title)} in ${file}` : file; + return title ? title : file; } function formatReplayTestProgressStatusLabel(event: ReplayTestCaseProgressEvent): string { @@ -151,15 +195,30 @@ function colorizeProgressMarker(text: string, format: Parameters 0 ? `, ${data.failed} failed` : ''; const flakySuffix = flakyCount > 0 ? `, ${flakyCount} flaky` : ''; - const durationSuffix = durationMs !== undefined ? ` in ${formatDurationSeconds(durationMs)}` : ''; - return `Test summary: ${data.passed} passed, ${data.failed} failed${flakySuffix}${durationSuffix}`; + const durationSuffix = + durationMs !== undefined ? ` in ${formatReplayDuration(durationMs, { useColor })}` : ''; + return `Test summary: ${passed} ${total}${failedSuffix}${flakySuffix}${durationSuffix}`; +} + +function formatReplaySummaryPassed(passed: number, options: { useColor?: boolean } = {}): string { + const text = `${passed} passed`; + return options.useColor ? colorize(text, 'green') : text; +} + +function formatReplaySummaryTotal(total: number, options: { useColor?: boolean } = {}): string { + const text = `(${total})`; + return options.useColor ? colorize(text, 'dim') : text; +} + +function formatReplayDuration(durationMs: number, options: { useColor?: boolean } = {}): string { + const duration = formatDurationSeconds(durationMs); + return options.useColor ? colorize(duration, 'yellow') : duration; } function replayFlakyStatusIcon(): string { @@ -97,6 +118,8 @@ function renderReplayFailureBody( context: ReplayTestReporterContext, indent: string, ): void { + const fileLine = replayTestFailureFileLine(result); + if (fileLine) context.writeStdout(`${indent}${fileLine}\n`); context.writeStdout(`${indent}${result.error?.message ?? 'Unknown test failure'}\n`); for (const line of replayFailureConsoleLines(result)) { context.writeStdout(`${indent}${line}\n`); diff --git a/src/cli-test-reporters/format.ts b/src/cli-test-reporters/format.ts index 9bf8cfc1a..d465271ad 100644 --- a/src/cli-test-reporters/format.ts +++ b/src/cli-test-reporters/format.ts @@ -24,7 +24,7 @@ export function isFlakyReplayTestResult( export function replayTestDisplayNameWithFile(result: ReplaySuiteTestResult): string { const title = replayTestTitle(result); const filename = path.basename(result.file); - const base = title && title.length > 0 ? `${JSON.stringify(title)} in ${filename}` : filename; + const base = title && title.length > 0 ? title : filename; return `${base}${formatReplayTestShardSuffix(result)}`; } @@ -41,6 +41,10 @@ export function replayArtifactsLine( : undefined; } +export function replayTestFailureFileLine(result: FailedReplayTestResult): string | undefined { + return replayTestTitle(result) ? `file: ${path.basename(result.file)}` : undefined; +} + export function replayErrorHintLine(error: ReplayTestError): string | undefined { return error.hint ? `hint: ${error.hint}` : undefined; } @@ -96,6 +100,10 @@ export function appendReplayTestShardMetadata( lines, typeof result.deviceId === 'string' ? `deviceId: ${result.deviceId}` : undefined, ); + appendOptionalLine( + lines, + typeof result.deviceName === 'string' ? `deviceName: ${result.deviceName}` : undefined, + ); } export function replayTestWarningLines(result: ReplaySuiteTestResult): string[] { @@ -132,6 +140,13 @@ function replayTestTitle(result: ReplaySuiteTestResult): string | undefined { export function formatReplayTestShardSuffix(result: ReplaySuiteTestResult): string { if (!('shardIndex' in result) || typeof result.shardIndex !== 'number') return ''; const shardCount = typeof result.shardCount === 'number' ? result.shardCount : '?'; - const device = typeof result.deviceId === 'string' ? ` ${result.deviceId}` : ''; - return ` [shard ${result.shardIndex + 1}/${shardCount}${device}]`; + const device = replayTestShardDeviceName(result); + return ` [${result.shardIndex + 1}/${shardCount}${device ? ` ${device}` : ''}]`; +} + +function replayTestShardDeviceName(result: ReplaySuiteTestResult): string | undefined { + const name = 'deviceName' in result ? result.deviceName?.trim() : undefined; + if (name) return name; + const id = 'deviceId' in result ? result.deviceId?.trim() : undefined; + return id || undefined; } diff --git a/src/commands/management/prepare.ts b/src/commands/management/prepare.ts index 2e100890a..8e8c8a488 100644 --- a/src/commands/management/prepare.ts +++ b/src/commands/management/prepare.ts @@ -32,7 +32,7 @@ const prepareCliSchema = { usageOverride: 'prepare ios-runner --platform ios|macos [--timeout ]', listUsageOverride: 'prepare', helpDescription: - 'Prepare platform helper infrastructure. ios-runner builds/reuses, starts, and health-checks the XCTest runner so later Apple snapshots and interactions do not pay first-use startup cost. In CI, run it after boot/install and before replay/test; if replay/test starts a separate daemon, run clean:daemon after prepare to release the prepared runner lease. It is not a recovery step for "runner already owned by another agent-device daemon"; stop or clean the owning daemon on the Mac with simulator access instead. Runner build/start output is written to the session runner.log; daemon.log is for daemon lifecycle/startup issues.', + 'Prepare platform helper infrastructure. ios-runner builds/reuses, starts, and health-checks the XCTest runner so later Apple snapshots and interactions do not pay first-use startup cost. In CI, run it after boot/install and before replay/test; if replay/test starts a separate daemon, stop the prepare daemon before replay/test so it does not keep the prepared runner lease. It is not a recovery step for "runner already owned by another agent-device daemon"; stop the owning daemon on the Mac with simulator access instead. Runner build/start output is written to the session runner.log; daemon.log is for daemon lifecycle/startup issues.', summary: 'Pre-warm platform helpers, especially the iOS/macOS XCTest runner before Apple automation', positionalArgs: ['ios-runner'], diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index c7efc4a5c..0b7b8d00c 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -740,7 +740,9 @@ test('parseMaestroReplayFlow parses the test-app Maestro suite fixture', () => { assert.deepEqual( parsed.actions.map((entry) => entry.command), [ + 'open', '__maestroAssertVisible', + '__maestroScrollUntilVisible', '__maestroTapOn', '__maestroAssertVisible', '__maestroTapOn', @@ -760,4 +762,5 @@ test('parseMaestroReplayFlow parses the test-app Maestro suite fixture', () => { '__maestroAssertVisible', ], ); + assert.equal(parsed.actions[0]?.flags.clearAppState, true); }); diff --git a/src/daemon/handlers/__tests__/session-test-suite.test.ts b/src/daemon/handlers/__tests__/session-test-suite.test.ts index 0c4db5d61..ba581ea46 100644 --- a/src/daemon/handlers/__tests__/session-test-suite.test.ts +++ b/src/daemon/handlers/__tests__/session-test-suite.test.ts @@ -509,6 +509,12 @@ test('test --shard-all runs each runnable entry on each selected device', async 'emulator-5556', 'emulator-5556', ]); + expect(tests.map((entry) => entry.deviceName)).toEqual([ + 'Pixel 8', + 'Pixel 8', + 'Pixel 8 Pro', + 'Pixel 8 Pro', + ]); expect(tests.map((entry) => entry.artifactsDir)).toEqual([ expect.stringContaining(`${path.sep}shard-1${path.sep}01-login`), expect.stringContaining(`${path.sep}shard-1${path.sep}02-pay`), @@ -567,6 +573,9 @@ test('test --shard-split distributes runnable entries by modulo and keeps skips expect(tests.filter((entry) => entry.status === 'passed').map((entry) => entry.deviceId)).toEqual( ['emulator-5554', 'emulator-5554', 'emulator-5556'], ); + expect( + tests.filter((entry) => entry.status === 'passed').map((entry) => entry.deviceName), + ).toEqual(['Pixel 8', 'Pixel 8', 'Pixel 8 Pro']); expect(invoked.map((req) => req.flags?.serial).sort()).toEqual([ 'emulator-5554', 'emulator-5554', diff --git a/src/daemon/handlers/session-test-attempt.ts b/src/daemon/handlers/session-test-attempt.ts index c79a081be..c286f3c70 100644 --- a/src/daemon/handlers/session-test-attempt.ts +++ b/src/daemon/handlers/session-test-attempt.ts @@ -243,6 +243,7 @@ function emitReplayTestRetryProgress( durationMs: attempt.durationMs, retrying: true, message: attempt.response.error.message, + hint: attempt.response.error.hint, session: attempt.sessionName, artifactsDir: context.testArtifactsDir, ...replayTestProgressShardMetadata(params.shard), @@ -322,6 +323,7 @@ function buildReplayTestFailedResult( session: outcome.finalSessionName, artifactsDir: context.testArtifactsDir, message: error.message, + hint: error.hint, ...replayTestProgressShardMetadata(shard), }); return { @@ -379,18 +381,19 @@ function readReplayResponseSnapshotDiagnostics(response: DaemonResponse | undefi function replayTestShardResultMetadata( shard: ReplayTestShardContext | undefined, -): Pick { +): Pick { return replayTestProgressShardMetadata(shard); } function replayTestProgressShardMetadata( shard: ReplayTestShardContext | undefined, -): Pick { +): Pick { return shard ? { shardIndex: shard.shardIndex, shardCount: shard.shardCount, deviceId: shard.device.id, + deviceName: shard.device.name, } : {}; } diff --git a/src/daemon/handlers/session-test.ts b/src/daemon/handlers/session-test.ts index ac837c029..13f27ad93 100644 --- a/src/daemon/handlers/session-test.ts +++ b/src/daemon/handlers/session-test.ts @@ -230,6 +230,7 @@ function buildUnexpectedShardFailure( shardIndex: shard.shardIndex, shardCount: shard.shardCount, deviceId: shard.device.id, + deviceName: shard.device.name, }; } diff --git a/src/daemon/request-progress.ts b/src/daemon/request-progress.ts index 88aed6d81..1b013933d 100644 --- a/src/daemon/request-progress.ts +++ b/src/daemon/request-progress.ts @@ -25,11 +25,13 @@ export type ReplayTestProgressEvent = { durationMs?: number; retrying?: boolean; message?: string; + hint?: string; session?: string; artifactsDir?: string; shardIndex?: number; shardCount?: number; deviceId?: string; + deviceName?: string; }; export type CommandProgressEvent = { diff --git a/src/daemon/types.ts b/src/daemon/types.ts index 9ce53132a..6ccda5a1f 100644 --- a/src/daemon/types.ts +++ b/src/daemon/types.ts @@ -76,6 +76,7 @@ export type ReplaySuiteTestPassed = { shardIndex?: number; shardCount?: number; deviceId?: string; + deviceName?: string; snapshotDiagnostics?: SnapshotDiagnosticsSummary; }; @@ -91,6 +92,7 @@ export type ReplaySuiteTestFailed = { shardIndex?: number; shardCount?: number; deviceId?: string; + deviceName?: string; snapshotDiagnostics?: SnapshotDiagnosticsSummary; }; diff --git a/src/platforms/ios/__tests__/runner-session.test.ts b/src/platforms/ios/__tests__/runner-session.test.ts index 091fc8d29..16c890faa 100644 --- a/src/platforms/ios/__tests__/runner-session.test.ts +++ b/src/platforms/ios/__tests__/runner-session.test.ts @@ -772,10 +772,26 @@ test('runner session startup rejects live foreign runner lease', async () => { String((thrown as { details?: Record }).details?.hint), /Do not run prepare ios-runner/, ); + assert.match( + String((thrown as { details?: Record }).details?.hint), + /^If it is stuck, stop the owning agent-device daemon for AGENT_DEVICE_STATE_DIR='\/tmp\/agent-device-owner' and retry/, + ); + assert.doesNotMatch( + String((thrown as { details?: Record }).details?.hint), + /pnpm|clean:daemon/, + ); assert.match( String((thrown as { details?: Record }).details?.hint), /PID \d+ with AGENT_DEVICE_STATE_DIR=\/tmp\/agent-device-owner/, ); + assert.doesNotMatch( + String((thrown as { details?: Record }).details?.hint), + /AGENT_DEVICE_STATE_DIR=\/tmp\/agent-device-owner\./, + ); + assert.doesNotMatch( + String((thrown as { details?: Record }).details?.hint), + /Current daemon state dir/, + ); assert.equal(mockRunCmdBackground.mock.calls.length, 0); assert.equal( mockRunAppleToolCommand.mock.calls.some((call) => call[0] === 'pkill'), @@ -825,10 +841,19 @@ test('runner session busy error includes logical lease context after admission', deviceKey: device.id, }); assert.match(String(thrown.details?.hint), /five-minute inactivity lease expires/); + assert.match( + String(thrown.details?.hint), + /^If it is stuck, stop the owning agent-device daemon for AGENT_DEVICE_STATE_DIR='\/tmp\/agent-device-owner' and retry/, + ); + assert.doesNotMatch(String(thrown.details?.hint), /pnpm|clean:daemon/); assert.match( String(thrown.details?.hint), /Runner owner: PID \d+ with AGENT_DEVICE_STATE_DIR=\/tmp\/agent-device-owner/, ); + assert.doesNotMatch( + String(thrown.details?.hint), + /AGENT_DEVICE_STATE_DIR=\/tmp\/agent-device-owner\./, + ); assert.equal(mockRunCmdBackground.mock.calls.length, 0); }); diff --git a/src/platforms/ios/runner-lease.ts b/src/platforms/ios/runner-lease.ts index 86e052eb8..6af2882a2 100644 --- a/src/platforms/ios/runner-lease.ts +++ b/src/platforms/ios/runner-lease.ts @@ -188,26 +188,44 @@ function buildBusyRunnerLeaseHint( lease: RunnerLease, logicalLeaseContext?: RunnerLogicalLeaseContext, ): string { - const owner = `PID ${lease.ownerPid}`; - const stateDir = lease.ownerStateDir ? ` with AGENT_DEVICE_STATE_DIR=${lease.ownerStateDir}` : ''; - const currentStateDir = readCurrentStateDir(); - const current = - currentStateDir && currentStateDir !== lease.ownerStateDir - ? ` Current daemon state dir is ${currentStateDir}.` - : ''; + const owner = buildRunnerOwnerHint(lease); + const cleanup = buildBusyRunnerLeaseCleanupHint(lease); if (logicalLeaseContext) { return [ - `The device is busy because another active device lease owns it, or the runner is owned by another daemon/process after lease admission. Runner owner: ${owner}${stateDir}.${current}`, + cleanup, + owner, + 'The device is busy because another active device lease owns it, or the runner is owned by another daemon/process after lease admission.', 'Retry after the owning session closes or after the five-minute inactivity lease expires.', - 'If this persists after expiry, inspect the runner owner details and clean the stale daemon state on the machine with simulator access.', ].join(' '); } return [ - `Runner owner details: ${owner}${stateDir}.${current} Retry after the owning runner finishes.`, - 'Do not run prepare ios-runner from another daemon/client to recover this; a live foreign runner lease cannot be released by the remote client.', + cleanup, + owner, + 'If the runner is still active, wait for it to finish. Do not run prepare ios-runner from another daemon/client to recover this.', ].join(' '); } +function buildRunnerOwnerHint(lease: RunnerLease): string { + const owner = `Runner owner: PID ${lease.ownerPid}`; + if (lease.ownerStateDir) return `${owner} with AGENT_DEVICE_STATE_DIR=${lease.ownerStateDir}`; + return `${owner}.`; +} + +function buildBusyRunnerLeaseCleanupHint(lease: RunnerLease): string { + if (lease.ownerStateDir) { + return `If it is stuck, stop the owning agent-device daemon for ${formatEnvAssignment('AGENT_DEVICE_STATE_DIR', lease.ownerStateDir)} and retry.`; + } + return 'If it is stuck, stop the owning agent-device daemon and retry.'; +} + +function formatEnvAssignment(name: string, value: string): string { + return `${name}=${shellQuote(value)}`; +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'`; +} + export async function cleanupOwnedRunnerLease( deviceId: string, cleanup: RunnerLeaseCleanupAdapter, diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index 1c92fa5bd..f12ab5e06 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -1391,7 +1391,8 @@ test('usageForCommand documents prepare ios-runner', () => { assert.match(help, /--timeout /); assert.match(help, /XCTest runner/); assert.match(help, /separate daemon/); - assert.match(help, /clean:daemon after prepare/); + assert.match(help, /stop the prepare daemon before replay\/test/); + assert.doesNotMatch(help, /clean:daemon|pnpm/); assert.match( help, /not a recovery step for "runner already owned by another agent-device daemon"/, diff --git a/src/utils/__tests__/daemon-client.test.ts b/src/utils/__tests__/daemon-client.test.ts index 3d8bd4201..ba7e197b1 100644 --- a/src/utils/__tests__/daemon-client.test.ts +++ b/src/utils/__tests__/daemon-client.test.ts @@ -558,8 +558,8 @@ test('sendToDaemon prints replay test progress before the socket response', asyn }); assert.deepEqual(response, { ok: true, data: { via: 'socket' } }); - assert.match(stderr, /✓ "Login flow" in 01-login\.ad \(1\.23s\)/); - assert.equal(stderr.match(/✓ "Login flow" in 01-login\.ad \(1\.23s\)/g)?.length, 1); + assert.match(stderr, /✓ Login flow 1\.23s/); + assert.equal(stderr.match(/✓ Login flow 1\.23s/g)?.length, 1); assert.match(stderr, /steps \(attempt 2\):/); assert.match(stderr, /open "Demo" \(line 3, 0\.25s\)/); assert.match(stderr, /assertVisible "text=\\"Home\\"" "3000" \(line 4, 0\.75s\)/); @@ -663,7 +663,7 @@ test('sendToDaemon prints replay test progress before the HTTP NDJSON response', assert.deepEqual(response, { ok: true, data: { via: 'http-progress' } }); }); assert.deepEqual(seenPaths, ['GET /agent-device/health', 'POST /agent-device/rpc']); - assert.match(stderr, /✓ "Payments flow" in 02-payments\.ad \(2\.50s\)/); + assert.match(stderr, /✓ Payments flow 2\.50s/); } finally { (http as unknown as { request: typeof http.request }).request = originalHttpRequest; process.stderr.write = originalStderrWrite; diff --git a/src/utils/cli-help.ts b/src/utils/cli-help.ts index 62ccf53f5..bc0e30986 100644 --- a/src/utils/cli-help.ts +++ b/src/utils/cli-help.ts @@ -155,7 +155,7 @@ Bootstrap: agent-device prepare ios-runner --platform ios --timeout 240000 If app id is unknown, plan devices, apps, then open . Discovery is not enough when the task asks to open/start the app. Install arguments are app/package id then artifact path. If the task says install, use install; use reinstall only when explicitly requested. Fresh runtime state is open --relaunch after install. - In Apple CI, run prepare ios-runner after boot/install and before replay/test. prepare ios-runner builds/reuses the XCTest runner, health-checks it with a lightweight command, and retries one stuck/non-connecting runner launch before the first snapshot pays that setup cost. It is not a recovery step for "runner already owned by another agent-device daemon"; stop or clean the owning daemon on the Mac with simulator access instead. If the replay/test step starts a separate daemon, run clean:daemon after prepare so the prepared runner does not keep a live lease owned by the prepare daemon. + In Apple CI, run prepare ios-runner after boot/install and before replay/test. prepare ios-runner builds/reuses the XCTest runner, health-checks it with a lightweight command, and retries one stuck/non-connecting runner launch before the first snapshot pays that setup cost. It is not a recovery step for "runner already owned by another agent-device daemon"; stop the owning daemon on the Mac with simulator access instead. If the replay/test step starts a separate daemon, stop the prepare daemon before replay/test so the prepared runner does not keep a live lease owned by that daemon. CI may cache ~/.agent-device/ios-runner/derived with an exact key that includes the agent-device package and Xcode version. Avoid broad restore-key fallbacks; prepare ios-runner already recovers bad restored runner artifacts and one retryable non-connecting runner launch. Runner build/start output is written to the session's runner.log; daemon.log is for daemon lifecycle/startup issues. Do not open artifact paths or invent package ids. If apps lookup misses the target and no URL/artifact is provided, ask or stop.