Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,8 @@ jobs:
- name: Test the epoch guard
run: node --test --test-concurrency=1 scripts/protocol-epoch-check.test.mjs

- name: Test AX tree audit contract
run: node --test scripts/ax-tree-audit.test.mjs
- name: Test Computer Use script contracts
run: node --test scripts/ax-tree-audit.test.mjs scripts/computer-use/lab-root.test.mjs

- name: Verify ASF npm preflight policy
run: npm run check:asf-npm
Expand Down
24 changes: 22 additions & 2 deletions docs/computer-use-evidence-classes.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,30 @@ real-machine qualification runner was removed. The five-round process-restart
runner remains as a non-qualifying soak after its qualification checks moved
into the canonical Runtime-backed harness.

The canonical operator commands are:
### Lab fixture setup

```text
The `real-ax` and `restart-soak` commands require a local checkout of the
[Codex Computer Use Lab](https://github.com/hqhq1025/codex-computer-use-lab).
Clone it outside this repository and export its absolute repository root:

```bash
git clone https://github.com/hqhq1025/codex-computer-use-lab.git ../codex-computer-use-lab
export MAKA_CU_AX_MODEL_LAB_ROOT="$(cd ../codex-computer-use-lab && pwd)"
```

The path must contain `test-app/launch.sh`. The launchers invoke that script,
which builds the fixture when its application bundle is absent.

With the variable exported, run the canonical operator commands:

```bash
npm run computer-use -- real-ax
npm run computer-use -- real-ax --scenario restart-recovery
npm run computer-use -- real-model
```

The non-qualifying five-round restart soak uses the same fixture checkout:

```bash
npm run computer-use -- restart-soak
```
6 changes: 4 additions & 2 deletions docs/computer-use-provider-evidence.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,10 @@ Qualification also keeps three fail-closed invariants:

The old direct real-machine qualification runner was removed. The five-round
restart runner remains available as `npm run computer-use -- restart-soak`,
but is regression-only and cannot satisfy a provider matrix cell. There is one
qualification path rather than parallel evidence standards.
using the `MAKA_CU_AX_MODEL_LAB_ROOT` fixture checkout described in
[Lab fixture setup](./computer-use-evidence-classes.md#lab-fixture-setup). The
runner is regression-only and cannot satisfy a provider matrix cell. There is
one qualification path rather than parallel evidence standards.

## Next Layer

Expand Down
56 changes: 56 additions & 0 deletions scripts/computer-use/lab-root.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import { accessSync, constants, realpathSync, statSync } from 'node:fs';
import { isAbsolute, join } from 'node:path';

export function requireComputerUseLabRoot(env = process.env) {
const configuredRoot = env.MAKA_CU_AX_MODEL_LAB_ROOT;
if (typeof configuredRoot !== 'string' || configuredRoot.trim() === '') {
throw new Error(
'MAKA_CU_AX_MODEL_LAB_ROOT is required: point it at a local checkout of the Codex CUA Lab fixture',
);
}
if (!isAbsolute(configuredRoot)) {
throw new Error('MAKA_CU_AX_MODEL_LAB_ROOT must be an absolute path');
}

let labRoot;
try {
labRoot = realpathSync(configuredRoot);
if (!statSync(labRoot).isDirectory()) throw new Error('Lab root is not a directory');
} catch (cause) {
throw new Error(
`MAKA_CU_AX_MODEL_LAB_ROOT must point to an existing directory: ${configuredRoot}`,
{ cause },
);
}

const launcherPath = join(labRoot, 'test-app', 'launch.sh');
try {
if (!statSync(launcherPath).isFile()) throw new Error('fixture launcher is not a file');
accessSync(launcherPath, constants.X_OK);
} catch (cause) {
throw new Error(
`MAKA_CU_AX_MODEL_LAB_ROOT must contain an executable test-app/launch.sh: ${launcherPath}`,
{ cause },
);
}
return labRoot;
}
117 changes: 117 additions & 0 deletions scripts/computer-use/lab-root.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import assert from 'node:assert/strict';
import {
chmod,
mkdir,
mkdtemp,
readFile,
realpath,
rm,
symlink,
writeFile,
} from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { test } from 'node:test';

import { requireComputerUseLabRoot } from './lab-root.mjs';

test('rejects missing and invalid Computer Use Lab roots', async () => {
const root = await mkdtemp(join(tmpdir(), 'maka-cu-lab-root-invalid-'));
try {
assert.throws(
() => requireComputerUseLabRoot({}),
new Error(
'MAKA_CU_AX_MODEL_LAB_ROOT is required: point it at a local checkout of the Codex CUA Lab fixture',
),
);
assert.throws(
() => requireComputerUseLabRoot({ MAKA_CU_AX_MODEL_LAB_ROOT: ' ' }),
/MAKA_CU_AX_MODEL_LAB_ROOT is required/,
);
assert.throws(
() => requireComputerUseLabRoot({ MAKA_CU_AX_MODEL_LAB_ROOT: 'relative/lab' }),
/MAKA_CU_AX_MODEL_LAB_ROOT must be an absolute path/,
);
assert.throws(
() => requireComputerUseLabRoot({ MAKA_CU_AX_MODEL_LAB_ROOT: join(root, 'missing') }),
/MAKA_CU_AX_MODEL_LAB_ROOT must point to an existing directory/,
);
assert.throws(
() => requireComputerUseLabRoot({ MAKA_CU_AX_MODEL_LAB_ROOT: root }),
/MAKA_CU_AX_MODEL_LAB_ROOT must contain an executable test-app\/launch\.sh/,
);
const nonExecutableRoot = join(root, 'non-executable');
const launcherPath = join(nonExecutableRoot, 'test-app', 'launch.sh');
await mkdir(join(nonExecutableRoot, 'test-app'), { recursive: true });
await writeFile(launcherPath, '#!/usr/bin/env bash\n');
await chmod(launcherPath, 0o644);
assert.throws(
() => requireComputerUseLabRoot({ MAKA_CU_AX_MODEL_LAB_ROOT: nonExecutableRoot }),
/MAKA_CU_AX_MODEL_LAB_ROOT must contain an executable test-app\/launch\.sh/,
);
} finally {
await rm(root, { recursive: true, force: true });
}
});

test('canonicalizes a valid Computer Use Lab root', async () => {
const root = await mkdtemp(join(tmpdir(), 'maka-cu-lab-root-valid-'));
try {
const fixtureRoot = join(root, 'fixture');
const linkedRoot = join(root, 'fixture-link');
const launcherPath = join(fixtureRoot, 'test-app', 'launch.sh');
await mkdir(join(fixtureRoot, 'test-app'), { recursive: true });
await writeFile(launcherPath, '#!/usr/bin/env bash\n');
await chmod(launcherPath, 0o755);
await symlink(fixtureRoot, linkedRoot);

assert.equal(
requireComputerUseLabRoot({ MAKA_CU_AX_MODEL_LAB_ROOT: linkedRoot }),
await realpath(fixtureRoot),
);
} finally {
await rm(root, { recursive: true, force: true });
}
});

test('Lab-backed entry points require the configured root', async () => {
const entryPoints = [
'process-restart-harness.mjs',
'process-restart-launcher.mjs',
'real-ax-harness.mjs',
'real-ax-launcher.mjs',
];

for (const entryPoint of entryPoints) {
const source = await readFile(new URL(entryPoint, import.meta.url), 'utf8');
assert.match(
source,
/const labRoot = requireComputerUseLabRoot\(\);/,
`${entryPoint} must require the configured Lab root`,
);
assert.doesNotMatch(
source,
/codex-computer-use-lab/i,
`${entryPoint} must not embed a contributor-specific Lab checkout`,
);
}
});
3 changes: 2 additions & 1 deletion scripts/computer-use/process-restart-harness.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,11 @@ import { tmpdir } from 'node:os';
import { join, relative, resolve } from 'node:path';
import { createMakaCuBackend } from '../../packages/computer-use/dist/index.js';
import { buildComputerUseTools } from '../../packages/runtime/dist/computer-use-tools.js';
import { requireComputerUseLabRoot } from './lab-root.mjs';

const repoRoot = new URL('../..', import.meta.url).pathname;
const binaryPath = join(repoRoot, 'apps/desktop/resources/bin/maka-cu');
const labRoot = '/Users/haoqing/Documents/Learning/codex-computer-use-lab';
const labRoot = requireComputerUseLabRoot();
const expectedAppPath = join(labRoot, 'test-app/build/Codex CUA Lab.app');
const statePath = join(labRoot, 'test-app/runtime/state.json');
const temporaryDirectory = process.env.MAKA_CU_RESTART_TEMP_DIR;
Expand Down
4 changes: 3 additions & 1 deletion scripts/computer-use/process-restart-launcher.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,13 @@ import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';

import { requireComputerUseLabRoot } from './lab-root.mjs';

const here = dirname(fileURLToPath(import.meta.url));
const repoRoot = join(here, '../..');
const harnessPath = join(here, 'process-restart-harness.mjs');
const monitorPath = join(here, 'real-e2e-monitor.swift');
const labRoot = '/Users/haoqing/Documents/Learning/codex-computer-use-lab';
const labRoot = requireComputerUseLabRoot();
const statePath = join(labRoot, 'test-app', 'runtime', 'state.json');
const SOAK_ROUNDS = 5;
const FIXTURE_BUNDLE_ID = 'com.openai.codex.cualab';
Expand Down
5 changes: 2 additions & 3 deletions scripts/computer-use/real-ax-harness.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,12 @@ import { buildComputerUseTools } from '../../packages/runtime/dist/computer-use-
import { getAIModel } from '../../packages/runtime/dist/model-factory.js';
import { createMakaCuBackend } from '../../packages/computer-use/dist/index.js';
import { createDirectRuntimeTurnLedger } from './direct-runtime-ledger.mjs';
import { requireComputerUseLabRoot } from './lab-root.mjs';
import { sanitizeCuDirectReport } from './report-sanitize.mjs';

const repoRoot = new URL('../..', import.meta.url).pathname;
const binaryPath = join(repoRoot, 'apps/desktop/resources/bin/maka-cu');
const labRoot =
process.env.MAKA_CU_AX_MODEL_LAB_ROOT ??
'/Users/haoqing/Documents/Learning/codex-computer-use-lab';
const labRoot = requireComputerUseLabRoot();
const expectedAppPath = join(labRoot, 'test-app/build/Codex CUA Lab.app');
const statePath = join(labRoot, 'test-app/runtime/state.json');
const fixturePID = Number(process.env.MAKA_CU_AX_MODEL_FIXTURE_PID);
Expand Down
9 changes: 3 additions & 6 deletions scripts/computer-use/real-ax-launcher.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,14 @@ import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';

import { requireComputerUseLabRoot } from './lab-root.mjs';

const here = dirname(fileURLToPath(import.meta.url));
const repoRoot = join(here, '../..');
const harnessPath = join(here, 'real-ax-harness.mjs');
const monitorPath = join(here, 'real-e2e-monitor.swift');
const inputAgeSource = join(here, 'physical-input-age.swift');
const labRoot = process.env.MAKA_CU_AX_MODEL_LAB_ROOT;
if (!labRoot) {
throw new Error(
'MAKA_CU_AX_MODEL_LAB_ROOT is required: point it at a local checkout of the Codex CUA Lab fixture',
);
}
const labRoot = requireComputerUseLabRoot();
const statePath = join(labRoot, 'test-app/runtime/state.json');
const fixtureBundleId = 'com.openai.codex.cualab';
const expectedAppPath = join(labRoot, 'test-app/build/Codex CUA Lab.app');
Expand Down