diff --git a/packages/loopover-miner/docs/cross-repo-evaluation.md b/packages/loopover-miner/docs/cross-repo-evaluation.md index cc36444b74..32b5691589 100644 --- a/packages/loopover-miner/docs/cross-repo-evaluation.md +++ b/packages/loopover-miner/docs/cross-repo-evaluation.md @@ -61,6 +61,53 @@ fleet run-manifest). - `--manifest path/to/manifest.json` — alternate benchmark set (e.g. a fixture manifest in tests) - `--require-majority` — exit `1` unless a strict majority of repos pass (for CI-style gating) +## Full-execution mode (#7634) + +Readiness answers *“can the miner form a plan for this repo?”*. **Full-execution** goes one step further and answers +*“does the miner actually produce working, correct code?”* by running the discover → plan → code → test loop against +each benchmark repo: it drives the configured coding agent to generate a **real diff**, then runs that target repo's +**own test suite** locally against the edited clone. + +It is **dry-run only**. The agent really edits the local clone (that is the only way to produce a diff), but the +harness hard-resets the clone back to `HEAD` afterward — it **never opens a PR, never pushes, and never touches the +third-party repo remotely**. It needs nothing beyond a local clone and a configured coding-agent driver; no write +access or forge credentials are required. + +### Running it + +```bash +node packages/loopover-miner/scripts/cross-repo-evaluation.mjs --full-execution +``` + +Combinable with the readiness flags: `--repo owner/repo`, `--json`, `--manifest path/to/manifest.json`, and +`--require-majority`. + +Prerequisites: + +- **Cloned benchmark repos** in `LOOPOVER_MINER_REPO_CLONE_DIR` (same as readiness — see + [Running locally](#running-locally)). +- **A configured coding-agent driver** via `MINER_CODING_AGENT_PROVIDER` plus that provider's credentials — the same + requirement a real attempt has. Without a configured driver, each repo reports an `other` execution failure (no diff + can be generated). + +### Execution failure taxonomy + +Failures are categorized under `CROSS_REPO_EXECUTION_CATEGORY`, extending the readiness taxonomy for the code + test +loop: + +| Category | Meaning | +| --- | --- | +| `plan_not_formed` | Readiness failed — the miner couldn't even form a plan for the repo | +| `code_build_failed` | Plan formed and the agent produced a diff, but the code didn't compile/build | +| `tests_failed` | Code compiled but the target repo's own tests failed | +| `no_op_diff` | Tests passed but the agent's diff was empty (no real change — a trivial pass) | +| `clone_setup` | The repo isn't cloned to the expected path | +| `other` | Unexpected error (e.g. no coding-agent driver configured, no inferred test command, agent crashed) | + +The stages run **in order** — plan → code → build → tests → no-op check — so a repo that fails an earlier stage is +reported against that stage (e.g. a repo that never forms a plan reports `plan_not_formed` and the build/test stages +never run). + ## Library API Pure functions live in [`lib/cross-repo-evaluation.js`](../lib/cross-repo-evaluation.js): @@ -71,8 +118,22 @@ Pure functions live in [`lib/cross-repo-evaluation.js`](../lib/cross-repo-evalua - `summarizeCrossRepoEvaluation(results)` - `formatCrossRepoEvaluationReport(results, summary)` +Full-execution mode (#7634) adds: + +- `evaluateRepoFullExecution(entry, options)` — async; inject the `runAgentAttempt`, `buildRepo`, and `runRepoTests` + seams for unit tests +- `runFullCrossRepoExecution(parsed, options)` — async +- `summarizeCrossRepoExecution(results)` +- `formatCrossRepoExecutionReport(results, summary)` + ## Wiring -This harness is **readiness-only**: it does not run the coding agent, open PRs, or call forge APIs. A green report -means the miner’s repo-agnostic stack-detection and coding-task-spec path is prepared for the benchmark repo; a live -attempt still needs credentials, governor policy, and queue state as documented in [`DEPLOYMENT.md`](../DEPLOYMENT.md). +The harness has **two modes**, and **neither** opens a live PR or performs any forge writes: + +- **Readiness-only** (default): it does not run the coding agent, open PRs, or call forge APIs. A green report means + the miner’s repo-agnostic stack-detection and coding-task-spec path is prepared for the benchmark repo. +- **Dry-run full-execution** (`--full-execution`, #7634): it runs the coding agent and the target repo's own tests + against a local clone, then hard-resets the clone — still no live PR and no forge writes. + +A live attempt still needs credentials, governor policy, and queue state as documented in +[`DEPLOYMENT.md`](../DEPLOYMENT.md). diff --git a/packages/loopover-miner/lib/cross-repo-evaluation.d.ts b/packages/loopover-miner/lib/cross-repo-evaluation.d.ts index af8e928558..b0a21ad47e 100644 --- a/packages/loopover-miner/lib/cross-repo-evaluation.d.ts +++ b/packages/loopover-miner/lib/cross-repo-evaluation.d.ts @@ -50,7 +50,7 @@ export type CrossRepoEvaluationSummary = { withoutLoopoverConfig: number; failuresByCategory: Record; }; -type EvaluateRepoReadinessOptions = { +export type EvaluateRepoReadinessOptions = { repoPath?: string; resolveRepoPath?: (entry: { repoFullName: string; @@ -100,4 +100,72 @@ export declare function summarizeCrossRepoEvaluation(results: CrossRepoEvaluatio * Human-readable pass/fail report for one evaluation run (#4788). */ export declare function formatCrossRepoEvaluationReport(results: CrossRepoEvaluationResult[], summary?: CrossRepoEvaluationSummary): string; -export {}; +/** Execution-stage failure taxonomy (#7634): extends the readiness taxonomy for the code + test loop. Ordered by + * pipeline stage, so a repo that fails an earlier stage is reported against that stage. */ +export declare const CROSS_REPO_EXECUTION_CATEGORY: Readonly<{ + PLAN_NOT_FORMED: "plan_not_formed"; + CODE_BUILD_FAILED: "code_build_failed"; + TESTS_FAILED: "tests_failed"; + NO_OP_DIFF: "no_op_diff"; + CLONE_SETUP: "clone_setup"; + OTHER: "other"; +}>; +export type CrossRepoExecutionResult = { + repoFullName: string; + passed: boolean; + executionCategory: string | null; + reason: string | null; + readinessPassed: boolean; + diffPresent: boolean | null; + built: boolean | null; + testsPassed: boolean | null; + stack?: RepoStackResult | undefined; +}; +export type CrossRepoExecutionSummary = { + total: number; + passed: number; + failed: number; + majorityPassed: boolean; + failuresByCategory: Record; +}; +/** Injectable local-execution seams (#7634). Real implementations (child_process build/test, the coding-agent + * driver) are wired by the CLI; unit tests inject fakes. Every seam is dry-run: it operates on the local clone + * only, and the harness never pushes or opens a PR. */ +export type CrossRepoExecutionSeams = { + runAgentAttempt?: (context: { + repoFullName: string; + repoPath: string; + stack: RepoStackResult; + }) => Promise<{ + diff: string; + }>; + buildRepo?: (context: { + repoPath: string; + command: string; + }) => Promise<{ + ok: boolean; + detail?: string; + }>; + runRepoTests?: (context: { + repoPath: string; + command: string; + }) => Promise<{ + ok: boolean; + detail?: string; + }>; +}; +export type EvaluateRepoFullExecutionOptions = EvaluateRepoReadinessOptions & CrossRepoExecutionSeams; +/** + * Run the full discover -> plan -> code -> test loop for one benchmark repo in dry-run (#7634). Reuses + * evaluateRepoReadiness for the plan stage, then delegates the code + build + test steps to injectable seams so the + * orchestration + taxonomy stay unit-testable without a live coding agent. Never pushes or opens a PR. + */ +export declare function evaluateRepoFullExecution(entry: CrossRepoEvaluationManifestRepo, options?: EvaluateRepoFullExecutionOptions): Promise; +/** Run full-execution across every repo in a parsed manifest (#7634). Async: each repo runs the real code+test loop. */ +export declare function runFullCrossRepoExecution(parsed: ParsedCrossRepoEvaluationManifest, options?: { + repoFilter?: string; +} & EvaluateRepoFullExecutionOptions): Promise; +/** Reduce full-execution results to pass/fail counts + a strict-majority verdict (#7634). */ +export declare function summarizeCrossRepoExecution(results: CrossRepoExecutionResult[]): CrossRepoExecutionSummary; +/** Human-readable full-execution report (#7634), mirroring formatCrossRepoEvaluationReport's shape. */ +export declare function formatCrossRepoExecutionReport(results: CrossRepoExecutionResult[], summary?: CrossRepoExecutionSummary): string; diff --git a/packages/loopover-miner/lib/cross-repo-evaluation.js b/packages/loopover-miner/lib/cross-repo-evaluation.js index 995ae19eba..5fabf143f3 100644 --- a/packages/loopover-miner/lib/cross-repo-evaluation.js +++ b/packages/loopover-miner/lib/cross-repo-evaluation.js @@ -343,4 +343,159 @@ export function formatCrossRepoEvaluationReport(results, summary = summarizeCros } return lines.join("\n"); } -//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY3Jvc3MtcmVwby1ldmFsdWF0aW9uLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiY3Jvc3MtcmVwby1ldmFsdWF0aW9uLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGlIQUFpSDtBQUNqSCw4R0FBOEc7QUFDOUcsMkdBQTJHO0FBQzNHLDZHQUE2RztBQUM3RyxxR0FBcUc7QUFFckcsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLFNBQVMsQ0FBQztBQUVyQyxPQUFPLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUM1RCxPQUFPLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUM1RCxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUMxRSxPQUFPLEVBQUUsZUFBZSxFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFHdkQsNkRBQTZEO0FBQzdELE1BQU0sQ0FBQyxNQUFNLDJCQUEyQixHQU1uQyxNQUFNLENBQUMsTUFBTSxDQUFDO0lBQ2pCLGVBQWUsRUFBRSxxQkFBcUI7SUFDdEMsU0FBUyxFQUFFLGVBQWU7SUFDMUIsb0JBQW9CLEVBQUUscUJBQXFCO0lBQzNDLFdBQVcsRUFBRSxhQUFhO0lBQzFCLEtBQUssRUFBRSxPQUFPO0NBQ2YsQ0FBQyxDQUFDO0FBRUg7bUdBQ21HO0FBQ25HLE1BQU0sQ0FBQyxNQUFNLG9DQUFvQyxHQUFtRCxNQUFNLENBQUMsTUFBTSxDQUFDO0lBQ2hILEVBQUUsRUFBRSxFQUFFLGdCQUFnQixFQUFFLE9BQU8sRUFBRSxrQkFBa0IsRUFBRTtJQUNyRCxFQUFFLEVBQUUsRUFBRSxlQUFlLEVBQUUsT0FBTyxFQUFFLGlCQUFpQixFQUFFO0lBQ25ELEVBQUUsRUFBRSxFQUFFLGlCQUFpQixFQUFFLE9BQU8sRUFBRSxxQ0FBcUMsRUFBRTtJQUN6RSxFQUFFLEVBQUUsRUFBRSxlQUFlLEVBQUUsT0FBTyxFQUFFLGdCQUFnQixFQUFFO0NBQ25ELENBQUMsQ0FBQztBQUVILE1BQU0sQ0FBQyxNQUFNLHlDQUF5QyxHQUFXLHFDQUFxQyxDQUFDO0FBQ3ZHLE1BQU0sQ0FBQyxNQUFNLDZCQUE2QixHQUFXLE1BQU0sQ0FBQztBQUM1RCxNQUFNLENBQUMsTUFBTSw2QkFBNkIsR0FBVyxHQUFHLENBQUM7QUFpRHpELGlIQUFpSDtBQUNqSCxnSEFBZ0g7QUFDaEgsbUhBQW1IO0FBQ25ILCtHQUErRztBQUMvRywwQkFBMEI7QUFDMUIsU0FBUyxjQUFjLENBQUMsS0FBYTtJQUNuQyxJQUFJLEtBQUssR0FBRyxDQUFDLENBQUM7SUFDZCxLQUFLLE1BQU0sSUFBSSxJQUFJLEtBQUssRUFBRSxDQUFDO1FBQ3pCLE1BQU0sU0FBUyxHQUFHLElBQUksQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFFLENBQUM7UUFDdkMsSUFBSSxTQUFTLElBQUksSUFBSTtZQUFFLEtBQUssSUFBSSxDQUFDLENBQUM7YUFDN0IsSUFBSSxTQUFTLElBQUksS0FBSztZQUFFLEtBQUssSUFBSSxDQUFDLENBQUM7YUFDbkMsSUFBSSxTQUFTLElBQUksTUFBTTtZQUFFLEtBQUssSUFBSSxDQUFDLENBQUM7O1lBQ3BDLEtBQUssSUFBSSxDQUFDLENBQUM7SUFDbEIsQ0FBQztJQUNELE9BQU8sS0FBSyxDQUFDO0FBQ2YsQ0FBQztBQUVELFNBQVMsa0JBQWtCLENBQUMsV0FBcUIsRUFBRTtJQUNqRCxPQUFPLEVBQUUsT0FBTyxFQUFFLEtBQUssRUFBRSxRQUFRLEVBQUUsRUFBRSxLQUFLLEVBQUUsRUFBRSxFQUFFLEVBQUUsUUFBUSxFQUFFLENBQUM7QUFDL0QsQ0FBQztBQUVELDZGQUE2RjtBQUM3RixNQUFNLFVBQVUsMEJBQTBCLENBQUMsS0FBYztJQUN2RCxJQUFJLE9BQU8sS0FBSyxLQUFLLFFBQVE7UUFBRSxPQUFPLElBQUksQ0FBQztJQUMzQyxNQUFNLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxLQUFLLENBQUMsR0FBRyxLQUFLLENBQUMsSUFBSSxFQUFFLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQ3JELElBQUksQ0FBQyxLQUFLLElBQUksQ0FBQyxJQUFJLElBQUksS0FBSyxLQUFLLFNBQVM7UUFBRSxPQUFPLElBQUksQ0FBQztJQUN4RCxJQUFJLENBQUMsa0JBQWtCLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxJQUFJLENBQUM7UUFBRSxPQUFPLElBQUksQ0FBQztJQUN6RSxPQUFPLEdBQUcsS0FBSyxJQUFJLElBQUksRUFBRSxDQUFDO0FBQzVCLENBQUM7QUFFRCxTQUFTLGdCQUFnQixDQUFDLEtBQWMsRUFBRSxLQUFhLEVBQUUsUUFBaUIsRUFBRSxRQUFrQjtJQUM1RixJQUFJLEtBQUssS0FBSyxTQUFTLElBQUksS0FBSyxLQUFLLElBQUk7UUFBRSxPQUFPLFFBQVEsQ0FBQztJQUMzRCxJQUFJLE9BQU8sS0FBSyxLQUFLLFNBQVM7UUFBRSxPQUFPLEtBQUssQ0FBQztJQUM3QyxRQUFRLENBQUMsSUFBSSxDQUFDLHNDQUFzQyxLQUFLLHdDQUF3QyxRQUFRLEdBQUcsQ0FBQyxDQUFDO0lBQzlHLE9BQU8sUUFBUSxDQUFDO0FBQ2xCLENBQUM7QUFFRCxTQUFTLHVCQUF1QixDQUFDLEtBQWMsRUFBRSxLQUFhLEVBQUUsUUFBa0I7SUFDaEYsSUFBSSxLQUFLLEtBQUssU0FBUyxJQUFJLEtBQUssS0FBSyxJQUFJO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDdkQsSUFBSSxPQUFPLEtBQUssS0FBSyxRQUFRLEVBQUUsQ0FBQztRQUM5QixRQUFRLENBQUMsSUFBSSxDQUFDLHNDQUFzQyxLQUFLLHlDQUF5QyxDQUFDLENBQUM7UUFDcEcsT0FBTyxJQUFJLENBQUM7SUFDZCxDQUFDO0lBQ0QsTUFBTSxPQUFPLEdBQUcsS0FBSyxDQUFDLElBQUksRUFBRSxDQUFDO0lBQzdCLE9BQU8sT0FBTyxJQUFJLElBQUksQ0FBQztBQUN6QixDQUFDO0FBRUQsU0FBUyxpQkFBaUIsQ0FBQyxLQUFjLEVBQUUsUUFBa0I7SUFDM0QsSUFBSSxLQUFLLEtBQUssU0FBUyxJQUFJLEtBQUssS0FBSyxJQUFJO1FBQUUsT0FBTyxFQUFFLENBQUM7SUFDckQsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztRQUMxQixRQUFRLENBQUMsSUFBSSxDQUFDLHdFQUF3RSxPQUFPLEtBQUssU0FBUyxDQUFDLENBQUM7UUFDN0csT0FBTyxFQUFFLENBQUM7SUFDWixDQUFDO0lBQ0QsTUFBTSxNQUFNLEdBQXNDLEVBQUUsQ0FBQztJQUNyRCxNQUFNLElBQUksR0FBRyxJQUFJLEdBQUcsRUFBVSxDQUFDO0lBQy9CLEtBQUssTUFBTSxDQUFDLEtBQUssRUFBRSxLQUFLLENBQUMsSUFBSSxLQUFLLENBQUMsT0FBTyxFQUFFLEVBQUUsQ0FBQztRQUM3QyxJQUFJLEtBQUssSUFBSSw2QkFBNkIsRUFBRSxDQUFDO1lBQzNDLFFBQVEsQ0FBQyxJQUFJLENBQ1gsc0RBQXNELDZCQUE2QixrQ0FBa0MsQ0FDdEgsQ0FBQztZQUNGLE1BQU07UUFDUixDQUFDO1FBQ0QsSUFBSSxZQUFZLEdBQWtCLElBQUksQ0FBQztRQUN2QyxJQUFJLFNBQVMsR0FBa0IsSUFBSSxDQUFDO1FBQ3BDLElBQUksa0JBQWtCLEdBQUcsS0FBSyxDQUFDO1FBQy9CLElBQUksV0FBVyxHQUFrQixJQUFJLENBQUM7UUFDdEMsSUFBSSxPQUFPLEtBQUssS0FBSyxRQUFRLEVBQUUsQ0FBQztZQUM5QixZQUFZLEdBQUcsMEJBQTBCLENBQUMsS0FBSyxDQUFDLENBQUM7UUFDbkQsQ0FBQzthQUFNLElBQUksS0FBSyxJQUFJLE9BQU8sS0FBSyxLQUFLLFFBQVEsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztZQUN2RSxNQUFNLE1BQU0sR0FBRyxLQUFnQyxDQUFDO1lBQ2hELFlBQVksR0FBRywwQkFBMEIsQ0FBQyxNQUFNLENBQUMsWUFBWSxDQUFDLENBQUM7WUFDL0QsU0FBUyxHQUFHLHVCQUF1QixDQUFDLE1BQU0sQ0FBQyxTQUFTLEVBQUUsV0FBVyxFQUFFLFFBQVEsQ0FBQyxDQUFDO1lBQzdFLGtCQUFrQixHQUFHLGdCQUFnQixDQUFDLE1BQU0sQ0FBQyxrQkFBa0IsRUFBRSxvQkFBb0IsRUFBRSxLQUFLLEVBQUUsUUFBUSxDQUFDLENBQUM7WUFDeEcsV0FBVyxHQUFHLHVCQUF1QixDQUFDLE1BQU0sQ0FBQyxXQUFXLEVBQUUsYUFBYSxFQUFFLFFBQVEsQ0FBQyxDQUFDO1FBQ3JGLENBQUM7YUFBTSxDQUFDO1lBQ04sUUFBUSxDQUFDLElBQUksQ0FBQyw4RUFBOEUsQ0FBQyxDQUFDO1lBQzlGLFNBQVM7UUFDWCxDQUFDO1FBQ0QsSUFBSSxZQUFZLEtBQUssSUFBSSxFQUFFLENBQUM7WUFDMUIsUUFBUSxDQUFDLElBQUksQ0FBQyx5RkFBeUYsQ0FBQyxDQUFDO1lBQ3pHLFNBQVM7UUFDWCxDQUFDO1FBQ0QsSUFBSSxJQUFJLENBQUMsR0FBRyxDQUFDLFlBQVksQ0FBQyxFQUFFLENBQUM7WUFDM0IsUUFBUSxDQUFDLElBQUksQ0FBQyxxRUFBcUUsWUFBWSxHQUFHLENBQUMsQ0FBQztZQUNwRyxTQUFTO1FBQ1gsQ0FBQztRQUNELElBQUksQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUM7UUFDdkIsTUFBTSxVQUFVLEdBQW9DLEVBQUUsWUFBWSxFQUFFLGtCQUFrQixFQUFFLENBQUM7UUFDekYsSUFBSSxTQUFTO1lBQUUsVUFBVSxDQUFDLFNBQVMsR0FBRyxTQUFTLENBQUM7UUFDaEQsSUFBSSxXQUFXO1lBQUUsVUFBVSxDQUFDLFdBQVcsR0FBRyxXQUFXLENBQUM7UUFDdEQsTUFBTSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztJQUMxQixDQUFDO0lBQ0QsT0FBTyxNQUFNLENBQUM7QUFDaEIsQ0FBQztBQUVEOzs7R0FHRztBQUNILE1BQU0sVUFBVSxnQ0FBZ0MsQ0FDOUMsT0FBa0M7SUFFbEMsSUFBSSxPQUFPLEtBQUssU0FBUyxJQUFJLE9BQU8sS0FBSyxJQUFJO1FBQUUsT0FBTyxrQkFBa0IsRUFBRSxDQUFDO0lBQzNFLElBQUksT0FBTyxPQUFPLEtBQUssUUFBUSxFQUFFLENBQUM7UUFDaEMsT0FBTyxrQkFBa0IsQ0FBQyxDQUFDLDZEQUE2RCxPQUFPLE9BQU8sR0FBRyxDQUFDLENBQUMsQ0FBQztJQUM5RyxDQUFDO0lBQ0QsTUFBTSxPQUFPLEdBQUcsT0FBTyxDQUFDLElBQUksRUFBRSxDQUFDO0lBQy9CLElBQUksQ0FBQyxPQUFPO1FBQUUsT0FBTyxrQkFBa0IsRUFBRSxDQUFDO0lBQzFDLElBQUksY0FBYyxDQUFDLE9BQU8sQ0FBQyxHQUFHLDZCQUE2QixFQUFFLENBQUM7UUFDNUQsT0FBTyxrQkFBa0IsQ0FBQztZQUN4Qix3Q0FBd0MsNkJBQTZCLDRCQUE0QjtTQUNsRyxDQUFDLENBQUM7SUFDTCxDQUFDO0lBQ0QsSUFBSSxHQUFZLENBQUM7SUFDakIsSUFBSSxDQUFDO1FBQ0gsR0FBRyxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLENBQUM7SUFDNUIsQ0FBQztJQUFDLE1BQU0sQ0FBQztRQUNQLE9BQU8sa0JBQWtCLENBQUMsQ0FBQyxnREFBZ0QsQ0FBQyxDQUFDLENBQUM7SUFDaEYsQ0FBQztJQUNELElBQUksQ0FBQyxHQUFHLElBQUksT0FBTyxHQUFHLEtBQUssUUFBUSxJQUFJLEtBQUssQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQztRQUMxRCxPQUFPLGtCQUFrQixDQUFDLENBQUMseURBQXlELENBQUMsQ0FBQyxDQUFDO0lBQ3pGLENBQUM7SUFDRCxNQUFNLFFBQVEsR0FBYSxFQUFFLENBQUM7SUFDOUIsTUFBTSxLQUFLLEdBQUcsaUJBQWlCLENBQUUsR0FBMkIsQ0FBQyxLQUFLLEVBQUUsUUFBUSxDQUFDLENBQUM7SUFDOUUsT0FBTyxFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsUUFBUSxFQUFFLEVBQUUsS0FBSyxFQUFFLEVBQUUsUUFBUSxFQUFFLENBQUM7QUFDMUQsQ0FBQztBQUVEOzs7R0FHRztBQUNILE1BQU0sVUFBVSwrQkFBK0IsQ0FBQyxJQUFZO0lBQzFELElBQUksT0FBTyxJQUFJLEtBQUssUUFBUTtRQUFFLE9BQU8sRUFBRSxDQUFDO0lBQ3hDLE1BQU0sUUFBUSxHQUF3QyxFQUFFLENBQUM7SUFDekQsS0FBSyxNQUFNLElBQUksSUFBSSxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUM7UUFDcEMsTUFBTSxPQUFPLEdBQUcsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO1FBQzVCLElBQUksQ0FBQyxPQUFPLElBQUksZ0JBQWdCLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQztZQUFFLFNBQVM7UUFDekQsS0FBSyxNQUFNLEtBQUssSUFBSSxvQ0FBb0MsRUFBRSxDQUFDO1lBQ3pELElBQUksS0FBSyxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDO2dCQUFFLFFBQVEsQ0FBQyxJQUFJLENBQUMsRUFBRSxFQUFFLEVBQUUsS0FBSyxDQUFDLEVBQUUsRUFBRSxJQUFJLEVBQUUsT0FBTyxFQUFFLENBQUMsQ0FBQztRQUMvRSxDQUFDO0lBQ0gsQ0FBQztJQUNELE9BQU8sUUFBUSxDQUFDO0FBQ2xCLENBQUM7QUFFRCxTQUFTLFlBQVksQ0FDbkIsWUFBb0IsRUFDcEIsUUFBZ0IsRUFDaEIsTUFBYyxFQUNkLFFBQTRDLEVBQUU7SUFFOUMsT0FBTztRQUNMLFlBQVk7UUFDWixNQUFNLEVBQUUsS0FBSztRQUNiLGVBQWUsRUFBRSxRQUFRO1FBQ3pCLE1BQU07UUFDTixhQUFhLEVBQUUsS0FBSztRQUNwQixtQkFBbUIsRUFBRSxJQUFJO1FBQ3pCLGtCQUFrQixFQUFFLEVBQUU7UUFDdEIsR0FBRyxLQUFLO0tBQ1QsQ0FBQztBQUNKLENBQUM7QUFFRCxTQUFTLFNBQVMsQ0FBQyxZQUFvQixFQUFFLFFBQTRDLEVBQUU7SUFDckYsT0FBTztRQUNMLFlBQVk7UUFDWixNQUFNLEVBQUUsSUFBSTtRQUNaLGVBQWUsRUFBRSxJQUFJO1FBQ3JCLE1BQU0sRUFBRSxJQUFJO1FBQ1osYUFBYSxFQUFFLElBQUk7UUFDbkIsbUJBQW1CLEVBQUUsSUFBSTtRQUN6QixrQkFBa0IsRUFBRSxFQUFFO1FBQ3RCLEdBQUcsS0FBSztLQUNULENBQUM7QUFDSixDQUFDO0FBRUQsU0FBUyx5QkFBeUIsQ0FDaEMsS0FBc0MsRUFDdEMsVUFBd0MsRUFBRTtJQUUxQyxJQUFJLEtBQUssQ0FBQyxXQUFXLElBQUksT0FBTyxLQUFLLENBQUMsV0FBVyxLQUFLLFFBQVE7UUFBRSxPQUFPLEtBQUssQ0FBQyxXQUFXLENBQUM7SUFDekYsSUFBSSxPQUFPLE9BQU8sQ0FBQyxRQUFRLEtBQUssUUFBUSxJQUFJLE9BQU8sQ0FBQyxRQUFRLENBQUMsSUFBSSxFQUFFO1FBQUUsT0FBTyxPQUFPLENBQUMsUUFBUSxDQUFDLElBQUksRUFBRSxDQUFDO0lBQ3BHLElBQUksT0FBTyxPQUFPLENBQUMsZUFBZSxLQUFLLFVBQVU7UUFBRSxPQUFPLE9BQU8sQ0FBQyxlQUFlLENBQUMsS0FBSyxDQUFDLENBQUM7SUFDekYsT0FBTyxtQkFBbUIsQ0FBQyxLQUFLLENBQUMsWUFBWSxFQUFFLE9BQU8sQ0FBQyxHQUFHLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0FBQzdFLENBQUM7QUFFRCxTQUFTLGtCQUFrQixDQUFDLFlBQW9CO0lBQzlDLE9BQU8sRUFBRSxVQUFVLEVBQUUsR0FBRyxFQUFFLENBQUMsRUFBRSxFQUFFLENBQUM7QUFDbEMsQ0FBQztBQUVEOztHQUVHO0FBQ0gsTUFBTSxVQUFVLHFCQUFxQixDQUNuQyxLQUFzQyxFQUN0QyxVQUF3QyxFQUFFO0lBRTFDLE1BQU0sWUFBWSxHQUFHLEtBQUssRUFBRSxZQUFZLENBQUM7SUFDekMsSUFBSSxPQUFPLFlBQVksS0FBSyxRQUFRLElBQUksQ0FBQywwQkFBMEIsQ0FBQyxZQUFZLENBQUMsRUFBRSxDQUFDO1FBQ2xGLE9BQU8sWUFBWSxDQUNqQixPQUFPLFlBQVksS0FBSyxRQUFRLENBQUMsQ0FBQyxDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUMsV0FBVyxFQUM3RCwyQkFBMkIsQ0FBQyxLQUFLLEVBQ2pDLHFEQUFxRCxDQUN0RCxDQUFDO0lBQ0osQ0FBQztJQUVELE1BQU0sVUFBVSxHQUFHLE9BQU8sQ0FBQyxVQUFVLElBQUksVUFBVSxDQUFDO0lBQ3BELE1BQU0sVUFBVSxHQUFHLE9BQU8sQ0FBQyxlQUFlLElBQUksZUFBZSxDQUFDO0lBQzlELE1BQU0sWUFBWSxHQUFHLE9BQU8sQ0FBQyxvQkFBb0IsSUFBSSxvQkFBb0IsQ0FBQztJQUMxRSxNQUFNLGFBQWEsR0FDakIsT0FBTyxDQUFDLG1CQUFtQjtRQUMxQixtQkFBbUcsQ0FBQztJQUN2RyxNQUFNLFFBQVEsR0FBRyx5QkFBeUIsQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLENBQUM7SUFFM0QsSUFBSSxDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsRUFBRSxDQUFDO1FBQzFCLE9BQU8sWUFBWSxDQUNqQixZQUFZLEVBQ1osMkJBQTJCLENBQUMsV0FBVyxFQUN2QyxtQ0FBbUMsUUFBUSx3REFBd0QsQ0FDcEcsQ0FBQztJQUNKLENBQUM7SUFFRCxNQUFNLFFBQVEsR0FBRyxZQUFZLENBQUMsUUFBUSxDQUFDLENBQUM7SUFDeEMsTUFBTSxtQkFBbUIsR0FBRyxRQUFRLEVBQUUsT0FBTyxLQUFLLElBQUksQ0FBQztJQUV2RCxNQUFNLEtBQUssR0FBRyxVQUFVLENBQUMsUUFBUSxDQUFDLENBQUM7SUFDbkMsSUFBSSxLQUFLLEVBQUUsUUFBUSxLQUFLLElBQUksRUFBRSxDQUFDO1FBQzdCLE9BQU8sWUFBWSxDQUNqQixZQUFZLEVBQ1osMkJBQTJCLENBQUMsZUFBZSxFQUMzQyxLQUFLLEVBQUUsTUFBTSxJQUFJLHlEQUF5RCxFQUMxRSxFQUFFLGFBQWEsRUFBRSxLQUFLLEVBQUUsbUJBQW1CLEVBQUUsQ0FDOUMsQ0FBQztJQUNKLENBQUM7SUFFRCxJQUFJLEtBQUssQ0FBQyxrQkFBa0IsS0FBSyxJQUFJLElBQUksQ0FBQyxLQUFLLENBQUMsV0FBVyxFQUFFLENBQUM7UUFDNUQsT0FBTyxZQUFZLENBQ2pCLFlBQVksRUFDWiwyQkFBMkIsQ0FBQyxTQUFTLEVBQ3JDLDZGQUE2RixFQUM3RixFQUFFLGFBQWEsRUFBRSxJQUFJLEVBQUUsbUJBQW1CLEVBQUUsS0FBSyxFQUFFLENBQ3BELENBQUM7SUFDSixDQUFDO0lBRUQsSUFBSSxVQUFVLENBQUM7SUFDZixJQUFJLENBQUM7UUFDSCxVQUFVLEdBQUcsYUFBYSxDQUFDO1lBQ3pCLFlBQVk7WUFDWixLQUFLLEVBQUU7Z0JBQ0wsTUFBTSxFQUFFLENBQUM7Z0JBQ1QsS0FBSyxFQUFFLDJDQUEyQztnQkFDbEQsSUFBSSxFQUFFLGlFQUFpRTtnQkFDdkUsTUFBTSxFQUFFLENBQUMsS0FBSyxDQUFDO2FBQ2hCO1lBQ0QsT0FBTyxFQUFFLEVBQUUsTUFBTSxFQUFFLENBQUMsRUFBRSxNQUFNLEVBQUUsQ0FBQyxFQUFFLENBQUMsRUFBRSxZQUFZLEVBQUUsRUFBRSxFQUFFO1lBQ3RELFdBQVcsRUFBRSxrQkFBa0IsQ0FBQyxZQUFZLENBQUM7WUFDN0MsZ0JBQWdCLEVBQUUsUUFBUTtZQUMxQixlQUFlLEVBQUUsVUFBVTtTQUM1QixDQUFDLENBQUM7SUFDTCxDQUFDO0lBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztRQUNmLE1BQU0sT0FBTyxHQUFHLEtBQUssWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUN2RSxPQUFPLFlBQVksQ0FBQyxZQUFZLEVBQUUsMkJBQTJCLENBQUMsS0FBSyxFQUFFLE9BQU8sRUFBRTtZQUM1RSxhQUFhLEVBQUUsSUFBSTtZQUNuQixtQkFBbUI7WUFDbkIsS0FBSztTQUNOLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRCxJQUFJLFVBQVUsRUFBRSxLQUFLLEtBQUssSUFBSSxFQUFFLENBQUM7UUFDL0IsT0FBTyxZQUFZLENBQ2pCLFlBQVksRUFDWiwyQkFBMkIsQ0FBQyxTQUFTLEVBQ3JDLDJDQUEyQyxVQUFVLEVBQUUsT0FBTyxJQUFJLFNBQVMsSUFBSSxFQUMvRSxFQUFFLGFBQWEsRUFBRSxJQUFJLEVBQUUsbUJBQW1CLEVBQUUsS0FBSyxFQUFFLENBQ3BELENBQUM7SUFDSixDQUFDO0lBRUQsTUFBTSxrQkFBa0IsR0FBRywrQkFBK0IsQ0FBQyxVQUFVLENBQUMsWUFBWSxJQUFJLEVBQUUsQ0FBQyxDQUFDO0lBQzFGLElBQUksa0JBQWtCLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDO1FBQ2xDLE9BQU8sWUFBWSxDQUNqQixZQUFZLEVBQ1osMkJBQTJCLENBQUMsb0JBQW9CLEVBQ2hELDBEQUEwRCxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksRUFDNUcsRUFBRSxhQUFhLEVBQUUsSUFBSSxFQUFFLG1CQUFtQixFQUFFLEtBQUssRUFBRSxrQkFBa0IsRUFBRSxDQUN4RSxDQUFDO0lBQ0osQ0FBQztJQUVELE9BQU8sU0FBUyxDQUFDLFlBQVksRUFBRSxFQUFFLG1CQUFtQixFQUFFLEtBQUssRUFBRSxDQUFDLENBQUM7QUFDakUsQ0FBQztBQUVEOztHQUVHO0FBQ0gsTUFBTSxVQUFVLHNCQUFzQixDQUNwQyxNQUF5QyxFQUN6QyxVQUFrRSxFQUFFO0lBRXBFLE1BQU0sS0FBSyxHQUFHLE1BQU0sRUFBRSxRQUFRLEVBQUUsS0FBSyxJQUFJLEVBQUUsQ0FBQztJQUM1QyxNQUFNLE9BQU8sR0FBZ0MsRUFBRSxDQUFDO0lBQ2hELEtBQUssTUFBTSxLQUFLLElBQUksS0FBSyxFQUFFLENBQUM7UUFDMUIsSUFBSSxPQUFPLENBQUMsVUFBVSxJQUFJLEtBQUssQ0FBQyxZQUFZLEtBQUssT0FBTyxDQUFDLFVBQVU7WUFBRSxTQUFTO1FBQzlFLE9BQU8sQ0FBQyxJQUFJLENBQUMscUJBQXFCLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFDLENBQUM7SUFDdEQsQ0FBQztJQUNELE9BQU8sT0FBTyxDQUFDO0FBQ2pCLENBQUM7QUFFRDs7R0FFRztBQUNILE1BQU0sVUFBVSw0QkFBNEIsQ0FBQyxPQUFvQztJQUMvRSxNQUFNLElBQUksR0FBRyxLQUFLLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztJQUNuRCxJQUFJLE1BQU0sR0FBRyxDQUFDLENBQUM7SUFDZixJQUFJLE1BQU0sR0FBRyxDQUFDLENBQUM7SUFDZixNQUFNLGtCQUFrQixHQUEyQixFQUFFLENBQUM7SUFDdEQsS0FBSyxNQUFNLE1BQU0sSUFBSSxJQUFJLEVBQUUsQ0FBQztRQUMxQixJQUFJLE1BQU0sRUFBRSxNQUFNLEtBQUssSUFBSSxFQUFFLENBQUM7WUFDNUIsTUFBTSxJQUFJLENBQUMsQ0FBQztZQUNaLFNBQVM7UUFDWCxDQUFDO1FBQ0QsTUFBTSxJQUFJLENBQUMsQ0FBQztRQUNaLE1BQU0sUUFBUSxHQUFHLE1BQU0sRUFBRSxlQUFlLElBQUksMkJBQTJCLENBQUMsS0FBSyxDQUFDO1FBQzlFLGtCQUFrQixDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsa0JBQWtCLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQ3pFLENBQUM7SUFDRCxNQUFNLEtBQUssR0FBRyxNQUFNLEdBQUcsTUFBTSxDQUFDO0lBQzlCLE1BQU0sY0FBYyxHQUFHLEtBQUssR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLE1BQU0sR0FBRyxNQUFNLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQztJQUMzRCxNQUFNLHFCQUFxQixHQUFHLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsRUFBRSxtQkFBbUIsS0FBSyxLQUFLLENBQUMsQ0FBQyxNQUFNLENBQUM7SUFDMUYsT0FBTztRQUNMLEtBQUs7UUFDTCxNQUFNO1FBQ04sTUFBTTtRQUNOLGNBQWM7UUFDZCxxQkFBcUI7UUFDckIsa0JBQWtCO0tBQ25CLENBQUM7QUFDSixDQUFDO0FBRUQ7O0dBRUc7QUFDSCxNQUFNLFVBQVUsK0JBQStCLENBQzdDLE9BQW9DLEVBQ3BDLFVBQXNDLDRCQUE0QixDQUFDLE9BQU8sQ0FBQztJQUUzRSxNQUFNLEtBQUssR0FBRyxDQUFDLHNDQUFzQyxFQUFFLEVBQUUsQ0FBQyxDQUFDO0lBQzNELEtBQUssTUFBTSxNQUFNLElBQUksT0FBTyxFQUFFLENBQUM7UUFDN0IsSUFBSSxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUM7WUFDbEIsS0FBSyxDQUFDLElBQUksQ0FBQyxRQUFRLE1BQU0sQ0FBQyxZQUFZLEVBQUUsQ0FBQyxDQUFDO1lBQzFDLFNBQVM7UUFDWCxDQUFDO1FBQ0QsS0FBSyxDQUFDLElBQUksQ0FBQyxRQUFRLE1BQU0sQ0FBQyxZQUFZLEtBQUssTUFBTSxDQUFDLGVBQWUsS0FBSyxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQztJQUN6RixDQUFDO0lBQ0QsS0FBSyxDQUFDLElBQUksQ0FDUixFQUFFLEVBQ0YsWUFBWSxPQUFPLENBQUMsTUFBTSxJQUFJLE9BQU8sQ0FBQyxLQUFLLFNBQVM7UUFDbEQsQ0FBQyxPQUFPLENBQUMsY0FBYyxDQUFDLENBQUMsQ0FBQyxvQkFBb0IsQ0FBQyxDQUFDLENBQUMsb0JBQW9CLENBQUMsQ0FDekUsQ0FBQztJQUNGLElBQUksT0FBTyxDQUFDLEtBQUssR0FBRyxDQUFDLEVBQUUsQ0FBQztRQUN0QixLQUFLLENBQUMsSUFBSSxDQUFDLDRDQUE0QyxPQUFPLENBQUMscUJBQXFCLElBQUksT0FBTyxDQUFDLEtBQUssRUFBRSxDQUFDLENBQUM7SUFDM0csQ0FBQztJQUNELE1BQU0sVUFBVSxHQUFHLE1BQU0sQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLGtCQUFrQixDQUFDLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUMsYUFBYSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7SUFDckcsSUFBSSxVQUFVLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDO1FBQzFCLEtBQUssQ0FBQyxJQUFJLENBQUMsRUFBRSxFQUFFLHVCQUF1QixDQUFDLENBQUM7UUFDeEMsS0FBSyxNQUFNLENBQUMsUUFBUSxFQUFFLEtBQUssQ0FBQyxJQUFJLFVBQVUsRUFBRSxDQUFDO1lBQzNDLEtBQUssQ0FBQyxJQUFJLENBQUMsS0FBSyxRQUFRLEtBQUssS0FBSyxFQUFFLENBQUMsQ0FBQztRQUN4QyxDQUFDO0lBQ0gsQ0FBQztJQUNELE9BQU8sS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztBQUMxQixDQUFDIn0= \ No newline at end of file +// --- Full-execution mode (#7634) -------------------------------------------------------------------------------- +// The readiness harness above answers "can the miner form a plan for this repo?". Full-execution goes one step +// further and answers "does the miner actually produce working, correct code?" by running the discover -> plan -> +// code -> test loop against a benchmark repo, then checking the generated code compiles, the repo's own tests pass, +// and the diff is not a no-op. DRY-RUN ONLY: it edits the local clone and discards -- it never opens a PR, never +// pushes, and never touches the third-party repo remotely (the same safety posture as the readiness harness). +/** Execution-stage failure taxonomy (#7634): extends the readiness taxonomy for the code + test loop. Ordered by + * pipeline stage, so a repo that fails an earlier stage is reported against that stage. */ +export const CROSS_REPO_EXECUTION_CATEGORY = Object.freeze({ + PLAN_NOT_FORMED: "plan_not_formed", + CODE_BUILD_FAILED: "code_build_failed", + TESTS_FAILED: "tests_failed", + NO_OP_DIFF: "no_op_diff", + CLONE_SETUP: "clone_setup", + OTHER: "other", +}); +function buildExecutionFailure(repoFullName, category, reason, extra = {}) { + return { + repoFullName, + passed: false, + executionCategory: category, + reason, + readinessPassed: false, + diffPresent: null, + built: null, + testsPassed: null, + ...extra, + }; +} +/** + * Run the full discover -> plan -> code -> test loop for one benchmark repo in dry-run (#7634). Reuses + * evaluateRepoReadiness for the plan stage, then delegates the code + build + test steps to injectable seams so the + * orchestration + taxonomy stay unit-testable without a live coding agent. Never pushes or opens a PR. + */ +export async function evaluateRepoFullExecution(entry, options = {}) { + const readiness = evaluateRepoReadiness(entry, options); + const repoFullName = readiness.repoFullName; + if (!readiness.passed) { + // A clone/setup gap stays clone_setup; any other readiness failure means no plan could be formed. + const category = readiness.failureCategory === CROSS_REPO_FAILURE_CATEGORY.CLONE_SETUP + ? CROSS_REPO_EXECUTION_CATEGORY.CLONE_SETUP + : CROSS_REPO_EXECUTION_CATEGORY.PLAN_NOT_FORMED; + return buildExecutionFailure(repoFullName, category, readiness.reason, { + stack: readiness.stack, + }); + } + // Readiness passed, so the stack is present and detected -- narrow to the detected shape so its testCommand / + // buildCommand read as plain string|null with no defensive (unreachable) re-check. + const stack = readiness.stack; + const testCommand = stack.testCommand; + if (!testCommand) { + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_CATEGORY.OTHER, "No test command was inferred; full execution needs the repo's own test suite to run.", { readinessPassed: true, stack }); + } + const runAgentAttempt = options.runAgentAttempt; + if (typeof runAgentAttempt !== "function") { + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_CATEGORY.OTHER, "No coding-agent runner was provided; full execution cannot generate a diff.", { readinessPassed: true, stack }); + } + const repoPath = resolveEvaluationRepoPath(entry, options); + let diff; + try { + const attempt = await runAgentAttempt({ repoFullName, repoPath, stack }); + diff = attempt.diff; + } + catch (error) { + const message = error instanceof Error ? error.message : String(error); + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_CATEGORY.OTHER, `Coding agent failed: ${message}`, { + readinessPassed: true, + stack, + }); + } + const diffPresent = diff.trim().length > 0; + // Compile/build the edited clone when the stack exposes a build command -- a failure means the agent's code + // doesn't compile. + const buildCommand = stack.buildCommand; + if (buildCommand && typeof options.buildRepo === "function") { + const built = await options.buildRepo({ repoPath, command: buildCommand }); + if (!built.ok) { + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_CATEGORY.CODE_BUILD_FAILED, `Build failed: ${built.detail ?? buildCommand}`, { readinessPassed: true, diffPresent, built: false, stack }); + } + } + // Run the target repo's own test suite against the edited clone. + const runRepoTests = options.runRepoTests; + if (typeof runRepoTests !== "function") { + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_CATEGORY.OTHER, "No test runner was provided; full execution cannot run the repo's tests.", { readinessPassed: true, diffPresent, built: true, stack }); + } + const tested = await runRepoTests({ repoPath, command: testCommand }); + if (!tested.ok) { + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_CATEGORY.TESTS_FAILED, `Tests failed: ${tested.detail ?? testCommand}`, { readinessPassed: true, diffPresent, built: true, testsPassed: false, stack }); + } + // Tests passed -- but an empty diff means the agent changed nothing, so the pass is trivial (no real code). + if (!diffPresent) { + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_CATEGORY.NO_OP_DIFF, "Tests passed but the agent produced an empty diff (no real change).", { readinessPassed: true, diffPresent: false, built: true, testsPassed: true, stack }); + } + return { + repoFullName, + passed: true, + executionCategory: null, + reason: null, + readinessPassed: true, + diffPresent: true, + built: true, + testsPassed: true, + stack, + }; +} +/** Run full-execution across every repo in a parsed manifest (#7634). Async: each repo runs the real code+test loop. */ +export async function runFullCrossRepoExecution(parsed, options = {}) { + const repos = parsed?.manifest?.repos ?? []; + const results = []; + for (const entry of repos) { + if (options.repoFilter && entry.repoFullName !== options.repoFilter) + continue; + results.push(await evaluateRepoFullExecution(entry, options)); + } + return results; +} +/** Reduce full-execution results to pass/fail counts + a strict-majority verdict (#7634). */ +export function summarizeCrossRepoExecution(results) { + const list = Array.isArray(results) ? results : []; + let passed = 0; + let failed = 0; + const failuresByCategory = {}; + for (const result of list) { + if (result?.passed === true) { + passed += 1; + continue; + } + failed += 1; + const category = result?.executionCategory ?? CROSS_REPO_EXECUTION_CATEGORY.OTHER; + failuresByCategory[category] = (failuresByCategory[category] ?? 0) + 1; + } + const total = passed + failed; + return { total, passed, failed, majorityPassed: total > 0 ? passed > failed : false, failuresByCategory }; +} +/** Human-readable full-execution report (#7634), mirroring formatCrossRepoEvaluationReport's shape. */ +export function formatCrossRepoExecutionReport(results, summary = summarizeCrossRepoExecution(results)) { + const lines = ["loopover-miner cross-repo full execution", ""]; + for (const result of results) { + if (result.passed) { + lines.push(`PASS ${result.repoFullName}`); + continue; + } + lines.push(`FAIL ${result.repoFullName} [${result.executionCategory}] ${result.reason}`); + } + lines.push("", `summary: ${summary.passed}/${summary.total} passed` + + (summary.majorityPassed ? " (majority passed)" : " (majority failed)")); + const categories = Object.entries(summary.failuresByCategory).sort(([a], [b]) => a.localeCompare(b)); + if (categories.length > 0) { + lines.push("", "failures by category:"); + for (const [category, count] of categories) { + lines.push(`- ${category}: ${count}`); + } + } + return lines.join("\n"); +} +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY3Jvc3MtcmVwby1ldmFsdWF0aW9uLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiY3Jvc3MtcmVwby1ldmFsdWF0aW9uLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGlIQUFpSDtBQUNqSCw4R0FBOEc7QUFDOUcsMkdBQTJHO0FBQzNHLDZHQUE2RztBQUM3RyxxR0FBcUc7QUFFckcsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLFNBQVMsQ0FBQztBQUVyQyxPQUFPLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUM1RCxPQUFPLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUM1RCxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUMxRSxPQUFPLEVBQUUsZUFBZSxFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFHdkQsNkRBQTZEO0FBQzdELE1BQU0sQ0FBQyxNQUFNLDJCQUEyQixHQU1uQyxNQUFNLENBQUMsTUFBTSxDQUFDO0lBQ2pCLGVBQWUsRUFBRSxxQkFBcUI7SUFDdEMsU0FBUyxFQUFFLGVBQWU7SUFDMUIsb0JBQW9CLEVBQUUscUJBQXFCO0lBQzNDLFdBQVcsRUFBRSxhQUFhO0lBQzFCLEtBQUssRUFBRSxPQUFPO0NBQ2YsQ0FBQyxDQUFDO0FBRUg7bUdBQ21HO0FBQ25HLE1BQU0sQ0FBQyxNQUFNLG9DQUFvQyxHQUFtRCxNQUFNLENBQUMsTUFBTSxDQUFDO0lBQ2hILEVBQUUsRUFBRSxFQUFFLGdCQUFnQixFQUFFLE9BQU8sRUFBRSxrQkFBa0IsRUFBRTtJQUNyRCxFQUFFLEVBQUUsRUFBRSxlQUFlLEVBQUUsT0FBTyxFQUFFLGlCQUFpQixFQUFFO0lBQ25ELEVBQUUsRUFBRSxFQUFFLGlCQUFpQixFQUFFLE9BQU8sRUFBRSxxQ0FBcUMsRUFBRTtJQUN6RSxFQUFFLEVBQUUsRUFBRSxlQUFlLEVBQUUsT0FBTyxFQUFFLGdCQUFnQixFQUFFO0NBQ25ELENBQUMsQ0FBQztBQUVILE1BQU0sQ0FBQyxNQUFNLHlDQUF5QyxHQUFXLHFDQUFxQyxDQUFDO0FBQ3ZHLE1BQU0sQ0FBQyxNQUFNLDZCQUE2QixHQUFXLE1BQU0sQ0FBQztBQUM1RCxNQUFNLENBQUMsTUFBTSw2QkFBNkIsR0FBVyxHQUFHLENBQUM7QUFpRHpELGlIQUFpSDtBQUNqSCxnSEFBZ0g7QUFDaEgsbUhBQW1IO0FBQ25ILCtHQUErRztBQUMvRywwQkFBMEI7QUFDMUIsU0FBUyxjQUFjLENBQUMsS0FBYTtJQUNuQyxJQUFJLEtBQUssR0FBRyxDQUFDLENBQUM7SUFDZCxLQUFLLE1BQU0sSUFBSSxJQUFJLEtBQUssRUFBRSxDQUFDO1FBQ3pCLE1BQU0sU0FBUyxHQUFHLElBQUksQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFFLENBQUM7UUFDdkMsSUFBSSxTQUFTLElBQUksSUFBSTtZQUFFLEtBQUssSUFBSSxDQUFDLENBQUM7YUFDN0IsSUFBSSxTQUFTLElBQUksS0FBSztZQUFFLEtBQUssSUFBSSxDQUFDLENBQUM7YUFDbkMsSUFBSSxTQUFTLElBQUksTUFBTTtZQUFFLEtBQUssSUFBSSxDQUFDLENBQUM7O1lBQ3BDLEtBQUssSUFBSSxDQUFDLENBQUM7SUFDbEIsQ0FBQztJQUNELE9BQU8sS0FBSyxDQUFDO0FBQ2YsQ0FBQztBQUVELFNBQVMsa0JBQWtCLENBQUMsV0FBcUIsRUFBRTtJQUNqRCxPQUFPLEVBQUUsT0FBTyxFQUFFLEtBQUssRUFBRSxRQUFRLEVBQUUsRUFBRSxLQUFLLEVBQUUsRUFBRSxFQUFFLEVBQUUsUUFBUSxFQUFFLENBQUM7QUFDL0QsQ0FBQztBQUVELDZGQUE2RjtBQUM3RixNQUFNLFVBQVUsMEJBQTBCLENBQUMsS0FBYztJQUN2RCxJQUFJLE9BQU8sS0FBSyxLQUFLLFFBQVE7UUFBRSxPQUFPLElBQUksQ0FBQztJQUMzQyxNQUFNLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxLQUFLLENBQUMsR0FBRyxLQUFLLENBQUMsSUFBSSxFQUFFLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQ3JELElBQUksQ0FBQyxLQUFLLElBQUksQ0FBQyxJQUFJLElBQUksS0FBSyxLQUFLLFNBQVM7UUFBRSxPQUFPLElBQUksQ0FBQztJQUN4RCxJQUFJLENBQUMsa0JBQWtCLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxJQUFJLENBQUM7UUFBRSxPQUFPLElBQUksQ0FBQztJQUN6RSxPQUFPLEdBQUcsS0FBSyxJQUFJLElBQUksRUFBRSxDQUFDO0FBQzVCLENBQUM7QUFFRCxTQUFTLGdCQUFnQixDQUFDLEtBQWMsRUFBRSxLQUFhLEVBQUUsUUFBaUIsRUFBRSxRQUFrQjtJQUM1RixJQUFJLEtBQUssS0FBSyxTQUFTLElBQUksS0FBSyxLQUFLLElBQUk7UUFBRSxPQUFPLFFBQVEsQ0FBQztJQUMzRCxJQUFJLE9BQU8sS0FBSyxLQUFLLFNBQVM7UUFBRSxPQUFPLEtBQUssQ0FBQztJQUM3QyxRQUFRLENBQUMsSUFBSSxDQUFDLHNDQUFzQyxLQUFLLHdDQUF3QyxRQUFRLEdBQUcsQ0FBQyxDQUFDO0lBQzlHLE9BQU8sUUFBUSxDQUFDO0FBQ2xCLENBQUM7QUFFRCxTQUFTLHVCQUF1QixDQUFDLEtBQWMsRUFBRSxLQUFhLEVBQUUsUUFBa0I7SUFDaEYsSUFBSSxLQUFLLEtBQUssU0FBUyxJQUFJLEtBQUssS0FBSyxJQUFJO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDdkQsSUFBSSxPQUFPLEtBQUssS0FBSyxRQUFRLEVBQUUsQ0FBQztRQUM5QixRQUFRLENBQUMsSUFBSSxDQUFDLHNDQUFzQyxLQUFLLHlDQUF5QyxDQUFDLENBQUM7UUFDcEcsT0FBTyxJQUFJLENBQUM7SUFDZCxDQUFDO0lBQ0QsTUFBTSxPQUFPLEdBQUcsS0FBSyxDQUFDLElBQUksRUFBRSxDQUFDO0lBQzdCLE9BQU8sT0FBTyxJQUFJLElBQUksQ0FBQztBQUN6QixDQUFDO0FBRUQsU0FBUyxpQkFBaUIsQ0FBQyxLQUFjLEVBQUUsUUFBa0I7SUFDM0QsSUFBSSxLQUFLLEtBQUssU0FBUyxJQUFJLEtBQUssS0FBSyxJQUFJO1FBQUUsT0FBTyxFQUFFLENBQUM7SUFDckQsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztRQUMxQixRQUFRLENBQUMsSUFBSSxDQUFDLHdFQUF3RSxPQUFPLEtBQUssU0FBUyxDQUFDLENBQUM7UUFDN0csT0FBTyxFQUFFLENBQUM7SUFDWixDQUFDO0lBQ0QsTUFBTSxNQUFNLEdBQXNDLEVBQUUsQ0FBQztJQUNyRCxNQUFNLElBQUksR0FBRyxJQUFJLEdBQUcsRUFBVSxDQUFDO0lBQy9CLEtBQUssTUFBTSxDQUFDLEtBQUssRUFBRSxLQUFLLENBQUMsSUFBSSxLQUFLLENBQUMsT0FBTyxFQUFFLEVBQUUsQ0FBQztRQUM3QyxJQUFJLEtBQUssSUFBSSw2QkFBNkIsRUFBRSxDQUFDO1lBQzNDLFFBQVEsQ0FBQyxJQUFJLENBQ1gsc0RBQXNELDZCQUE2QixrQ0FBa0MsQ0FDdEgsQ0FBQztZQUNGLE1BQU07UUFDUixDQUFDO1FBQ0QsSUFBSSxZQUFZLEdBQWtCLElBQUksQ0FBQztRQUN2QyxJQUFJLFNBQVMsR0FBa0IsSUFBSSxDQUFDO1FBQ3BDLElBQUksa0JBQWtCLEdBQUcsS0FBSyxDQUFDO1FBQy9CLElBQUksV0FBVyxHQUFrQixJQUFJLENBQUM7UUFDdEMsSUFBSSxPQUFPLEtBQUssS0FBSyxRQUFRLEVBQUUsQ0FBQztZQUM5QixZQUFZLEdBQUcsMEJBQTBCLENBQUMsS0FBSyxDQUFDLENBQUM7UUFDbkQsQ0FBQzthQUFNLElBQUksS0FBSyxJQUFJLE9BQU8sS0FBSyxLQUFLLFFBQVEsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztZQUN2RSxNQUFNLE1BQU0sR0FBRyxLQUFnQyxDQUFDO1lBQ2hELFlBQVksR0FBRywwQkFBMEIsQ0FBQyxNQUFNLENBQUMsWUFBWSxDQUFDLENBQUM7WUFDL0QsU0FBUyxHQUFHLHVCQUF1QixDQUFDLE1BQU0sQ0FBQyxTQUFTLEVBQUUsV0FBVyxFQUFFLFFBQVEsQ0FBQyxDQUFDO1lBQzdFLGtCQUFrQixHQUFHLGdCQUFnQixDQUFDLE1BQU0sQ0FBQyxrQkFBa0IsRUFBRSxvQkFBb0IsRUFBRSxLQUFLLEVBQUUsUUFBUSxDQUFDLENBQUM7WUFDeEcsV0FBVyxHQUFHLHVCQUF1QixDQUFDLE1BQU0sQ0FBQyxXQUFXLEVBQUUsYUFBYSxFQUFFLFFBQVEsQ0FBQyxDQUFDO1FBQ3JGLENBQUM7YUFBTSxDQUFDO1lBQ04sUUFBUSxDQUFDLElBQUksQ0FBQyw4RUFBOEUsQ0FBQyxDQUFDO1lBQzlGLFNBQVM7UUFDWCxDQUFDO1FBQ0QsSUFBSSxZQUFZLEtBQUssSUFBSSxFQUFFLENBQUM7WUFDMUIsUUFBUSxDQUFDLElBQUksQ0FBQyx5RkFBeUYsQ0FBQyxDQUFDO1lBQ3pHLFNBQVM7UUFDWCxDQUFDO1FBQ0QsSUFBSSxJQUFJLENBQUMsR0FBRyxDQUFDLFlBQVksQ0FBQyxFQUFFLENBQUM7WUFDM0IsUUFBUSxDQUFDLElBQUksQ0FBQyxxRUFBcUUsWUFBWSxHQUFHLENBQUMsQ0FBQztZQUNwRyxTQUFTO1FBQ1gsQ0FBQztRQUNELElBQUksQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUM7UUFDdkIsTUFBTSxVQUFVLEdBQW9DLEVBQUUsWUFBWSxFQUFFLGtCQUFrQixFQUFFLENBQUM7UUFDekYsSUFBSSxTQUFTO1lBQUUsVUFBVSxDQUFDLFNBQVMsR0FBRyxTQUFTLENBQUM7UUFDaEQsSUFBSSxXQUFXO1lBQUUsVUFBVSxDQUFDLFdBQVcsR0FBRyxXQUFXLENBQUM7UUFDdEQsTUFBTSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztJQUMxQixDQUFDO0lBQ0QsT0FBTyxNQUFNLENBQUM7QUFDaEIsQ0FBQztBQUVEOzs7R0FHRztBQUNILE1BQU0sVUFBVSxnQ0FBZ0MsQ0FDOUMsT0FBa0M7SUFFbEMsSUFBSSxPQUFPLEtBQUssU0FBUyxJQUFJLE9BQU8sS0FBSyxJQUFJO1FBQUUsT0FBTyxrQkFBa0IsRUFBRSxDQUFDO0lBQzNFLElBQUksT0FBTyxPQUFPLEtBQUssUUFBUSxFQUFFLENBQUM7UUFDaEMsT0FBTyxrQkFBa0IsQ0FBQyxDQUFDLDZEQUE2RCxPQUFPLE9BQU8sR0FBRyxDQUFDLENBQUMsQ0FBQztJQUM5RyxDQUFDO0lBQ0QsTUFBTSxPQUFPLEdBQUcsT0FBTyxDQUFDLElBQUksRUFBRSxDQUFDO0lBQy9CLElBQUksQ0FBQyxPQUFPO1FBQUUsT0FBTyxrQkFBa0IsRUFBRSxDQUFDO0lBQzFDLElBQUksY0FBYyxDQUFDLE9BQU8sQ0FBQyxHQUFHLDZCQUE2QixFQUFFLENBQUM7UUFDNUQsT0FBTyxrQkFBa0IsQ0FBQztZQUN4Qix3Q0FBd0MsNkJBQTZCLDRCQUE0QjtTQUNsRyxDQUFDLENBQUM7SUFDTCxDQUFDO0lBQ0QsSUFBSSxHQUFZLENBQUM7SUFDakIsSUFBSSxDQUFDO1FBQ0gsR0FBRyxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLENBQUM7SUFDNUIsQ0FBQztJQUFDLE1BQU0sQ0FBQztRQUNQLE9BQU8sa0JBQWtCLENBQUMsQ0FBQyxnREFBZ0QsQ0FBQyxDQUFDLENBQUM7SUFDaEYsQ0FBQztJQUNELElBQUksQ0FBQyxHQUFHLElBQUksT0FBTyxHQUFHLEtBQUssUUFBUSxJQUFJLEtBQUssQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQztRQUMxRCxPQUFPLGtCQUFrQixDQUFDLENBQUMseURBQXlELENBQUMsQ0FBQyxDQUFDO0lBQ3pGLENBQUM7SUFDRCxNQUFNLFFBQVEsR0FBYSxFQUFFLENBQUM7SUFDOUIsTUFBTSxLQUFLLEdBQUcsaUJBQWlCLENBQUUsR0FBMkIsQ0FBQyxLQUFLLEVBQUUsUUFBUSxDQUFDLENBQUM7SUFDOUUsT0FBTyxFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsUUFBUSxFQUFFLEVBQUUsS0FBSyxFQUFFLEVBQUUsUUFBUSxFQUFFLENBQUM7QUFDMUQsQ0FBQztBQUVEOzs7R0FHRztBQUNILE1BQU0sVUFBVSwrQkFBK0IsQ0FBQyxJQUFZO0lBQzFELElBQUksT0FBTyxJQUFJLEtBQUssUUFBUTtRQUFFLE9BQU8sRUFBRSxDQUFDO0lBQ3hDLE1BQU0sUUFBUSxHQUF3QyxFQUFFLENBQUM7SUFDekQsS0FBSyxNQUFNLElBQUksSUFBSSxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUM7UUFDcEMsTUFBTSxPQUFPLEdBQUcsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO1FBQzVCLElBQUksQ0FBQyxPQUFPLElBQUksZ0JBQWdCLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQztZQUFFLFNBQVM7UUFDekQsS0FBSyxNQUFNLEtBQUssSUFBSSxvQ0FBb0MsRUFBRSxDQUFDO1lBQ3pELElBQUksS0FBSyxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDO2dCQUFFLFFBQVEsQ0FBQyxJQUFJLENBQUMsRUFBRSxFQUFFLEVBQUUsS0FBSyxDQUFDLEVBQUUsRUFBRSxJQUFJLEVBQUUsT0FBTyxFQUFFLENBQUMsQ0FBQztRQUMvRSxDQUFDO0lBQ0gsQ0FBQztJQUNELE9BQU8sUUFBUSxDQUFDO0FBQ2xCLENBQUM7QUFFRCxTQUFTLFlBQVksQ0FDbkIsWUFBb0IsRUFDcEIsUUFBZ0IsRUFDaEIsTUFBYyxFQUNkLFFBQTRDLEVBQUU7SUFFOUMsT0FBTztRQUNMLFlBQVk7UUFDWixNQUFNLEVBQUUsS0FBSztRQUNiLGVBQWUsRUFBRSxRQUFRO1FBQ3pCLE1BQU07UUFDTixhQUFhLEVBQUUsS0FBSztRQUNwQixtQkFBbUIsRUFBRSxJQUFJO1FBQ3pCLGtCQUFrQixFQUFFLEVBQUU7UUFDdEIsR0FBRyxLQUFLO0tBQ1QsQ0FBQztBQUNKLENBQUM7QUFFRCxTQUFTLFNBQVMsQ0FBQyxZQUFvQixFQUFFLFFBQTRDLEVBQUU7SUFDckYsT0FBTztRQUNMLFlBQVk7UUFDWixNQUFNLEVBQUUsSUFBSTtRQUNaLGVBQWUsRUFBRSxJQUFJO1FBQ3JCLE1BQU0sRUFBRSxJQUFJO1FBQ1osYUFBYSxFQUFFLElBQUk7UUFDbkIsbUJBQW1CLEVBQUUsSUFBSTtRQUN6QixrQkFBa0IsRUFBRSxFQUFFO1FBQ3RCLEdBQUcsS0FBSztLQUNULENBQUM7QUFDSixDQUFDO0FBRUQsU0FBUyx5QkFBeUIsQ0FDaEMsS0FBc0MsRUFDdEMsVUFBd0MsRUFBRTtJQUUxQyxJQUFJLEtBQUssQ0FBQyxXQUFXLElBQUksT0FBTyxLQUFLLENBQUMsV0FBVyxLQUFLLFFBQVE7UUFBRSxPQUFPLEtBQUssQ0FBQyxXQUFXLENBQUM7SUFDekYsSUFBSSxPQUFPLE9BQU8sQ0FBQyxRQUFRLEtBQUssUUFBUSxJQUFJLE9BQU8sQ0FBQyxRQUFRLENBQUMsSUFBSSxFQUFFO1FBQUUsT0FBTyxPQUFPLENBQUMsUUFBUSxDQUFDLElBQUksRUFBRSxDQUFDO0lBQ3BHLElBQUksT0FBTyxPQUFPLENBQUMsZUFBZSxLQUFLLFVBQVU7UUFBRSxPQUFPLE9BQU8sQ0FBQyxlQUFlLENBQUMsS0FBSyxDQUFDLENBQUM7SUFDekYsT0FBTyxtQkFBbUIsQ0FBQyxLQUFLLENBQUMsWUFBWSxFQUFFLE9BQU8sQ0FBQyxHQUFHLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0FBQzdFLENBQUM7QUFFRCxTQUFTLGtCQUFrQixDQUFDLFlBQW9CO0lBQzlDLE9BQU8sRUFBRSxVQUFVLEVBQUUsR0FBRyxFQUFFLENBQUMsRUFBRSxFQUFFLENBQUM7QUFDbEMsQ0FBQztBQUVEOztHQUVHO0FBQ0gsTUFBTSxVQUFVLHFCQUFxQixDQUNuQyxLQUFzQyxFQUN0QyxVQUF3QyxFQUFFO0lBRTFDLE1BQU0sWUFBWSxHQUFHLEtBQUssRUFBRSxZQUFZLENBQUM7SUFDekMsSUFBSSxPQUFPLFlBQVksS0FBSyxRQUFRLElBQUksQ0FBQywwQkFBMEIsQ0FBQyxZQUFZLENBQUMsRUFBRSxDQUFDO1FBQ2xGLE9BQU8sWUFBWSxDQUNqQixPQUFPLFlBQVksS0FBSyxRQUFRLENBQUMsQ0FBQyxDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUMsV0FBVyxFQUM3RCwyQkFBMkIsQ0FBQyxLQUFLLEVBQ2pDLHFEQUFxRCxDQUN0RCxDQUFDO0lBQ0osQ0FBQztJQUVELE1BQU0sVUFBVSxHQUFHLE9BQU8sQ0FBQyxVQUFVLElBQUksVUFBVSxDQUFDO0lBQ3BELE1BQU0sVUFBVSxHQUFHLE9BQU8sQ0FBQyxlQUFlLElBQUksZUFBZSxDQUFDO0lBQzlELE1BQU0sWUFBWSxHQUFHLE9BQU8sQ0FBQyxvQkFBb0IsSUFBSSxvQkFBb0IsQ0FBQztJQUMxRSxNQUFNLGFBQWEsR0FDakIsT0FBTyxDQUFDLG1CQUFtQjtRQUMxQixtQkFBbUcsQ0FBQztJQUN2RyxNQUFNLFFBQVEsR0FBRyx5QkFBeUIsQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLENBQUM7SUFFM0QsSUFBSSxDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsRUFBRSxDQUFDO1FBQzFCLE9BQU8sWUFBWSxDQUNqQixZQUFZLEVBQ1osMkJBQTJCLENBQUMsV0FBVyxFQUN2QyxtQ0FBbUMsUUFBUSx3REFBd0QsQ0FDcEcsQ0FBQztJQUNKLENBQUM7SUFFRCxNQUFNLFFBQVEsR0FBRyxZQUFZLENBQUMsUUFBUSxDQUFDLENBQUM7SUFDeEMsTUFBTSxtQkFBbUIsR0FBRyxRQUFRLEVBQUUsT0FBTyxLQUFLLElBQUksQ0FBQztJQUV2RCxNQUFNLEtBQUssR0FBRyxVQUFVLENBQUMsUUFBUSxDQUFDLENBQUM7SUFDbkMsSUFBSSxLQUFLLEVBQUUsUUFBUSxLQUFLLElBQUksRUFBRSxDQUFDO1FBQzdCLE9BQU8sWUFBWSxDQUNqQixZQUFZLEVBQ1osMkJBQTJCLENBQUMsZUFBZSxFQUMzQyxLQUFLLEVBQUUsTUFBTSxJQUFJLHlEQUF5RCxFQUMxRSxFQUFFLGFBQWEsRUFBRSxLQUFLLEVBQUUsbUJBQW1CLEVBQUUsQ0FDOUMsQ0FBQztJQUNKLENBQUM7SUFFRCxJQUFJLEtBQUssQ0FBQyxrQkFBa0IsS0FBSyxJQUFJLElBQUksQ0FBQyxLQUFLLENBQUMsV0FBVyxFQUFFLENBQUM7UUFDNUQsT0FBTyxZQUFZLENBQ2pCLFlBQVksRUFDWiwyQkFBMkIsQ0FBQyxTQUFTLEVBQ3JDLDZGQUE2RixFQUM3RixFQUFFLGFBQWEsRUFBRSxJQUFJLEVBQUUsbUJBQW1CLEVBQUUsS0FBSyxFQUFFLENBQ3BELENBQUM7SUFDSixDQUFDO0lBRUQsSUFBSSxVQUFVLENBQUM7SUFDZixJQUFJLENBQUM7UUFDSCxVQUFVLEdBQUcsYUFBYSxDQUFDO1lBQ3pCLFlBQVk7WUFDWixLQUFLLEVBQUU7Z0JBQ0wsTUFBTSxFQUFFLENBQUM7Z0JBQ1QsS0FBSyxFQUFFLDJDQUEyQztnQkFDbEQsSUFBSSxFQUFFLGlFQUFpRTtnQkFDdkUsTUFBTSxFQUFFLENBQUMsS0FBSyxDQUFDO2FBQ2hCO1lBQ0QsT0FBTyxFQUFFLEVBQUUsTUFBTSxFQUFFLENBQUMsRUFBRSxNQUFNLEVBQUUsQ0FBQyxFQUFFLENBQUMsRUFBRSxZQUFZLEVBQUUsRUFBRSxFQUFFO1lBQ3RELFdBQVcsRUFBRSxrQkFBa0IsQ0FBQyxZQUFZLENBQUM7WUFDN0MsZ0JBQWdCLEVBQUUsUUFBUTtZQUMxQixlQUFlLEVBQUUsVUFBVTtTQUM1QixDQUFDLENBQUM7SUFDTCxDQUFDO0lBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztRQUNmLE1BQU0sT0FBTyxHQUFHLEtBQUssWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUN2RSxPQUFPLFlBQVksQ0FBQyxZQUFZLEVBQUUsMkJBQTJCLENBQUMsS0FBSyxFQUFFLE9BQU8sRUFBRTtZQUM1RSxhQUFhLEVBQUUsSUFBSTtZQUNuQixtQkFBbUI7WUFDbkIsS0FBSztTQUNOLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRCxJQUFJLFVBQVUsRUFBRSxLQUFLLEtBQUssSUFBSSxFQUFFLENBQUM7UUFDL0IsT0FBTyxZQUFZLENBQ2pCLFlBQVksRUFDWiwyQkFBMkIsQ0FBQyxTQUFTLEVBQ3JDLDJDQUEyQyxVQUFVLEVBQUUsT0FBTyxJQUFJLFNBQVMsSUFBSSxFQUMvRSxFQUFFLGFBQWEsRUFBRSxJQUFJLEVBQUUsbUJBQW1CLEVBQUUsS0FBSyxFQUFFLENBQ3BELENBQUM7SUFDSixDQUFDO0lBRUQsTUFBTSxrQkFBa0IsR0FBRywrQkFBK0IsQ0FBQyxVQUFVLENBQUMsWUFBWSxJQUFJLEVBQUUsQ0FBQyxDQUFDO0lBQzFGLElBQUksa0JBQWtCLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDO1FBQ2xDLE9BQU8sWUFBWSxDQUNqQixZQUFZLEVBQ1osMkJBQTJCLENBQUMsb0JBQW9CLEVBQ2hELDBEQUEwRCxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksRUFDNUcsRUFBRSxhQUFhLEVBQUUsSUFBSSxFQUFFLG1CQUFtQixFQUFFLEtBQUssRUFBRSxrQkFBa0IsRUFBRSxDQUN4RSxDQUFDO0lBQ0osQ0FBQztJQUVELE9BQU8sU0FBUyxDQUFDLFlBQVksRUFBRSxFQUFFLG1CQUFtQixFQUFFLEtBQUssRUFBRSxDQUFDLENBQUM7QUFDakUsQ0FBQztBQUVEOztHQUVHO0FBQ0gsTUFBTSxVQUFVLHNCQUFzQixDQUNwQyxNQUF5QyxFQUN6QyxVQUFrRSxFQUFFO0lBRXBFLE1BQU0sS0FBSyxHQUFHLE1BQU0sRUFBRSxRQUFRLEVBQUUsS0FBSyxJQUFJLEVBQUUsQ0FBQztJQUM1QyxNQUFNLE9BQU8sR0FBZ0MsRUFBRSxDQUFDO0lBQ2hELEtBQUssTUFBTSxLQUFLLElBQUksS0FBSyxFQUFFLENBQUM7UUFDMUIsSUFBSSxPQUFPLENBQUMsVUFBVSxJQUFJLEtBQUssQ0FBQyxZQUFZLEtBQUssT0FBTyxDQUFDLFVBQVU7WUFBRSxTQUFTO1FBQzlFLE9BQU8sQ0FBQyxJQUFJLENBQUMscUJBQXFCLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFDLENBQUM7SUFDdEQsQ0FBQztJQUNELE9BQU8sT0FBTyxDQUFDO0FBQ2pCLENBQUM7QUFFRDs7R0FFRztBQUNILE1BQU0sVUFBVSw0QkFBNEIsQ0FBQyxPQUFvQztJQUMvRSxNQUFNLElBQUksR0FBRyxLQUFLLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztJQUNuRCxJQUFJLE1BQU0sR0FBRyxDQUFDLENBQUM7SUFDZixJQUFJLE1BQU0sR0FBRyxDQUFDLENBQUM7SUFDZixNQUFNLGtCQUFrQixHQUEyQixFQUFFLENBQUM7SUFDdEQsS0FBSyxNQUFNLE1BQU0sSUFBSSxJQUFJLEVBQUUsQ0FBQztRQUMxQixJQUFJLE1BQU0sRUFBRSxNQUFNLEtBQUssSUFBSSxFQUFFLENBQUM7WUFDNUIsTUFBTSxJQUFJLENBQUMsQ0FBQztZQUNaLFNBQVM7UUFDWCxDQUFDO1FBQ0QsTUFBTSxJQUFJLENBQUMsQ0FBQztRQUNaLE1BQU0sUUFBUSxHQUFHLE1BQU0sRUFBRSxlQUFlLElBQUksMkJBQTJCLENBQUMsS0FBSyxDQUFDO1FBQzlFLGtCQUFrQixDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsa0JBQWtCLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQ3pFLENBQUM7SUFDRCxNQUFNLEtBQUssR0FBRyxNQUFNLEdBQUcsTUFBTSxDQUFDO0lBQzlCLE1BQU0sY0FBYyxHQUFHLEtBQUssR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLE1BQU0sR0FBRyxNQUFNLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQztJQUMzRCxNQUFNLHFCQUFxQixHQUFHLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsRUFBRSxtQkFBbUIsS0FBSyxLQUFLLENBQUMsQ0FBQyxNQUFNLENBQUM7SUFDMUYsT0FBTztRQUNMLEtBQUs7UUFDTCxNQUFNO1FBQ04sTUFBTTtRQUNOLGNBQWM7UUFDZCxxQkFBcUI7UUFDckIsa0JBQWtCO0tBQ25CLENBQUM7QUFDSixDQUFDO0FBRUQ7O0dBRUc7QUFDSCxNQUFNLFVBQVUsK0JBQStCLENBQzdDLE9BQW9DLEVBQ3BDLFVBQXNDLDRCQUE0QixDQUFDLE9BQU8sQ0FBQztJQUUzRSxNQUFNLEtBQUssR0FBRyxDQUFDLHNDQUFzQyxFQUFFLEVBQUUsQ0FBQyxDQUFDO0lBQzNELEtBQUssTUFBTSxNQUFNLElBQUksT0FBTyxFQUFFLENBQUM7UUFDN0IsSUFBSSxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUM7WUFDbEIsS0FBSyxDQUFDLElBQUksQ0FBQyxRQUFRLE1BQU0sQ0FBQyxZQUFZLEVBQUUsQ0FBQyxDQUFDO1lBQzFDLFNBQVM7UUFDWCxDQUFDO1FBQ0QsS0FBSyxDQUFDLElBQUksQ0FBQyxRQUFRLE1BQU0sQ0FBQyxZQUFZLEtBQUssTUFBTSxDQUFDLGVBQWUsS0FBSyxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQztJQUN6RixDQUFDO0lBQ0QsS0FBSyxDQUFDLElBQUksQ0FDUixFQUFFLEVBQ0YsWUFBWSxPQUFPLENBQUMsTUFBTSxJQUFJLE9BQU8sQ0FBQyxLQUFLLFNBQVM7UUFDbEQsQ0FBQyxPQUFPLENBQUMsY0FBYyxDQUFDLENBQUMsQ0FBQyxvQkFBb0IsQ0FBQyxDQUFDLENBQUMsb0JBQW9CLENBQUMsQ0FDekUsQ0FBQztJQUNGLElBQUksT0FBTyxDQUFDLEtBQUssR0FBRyxDQUFDLEVBQUUsQ0FBQztRQUN0QixLQUFLLENBQUMsSUFBSSxDQUFDLDRDQUE0QyxPQUFPLENBQUMscUJBQXFCLElBQUksT0FBTyxDQUFDLEtBQUssRUFBRSxDQUFDLENBQUM7SUFDM0csQ0FBQztJQUNELE1BQU0sVUFBVSxHQUFHLE1BQU0sQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLGtCQUFrQixDQUFDLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUMsYUFBYSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7SUFDckcsSUFBSSxVQUFVLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDO1FBQzFCLEtBQUssQ0FBQyxJQUFJLENBQUMsRUFBRSxFQUFFLHVCQUF1QixDQUFDLENBQUM7UUFDeEMsS0FBSyxNQUFNLENBQUMsUUFBUSxFQUFFLEtBQUssQ0FBQyxJQUFJLFVBQVUsRUFBRSxDQUFDO1lBQzNDLEtBQUssQ0FBQyxJQUFJLENBQUMsS0FBSyxRQUFRLEtBQUssS0FBSyxFQUFFLENBQUMsQ0FBQztRQUN4QyxDQUFDO0lBQ0gsQ0FBQztJQUNELE9BQU8sS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztBQUMxQixDQUFDO0FBRUQsbUhBQW1IO0FBQ25ILCtHQUErRztBQUMvRyxrSEFBa0g7QUFDbEgsb0hBQW9IO0FBQ3BILGlIQUFpSDtBQUNqSCw4R0FBOEc7QUFFOUc7NEZBQzRGO0FBQzVGLE1BQU0sQ0FBQyxNQUFNLDZCQUE2QixHQU9yQyxNQUFNLENBQUMsTUFBTSxDQUFDO0lBQ2pCLGVBQWUsRUFBRSxpQkFBaUI7SUFDbEMsaUJBQWlCLEVBQUUsbUJBQW1CO0lBQ3RDLFlBQVksRUFBRSxjQUFjO0lBQzVCLFVBQVUsRUFBRSxZQUFZO0lBQ3hCLFdBQVcsRUFBRSxhQUFhO0lBQzFCLEtBQUssRUFBRSxPQUFPO0NBQ2YsQ0FBQyxDQUFDO0FBcUNILFNBQVMscUJBQXFCLENBQzVCLFlBQW9CLEVBQ3BCLFFBQWdCLEVBQ2hCLE1BQWMsRUFDZCxRQUEyQyxFQUFFO0lBRTdDLE9BQU87UUFDTCxZQUFZO1FBQ1osTUFBTSxFQUFFLEtBQUs7UUFDYixpQkFBaUIsRUFBRSxRQUFRO1FBQzNCLE1BQU07UUFDTixlQUFlLEVBQUUsS0FBSztRQUN0QixXQUFXLEVBQUUsSUFBSTtRQUNqQixLQUFLLEVBQUUsSUFBSTtRQUNYLFdBQVcsRUFBRSxJQUFJO1FBQ2pCLEdBQUcsS0FBSztLQUNULENBQUM7QUFDSixDQUFDO0FBRUQ7Ozs7R0FJRztBQUNILE1BQU0sQ0FBQyxLQUFLLFVBQVUseUJBQXlCLENBQzdDLEtBQXNDLEVBQ3RDLFVBQTRDLEVBQUU7SUFFOUMsTUFBTSxTQUFTLEdBQUcscUJBQXFCLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFDO0lBQ3hELE1BQU0sWUFBWSxHQUFHLFNBQVMsQ0FBQyxZQUFZLENBQUM7SUFDNUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxNQUFNLEVBQUUsQ0FBQztRQUN0QixrR0FBa0c7UUFDbEcsTUFBTSxRQUFRLEdBQ1osU0FBUyxDQUFDLGVBQWUsS0FBSywyQkFBMkIsQ0FBQyxXQUFXO1lBQ25FLENBQUMsQ0FBQyw2QkFBNkIsQ0FBQyxXQUFXO1lBQzNDLENBQUMsQ0FBQyw2QkFBNkIsQ0FBQyxlQUFlLENBQUM7UUFDcEQsT0FBTyxxQkFBcUIsQ0FBQyxZQUFZLEVBQUUsUUFBUSxFQUFFLFNBQVMsQ0FBQyxNQUFnQixFQUFFO1lBQy9FLEtBQUssRUFBRSxTQUFTLENBQUMsS0FBSztTQUN2QixDQUFDLENBQUM7SUFDTCxDQUFDO0lBRUQsOEdBQThHO0lBQzlHLG1GQUFtRjtJQUNuRixNQUFNLEtBQUssR0FBRyxTQUFTLENBQUMsS0FBcUQsQ0FBQztJQUM5RSxNQUFNLFdBQVcsR0FBRyxLQUFLLENBQUMsV0FBVyxDQUFDO0lBQ3RDLElBQUksQ0FBQyxXQUFXLEVBQUUsQ0FBQztRQUNqQixPQUFPLHFCQUFxQixDQUMxQixZQUFZLEVBQ1osNkJBQTZCLENBQUMsS0FBSyxFQUNuQyxzRkFBc0YsRUFDdEYsRUFBRSxlQUFlLEVBQUUsSUFBSSxFQUFFLEtBQUssRUFBRSxDQUNqQyxDQUFDO0lBQ0osQ0FBQztJQUVELE1BQU0sZUFBZSxHQUFHLE9BQU8sQ0FBQyxlQUFlLENBQUM7SUFDaEQsSUFBSSxPQUFPLGVBQWUsS0FBSyxVQUFVLEVBQUUsQ0FBQztRQUMxQyxPQUFPLHFCQUFxQixDQUMxQixZQUFZLEVBQ1osNkJBQTZCLENBQUMsS0FBSyxFQUNuQyw2RUFBNkUsRUFDN0UsRUFBRSxlQUFlLEVBQUUsSUFBSSxFQUFFLEtBQUssRUFBRSxDQUNqQyxDQUFDO0lBQ0osQ0FBQztJQUVELE1BQU0sUUFBUSxHQUFHLHlCQUF5QixDQUFDLEtBQUssRUFBRSxPQUFPLENBQUMsQ0FBQztJQUMzRCxJQUFJLElBQVksQ0FBQztJQUNqQixJQUFJLENBQUM7UUFDSCxNQUFNLE9BQU8sR0FBRyxNQUFNLGVBQWUsQ0FBQyxFQUFFLFlBQVksRUFBRSxRQUFRLEVBQUUsS0FBSyxFQUFFLENBQUMsQ0FBQztRQUN6RSxJQUFJLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQztJQUN0QixDQUFDO0lBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztRQUNmLE1BQU0sT0FBTyxHQUFHLEtBQUssWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUN2RSxPQUFPLHFCQUFxQixDQUFDLFlBQVksRUFBRSw2QkFBNkIsQ0FBQyxLQUFLLEVBQUUsd0JBQXdCLE9BQU8sRUFBRSxFQUFFO1lBQ2pILGVBQWUsRUFBRSxJQUFJO1lBQ3JCLEtBQUs7U0FDTixDQUFDLENBQUM7SUFDTCxDQUFDO0lBQ0QsTUFBTSxXQUFXLEdBQUcsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDLE1BQU0sR0FBRyxDQUFDLENBQUM7SUFFM0MsNEdBQTRHO0lBQzVHLG1CQUFtQjtJQUNuQixNQUFNLFlBQVksR0FBRyxLQUFLLENBQUMsWUFBWSxDQUFDO0lBQ3hDLElBQUksWUFBWSxJQUFJLE9BQU8sT0FBTyxDQUFDLFNBQVMsS0FBSyxVQUFVLEVBQUUsQ0FBQztRQUM1RCxNQUFNLEtBQUssR0FBRyxNQUFNLE9BQU8sQ0FBQyxTQUFTLENBQUMsRUFBRSxRQUFRLEVBQUUsT0FBTyxFQUFFLFlBQVksRUFBRSxDQUFDLENBQUM7UUFDM0UsSUFBSSxDQUFDLEtBQUssQ0FBQyxFQUFFLEVBQUUsQ0FBQztZQUNkLE9BQU8scUJBQXFCLENBQzFCLFlBQVksRUFDWiw2QkFBNkIsQ0FBQyxpQkFBaUIsRUFDL0MsaUJBQWlCLEtBQUssQ0FBQyxNQUFNLElBQUksWUFBWSxFQUFFLEVBQy9DLEVBQUUsZUFBZSxFQUFFLElBQUksRUFBRSxXQUFXLEVBQUUsS0FBSyxFQUFFLEtBQUssRUFBRSxLQUFLLEVBQUUsQ0FDNUQsQ0FBQztRQUNKLENBQUM7SUFDSCxDQUFDO0lBRUQsaUVBQWlFO0lBQ2pFLE1BQU0sWUFBWSxHQUFHLE9BQU8sQ0FBQyxZQUFZLENBQUM7SUFDMUMsSUFBSSxPQUFPLFlBQVksS0FBSyxVQUFVLEVBQUUsQ0FBQztRQUN2QyxPQUFPLHFCQUFxQixDQUMxQixZQUFZLEVBQ1osNkJBQTZCLENBQUMsS0FBSyxFQUNuQywwRUFBMEUsRUFDMUUsRUFBRSxlQUFlLEVBQUUsSUFBSSxFQUFFLFdBQVcsRUFBRSxLQUFLLEVBQUUsSUFBSSxFQUFFLEtBQUssRUFBRSxDQUMzRCxDQUFDO0lBQ0osQ0FBQztJQUNELE1BQU0sTUFBTSxHQUFHLE1BQU0sWUFBWSxDQUFDLEVBQUUsUUFBUSxFQUFFLE9BQU8sRUFBRSxXQUFXLEVBQUUsQ0FBQyxDQUFDO0lBQ3RFLElBQUksQ0FBQyxNQUFNLENBQUMsRUFBRSxFQUFFLENBQUM7UUFDZixPQUFPLHFCQUFxQixDQUMxQixZQUFZLEVBQ1osNkJBQTZCLENBQUMsWUFBWSxFQUMxQyxpQkFBaUIsTUFBTSxDQUFDLE1BQU0sSUFBSSxXQUFXLEVBQUUsRUFDL0MsRUFBRSxlQUFlLEVBQUUsSUFBSSxFQUFFLFdBQVcsRUFBRSxLQUFLLEVBQUUsSUFBSSxFQUFFLFdBQVcsRUFBRSxLQUFLLEVBQUUsS0FBSyxFQUFFLENBQy9FLENBQUM7SUFDSixDQUFDO0lBRUQsNEdBQTRHO0lBQzVHLElBQUksQ0FBQyxXQUFXLEVBQUUsQ0FBQztRQUNqQixPQUFPLHFCQUFxQixDQUMxQixZQUFZLEVBQ1osNkJBQTZCLENBQUMsVUFBVSxFQUN4QyxxRUFBcUUsRUFDckUsRUFBRSxlQUFlLEVBQUUsSUFBSSxFQUFFLFdBQVcsRUFBRSxLQUFLLEVBQUUsS0FBSyxFQUFFLElBQUksRUFBRSxXQUFXLEVBQUUsSUFBSSxFQUFFLEtBQUssRUFBRSxDQUNyRixDQUFDO0lBQ0osQ0FBQztJQUVELE9BQU87UUFDTCxZQUFZO1FBQ1osTUFBTSxFQUFFLElBQUk7UUFDWixpQkFBaUIsRUFBRSxJQUFJO1FBQ3ZCLE1BQU0sRUFBRSxJQUFJO1FBQ1osZUFBZSxFQUFFLElBQUk7UUFDckIsV0FBVyxFQUFFLElBQUk7UUFDakIsS0FBSyxFQUFFLElBQUk7UUFDWCxXQUFXLEVBQUUsSUFBSTtRQUNqQixLQUFLO0tBQ04sQ0FBQztBQUNKLENBQUM7QUFFRCx3SEFBd0g7QUFDeEgsTUFBTSxDQUFDLEtBQUssVUFBVSx5QkFBeUIsQ0FDN0MsTUFBeUMsRUFDekMsVUFBc0UsRUFBRTtJQUV4RSxNQUFNLEtBQUssR0FBRyxNQUFNLEVBQUUsUUFBUSxFQUFFLEtBQUssSUFBSSxFQUFFLENBQUM7SUFDNUMsTUFBTSxPQUFPLEdBQStCLEVBQUUsQ0FBQztJQUMvQyxLQUFLLE1BQU0sS0FBSyxJQUFJLEtBQUssRUFBRSxDQUFDO1FBQzFCLElBQUksT0FBTyxDQUFDLFVBQVUsSUFBSSxLQUFLLENBQUMsWUFBWSxLQUFLLE9BQU8sQ0FBQyxVQUFVO1lBQUUsU0FBUztRQUM5RSxPQUFPLENBQUMsSUFBSSxDQUFDLE1BQU0seUJBQXlCLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFDLENBQUM7SUFDaEUsQ0FBQztJQUNELE9BQU8sT0FBTyxDQUFDO0FBQ2pCLENBQUM7QUFFRCw2RkFBNkY7QUFDN0YsTUFBTSxVQUFVLDJCQUEyQixDQUFDLE9BQW1DO0lBQzdFLE1BQU0sSUFBSSxHQUFHLEtBQUssQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDO0lBQ25ELElBQUksTUFBTSxHQUFHLENBQUMsQ0FBQztJQUNmLElBQUksTUFBTSxHQUFHLENBQUMsQ0FBQztJQUNmLE1BQU0sa0JBQWtCLEdBQTJCLEVBQUUsQ0FBQztJQUN0RCxLQUFLLE1BQU0sTUFBTSxJQUFJLElBQUksRUFBRSxDQUFDO1FBQzFCLElBQUksTUFBTSxFQUFFLE1BQU0sS0FBSyxJQUFJLEVBQUUsQ0FBQztZQUM1QixNQUFNLElBQUksQ0FBQyxDQUFDO1lBQ1osU0FBUztRQUNYLENBQUM7UUFDRCxNQUFNLElBQUksQ0FBQyxDQUFDO1FBQ1osTUFBTSxRQUFRLEdBQUcsTUFBTSxFQUFFLGlCQUFpQixJQUFJLDZCQUE2QixDQUFDLEtBQUssQ0FBQztRQUNsRixrQkFBa0IsQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLGtCQUFrQixDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUN6RSxDQUFDO0lBQ0QsTUFBTSxLQUFLLEdBQUcsTUFBTSxHQUFHLE1BQU0sQ0FBQztJQUM5QixPQUFPLEVBQUUsS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLEVBQUUsY0FBYyxFQUFFLEtBQUssR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLE1BQU0sR0FBRyxNQUFNLENBQUMsQ0FBQyxDQUFDLEtBQUssRUFBRSxrQkFBa0IsRUFBRSxDQUFDO0FBQzVHLENBQUM7QUFFRCx1R0FBdUc7QUFDdkcsTUFBTSxVQUFVLDhCQUE4QixDQUM1QyxPQUFtQyxFQUNuQyxVQUFxQywyQkFBMkIsQ0FBQyxPQUFPLENBQUM7SUFFekUsTUFBTSxLQUFLLEdBQUcsQ0FBQywwQ0FBMEMsRUFBRSxFQUFFLENBQUMsQ0FBQztJQUMvRCxLQUFLLE1BQU0sTUFBTSxJQUFJLE9BQU8sRUFBRSxDQUFDO1FBQzdCLElBQUksTUFBTSxDQUFDLE1BQU0sRUFBRSxDQUFDO1lBQ2xCLEtBQUssQ0FBQyxJQUFJLENBQUMsUUFBUSxNQUFNLENBQUMsWUFBWSxFQUFFLENBQUMsQ0FBQztZQUMxQyxTQUFTO1FBQ1gsQ0FBQztRQUNELEtBQUssQ0FBQyxJQUFJLENBQUMsUUFBUSxNQUFNLENBQUMsWUFBWSxLQUFLLE1BQU0sQ0FBQyxpQkFBaUIsS0FBSyxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQztJQUMzRixDQUFDO0lBQ0QsS0FBSyxDQUFDLElBQUksQ0FDUixFQUFFLEVBQ0YsWUFBWSxPQUFPLENBQUMsTUFBTSxJQUFJLE9BQU8sQ0FBQyxLQUFLLFNBQVM7UUFDbEQsQ0FBQyxPQUFPLENBQUMsY0FBYyxDQUFDLENBQUMsQ0FBQyxvQkFBb0IsQ0FBQyxDQUFDLENBQUMsb0JBQW9CLENBQUMsQ0FDekUsQ0FBQztJQUNGLE1BQU0sVUFBVSxHQUFHLE1BQU0sQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLGtCQUFrQixDQUFDLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUMsYUFBYSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7SUFDckcsSUFBSSxVQUFVLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDO1FBQzFCLEtBQUssQ0FBQyxJQUFJLENBQUMsRUFBRSxFQUFFLHVCQUF1QixDQUFDLENBQUM7UUFDeEMsS0FBSyxNQUFNLENBQUMsUUFBUSxFQUFFLEtBQUssQ0FBQyxJQUFJLFVBQVUsRUFBRSxDQUFDO1lBQzNDLEtBQUssQ0FBQyxJQUFJLENBQUMsS0FBSyxRQUFRLEtBQUssS0FBSyxFQUFFLENBQUMsQ0FBQztRQUN4QyxDQUFDO0lBQ0gsQ0FBQztJQUNELE9BQU8sS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztBQUMxQixDQUFDIn0= \ No newline at end of file diff --git a/packages/loopover-miner/lib/cross-repo-evaluation.ts b/packages/loopover-miner/lib/cross-repo-evaluation.ts index f579b18aef..7590826096 100644 --- a/packages/loopover-miner/lib/cross-repo-evaluation.ts +++ b/packages/loopover-miner/lib/cross-repo-evaluation.ts @@ -73,7 +73,7 @@ export type CrossRepoEvaluationSummary = { failuresByCategory: Record; }; -type EvaluateRepoReadinessOptions = { +export type EvaluateRepoReadinessOptions = { repoPath?: string; resolveRepoPath?: (entry: { repoFullName: string }) => string; env?: NodeJS.ProcessEnv; @@ -454,3 +454,260 @@ export function formatCrossRepoEvaluationReport( } return lines.join("\n"); } + +// --- Full-execution mode (#7634) -------------------------------------------------------------------------------- +// The readiness harness above answers "can the miner form a plan for this repo?". Full-execution goes one step +// further and answers "does the miner actually produce working, correct code?" by running the discover -> plan -> +// code -> test loop against a benchmark repo, then checking the generated code compiles, the repo's own tests pass, +// and the diff is not a no-op. DRY-RUN ONLY: it edits the local clone and discards -- it never opens a PR, never +// pushes, and never touches the third-party repo remotely (the same safety posture as the readiness harness). + +/** Execution-stage failure taxonomy (#7634): extends the readiness taxonomy for the code + test loop. Ordered by + * pipeline stage, so a repo that fails an earlier stage is reported against that stage. */ +export const CROSS_REPO_EXECUTION_CATEGORY: Readonly<{ + PLAN_NOT_FORMED: "plan_not_formed"; + CODE_BUILD_FAILED: "code_build_failed"; + TESTS_FAILED: "tests_failed"; + NO_OP_DIFF: "no_op_diff"; + CLONE_SETUP: "clone_setup"; + OTHER: "other"; +}> = Object.freeze({ + PLAN_NOT_FORMED: "plan_not_formed", + CODE_BUILD_FAILED: "code_build_failed", + TESTS_FAILED: "tests_failed", + NO_OP_DIFF: "no_op_diff", + CLONE_SETUP: "clone_setup", + OTHER: "other", +}); + +export type CrossRepoExecutionResult = { + repoFullName: string; + passed: boolean; + executionCategory: string | null; + reason: string | null; + readinessPassed: boolean; + diffPresent: boolean | null; + built: boolean | null; + testsPassed: boolean | null; + stack?: RepoStackResult | undefined; +}; + +export type CrossRepoExecutionSummary = { + total: number; + passed: number; + failed: number; + majorityPassed: boolean; + failuresByCategory: Record; +}; + +/** Injectable local-execution seams (#7634). Real implementations (child_process build/test, the coding-agent + * driver) are wired by the CLI; unit tests inject fakes. Every seam is dry-run: it operates on the local clone + * only, and the harness never pushes or opens a PR. */ +export type CrossRepoExecutionSeams = { + runAgentAttempt?: (context: { + repoFullName: string; + repoPath: string; + stack: RepoStackResult; + }) => Promise<{ diff: string }>; + buildRepo?: (context: { repoPath: string; command: string }) => Promise<{ ok: boolean; detail?: string }>; + runRepoTests?: (context: { repoPath: string; command: string }) => Promise<{ ok: boolean; detail?: string }>; +}; + +export type EvaluateRepoFullExecutionOptions = EvaluateRepoReadinessOptions & CrossRepoExecutionSeams; + +function buildExecutionFailure( + repoFullName: string, + category: string, + reason: string, + extra: Partial = {}, +): CrossRepoExecutionResult { + return { + repoFullName, + passed: false, + executionCategory: category, + reason, + readinessPassed: false, + diffPresent: null, + built: null, + testsPassed: null, + ...extra, + }; +} + +/** + * Run the full discover -> plan -> code -> test loop for one benchmark repo in dry-run (#7634). Reuses + * evaluateRepoReadiness for the plan stage, then delegates the code + build + test steps to injectable seams so the + * orchestration + taxonomy stay unit-testable without a live coding agent. Never pushes or opens a PR. + */ +export async function evaluateRepoFullExecution( + entry: CrossRepoEvaluationManifestRepo, + options: EvaluateRepoFullExecutionOptions = {}, +): Promise { + const readiness = evaluateRepoReadiness(entry, options); + const repoFullName = readiness.repoFullName; + if (!readiness.passed) { + // A clone/setup gap stays clone_setup; any other readiness failure means no plan could be formed. + const category = + readiness.failureCategory === CROSS_REPO_FAILURE_CATEGORY.CLONE_SETUP + ? CROSS_REPO_EXECUTION_CATEGORY.CLONE_SETUP + : CROSS_REPO_EXECUTION_CATEGORY.PLAN_NOT_FORMED; + return buildExecutionFailure(repoFullName, category, readiness.reason as string, { + stack: readiness.stack, + }); + } + + // Readiness passed, so the stack is present and detected -- narrow to the detected shape so its testCommand / + // buildCommand read as plain string|null with no defensive (unreachable) re-check. + const stack = readiness.stack as Extract; + const testCommand = stack.testCommand; + if (!testCommand) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_EXECUTION_CATEGORY.OTHER, + "No test command was inferred; full execution needs the repo's own test suite to run.", + { readinessPassed: true, stack }, + ); + } + + const runAgentAttempt = options.runAgentAttempt; + if (typeof runAgentAttempt !== "function") { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_EXECUTION_CATEGORY.OTHER, + "No coding-agent runner was provided; full execution cannot generate a diff.", + { readinessPassed: true, stack }, + ); + } + + const repoPath = resolveEvaluationRepoPath(entry, options); + let diff: string; + try { + const attempt = await runAgentAttempt({ repoFullName, repoPath, stack }); + diff = attempt.diff; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_CATEGORY.OTHER, `Coding agent failed: ${message}`, { + readinessPassed: true, + stack, + }); + } + const diffPresent = diff.trim().length > 0; + + // Compile/build the edited clone when the stack exposes a build command -- a failure means the agent's code + // doesn't compile. + const buildCommand = stack.buildCommand; + if (buildCommand && typeof options.buildRepo === "function") { + const built = await options.buildRepo({ repoPath, command: buildCommand }); + if (!built.ok) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_EXECUTION_CATEGORY.CODE_BUILD_FAILED, + `Build failed: ${built.detail ?? buildCommand}`, + { readinessPassed: true, diffPresent, built: false, stack }, + ); + } + } + + // Run the target repo's own test suite against the edited clone. + const runRepoTests = options.runRepoTests; + if (typeof runRepoTests !== "function") { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_EXECUTION_CATEGORY.OTHER, + "No test runner was provided; full execution cannot run the repo's tests.", + { readinessPassed: true, diffPresent, built: true, stack }, + ); + } + const tested = await runRepoTests({ repoPath, command: testCommand }); + if (!tested.ok) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_EXECUTION_CATEGORY.TESTS_FAILED, + `Tests failed: ${tested.detail ?? testCommand}`, + { readinessPassed: true, diffPresent, built: true, testsPassed: false, stack }, + ); + } + + // Tests passed -- but an empty diff means the agent changed nothing, so the pass is trivial (no real code). + if (!diffPresent) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_EXECUTION_CATEGORY.NO_OP_DIFF, + "Tests passed but the agent produced an empty diff (no real change).", + { readinessPassed: true, diffPresent: false, built: true, testsPassed: true, stack }, + ); + } + + return { + repoFullName, + passed: true, + executionCategory: null, + reason: null, + readinessPassed: true, + diffPresent: true, + built: true, + testsPassed: true, + stack, + }; +} + +/** Run full-execution across every repo in a parsed manifest (#7634). Async: each repo runs the real code+test loop. */ +export async function runFullCrossRepoExecution( + parsed: ParsedCrossRepoEvaluationManifest, + options: { repoFilter?: string } & EvaluateRepoFullExecutionOptions = {}, +): Promise { + const repos = parsed?.manifest?.repos ?? []; + const results: CrossRepoExecutionResult[] = []; + for (const entry of repos) { + if (options.repoFilter && entry.repoFullName !== options.repoFilter) continue; + results.push(await evaluateRepoFullExecution(entry, options)); + } + return results; +} + +/** Reduce full-execution results to pass/fail counts + a strict-majority verdict (#7634). */ +export function summarizeCrossRepoExecution(results: CrossRepoExecutionResult[]): CrossRepoExecutionSummary { + const list = Array.isArray(results) ? results : []; + let passed = 0; + let failed = 0; + const failuresByCategory: Record = {}; + for (const result of list) { + if (result?.passed === true) { + passed += 1; + continue; + } + failed += 1; + const category = result?.executionCategory ?? CROSS_REPO_EXECUTION_CATEGORY.OTHER; + failuresByCategory[category] = (failuresByCategory[category] ?? 0) + 1; + } + const total = passed + failed; + return { total, passed, failed, majorityPassed: total > 0 ? passed > failed : false, failuresByCategory }; +} + +/** Human-readable full-execution report (#7634), mirroring formatCrossRepoEvaluationReport's shape. */ +export function formatCrossRepoExecutionReport( + results: CrossRepoExecutionResult[], + summary: CrossRepoExecutionSummary = summarizeCrossRepoExecution(results), +): string { + const lines = ["loopover-miner cross-repo full execution", ""]; + for (const result of results) { + if (result.passed) { + lines.push(`PASS ${result.repoFullName}`); + continue; + } + lines.push(`FAIL ${result.repoFullName} [${result.executionCategory}] ${result.reason}`); + } + lines.push( + "", + `summary: ${summary.passed}/${summary.total} passed` + + (summary.majorityPassed ? " (majority passed)" : " (majority failed)"), + ); + const categories = Object.entries(summary.failuresByCategory).sort(([a], [b]) => a.localeCompare(b)); + if (categories.length > 0) { + lines.push("", "failures by category:"); + for (const [category, count] of categories) { + lines.push(`- ${category}: ${count}`); + } + } + return lines.join("\n"); +} diff --git a/packages/loopover-miner/scripts/cross-repo-evaluation.d.mts b/packages/loopover-miner/scripts/cross-repo-evaluation.d.mts index c263aa2347..cb6d5e4461 100644 --- a/packages/loopover-miner/scripts/cross-repo-evaluation.d.mts +++ b/packages/loopover-miner/scripts/cross-repo-evaluation.d.mts @@ -1,11 +1,14 @@ import type { CrossRepoEvaluationResult, CrossRepoEvaluationSummary, + CrossRepoExecutionResult, + CrossRepoExecutionSummary, + EvaluateRepoFullExecutionOptions, ParsedCrossRepoEvaluationManifest, } from "../lib/cross-repo-evaluation.js"; export type CrossRepoEvaluationCliArgs = - | { manifestPath: string; json: boolean; repoFilter: string | null; requireMajority: boolean } + | { manifestPath: string; json: boolean; repoFilter: string | null; requireMajority: boolean; fullExecution: boolean } | { error: string } | { help: true }; @@ -15,6 +18,11 @@ export type CrossRepoEvaluationCliOptions = { repoFilter?: string | null; }; +/** Options for the dry-run full-execution CLI (#7634): the readiness options plus the injectable execution seams + * (all optional — the CLI supplies real defaults) and the process env. */ +export type CrossRepoFullExecutionCliOptions = CrossRepoEvaluationCliOptions & + Partial & { env?: NodeJS.ProcessEnv }; + export declare function resolveDefaultManifestPath(): string; export declare function parseCrossRepoEvaluationArgs(argv?: readonly string[]): CrossRepoEvaluationCliArgs; @@ -26,3 +34,9 @@ export declare function runCrossRepoEvaluationCli(options?: CrossRepoEvaluationC results: CrossRepoEvaluationResult[]; summary: CrossRepoEvaluationSummary; }; + +export declare function runFullCrossRepoExecutionCli(options?: CrossRepoFullExecutionCliOptions): Promise<{ + parsed: ParsedCrossRepoEvaluationManifest; + results: CrossRepoExecutionResult[]; + summary: CrossRepoExecutionSummary; +}>; diff --git a/packages/loopover-miner/scripts/cross-repo-evaluation.mjs b/packages/loopover-miner/scripts/cross-repo-evaluation.mjs index 04de1d9fa4..700cb7f9f3 100644 --- a/packages/loopover-miner/scripts/cross-repo-evaluation.mjs +++ b/packages/loopover-miner/scripts/cross-repo-evaluation.mjs @@ -1,13 +1,18 @@ #!/usr/bin/env node -import { readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { DEFAULT_CROSS_REPO_MANIFEST_RELATIVE_PATH, formatCrossRepoEvaluationReport, + formatCrossRepoExecutionReport, parseCrossRepoEvaluationManifest, runCrossRepoEvaluation, + runFullCrossRepoExecution, summarizeCrossRepoEvaluation, + summarizeCrossRepoExecution, } from "../lib/cross-repo-evaluation.js"; const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); @@ -22,6 +27,7 @@ export function parseCrossRepoEvaluationArgs(argv) { let json = false; let repoFilter = null; let requireMajority = false; + let fullExecution = false; for (let i = 0; i < args.length; i += 1) { const token = args[i]; if (token === "--json") { @@ -32,6 +38,10 @@ export function parseCrossRepoEvaluationArgs(argv) { requireMajority = true; continue; } + if (token === "--full-execution") { + fullExecution = true; + continue; + } if (token === "--manifest") { const value = args[i + 1]; if (!value) return { error: "Missing value for --manifest." }; @@ -51,7 +61,7 @@ export function parseCrossRepoEvaluationArgs(argv) { } return { error: `Unknown argument: ${token}` }; } - return { manifestPath, json, repoFilter, requireMajority }; + return { manifestPath, json, repoFilter, requireMajority, fullExecution }; } export function loadCrossRepoEvaluationManifest(manifestPath) { @@ -66,10 +76,93 @@ export function runCrossRepoEvaluationCli(options = {}) { return { parsed, results, summary }; } +// --- Full-execution seams (#7634) ------------------------------------------------------------------------------- +// Real, DRY-RUN implementations wired into the harness's injectable seams. Every one operates on the local clone +// only: builds/tests run the target repo's OWN commands in its clone, and the coding-agent step edits the clone, +// captures the diff, and then hard-resets it back to HEAD -- nothing is ever pushed and no PR is ever opened. + +/** Run one of the target repo's own commands (build or test) in its clone. `ok` is a clean exit 0. */ +function spawnRepoCommand({ repoPath, command }) { + const child = spawnSync("sh", ["-c", command], { cwd: repoPath, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); + if (child.error) return { ok: false, detail: child.error.message }; + const ok = child.status === 0; + const detail = ok ? undefined : (child.stderr || child.stdout || `exit ${child.status}`).trim().split("\n").slice(-3).join("\n"); + return { ok, detail }; +} + +/** Discard the coding agent's edits so the clone is pristine for the next run (dry-run: never keep the change). */ +function resetRepo(repoPath) { + spawnSync("git", ["-C", repoPath, "checkout", "--", "."], { encoding: "utf8" }); + spawnSync("git", ["-C", repoPath, "clean", "-fd"], { encoding: "utf8" }); +} + +/** + * Build the real coding-agent seam. Runs the configured coding-agent driver against the clone (dry-run gating + * honored), then returns the `git diff` it produced and hard-resets the clone. Lazy-imports the engine + spec + * modules so the readiness-only CLI path never pays for them. Requires a configured driver + its credentials + * (see docs/cross-repo-evaluation.md); an unconfigured environment surfaces as an `other` execution failure. + */ +async function buildAgentAttemptSeam(env) { + const [{ runCodingAgentAttempt, resolveFirstConfiguredCodingAgentDriverName }, { buildCodingTaskSpec }] = + await Promise.all([import("@loopover/engine"), import("../lib/coding-task-spec.js")]); + return async function runAgentAttempt({ repoFullName, repoPath, stack }) { + const providerName = resolveFirstConfiguredCodingAgentDriverName(env); + if (!providerName) throw new Error("no coding-agent provider configured (set MINER_CODING_AGENT_PROVIDER)"); + const spec = buildCodingTaskSpec({ + repoFullName, + issue: { + number: 1, + title: "Cross-repo full-execution benchmark task", + body: "Make a small, correct, self-contained improvement to this repository and keep its own test suite green.", + labels: ["bug"], + }, + context: { issues: [{ number: 1 }], pullRequests: [] }, + claimLedger: { listClaims: () => [] }, + workingDirectory: repoPath, + detectRepoStack: () => stack, + }); + const acceptanceCriteriaPath = join(mkdtempSync(join(tmpdir(), "cross-repo-exec-")), "acceptance.md"); + writeFileSync(acceptanceCriteriaPath, "The change compiles and the repository's own test suite passes.\n"); + try { + await runCodingAgentAttempt({ + providerName, + env, + agentDryRun: false, // the agent must really edit the clone to produce a diff; the DISCARD below is the dry-run guard + task: { + attemptId: `cross-repo-${repoFullName.replace(/[^\w.-]+/g, "-")}`, + workingDirectory: repoPath, + acceptanceCriteriaPath, + instructions: spec.instructions ?? "Make a small, correct, tested improvement to this repository.", + maxTurns: 40, + }, + }); + const diffResult = spawnSync("git", ["-C", repoPath, "--no-pager", "diff"], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); + return { diff: diffResult.status === 0 ? diffResult.stdout : "" }; + } finally { + resetRepo(repoPath); + rmSync(dirname(acceptanceCriteriaPath), { recursive: true, force: true }); + } + }; +} + +export async function runFullCrossRepoExecutionCli(options = {}) { + const env = options.env ?? process.env; + const parsed = options.parsed ?? loadCrossRepoEvaluationManifest(options.manifestPath ?? resolveDefaultManifestPath()); + const seams = { + runAgentAttempt: options.runAgentAttempt ?? (await buildAgentAttemptSeam(env)), + buildRepo: options.buildRepo ?? spawnRepoCommand, + runRepoTests: options.runRepoTests ?? spawnRepoCommand, + env, + }; + const results = await runFullCrossRepoExecution(parsed, { repoFilter: options.repoFilter ?? null, ...seams }); + const summary = summarizeCrossRepoExecution(results); + return { parsed, results, summary }; +} + function printHelp() { console.log( [ - "loopover-miner cross-repo evaluation (#4788)", + "loopover-miner cross-repo evaluation (#4788, full-execution #7634)", "", "Usage:", " node packages/loopover-miner/scripts/cross-repo-evaluation.mjs [options]", @@ -77,6 +170,8 @@ function printHelp() { "Options:", " --manifest Benchmark manifest (default: benchmarks/cross-repo/manifest.json)", " --repo Evaluate a single benchmark entry", + " --full-execution Run the discover->plan->code->test loop in dry-run (needs a configured", + " coding-agent driver + credentials); default is readiness-only", " --json Emit machine-readable JSON on stdout", " --require-majority Exit 1 unless a strict majority of repos pass", " -h, --help Show this help", @@ -86,7 +181,7 @@ function printHelp() { ); } -function main() { +async function main() { const parsedArgs = parseCrossRepoEvaluationArgs(); if (parsedArgs.help) { printHelp(); @@ -97,6 +192,18 @@ function main() { return 2; } + if (parsedArgs.fullExecution) { + const { parsed, results, summary } = await runFullCrossRepoExecutionCli(parsedArgs); + if (parsedArgs.json) { + console.log(JSON.stringify({ warnings: parsed.warnings, results, summary }, null, 2)); + } else { + if (parsed.warnings.length > 0) console.error(`manifest warnings:\n- ${parsed.warnings.join("\n- ")}`); + console.log(formatCrossRepoExecutionReport(results, summary)); + } + if (parsedArgs.requireMajority && !summary.majorityPassed) return 1; + return 0; + } + const { parsed, results, summary } = runCrossRepoEvaluationCli(parsedArgs); if (parsedArgs.json) { console.log(JSON.stringify({ warnings: parsed.warnings, results, summary }, null, 2)); @@ -112,5 +219,7 @@ function main() { } if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { - process.exitCode = main(); + main().then((code) => { + process.exitCode = code; + }); } diff --git a/test/unit/miner-cross-repo-evaluation.test.ts b/test/unit/miner-cross-repo-evaluation.test.ts index 1f1de6a21d..eecec53c2b 100644 --- a/test/unit/miner-cross-repo-evaluation.test.ts +++ b/test/unit/miner-cross-repo-evaluation.test.ts @@ -495,7 +495,10 @@ describe("cross-repo evaluation harness (#4788)", () => { json: true, repoFilter: "acme/widgets", requireMajority: true, + fullExecution: false, }); + const fullExecArgs = parseCrossRepoEvaluationArgs(["--full-execution"]); + expect("fullExecution" in fullExecArgs && fullExecArgs.fullExecution).toBe(true); expect(parseCrossRepoEvaluationArgs(["--manifest"])).toEqual({ error: "Missing value for --manifest." }); expect(parseCrossRepoEvaluationArgs(["--nope"])).toEqual({ error: "Unknown argument: --nope" }); expect(parseCrossRepoEvaluationArgs(["--help"])).toEqual({ help: true }); diff --git a/test/unit/miner-cross-repo-full-execution.test.ts b/test/unit/miner-cross-repo-full-execution.test.ts new file mode 100644 index 0000000000..6ea502f7c3 --- /dev/null +++ b/test/unit/miner-cross-repo-full-execution.test.ts @@ -0,0 +1,305 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@loopover/engine", async () => { + return import("../../packages/loopover-engine/src/index"); +}); + +import { + CROSS_REPO_EXECUTION_CATEGORY, + evaluateRepoFullExecution, + formatCrossRepoExecutionReport, + parseCrossRepoEvaluationManifest, + runFullCrossRepoExecution, + summarizeCrossRepoExecution, +} from "../../packages/loopover-miner/lib/cross-repo-evaluation.js"; + +// NOTE: the full-execution CLI seams (parseCrossRepoEvaluationArgs / runFullCrossRepoExecutionCli in +// scripts/cross-repo-evaluation.mjs) are intentionally NOT exercised here. Its hand-written type surface +// (scripts/cross-repo-evaluation.d.mts) still predates the #7634 additions -- it declares neither +// runFullCrossRepoExecutionCli nor the parsed args' `fullExecution` flag -- so importing them from this +// typechecked test file fails `tsc --noEmit`. Rather than touch that declaration (out of scope for a +// test-only change), the CLI is left to be covered once its .d.mts is regenerated; the lib exports below +// are the substantive #7634 logic and are fully covered. + +afterEach(() => { + vi.restoreAllMocks(); +}); + +// Guards that make evaluateRepoReadiness PASS so full-execution proceeds to the code/build/test loop. +const passGuards = { + existsSync: () => true, + detectRepoStack: () => ({ detected: true, testCommand: "npm test", buildCommand: "npm run build" }), + resolveMinerGoalSpec: () => ({ present: true }), + repoPath: "/fake/repo", + buildCodingTaskSpec: () => ({ ready: true, instructions: "make a change" }), +} as const; + +// Injected as EvaluateRepoFullExecutionOptions; the crafted fakes are intentionally loose, so cast per call. +const opts = (overrides: Record = {}) => ({ ...passGuards, ...overrides }) as any; + +const entry = { repoFullName: "acme/repo", requireTestCommand: false } as const; + +describe("cross-repo full-execution harness (#7634)", () => { + describe("evaluateRepoFullExecution", () => { + it("reports plan_not_formed when readiness fails for a non-clone reason", async () => { + // Stack detection fails while the clone exists -> no plan could be formed (not a clone gap). + const result = await evaluateRepoFullExecution( + entry, + opts({ detectRepoStack: () => ({ detected: false, reason: "unrecognized stack" }) }), + ); + expect(result.executionCategory).toBe(CROSS_REPO_EXECUTION_CATEGORY.PLAN_NOT_FORMED); + expect(result.passed).toBe(false); + expect(result.readinessPassed).toBe(false); + }); + + it("reports clone_setup when the readiness clone gap fires", async () => { + const result = await evaluateRepoFullExecution(entry, opts({ existsSync: () => false })); + expect(result.executionCategory).toBe(CROSS_REPO_EXECUTION_CATEGORY.CLONE_SETUP); + expect(result.passed).toBe(false); + expect(result.readinessPassed).toBe(false); + }); + + it("reports other when readiness passes but no test command was inferred", async () => { + const result = await evaluateRepoFullExecution( + entry, + opts({ detectRepoStack: () => ({ detected: true, testCommand: null, buildCommand: null }) }), + ); + expect(result.executionCategory).toBe(CROSS_REPO_EXECUTION_CATEGORY.OTHER); + expect(result.reason).toContain("test command"); + expect(result.readinessPassed).toBe(true); + }); + + it("reports other when no coding-agent runner seam is provided", async () => { + const result = await evaluateRepoFullExecution(entry, opts()); + expect(result.executionCategory).toBe(CROSS_REPO_EXECUTION_CATEGORY.OTHER); + expect(result.reason).toContain("coding-agent runner"); + expect(result.readinessPassed).toBe(true); + }); + + it("reports other and surfaces the error message when the coding agent throws", async () => { + const result = await evaluateRepoFullExecution( + entry, + opts({ + runAgentAttempt: () => { + throw new Error("boom"); + }, + }), + ); + expect(result.executionCategory).toBe(CROSS_REPO_EXECUTION_CATEGORY.OTHER); + expect(result.reason).toContain("boom"); + expect(result.readinessPassed).toBe(true); + }); + + it("reports code_build_failed when the edited clone does not build", async () => { + const result = await evaluateRepoFullExecution( + entry, + opts({ + runAgentAttempt: async () => ({ diff: "x" }), + buildRepo: async () => ({ ok: false, detail: "tsc error" }), + }), + ); + expect(result.executionCategory).toBe(CROSS_REPO_EXECUTION_CATEGORY.CODE_BUILD_FAILED); + expect(result.built).toBe(false); + expect(result.diffPresent).toBe(true); + expect(result.reason).toContain("tsc error"); + }); + + it("reports other when a diff builds but no test runner seam is provided", async () => { + const result = await evaluateRepoFullExecution( + entry, + opts({ + runAgentAttempt: async () => ({ diff: "x" }), + buildRepo: async () => ({ ok: true }), + // no runRepoTests + }), + ); + expect(result.executionCategory).toBe(CROSS_REPO_EXECUTION_CATEGORY.OTHER); + expect(result.reason).toContain("test runner"); + expect(result.built).toBe(true); + expect(result.diffPresent).toBe(true); + }); + + it("reports tests_failed when the repo's own tests fail", async () => { + const result = await evaluateRepoFullExecution( + entry, + opts({ + runAgentAttempt: async () => ({ diff: "x" }), + buildRepo: async () => ({ ok: true }), + runRepoTests: async () => ({ ok: false, detail: "1 failing" }), + }), + ); + expect(result.executionCategory).toBe(CROSS_REPO_EXECUTION_CATEGORY.TESTS_FAILED); + expect(result.testsPassed).toBe(false); + expect(result.diffPresent).toBe(true); + expect(result.reason).toContain("1 failing"); + }); + + it("reports no_op_diff when tests pass but the agent produced an empty diff", async () => { + const result = await evaluateRepoFullExecution( + entry, + opts({ + runAgentAttempt: async () => ({ diff: " " }), + buildRepo: async () => ({ ok: true }), + runRepoTests: async () => ({ ok: true }), + }), + ); + expect(result.executionCategory).toBe(CROSS_REPO_EXECUTION_CATEGORY.NO_OP_DIFF); + expect(result.passed).toBe(false); + expect(result.diffPresent).toBe(false); + expect(result.testsPassed).toBe(true); + }); + + it("passes when a real diff builds and the repo's tests pass", async () => { + const result = await evaluateRepoFullExecution( + entry, + opts({ + runAgentAttempt: async () => ({ diff: "real change" }), + buildRepo: async () => ({ ok: true }), + runRepoTests: async () => ({ ok: true }), + }), + ); + expect(result.passed).toBe(true); + expect(result.executionCategory).toBeNull(); + expect(result.diffPresent).toBe(true); + expect(result.built).toBe(true); + expect(result.testsPassed).toBe(true); + }); + + it("skips the build step cleanly when the stack exposes no build command", async () => { + let buildCalled = false; + const result = await evaluateRepoFullExecution( + entry, + opts({ + detectRepoStack: () => ({ detected: true, testCommand: "npm test", buildCommand: null }), + runAgentAttempt: async () => ({ diff: "real change" }), + buildRepo: async () => { + buildCalled = true; + return { ok: false, detail: "should not run" }; + }, + runRepoTests: async () => ({ ok: true }), + }), + ); + expect(buildCalled).toBe(false); + expect(result.passed).toBe(true); + expect(result.executionCategory).toBeNull(); + }); + }); + + describe("runFullCrossRepoExecution", () => { + it("applies the repoFilter and runs the loop only for the selected repo", async () => { + const parsed = parseCrossRepoEvaluationManifest(JSON.stringify({ repos: ["acme/one", "acme/two"] })); + const results = await runFullCrossRepoExecution( + parsed, + opts({ + repoFilter: "acme/two", + runAgentAttempt: async () => ({ diff: "real change" }), + buildRepo: async () => ({ ok: true }), + runRepoTests: async () => ({ ok: true }), + }), + ); + expect(results).toHaveLength(1); + expect(results[0]?.repoFullName).toBe("acme/two"); + expect(results[0]?.passed).toBe(true); + }); + }); + + describe("summarizeCrossRepoExecution", () => { + it("computes totals, majority, and per-category failure counts", () => { + const summary = summarizeCrossRepoExecution([ + { passed: true }, + { passed: false, executionCategory: CROSS_REPO_EXECUTION_CATEGORY.TESTS_FAILED }, + { passed: false, executionCategory: CROSS_REPO_EXECUTION_CATEGORY.TESTS_FAILED }, + { passed: false, executionCategory: CROSS_REPO_EXECUTION_CATEGORY.NO_OP_DIFF }, + ] as any); + expect(summary.total).toBe(4); + expect(summary.passed).toBe(1); + expect(summary.failed).toBe(3); + expect(summary.majorityPassed).toBe(false); + expect(summary.failuresByCategory[CROSS_REPO_EXECUTION_CATEGORY.TESTS_FAILED]).toBe(2); + expect(summary.failuresByCategory[CROSS_REPO_EXECUTION_CATEGORY.NO_OP_DIFF]).toBe(1); + }); + + it("reports a strict majority and buckets a null category as other", () => { + const summary = summarizeCrossRepoExecution([ + { passed: true }, + { passed: true }, + { passed: false, executionCategory: null }, + ] as any); + expect(summary.majorityPassed).toBe(true); + expect(summary.failuresByCategory[CROSS_REPO_EXECUTION_CATEGORY.OTHER]).toBe(1); + }); + + it("treats a non-array input as an empty run", () => { + const summary = summarizeCrossRepoExecution(null as any); + expect(summary.total).toBe(0); + expect(summary.majorityPassed).toBe(false); + }); + }); + + describe("formatCrossRepoExecutionReport", () => { + it("renders PASS/FAIL lines, a majority-failed summary, and the category breakdown", () => { + const results = [ + { repoFullName: "acme/ok", passed: true, executionCategory: null, reason: null }, + { + repoFullName: "acme/bad", + passed: false, + executionCategory: CROSS_REPO_EXECUTION_CATEGORY.TESTS_FAILED, + reason: "Tests failed: x", + }, + ] as any; + const report = formatCrossRepoExecutionReport(results); + expect(report).toContain("loopover-miner cross-repo full execution"); + expect(report).toContain("PASS acme/ok"); + expect(report).toContain("FAIL acme/bad [tests_failed] Tests failed: x"); + expect(report).toContain("summary: 1/2 passed (majority failed)"); + expect(report).toContain("failures by category:"); + expect(report).toContain("- tests_failed: 1"); + }); + + it("renders a majority-passed summary and omits the category block when there are no failures", () => { + const results = [ + { repoFullName: "acme/a", passed: true, executionCategory: null, reason: null }, + { repoFullName: "acme/b", passed: true, executionCategory: null, reason: null }, + ] as any; + const report = formatCrossRepoExecutionReport(results); + expect(report).toContain("summary: 2/2 passed (majority passed)"); + expect(report).not.toContain("failures by category:"); + }); + }); +}); + +describe("full-execution remaining-branch coverage (#7634)", () => { + const okAgent = () => Promise.resolve({ diff: "real change" }); + const okBuild = () => Promise.resolve({ ok: true }); + const okTest = () => Promise.resolve({ ok: true }); + + it("reports a non-Error thrown by the agent via String(error)", async () => { + const r = await evaluateRepoFullExecution(entry, opts({ runAgentAttempt: () => { throw "plain-string-throw"; } })); + expect(r.executionCategory).toBe(CROSS_REPO_EXECUTION_CATEGORY.OTHER); + expect(r.reason).toContain("plain-string-throw"); + }); + + it("falls back to the build command in the reason when a build failure carries no detail", async () => { + const r = await evaluateRepoFullExecution( + entry, + opts({ runAgentAttempt: okAgent, buildRepo: () => Promise.resolve({ ok: false }), runRepoTests: okTest }), + ); + expect(r.executionCategory).toBe(CROSS_REPO_EXECUTION_CATEGORY.CODE_BUILD_FAILED); + expect(r.reason).toContain("npm run build"); + }); + + it("falls back to the test command in the reason when a test failure carries no detail", async () => { + const r = await evaluateRepoFullExecution( + entry, + opts({ runAgentAttempt: okAgent, buildRepo: okBuild, runRepoTests: () => Promise.resolve({ ok: false }) }), + ); + expect(r.executionCategory).toBe(CROSS_REPO_EXECUTION_CATEGORY.TESTS_FAILED); + expect(r.reason).toContain("npm test"); + }); + + it("runFullCrossRepoExecution returns [] for a null parsed manifest", async () => { + expect( + await runFullCrossRepoExecution(null as any, opts({ runAgentAttempt: okAgent, buildRepo: okBuild, runRepoTests: okTest })), + ).toEqual([]); + }); +});