diff --git a/.github/workflows/paper.yml b/.github/workflows/paper.yml index 7b9bf80..a5444a2 100644 --- a/.github/workflows/paper.yml +++ b/.github/workflows/paper.yml @@ -42,20 +42,29 @@ jobs: path: dist/ github-token: ${{ github.token }} run-id: ${{ inputs.qualification-run-id }} + - name: Create a clean paper qualification environment + id: qualification-runtime + run: | + uv venv --clear --python 3.12 "${{ runner.temp }}/paper-venv" + uv pip install \ + --python "${{ runner.temp }}/paper-venv/bin/python" \ + "psutil==7.2.2" - name: Bind the downloaded artifact to its successful qualification run + id: candidate-binding env: GITHUB_TOKEN: ${{ github.token }} run: >- - "$(uv python find 3.12)" scripts/qualification/qualify_paper.py candidate + "${{ runner.temp }}/paper-venv/bin/python" + scripts/qualification/qualify_paper.py candidate --artifacts-dir dist --candidate-sha "${{ inputs.candidate-sha }}" --qualification-run-id "${{ inputs.qualification-run-id }}" --repository "${{ github.repository }}" --checkout-root "${{ github.workspace }}" --output paper-evidence/candidate.json - - name: Install only the candidate wheel outside the checkout + - name: Install only the candidate ML4T wheel outside the checkout + id: candidate-install run: | - uv venv --python 3.12 "${{ runner.temp }}/paper-venv" uv pip install --python "${{ runner.temp }}/paper-venv/bin/python" dist/*.whl mkdir -p "${{ runner.temp }}/paper-run" "${{ runner.temp }}/paper-state" - name: Exercise Alpaca paper order lifecycle diff --git a/scripts/qualification/check_workflows.py b/scripts/qualification/check_workflows.py index a7a31c6..9fed646 100644 --- a/scripts/qualification/check_workflows.py +++ b/scripts/qualification/check_workflows.py @@ -250,6 +250,39 @@ def paper_soak_failures(paper_job: dict[str, Any]) -> list[str]: return failures +def paper_runtime_failures(paper_job: dict[str, Any]) -> list[str]: + """Reject paper qualification that depends on ambient runner packages.""" + steps = _steps(paper_job) + indexed = {step.get("id"): (index, step) for index, step in enumerate(steps) if step.get("id")} + required = ("qualification-runtime", "candidate-binding", "candidate-install") + if any(step_id not in indexed for step_id in required): + return ["paper qualification does not define its clean runtime and candidate steps"] + + runtime_index, runtime = indexed["qualification-runtime"] + binding_index, binding = indexed["candidate-binding"] + install_index, install = indexed["candidate-install"] + failures = [] + if not runtime_index < binding_index < install_index: + failures.append("paper qualification does not prepare its runtime before candidate use") + + runtime_text = str(runtime.get("run", "")) + paper_python = '"${{ runner.temp }}/paper-venv/bin/python"' + if "uv venv --clear --python 3.12" not in runtime_text: + failures.append("paper qualification does not create a clean Python 3.12 environment") + if paper_python not in runtime_text or '"psutil==7.2.2"' not in runtime_text: + failures.append("paper qualification does not install its pinned runtime dependency") + if not str(binding.get("run", "")).startswith( + f"{paper_python} scripts/qualification/qualify_paper.py candidate" + ): + failures.append("paper artifact binding does not use the clean qualification environment") + install_text = str(install.get("run", "")) + if paper_python not in install_text or "dist/*.whl" not in install_text: + failures.append( + "paper qualification does not install the candidate in its clean environment" + ) + return failures + + def validate_workflows(root: Path = WORKFLOW_ROOT) -> list[str]: paths = sorted(root.glob("*.yml")) workflows = {path.name: load_workflow(path) for path in paths} @@ -403,6 +436,7 @@ def validate_workflows(root: Path = WORKFLOW_ROOT) -> list[str]: failures.append("paper qualification is not manual-only") paper_job = paper.get("jobs", {}).get("paper", {}) failures.extend(paper_soak_failures(paper_job)) + failures.extend(paper_runtime_failures(paper_job)) if paper_job.get("environment") != "paper": failures.append("paper qualification does not use the protected paper environment") if "secrets." not in json.dumps(paper_job, sort_keys=True): @@ -410,10 +444,6 @@ def validate_workflows(root: Path = WORKFLOW_ROOT) -> list[str]: paper_run_text = _run_text(paper_job) if "qualify_paper.py candidate" not in paper_run_text: failures.append("paper qualification does not bind the downloaded candidate artifact") - if '"$(uv python find 3.12)" scripts/qualification/qualify_paper.py candidate' not in ( - paper_run_text - ): - failures.append("paper artifact binding does not use the uv-managed Python interpreter") for provider in ("alpaca", "ib"): for phase in ("exercise", "restart"): if ( diff --git a/tests/unit/test_workflow_policy.py b/tests/unit/test_workflow_policy.py index d34cc74..6a19fde 100644 --- a/tests/unit/test_workflow_policy.py +++ b/tests/unit/test_workflow_policy.py @@ -8,6 +8,7 @@ WORKFLOW_ROOT, action_pin_failures, load_workflow, + paper_runtime_failures, paper_soak_failures, promotion_failures, release_recovery_failures, @@ -36,6 +37,23 @@ def test_paper_soak_requires_every_short_provider_check() -> None: assert any("ib-exercise" in failure for failure in paper_soak_failures(seeded_job)) +def test_paper_qualification_uses_a_clean_explicit_runtime() -> None: + paper = load_workflow(WORKFLOW_ROOT / "paper.yml") + paper_job = paper["jobs"]["paper"] + + assert paper_runtime_failures(paper_job) == [] + + seeded_job = deepcopy(paper_job) + runtime = next( + step for step in seeded_job["steps"] if step.get("id") == "qualification-runtime" + ) + runtime["run"] = str(runtime["run"]).replace('"psutil==7.2.2"', "") + + assert any( + "pinned runtime dependency" in failure for failure in paper_runtime_failures(seeded_job) + ) + + @pytest.mark.parametrize( ("mutation", "expected"), [