Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions .github/workflows/paper.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,20 +42,29 @@ jobs:
path: dist/
github-token: ${{ github.token }}
run-id: ${{ inputs.qualification-run-id }}
- name: Create a clean paper qualification environment
id: qualification-runtime
run: |
uv venv --clear --python 3.12 "${{ runner.temp }}/paper-venv"
uv pip install \
--python "${{ runner.temp }}/paper-venv/bin/python" \
"psutil==7.2.2"
- name: Bind the downloaded artifact to its successful qualification run
id: candidate-binding
env:
GITHUB_TOKEN: ${{ github.token }}
run: >-
"$(uv python find 3.12)" scripts/qualification/qualify_paper.py candidate
"${{ runner.temp }}/paper-venv/bin/python"
scripts/qualification/qualify_paper.py candidate
--artifacts-dir dist
--candidate-sha "${{ inputs.candidate-sha }}"
--qualification-run-id "${{ inputs.qualification-run-id }}"
--repository "${{ github.repository }}"
--checkout-root "${{ github.workspace }}"
--output paper-evidence/candidate.json
- name: Install only the candidate wheel outside the checkout
- name: Install only the candidate ML4T wheel outside the checkout
id: candidate-install
run: |
uv venv --python 3.12 "${{ runner.temp }}/paper-venv"
uv pip install --python "${{ runner.temp }}/paper-venv/bin/python" dist/*.whl
mkdir -p "${{ runner.temp }}/paper-run" "${{ runner.temp }}/paper-state"
- name: Exercise Alpaca paper order lifecycle
Expand Down
38 changes: 34 additions & 4 deletions scripts/qualification/check_workflows.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,39 @@ def paper_soak_failures(paper_job: dict[str, Any]) -> list[str]:
return failures


def paper_runtime_failures(paper_job: dict[str, Any]) -> list[str]:
"""Reject paper qualification that depends on ambient runner packages."""
steps = _steps(paper_job)
indexed = {step.get("id"): (index, step) for index, step in enumerate(steps) if step.get("id")}
required = ("qualification-runtime", "candidate-binding", "candidate-install")
if any(step_id not in indexed for step_id in required):
return ["paper qualification does not define its clean runtime and candidate steps"]

runtime_index, runtime = indexed["qualification-runtime"]
binding_index, binding = indexed["candidate-binding"]
install_index, install = indexed["candidate-install"]
failures = []
if not runtime_index < binding_index < install_index:
failures.append("paper qualification does not prepare its runtime before candidate use")

runtime_text = str(runtime.get("run", ""))
paper_python = '"${{ runner.temp }}/paper-venv/bin/python"'
if "uv venv --clear --python 3.12" not in runtime_text:
failures.append("paper qualification does not create a clean Python 3.12 environment")
if paper_python not in runtime_text or '"psutil==7.2.2"' not in runtime_text:
failures.append("paper qualification does not install its pinned runtime dependency")
if not str(binding.get("run", "")).startswith(
f"{paper_python} scripts/qualification/qualify_paper.py candidate"
):
failures.append("paper artifact binding does not use the clean qualification environment")
install_text = str(install.get("run", ""))
if paper_python not in install_text or "dist/*.whl" not in install_text:
failures.append(
"paper qualification does not install the candidate in its clean environment"
)
return failures


def validate_workflows(root: Path = WORKFLOW_ROOT) -> list[str]:
paths = sorted(root.glob("*.yml"))
workflows = {path.name: load_workflow(path) for path in paths}
Expand Down Expand Up @@ -403,17 +436,14 @@ def validate_workflows(root: Path = WORKFLOW_ROOT) -> list[str]:
failures.append("paper qualification is not manual-only")
paper_job = paper.get("jobs", {}).get("paper", {})
failures.extend(paper_soak_failures(paper_job))
failures.extend(paper_runtime_failures(paper_job))
if paper_job.get("environment") != "paper":
failures.append("paper qualification does not use the protected paper environment")
if "secrets." not in json.dumps(paper_job, sort_keys=True):
failures.append("paper qualification has no explicit protected credential inputs")
paper_run_text = _run_text(paper_job)
if "qualify_paper.py candidate" not in paper_run_text:
failures.append("paper qualification does not bind the downloaded candidate artifact")
if '"$(uv python find 3.12)" scripts/qualification/qualify_paper.py candidate' not in (
paper_run_text
):
failures.append("paper artifact binding does not use the uv-managed Python interpreter")
for provider in ("alpaca", "ib"):
for phase in ("exercise", "restart"):
if (
Expand Down
18 changes: 18 additions & 0 deletions tests/unit/test_workflow_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
WORKFLOW_ROOT,
action_pin_failures,
load_workflow,
paper_runtime_failures,
paper_soak_failures,
promotion_failures,
release_recovery_failures,
Expand Down Expand Up @@ -36,6 +37,23 @@ def test_paper_soak_requires_every_short_provider_check() -> None:
assert any("ib-exercise" in failure for failure in paper_soak_failures(seeded_job))


def test_paper_qualification_uses_a_clean_explicit_runtime() -> None:
paper = load_workflow(WORKFLOW_ROOT / "paper.yml")
paper_job = paper["jobs"]["paper"]

assert paper_runtime_failures(paper_job) == []

seeded_job = deepcopy(paper_job)
runtime = next(
step for step in seeded_job["steps"] if step.get("id") == "qualification-runtime"
)
runtime["run"] = str(runtime["run"]).replace('"psutil==7.2.2"', "")

assert any(
"pinned runtime dependency" in failure for failure in paper_runtime_failures(seeded_job)
)


@pytest.mark.parametrize(
("mutation", "expected"),
[
Expand Down