diff --git a/.github/scripts/verify-pypi-release.py b/.github/scripts/verify-pypi-release.py new file mode 100755 index 0000000..e8396ff --- /dev/null +++ b/.github/scripts/verify-pypi-release.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import TypedDict + +PYPI_JSON_URL = "https://pypi.org/pypi/rstreamlabs-rstream/{version}/json" + + +class PublishedFile(TypedDict): + digests: dict[str, str] + filename: str + + +def distribution_digests(directory: Path) -> dict[str, str]: + distributions = sorted( + path + for path in directory.iterdir() + if path.name != "SHA256SUMS" and path.is_file() + ) + if not distributions: + raise ValueError(f"no distributions found in {directory}") + return { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in distributions + } + + +def published_digests(version: str) -> dict[str, str] | None: + request = urllib.request.Request( + PYPI_JSON_URL.format(version=version), + headers={"User-Agent": "rstream-release-verifier/1"}, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + payload = json.load(response) + except urllib.error.HTTPError as error: + if error.code == 404: + return None + raise + files: list[PublishedFile] = payload["urls"] + return {item["filename"]: item["digests"]["sha256"] for item in files} + + +def matches_release(version: str, expected: dict[str, str]) -> bool | None: + actual = published_digests(version) + if actual is None: + return None + if actual != expected: + missing = sorted(expected.keys() - actual.keys()) + unexpected = sorted(actual.keys() - expected.keys()) + mismatched = sorted( + filename + for filename in expected.keys() & actual.keys() + if expected[filename] != actual[filename] + ) + raise ValueError( + "PyPI release differs from the candidate " + f"(missing={missing}, unexpected={unexpected}, mismatched={mismatched})" + ) + return True + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("version") + parser.add_argument("directory", type=Path) + parser.add_argument("--wait", action="store_true") + parser.add_argument("--github-output", type=Path) + arguments = parser.parse_args() + + expected = distribution_digests(arguments.directory) + attempts = 60 if arguments.wait else 1 + for attempt in range(attempts): + result = matches_release(arguments.version, expected) + if result: + if arguments.github_output: + with arguments.github_output.open("a", encoding="utf-8") as output: + output.write("publish=false\n") + return 0 + if attempt + 1 < attempts: + time.sleep(10) + + if arguments.wait: + print( + f"PyPI release {arguments.version} did not become visible", + file=sys.stderr, + ) + return 1 + if arguments.github_output: + with arguments.github_output.open("a", encoding="utf-8") as output: + output.write("publish=true\n") + return 0 + print(f"PyPI release {arguments.version} is not published", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 00884a5..937bd1c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,27 +1,108 @@ -name: Publish +name: Promote stable release + on: - release: - types: - - published workflow_dispatch: + inputs: + release_tag: + description: Reviewed candidate tag to publish + required: true + type: string + permissions: - contents: read + actions: read + contents: write id-token: write + +concurrency: + group: stable-release + cancel-in-progress: false + jobs: publish: - runs-on: ubuntu-latest + name: Publish approved Python distributions + if: ${{ github.actor == vars.CI_ALLOWED_ACTOR }} environment: pypi + runs-on: ubuntu-latest steps: + - name: Find matching release candidate + id: release + shell: bash + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + set -euo pipefail + if [[ ! "$RELEASE_TAG" =~ ^v[0-9]+(\.[0-9]+){2}$ ]]; then + echo "invalid release tag: ${RELEASE_TAG}" >&2 + exit 1 + fi + if [[ "$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${RELEASE_TAG}" --jq '.draft')" != true ]]; then + echo "GitHub release ${RELEASE_TAG} must still be a draft" >&2 + exit 1 + fi + version=${RELEASE_TAG#v} + latest_tag=$(gh api "repos/${GITHUB_REPOSITORY}/releases/latest" --jq '.tag_name' 2>/dev/null || true) + if [[ -n "$latest_tag" && "$(printf '%s\n' "${latest_tag#v}" "$version" | sort -V | tail -n 1)" != "$version" ]]; then + echo "refusing to promote ${RELEASE_TAG} after newer release ${latest_tag}" >&2 + exit 1 + fi + tag_ref=$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/${RELEASE_TAG}") + tag_sha=$(jq -r '.object.sha' <<<"$tag_ref") + if [[ "$(jq -r '.object.type' <<<"$tag_ref")" == tag ]]; then + tag_sha=$(gh api "repos/${GITHUB_REPOSITORY}/git/tags/${tag_sha}" --jq '.object.sha') + fi + candidate_run=$(gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/release-candidate.yml/runs" \ + -f branch="$RELEASE_TAG" -f event=push -f status=success -f per_page=20 \ + --jq ".workflow_runs | map(select(.head_sha == \"${tag_sha}\")) | first | .id") + if [[ -z "$candidate_run" || "$candidate_run" == null ]]; then + echo "no successful release candidate run matches ${RELEASE_TAG}" >&2 + exit 1 + fi + { + echo "tag=${RELEASE_TAG}" + echo "version=${version}" + echo "run_id=${candidate_run}" + } >> "$GITHUB_OUTPUT" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: + ref: ${{ steps.release.outputs.tag }} persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - python-version: "3.13" - - name: Build distribution + name: release-candidate-${{ steps.release.outputs.version }} + path: candidate + repository: ${{ github.repository }} + run-id: ${{ steps.release.outputs.run_id }} + github-token: ${{ github.token }} + - name: Verify candidate integrity shell: bash - run: | - python -m pip install --upgrade pip build==1.5.0 - python -m build - python scripts/verify_distribution.py dist + run: (cd candidate && sha256sum --check SHA256SUMS) + - name: Check PyPI publication state + id: pypi + env: + VERSION: ${{ steps.release.outputs.version }} + run: >- + python .github/scripts/verify-pypi-release.py + "$VERSION" candidate/packages --github-output "$GITHUB_OUTPUT" - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + if: ${{ steps.pypi.outputs.publish == 'true' }} + with: + packages-dir: candidate/packages + - name: Verify PyPI release + env: + VERSION: ${{ steps.release.outputs.version }} + run: >- + python .github/scripts/verify-pypi-release.py + "$VERSION" candidate/packages --wait + - name: Publish GitHub release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ steps.release.outputs.tag }} + run: | + set -euo pipefail + gh release edit "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --draft=false --latest + if [[ "$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${RELEASE_TAG}" --jq '.draft')" != false ]]; then + echo "GitHub release is still a draft: ${RELEASE_TAG}" >&2 + exit 1 + fi diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml new file mode 100644 index 0000000..46a1157 --- /dev/null +++ b/.github/workflows/release-candidate.yml @@ -0,0 +1,58 @@ +name: Build release candidate + +on: + push: + tags: + - "v*" + +permissions: + contents: read + +concurrency: + group: release-candidate-${{ github.ref }} + cancel-in-progress: false + +jobs: + build: + name: Build immutable Python distributions + if: ${{ github.actor == vars.CI_ALLOWED_ACTOR }} + runs-on: ubuntu-latest + steps: + - name: Validate release tag + shell: bash + run: | + set -euo pipefail + version="${GITHUB_REF_NAME#v}" + if [[ ! "$version" =~ ^[0-9]+(\.[0-9]+){2}$ ]]; then + echo "invalid release version: ${version}" >&2 + exit 1 + fi + printf 'VERSION=%s\n' "$version" >> "$GITHUB_ENV" + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: "3.13" + - name: Build and verify distributions + shell: bash + run: | + set -euo pipefail + package_version=$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])') + if [[ "$package_version" != "$VERSION" ]]; then + echo "tag version ${VERSION} does not match package version ${package_version}" >&2 + exit 1 + fi + python -m pip install --upgrade pip build==1.5.0 + python -m build + python scripts/verify_distribution.py dist + mkdir -p candidate/packages + mv dist/* candidate/packages/ + (cd candidate && sha256sum -- packages/* > SHA256SUMS) + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: release-candidate-${{ env.VERSION }} + path: candidate + if-no-files-found: error + compression-level: 0 + retention-days: 90 diff --git a/release-please-config.json b/release-please-config.json index 93e653d..61c3ca6 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -3,6 +3,8 @@ "pull-request-header": "New release created", "packages": { ".": { + "draft": true, + "force-tag-creation": true, "release-type": "python", "include-v-in-tag": true, "extra-files": [ diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py new file mode 100644 index 0000000..0784c61 --- /dev/null +++ b/tests/test_release_workflows.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import json +from pathlib import Path + +ROOT = Path(__file__).parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_release_candidate_does_not_publish() -> None: + workflow = read(".github/workflows/release-candidate.yml") + assert "actions/upload-artifact@" in workflow + assert "pypa/gh-action-pypi-publish@" not in workflow + assert "workflow_dispatch:" not in workflow + + +def test_release_promotion_publishes_github_release_last() -> None: + workflow = read(".github/workflows/publish.yml") + assert "environment: pypi" in workflow + pypi = workflow.index("Verify PyPI release") + github_release = workflow.index("Publish GitHub release") + assert github_release > pypi + + +def test_release_please_creates_tagged_draft() -> None: + config = json.loads(read("release-please-config.json")) + package = config["packages"]["."] + assert package["draft"] is True + assert package["force-tag-creation"] is True