ci: promote immutable PyPI release candidates

.github/scripts/verify-pypi-release.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import TypedDict
+
+PYPI_JSON_URL = "https://pypi.org/pypi/rstreamlabs-rstream/{version}/json"
+
+
+class PublishedFile(TypedDict):
+    digests: dict[str, str]
+    filename: str
+
+
+def distribution_digests(directory: Path) -> dict[str, str]:
+    distributions = sorted(
+        path
+        for path in directory.iterdir()
+        if path.name != "SHA256SUMS" and path.is_file()
+    )
+    if not distributions:
+        raise ValueError(f"no distributions found in {directory}")
+    return {
+        path.name: hashlib.sha256(path.read_bytes()).hexdigest()
+        for path in distributions
+    }
+
+
+def published_digests(version: str) -> dict[str, str] | None:
+    request = urllib.request.Request(
+        PYPI_JSON_URL.format(version=version),
+        headers={"User-Agent": "rstream-release-verifier/1"},
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=30) as response:
+            payload = json.load(response)
+    except urllib.error.HTTPError as error:
+        if error.code == 404:
+            return None
+        raise
+    files: list[PublishedFile] = payload["urls"]
+    return {item["filename"]: item["digests"]["sha256"] for item in files}
+
+
+def matches_release(version: str, expected: dict[str, str]) -> bool | None:
+    actual = published_digests(version)
+    if actual is None:
+        return None
+    if actual != expected:
+        missing = sorted(expected.keys() - actual.keys())
+        unexpected = sorted(actual.keys() - expected.keys())
+        mismatched = sorted(
+            filename
+            for filename in expected.keys() & actual.keys()
+            if expected[filename] != actual[filename]
+        )
+        raise ValueError(
+            "PyPI release differs from the candidate "
+            f"(missing={missing}, unexpected={unexpected}, mismatched={mismatched})"
+        )
+    return True
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("version")
+    parser.add_argument("directory", type=Path)
+    parser.add_argument("--wait", action="store_true")
+    parser.add_argument("--github-output", type=Path)
+    arguments = parser.parse_args()
+
+    expected = distribution_digests(arguments.directory)
+    attempts = 60 if arguments.wait else 1
+    for attempt in range(attempts):
+        result = matches_release(arguments.version, expected)
+        if result:
+            if arguments.github_output:
+                with arguments.github_output.open("a", encoding="utf-8") as output:
+                    output.write("publish=false\n")
+            return 0
+        if attempt + 1 < attempts:
+            time.sleep(10)
+
+    if arguments.wait:
+        print(
+            f"PyPI release {arguments.version} did not become visible",
+            file=sys.stderr,
+        )
+        return 1
+    if arguments.github_output:
+        with arguments.github_output.open("a", encoding="utf-8") as output:
+            output.write("publish=true\n")
+        return 0
+    print(f"PyPI release {arguments.version} is not published", file=sys.stderr)
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/workflows/publish.yml

@@ -1,27 +1,108 @@
-name: Publish
+name: Promote stable release
+
 on:
-  release:
-    types:
-      - published
   workflow_dispatch:
+    inputs:
+      release_tag:
+        description: Reviewed candidate tag to publish
+        required: true
+        type: string
+
 permissions:
-  contents: read
+  actions: read
+  contents: write
   id-token: write
+
+concurrency:
+  group: stable-release
+  cancel-in-progress: false
+
 jobs:
   publish:
-    runs-on: ubuntu-latest
+    name: Publish approved Python distributions
+    if: ${{ github.actor == vars.CI_ALLOWED_ACTOR }}
     environment: pypi
+    runs-on: ubuntu-latest
     steps:
+      - name: Find matching release candidate
+        id: release
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_TAG: ${{ inputs.release_tag }}
+        run: |
+          set -euo pipefail
+          if [[ ! "$RELEASE_TAG" =~ ^v[0-9]+(\.[0-9]+){2}$ ]]; then
+            echo "invalid release tag: ${RELEASE_TAG}" >&2
+            exit 1
+          fi
+          if [[ "$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${RELEASE_TAG}" --jq '.draft')" != true ]]; then
+            echo "GitHub release ${RELEASE_TAG} must still be a draft" >&2
+            exit 1
+          fi
+          version=${RELEASE_TAG#v}
+          latest_tag=$(gh api "repos/${GITHUB_REPOSITORY}/releases/latest" --jq '.tag_name' 2>/dev/null || true)
+          if [[ -n "$latest_tag" && "$(printf '%s\n' "${latest_tag#v}" "$version" | sort -V | tail -n 1)" != "$version" ]]; then
+            echo "refusing to promote ${RELEASE_TAG} after newer release ${latest_tag}" >&2
+            exit 1
+          fi
+          tag_ref=$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/${RELEASE_TAG}")
+          tag_sha=$(jq -r '.object.sha' <<<"$tag_ref")
+          if [[ "$(jq -r '.object.type' <<<"$tag_ref")" == tag ]]; then
+            tag_sha=$(gh api "repos/${GITHUB_REPOSITORY}/git/tags/${tag_sha}" --jq '.object.sha')
+          fi
+          candidate_run=$(gh api --method GET \
+            "repos/${GITHUB_REPOSITORY}/actions/workflows/release-candidate.yml/runs" \
+            -f branch="$RELEASE_TAG" -f event=push -f status=success -f per_page=20 \
+            --jq ".workflow_runs | map(select(.head_sha == \"${tag_sha}\")) | first | .id")
+          if [[ -z "$candidate_run" || "$candidate_run" == null ]]; then
+            echo "no successful release candidate run matches ${RELEASE_TAG}" >&2
+            exit 1
+          fi
+          {
+            echo "tag=${RELEASE_TAG}"
+            echo "version=${version}"
+            echo "run_id=${candidate_run}"
+          } >> "$GITHUB_OUTPUT"
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
         with:
+          ref: ${{ steps.release.outputs.tag }}
           persist-credentials: false
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
-          python-version: "3.13"
-      - name: Build distribution
+          name: release-candidate-${{ steps.release.outputs.version }}
+          path: candidate
+          repository: ${{ github.repository }}
+          run-id: ${{ steps.release.outputs.run_id }}
+          github-token: ${{ github.token }}
+      - name: Verify candidate integrity
         shell: bash
-        run: |
-          python -m pip install --upgrade pip build==1.5.0
-          python -m build
-          python scripts/verify_distribution.py dist
+        run: (cd candidate && sha256sum --check SHA256SUMS)
+      - name: Check PyPI publication state
+        id: pypi
+        env:
+          VERSION: ${{ steps.release.outputs.version }}
+        run: >-
+          python .github/scripts/verify-pypi-release.py
+          "$VERSION" candidate/packages --github-output "$GITHUB_OUTPUT"
       - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1
+        if: ${{ steps.pypi.outputs.publish == 'true' }}
+        with:
+          packages-dir: candidate/packages
+      - name: Verify PyPI release
+        env:
+          VERSION: ${{ steps.release.outputs.version }}
+        run: >-
+          python .github/scripts/verify-pypi-release.py
+          "$VERSION" candidate/packages --wait
+      - name: Publish GitHub release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_TAG: ${{ steps.release.outputs.tag }}
+        run: |
+          set -euo pipefail
+          gh release edit "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --draft=false --latest
+          if [[ "$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${RELEASE_TAG}" --jq '.draft')" != false ]]; then
+            echo "GitHub release is still a draft: ${RELEASE_TAG}" >&2
+            exit 1
+          fi

.github/workflows/release-candidate.yml

@@ -0,0 +1,58 @@
+name: Build release candidate
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: release-candidate-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Build immutable Python distributions
+    if: ${{ github.actor == vars.CI_ALLOWED_ACTOR }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Validate release tag
+        shell: bash
+        run: |
+          set -euo pipefail
+          version="${GITHUB_REF_NAME#v}"
+          if [[ ! "$version" =~ ^[0-9]+(\.[0-9]+){2}$ ]]; then
+            echo "invalid release version: ${version}" >&2
+            exit 1
+          fi
+          printf 'VERSION=%s\n' "$version" >> "$GITHUB_ENV"
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
+        with:
+          python-version: "3.13"
+      - name: Build and verify distributions
+        shell: bash
+        run: |
+          set -euo pipefail
+          package_version=$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')
+          if [[ "$package_version" != "$VERSION" ]]; then
+            echo "tag version ${VERSION} does not match package version ${package_version}" >&2
+            exit 1
+          fi
+          python -m pip install --upgrade pip build==1.5.0
+          python -m build
+          python scripts/verify_distribution.py dist
+          mkdir -p candidate/packages
+          mv dist/* candidate/packages/
+          (cd candidate && sha256sum -- packages/* > SHA256SUMS)
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: release-candidate-${{ env.VERSION }}
+          path: candidate
+          if-no-files-found: error
+          compression-level: 0
+          retention-days: 90

release-please-config.json

@@ -3,6 +3,8 @@
   "pull-request-header": "New release created",
   "packages": {
     ".": {
+      "draft": true,
+      "force-tag-creation": true,
       "release-type": "python",
       "include-v-in-tag": true,
       "extra-files": [

tests/test_release_workflows.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+ROOT = Path(__file__).parents[1]
+
+
+def read(path: str) -> str:
+    return (ROOT / path).read_text(encoding="utf-8")
+
+
+def test_release_candidate_does_not_publish() -> None:
+    workflow = read(".github/workflows/release-candidate.yml")
+    assert "actions/upload-artifact@" in workflow
+    assert "pypa/gh-action-pypi-publish@" not in workflow
+    assert "workflow_dispatch:" not in workflow
+
+
+def test_release_promotion_publishes_github_release_last() -> None:
+    workflow = read(".github/workflows/publish.yml")
+    assert "environment: pypi" in workflow
+    pypi = workflow.index("Verify PyPI release")
+    github_release = workflow.index("Publish GitHub release")
+    assert github_release > pypi
+
+
+def test_release_please_creates_tagged_draft() -> None:
+    config = json.loads(read("release-please-config.json"))
+    package = config["packages"]["."]
+    assert package["draft"] is True
+    assert package["force-tag-creation"] is True