fix(release): keep the release a draft - the uploader was publishing it

.github/workflows/aur-publish.yml

@@ -80,8 +80,15 @@ jobs:
           printf '%s\n' "$AUR_HOSTKEY" > ~/.ssh/known_hosts
           # A mismatch means either a legitimate rotation (update the pin) or
           # something worth stopping the release for.
-          if ! ssh-keyscan -t ed25519 aur.archlinux.org 2>/dev/null | grep -qxF "$AUR_HOSTKEY"; then
-            echo "::error::aur.archlinux.org host key does not match the pinned one"
+          # Distinguish "the scan failed" from "the key changed". Piping
+          # straight into grep reports a compromise for any network blip, and
+          # the exit code of ssh-keyscan is swallowed - a scary message for the
+          # wrong reason is worse than no message.
+          scanned=$(ssh-keyscan -t ed25519 aur.archlinux.org 2>/dev/null || true)
+          if [ -z "$scanned" ]; then
+            echo "::warning::could not reach aur.archlinux.org to cross-check the host key; continuing with the pinned one"
+          elif ! printf '%s\n' "$scanned" | grep -qxF "$AUR_HOSTKEY"; then
+            echo "::error::aur.archlinux.org presented a DIFFERENT host key than the pinned one. Either it rotated (update the pin) or something is wrong. Stopping."
             exit 1
           fi
 

.github/workflows/release-please.yml

@@ -117,11 +117,39 @@ jobs:
           file "${{ matrix.asset }}" | grep -q 'x86_64' \
             || { echo "::error::${{ matrix.asset }} is not an x86_64 binary"; exit 1; }
 
+      # `draft: true` is LOAD-BEARING, not a preference. Without it this action
+      # calls finalizeRelease() after uploading, which PATCHes draft:false - so
+      # the first matrix leg to finish would publish v0.10.0 as /releases/latest
+      # carrying one of four binaries and zero signatures, and the sign job's
+      # closing `--draft=false` would be a silent no-op. Every check would still
+      # be green. That is the v0.7.0 incident, reached automatically.
+      #
+      # The action's own action.yml says it: "When reusing an existing draft
+      # release, set this to true to keep it draft; omit it to publish after
+      # upload." Verified in the pinned build: an absent input parses to
+      # `undefined`, and the only guard is
+      # `input_draft === true || release.draft === false`.
       - name: Upload binary to GitHub Release
         uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2
         with:
           tag_name: ${{ needs.release-please.outputs.tag_name }}
           files: ${{ matrix.asset }}
+          draft: true
+
+      # Assert the hold actually survived, so a future action bump cannot
+      # silently reintroduce the above. Cheap, and it fails the leg rather than
+      # the release.
+      - name: The release must still be a draft
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ needs.release-please.outputs.tag_name }}
+        run: |
+          state=$(gh release view "$TAG" --json isDraft --jq .isDraft)
+          [ "$state" = "true" ] || {
+            echo "::error::$TAG is no longer a draft after upload - it is public without signatures. Re-draft it NOW: gh release edit $TAG --draft=true"
+            exit 1
+          }
 
   # Self-update verification is fail-closed: a release without .sig assets
   # bricks "Download update" for every existing install (this happened on
@@ -164,6 +192,10 @@ jobs:
           fi
           printf '%s' "$KEY" > /tmp/colony-release.pem
           chmod 600 /tmp/colony-release.pem
+          # The rm below is skipped when sign-release.sh exits non-zero under
+          # `bash -e`, leaving the private key on the runner's disk for the rest
+          # of the job.
+          trap 'rm -f /tmp/colony-release.pem' EXIT
           for a in $ASSETS; do PATHS="$PATHS dist/$a"; done
           COLONY_SIGNING_KEY=/tmp/colony-release.pem \
           COLONY_RELEASE_VERSION="$TAG" \

docs/release-signing.md

@@ -51,15 +51,63 @@ The private key never lives in the repo. Point `COLONY_SIGNING_KEY` at the
 ed25519 private key (PEM), set `COLONY_RELEASE_VERSION` to the release tag (it is
 bound into each sidecar), and run:
 
+**Sign the PUBLISHED bytes, never a local rebuild.** Download the assets from
+the release first:
+
 ```sh
+gh release download v1.2.3 --dir dist \
+  --pattern colony-linux --pattern colony-windows.exe \
+  --pattern colony-macos --pattern colony-macos-x86
+
 COLONY_SIGNING_KEY=/path/to/colony-release.pem \
 COLONY_RELEASE_VERSION=v1.2.3 \
-  ./scripts/sign-release.sh colony-linux colony-windows.exe colony-macos colony-macos-x86
+  ./scripts/sign-release.sh dist/colony-linux dist/colony-windows.exe \
+                            dist/colony-macos dist/colony-macos-x86
+
+gh release upload v1.2.3 dist/*.sig dist/*.meta --clobber
 ```
 
+The download step is not optional. `sign-release.sh` hashes whatever local file
+you hand it into the `.meta` sidecar, and the client then enforces that digest
+against the bytes it downloaded. Rust release builds are not bit-reproducible
+across machines, so signing a fresh `cargo build --release` produces a sidecar
+whose digest does not match what users receive - and every install fails
+verification, which is worse than being unsigned because it also fails
+fail-closed.
+
 For each asset this writes `<asset>.sig`, `<asset>.meta` and `<asset>.meta.sig`,
 every signature self-verified before it is kept. Upload all of them as release
-assets.
+assets, then confirm the count:
+
+```sh
+gh release view v1.2.3 --json assets --jq '.assets|length'   # must be 16
+```
+
+### If a release goes wrong
+
+`gh release view <tag> --json isDraft,assets --jq '{draft:.isDraft, n:(.assets|length)}'`
+tells you which state you are in.
+
+**Still a draft, incomplete.** Nobody is affected: `/releases/latest` still
+points at the previous version and no client has been offered anything. Fix the
+cause and use **"Re-run failed jobs"**.
+
+**Never "Re-run all jobs".** release-please re-runs against a `main` whose
+release already exists, emits an empty `release_created`, and every downstream
+job skips - while the run reports green. That looks like a successful recovery
+and is the opposite of one.
+
+**Published but unsigned or partial.** Clients are being offered an update that
+cannot be applied. Take it out of `latest` first, then complete it:
+
+```sh
+gh release edit <tag> --prerelease          # assets exist; keeps their URLs alive
+# or: gh release edit <tag> --draft=true    # release is empty anyway
+gh api repos/Project-Colony/Colony/releases/latest --jq .tag_name   # confirm the fallback
+```
+
+then follow the manual signing procedure above and re-publish with
+`gh release edit <tag> --draft=false --prerelease=false`.
 
 ### In CI (the normal path)