fix(release): Enhance signing logic to support retry

.github/workflows/release.yml

@@ -8,7 +8,6 @@ on:
 permissions:
   contents: write
   packages: write
-  id-token: write  # Required for cosign keyless signing via Sigstore
 
 jobs:
   release:
@@ -38,9 +37,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Install cosign
-        uses: sigstore/cosign-installer@v3
-
+      # Required by the sboms stanza in .goreleaser.yaml (binary SBOMs).
       - name: Install syft
         uses: anchore/sbom-action/download-syft@v0
 
@@ -59,5 +56,15 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Sign multi-arch manifests and attach SBOM attestations (by digest)
-        run: .github/workflows/scripts/sign-and-attest.sh
+  # Signing lives in its own workflow so that it can also be dispatched on its
+  # own to sign a release published earlier. See .github/workflows/sign.yml.
+  sign:
+    name: Sign
+    needs: release
+    permissions:
+      contents: read
+      packages: write  # push cosign signatures and attestations to GHCR
+      id-token: write  # required for cosign keyless signing via Sigstore
+    uses: ./.github/workflows/sign.yml
+    with:
+      tag: ${{ github.ref_name }}

.github/workflows/scripts/sign-and-attest.sh

@@ -1,72 +1,136 @@
 #!/usr/bin/env bash
 #
-# Sign container images and attach SBOM attestations for a release.
+# Sign a release's container image and attach its SBOM attestation.
 #
-# Called by the Release workflow (.github/workflows/release.yml) after
-# GoReleaser pushes multi-arch manifest lists. For each release tag the
-# script resolves the registry-side digest, deduplicates, and then:
+# Called by the Sign workflow (.github/workflows/sign.yml), which the Release
+# workflow invokes once GoReleaser has pushed the multi-arch manifest lists, and
+# which can also be dispatched on its own to sign a release published earlier.
+# For the requested release tag the script resolves the registry-side digest and:
 #   1. Signs the manifest list by digest with cosign (keyless / Sigstore OIDC).
 #   2. Generates an SPDX JSON SBOM with syft against the digest reference.
 #   3. Attaches the SBOM as a signed in-toto attestation with cosign.
 #
 # Signing by digest (name@sha256:...) is the cosign-recommended model;
-# signatures are discoverable from any tag pointing at that digest, so
-# signing once covers both the version tag and :latest.
+# signatures are discoverable from any tag pointing at that digest, so signing
+# the release tag's manifest list also covers :latest whenever :latest points at
+# that same manifest list. Signing is always scoped to the requested release
+# tag: a dispatch that re-signs an older release must never sign whatever
+# :latest happens to point at now.
+#
+# Every registry operation is retried with exponential backoff, and each step
+# is skipped when its artifact is already present. GHCR's token service
+# intermittently rejects push-scoped token requests with "DENIED: denied" even
+# when the credentials are valid and were granted the same scope seconds
+# earlier — release run 31604663860 published 0.2.0 unsigned that way, after
+# the images and the GitHub release had already gone out. A transient registry
+# failure must not cost a release its signature, and re-running this script
+# against an already-signed digest must be a cheap no-op so that recovery is
+# just "run it again".
 #
 # Prerequisites (installed by the workflow):
 #   - docker buildx  (docker/setup-buildx-action)
 #   - cosign         (sigstore/cosign-installer)
 #   - syft           (anchore/sbom-action/download-syft)
 #
 # Environment:
-#   GITHUB_REF_NAME  – the git tag pushed (e.g. "0.0.20"), set automatically
-#                      by GitHub Actions.
+#   RELEASE_TAG          – the release tag to sign (e.g. "0.0.20"), passed by the
+#                          Sign workflow. Falls back to GITHUB_REF_NAME, which
+#                          GitHub Actions sets to the tag on a tag-triggered run.
+#   RETRY_MAX_ATTEMPTS   – attempts per registry operation (default 5).
+#   RETRY_INITIAL_DELAY  – seconds before the first retry, doubling on each
+#                          subsequent attempt (default 5).
 
 set -euo pipefail
 
-if [ -z "${GITHUB_REF_NAME:-}" ]; then
-  echo "ERROR: GITHUB_REF_NAME is not set"
+RELEASE_TAG="${RELEASE_TAG:-${GITHUB_REF_NAME:-}}"
+if [ -z "${RELEASE_TAG}" ]; then
+  echo "ERROR: neither RELEASE_TAG nor GITHUB_REF_NAME is set"
   exit 1
 fi
 
 IMAGE="ghcr.io/observiq/bindplane-operator"
-TAGS=(
-  "${GITHUB_REF_NAME}"
-  "latest"
-)
 
-declare -A TAG_TO_DIGEST
-declare -A SEEN_DIGESTS
+RETRY_MAX_ATTEMPTS="${RETRY_MAX_ATTEMPTS:-5}"
+RETRY_INITIAL_DELAY="${RETRY_INITIAL_DELAY:-5}"
 
-echo "Resolving registry digests for ${IMAGE} tags:"
-for TAG in "${TAGS[@]}"; do
-  DIGEST=$(docker buildx imagetools inspect "${IMAGE}:${TAG}" --format '{{ .Manifest.Digest }}')
-  if [ -z "${DIGEST}" ]; then
-    echo "ERROR: failed to resolve digest for ${IMAGE}:${TAG}"
-    exit 1
-  fi
-  TAG_TO_DIGEST["${TAG}"]="${DIGEST}"
-  echo "  ${TAG} -> ${DIGEST}"
-done
+# retry <command> [args...]
+# Runs the command, retrying with exponential backoff until it succeeds or
+# RETRY_MAX_ATTEMPTS is exhausted. Diagnostics go to stderr so that callers can
+# safely capture the command's stdout via command substitution.
+retry() {
+  local attempt=1
+  local delay="${RETRY_INITIAL_DELAY}"
+  until "$@"; do
+    if [ "${attempt}" -ge "${RETRY_MAX_ATTEMPTS}" ]; then
+      echo "ERROR: '$*' failed after ${RETRY_MAX_ATTEMPTS} attempts" >&2
+      return 1
+    fi
+    echo "  attempt ${attempt}/${RETRY_MAX_ATTEMPTS} of '$*' failed, retrying in ${delay}s" >&2
+    sleep "${delay}"
+    attempt=$((attempt + 1))
+    delay=$((delay * 2))
+  done
+}
 
-for TAG in "${TAGS[@]}"; do
-  DIGEST="${TAG_TO_DIGEST[${TAG}]}"
-  if [ -n "${SEEN_DIGESTS[${DIGEST}]:-}" ]; then
-    echo "Digest ${DIGEST} already signed and attested (covered by tag ${SEEN_DIGESTS[${DIGEST}]}), skipping ${TAG}"
-    continue
+# inspect_digest <tag>
+# Prints the manifest digest for a tag. Fails on anything that is not a
+# well-formed digest so that a garbled or partial registry response is retried
+# rather than propagated into a signing reference.
+inspect_digest() {
+  local digest
+  if ! digest=$(docker buildx imagetools inspect "${IMAGE}:${1}" --format '{{ .Manifest.Digest }}'); then
+    return 1
+  fi
+  if [[ ! "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+    echo "ERROR: unexpected digest '${digest}' for ${IMAGE}:${1}" >&2
+    return 1
   fi
-  SEEN_DIGESTS["${DIGEST}"]="${TAG}"
+  printf '%s\n' "${digest}"
+}
 
-  REF="${IMAGE}@${DIGEST}"
+# artifact_exists <tag>
+# True when the tag already resolves in the registry. Deliberately not retried:
+# a missing artifact is the expected answer for a fresh release, and a false
+# negative only costs a redundant signing attempt.
+artifact_exists() {
+  docker buildx imagetools inspect "${IMAGE}:${1}" >/dev/null 2>&1
+}
 
+echo "Resolving registry digest for ${IMAGE}:"
+DIGEST=$(retry inspect_digest "${RELEASE_TAG}")
+echo "  ${RELEASE_TAG} -> ${DIGEST}"
+
+# Reported for operator visibility only — :latest needs no separate signature
+# when it points at the same manifest list, and must not be signed when it does
+# not (see the note on tag scoping above). Best effort: a failure to resolve
+# :latest is not a reason to fail signing the release tag.
+LATEST_DIGEST=$(inspect_digest "latest" 2>/dev/null || true)
+if [ "${LATEST_DIGEST}" = "${DIGEST}" ]; then
+  echo "  latest -> ${DIGEST} (same manifest list, covered by this signature)"
+elif [ -n "${LATEST_DIGEST}" ]; then
+  echo "  latest -> ${LATEST_DIGEST} (different manifest list, not signed by this run)"
+fi
+
+REF="${IMAGE}@${DIGEST}"
+# cosign derives its artifact tags from the digest, replacing the algorithm
+# separator: sha256:abc... -> sha256-abc....sig / sha256-abc....att
+COSIGN_TAG="${DIGEST/:/-}"
+
+if artifact_exists "${COSIGN_TAG}.sig"; then
+  echo "Signature already present for ${REF}, skipping"
+else
   echo "Signing ${REF}"
-  cosign sign --yes "${REF}"
+  retry cosign sign --yes "${REF}"
+fi
 
+if artifact_exists "${COSIGN_TAG}.att"; then
+  echo "SBOM attestation already present for ${REF}, skipping"
+else
   echo "Generating SBOM for ${REF}"
-  syft "${REF}" -o spdx-json=sbom.spdx.json
+  retry syft "${REF}" -o spdx-json=sbom.spdx.json
 
   echo "Attaching SBOM attestation to ${REF}"
-  cosign attest --yes --predicate sbom.spdx.json --type spdxjson "${REF}"
+  retry cosign attest --yes --predicate sbom.spdx.json --type spdxjson "${REF}"
+fi
 
-  rm -f sbom.spdx.json
-done
+rm -f sbom.spdx.json

.github/workflows/sign.yml

@@ -0,0 +1,76 @@
+name: Sign
+
+# Signs a release's container image and attaches its SBOM attestation.
+#
+# Two entry points:
+#   1. workflow_call   – invoked by the Release workflow (release.yml) once
+#                        GoReleaser has published the images, so every release
+#                        is signed as part of the release run.
+#   2. workflow_dispatch – run manually to sign a release that was published
+#                        earlier, e.g. after a transient registry failure left
+#                        a release unsigned. Dispatch from the default branch
+#                        and pass the release tag.
+#
+# Both paths sign with cosign keyless (Sigstore OIDC), so the certificate
+# identity is a GitHub Actions workflow ref under
+# https://github.com/observIQ/bindplane-operator/ — which is what the verify
+# commands in docs/releases.md match on.
+
+on:
+  workflow_call:
+    inputs:
+      tag:
+        description: 'Release tag whose published image should be signed and attested (e.g. 0.2.0).'
+        type: string
+        required: true
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Release tag whose published image should be signed and attested (e.g. 0.2.0). The image must already exist in the registry.'
+        type: string
+        required: true
+
+jobs:
+  sign:
+    name: Sign and attest
+    runs-on: ubuntu-latest-8-cores
+    permissions:
+      contents: read
+      packages: write  # push cosign signatures and attestations to GHCR
+      id-token: write  # required for cosign keyless signing via Sigstore
+    steps:
+      - name: Validate tag format
+        run: |
+          if [[ "${TAG}" == v* ]]; then
+            echo "Error: tag '${TAG}' must not have a 'v' prefix (e.g. use '1.0.0' not 'v1.0.0')"
+            exit 1
+          fi
+        env:
+          TAG: ${{ inputs.tag }}
+
+      # Checked out at the ref this workflow runs on, not at the release tag, so
+      # that a re-signing dispatch always uses the current signing script rather
+      # than the version shipped with the tag being signed.
+      - name: Clone the code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Install cosign
+        uses: sigstore/cosign-installer@v3
+
+      - name: Install syft
+        uses: anchore/sbom-action/download-syft@v0
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Sign multi-arch manifest and attach SBOM attestation (by digest)
+        run: .github/workflows/scripts/sign-and-attest.sh
+        env:
+          RELEASE_TAG: ${{ inputs.tag }}

docs/releases.md

@@ -14,9 +14,17 @@ Every release container image is signed using
 signature is produced in GitHub Actions using an OIDC identity token issued
 by GitHub, so there are no long-lived signing keys to manage or rotate.
 
-A valid signature proves that the image was built by the
-`observiq/bindplane-operator` GitHub Actions release workflow and has not
-been tampered with.
+A valid signature proves that the image was built and signed by an
+`observiq/bindplane-operator` GitHub Actions workflow and has not been
+tampered with.
+
+Signing runs in its own workflow (`.github/workflows/sign.yml`), which the
+release workflow invokes once the images are published. Because signing is a
+separate workflow, it can also be re-run on its own — see
+[Signing a release after the fact](#signing-a-release-after-the-fact).
+
+The image is signed by digest, so a single signature covers both the version
+tag and `latest` while they point at the same manifest list.
 
 ### Verifying the signature
 
@@ -31,6 +39,31 @@ A successful verification prints the signing certificate details and
 Rekor transparency log entry. A failed verification exits non-zero with
 an error message.
 
+The identity is matched with `--certificate-identity-regexp` against the
+repository prefix rather than a single workflow ref, so the same command
+verifies releases regardless of which repository workflow produced the
+signature.
+
+### Signing a release after the fact
+
+If a release is published but its signature or SBOM attestation is missing —
+for example because the registry rejected the upload during the release run —
+the Sign workflow can be dispatched on its own against the already-published
+images:
+
+```bash
+gh workflow run sign.yml --repo observiq/bindplane-operator -f tag=0.2.0
+```
+
+Dispatch from the default branch (the workflow signs whatever tag is passed in
+`tag`, and always uses the current signing script). The run is safe to repeat:
+signing and attestation are each skipped when the artifact is already present,
+so a re-run only fills in what is missing, and only the requested tag's
+manifest list is ever signed.
+
+To confirm the missing artifacts afterwards, use
+[`cosign tree`](#checking-supply-chain-artifacts).
+
 ## Software Bill of Materials (SBOM)
 
 An SBOM is a machine-readable inventory of every dependency (libraries,

go.mod

@@ -1,6 +1,6 @@
 module github.com/observiq/bindplane-operator
 
-go 1.26.5
+go 1.26.6
 
 require (
 	github.com/argoproj/argo-rollouts v1.9.1

tools/go.mod

@@ -1,6 +1,6 @@
 module github.com/observiq/bindplane-operator/tools
 
-go 1.26.5
+go 1.26.6
 
 tool (
 	github.com/elastic/crd-ref-docs