diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0295946..a15f945 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,6 @@ on: permissions: contents: write packages: write - id-token: write # Required for cosign keyless signing via Sigstore jobs: release: @@ -38,9 +37,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Install cosign - uses: sigstore/cosign-installer@v3 - + # Required by the sboms stanza in .goreleaser.yaml (binary SBOMs). - name: Install syft uses: anchore/sbom-action/download-syft@v0 @@ -59,5 +56,15 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Sign multi-arch manifests and attach SBOM attestations (by digest) - run: .github/workflows/scripts/sign-and-attest.sh + # Signing lives in its own workflow so that it can also be dispatched on its + # own to sign a release published earlier. See .github/workflows/sign.yml. + sign: + name: Sign + needs: release + permissions: + contents: read + packages: write # push cosign signatures and attestations to GHCR + id-token: write # required for cosign keyless signing via Sigstore + uses: ./.github/workflows/sign.yml + with: + tag: ${{ github.ref_name }} diff --git a/.github/workflows/scripts/sign-and-attest.sh b/.github/workflows/scripts/sign-and-attest.sh index a77ee3e..07e2320 100755 --- a/.github/workflows/scripts/sign-and-attest.sh +++ b/.github/workflows/scripts/sign-and-attest.sh @@ -1,17 +1,31 @@ #!/usr/bin/env bash # -# Sign container images and attach SBOM attestations for a release. +# Sign a release's container image and attach its SBOM attestation. # -# Called by the Release workflow (.github/workflows/release.yml) after -# GoReleaser pushes multi-arch manifest lists. For each release tag the -# script resolves the registry-side digest, deduplicates, and then: +# Called by the Sign workflow (.github/workflows/sign.yml), which the Release +# workflow invokes once GoReleaser has pushed the multi-arch manifest lists, and +# which can also be dispatched on its own to sign a release published earlier. +# For the requested release tag the script resolves the registry-side digest and: # 1. Signs the manifest list by digest with cosign (keyless / Sigstore OIDC). # 2. Generates an SPDX JSON SBOM with syft against the digest reference. # 3. Attaches the SBOM as a signed in-toto attestation with cosign. # # Signing by digest (name@sha256:...) is the cosign-recommended model; -# signatures are discoverable from any tag pointing at that digest, so -# signing once covers both the version tag and :latest. +# signatures are discoverable from any tag pointing at that digest, so signing +# the release tag's manifest list also covers :latest whenever :latest points at +# that same manifest list. Signing is always scoped to the requested release +# tag: a dispatch that re-signs an older release must never sign whatever +# :latest happens to point at now. +# +# Every registry operation is retried with exponential backoff, and each step +# is skipped when its artifact is already present. GHCR's token service +# intermittently rejects push-scoped token requests with "DENIED: denied" even +# when the credentials are valid and were granted the same scope seconds +# earlier — release run 31604663860 published 0.2.0 unsigned that way, after +# the images and the GitHub release had already gone out. A transient registry +# failure must not cost a release its signature, and re-running this script +# against an already-signed digest must be a cheap no-op so that recovery is +# just "run it again". # # Prerequisites (installed by the workflow): # - docker buildx (docker/setup-buildx-action) @@ -19,54 +33,104 @@ # - syft (anchore/sbom-action/download-syft) # # Environment: -# GITHUB_REF_NAME – the git tag pushed (e.g. "0.0.20"), set automatically -# by GitHub Actions. +# RELEASE_TAG – the release tag to sign (e.g. "0.0.20"), passed by the +# Sign workflow. Falls back to GITHUB_REF_NAME, which +# GitHub Actions sets to the tag on a tag-triggered run. +# RETRY_MAX_ATTEMPTS – attempts per registry operation (default 5). +# RETRY_INITIAL_DELAY – seconds before the first retry, doubling on each +# subsequent attempt (default 5). set -euo pipefail -if [ -z "${GITHUB_REF_NAME:-}" ]; then - echo "ERROR: GITHUB_REF_NAME is not set" +RELEASE_TAG="${RELEASE_TAG:-${GITHUB_REF_NAME:-}}" +if [ -z "${RELEASE_TAG}" ]; then + echo "ERROR: neither RELEASE_TAG nor GITHUB_REF_NAME is set" exit 1 fi IMAGE="ghcr.io/observiq/bindplane-operator" -TAGS=( - "${GITHUB_REF_NAME}" - "latest" -) -declare -A TAG_TO_DIGEST -declare -A SEEN_DIGESTS +RETRY_MAX_ATTEMPTS="${RETRY_MAX_ATTEMPTS:-5}" +RETRY_INITIAL_DELAY="${RETRY_INITIAL_DELAY:-5}" -echo "Resolving registry digests for ${IMAGE} tags:" -for TAG in "${TAGS[@]}"; do - DIGEST=$(docker buildx imagetools inspect "${IMAGE}:${TAG}" --format '{{ .Manifest.Digest }}') - if [ -z "${DIGEST}" ]; then - echo "ERROR: failed to resolve digest for ${IMAGE}:${TAG}" - exit 1 - fi - TAG_TO_DIGEST["${TAG}"]="${DIGEST}" - echo " ${TAG} -> ${DIGEST}" -done +# retry [args...] +# Runs the command, retrying with exponential backoff until it succeeds or +# RETRY_MAX_ATTEMPTS is exhausted. Diagnostics go to stderr so that callers can +# safely capture the command's stdout via command substitution. +retry() { + local attempt=1 + local delay="${RETRY_INITIAL_DELAY}" + until "$@"; do + if [ "${attempt}" -ge "${RETRY_MAX_ATTEMPTS}" ]; then + echo "ERROR: '$*' failed after ${RETRY_MAX_ATTEMPTS} attempts" >&2 + return 1 + fi + echo " attempt ${attempt}/${RETRY_MAX_ATTEMPTS} of '$*' failed, retrying in ${delay}s" >&2 + sleep "${delay}" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done +} -for TAG in "${TAGS[@]}"; do - DIGEST="${TAG_TO_DIGEST[${TAG}]}" - if [ -n "${SEEN_DIGESTS[${DIGEST}]:-}" ]; then - echo "Digest ${DIGEST} already signed and attested (covered by tag ${SEEN_DIGESTS[${DIGEST}]}), skipping ${TAG}" - continue +# inspect_digest +# Prints the manifest digest for a tag. Fails on anything that is not a +# well-formed digest so that a garbled or partial registry response is retried +# rather than propagated into a signing reference. +inspect_digest() { + local digest + if ! digest=$(docker buildx imagetools inspect "${IMAGE}:${1}" --format '{{ .Manifest.Digest }}'); then + return 1 + fi + if [[ ! "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: unexpected digest '${digest}' for ${IMAGE}:${1}" >&2 + return 1 fi - SEEN_DIGESTS["${DIGEST}"]="${TAG}" + printf '%s\n' "${digest}" +} - REF="${IMAGE}@${DIGEST}" +# artifact_exists +# True when the tag already resolves in the registry. Deliberately not retried: +# a missing artifact is the expected answer for a fresh release, and a false +# negative only costs a redundant signing attempt. +artifact_exists() { + docker buildx imagetools inspect "${IMAGE}:${1}" >/dev/null 2>&1 +} +echo "Resolving registry digest for ${IMAGE}:" +DIGEST=$(retry inspect_digest "${RELEASE_TAG}") +echo " ${RELEASE_TAG} -> ${DIGEST}" + +# Reported for operator visibility only — :latest needs no separate signature +# when it points at the same manifest list, and must not be signed when it does +# not (see the note on tag scoping above). Best effort: a failure to resolve +# :latest is not a reason to fail signing the release tag. +LATEST_DIGEST=$(inspect_digest "latest" 2>/dev/null || true) +if [ "${LATEST_DIGEST}" = "${DIGEST}" ]; then + echo " latest -> ${DIGEST} (same manifest list, covered by this signature)" +elif [ -n "${LATEST_DIGEST}" ]; then + echo " latest -> ${LATEST_DIGEST} (different manifest list, not signed by this run)" +fi + +REF="${IMAGE}@${DIGEST}" +# cosign derives its artifact tags from the digest, replacing the algorithm +# separator: sha256:abc... -> sha256-abc....sig / sha256-abc....att +COSIGN_TAG="${DIGEST/:/-}" + +if artifact_exists "${COSIGN_TAG}.sig"; then + echo "Signature already present for ${REF}, skipping" +else echo "Signing ${REF}" - cosign sign --yes "${REF}" + retry cosign sign --yes "${REF}" +fi +if artifact_exists "${COSIGN_TAG}.att"; then + echo "SBOM attestation already present for ${REF}, skipping" +else echo "Generating SBOM for ${REF}" - syft "${REF}" -o spdx-json=sbom.spdx.json + retry syft "${REF}" -o spdx-json=sbom.spdx.json echo "Attaching SBOM attestation to ${REF}" - cosign attest --yes --predicate sbom.spdx.json --type spdxjson "${REF}" + retry cosign attest --yes --predicate sbom.spdx.json --type spdxjson "${REF}" +fi - rm -f sbom.spdx.json -done +rm -f sbom.spdx.json diff --git a/.github/workflows/sign.yml b/.github/workflows/sign.yml new file mode 100644 index 0000000..e9ebbcb --- /dev/null +++ b/.github/workflows/sign.yml @@ -0,0 +1,76 @@ +name: Sign + +# Signs a release's container image and attaches its SBOM attestation. +# +# Two entry points: +# 1. workflow_call – invoked by the Release workflow (release.yml) once +# GoReleaser has published the images, so every release +# is signed as part of the release run. +# 2. workflow_dispatch – run manually to sign a release that was published +# earlier, e.g. after a transient registry failure left +# a release unsigned. Dispatch from the default branch +# and pass the release tag. +# +# Both paths sign with cosign keyless (Sigstore OIDC), so the certificate +# identity is a GitHub Actions workflow ref under +# https://github.com/observIQ/bindplane-operator/ — which is what the verify +# commands in docs/releases.md match on. + +on: + workflow_call: + inputs: + tag: + description: 'Release tag whose published image should be signed and attested (e.g. 0.2.0).' + type: string + required: true + workflow_dispatch: + inputs: + tag: + description: 'Release tag whose published image should be signed and attested (e.g. 0.2.0). The image must already exist in the registry.' + type: string + required: true + +jobs: + sign: + name: Sign and attest + runs-on: ubuntu-latest-8-cores + permissions: + contents: read + packages: write # push cosign signatures and attestations to GHCR + id-token: write # required for cosign keyless signing via Sigstore + steps: + - name: Validate tag format + run: | + if [[ "${TAG}" == v* ]]; then + echo "Error: tag '${TAG}' must not have a 'v' prefix (e.g. use '1.0.0' not 'v1.0.0')" + exit 1 + fi + env: + TAG: ${{ inputs.tag }} + + # Checked out at the ref this workflow runs on, not at the release tag, so + # that a re-signing dispatch always uses the current signing script rather + # than the version shipped with the tag being signed. + - name: Clone the code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + + - name: Install syft + uses: anchore/sbom-action/download-syft@v0 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Sign multi-arch manifest and attach SBOM attestation (by digest) + run: .github/workflows/scripts/sign-and-attest.sh + env: + RELEASE_TAG: ${{ inputs.tag }} diff --git a/docs/releases.md b/docs/releases.md index 82050cb..2b280ac 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -14,9 +14,17 @@ Every release container image is signed using signature is produced in GitHub Actions using an OIDC identity token issued by GitHub, so there are no long-lived signing keys to manage or rotate. -A valid signature proves that the image was built by the -`observiq/bindplane-operator` GitHub Actions release workflow and has not -been tampered with. +A valid signature proves that the image was built and signed by an +`observiq/bindplane-operator` GitHub Actions workflow and has not been +tampered with. + +Signing runs in its own workflow (`.github/workflows/sign.yml`), which the +release workflow invokes once the images are published. Because signing is a +separate workflow, it can also be re-run on its own — see +[Signing a release after the fact](#signing-a-release-after-the-fact). + +The image is signed by digest, so a single signature covers both the version +tag and `latest` while they point at the same manifest list. ### Verifying the signature @@ -31,6 +39,31 @@ A successful verification prints the signing certificate details and Rekor transparency log entry. A failed verification exits non-zero with an error message. +The identity is matched with `--certificate-identity-regexp` against the +repository prefix rather than a single workflow ref, so the same command +verifies releases regardless of which repository workflow produced the +signature. + +### Signing a release after the fact + +If a release is published but its signature or SBOM attestation is missing — +for example because the registry rejected the upload during the release run — +the Sign workflow can be dispatched on its own against the already-published +images: + +```bash +gh workflow run sign.yml --repo observiq/bindplane-operator -f tag=0.2.0 +``` + +Dispatch from the default branch (the workflow signs whatever tag is passed in +`tag`, and always uses the current signing script). The run is safe to repeat: +signing and attestation are each skipped when the artifact is already present, +so a re-run only fills in what is missing, and only the requested tag's +manifest list is ever signed. + +To confirm the missing artifacts afterwards, use +[`cosign tree`](#checking-supply-chain-artifacts). + ## Software Bill of Materials (SBOM) An SBOM is a machine-readable inventory of every dependency (libraries, diff --git a/go.mod b/go.mod index 6bd2189..316f780 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/observiq/bindplane-operator -go 1.26.5 +go 1.26.6 require ( github.com/argoproj/argo-rollouts v1.9.1 diff --git a/tools/go.mod b/tools/go.mod index ac560ec..763ceb2 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -1,6 +1,6 @@ module github.com/observiq/bindplane-operator/tools -go 1.26.5 +go 1.26.6 tool ( github.com/elastic/crd-ref-docs