diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000000..8859cae25a --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,819 @@ +name: Publish Docker release + +on: + workflow_dispatch: + inputs: + version: + description: Published D2 release tag to publish to Docker Hub + required: true + type: string + publish_latest: + description: Also update latest (stable releases only) + required: true + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: docker-production-release + cancel-in-progress: false + +env: + DOCKER_IMAGE: terrastruct/d2 + +jobs: + validate: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + version: ${{ steps.release.outputs.version }} + release_commit: ${{ steps.release.outputs.release_commit }} + release_id: ${{ steps.release.outputs.release_id }} + release_prerelease: ${{ steps.release.outputs.release_prerelease }} + amd64_asset_id: ${{ steps.release.outputs.amd64_asset_id }} + amd64_asset_digest: ${{ steps.release.outputs.amd64_asset_digest }} + arm64_asset_id: ${{ steps.release.outputs.arm64_asset_id }} + arm64_asset_digest: ${{ steps.release.outputs.arm64_asset_digest }} + steps: + - name: Validate protected master dispatch and published release + id: release + env: + GH_TOKEN: ${{ github.token }} + PUBLISH_LATEST: ${{ inputs.publish_latest }} + REF_PROTECTED: ${{ github.ref_protected }} + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + + if [[ "$GITHUB_REF" != refs/heads/master || "$REF_PROTECTED" != true ]]; then + echo "Run this workflow from the protected master branch" >&2 + exit 1 + fi + + if [[ "${#VERSION}" -gt 128 ]] || + ! printf '%s\n' "$VERSION" | grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z]+([.-][0-9A-Za-z]+)*)?$'; then + echo "version must be a v-prefixed semantic version usable as a Docker tag" >&2 + exit 1 + fi + + release_json=$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$VERSION") + if [[ $(jq -r '.draft' <<<"$release_json") != false ]]; then + echo "$VERSION is a draft release" >&2 + exit 1 + fi + if [[ $(jq -r '.published_at' <<<"$release_json") == null ]]; then + echo "$VERSION has not been published" >&2 + exit 1 + fi + if [[ $(jq -r '.tag_name' <<<"$release_json") != "$VERSION" ]]; then + echo "release tag does not match $VERSION" >&2 + exit 1 + fi + release_id=$(jq -r '.id' <<<"$release_json") + release_prerelease=$(jq -r '.prerelease' <<<"$release_json") + if ! printf '%s\n' "$release_id" | grep -Eq '^[0-9]+$'; then + echo "$VERSION does not have a valid GitHub release ID" >&2 + exit 1 + fi + if [[ "$release_prerelease" != true && "$release_prerelease" != false ]]; then + echo "$VERSION does not have a valid prerelease state" >&2 + exit 1 + fi + if [[ "$PUBLISH_LATEST" == true ]] && + { [[ "$release_prerelease" == true ]] || [[ "$VERSION" == *-* ]]; }; then + echo "publish_latest cannot be used for a prerelease" >&2 + exit 1 + fi + + resolve_tag_commit() { + local depth object_json object_sha object_type + depth=0 + object_json=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$VERSION") + object_sha=$(jq -r '.object.sha' <<<"$object_json") + object_type=$(jq -r '.object.type' <<<"$object_json") + while [[ "$object_type" == tag ]]; do + depth=$((depth + 1)) + if [[ "$depth" -gt 8 ]] || + ! printf '%s\n' "$object_sha" | grep -Eq '^[0-9a-f]{40}$'; then + echo "could not safely peel tag $VERSION" >&2 + return 1 + fi + object_json=$(gh api "repos/$GITHUB_REPOSITORY/git/tags/$object_sha") + object_sha=$(jq -r '.object.sha' <<<"$object_json") + object_type=$(jq -r '.object.type' <<<"$object_json") + done + if [[ "$object_type" != commit ]]; then + echo "$VERSION does not peel to a commit" >&2 + return 1 + fi + printf '%s\n' "$object_sha" + } + release_commit=$(resolve_tag_commit) + if ! printf '%s\n' "$release_commit" | grep -Eq '^[0-9a-f]{40}$'; then + echo "could not resolve $VERSION to an immutable commit" >&2 + exit 1 + fi + compare_status=$(gh api \ + "repos/$GITHUB_REPOSITORY/compare/$release_commit...$GITHUB_SHA" \ + --jq '.status') + if [[ "$compare_status" != ahead && "$compare_status" != identical ]]; then + echo "$VERSION does not resolve to an ancestor of the protected master dispatch" >&2 + exit 1 + fi + + for arch in amd64 arm64; do + asset="d2-$VERSION-linux-$arch.tar.gz" + asset_count=$(jq --arg asset "$asset" \ + '[.assets[] | select(.name == $asset and .state == "uploaded" and .size > 0)] | length' \ + <<<"$release_json") + if [[ "$asset_count" -ne 1 ]]; then + echo "$VERSION must contain exactly one uploaded, non-empty $asset" >&2 + exit 1 + fi + + asset_id=$(jq -r --arg asset "$asset" \ + '.assets[] | select(.name == $asset) | .id' <<<"$release_json") + asset_digest=$(jq -r --arg asset "$asset" \ + '.assets[] | select(.name == $asset) | .digest' <<<"$release_json") + if ! printf '%s\n' "$asset_id" | grep -Eq '^[0-9]+$'; then + echo "$asset does not have a valid asset ID" >&2 + exit 1 + fi + if ! printf '%s\n' "$asset_digest" | grep -Eq '^sha256:[0-9a-f]{64}$'; then + echo "$asset does not have a valid GitHub SHA-256 digest" >&2 + exit 1 + fi + + if [[ "$arch" == amd64 ]]; then + amd64_asset_id=$asset_id + amd64_asset_digest=$asset_digest + else + arm64_asset_id=$asset_id + arm64_asset_digest=$asset_digest + fi + done + + { + echo "version=$VERSION" + echo "release_commit=$release_commit" + echo "release_id=$release_id" + echo "release_prerelease=$release_prerelease" + echo "amd64_asset_id=$amd64_asset_id" + echo "amd64_asset_digest=$amd64_asset_digest" + echo "arm64_asset_id=$arm64_asset_id" + echo "arm64_asset_digest=$arm64_asset_digest" + } >>"$GITHUB_OUTPUT" + + preflight: + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 5 + environment: docker-release + env: + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + VERSION: ${{ needs.validate.outputs.version }} + steps: + - name: Log in to Docker Hub + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + set -euo pipefail + test "$DOCKERHUB_USERNAME" = terrastruct + test -n "$DOCKERHUB_TOKEN" + printf '%s' "$DOCKERHUB_TOKEN" | \ + docker login --username "$DOCKERHUB_USERNAME" --password-stdin + + - name: Verify Docker Hub version-tag immutability + run: | + set -euo pipefail + expected_rule='^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+([.-][0-9A-Za-z]+)*)?$' + curl -fsSL 'https://hub.docker.com/v2/namespaces/terrastruct/repositories/d2' | \ + jq -e --arg rule "$expected_rule" ' + .immutable_tags_settings.enabled == true and + .immutable_tags_settings.rules == [$rule] + ' >/dev/null + + - name: Refuse to overwrite an immutable version tag + run: | + set -euo pipefail + image="$DOCKER_IMAGE" + registry_token=$(curl -fsSL \ + 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:terrastruct/d2:pull' | \ + jq -er '.token') + if ! status=$(curl -sS -o /dev/null -w '%{http_code}' --head \ + -H "Authorization: Bearer $registry_token" \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \ + "https://registry-1.docker.io/v2/terrastruct/d2/manifests/$VERSION"); then + echo "Docker Hub version-tag preflight request failed" >&2 + exit 1 + fi + case "$status" in + 404) ;; + 200) + echo "$image:$VERSION already exists; version tags are immutable" >&2 + exit 1 + ;; + *) + echo "Docker Hub version-tag preflight returned HTTP $status" >&2 + exit 1 + ;; + esac + + - name: Clean up Docker credentials + if: always() + run: docker logout >/dev/null 2>&1 || true + + build-amd64: + needs: [validate, preflight] + runs-on: ubuntu-24.04 + timeout-minutes: 60 + environment: docker-release + env: + ARCH: amd64 + ASSET_DIGEST: ${{ needs.validate.outputs.amd64_asset_digest }} + ASSET_ID: ${{ needs.validate.outputs.amd64_asset_id }} + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + PLATFORM: linux/amd64 + VERSION: ${{ needs.validate.outputs.version }} + outputs: + digest: ${{ steps.build.outputs.digest }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.validate.outputs.release_commit }} + persist-credentials: false + + - name: Download and verify exact release archive + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + asset="d2-$VERSION-linux-$ARCH.tar.gz" + archive="$RUNNER_TEMP/docker-context/$asset" + mkdir -p "$RUNNER_TEMP/docker-context" + gh api \ + --method GET \ + -H 'Accept: application/octet-stream' \ + "repos/$GITHUB_REPOSITORY/releases/assets/$ASSET_ID" >"$archive" + printf '%s %s\n' "${ASSET_DIGEST#sha256:}" "$archive" | sha256sum -c - + tar -tzf "$archive" >/dev/null + cp ci/release/docker/entrypoint.sh "$RUNNER_TEMP/docker-context/entrypoint.sh" + + - name: Log in to Docker Hub + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + set -euo pipefail + test "$DOCKERHUB_USERNAME" = terrastruct + test -n "$DOCKERHUB_TOKEN" + printf '%s' "$DOCKERHUB_TOKEN" | \ + docker login --username "$DOCKERHUB_USERNAME" --password-stdin + + - name: Build and push by digest with provenance + id: build + run: | + set -euo pipefail + builder="d2-release-$ARCH" + image="$DOCKER_IMAGE" + metadata="$RUNNER_TEMP/build-metadata.json" + + docker buildx create --name "$builder" --driver docker-container --use + docker buildx inspect --bootstrap + docker buildx build \ + --platform "$PLATFORM" \ + --provenance=mode=max \ + --output "type=image,name=$image,push-by-digest=true,name-canonical=true,push=true" \ + --metadata-file "$metadata" \ + --file ci/release/docker/Dockerfile \ + "$RUNNER_TEMP/docker-context" + + digest=$(jq -er '."containerimage.digest"' "$metadata") + if ! printf '%s\n' "$digest" | grep -Eq '^sha256:[0-9a-f]{64}$'; then + echo "build did not return a valid image digest" >&2 + exit 1 + fi + echo "digest=$digest" >>"$GITHUB_OUTPUT" + + - name: Verify image natively + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + set -euo pipefail + image="$DOCKER_IMAGE@$DIGEST" + smoke="$RUNNER_TEMP/smoke" + mkdir -p "$smoke" + printf 'x -> y\n' >"$smoke/input.d2" + + version_output=$(docker run --rm --platform "$PLATFORM" "$image" --version) + version_output=${version_output%$'\r'} + printf '%s\n' "$version_output" + test "$version_output" = "$VERSION" + + docker run --rm --platform "$PLATFORM" \ + -u "$(id -u):$(id -g)" \ + -v "$smoke:/home/debian/src" \ + "$image" input.d2 output.svg + test -s "$smoke/output.svg" + grep -q '/dev/null 2>&1 || true + docker buildx rm "d2-release-$ARCH" >/dev/null 2>&1 || true + + build-arm64: + needs: [validate, preflight] + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + environment: docker-release + env: + ARCH: arm64 + ASSET_DIGEST: ${{ needs.validate.outputs.arm64_asset_digest }} + ASSET_ID: ${{ needs.validate.outputs.arm64_asset_id }} + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + PLATFORM: linux/arm64 + VERSION: ${{ needs.validate.outputs.version }} + outputs: + digest: ${{ steps.build.outputs.digest }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.validate.outputs.release_commit }} + persist-credentials: false + + - name: Download and verify exact release archive + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + asset="d2-$VERSION-linux-$ARCH.tar.gz" + archive="$RUNNER_TEMP/docker-context/$asset" + mkdir -p "$RUNNER_TEMP/docker-context" + gh api \ + --method GET \ + -H 'Accept: application/octet-stream' \ + "repos/$GITHUB_REPOSITORY/releases/assets/$ASSET_ID" >"$archive" + printf '%s %s\n' "${ASSET_DIGEST#sha256:}" "$archive" | sha256sum -c - + tar -tzf "$archive" >/dev/null + cp ci/release/docker/entrypoint.sh "$RUNNER_TEMP/docker-context/entrypoint.sh" + + - name: Log in to Docker Hub + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + set -euo pipefail + test "$DOCKERHUB_USERNAME" = terrastruct + test -n "$DOCKERHUB_TOKEN" + printf '%s' "$DOCKERHUB_TOKEN" | \ + docker login --username "$DOCKERHUB_USERNAME" --password-stdin + + - name: Build and push by digest with provenance + id: build + run: | + set -euo pipefail + builder="d2-release-$ARCH" + image="$DOCKER_IMAGE" + metadata="$RUNNER_TEMP/build-metadata.json" + + docker buildx create --name "$builder" --driver docker-container --use + docker buildx inspect --bootstrap + docker buildx build \ + --platform "$PLATFORM" \ + --provenance=mode=max \ + --output "type=image,name=$image,push-by-digest=true,name-canonical=true,push=true" \ + --metadata-file "$metadata" \ + --file ci/release/docker/Dockerfile \ + "$RUNNER_TEMP/docker-context" + + digest=$(jq -er '."containerimage.digest"' "$metadata") + if ! printf '%s\n' "$digest" | grep -Eq '^sha256:[0-9a-f]{64}$'; then + echo "build did not return a valid image digest" >&2 + exit 1 + fi + echo "digest=$digest" >>"$GITHUB_OUTPUT" + + - name: Verify image natively + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + set -euo pipefail + image="$DOCKER_IMAGE@$DIGEST" + smoke="$RUNNER_TEMP/smoke" + mkdir -p "$smoke" + printf 'x -> y\n' >"$smoke/input.d2" + + version_output=$(docker run --rm --platform "$PLATFORM" "$image" --version) + version_output=${version_output%$'\r'} + printf '%s\n' "$version_output" + test "$version_output" = "$VERSION" + + docker run --rm --platform "$PLATFORM" \ + -u "$(id -u):$(id -g)" \ + -v "$smoke:/home/debian/src" \ + "$image" input.d2 output.svg + test -s "$smoke/output.svg" + grep -q '/dev/null 2>&1 || true + docker buildx rm "d2-release-$ARCH" >/dev/null 2>&1 || true + + publish-version: + needs: [validate, build-amd64, build-arm64] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + environment: docker-release + env: + AMD64_ASSET_DIGEST: ${{ needs.validate.outputs.amd64_asset_digest }} + AMD64_ASSET_ID: ${{ needs.validate.outputs.amd64_asset_id }} + AMD64_DIGEST: ${{ needs.build-amd64.outputs.digest }} + ARM64_ASSET_DIGEST: ${{ needs.validate.outputs.arm64_asset_digest }} + ARM64_ASSET_ID: ${{ needs.validate.outputs.arm64_asset_id }} + ARM64_DIGEST: ${{ needs.build-arm64.outputs.digest }} + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + EXPECTED_RELEASE_COMMIT: ${{ needs.validate.outputs.release_commit }} + EXPECTED_RELEASE_ID: ${{ needs.validate.outputs.release_id }} + EXPECTED_RELEASE_PRERELEASE: ${{ needs.validate.outputs.release_prerelease }} + VERSION: ${{ needs.validate.outputs.version }} + outputs: + digest: ${{ steps.version.outputs.digest }} + steps: + - name: Log in to Docker Hub + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + set -euo pipefail + test "$DOCKERHUB_USERNAME" = terrastruct + test -n "$DOCKERHUB_TOKEN" + printf '%s' "$DOCKERHUB_TOKEN" | \ + docker login --username "$DOCKERHUB_USERNAME" --password-stdin + + - name: Validate candidate manifest before publishing a tag + id: candidate + run: | + set -euo pipefail + image="$DOCKER_IMAGE" + candidate="$RUNNER_TEMP/candidate-manifest.json" + + for digest in "$AMD64_DIGEST" "$ARM64_DIGEST"; do + if ! printf '%s\n' "$digest" | grep -Eq '^sha256:[0-9a-f]{64}$'; then + echo "invalid architecture digest" >&2 + exit 1 + fi + done + + docker buildx imagetools create \ + --dry-run \ + --progress=none \ + "$image@$AMD64_DIGEST" \ + "$image@$ARM64_DIGEST" >"$candidate" + if [[ $(tail -c 1 "$candidate" | od -An -tx1 | tr -d ' \n') != 0a ]]; then + echo "candidate manifest output does not end in the expected newline" >&2 + exit 1 + fi + candidate_size=$(wc -c <"$candidate" | tr -d ' ') + candidate_digest="sha256:$(dd if="$candidate" bs=1 count="$((candidate_size - 1))" 2>/dev/null | \ + sha256sum | awk '{print $1}')" + if ! printf '%s\n' "$candidate_digest" | grep -Eq '^sha256:[0-9a-f]{64}$'; then + echo "candidate manifest does not have a valid digest" >&2 + exit 1 + fi + + jq -e ' + def runtime: + .platform.os == "linux" and + (.platform.architecture == "amd64" or .platform.architecture == "arm64"); + def attestation: + .platform.os == "unknown" and + .platform.architecture == "unknown" and + .annotations["vnd.docker.reference.type"] == "attestation-manifest"; + .schemaVersion == 2 and + (.manifests | type == "array") and + ([.manifests[] | select(runtime)] | length == 2) and + ([.manifests[] | select(runtime and .platform.architecture == "amd64")] | length == 1) and + ([.manifests[] | select(runtime and .platform.architecture == "arm64")] | length == 1) and + ([.manifests[] | select(attestation)] | length == 2) and + (.manifests | length == 4) and + (([.manifests[] | select(runtime) | .digest] | sort) == + ([.manifests[] | select(attestation) | + .annotations["vnd.docker.reference.digest"]] | sort)) + ' "$candidate" >/dev/null + + registry_token=$(curl -fsSL \ + 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:terrastruct/d2:pull' | \ + jq -er '.token') + if ! status=$(curl -sS -o /dev/null -D "$RUNNER_TEMP/existing-version-headers" \ + -w '%{http_code}' --head \ + -H "Authorization: Bearer $registry_token" \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \ + "https://registry-1.docker.io/v2/terrastruct/d2/manifests/$VERSION"); then + echo "Docker Hub version-tag revalidation request failed" >&2 + exit 1 + fi + if [[ "$status" == 200 ]]; then + existing_digest=$(tr -d '\r' <"$RUNNER_TEMP/existing-version-headers" | \ + awk 'tolower($1) == "docker-content-digest:" {print $2}' | tail -n 1) + if [[ "$existing_digest" != "$candidate_digest" ]]; then + echo "$image:$VERSION exists at a digest different from this run's candidate" >&2 + exit 1 + fi + docker buildx imagetools inspect --raw "$image:$VERSION" \ + >"$RUNNER_TEMP/existing-version-manifest.json" + jq -S '.manifests | sort_by(.digest)' "$candidate" \ + >"$RUNNER_TEMP/candidate-descriptors.json" + jq -S '.manifests | sort_by(.digest)' "$RUNNER_TEMP/existing-version-manifest.json" \ + >"$RUNNER_TEMP/existing-version-descriptors.json" + if ! cmp -s \ + "$RUNNER_TEMP/candidate-descriptors.json" \ + "$RUNNER_TEMP/existing-version-descriptors.json"; then + diff -u \ + "$RUNNER_TEMP/candidate-descriptors.json" \ + "$RUNNER_TEMP/existing-version-descriptors.json" || true + echo "$image:$VERSION exists but does not match this run's verified candidate" >&2 + exit 1 + fi + version_exists=true + elif [[ "$status" == 404 ]]; then + version_exists=false + else + echo "Docker Hub version-tag revalidation returned HTTP $status" >&2 + exit 1 + fi + { + echo "candidate_digest=$candidate_digest" + echo "version_exists=$version_exists" + } >>"$GITHUB_OUTPUT" + + - name: Revalidate immutable release inputs + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + + release_json=$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$VERSION") + if [[ $(jq -r '.id' <<<"$release_json") != "$EXPECTED_RELEASE_ID" ]] || + [[ $(jq -r '.tag_name' <<<"$release_json") != "$VERSION" ]] || + [[ $(jq -r '.draft' <<<"$release_json") != false ]] || + [[ $(jq -r '.published_at' <<<"$release_json") == null ]] || + [[ $(jq -r '.prerelease' <<<"$release_json") != "$EXPECTED_RELEASE_PRERELEASE" ]]; then + echo "$VERSION release identity or publication state changed after validation" >&2 + exit 1 + fi + + resolve_tag_commit() { + local depth object_json object_sha object_type + depth=0 + object_json=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$VERSION") + object_sha=$(jq -r '.object.sha' <<<"$object_json") + object_type=$(jq -r '.object.type' <<<"$object_json") + while [[ "$object_type" == tag ]]; do + depth=$((depth + 1)) + if [[ "$depth" -gt 8 ]] || + ! printf '%s\n' "$object_sha" | grep -Eq '^[0-9a-f]{40}$'; then + echo "could not safely peel tag $VERSION" >&2 + return 1 + fi + object_json=$(gh api "repos/$GITHUB_REPOSITORY/git/tags/$object_sha") + object_sha=$(jq -r '.object.sha' <<<"$object_json") + object_type=$(jq -r '.object.type' <<<"$object_json") + done + if [[ "$object_type" != commit ]]; then + echo "$VERSION does not peel to a commit" >&2 + return 1 + fi + printf '%s\n' "$object_sha" + } + current_release_commit=$(resolve_tag_commit) + if [[ "$current_release_commit" != "$EXPECTED_RELEASE_COMMIT" ]]; then + echo "$VERSION tag moved after validation" >&2 + exit 1 + fi + + for arch in amd64 arm64; do + asset="d2-$VERSION-linux-$arch.tar.gz" + asset_count=$(jq --arg asset "$asset" \ + '[.assets[] | select(.name == $asset and .state == "uploaded" and .size > 0)] | length' \ + <<<"$release_json") + if [[ "$asset_count" -ne 1 ]]; then + echo "$VERSION no longer contains exactly one uploaded, non-empty $asset" >&2 + exit 1 + fi + asset_id=$(jq -r --arg asset "$asset" \ + '.assets[] | select(.name == $asset) | .id' <<<"$release_json") + asset_digest=$(jq -r --arg asset "$asset" \ + '.assets[] | select(.name == $asset) | .digest' <<<"$release_json") + if [[ "$arch" == amd64 ]]; then + expected_asset_id=$AMD64_ASSET_ID + expected_asset_digest=$AMD64_ASSET_DIGEST + else + expected_asset_id=$ARM64_ASSET_ID + expected_asset_digest=$ARM64_ASSET_DIGEST + fi + if [[ "$asset_id" != "$expected_asset_id" ]] || + [[ "$asset_digest" != "$expected_asset_digest" ]]; then + echo "$asset ID or digest changed after validation" >&2 + exit 1 + fi + done + + - name: Publish and verify immutable version manifest + id: version + env: + CANDIDATE_DIGEST: ${{ steps.candidate.outputs.candidate_digest }} + VERSION_EXISTS: ${{ steps.candidate.outputs.version_exists }} + run: | + set -euo pipefail + image="$DOCKER_IMAGE" + candidate="$RUNNER_TEMP/candidate-manifest.json" + metadata="$RUNNER_TEMP/version-metadata.json" + published="$RUNNER_TEMP/version-manifest.json" + + if [[ "$VERSION_EXISTS" != true ]]; then + docker buildx imagetools create \ + --tag "$image:$VERSION" \ + --metadata-file "$metadata" \ + "$image@$AMD64_DIGEST" \ + "$image@$ARM64_DIGEST" + fi + docker buildx imagetools inspect --raw "$image:$VERSION" >"$published" + + jq -S '.manifests | sort_by(.digest)' "$candidate" >"$RUNNER_TEMP/candidate-descriptors.json" + jq -S '.manifests | sort_by(.digest)' "$published" >"$RUNNER_TEMP/published-descriptors.json" + if ! cmp -s "$RUNNER_TEMP/candidate-descriptors.json" "$RUNNER_TEMP/published-descriptors.json"; then + diff -u "$RUNNER_TEMP/candidate-descriptors.json" "$RUNNER_TEMP/published-descriptors.json" || true + echo "published version manifest does not match the verified candidate" >&2 + exit 1 + fi + + registry_token=$(curl -fsSL \ + 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:terrastruct/d2:pull' | \ + jq -er '.token') + if ! status=$(curl -sS -o /dev/null -D "$RUNNER_TEMP/version-headers" \ + -w '%{http_code}' --head \ + -H "Authorization: Bearer $registry_token" \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \ + "https://registry-1.docker.io/v2/terrastruct/d2/manifests/$VERSION"); then + echo "Docker Hub version-manifest verification request failed" >&2 + exit 1 + fi + if [[ "$status" != 200 ]]; then + echo "Docker Hub version-manifest verification returned HTTP $status" >&2 + exit 1 + fi + digest=$(tr -d '\r' <"$RUNNER_TEMP/version-headers" | \ + awk 'tolower($1) == "docker-content-digest:" {print $2}' | tail -n 1) + if ! printf '%s\n' "$digest" | grep -Eq '^sha256:[0-9a-f]{64}$'; then + echo "version publication did not return a valid manifest digest" >&2 + exit 1 + fi + if [[ "$digest" != "$CANDIDATE_DIGEST" ]]; then + echo "registry manifest digest does not match the verified candidate" >&2 + exit 1 + fi + if [[ "$VERSION_EXISTS" != true ]]; then + created_digest=$(jq -er '."containerimage.descriptor".digest' "$metadata") + if [[ "$created_digest" != "$digest" ]]; then + echo "registry manifest digest does not match the publication result" >&2 + exit 1 + fi + fi + docker buildx imagetools inspect "$image:$VERSION" + echo "digest=$digest" >>"$GITHUB_OUTPUT" + + - name: Record publication + env: + RELEASE_COMMIT: ${{ needs.validate.outputs.release_commit }} + VERSION_DIGEST: ${{ steps.version.outputs.digest }} + run: | + { + echo "Published and verified \`$DOCKER_IMAGE:$VERSION\`." + echo + echo "- Release commit: \`$RELEASE_COMMIT\`" + echo "- amd64 release asset: ID \`$AMD64_ASSET_ID\`, \`$AMD64_ASSET_DIGEST\`" + echo "- arm64 release asset: ID \`$ARM64_ASSET_ID\`, \`$ARM64_ASSET_DIGEST\`" + echo "- amd64 image: \`$AMD64_DIGEST\`" + echo "- arm64 image: \`$ARM64_DIGEST\`" + echo "- Version manifest: \`$VERSION_DIGEST\`" + } >>"$GITHUB_STEP_SUMMARY" + + - name: Clean up Docker credentials + if: always() + run: docker logout >/dev/null 2>&1 || true + + publish-latest: + if: inputs.publish_latest + needs: [validate, publish-version] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + environment: docker-release + env: + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + EXPECTED_RELEASE_ID: ${{ needs.validate.outputs.release_id }} + VERSION: ${{ needs.validate.outputs.version }} + VERSION_DIGEST: ${{ needs.publish-version.outputs.digest }} + steps: + - name: Log in to Docker Hub + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + set -euo pipefail + test "$DOCKERHUB_USERNAME" = terrastruct + test -n "$DOCKERHUB_TOKEN" + printf '%s' "$DOCKERHUB_TOKEN" | \ + docker login --username "$DOCKERHUB_USERNAME" --password-stdin + + - name: Confirm release is still stable and published + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + release_json=$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$VERSION") + if [[ $(jq -r '.id' <<<"$release_json") != "$EXPECTED_RELEASE_ID" ]] || + [[ $(jq -r '.tag_name' <<<"$release_json") != "$VERSION" ]] || + [[ $(jq -r '.draft' <<<"$release_json") != false ]] || + [[ $(jq -r '.published_at' <<<"$release_json") == null ]] || + [[ $(jq -r '.prerelease' <<<"$release_json") != false ]]; then + echo "$VERSION is no longer the same published stable release" >&2 + exit 1 + fi + + - name: Publish latest from the immutable version digest + run: | + set -euo pipefail + image="$DOCKER_IMAGE" + version_manifest="$RUNNER_TEMP/version-manifest.json" + latest_manifest="$RUNNER_TEMP/latest-manifest.json" + + if ! printf '%s\n' "$VERSION_DIGEST" | grep -Eq '^sha256:[0-9a-f]{64}$'; then + echo "invalid immutable version manifest digest" >&2 + exit 1 + fi + docker buildx imagetools inspect --raw "$image@$VERSION_DIGEST" >"$version_manifest" + docker buildx imagetools create \ + --tag "$image:latest" \ + "$image@$VERSION_DIGEST" + docker buildx imagetools inspect --raw "$image:latest" >"$latest_manifest" + + jq -S '.manifests | sort_by(.digest)' "$version_manifest" \ + >"$RUNNER_TEMP/version-descriptors.json" + jq -S '.manifests | sort_by(.digest)' "$latest_manifest" \ + >"$RUNNER_TEMP/latest-descriptors.json" + if ! cmp -s \ + "$RUNNER_TEMP/version-descriptors.json" \ + "$RUNNER_TEMP/latest-descriptors.json"; then + diff -u \ + "$RUNNER_TEMP/version-descriptors.json" \ + "$RUNNER_TEMP/latest-descriptors.json" || true + echo "latest does not match the verified version manifest" >&2 + exit 1 + fi + registry_token=$(curl -fsSL \ + 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:terrastruct/d2:pull' | \ + jq -er '.token') + if ! status=$(curl -sS -o /dev/null -D "$RUNNER_TEMP/latest-headers" \ + -w '%{http_code}' --head \ + -H "Authorization: Bearer $registry_token" \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \ + 'https://registry-1.docker.io/v2/terrastruct/d2/manifests/latest'); then + echo "Docker Hub latest-manifest verification request failed" >&2 + exit 1 + fi + if [[ "$status" != 200 ]]; then + echo "Docker Hub latest-manifest verification returned HTTP $status" >&2 + exit 1 + fi + latest_digest=$(tr -d '\r' <"$RUNNER_TEMP/latest-headers" | \ + awk 'tolower($1) == "docker-content-digest:" {print $2}' | tail -n 1) + if [[ "$latest_digest" != "$VERSION_DIGEST" ]]; then + echo "latest does not resolve to the immutable version manifest digest" >&2 + exit 1 + fi + docker buildx imagetools inspect "$image:latest" + + - name: Record latest promotion + run: | + echo "Updated and verified \`$DOCKER_IMAGE:latest\` from \`$DOCKER_IMAGE@$VERSION_DIGEST\`." \ + >>"$GITHUB_STEP_SUMMARY" + + - name: Clean up Docker credentials + if: always() + run: docker logout >/dev/null 2>&1 || true diff --git a/ci/release/README.md b/ci/release/README.md index 9a10115110..79c5638201 100644 --- a/ci/release/README.md +++ b/ci/release/README.md @@ -25,10 +25,53 @@ it depends on from ../sub/lib. Use `--host-only` to build only the release for the host's `$OS-$ARCH` pair. -### build_docker.sh - -Helper script called by build.sh to build D2 on each linux runner inside Docker. -The Dockerfile is in ./linux/Dockerfile +### Docker image helper + +`./docker/build.sh` is retained as a load-only local development helper. It rejects +`--push`, `--latest`, and inherited `RELEASE` publishing. The release script no longer +calls it or publishes Docker tags from a legacy AWS SSH builder. + +### Production Docker publishing + +The manually dispatched `Publish Docker release` GitHub workflow is the production path +for Docker Hub. It replaces only the Docker portion of the legacy AWS release path; the +other release asset builders are unchanged. + +Run it from the protected `master` branch after the GitHub release is published. Enter the +exact v-prefixed release version and leave `publish_latest` off unless this stable release +should become the default Docker image. The workflow rejects `publish_latest` for a GitHub +prerelease or a semver prerelease. + +The `docker-release` GitHub environment must provide a `DOCKERHUB_USERNAME` variable and a +`DOCKERHUB_TOKEN` secret with write access to that user's `d2` repository. The current +production namespace is fixed to `terrastruct/d2`, so `DOCKERHUB_USERNAME` must be +`terrastruct`. The environment's deployment branch policy must allow only protected +branches. Docker Hub tag immutability must remain enabled for version tags with +`^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+([.-][0-9A-Za-z]+)*)?$`; `latest` remains outside +that rule and mutable. This repository-side rule is the final overwrite guard. + +Before publishing a production tag, the workflow: + +1. verifies the protected-master dispatch, published non-draft semver GitHub release, exact + Linux amd64 and arm64 asset IDs, and their GitHub SHA-256 digests; +2. explicitly peels the Git tag to a commit, requires that commit to be an ancestor of the + dispatched `master` commit, and uses that release commit's Dockerfile; +3. builds on native GitHub-hosted amd64 and arm64 runners and pushes only untagged digests + with provenance; +4. runs native version, SVG, and PNG smoke tests for both digests; and +5. dry-runs and validates the exact two-platform manifest, including provenance + attestations; then +6. immediately re-fetches the release and re-peels its tag, requiring the release ID, + publication/prerelease state, tag commit, asset IDs, and asset digests to be unchanged. + +Only then does it create the requested version tag. Existing version tags are immutable, +so the workflow refuses to overwrite one. If `publish_latest` was explicitly selected, it +updates `latest` from the immutable version-manifest digest only after the version manifest +is published and verified. If verification or `latest` promotion fails after the version +tag was created, use GitHub's **Re-run failed jobs** action: the same run accepts the +existing version tag only when its descriptors exactly match that run's verified candidate, +and the separate `latest` job can retry without touching the version tag. A new dispatch +still refuses any existing version tag during preflight. ### Docker continuity test @@ -39,11 +82,9 @@ runners, verifies the images, and publishes only `terrastruct/d2:continuity-test-`. It never updates a release version tag or `latest`. -Before dispatching it, configure the `docker-release` GitHub environment with a -`DOCKERHUB_USERNAME` variable and a `DOCKERHUB_TOKEN` secret that can write to that user's -`d2` repository. Dispatch the workflow from the protected `master` branch. The version input -must name a published, non-draft GitHub release with both Linux archives; `v0.7.1` is the -default continuity fixture. Delete the continuity-test tag in Docker Hub after reviewing the +Dispatch the workflow from the protected `master` branch. The version input must name a +published, non-draft GitHub release with both Linux archives; `v0.7.1` is the default +continuity fixture. Delete the continuity-test tag in Docker Hub after reviewing the workflow summary and manifest. The `v0.7.1` fixture embeds playwright-go v0.4702.0, whose original driver CDN no longer @@ -53,12 +94,12 @@ tarball and the image's Node runtime. Browser payloads use Playwright's current The published D2 archive remains unchanged. Other release versions use their tagged Dockerfile without this compatibility step. -This test does not disable or replace the existing release script's Docker publishing path. +This remains a non-production regression test. The separate `Publish Docker release` +workflow is the production publisher. ### _build.sh -Called by build.sh (with --local or macOS) or build_docker.sh (on linux) to create the -release archive. +Called by build.sh to create a release archive. Do not invoke directly. If you want to produce a build for a single platform run build.sh as so: diff --git a/ci/release/aws/ensure.sh b/ci/release/aws/ensure.sh index 7946759989..7579ad3270 100755 --- a/ci/release/aws/ensure.sh +++ b/ci/release/aws/ensure.sh @@ -314,45 +314,7 @@ export DEBIAN_FRONTEND=noninteractive sudo -E apt-get update -y sudo -E apt-get dist-upgrade -y sudo -E apt-get update -y -sudo -E apt-get install -y build-essential rsync - -# Docker from https://docs.docker.com/engine/install/ubuntu/ -sudo -E apt-get -y install \ - ca-certificates \ - curl \ - gnupg \ - lsb-release -sudo mkdir -p /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --yes --dearmor -o /etc/apt/keyrings/docker.gpg -echo \ - "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - \$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null -sudo -E apt-get update -y -sudo -E apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin -sudo groupadd docker || true -sudo usermod -aG docker \$USER - -printf %s '$CI_DOCKER_TOKEN' | docker login -u terrastruct --password-stdin - -# For building images cross platform from the arm64 instance. -# We could use QEMU with: -# sudo -E apt-get install -y qemu qemu-user-static -# But we don't as playwright dependencies do not install on QEMU on either arm64 or amd64. -if [ "\$(uname -m)" = aarch64 ]; then - if [ "\$(stat -c '%a' ~/.ssh/id_ed25519 2>/dev/null)" != 600 ]; then - echo '$CI_TSTRUCT_ID_ED25519' >~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 - fi - if ! docker context ls | grep -qF ci-d2-linux-amd64; then - docker context create ci-d2-linux-amd64 --docker "host=ssh://$CI_D2_LINUX_AMD64" - fi - if ! docker buildx ls | grep -qF 'd2 *'; then - docker buildx create --use --name d2 --platform linux/arm64 default - fi - if ! docker buildx inspect d2 | grep -qF ci-d2-linux-amd64; then - docker buildx create --append --name d2 --platform linux/amd64 ci-d2-linux-amd64 - fi -fi +sudo -E apt-get install -y build-essential ca-certificates curl rsync mkdir -p \$HOME/.local/bin mkdir -p \$HOME/.local/share/man diff --git a/ci/release/build.sh b/ci/release/build.sh index 07a0ff0124..db21a186f6 100755 --- a/ci/release/build.sh +++ b/ci/release/build.sh @@ -44,15 +44,6 @@ Flags: --uninstall Ensure a release using --host-only and uninstall it. ---push-docker - Push the built docker image. Unfortunately dockerx requires the multi-arch images be - pushed if required in the same invocation as build. dockerx cannot load multi-arch - images into the daemon for push later. It's not slow though to use --push-docker after - building the image as nearly all artifacts are cached. - Automatically set if called from release.sh - ---latest-docker - Mark the built image with the latest tag. Automatically set if called from release.sh EOF } @@ -97,14 +88,6 @@ main() { UNINSTALL=1 HOST_ONLY=1 ;; - push-docker) - flag_noarg && shift "$FLAGSHIFT" - PUSH_DOCKER=1 - ;; - latest-docker) - flag_noarg && shift "$FLAGSHIFT" - LATEST_DOCKER=1 - ;; *) flag_errusage "unrecognized flag $FLAGRAW" ;; @@ -141,7 +124,6 @@ main() { runjob windows/arm64 'OS=windows ARCH=arm64 build' & waitjobs - runjob linux/docker build_docker & runjob windows/amd64/msi 'OS=windows ARCH=amd64 build_windows_msi' & waitjobs } @@ -169,29 +151,6 @@ build_local() { sh_c ./ci/release/_build.sh } -build_docker() { - if [ -n "${LOCAL-}" ]; then - sh_c ./ci/release/docker/build.sh \ - --version="$VERSION" \ - ${PUSH_DOCKER:+--push} \ - ${LATEST_DOCKER:+--latest} - return 0 - fi - - sh_c lockfile_ssh "$CI_D2_LINUX_ARM64" .d2-build-lock - sh_c gitsync "$CI_D2_LINUX_ARM64" src/d2 - sh_c rsync --archive --human-readable \ - "$BUILD_DIR/d2-$VERSION"-linux-*.tar.gz \ - "$CI_D2_LINUX_ARM64:src/d2/$BUILD_DIR/" - sh_c ssh "$CI_D2_LINUX_ARM64" \ - "D2_DOCKER_IMAGE=${D2_DOCKER_IMAGE-}" \ - "RELEASE=${RELEASE-}" \ - ./src/d2/ci/release/docker/build.sh \ - --version="$VERSION" \ - ${PUSH_DOCKER:+--push} \ - ${LATEST_DOCKER:+--latest} -} - build_windows_msi() { REMOTE_HOST=$CI_D2_WINDOWS_AMD64 diff --git a/ci/release/docker/build.sh b/ci/release/docker/build.sh index 70db313dcf..1a0dc3d984 100755 --- a/ci/release/docker/build.sh +++ b/ci/release/docker/build.sh @@ -6,7 +6,10 @@ cd -- "$(dirname "$0")/../../.." help() { cat <&2 + return 2 ;; latest) flag_noarg && shift "$FLAGSHIFT" - LATEST=1 + echo "--latest is disabled; use the Publish Docker release GitHub workflow" >&2 + return 2 ;; version) flag_reqarg && shift "$FLAGSHIFT" @@ -36,6 +41,11 @@ main() { done shift "$FLAGSHIFT" + if [ -n "${RELEASE-}" ]; then + echo "RELEASE-driven Docker publishing is disabled; use the Publish Docker release GitHub workflow" >&2 + return 2 + fi + if [ -z "${VERSION-}" ]; then VERSION=$(readlink ./ci/release/build/latest) fi @@ -49,14 +59,7 @@ main() { ./ci/release/docker/entrypoint.sh \ "./ci/release/build/$VERSION/docker/entrypoint.sh" - flags='--load' - if [ -n "${PUSH-}" -o -n "${RELEASE-}" ]; then - flags='--push --platform linux/amd64,linux/arm64' - fi - if [ -n "${LATEST-}" -o -n "${RELEASE-}" ]; then - flags="$flags -t $D2_DOCKER_IMAGE:latest" - fi - sh_c docker buildx build $flags \ + sh_c docker buildx build --load \ -t "$D2_DOCKER_IMAGE:$VERSION" \ -f ./ci/release/docker/Dockerfile "./ci/release/build/$VERSION/docker" }