Mirror history: skip re-synchronizing already-mirrored (and promoted) digests

.github/actions/mirror-history/action.yml

@@ -0,0 +1,198 @@
+# Composite action: mirror-history
+#
+# Maintain a durable, deletion-surviving record of the source digests a mirror
+# has already synchronized into a quarantine repository. The record is a small
+# OCI artifact stored under the reserved tag `<dest-image>:mirror-history` (an
+# append-only JSON log keyed by source tag + source digest). Because it is a
+# separate tag it survives image-tag deletion during promotion, so a digest that
+# was mirrored once is not re-synchronized after it has been promoted out of and
+# deleted from quarantine.
+#
+# Two operations:
+#   check  - output already-synchronized=true|false for a (source-tag,
+#            source-digest) pair.
+#   record - append an entry for the just-mirrored image and push the updated
+#            history artifact (created on first use).
+#
+# The action assumes oras is already authenticated to the destination registry
+# (the mirror workflow logs in to GHCR with oras).
+#
+# Terminology and the action catalogue: see docs/reference/workflow-actions.md.
+name: mirror-history
+description: Record and query the digests a mirror has already synchronized (OCI artifact per repo).
+
+inputs:
+  operation:
+    description: "Operation to perform: 'check' or 'record'."
+    required: true
+  dest-image:
+    description: "Fully qualified destination image without tag (e.g. ghcr.io/owner/quarantine/python)."
+    required: true
+  source-tag:
+    description: "Source image tag being synchronized (e.g. 3.14-slim)."
+    required: true
+  source-digest:
+    description: "Source manifest digest being synchronized (sha256:...)."
+    required: true
+  source-image:
+    description: "Fully qualified source image without tag (e.g. docker.io/library/python). Required for 'record'."
+    required: false
+    default: ""
+  dest-tag:
+    description: "Destination image tag written in quarantine. Required for 'record'."
+    required: false
+    default: ""
+  force:
+    description: "'true' when the sync was a force run (recorded on the entry)."
+    required: false
+    default: "false"
+
+outputs:
+  already-synchronized:
+    description: "'true' when the (source-tag, source-digest) pair is already in the history ('check' only; empty for 'record')."
+    value: ${{ steps.history.outputs.already-synchronized }}
+
+runs:
+  using: composite
+  steps:
+    - name: mirror-history ${{ inputs.operation }} for ${{ inputs.dest-image }}
+      id: history
+      shell: bash
+      env:
+        OPERATION: "${{ inputs.operation }}"
+        DEST_IMAGE: "${{ inputs.dest-image }}"
+        SOURCE_IMAGE: "${{ inputs.source-image }}"
+        SOURCE_TAG: "${{ inputs.source-tag }}"
+        SOURCE_DIGEST: "${{ inputs.source-digest }}"
+        DEST_TAG: "${{ inputs.dest-tag }}"
+        FORCE: "${{ inputs.force }}"
+        RUN_ID: "${{ github.run_id }}"
+        RUN_ATTEMPT: "${{ github.run_attempt }}"
+        SERVER_URL: "${{ github.server_url }}"
+        REPOSITORY: "${{ github.repository }}"
+      run: |
+        set -euo pipefail
+
+        HISTORY_TAG="mirror-history"
+        MEDIA_TYPE="application/vnd.cssc.mirror-history.v1+json"
+        HISTORY_REF="${DEST_IMAGE}:${HISTORY_TAG}"
+
+        case "${OPERATION}" in
+          check|record) : ;;
+          *)
+            echo "::error::mirror-history: unknown operation '${OPERATION}' (expected 'check' or 'record')."
+            exit 1
+            ;;
+        esac
+
+        # Normalize the force flag to a JSON boolean.
+        if [ "${FORCE}" = "true" ]; then force_json="true"; else force_json="false"; fi
+
+        work="$(mktemp -d)"
+        history_file="${work}/mirror-history.json"
+
+        # Fetch the existing history artifact into ${history_file}. A missing tag
+        # is not an error (the repo has no history yet) but any other failure is,
+        # so a transient registry problem is never silently treated as "empty".
+        load_history() {
+          local err
+          if err="$(oras manifest fetch "${HISTORY_REF}" 2>&1 >/dev/null)"; then
+            oras pull "${HISTORY_REF}" -o "${work}" >/dev/null
+            if [ ! -f "${history_file}" ]; then
+              echo "::error::mirror-history: ${HISTORY_REF} exists but did not yield mirror-history.json."
+              return 1
+            fi
+            return 0
+          fi
+          if printf '%s' "${err}" | grep -qiE "not found|manifest unknown|name unknown|404"; then
+            echo "No existing history at ${HISTORY_REF}; starting a new log."
+            jq -n \
+              --arg image "${DEST_IMAGE}" \
+              --arg source "${SOURCE_IMAGE}" \
+              '{schemaVersion: 1, image: $image, source: $source, entries: []}' \
+              > "${history_file}"
+            return 0
+          fi
+          echo "::error::mirror-history: failed to read ${HISTORY_REF}."
+          printf '%s\n' "${err}"
+          return 1
+        }
+
+        load_history
+
+        if [ "${OPERATION}" = "check" ]; then
+          match="$(jq -r \
+            --arg t "${SOURCE_TAG}" \
+            --arg d "${SOURCE_DIGEST}" \
+            '[.entries[]? | select(.sourceTag == $t and .sourceDigest == $d)] | length' \
+            "${history_file}")"
+          if [ "${match}" -gt 0 ]; then
+            echo "Digest ${SOURCE_DIGEST} for tag ${SOURCE_TAG} is already synchronized."
+            echo "already-synchronized=true" >> "${GITHUB_OUTPUT}"
+          else
+            echo "Digest ${SOURCE_DIGEST} for tag ${SOURCE_TAG} has not been synchronized yet."
+            echo "already-synchronized=false" >> "${GITHUB_OUTPUT}"
+          fi
+          exit 0
+        fi
+
+        # operation == record
+        if [ -z "${DEST_TAG}" ]; then
+          echo "::error::mirror-history: 'record' requires dest-tag."
+          exit 1
+        fi
+        if [ -z "${SOURCE_IMAGE}" ]; then
+          echo "::error::mirror-history: 'record' requires source-image."
+          exit 1
+        fi
+
+        synced_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+        run_url="${SERVER_URL}/${REPOSITORY}/actions/runs/${RUN_ID}"
+
+        # Append a new entry (append-only: a force re-sync of an already-recorded
+        # digest still adds an entry, keeping the log a complete audit trail).
+        # Also refresh image/source in case the log predates this action.
+        updated_file="${work}/mirror-history.next.json"
+        jq \
+          --arg image "${DEST_IMAGE}" \
+          --arg source "${SOURCE_IMAGE}" \
+          --arg t "${SOURCE_TAG}" \
+          --arg d "${SOURCE_DIGEST}" \
+          --arg dt "${DEST_TAG}" \
+          --arg at "${synced_at}" \
+          --arg url "${run_url}" \
+          --arg rid "${RUN_ID}" \
+          --arg ra "${RUN_ATTEMPT}" \
+          --argjson force "${force_json}" \
+          '.schemaVersion = 1
+           | .image = $image
+           | .source = $source
+           | .entries = ((.entries // []) + [{
+               sourceTag: $t,
+               sourceDigest: $d,
+               destTag: $dt,
+               syncedAt: $at,
+               runUrl: $url,
+               runId: $rid,
+               runAttempt: $ra,
+               force: $force
+             }])' \
+          "${history_file}" > "${updated_file}"
+        mv "${updated_file}" "${history_file}"
+
+        count="$(jq -r '.entries | length' "${history_file}")"
+        echo "Recorded ${SOURCE_TAG}@${SOURCE_DIGEST}; history now has ${count} entr$( [ "${count}" -eq 1 ] && echo y || echo ies )."
+
+        # Push the updated artifact under the reserved tag. --artifact-type sets
+        # manifest.artifactType and uses the standard empty config; the JSON log
+        # is the single layer. cd into the work dir so the layer's title
+        # annotation is the bare filename (restores as mirror-history.json).
+        ( cd "${work}" && oras push "${HISTORY_REF}" \
+            --artifact-type "${MEDIA_TYPE}" \
+            --annotation "org.opencontainers.image.title=mirror-history.json" \
+            --annotation "com.toddysm.mirror-history.count=${count}" \
+            --annotation "com.toddysm.mirror-history.updated=${synced_at}" \
+            "mirror-history.json:${MEDIA_TYPE}" >/dev/null )
+        echo "Pushed updated history to ${HISTORY_REF}."
+
+        echo "already-synchronized=" >> "${GITHUB_OUTPUT}"

.github/workflows/_mirror-image.yml

@@ -52,6 +52,11 @@ on:
         required: false
         default: true
         type: boolean
+      record_history:
+        description: "Maintain a mirror-history artifact (dest-image:mirror-history) and skip re-synchronizing a source digest already recorded for the tag (unless force). Records every synchronized digest."
+        required: false
+        default: true
+        type: boolean
     secrets:
       source_registry_username:
         description: "Username for source_login_registry. Required only when source_login_registry is set."
@@ -74,9 +79,10 @@ jobs:
         uses: imjasonh/setup-crane@31b88efe9de28ae0ffa220711af4b60be9435f6e # v0.4
 
       - name: Set up oras
-        # Needed when copying referrers or when attaching the acquisition
-        # provenance referrer; the plain crane-only path stays lightweight.
-        if: ${{ inputs.copy_referrers || inputs.record_acquisition_provenance }}
+        # Needed when copying referrers, attaching the acquisition provenance
+        # referrer, or maintaining the mirror-history artifact; the plain
+        # crane-only path stays lightweight.
+        if: ${{ inputs.copy_referrers || inputs.record_acquisition_provenance || inputs.record_history }}
         uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
         with:
           version: 1.3.1
@@ -99,10 +105,40 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ github.token }}
-          with-oras: ${{ inputs.copy_referrers || inputs.record_acquisition_provenance }}
+          with-oras: ${{ inputs.copy_referrers || inputs.record_acquisition_provenance || inputs.record_history }}
+
+      - name: Resolve source digest
+        # Resolve the source digest up front so the history can be consulted
+        # before any copy. crane is already authenticated to the source.
+        id: source
+        if: ${{ inputs.record_history }}
+        env:
+          SOURCE_REF: "${{ inputs.source_image }}:${{ inputs.source_tag }}"
+        shell: bash
+        run: |
+          set -euo pipefail
+          digest="$(crane digest "${SOURCE_REF}")"
+          echo "Source digest: ${digest}"
+          echo "digest=${digest}" >> "${GITHUB_OUTPUT}"
+
+      - name: Check mirror history
+        id: check
+        if: ${{ inputs.record_history }}
+        uses: ./.github/actions/mirror-history
+        with:
+          operation: check
+          dest-image: ${{ inputs.dest_image }}
+          source-image: ${{ inputs.source_image }}
+          source-tag: ${{ inputs.source_tag }}
+          source-digest: ${{ steps.source.outputs.digest }}
 
       - name: Mirror image
         id: mirror
+        # Skip the copy when this source digest for this tag has already been
+        # synchronized once (even if it was later promoted and deleted from
+        # quarantine), unless force is set. With history disabled the mirror
+        # always runs, preserving the original behaviour.
+        if: ${{ !inputs.record_history || steps.check.outputs.already-synchronized != 'true' || inputs.force }}
         uses: ./.github/actions/mirror-image
         with:
           source-image: ${{ inputs.source_image }}
@@ -132,6 +168,23 @@ jobs:
           copy-referrers: ${{ inputs.copy_referrers }}
           source-authenticated: ${{ inputs.source_login_registry != '' }}
 
+      - name: Record mirror history
+        # Record whenever the mirror actually ran (copied, or found the
+        # destination already up to date) — a run skipped by the history check
+        # leaves the mirror outputs empty. force runs append too, keeping the
+        # log a complete audit trail.
+        id: record
+        if: ${{ inputs.record_history && steps.mirror.outputs.digest != '' }}
+        uses: ./.github/actions/mirror-history
+        with:
+          operation: record
+          dest-image: ${{ inputs.dest_image }}
+          source-image: ${{ inputs.source_image }}
+          source-tag: ${{ inputs.source_tag }}
+          source-digest: ${{ steps.source.outputs.digest }}
+          dest-tag: ${{ inputs.dest_tag }}
+          force: ${{ inputs.force }}
+
       - name: Write job summary
         env:
           SOURCE_REF: "${{ inputs.source_image }}:${{ inputs.source_tag }}"
@@ -140,9 +193,20 @@ jobs:
           DIGEST: "${{ steps.mirror.outputs.digest }}"
           PREVIOUS_DIGEST: "${{ steps.mirror.outputs.previous-digest }}"
           REFERRERS_NOTE: "${{ steps.mirror.outputs.referrers-note }}"
+          SKIPPED: "${{ inputs.record_history && steps.mirror.outputs.digest == '' }}"
+          SOURCE_DIGEST: "${{ steps.source.outputs.digest }}"
         run: |
           set -euo pipefail
-          if [ "${COPIED}" = "true" ]; then
+          if [ "${SKIPPED}" = "true" ]; then
+            {
+              echo "### Mirror image: skipped, already synchronized :fast_forward:"
+              echo ""
+              echo "- **Source:** \`${SOURCE_REF}\`"
+              echo "- **Destination:** \`${DEST_REF}\`"
+              echo "- **Digest:** \`${SOURCE_DIGEST}\`"
+              echo "- This source digest was mirrored before; not re-synchronized."
+            } >> "${GITHUB_STEP_SUMMARY}"
+          elif [ "${COPIED}" = "true" ]; then
             {
               echo "### Mirror image: copied :inbox_tray:"
               echo ""

apps/python-app/libs/cssc_common/cssc_common/__init__.py

@@ -12,15 +12,19 @@
 from .cache import TTLCache
 from .config import GitHubSettings, github_settings
 from .github import GitHubClient
-from .models import Cve, MirroredImage, PromotionIssue, Tag
+from .models import Cve, MirroredImage, MirrorHistoryEntry, PromotionIssue, Tag
+from .registry import MIRROR_HISTORY_TAG, OciRegistryClient
 
 __all__ = [
     "TTLCache",
     "GitHubSettings",
     "github_settings",
     "GitHubClient",
+    "OciRegistryClient",
+    "MIRROR_HISTORY_TAG",
     "Cve",
     "MirroredImage",
+    "MirrorHistoryEntry",
     "PromotionIssue",
     "Tag",
 ]

apps/python-app/libs/cssc_common/cssc_common/config.py

@@ -16,6 +16,7 @@ class GitHubSettings:
     cache_ttl: int
     token: str | None = None
     owner_type: str = "user"
+    username: str | None = None
 
 
 def github_settings() -> GitHubSettings:
@@ -27,6 +28,9 @@ def github_settings() -> GitHubSettings:
     * ``GITHUB_OWNER`` / ``GITHUB_REPO`` — repository coordinates.
     * ``GITHUB_OWNER_TYPE`` — ``user`` (default) or ``org``; selects the
       ``/users/{owner}`` vs ``/orgs/{owner}`` Packages API root.
+    * ``GITHUB_USERNAME`` — the authenticating user's login, used as the
+      registry token-exchange Basic-auth username (defaults to ``GITHUB_OWNER``,
+      which is only correct when the owner is a user account).
     * ``GITHUB_API_URL`` — defaults to ``https://api.github.com``.
     * ``CACHE_TTL_SECONDS`` — response cache TTL (defaults to ``60``).
     """
@@ -38,4 +42,5 @@ def github_settings() -> GitHubSettings:
         cache_ttl=int(os.environ.get("CACHE_TTL_SECONDS", "60")),
         token=os.environ.get("GITHUB_TOKEN") or None,
         owner_type=os.environ.get("GITHUB_OWNER_TYPE", "user"),
+        username=os.environ.get("GITHUB_USERNAME") or None,
     )

apps/python-app/libs/cssc_common/cssc_common/models.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 
 class Tag(BaseModel):
@@ -41,3 +41,22 @@ class Cve(BaseModel):
 
     id: str
     url: str
+
+
+class MirrorHistoryEntry(BaseModel):
+    """One recorded synchronization from the mirror-history artifact.
+
+    Accepts the camelCase keys used in the on-registry JSON and exposes
+    snake_case attributes to the services.
+    """
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    source_tag: str | None = Field(default=None, alias="sourceTag")
+    source_digest: str | None = Field(default=None, alias="sourceDigest")
+    dest_tag: str | None = Field(default=None, alias="destTag")
+    synced_at: str | None = Field(default=None, alias="syncedAt")
+    run_url: str | None = Field(default=None, alias="runUrl")
+    run_id: str | None = Field(default=None, alias="runId")
+    run_attempt: str | None = Field(default=None, alias="runAttempt")
+    force: bool = False