diff --git a/.github/actions/mirror-history/action.yml b/.github/actions/mirror-history/action.yml new file mode 100644 index 0000000..438a836 --- /dev/null +++ b/.github/actions/mirror-history/action.yml @@ -0,0 +1,198 @@ +# Composite action: mirror-history +# +# Maintain a durable, deletion-surviving record of the source digests a mirror +# has already synchronized into a quarantine repository. The record is a small +# OCI artifact stored under the reserved tag `:mirror-history` (an +# append-only JSON log keyed by source tag + source digest). Because it is a +# separate tag it survives image-tag deletion during promotion, so a digest that +# was mirrored once is not re-synchronized after it has been promoted out of and +# deleted from quarantine. +# +# Two operations: +# check - output already-synchronized=true|false for a (source-tag, +# source-digest) pair. +# record - append an entry for the just-mirrored image and push the updated +# history artifact (created on first use). +# +# The action assumes oras is already authenticated to the destination registry +# (the mirror workflow logs in to GHCR with oras). +# +# Terminology and the action catalogue: see docs/reference/workflow-actions.md. +name: mirror-history +description: Record and query the digests a mirror has already synchronized (OCI artifact per repo). + +inputs: + operation: + description: "Operation to perform: 'check' or 'record'." + required: true + dest-image: + description: "Fully qualified destination image without tag (e.g. ghcr.io/owner/quarantine/python)." + required: true + source-tag: + description: "Source image tag being synchronized (e.g. 3.14-slim)." + required: true + source-digest: + description: "Source manifest digest being synchronized (sha256:...)." + required: true + source-image: + description: "Fully qualified source image without tag (e.g. docker.io/library/python). Required for 'record'." + required: false + default: "" + dest-tag: + description: "Destination image tag written in quarantine. Required for 'record'." + required: false + default: "" + force: + description: "'true' when the sync was a force run (recorded on the entry)." + required: false + default: "false" + +outputs: + already-synchronized: + description: "'true' when the (source-tag, source-digest) pair is already in the history ('check' only; empty for 'record')." + value: ${{ steps.history.outputs.already-synchronized }} + +runs: + using: composite + steps: + - name: mirror-history ${{ inputs.operation }} for ${{ inputs.dest-image }} + id: history + shell: bash + env: + OPERATION: "${{ inputs.operation }}" + DEST_IMAGE: "${{ inputs.dest-image }}" + SOURCE_IMAGE: "${{ inputs.source-image }}" + SOURCE_TAG: "${{ inputs.source-tag }}" + SOURCE_DIGEST: "${{ inputs.source-digest }}" + DEST_TAG: "${{ inputs.dest-tag }}" + FORCE: "${{ inputs.force }}" + RUN_ID: "${{ github.run_id }}" + RUN_ATTEMPT: "${{ github.run_attempt }}" + SERVER_URL: "${{ github.server_url }}" + REPOSITORY: "${{ github.repository }}" + run: | + set -euo pipefail + + HISTORY_TAG="mirror-history" + MEDIA_TYPE="application/vnd.cssc.mirror-history.v1+json" + HISTORY_REF="${DEST_IMAGE}:${HISTORY_TAG}" + + case "${OPERATION}" in + check|record) : ;; + *) + echo "::error::mirror-history: unknown operation '${OPERATION}' (expected 'check' or 'record')." + exit 1 + ;; + esac + + # Normalize the force flag to a JSON boolean. + if [ "${FORCE}" = "true" ]; then force_json="true"; else force_json="false"; fi + + work="$(mktemp -d)" + history_file="${work}/mirror-history.json" + + # Fetch the existing history artifact into ${history_file}. A missing tag + # is not an error (the repo has no history yet) but any other failure is, + # so a transient registry problem is never silently treated as "empty". + load_history() { + local err + if err="$(oras manifest fetch "${HISTORY_REF}" 2>&1 >/dev/null)"; then + oras pull "${HISTORY_REF}" -o "${work}" >/dev/null + if [ ! -f "${history_file}" ]; then + echo "::error::mirror-history: ${HISTORY_REF} exists but did not yield mirror-history.json." + return 1 + fi + return 0 + fi + if printf '%s' "${err}" | grep -qiE "not found|manifest unknown|name unknown|404"; then + echo "No existing history at ${HISTORY_REF}; starting a new log." + jq -n \ + --arg image "${DEST_IMAGE}" \ + --arg source "${SOURCE_IMAGE}" \ + '{schemaVersion: 1, image: $image, source: $source, entries: []}' \ + > "${history_file}" + return 0 + fi + echo "::error::mirror-history: failed to read ${HISTORY_REF}." + printf '%s\n' "${err}" + return 1 + } + + load_history + + if [ "${OPERATION}" = "check" ]; then + match="$(jq -r \ + --arg t "${SOURCE_TAG}" \ + --arg d "${SOURCE_DIGEST}" \ + '[.entries[]? | select(.sourceTag == $t and .sourceDigest == $d)] | length' \ + "${history_file}")" + if [ "${match}" -gt 0 ]; then + echo "Digest ${SOURCE_DIGEST} for tag ${SOURCE_TAG} is already synchronized." + echo "already-synchronized=true" >> "${GITHUB_OUTPUT}" + else + echo "Digest ${SOURCE_DIGEST} for tag ${SOURCE_TAG} has not been synchronized yet." + echo "already-synchronized=false" >> "${GITHUB_OUTPUT}" + fi + exit 0 + fi + + # operation == record + if [ -z "${DEST_TAG}" ]; then + echo "::error::mirror-history: 'record' requires dest-tag." + exit 1 + fi + if [ -z "${SOURCE_IMAGE}" ]; then + echo "::error::mirror-history: 'record' requires source-image." + exit 1 + fi + + synced_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + run_url="${SERVER_URL}/${REPOSITORY}/actions/runs/${RUN_ID}" + + # Append a new entry (append-only: a force re-sync of an already-recorded + # digest still adds an entry, keeping the log a complete audit trail). + # Also refresh image/source in case the log predates this action. + updated_file="${work}/mirror-history.next.json" + jq \ + --arg image "${DEST_IMAGE}" \ + --arg source "${SOURCE_IMAGE}" \ + --arg t "${SOURCE_TAG}" \ + --arg d "${SOURCE_DIGEST}" \ + --arg dt "${DEST_TAG}" \ + --arg at "${synced_at}" \ + --arg url "${run_url}" \ + --arg rid "${RUN_ID}" \ + --arg ra "${RUN_ATTEMPT}" \ + --argjson force "${force_json}" \ + '.schemaVersion = 1 + | .image = $image + | .source = $source + | .entries = ((.entries // []) + [{ + sourceTag: $t, + sourceDigest: $d, + destTag: $dt, + syncedAt: $at, + runUrl: $url, + runId: $rid, + runAttempt: $ra, + force: $force + }])' \ + "${history_file}" > "${updated_file}" + mv "${updated_file}" "${history_file}" + + count="$(jq -r '.entries | length' "${history_file}")" + echo "Recorded ${SOURCE_TAG}@${SOURCE_DIGEST}; history now has ${count} entr$( [ "${count}" -eq 1 ] && echo y || echo ies )." + + # Push the updated artifact under the reserved tag. --artifact-type sets + # manifest.artifactType and uses the standard empty config; the JSON log + # is the single layer. cd into the work dir so the layer's title + # annotation is the bare filename (restores as mirror-history.json). + ( cd "${work}" && oras push "${HISTORY_REF}" \ + --artifact-type "${MEDIA_TYPE}" \ + --annotation "org.opencontainers.image.title=mirror-history.json" \ + --annotation "com.toddysm.mirror-history.count=${count}" \ + --annotation "com.toddysm.mirror-history.updated=${synced_at}" \ + "mirror-history.json:${MEDIA_TYPE}" >/dev/null ) + echo "Pushed updated history to ${HISTORY_REF}." + + echo "already-synchronized=" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/_mirror-image.yml b/.github/workflows/_mirror-image.yml index 3b7319e..e8f56ff 100644 --- a/.github/workflows/_mirror-image.yml +++ b/.github/workflows/_mirror-image.yml @@ -52,6 +52,11 @@ on: required: false default: true type: boolean + record_history: + description: "Maintain a mirror-history artifact (dest-image:mirror-history) and skip re-synchronizing a source digest already recorded for the tag (unless force). Records every synchronized digest." + required: false + default: true + type: boolean secrets: source_registry_username: description: "Username for source_login_registry. Required only when source_login_registry is set." @@ -74,9 +79,10 @@ jobs: uses: imjasonh/setup-crane@31b88efe9de28ae0ffa220711af4b60be9435f6e # v0.4 - name: Set up oras - # Needed when copying referrers or when attaching the acquisition - # provenance referrer; the plain crane-only path stays lightweight. - if: ${{ inputs.copy_referrers || inputs.record_acquisition_provenance }} + # Needed when copying referrers, attaching the acquisition provenance + # referrer, or maintaining the mirror-history artifact; the plain + # crane-only path stays lightweight. + if: ${{ inputs.copy_referrers || inputs.record_acquisition_provenance || inputs.record_history }} uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0 with: version: 1.3.1 @@ -99,10 +105,40 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ github.token }} - with-oras: ${{ inputs.copy_referrers || inputs.record_acquisition_provenance }} + with-oras: ${{ inputs.copy_referrers || inputs.record_acquisition_provenance || inputs.record_history }} + + - name: Resolve source digest + # Resolve the source digest up front so the history can be consulted + # before any copy. crane is already authenticated to the source. + id: source + if: ${{ inputs.record_history }} + env: + SOURCE_REF: "${{ inputs.source_image }}:${{ inputs.source_tag }}" + shell: bash + run: | + set -euo pipefail + digest="$(crane digest "${SOURCE_REF}")" + echo "Source digest: ${digest}" + echo "digest=${digest}" >> "${GITHUB_OUTPUT}" + + - name: Check mirror history + id: check + if: ${{ inputs.record_history }} + uses: ./.github/actions/mirror-history + with: + operation: check + dest-image: ${{ inputs.dest_image }} + source-image: ${{ inputs.source_image }} + source-tag: ${{ inputs.source_tag }} + source-digest: ${{ steps.source.outputs.digest }} - name: Mirror image id: mirror + # Skip the copy when this source digest for this tag has already been + # synchronized once (even if it was later promoted and deleted from + # quarantine), unless force is set. With history disabled the mirror + # always runs, preserving the original behaviour. + if: ${{ !inputs.record_history || steps.check.outputs.already-synchronized != 'true' || inputs.force }} uses: ./.github/actions/mirror-image with: source-image: ${{ inputs.source_image }} @@ -132,6 +168,23 @@ jobs: copy-referrers: ${{ inputs.copy_referrers }} source-authenticated: ${{ inputs.source_login_registry != '' }} + - name: Record mirror history + # Record whenever the mirror actually ran (copied, or found the + # destination already up to date) — a run skipped by the history check + # leaves the mirror outputs empty. force runs append too, keeping the + # log a complete audit trail. + id: record + if: ${{ inputs.record_history && steps.mirror.outputs.digest != '' }} + uses: ./.github/actions/mirror-history + with: + operation: record + dest-image: ${{ inputs.dest_image }} + source-image: ${{ inputs.source_image }} + source-tag: ${{ inputs.source_tag }} + source-digest: ${{ steps.source.outputs.digest }} + dest-tag: ${{ inputs.dest_tag }} + force: ${{ inputs.force }} + - name: Write job summary env: SOURCE_REF: "${{ inputs.source_image }}:${{ inputs.source_tag }}" @@ -140,9 +193,20 @@ jobs: DIGEST: "${{ steps.mirror.outputs.digest }}" PREVIOUS_DIGEST: "${{ steps.mirror.outputs.previous-digest }}" REFERRERS_NOTE: "${{ steps.mirror.outputs.referrers-note }}" + SKIPPED: "${{ inputs.record_history && steps.mirror.outputs.digest == '' }}" + SOURCE_DIGEST: "${{ steps.source.outputs.digest }}" run: | set -euo pipefail - if [ "${COPIED}" = "true" ]; then + if [ "${SKIPPED}" = "true" ]; then + { + echo "### Mirror image: skipped, already synchronized :fast_forward:" + echo "" + echo "- **Source:** \`${SOURCE_REF}\`" + echo "- **Destination:** \`${DEST_REF}\`" + echo "- **Digest:** \`${SOURCE_DIGEST}\`" + echo "- This source digest was mirrored before; not re-synchronized." + } >> "${GITHUB_STEP_SUMMARY}" + elif [ "${COPIED}" = "true" ]; then { echo "### Mirror image: copied :inbox_tray:" echo "" diff --git a/apps/python-app/libs/cssc_common/cssc_common/__init__.py b/apps/python-app/libs/cssc_common/cssc_common/__init__.py index 56ac2e4..5a5f961 100644 --- a/apps/python-app/libs/cssc_common/cssc_common/__init__.py +++ b/apps/python-app/libs/cssc_common/cssc_common/__init__.py @@ -12,15 +12,19 @@ from .cache import TTLCache from .config import GitHubSettings, github_settings from .github import GitHubClient -from .models import Cve, MirroredImage, PromotionIssue, Tag +from .models import Cve, MirroredImage, MirrorHistoryEntry, PromotionIssue, Tag +from .registry import MIRROR_HISTORY_TAG, OciRegistryClient __all__ = [ "TTLCache", "GitHubSettings", "github_settings", "GitHubClient", + "OciRegistryClient", + "MIRROR_HISTORY_TAG", "Cve", "MirroredImage", + "MirrorHistoryEntry", "PromotionIssue", "Tag", ] diff --git a/apps/python-app/libs/cssc_common/cssc_common/config.py b/apps/python-app/libs/cssc_common/cssc_common/config.py index 57e1a94..0b0f6b1 100644 --- a/apps/python-app/libs/cssc_common/cssc_common/config.py +++ b/apps/python-app/libs/cssc_common/cssc_common/config.py @@ -16,6 +16,7 @@ class GitHubSettings: cache_ttl: int token: str | None = None owner_type: str = "user" + username: str | None = None def github_settings() -> GitHubSettings: @@ -27,6 +28,9 @@ def github_settings() -> GitHubSettings: * ``GITHUB_OWNER`` / ``GITHUB_REPO`` — repository coordinates. * ``GITHUB_OWNER_TYPE`` — ``user`` (default) or ``org``; selects the ``/users/{owner}`` vs ``/orgs/{owner}`` Packages API root. + * ``GITHUB_USERNAME`` — the authenticating user's login, used as the + registry token-exchange Basic-auth username (defaults to ``GITHUB_OWNER``, + which is only correct when the owner is a user account). * ``GITHUB_API_URL`` — defaults to ``https://api.github.com``. * ``CACHE_TTL_SECONDS`` — response cache TTL (defaults to ``60``). """ @@ -38,4 +42,5 @@ def github_settings() -> GitHubSettings: cache_ttl=int(os.environ.get("CACHE_TTL_SECONDS", "60")), token=os.environ.get("GITHUB_TOKEN") or None, owner_type=os.environ.get("GITHUB_OWNER_TYPE", "user"), + username=os.environ.get("GITHUB_USERNAME") or None, ) diff --git a/apps/python-app/libs/cssc_common/cssc_common/models.py b/apps/python-app/libs/cssc_common/cssc_common/models.py index 15d8cdc..9446167 100644 --- a/apps/python-app/libs/cssc_common/cssc_common/models.py +++ b/apps/python-app/libs/cssc_common/cssc_common/models.py @@ -2,7 +2,7 @@ from __future__ import annotations -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class Tag(BaseModel): @@ -41,3 +41,22 @@ class Cve(BaseModel): id: str url: str + + +class MirrorHistoryEntry(BaseModel): + """One recorded synchronization from the mirror-history artifact. + + Accepts the camelCase keys used in the on-registry JSON and exposes + snake_case attributes to the services. + """ + + model_config = ConfigDict(populate_by_name=True) + + source_tag: str | None = Field(default=None, alias="sourceTag") + source_digest: str | None = Field(default=None, alias="sourceDigest") + dest_tag: str | None = Field(default=None, alias="destTag") + synced_at: str | None = Field(default=None, alias="syncedAt") + run_url: str | None = Field(default=None, alias="runUrl") + run_id: str | None = Field(default=None, alias="runId") + run_attempt: str | None = Field(default=None, alias="runAttempt") + force: bool = False diff --git a/apps/python-app/libs/cssc_common/cssc_common/registry.py b/apps/python-app/libs/cssc_common/cssc_common/registry.py new file mode 100644 index 0000000..38a5824 --- /dev/null +++ b/apps/python-app/libs/cssc_common/cssc_common/registry.py @@ -0,0 +1,121 @@ +"""A minimal OCI registry read client for small JSON artifacts. + +The dashboard reads the ``mirror-history`` artifact straight from the registry +(GHCR by default): it resolves a tag to a manifest, then pulls the single JSON +layer blob. This is deliberately tiny — just enough to fetch one JSON artifact — +and takes an injectable ``httpx`` client/transport so it is trivial to unit test +with :class:`httpx.MockTransport`. +""" + +from __future__ import annotations + +import base64 +from typing import Any + +import httpx + +# Reserved tag under which the mirror workflows store the per-repo history +# artifact. It is never a valid upstream image tag to mirror. +MIRROR_HISTORY_TAG = "mirror-history" + +_MANIFEST_ACCEPT = ", ".join( + [ + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + ] +) + + +class OciRegistryClient: + """Fetch a single JSON artifact (manifest + first layer blob) from a registry.""" + + def __init__( + self, + *, + registry: str = "ghcr.io", + owner: str = "", + token: str | None = None, + username: str | None = None, + client: httpx.Client | None = None, + transport: httpx.BaseTransport | None = None, + timeout: float = 10.0, + ) -> None: + self._registry = registry + self._owner = owner + self._token = token + # The token endpoint's Basic-auth username is the authenticating GitHub + # user, which is not necessarily the package owner (the owner may be an + # org). Default to the owner only as a fallback. + self._username = username or owner or "x" + + if client is not None: + self._client = client + self._owns_client = False + else: + self._client = httpx.Client( + base_url=f"https://{registry}", + timeout=timeout, + transport=transport, + ) + self._owns_client = True + + def _repository(self, name: str) -> str: + return f"{self._owner}/{name}" if self._owner else name + + def _bearer(self, repository: str) -> str | None: + """Exchange for a pull token. Anonymous for public repos; Basic-auth + with the configured token otherwise.""" + + headers: dict[str, str] = {} + if self._token: + basic = base64.b64encode( + f"{self._username}:{self._token}".encode() + ).decode() + headers["Authorization"] = f"Basic {basic}" + response = self._client.get( + "/token", + params={ + "service": self._registry, + "scope": f"repository:{repository}:pull", + }, + headers=headers, + ) + response.raise_for_status() + data = response.json() + return data.get("token") or data.get("access_token") + + def fetch_json_artifact( + self, name: str, reference: str + ) -> dict[str, Any] | None: + """Return the parsed JSON of the artifact's first layer blob. + + ``None`` when the reference does not exist (a missing tag is not an + error), so callers can treat it as "no history yet". + """ + + repository = self._repository(name) + bearer = self._bearer(repository) + auth = {"Authorization": f"Bearer {bearer}"} if bearer else {} + + manifest_response = self._client.get( + f"/v2/{repository}/manifests/{reference}", + headers={"Accept": _MANIFEST_ACCEPT, **auth}, + ) + if manifest_response.status_code == 404: + return None + manifest_response.raise_for_status() + + layers = manifest_response.json().get("layers") or [] + if not layers or not layers[0].get("digest"): + return None + + blob_response = self._client.get( + f"/v2/{repository}/blobs/{layers[0]['digest']}", + headers=auth, + ) + blob_response.raise_for_status() + return blob_response.json() + + def close(self) -> None: + if self._owns_client: + self._client.close() diff --git a/apps/python-app/services/dashboard-web/src/dashboard_web/clients.py b/apps/python-app/services/dashboard-web/src/dashboard_web/clients.py index 982982c..f5b310d 100644 --- a/apps/python-app/services/dashboard-web/src/dashboard_web/clients.py +++ b/apps/python-app/services/dashboard-web/src/dashboard_web/clients.py @@ -28,6 +28,16 @@ def get_packages(self, namespace: str) -> list[dict[str, Any]]: response.raise_for_status() return response.json() + def get_tags(self, name: str) -> list[dict[str, Any]]: + response = self._client.get(f"{self._base}/packages/{name}/tags") + response.raise_for_status() + return response.json() + + def get_history(self, name: str) -> list[dict[str, Any]]: + response = self._client.get(f"{self._base}/packages/{name}/history") + response.raise_for_status() + return response.json() + class IssuesServiceClient: def __init__( diff --git a/apps/python-app/services/dashboard-web/src/dashboard_web/stages/acquisition.py b/apps/python-app/services/dashboard-web/src/dashboard_web/stages/acquisition.py index 6f49b57..c346ad2 100644 --- a/apps/python-app/services/dashboard-web/src/dashboard_web/stages/acquisition.py +++ b/apps/python-app/services/dashboard-web/src/dashboard_web/stages/acquisition.py @@ -11,6 +11,11 @@ from ..clients import IssuesServiceClient, PackagesServiceClient from .base import Stage +# Reserved tag under which the mirror workflows store the per-repo history +# artifact. It is not a real image tag, so it is excluded when deciding whether +# a repository still holds a quarantined image. +MIRROR_HISTORY_TAG = "mirror-history" + class AcquisitionProvider: stage = Stage( @@ -68,6 +73,18 @@ def get_data(self) -> dict[str, Any]: for pkg in packages: name = pkg.get("name", "") + # Two per-package reads (tags + history): acceptable at the demo's + # scale of a handful of quarantine repos; tags are served from + # packages-service's cached GitHub responses. + # A repository still holds a quarantined image only if it has a tag + # other than the reserved history tag; a history-only package (its + # image already promoted and deleted) is not "in quarantine". + real_tags = [ + tag.get("tag") + for tag in self._packages.get_tags(name) + if tag.get("tag") and tag.get("tag") != MIRROR_HISTORY_TAG + ] + synchronized = self._packages.get_history(name) issues: list[dict[str, Any]] = [] for issue in all_issues: if self._matches(issue.get("image") or "", name): @@ -79,9 +96,10 @@ def get_data(self) -> dict[str, Any]: "name": name, "visibility": pkg.get("visibility"), "updated_at": pkg.get("updated_at"), - "tag_count": pkg.get("tag_count"), - "in_quarantine": True, + "tag_count": len(real_tags), + "in_quarantine": bool(real_tags), "issues": issues, + "synchronized": synchronized, } ) @@ -107,6 +125,7 @@ def get_data(self) -> dict[str, Any]: "tag_count": None, "in_quarantine": False, "issues": issues, + "synchronized": [], } ) diff --git a/apps/python-app/services/dashboard-web/src/dashboard_web/static/css/styles.css b/apps/python-app/services/dashboard-web/src/dashboard_web/static/css/styles.css index cc1b5ce..b92ea73 100644 --- a/apps/python-app/services/dashboard-web/src/dashboard_web/static/css/styles.css +++ b/apps/python-app/services/dashboard-web/src/dashboard_web/static/css/styles.css @@ -108,6 +108,17 @@ main { border-color: var(--open); } +details.synchronized { + margin-top: 0.75rem; +} + +details.synchronized > summary { + cursor: pointer; + color: var(--muted); + font-size: 0.9rem; + padding: 0.25rem 0; +} + table.issues { width: 100%; border-collapse: collapse; diff --git a/apps/python-app/services/dashboard-web/src/dashboard_web/templates/stages/acquisition.html b/apps/python-app/services/dashboard-web/src/dashboard_web/templates/stages/acquisition.html index 2549cc7..146357b 100644 --- a/apps/python-app/services/dashboard-web/src/dashboard_web/templates/stages/acquisition.html +++ b/apps/python-app/services/dashboard-web/src/dashboard_web/templates/stages/acquisition.html @@ -50,6 +50,31 @@

{% endif %} + {% if image.synchronized %} +
+ Synchronized ({{ image.synchronized|length }}) + + + + + + + + + + + {% for entry in image.synchronized %} + + + + + + + {% endfor %} + +
Source tagDigestSyncedRun
{% if entry.source_tag %}{{ entry.source_tag }}{% else %}—{% endif %}{% if entry.source_digest %}{{ entry.source_digest[:19] }}…{% else %}—{% endif %}{{ entry.synced_at or "—" }}{% if entry.force %} forced{% endif %}{% if entry.run_url %}run{% else %}—{% endif %}
+
+ {% endif %} {% endfor %} {% endif %} diff --git a/apps/python-app/services/dashboard-web/tests/test_acquisition.py b/apps/python-app/services/dashboard-web/tests/test_acquisition.py index 6983e30..f3a0a89 100644 --- a/apps/python-app/services/dashboard-web/tests/test_acquisition.py +++ b/apps/python-app/services/dashboard-web/tests/test_acquisition.py @@ -2,12 +2,20 @@ class FakePackages: - def __init__(self, data): + def __init__(self, data, tags=None, history=None): self._data = data + self._tags = tags or {} + self._history = history or {} def get_packages(self, namespace): return self._data + def get_tags(self, name): + return self._tags.get(name, []) + + def get_history(self, name): + return self._history.get(name, []) + class FakeIssues: def __init__(self, data): @@ -133,13 +141,63 @@ def test_pending_issue_with_package_is_not_duplicated(): "blocking_cves": [], } ] - data = AcquisitionProvider(FakePackages(packages), FakeIssues(issues)).get_data() + data = AcquisitionProvider( + FakePackages(packages, tags={"quarantine/node": [{"tag": "26-alpine"}]}), + FakeIssues(issues), + ).get_data() # Shown under its package card only — no duplicate orphan card. assert len(data["images"]) == 1 assert data["images"][0]["in_quarantine"] is True assert [i["number"] for i in data["images"][0]["issues"]] == [88] +def test_synchronized_history_is_surfaced(): + history = { + "quarantine/python": [ + { + "source_tag": "3.14-slim", + "source_digest": "sha256:aaaa", + "synced_at": "2026-07-30T06:00:00Z", + "run_url": "https://github.com/toddysm/cssc-framework/actions/runs/1", + "force": False, + } + ] + } + provider = AcquisitionProvider( + FakePackages( + PACKAGES, + tags={"quarantine/python": [{"tag": "3.14-slim"}]}, + history=history, + ), + FakeIssues([]), + ) + image = provider.get_data()["images"][0] + assert image["in_quarantine"] is True + assert image["tag_count"] == 1 + assert [e["source_digest"] for e in image["synchronized"]] == ["sha256:aaaa"] + + +def test_history_only_package_is_not_in_quarantine(): + # The image was promoted and deleted; only the reserved history tag remains. + history = { + "quarantine/python": [ + {"source_tag": "3.14-slim", "source_digest": "sha256:aaaa"} + ] + } + provider = AcquisitionProvider( + FakePackages( + PACKAGES, + tags={"quarantine/python": [{"tag": "mirror-history"}]}, + history=history, + ), + FakeIssues([]), + ) + image = provider.get_data()["images"][0] + assert image["in_quarantine"] is False + assert image["tag_count"] == 0 + assert len(image["synchronized"]) == 1 + + def test_cve_url_normalizes_missing_trailing_slash(): provider = AcquisitionProvider( FakePackages(PACKAGES), diff --git a/apps/python-app/services/dashboard-web/tests/test_app.py b/apps/python-app/services/dashboard-web/tests/test_app.py index e230935..6e50c75 100644 --- a/apps/python-app/services/dashboard-web/tests/test_app.py +++ b/apps/python-app/services/dashboard-web/tests/test_app.py @@ -16,6 +16,20 @@ def get_packages(self, namespace): } ] + def get_tags(self, name): + return [{"tag": "3.14-slim", "digest": "sha256:aaaa"}] + + def get_history(self, name): + return [ + { + "source_tag": "3.14-slim", + "source_digest": "sha256:aaaabbbbcccc", + "synced_at": "2026-07-30T06:00:00Z", + "run_url": "https://github.com/toddysm/cssc-framework/actions/runs/9", + "force": False, + } + ] + class FakeIssues: def get_issues(self, image=None, tag=None, state="all"): @@ -65,6 +79,8 @@ def test_fragment_renders_table_with_cve_links(): assert 'target="_blank"' in body assert 'rel="noopener"' in body assert "Open" in body + assert "Synchronized" in body + assert 'href="https://github.com/toddysm/cssc-framework/actions/runs/9"' in body def test_unknown_stage_returns_404(): diff --git a/apps/python-app/services/packages-service/src/packages_service/app.py b/apps/python-app/services/packages-service/src/packages_service/app.py index 1bc428a..a4f0bbe 100644 --- a/apps/python-app/services/packages-service/src/packages_service/app.py +++ b/apps/python-app/services/packages-service/src/packages_service/app.py @@ -4,7 +4,7 @@ from fastapi import FastAPI, Query -from cssc_common import GitHubClient, github_settings +from cssc_common import GitHubClient, OciRegistryClient, github_settings from .client import PackagesClient @@ -19,7 +19,10 @@ def build_client() -> PackagesClient: api_url=settings.api_url, cache_ttl=settings.cache_ttl, ) - return PackagesClient(github) + registry = OciRegistryClient( + owner=settings.owner, token=settings.token, username=settings.username + ) + return PackagesClient(github, registry) def create_app(client: PackagesClient | None = None) -> FastAPI: @@ -53,6 +56,10 @@ def list_packages(namespace: str = Query("quarantine")) -> list[dict]: def list_tags(name: str) -> list[dict]: return [tag.model_dump() for tag in get_client().list_tags(name)] + @app.get("/packages/{name:path}/history") + def get_history(name: str) -> list[dict]: + return [entry.model_dump() for entry in get_client().get_history(name)] + return app diff --git a/apps/python-app/services/packages-service/src/packages_service/client.py b/apps/python-app/services/packages-service/src/packages_service/client.py index 018473e..c5637df 100644 --- a/apps/python-app/services/packages-service/src/packages_service/client.py +++ b/apps/python-app/services/packages-service/src/packages_service/client.py @@ -4,14 +4,26 @@ from urllib.parse import quote -from cssc_common import GitHubClient, MirroredImage, Tag +from cssc_common import ( + MIRROR_HISTORY_TAG, + GitHubClient, + MirroredImage, + MirrorHistoryEntry, + OciRegistryClient, + Tag, +) class PackagesClient: """Read container packages and their tags for the configured owner.""" - def __init__(self, github: GitHubClient) -> None: + def __init__( + self, + github: GitHubClient, + registry: OciRegistryClient | None = None, + ) -> None: self._gh = github + self._registry = registry def _owner_root(self) -> str: """Return the Packages API root for the owner (user vs org). @@ -75,3 +87,19 @@ def list_tags(self, name: str) -> list[Tag]: ) ) return tags + + def get_history(self, name: str) -> list[MirrorHistoryEntry]: + """Return the recorded synchronization history for a repository. + + Reads the ``:mirror-history`` OCI artifact from the registry. + Returns an empty list when no registry client is configured or the + repository has no history yet (the reserved tag is absent). + """ + + if self._registry is None: + return [] + document = self._registry.fetch_json_artifact(name, MIRROR_HISTORY_TAG) + if not document: + return [] + entries = document.get("entries") or [] + return [MirrorHistoryEntry.model_validate(entry) for entry in entries] diff --git a/apps/python-app/services/packages-service/tests/test_history.py b/apps/python-app/services/packages-service/tests/test_history.py new file mode 100644 index 0000000..b6a43f3 --- /dev/null +++ b/apps/python-app/services/packages-service/tests/test_history.py @@ -0,0 +1,114 @@ +import httpx +from fastapi.testclient import TestClient + +from cssc_common import GitHubClient, OciRegistryClient +from packages_service.app import create_app +from packages_service.client import PackagesClient + +HISTORY_DOC = { + "schemaVersion": 1, + "image": "ghcr.io/toddysm/quarantine/python", + "source": "docker.io/library/python", + "entries": [ + { + "sourceTag": "3.14-slim", + "sourceDigest": "sha256:aaaa", + "destTag": "3.14-slim", + "syncedAt": "2026-07-30T06:00:00Z", + "runUrl": "https://github.com/toddysm/cssc-framework/actions/runs/1", + "runId": "1", + "runAttempt": "1", + "force": False, + }, + { + "sourceTag": "3.14-slim", + "sourceDigest": "sha256:bbbb", + "destTag": "3.14-slim", + "syncedAt": "2026-08-13T06:00:00Z", + "runUrl": "https://github.com/toddysm/cssc-framework/actions/runs/2", + "runId": "2", + "runAttempt": "1", + "force": True, + }, + ], +} + +BLOB_DIGEST = "sha256:blob" + + +def _registry_handler(present: bool): + def handler(request: httpx.Request) -> httpx.Response: + path = request.url.path + if path == "/token": + return httpx.Response(200, json={"token": "t"}) + if path.endswith("/manifests/mirror-history"): + if not present: + return httpx.Response(404, json={"errors": [{"code": "MANIFEST_UNKNOWN"}]}) + return httpx.Response( + 200, + json={ + "schemaVersion": 2, + "artifactType": "application/vnd.cssc.mirror-history.v1+json", + "layers": [ + { + "mediaType": "application/vnd.cssc.mirror-history.v1+json", + "digest": BLOB_DIGEST, + } + ], + }, + ) + if path.endswith(f"/blobs/{BLOB_DIGEST}"): + return httpx.Response(200, json=HISTORY_DOC) + return httpx.Response(404, json={"message": "not found"}) + + return handler + + +def _app(present: bool = True): + gh_transport = httpx.MockTransport( + lambda r: httpx.Response(404, json={"message": "not found"}) + ) + github = GitHubClient( + client=httpx.Client(base_url="https://api.github.com", transport=gh_transport), + owner="toddysm", + repo="cssc-framework", + cache_ttl=0, + ) + registry = OciRegistryClient( + owner="toddysm", + token="tok", + client=httpx.Client( + base_url="https://ghcr.io", transport=httpx.MockTransport(_registry_handler(present)) + ), + ) + return create_app(PackagesClient(github, registry)) + + +def test_history_returns_entries(): + client = TestClient(_app(present=True)) + response = client.get("/packages/quarantine/python/history") + assert response.status_code == 200 + body = response.json() + assert [e["source_digest"] for e in body] == ["sha256:aaaa", "sha256:bbbb"] + assert body[0]["source_tag"] == "3.14-slim" + assert body[1]["force"] is True + + +def test_history_empty_when_tag_absent(): + client = TestClient(_app(present=False)) + response = client.get("/packages/quarantine/python/history") + assert response.status_code == 200 + assert response.json() == [] + + +def test_history_empty_without_registry(): + gh_transport = httpx.MockTransport( + lambda r: httpx.Response(404, json={"message": "not found"}) + ) + github = GitHubClient( + client=httpx.Client(base_url="https://api.github.com", transport=gh_transport), + owner="toddysm", + cache_ttl=0, + ) + client = TestClient(create_app(PackagesClient(github))) + assert client.get("/packages/quarantine/python/history").json() == [] diff --git a/docs/architecture/acquire/README.md b/docs/architecture/acquire/README.md index 1161a77..7d6b31f 100644 --- a/docs/architecture/acquire/README.md +++ b/docs/architecture/acquire/README.md @@ -8,6 +8,9 @@ controlled namespace before they are used. - [Image mirror workflows](image-mirror-workflows.md) — how the Docker Hub → GHCR mirroring actions are structured, the tooling they use, and what they do and deliberately do not do. +- [Mirror history](mirror-history.md) — a durable, deletion-surviving record of + the digests the mirror has already synchronized, so a promoted-and-deleted + image is not re-mirrored (implemented, pending end-to-end validation). ## Supply Chain Observability diff --git a/docs/architecture/acquire/image-mirror-workflows.md b/docs/architecture/acquire/image-mirror-workflows.md index 6e07a80..80e974f 100644 --- a/docs/architecture/acquire/image-mirror-workflows.md +++ b/docs/architecture/acquire/image-mirror-workflows.md @@ -55,6 +55,7 @@ It is triggered only through `workflow_call` and exposes these inputs: | `force` | no | `false` | Copy even when the source and destination digests match. | | `source_login_registry` | no | `""` | Registry to authenticate to before pulling the source (e.g. `dhi.io`). Empty means an anonymous public pull. | | `copy_referrers` | no | `false` | Also copy OCI referrer artifacts (SBOMs, provenance, VEX, signatures) attached to the image. Switches the copy to `oras`. Works for any image that has referrers, not just hardened images. | +| `record_history` | no | `true` | Maintain a per-repo `mirror-history` artifact and skip re-synchronizing a source digest already recorded for the tag (unless `force`). | It also accepts two optional secrets, used only for authenticated sources: @@ -161,6 +162,12 @@ Key tooling characteristics: per-platform child manifest. This is what lets downstream SBOM-based scanning read attestations straight from quarantine. It works for any image that has referrers, not just Docker Hardened Images. +- **Mirror history.** Before copying, the source digest is checked against a + durable per-repo [mirror-history artifact](mirror-history.md) + (`quarantine/:mirror-history`); a digest already synchronized for the + tag is skipped (even after it was promoted and deleted from quarantine), + unless `force`. Every synchronized digest is recorded. Controlled by the + `record_history` input (default on). - **Acquisition provenance.** After a successful copy, an [acquisition-provenance referrer](../../reference/acquisition-provenance.md) is attached to the mirrored image (index and each per-platform manifest) recording diff --git a/docs/architecture/acquire/mirror-history.md b/docs/architecture/acquire/mirror-history.md new file mode 100644 index 0000000..6dbc44f --- /dev/null +++ b/docs/architecture/acquire/mirror-history.md @@ -0,0 +1,285 @@ +# Mirror history: skip re-synchronizing already-mirrored digests + +- **Status:** implemented on branch `feature/mirror-history` (pending end-to-end validation) +- **Tracking issue:** [#157](https://github.com/toddysm/cssc-framework/issues/157) +- **Stage:** Acquire + +This document describes the durable **mirror history** that lets the image-mirror workflows +stop re-synchronizing a digest they have already acquired once — even after that +digest has been promoted out of quarantine and deleted. + +## Problem + +The mirror is **stateless**. The [`mirror-image`](../../reference/workflow-actions.md) +action decides whether to copy by comparing two live values: + +1. the **source** manifest digest (`crane digest docker.io/library/python:3.14-slim`), and +2. the **destination** digest (`crane digest ghcr.io//quarantine/python:3.14-slim`). + +It copies when they differ, treating a missing destination as "differs". + +That works while the destination sticks around, but the acquisition pipeline +deliberately removes it: + +1. The mirror copies `docker.io/library/python:3.14-slim` → `quarantine/python:3.14-slim`. +2. A promote-from-quarantine workflow copies it to `golden/python` and then + **deletes the tag from quarantine**. +3. Deleting the *last* tagged version of a GHCR package deletes the whole + `quarantine/python` package (documented GHCR behaviour; see the + [delete-image](../../reference/workflow-actions.md) action, which already has to + work around it). +4. The next scheduled mirror run reads the destination digest, finds **nothing**, + concludes "differs", and **re-synchronizes the exact digest that was already + acquired and promoted**. The image loops back into the pipeline. + +The system has no memory that this digest was already handled. + +## Goal + +Give the mirror a durable, deletion-surviving record of the digests it has +already synchronized for a given source tag, and skip copying a digest that is +already recorded — while preserving every existing behaviour (`force`, +`copy_referrers`, acquisition provenance, multi-arch, concurrency safety). + +Non-goals: pruning quarantine, discovering new tags, scanning, or changing the +promotion workflows. + +## Approach: a `mirror-history` OCI artifact per synchronized repo + +Store the history as a small OCI artifact in **each synchronized repo**, under a +reserved tag: + +``` +ghcr.io//quarantine/:mirror-history +``` + +The artifact carries a single JSON blob — an **append-only log** of every source +digest that has been synchronized, keyed by source tag. + +Why a **separate tag in the same repo** (rather than a referrer or a sibling +repo): + +- **It survives image deletion.** OCI referrers are attached to a subject digest + and are orphaned/removed when that image is deleted; a standalone tag is not. +- **It keeps the package alive.** Because `mirror-history` is always present, the + quarantine package is never reduced to zero tagged versions, so promotion's + tag delete no longer trips the GHCR "cannot delete the last tagged version → + whole package deleted" edge case. The history and the package persist together. +- **It is colocated**, matching the requirement that history live "in each repo + that is synchronized", and it is trivially discoverable (`crane manifest + quarantine/python:mirror-history`). + +### Artifact format + +An OCI **image manifest** (artifact) with one JSON blob layer: + +- **manifest artifactType:** `application/vnd.cssc.mirror-history.v1+json` + (pushed with `oras push --artifact-type`, so the manifest uses the standard + empty config `application/vnd.oci.empty.v1+json`). +- **layer mediaType:** `application/vnd.cssc.mirror-history.v1+json` +- **manifest annotations** (for `oras discover`/`crane manifest` visibility): + - `org.opencontainers.image.title=mirror-history.json` + - `com.toddysm.mirror-history.count=` + - `com.toddysm.mirror-history.updated=` + +The blob is the history document: + +```json +{ + "schemaVersion": 1, + "image": "ghcr.io/toddysm/quarantine/python", + "source": "docker.io/library/python", + "entries": [ + { + "sourceTag": "3.14-slim", + "sourceDigest": "sha256:aaaa1111...", + "destTag": "3.14-slim", + "syncedAt": "2026-07-30T06:00:00Z", + "runUrl": "https://github.com/toddysm/cssc-framework/actions/runs/123", + "runId": "123", + "runAttempt": "1", + "force": false + }, + { + "sourceTag": "3.14-slim", + "sourceDigest": "sha256:bbbb2222...", + "destTag": "3.14-slim", + "syncedAt": "2026-08-13T06:00:00Z", + "runUrl": "https://github.com/toddysm/cssc-framework/actions/runs/456", + "runId": "456", + "runAttempt": "1", + "force": false + }, + { + "sourceTag": "3.13-slim", + "sourceDigest": "sha256:cccc3333...", + "destTag": "3.13-slim", + "syncedAt": "2026-08-13T06:01:00Z", + "runUrl": "https://github.com/toddysm/cssc-framework/actions/runs/457", + "runId": "457", + "runAttempt": "1", + "force": true + } + ] +} +``` + +The first two entries (same `sourceTag`, different `sourceDigest`) are the common +case: upstream re-pointed `3.14-slim` to a new digest, so both were mirrored once +and both are recorded. A different tag (`3.13-slim`) lives in the same log. + +### Field reference + +| Field | Meaning | +| ----- | ------- | +| `schemaVersion` | Integer, currently `1`. Lets the format evolve. | +| `image` | Destination repo the history belongs to. | +| `source` | Upstream source image (without tag). | +| `entries[]` | Append-only list; never pruned (unbounded). Ordered chronologically, oldest first. | +| `entries[].sourceTag` | Source tag that was mirrored. Half of the dedupe key. | +| `entries[].sourceDigest` | Source manifest digest. A match on `(sourceTag, sourceDigest)` means "already synchronized". Because the copy preserves digests, this is also the destination digest. | +| `entries[].destTag` | Tag written in quarantine. | +| `entries[].syncedAt` | RFC 3339 UTC timestamp of the sync. | +| `entries[].runUrl` / `runId` / `runAttempt` | Workflow run that performed the sync (audit trail + the dashboard's run link). | +| `entries[].force` | `true` when this sync was a `force` run (bypassed the history check). | + +**Locked schema decisions:** + +- **No separate `destDigest` field** — the copy preserves digests, so `sourceDigest` + is also the destination digest; storing it twice would be redundant. +- **`force` appends a new entry** even when the same `(sourceTag, sourceDigest)` + is already recorded, so the log stays a complete, ordered audit trail rather + than mutating past entries. +- **Chronological order** — new entries are appended to the end (oldest → newest); + consumers (e.g. the dashboard) sort/reverse as needed. + +- **History key** = `(sourceTag, sourceDigest)`. A digest is "already + synchronized" when an entry exists with the same `sourceTag` **and** + `sourceDigest`. Scoping by source tag means that if two tags happen to point at + the same digest they are tracked independently, and a tag that upstream + re-points to a brand-new digest is (correctly) treated as new work. +- **Append-only, unbounded.** Entries are never removed; the log is the audit + trail of everything the mirror has ever acquired for that repo. + +### Changed mirror control flow + +```mermaid +flowchart TD + A[resolve source digest] --> B[read quarantine/:mirror-history] + B --> C{force?} + C -->|yes| G[copy image] + C -->|no| D{sourceTag+sourceDigest\nin history?} + D -->|yes| E[skip: already synchronized once] + D -->|no| F{destination present\nand digest matches?} + F -->|yes| E2[skip: up to date - record if missing] + F -->|no| G + G --> H[attach acquisition provenance\n] + H --> I[append entry to history + push\nquarantine/:mirror-history] + E2 --> I + E --> J[write job summary] + I --> J +``` + +Key points: + +- The **history check is a new short-circuit** that fires precisely in the case + the current digest compare misses: the source digest is known but the + destination is absent (promoted + deleted). +- The **existing digest short-circuit is kept** for the common "destination still + present and unchanged" case. If that path finds the destination up to date but + the digest is *not yet in the history* (e.g. first run after this feature + ships), it records it so future runs are covered. +- **`force`** skips the history check and copies, then **still records** the + digest (so the history stays a complete record). +- History is **read-modify-write**. The per-image concurrency group already + guarantees runs of the same image never overlap + (`cancel-in-progress: false`), so there is no race on the artifact. +- **`copy_referrers`** today always re-copies (referrers can change + independently of the subject digest). Under this design a **recorded digest + also suppresses the referrer re-copy**: once `(sourceTag, sourceDigest)` is in + the history the mirror skips, even when `copy_referrers` is true. This trades + routine referrer-freshness re-syncs for not re-pulling a promoted-and-deleted + image; a `force` run is the escape hatch when referrers must be refreshed + (resolved O2). + +### Where the logic lives + +- New composite action **`mirror-history`** under `.github/actions/mirror-history/` + with two operations: + - `check` — given repo + source tag + source digest, output + `already-synchronized=true|false`. + - `record` — append an entry and push the updated `:mirror-history` artifact + (create it on first use). + Uses `oras`/`crane` already available on the runner. +- `_mirror-image.yml` wires it in: a **check** step before `mirror-image` (gates + the copy) and a **record** step after a successful copy. The `mirror-image` + action itself is unchanged; gating happens at the workflow level via an `if:` + on the mirror step (or a new `skip` input), keeping the action single-purpose. + +## Interaction with existing behaviour + +| Concern | Behaviour | +| ------- | --------- | +| Acquisition provenance | Unchanged. Still attached only when a copy happened and the digest changed. A history-skip means no copy, so no new provenance — correct. | +| `force` | Bypasses history check, copies, records. | +| `copy_referrers` | A recorded digest suppresses the re-copy too (skips); an unrecorded digest copies with `oras` and records. `force` refreshes referrers on demand. | +| Multi-arch | Unaffected; history keys on the index/source digest. | +| Concurrency | Per-image group already serializes runs → safe read-modify-write. | +| Promotion / delete-image | Benefits: the `mirror-history` tag keeps the package alive, avoiding the last-tagged-version delete workaround. No change required in promote workflows. | +| Dashboard | Extended: the Acquisition view surfaces a per-repo **synchronized history** read from the `:mirror-history` artifact, and excludes the reserved `mirror-history` tag when deciding whether an image is actually present in quarantine (see [Dashboard integration](#dashboard-integration)). | + +## Dashboard integration + +Rather than hide the persisted package, the CSSC Dashboard's **Acquisition** view +is extended to *surface* what each repo has already synchronized (this resolves +O1). + +- **Read the history.** `packages-service` gains a capability to fetch and parse + the `:mirror-history` artifact for a `quarantine/` repo: resolve the + `mirror-history` tag manifest, read its single JSON layer blob, and return the + parsed `entries` (source tag, source digest, dest tag, `syncedAt`, run URL). It + reads the registry (GHCR v2) blob, not just the Packages API, since the entries + live in the artifact body. A new endpoint (e.g. `GET /packages/{name}/history`) + exposes it. +- **Correct the "in quarantine" signal.** The reserved `mirror-history` tag is + excluded when determining whether an image is actually present in quarantine, + so a history-only package (image already promoted and deleted) is no longer + shown as if an image were still awaiting promotion. +- **Render per repo.** Each Acquisition card shows, alongside its promotion + issues, a **Synchronized** list: the source tags/digests already mirrored (with + timestamp and run link), so it is clear what has flowed through the repo even + after the image itself has left quarantine. + +The access model is unchanged: only the outbound GitHub/registry read is +authenticated (the existing `read:packages` token); no new inbound auth. + +## Open questions for review + +_All design questions are currently resolved — see below._ + +### Resolved + +- **O1 — Dashboard visibility.** Resolved: **surface, don't hide**. The dashboard + is extended to read the `:mirror-history` artifact and show a per-repo + synchronized history, and to exclude the reserved tag from the "in quarantine" + signal (see [Dashboard integration](#dashboard-integration)). +- **O2 — `copy_referrers` + history.** Resolved: a recorded digest **also + suppresses** the referrer re-copy (skip once recorded); `force` refreshes + referrers on demand. +- **O3 — Bootstrapping.** Resolved: **no seeding**. Images mirrored+promoted + before this ships will re-mirror once (no history yet) and then record — this is + acceptable. +- **O4 — Reserved tag.** Resolved: the reserved tag name `mirror-history` is + confirmed and will be documented as never a valid upstream tag to mirror. + +## Deliverables (once approved) + +1. `mirror-history` composite action (`check` + `record`). +2. `_mirror-image.yml` wiring (check before copy, record after). +3. Dashboard integration: a `packages-service` history read + endpoint, and the + Acquisition view surfacing per-repo synchronized history (and excluding the + reserved tag from the "in quarantine" signal). +4. Reference + architecture docs (this doc finalized, action catalogue entry, a + `mirror-history` reference page, acquire index link). +5. End-to-end validation: mirror → promote → delete → re-run mirror shows + **skipped (already synchronized)** instead of re-copy. diff --git a/docs/reference/README.md b/docs/reference/README.md index 71aa519..6fc3322 100644 --- a/docs/reference/README.md +++ b/docs/reference/README.md @@ -8,3 +8,4 @@ Reference material and conventions for the `cssc-framework` repository. - [Image annotations](image-annotations.md) — OCI manifest annotations carried by the CSSC Dashboard images. - [Image attestations](image-attestations.md) — SBOM and provenance attestations published as OCI 1.1 referrers on the CSSC Dashboard images. - [Acquisition provenance](acquisition-provenance.md) — the OCI 1.1 referrer attached to mirrored images recording where they were acquired from. +- [Mirror history](mirror-history.md) — the per-repo OCI artifact recording the source digests a mirror has already synchronized (so promoted-and-deleted digests are not re-mirrored). diff --git a/docs/reference/mirror-history.md b/docs/reference/mirror-history.md new file mode 100644 index 0000000..24e4b30 --- /dev/null +++ b/docs/reference/mirror-history.md @@ -0,0 +1,85 @@ +# Mirror history artifact + +Every repository the mirror workflows synchronize carries a **mirror-history** +OCI artifact recording the source digests already acquired for that repository. +It is written by the [`mirror-history`](workflow-actions.md#mirror-history) +action from the [`_mirror-image.yml`](../../.github/workflows/_mirror-image.yml) +workflow, and it lets the mirror skip a digest it has already synchronized once — +even after that digest has been promoted out of and deleted from quarantine. + +The architecture and rationale are in +[docs/architecture/acquire/mirror-history.md](../architecture/acquire/mirror-history.md). + +## Location + +Stored under a **reserved tag** in the synchronized repository: + +``` +ghcr.io//quarantine/:mirror-history +``` + +`mirror-history` is never a valid upstream tag to mirror. Because it is a +separate tag it survives image-tag deletion during promotion (and keeps the +package alive, avoiding the GHCR "cannot delete the last tagged version" case). + +## Format + +- **manifest artifactType:** `application/vnd.cssc.mirror-history.v1+json` + (pushed with `oras push --artifact-type`; standard empty config). +- **layer mediaType:** `application/vnd.cssc.mirror-history.v1+json` +- **manifest annotations:** `org.opencontainers.image.title=mirror-history.json`, + `com.toddysm.mirror-history.count`, `com.toddysm.mirror-history.updated`. + +The single layer blob is an append-only JSON log: + +```json +{ + "schemaVersion": 1, + "image": "ghcr.io//quarantine/python", + "source": "docker.io/library/python", + "entries": [ + { + "sourceTag": "3.14-slim", + "sourceDigest": "sha256:...", + "destTag": "3.14-slim", + "syncedAt": "2026-07-30T06:00:00Z", + "runUrl": "https://github.com//cssc-framework/actions/runs/", + "runId": "", + "runAttempt": "1", + "force": false + } + ] +} +``` + +- **History key** = `(sourceTag, sourceDigest)`. A digest is "already + synchronized" when an entry matches both. `sourceDigest` is also the + destination digest (the copy preserves digests). +- **Append-only, unbounded, chronological.** Entries are never removed; a `force` + re-sync appends a new entry rather than mutating past ones. + +## Behaviour + +- On each mirror run the source digest is resolved and checked against the + history. If the `(tag, digest)` is already recorded the copy is **skipped** + (job summary: *skipped, already synchronized*), unless `force` is set. +- A recorded digest also suppresses the `copy_referrers` re-copy; `force` + refreshes referrers on demand. +- After a run that actually copied (or found the destination already up to date) + the digest is **recorded**. Skipped-by-history runs record nothing. + +Controlled by the `record_history` input of `_mirror-image.yml` (default on). + +## Retrieve + +```bash +# Inspect the artifact manifest (annotations show count + last-updated): +crane manifest ghcr.io//quarantine/python:mirror-history + +# Pull the JSON log: +oras pull -o out ghcr.io//quarantine/python:mirror-history +cat out/mirror-history.json | jq . +``` + +The CSSC Dashboard's Acquisition view surfaces this history per repository via +the `packages-service` `GET /packages/{name}/history` endpoint. diff --git a/docs/reference/workflow-actions.md b/docs/reference/workflow-actions.md index e481417..629149d 100644 --- a/docs/reference/workflow-actions.md +++ b/docs/reference/workflow-actions.md @@ -32,6 +32,7 @@ phrasings it replaces. | __evaluate-findings__ | Apply the severity threshold + CVE exceptions to produce a gate decision. | "gate on scan findings" | | __attach-scan-report__ | Attach the OCI scan-report referrer — an in-toto vulnerability attestation payload plus summary annotations — to a promoted image. | "attach scan-report attestation" | | __attach-acquisition-provenance__ | Attach the acquisition-provenance in-toto referrer to a mirrored image. | "acquisition provenance" | +| __mirror-history__ | Record/query the source digests a mirror has already synchronized, in a per-repo `:mirror-history` OCI artifact. | "mirror history", "already synchronized" | | __delete-image__ | Delete one tag from a GHCR repository via the Packages API. | "delete promoted tags from quarantine" | Standard nouns: @@ -111,6 +112,30 @@ external → quarantine acquisition, and only when the acquired digest changed. | `copy-referrers` | no | `false` | Whether the mirror copied referrers (records the copy method). | | `source-authenticated` | no | `false` | `true` when the mirror logged in to the source registry. | +### mirror-history + +Maintain a durable, deletion-surviving record of the source digests a mirror has +already synchronized into a repository. The record is an append-only JSON log +stored as an OCI artifact under the reserved tag `:mirror-history` +(artifact type `application/vnd.cssc.mirror-history.v1+json`, pushed with +`oras push --artifact-type`). Because it is a separate tag it survives image-tag +deletion during promotion, so a digest that was mirrored once is not +re-synchronized after it is promoted out of and deleted from quarantine. See +[mirror history](../architecture/acquire/mirror-history.md). + +| Input | Required | Default | Description | +| ----- | -------- | ------- | ----------- | +| `operation` | yes | — | `check` or `record`. | +| `dest-image` | yes | — | Destination image without tag (the repo the history belongs to). | +| `source-tag` | yes | — | Source tag being synchronized. | +| `source-digest` | yes | — | Source manifest digest being synchronized. | +| `source-image` | no | `""` | Source image without tag (recorded on the entry; used by `record`). | +| `dest-tag` | no | `""` | Destination tag written in quarantine (required for `record`). | +| `force` | no | `false` | Recorded on the entry when the sync was a force run. | + +Outputs: `already-synchronized` (`check` only — `true` when the `(source-tag, +source-digest)` pair is already recorded). + ### scan-image Scan one image filesystem with `trivy image`.