From a5adb51304068154ecba7aa36ebb46666d065ddb Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 31 Aug 2026 20:02:57 +0000 Subject: [PATCH 1/3] feat(devcontainer): add Mesa Dozen WSLg runtime (#41) --- .github/workflows/validate.yml | 18 +- AGENTS.md | 7 +- README.md | 43 +- classic-packages.lock | 4 + classic-vulkan-packages.lock | 23 + classic-vulkan-toolchain.json | 117 ++++ classic-vulkan-toolchain.spdx.json | 36 + linux/Dockerfile | 151 +++- tools/install_classic_vulkan_toolchain.py | 647 ++++++++++++++++++ .../test_install_classic_vulkan_toolchain.py | 147 ++++ tools/validate-vulkan-toolchain.sh | 191 ++++++ 11 files changed, 1367 insertions(+), 17 deletions(-) create mode 100644 classic-vulkan-packages.lock create mode 100644 classic-vulkan-toolchain.json create mode 100644 classic-vulkan-toolchain.spdx.json create mode 100644 tools/install_classic_vulkan_toolchain.py create mode 100644 tools/tests/test_install_classic_vulkan_toolchain.py create mode 100755 tools/validate-vulkan-toolchain.sh diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index bdc5f11..59e837f 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -38,11 +38,16 @@ jobs: .dockerignore | .github/actionlint.yaml | \ .github/workflows/* | linux/* | \ audio-toolchain.json | audio-toolchain.spdx.json | \ + classic-vulkan-toolchain.json | \ + classic-vulkan-toolchain.spdx.json | \ + classic-vulkan-packages.lock | \ toolchains.json | tools/audio/* | \ tools/build-sdl3-mixer.sh | \ + tools/install_classic_vulkan_toolchain.py | \ tools/require-image-checks.sh | \ tools/test-require-image-checks.sh | \ - tools/validate-toolchains.sh) + tools/validate-toolchains.sh | \ + tools/validate-vulkan-toolchain.sh) linux=true ;; esac @@ -53,13 +58,19 @@ jobs: classic-packages.lock | classic-toolchain.json | \ classic-shader-toolchain.json | \ classic-shader-toolchain.spdx.json | \ + classic-vulkan-toolchain.json | \ + classic-vulkan-toolchain.spdx.json | \ + classic-vulkan-packages.lock | \ tools/audio/* | tools/build-sdl3-mixer.sh | \ tools/require-image-checks.sh | \ tools/test-require-image-checks.sh | \ tools/validate-classic-check.sh | \ tools/validate-classic-image.sh | \ tools/install_classic_shader_toolchain.py | \ - tools/tests/test_install_classic_shader_toolchain.py) + tools/tests/test_install_classic_shader_toolchain.py | \ + tools/install_classic_vulkan_toolchain.py | \ + tools/validate-vulkan-toolchain.sh | \ + tools/tests/test_install_classic_vulkan_toolchain.py) classic=true ;; esac @@ -98,6 +109,9 @@ jobs: - name: Test Classic shader toolchain installer run: python3 -m unittest tools/tests/test_install_classic_shader_toolchain.py + - name: Test Classic Vulkan toolchain installer + run: python3 -m unittest tools/tests/test_install_classic_vulkan_toolchain.py + linux: name: Linux image needs: changes diff --git a/AGENTS.md b/AGENTS.md index 59876ec..9381c8a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,8 +23,11 @@ - The public Classic image's shader contract is defined by `classic-shader-toolchain.json`: retain the exact DXC/SPIRV-Cross archive and source checksums, upstream licenses, and `/usr/local/bin` tool paths. Its - pinned Lavapipe/Xvfb packages provide the equivalent public environment for - fork-safe GPU coverage; consumers still pin a released image digest. + `classic-vulkan-toolchain.json` contract builds the pinned Mesa Dozen driver + from `classic-vulkan-packages.lock`, carries only the dzn library and ICD + into both Linux images, and records the WSLg host mounts/environment. Its + pinned Lavapipe/Xvfb packages remain the fork-safe headless path; consumers + still pin a released image digest and supply WSLg adapter selection. - Keep a stable numeric runner UID when restoring a Classic ccache directory; the mode-1777 mount root supports non-root initialization but does not make ccache's owner-writable nested directories reusable across different UIDs. diff --git a/README.md b/README.md index 400e4f0..94c24b9 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,9 @@ docker run --rm atrinik-classic-build dxc --version docker run --rm atrinik-classic-build spirv-cross --help docker run --rm atrinik-classic-build \ sh -c 'test -f /usr/share/vulkan/icd.d/lvp_icd.json && \ - VK_DRIVER_FILES=/usr/share/vulkan/icd.d/lvp_icd.json vulkaninfo --summary' + test -f /usr/share/vulkan/icd.d/dzn_icd.x86_64.json && \ + VK_DRIVER_FILES=/usr/share/vulkan/icd.d/lvp_icd.json \ + xvfb-run -a vulkaninfo --summary' docker run --rm atrinik-linux-build \ atrinik-sdl3-mixer-probe \ /usr/local/share/atrinik/audio/opus-probe.opus @@ -179,6 +181,40 @@ Classic digest can serve both the shader-producing build and fork-safe GPU coverage jobs. The Classic consumer must update its separate workflow to pin that released digest; this image change does not rewrite consumer source. +The public Linux and Classic images also carry the Mesa Dozen Vulkan runtime +described by [`classic-vulkan-toolchain.json`](classic-vulkan-toolchain.json). +Mesa 26.0.8 is built once from its checksum-pinned source archive with only +the Wayland/D3D12 Vulkan path enabled; the build-only package closure is +recorded in [`classic-vulkan-packages.lock`](classic-vulkan-packages.lock) and +the source provenance in +[`classic-vulkan-toolchain.spdx.json`](classic-vulkan-toolchain.spdx.json). +Only those two custom-built artifacts—`/usr/lib/x86_64-linux-gnu/libvulkan_dzn.so` +and `/usr/share/vulkan/icd.d/dzn_icd.x86_64.json`—enter the runtime images +from the Mesa build stage. CI continues to select Lavapipe explicitly through +Xvfb. A WSLg consumer can +select Dozen after providing `/dev/dxg`, `/usr/lib/wsl`, and the WSLg runtime +mounts plus its adapter name: + +```sh +docker run --rm --gpus=all \ + --device=/dev/dxg \ + --volume /usr/lib/wsl:/usr/lib/wsl:ro \ + --volume /mnt/wslg:/mnt/wslg:ro \ + --env DISPLAY= \ + --env WAYLAND_DISPLAY=wayland-0 \ + --env XDG_RUNTIME_DIR=/mnt/wslg/runtime-dir \ + --env LD_LIBRARY_PATH=/usr/lib/wsl/lib \ + --env GALLIUM_DRIVER=d3d12 \ + --env MESA_D3D12_DEFAULT_ADAPTER_NAME=NVIDIA \ + --env VK_DRIVER_FILES=/usr/share/vulkan/icd.d/dzn_icd.x86_64.json \ + atrinik-classic-build vulkaninfo --summary +``` + +The adapter name is deliberately consumer-supplied; non-WSLg CI never selects +an NVIDIA device. The image validator runs the existing Lavapipe/Xvfb probe +and runs the Dozen probe only when all WSLg mounts/libraries and the consumer +adapter variable are present. + Classic runs as the unprivileged `ubuntu` user by default. `/cache/ccache` is a mode-1777 mount contract so CI can run with its own numeric UID and persist the directory without granting root. Consumers must still select ccache explicitly @@ -205,8 +241,9 @@ the digest, never a rolling tag. To update that pin: 1. Update the matching Ubuntu base digest and snapshot value in both `linux/Dockerfile` and `classic-toolchain.json`, refresh the exact direct versions in `classic-packages.lock`, and update the tool versions, shader - coordinates in `classic-shader-toolchain.json`, and pinned Classic - validation commit in `classic-toolchain.json`. + coordinates in `classic-shader-toolchain.json`, Mesa Dozen source + coordinates in `classic-vulkan-toolchain.json`, the Vulkan build lock, and + pinned Classic validation commit in `classic-toolchain.json`. 2. Build `classic-validation` and `classic-final`, run the repository checks, and compare compressed image size plus local client/server timings with the prior digest. diff --git a/classic-packages.lock b/classic-packages.lock index 0d854e3..0fc08a5 100644 --- a/classic-packages.lock +++ b/classic-packages.lock @@ -11,6 +11,7 @@ gcovr=7.2+really-2 git=1:2.53.0-1ubuntu1 jq=1.8.1-4ubuntu2 libcurl4-openssl-dev=8.18.0-1ubuntu2.3 +libdrm2=2.4.131-1 libgd-dev=2.3.3-13ubuntu2 libidn2-dev=2.3.8-4build1 libminiupnpc-dev=2.3.3-2build1 @@ -21,7 +22,9 @@ libsdl3-ttf-dev=3.2.2+ds-1build1 libssl-dev=3.5.5-1ubuntu3.3 libssl3t64=3.5.5-1ubuntu3.3 libsubunit-dev=1.4.4+1-1 +libudev1=259.5-0ubuntu3.3 libvulkan1=1.4.341.0-1 +libwayland-client0=1.24.0-2 libxml2-dev=2.15.2+dfsg-0.1ubuntu0.1 mesa-vulkan-drivers=26.0.3-1ubuntu1 ninja-build=1.13.2-1 @@ -33,4 +36,5 @@ python3-dev=3.14.3-0ubuntu2 vulkan-tools=1.4.341.0+dfsg1-1 xauth=1:1.1.2-1.1build1 xvfb=2:21.1.22-1ubuntu1 +zlib1g=1:1.3.dfsg+really1.3.1-1ubuntu3 zlib1g-dev=1:1.3.dfsg+really1.3.1-1ubuntu3 diff --git a/classic-vulkan-packages.lock b/classic-vulkan-packages.lock new file mode 100644 index 0000000..ac6de64 --- /dev/null +++ b/classic-vulkan-packages.lock @@ -0,0 +1,23 @@ +bison=2:3.8.2+dfsg-1build4 +build-essential=12.12ubuntu2.26.04.2 +ca-certificates=20260601~26.04.1 +directx-headers-dev=1.619.1-1 +flex=2.6.4-8.2build2 +libdrm-dev=2.4.131-1 +libssl3t64=3.5.5-1ubuntu3.3 +libudev-dev=259.5-0ubuntu3.3 +libvulkan-dev=1.4.341.0-1 +libwayland-dev=1.24.0-2 +meson=1.10.1-1ubuntu2 +ninja-build=1.13.2-1 +openssl=3.5.5-1ubuntu3.3 +openssl-provider-legacy=3.5.5-1ubuntu3.3 +pkgconf=2.5.1-4 +python3=3.14.3-0ubuntu2 +python3-dev=3.14.3-0ubuntu2 +python3-mako=1.3.10-3build1 +python3-markupsafe=3.0.3-1build1 +python3-setuptools=78.1.1-0.1build1 +python3-yaml=6.0.3-1build1 +wayland-protocols=1.47-1 +zlib1g-dev=1:1.3.dfsg+really1.3.1-1ubuntu3 diff --git a/classic-vulkan-toolchain.json b/classic-vulkan-toolchain.json new file mode 100644 index 0000000..bdd6276 --- /dev/null +++ b/classic-vulkan-toolchain.json @@ -0,0 +1,117 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "schema_version": 1, + "platform": "linux/amd64", + "target": "classic-final", + "image": "ghcr.io/atrinik/classic-build", + "base": { + "image": "ubuntu:26.04", + "digest": "sha256:678c6550cc43645e08669028bc177f50be4e7c5b8cca677067b1914d4afc7a03", + "apt_snapshot": "20260810T000000Z" + }, + "source": { + "repository": "mesa/mesa", + "tag": "mesa-26.0.8", + "version": "26.0.8", + "commit": "60e95b787857afbc9a00b693b91c0d9c8923a430", + "url": "https://archive.mesa3d.org/mesa-26.0.8.tar.xz", + "sha256": "caf1c0061a68e88dfa74967a7e780c0e85d65b6c4e334cd69095a5dc54ad78bc", + "archive_root": "mesa-26.0.8" + }, + "build": { + "system": "meson", + "version": "1.10.1", + "package_lock": "classic-vulkan-packages.lock", + "configure": { + "buildtype": "release", + "libdir": "lib/x86_64-linux-gnu", + "prefix": "/usr", + "wrap_mode": "nodownload" + }, + "options": { + "build-tests": false, + "display-info": "disabled", + "egl": "disabled", + "expat": "disabled", + "gallium-d3d12-graphics": "enabled", + "gallium-d3d12-video": "disabled", + "gallium-drivers": ["d3d12"], + "gallium-rusticl": false, + "gallium-va": "disabled", + "gbm": "disabled", + "glx": "disabled", + "gles1": "disabled", + "gles2": "disabled", + "html-docs": "disabled", + "install-mesa-clc": false, + "libunwind": "disabled", + "llvm": "disabled", + "lmsensors": "disabled", + "mesa-clc": "auto", + "opengl": false, + "perfetto": false, + "platforms": ["wayland"], + "shared-glapi": "disabled", + "spirv-tools": "disabled", + "teflon": false, + "valgrind": "disabled", + "vulkan-drivers": ["microsoft-experimental"], + "vulkan-manifest-per-architecture": true, + "xmlconfig": "disabled", + "zstd": "disabled" + } + }, + "install": { + "library": "/usr/lib/x86_64-linux-gnu/libvulkan_dzn.so", + "icd": "/usr/share/vulkan/icd.d/dzn_icd.x86_64.json", + "icd_library": "libvulkan_dzn.so", + "icd_api_version": "1.1.335" + }, + "runtime": { + "packages": [ + {"name": "libdrm2", "version": "2.4.131-1"}, + {"name": "libudev1", "version": "259.5-0ubuntu3.3"}, + {"name": "libvulkan1", "version": "1.4.341.0-1"}, + {"name": "libwayland-client0", "version": "1.24.0-2"}, + {"name": "mesa-vulkan-drivers", "version": "26.0.3-1ubuntu1"}, + {"name": "vulkan-tools", "version": "1.4.341.0+dfsg1-1"}, + {"name": "zlib1g", "version": "1:1.3.dfsg+really1.3.1-1ubuntu3"} + ], + "shared_library_sonames": [ + "libc.so.6", + "libdrm.so.2", + "libgcc_s.so.1", + "libm.so.6", + "libudev.so.1", + "libwayland-client.so.0", + "libz.so.1" + ], + "headless": { + "vulkan_icd": "/usr/share/vulkan/icd.d/lvp_icd.json", + "display_server": "Xvfb", + "commands": ["vulkaninfo", "Xvfb", "xvfb-run"] + }, + "wslg": { + "vulkan_icd": "/usr/share/vulkan/icd.d/dzn_icd.x86_64.json", + "required_mounts": [ + "/dev/dxg", + "/mnt/wslg/runtime-dir", + "/usr/lib/wsl" + ], + "required_host_libraries": [ + "/usr/lib/wsl/lib/libd3d12.so", + "/usr/lib/wsl/lib/libdxcore.so" + ], + "fixed_environment": { + "DISPLAY": "", + "GALLIUM_DRIVER": "d3d12", + "LD_LIBRARY_PATH": "/usr/lib/wsl/lib", + "VK_DRIVER_FILES": "/usr/share/vulkan/icd.d/dzn_icd.x86_64.json", + "WAYLAND_DISPLAY": "wayland-0", + "XDG_RUNTIME_DIR": "/mnt/wslg/runtime-dir" + }, + "consumer_environment": ["MESA_D3D12_DEFAULT_ADAPTER_NAME"], + "probe": ["vulkaninfo", "--summary"] + } + } +} diff --git a/classic-vulkan-toolchain.spdx.json b/classic-vulkan-toolchain.spdx.json new file mode 100644 index 0000000..593c433 --- /dev/null +++ b/classic-vulkan-toolchain.spdx.json @@ -0,0 +1,36 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "Atrinik Classic WSLg Vulkan Dozen runtime", + "documentNamespace": "https://github.com/atrinik/devcontainer/sbom/classic-vulkan-toolchain/60e95b787857afbc9a00b693b91c0d9c8923a430", + "creationInfo": { + "created": "2026-08-31T00:00:00Z", + "creators": ["Tool: Atrinik devcontainer vulkan-toolchain inventory"] + }, + "packages": [ + { + "SPDXID": "SPDXRef-Package-Mesa", + "name": "Mesa", + "versionInfo": "26.0.8", + "downloadLocation": "https://archive.mesa3d.org/mesa-26.0.8.tar.xz", + "filesAnalyzed": false, + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "caf1c0061a68e88dfa74967a7e780c0e85d65b6c4e334cd69095a5dc54ad78bc" + } + ], + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-Package-Mesa" + } + ] +} diff --git a/linux/Dockerfile b/linux/Dockerfile index 9c3655d..5d08b54 100644 --- a/linux/Dockerfile +++ b/linux/Dockerfile @@ -1,14 +1,83 @@ # syntax=docker/dockerfile:1@sha256:87999aa3d42bdc6bea60565083ee17e86d1f3339802f543c0d03998580f9cb89 +# The public Linux and Classic images share one checksum-pinned Mesa Dozen +# build. Only the resulting ICD and driver are copied into either runtime +# image; the source and build toolchain remain in this disposable stage. +FROM ubuntu:26.04@sha256:678c6550cc43645e08669028bc177f50be4e7c5b8cca677067b1914d4afc7a03 AS mesa-dozen + +ARG TARGETARCH +ENV DEBIAN_FRONTEND=noninteractive \ + UBUNTU_SNAPSHOT=20260810T000000Z + +COPY classic-vulkan-packages.lock /tmp/classic-vulkan-packages.lock +# The minimal base has no CA bundle. Bootstrap the exact TLS closure over the +# signed snapshot, then repeat the update with normal HTTPS verification. +RUN test "${TARGETARCH}" = amd64 \ + && test "$(dpkg --print-architecture)" = amd64 \ + && sed -i \ + -e "s|http://archive.ubuntu.com/ubuntu/|https://snapshot.ubuntu.com/ubuntu/${UBUNTU_SNAPSHOT}/|" \ + -e "s|http://security.ubuntu.com/ubuntu/|https://snapshot.ubuntu.com/ubuntu/${UBUNTU_SNAPSHOT}/|" \ + /etc/apt/sources.list.d/ubuntu.sources \ + && sed -i '/^Signed-By:/a Check-Valid-Until: no' \ + /etc/apt/sources.list.d/ubuntu.sources \ + && apt-get -o Acquire::https::Verify-Peer=false update \ + && awk -F= '$1 == "ca-certificates" \ + || $1 == "libssl3t64" \ + || $1 == "openssl" \ + || $1 == "openssl-provider-legacy" { print }' \ + /tmp/classic-vulkan-packages.lock \ + | xargs -r apt-get -o Acquire::https::Verify-Peer=false \ + install -y --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get update \ + && xargs -r apt-get install -y --no-install-recommends \ + < /tmp/classic-vulkan-packages.lock \ + && while IFS='=' read -r package version; do \ + test "$(dpkg-query --show --showformat='${Version}' "${package}")" = "${version}"; \ + done < /tmp/classic-vulkan-packages.lock \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*.deb + +COPY classic-vulkan-toolchain.json /tmp/classic-vulkan-toolchain.json +COPY tools/install_classic_vulkan_toolchain.py \ + /tmp/install_classic_vulkan_toolchain.py +RUN python3 /tmp/install_classic_vulkan_toolchain.py \ + --manifest /tmp/classic-vulkan-toolchain.json \ + --cache /tmp/classic-vulkan-downloads \ + --prefix /usr \ + --jobs "$(nproc)" \ + && ldconfig \ + && test -f /usr/lib/x86_64-linux-gnu/libvulkan_dzn.so \ + && test -f /usr/share/vulkan/icd.d/dzn_icd.x86_64.json \ + && rm -rf /tmp/classic-vulkan-downloads \ + /tmp/classic-vulkan-toolchain.json \ + /tmp/install_classic_vulkan_toolchain.py + FROM ubuntu:26.04@sha256:678c6550cc43645e08669028bc177f50be4e7c5b8cca677067b1914d4afc7a03 AS toolchain -ENV DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_FRONTEND=noninteractive \ + UBUNTU_SNAPSHOT=20260810T000000Z RUN rm -f /etc/apt/apt.conf.d/docker-clean +# Keep the public Linux image's exact Vulkan runtime closure on the same signed +# Ubuntu snapshot as the Classic image. The minimal base needs the same +# temporary TLS bootstrap before normal HTTPS verification can resume. +RUN sed -i \ + -e "s|http://archive.ubuntu.com/ubuntu/|https://snapshot.ubuntu.com/ubuntu/${UBUNTU_SNAPSHOT}/|" \ + -e "s|http://security.ubuntu.com/ubuntu/|https://snapshot.ubuntu.com/ubuntu/${UBUNTU_SNAPSHOT}/|" \ + /etc/apt/sources.list.d/ubuntu.sources \ + && sed -i '/^Signed-By:/a Check-Valid-Until: no' \ + /etc/apt/sources.list.d/ubuntu.sources + RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ - apt-get update \ + apt-get -o Acquire::https::Verify-Peer=false update \ + && apt-get -o Acquire::https::Verify-Peer=false install -y --no-install-recommends \ + ca-certificates \ + libssl3t64 \ + openssl \ + openssl-provider-legacy \ + && apt-get update \ && apt-get install -y --no-install-recommends \ alsa-utils \ build-essential \ @@ -34,6 +103,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ libasound2-plugins \ libclang-rt-dev \ libcurl4-openssl-dev \ + libdrm2=2.4.131-1 \ libgd-dev \ libidn2-dev \ libminiupnpc-dev \ @@ -43,11 +113,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ libsdl3-ttf-dev \ libssl-dev \ libsubunit-dev \ - libvulkan-dev \ + libudev1=259.5-0ubuntu3.3 \ + libvulkan-dev=1.4.341.0-1 \ + libvulkan1=1.4.341.0-1 \ + libwayland-client0=1.24.0-2 \ libxml2-dev \ libxml2-utils \ llvm \ - mesa-vulkan-drivers \ + mesa-vulkan-drivers=26.0.3-1ubuntu1 \ ninja-build \ openssh-client \ patch \ @@ -64,12 +137,21 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ tree \ unzip \ valgrind \ - vulkan-tools \ + vulkan-tools=1.4.341.0+dfsg1-1 \ xauth \ xvfb \ xz-utils \ zip \ - zlib1g-dev + zlib1g=1:1.3.dfsg+really1.3.1-1ubuntu3 \ + zlib1g-dev=1:1.3.dfsg+really1.3.1-1ubuntu3 + +COPY --from=mesa-dozen \ + /usr/lib/x86_64-linux-gnu/libvulkan_dzn.so \ + /usr/lib/x86_64-linux-gnu/libvulkan_dzn.so +COPY --from=mesa-dozen \ + /usr/share/vulkan/icd.d/dzn_icd.x86_64.json \ + /usr/share/vulkan/icd.d/dzn_icd.x86_64.json +RUN ldconfig # Declare independently updatable downloaded tools after the distribution # layer so a language/security tool bump can reuse the expensive native SDK. @@ -297,6 +379,12 @@ RUN clang --version \ COPY toolchains.json /usr/local/share/atrinik/toolchains.json COPY audio-toolchain.json /usr/local/share/atrinik/audio-toolchain.json COPY audio-toolchain.spdx.json /usr/local/share/atrinik/audio-toolchain.spdx.json +COPY classic-vulkan-toolchain.json \ + /usr/local/share/atrinik/classic-vulkan-toolchain.json +COPY classic-vulkan-toolchain.spdx.json \ + /usr/local/share/atrinik/classic-vulkan-toolchain.spdx.json +COPY classic-vulkan-packages.lock \ + /usr/local/share/atrinik/classic-vulkan-packages.lock FROM toolchain AS validation @@ -305,7 +393,11 @@ COPY .github/workflows /workspace/.github/workflows COPY toolchains.json /workspace/toolchains.json COPY audio-toolchain.json /workspace/audio-toolchain.json COPY audio-toolchain.spdx.json /workspace/audio-toolchain.spdx.json +COPY classic-vulkan-toolchain.json /workspace/classic-vulkan-toolchain.json +COPY classic-vulkan-toolchain.spdx.json /workspace/classic-vulkan-toolchain.spdx.json COPY tools/validate-toolchains.sh /workspace/tools/validate-toolchains.sh +COPY tools/validate-vulkan-toolchain.sh \ + /workspace/tools/validate-vulkan-toolchain.sh RUN actionlint \ -config-file /workspace/.github/actionlint.yaml \ @@ -316,11 +408,20 @@ RUN actionlint \ /usr/local/share/atrinik/audio-toolchain.json \ /workspace/audio-toolchain.spdx.json \ /usr/local/share/atrinik/audio-toolchain.spdx.json \ + && /workspace/tools/validate-vulkan-toolchain.sh \ + /workspace/classic-vulkan-toolchain.json \ + /usr/local/share/atrinik/classic-vulkan-toolchain.json \ + /workspace/classic-vulkan-toolchain.spdx.json \ + /usr/local/share/atrinik/classic-vulkan-toolchain.spdx.json \ && syft dir:/ --override-default-catalogers sbom-cataloger -o json \ - | jq -e --slurpfile inventory /workspace/audio-toolchain.json \ + | jq -e \ + --slurpfile inventory /workspace/audio-toolchain.json \ + --slurpfile vulkan /workspace/classic-vulkan-toolchain.json \ '([.artifacts[] | [.name, .version]] | sort) == \ (($inventory[0].dependencies + [$inventory[0].sdl_mixer]) \ - | map([.name, .version]) | sort)' \ + | map([.name, .version]) \ + | . + [["Mesa", $vulkan[0].source.version]] \ + | sort)' \ && syft dir:/ --select-catalogers +sbom-cataloger -o spdx-json \ | jq -e --slurpfile inventory /workspace/audio-toolchain.json ' \ .spdxVersion == "SPDX-2.3" \ @@ -334,6 +435,11 @@ RUN actionlint \ .name == "github.com/github/gh-stack" \ and (.versionInfo | startswith("v0.1.0")) \ ) \ + and any( \ + .packages[]; \ + .name == "Mesa" \ + and .versionInfo == "26.0.8" \ + ) \ and ( \ [.packages[] | [.name, .versionInfo]] \ | contains(( \ @@ -380,6 +486,14 @@ RUN test "${TARGETARCH}" = amd64 \ < /tmp/classic-packages.lock \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*.deb +COPY --from=mesa-dozen \ + /usr/lib/x86_64-linux-gnu/libvulkan_dzn.so \ + /usr/lib/x86_64-linux-gnu/libvulkan_dzn.so +COPY --from=mesa-dozen \ + /usr/share/vulkan/icd.d/dzn_icd.x86_64.json \ + /usr/share/vulkan/icd.d/dzn_icd.x86_64.json +RUN ldconfig + # Classic requires the same checksum-pinned SDL3_mixer contract as the broad # Linux image. Its codec dependencies are statically linked into the shared # library, while SDL3 and the remaining native libraries come from the frozen @@ -435,6 +549,12 @@ COPY classic-shader-toolchain.json \ /usr/local/share/atrinik/classic-shader-toolchain.json COPY classic-shader-toolchain.spdx.json \ /usr/local/share/atrinik/classic-shader-toolchain.spdx.json +COPY classic-vulkan-toolchain.json \ + /usr/local/share/atrinik/classic-vulkan-toolchain.json +COPY classic-vulkan-toolchain.spdx.json \ + /usr/local/share/atrinik/classic-vulkan-toolchain.spdx.json +COPY classic-vulkan-packages.lock \ + /usr/local/share/atrinik/classic-vulkan-packages.lock COPY audio-toolchain.json /usr/local/share/atrinik/audio-toolchain.json COPY audio-toolchain.spdx.json /usr/local/share/atrinik/audio-toolchain.spdx.json @@ -457,9 +577,13 @@ COPY classic-packages.lock /workspace/classic-packages.lock COPY classic-toolchain.json /workspace/classic-toolchain.json COPY classic-shader-toolchain.json /workspace/classic-shader-toolchain.json COPY classic-shader-toolchain.spdx.json /workspace/classic-shader-toolchain.spdx.json +COPY classic-vulkan-toolchain.json /workspace/classic-vulkan-toolchain.json +COPY classic-vulkan-toolchain.spdx.json /workspace/classic-vulkan-toolchain.spdx.json COPY audio-toolchain.json /workspace/audio-toolchain.json COPY linux/Dockerfile /workspace/linux/Dockerfile COPY tools/validate-classic-image.sh /workspace/tools/validate-classic-image.sh +COPY tools/validate-vulkan-toolchain.sh \ + /workspace/tools/validate-vulkan-toolchain.sh USER ubuntu RUN /workspace/tools/validate-classic-image.sh \ @@ -471,7 +595,13 @@ RUN /workspace/tools/validate-classic-image.sh \ /workspace/classic-shader-toolchain.json \ /usr/local/share/atrinik/classic-shader-toolchain.json \ /workspace/classic-shader-toolchain.spdx.json \ - /usr/local/share/atrinik/classic-shader-toolchain.spdx.json + /usr/local/share/atrinik/classic-shader-toolchain.spdx.json \ + && /workspace/tools/validate-vulkan-toolchain.sh \ + /workspace/classic-vulkan-toolchain.json \ + /usr/local/share/atrinik/classic-vulkan-toolchain.json \ + /workspace/classic-vulkan-toolchain.spdx.json \ + /usr/local/share/atrinik/classic-vulkan-toolchain.spdx.json \ + /workspace/classic-packages.lock USER root RUN syft dir:/ --select-catalogers +sbom-cataloger -o spdx-json \ @@ -482,7 +612,8 @@ RUN syft dir:/ --select-catalogers +sbom-cataloger -o spdx-json \ and ([.packages[].name] | index("SDL3_mixer") != null) \ and ([.packages[].name] | index("libogg") != null) \ and ([.packages[].name] | index("libopus") != null) \ - and ([.packages[].name] | index("libopusfile") != null)' + and ([.packages[].name] | index("libopusfile") != null) \ + and ([.packages[].name] | index("Mesa") != null)' FROM classic-ci AS classic-final diff --git a/tools/install_classic_vulkan_toolchain.py b/tools/install_classic_vulkan_toolchain.py new file mode 100644 index 0000000..4690e11 --- /dev/null +++ b/tools/install_classic_vulkan_toolchain.py @@ -0,0 +1,647 @@ +#!/usr/bin/env python3 +"""Build and install the checksum-locked Mesa Dozen Vulkan runtime.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import platform +import shutil +import stat +import subprocess +import tarfile +import tempfile +from typing import BinaryIO, Iterable +import urllib.parse +import urllib.request +import re + + +MAX_ARCHIVE_BYTES = 512 * 1024 * 1024 +MAX_EXPANDED_BYTES = 2 * 1024 * 1024 * 1024 +MAX_FILE_BYTES = 512 * 1024 * 1024 +MAX_MANIFEST_BYTES = 8 * 1024 * 1024 +MAX_MEMBERS = 100_000 +SHA256_RE = re.compile(r"[0-9a-f]{64}") +COMMIT_RE = re.compile(r"[0-9a-f]{40}") + + +class ToolchainError(RuntimeError): + """The Vulkan toolchain manifest, archive, or build failed validation.""" + + +def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: + value: dict[str, object] = {} + for key, item in pairs: + if key in value: + raise ToolchainError(f"duplicate Vulkan toolchain manifest key: {key}") + value[key] = item + return value + + +def load_json(path: Path, description: str) -> object: + try: + data = path.read_bytes() + if len(data) > MAX_MANIFEST_BYTES: + raise ToolchainError(f"{description} exceeds the JSON size limit") + return json.loads(data, object_pairs_hook=reject_duplicate_keys) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ToolchainError(f"cannot read {description} {path}: {error}") from error + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def require_keys(value: dict[str, object], expected: set[str], context: str) -> None: + actual = set(value) + if actual != expected: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + details = [] + if missing: + details.append(f"missing {', '.join(missing)}") + if extra: + details.append(f"unexpected {', '.join(extra)}") + raise ToolchainError(f"{context}: {'; '.join(details)}") + + +def locked_text(value: object, field: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise ToolchainError(f"{field} must be a non-empty trimmed string") + return value + + +def locked_digest(value: object, field: str) -> str: + value = locked_text(value, field) + if not SHA256_RE.fullmatch(value): + raise ToolchainError(f"{field} must be a lowercase SHA-256") + return value + + +def locked_commit(value: object, field: str) -> str: + value = locked_text(value, field) + if not COMMIT_RE.fullmatch(value): + raise ToolchainError(f"{field} must be a full lowercase Git SHA") + return value + + +def safe_relative_path(value: object, field: str) -> str: + value = locked_text(value, field) + if "\\" in value or "\x00" in value: + raise ToolchainError(f"{field} must be a safe relative path") + path = PurePosixPath(value) + if ( + value != path.as_posix() + or path.is_absolute() + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ToolchainError(f"{field} must be a safe relative path") + return value + + +def safe_absolute_path(value: object, field: str) -> str: + value = locked_text(value, field) + if "\\" in value or "\x00" in value: + raise ToolchainError(f"{field} must be a safe absolute path") + path = PurePosixPath(value) + if ( + value != path.as_posix() + or not path.is_absolute() + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ToolchainError(f"{field} must be a safe absolute path") + return value + + +def canonical_https(value: object, field: str) -> str: + value = locked_text(value, field) + try: + parsed = urllib.parse.urlsplit(value) + except ValueError as error: + raise ToolchainError(f"{field} must be a canonical HTTPS URL") from error + if ( + parsed.scheme != "https" + or not parsed.netloc + or parsed.query + or parsed.fragment + or parsed.username is not None + or parsed.password is not None + or parsed.port is not None + ): + raise ToolchainError(f"{field} must be a canonical HTTPS URL") + return value + + +def expected_options() -> dict[str, object]: + return { + "build-tests": False, + "display-info": "disabled", + "egl": "disabled", + "expat": "disabled", + "gallium-d3d12-graphics": "enabled", + "gallium-d3d12-video": "disabled", + "gallium-drivers": ["d3d12"], + "gallium-rusticl": False, + "gallium-va": "disabled", + "gbm": "disabled", + "glx": "disabled", + "gles1": "disabled", + "gles2": "disabled", + "html-docs": "disabled", + "install-mesa-clc": False, + "libunwind": "disabled", + "llvm": "disabled", + "lmsensors": "disabled", + "mesa-clc": "auto", + "opengl": False, + "perfetto": False, + "platforms": ["wayland"], + "shared-glapi": "disabled", + "spirv-tools": "disabled", + "teflon": False, + "valgrind": "disabled", + "vulkan-drivers": ["microsoft-experimental"], + "vulkan-manifest-per-architecture": True, + "xmlconfig": "disabled", + "zstd": "disabled", + } + + +def load_manifest(path: Path) -> dict[str, object]: + value = load_json(path, "Vulkan toolchain manifest") + if not isinstance(value, dict): + raise ToolchainError("Vulkan toolchain manifest root must be an object") + require_keys( + value, + {"$schema", "schema_version", "platform", "target", "image", "base", "source", "build", "install", "runtime"}, + "manifest root", + ) + if value["$schema"] != "https://json-schema.org/draft/2020-12/schema": + raise ToolchainError("manifest $schema is not the governed schema") + if value["schema_version"] != 1 or value["platform"] != "linux/amd64": + raise ToolchainError("unsupported Vulkan toolchain manifest") + if value["target"] != "classic-final" or value["image"] != "ghcr.io/atrinik/classic-build": + raise ToolchainError("manifest does not describe the public Classic image") + + base = value["base"] + if not isinstance(base, dict): + raise ToolchainError("manifest base must be an object") + require_keys(base, {"image", "digest", "apt_snapshot"}, "base") + if base != { + "image": "ubuntu:26.04", + "digest": "sha256:678c6550cc43645e08669028bc177f50be4e7c5b8cca677067b1914d4afc7a03", + "apt_snapshot": "20260810T000000Z", + }: + raise ToolchainError("manifest base does not match the governed Classic image") + + source = value["source"] + if not isinstance(source, dict): + raise ToolchainError("manifest source must be an object") + require_keys( + source, + {"repository", "tag", "version", "commit", "url", "sha256", "archive_root"}, + "source", + ) + if source["repository"] != "mesa/mesa": + raise ToolchainError("source.repository is not the governed Mesa repository") + if source["tag"] != "mesa-26.0.8" or source["version"] != "26.0.8": + raise ToolchainError("source is not the qualified Mesa 26.0.8 release") + locked_commit(source["commit"], "source.commit") + if source["commit"] != "60e95b787857afbc9a00b693b91c0d9c8923a430": + raise ToolchainError("source.commit is not the qualified Mesa release commit") + canonical_https(source["url"], "source.url") + if source["url"] != "https://archive.mesa3d.org/mesa-26.0.8.tar.xz": + raise ToolchainError("source.url does not match the governed Mesa archive") + locked_digest(source["sha256"], "source.sha256") + if source["sha256"] != "caf1c0061a68e88dfa74967a7e780c0e85d65b6c4e334cd69095a5dc54ad78bc": + raise ToolchainError("source.sha256 is not the qualified Mesa archive digest") + archive_root = safe_relative_path(source["archive_root"], "source.archive_root") + if archive_root != "mesa-26.0.8" or "/" in archive_root: + raise ToolchainError("source.archive_root is not the qualified archive root") + + build = value["build"] + if not isinstance(build, dict): + raise ToolchainError("manifest build must be an object") + require_keys(build, {"system", "version", "package_lock", "configure", "options"}, "build") + if build["system"] != "meson" or build["version"] != "1.10.1": + raise ToolchainError("manifest does not describe the pinned Meson build") + safe_relative_path(build["package_lock"], "build.package_lock") + if build["package_lock"] != "classic-vulkan-packages.lock": + raise ToolchainError("build.package_lock is not the governed package lock") + configure = build["configure"] + if not isinstance(configure, dict): + raise ToolchainError("build.configure must be an object") + require_keys(configure, {"buildtype", "libdir", "prefix", "wrap_mode"}, "build.configure") + if configure != { + "buildtype": "release", + "libdir": "lib/x86_64-linux-gnu", + "prefix": "/usr", + "wrap_mode": "nodownload", + }: + raise ToolchainError("build.configure is not the governed release configuration") + options = build["options"] + if not isinstance(options, dict) or options != expected_options(): + raise ToolchainError("build.options are not the governed narrow Dozen configuration") + + install = value["install"] + if not isinstance(install, dict): + raise ToolchainError("manifest install must be an object") + require_keys(install, {"library", "icd", "icd_library", "icd_api_version"}, "install") + if install != { + "library": "/usr/lib/x86_64-linux-gnu/libvulkan_dzn.so", + "icd": "/usr/share/vulkan/icd.d/dzn_icd.x86_64.json", + "icd_library": "libvulkan_dzn.so", + "icd_api_version": "1.1.335", + }: + raise ToolchainError("manifest install paths are not the governed Dozen artifacts") + safe_absolute_path(install["library"], "install.library") + safe_absolute_path(install["icd"], "install.icd") + safe_relative_path(install["icd_library"], "install.icd_library") + locked_text(install["icd_api_version"], "install.icd_api_version") + + runtime = value["runtime"] + if not isinstance(runtime, dict): + raise ToolchainError("manifest runtime must be an object") + require_keys(runtime, {"packages", "shared_library_sonames", "headless", "wslg"}, "runtime") + packages = runtime["packages"] + expected_packages = [ + {"name": "libdrm2", "version": "2.4.131-1"}, + {"name": "libudev1", "version": "259.5-0ubuntu3.3"}, + {"name": "libvulkan1", "version": "1.4.341.0-1"}, + {"name": "libwayland-client0", "version": "1.24.0-2"}, + {"name": "mesa-vulkan-drivers", "version": "26.0.3-1ubuntu1"}, + {"name": "vulkan-tools", "version": "1.4.341.0+dfsg1-1"}, + {"name": "zlib1g", "version": "1:1.3.dfsg+really1.3.1-1ubuntu3"}, + ] + if packages != expected_packages: + raise ToolchainError("runtime.packages are not the locked Vulkan runtime closure") + shared = runtime["shared_library_sonames"] + if shared != [ + "libc.so.6", + "libdrm.so.2", + "libgcc_s.so.1", + "libm.so.6", + "libudev.so.1", + "libwayland-client.so.0", + "libz.so.1", + ]: + raise ToolchainError("runtime.shared_library_sonames are incomplete") + + headless = runtime["headless"] + if not isinstance(headless, dict): + raise ToolchainError("runtime.headless must be an object") + require_keys(headless, {"vulkan_icd", "display_server", "commands"}, "runtime.headless") + if headless != { + "vulkan_icd": "/usr/share/vulkan/icd.d/lvp_icd.json", + "display_server": "Xvfb", + "commands": ["vulkaninfo", "Xvfb", "xvfb-run"], + }: + raise ToolchainError("runtime.headless is not the governed Lavapipe/Xvfb path") + safe_absolute_path(headless["vulkan_icd"], "runtime.headless.vulkan_icd") + + wslg = runtime["wslg"] + if not isinstance(wslg, dict): + raise ToolchainError("runtime.wslg must be an object") + require_keys( + wslg, + { + "vulkan_icd", + "required_mounts", + "required_host_libraries", + "fixed_environment", + "consumer_environment", + "probe", + }, + "runtime.wslg", + ) + if wslg["vulkan_icd"] != install["icd"]: + raise ToolchainError("runtime.wslg.vulkan_icd does not match install.icd") + if wslg["required_mounts"] != ["/dev/dxg", "/mnt/wslg/runtime-dir", "/usr/lib/wsl"]: + raise ToolchainError("runtime.wslg.required_mounts are incomplete") + if wslg["required_host_libraries"] != [ + "/usr/lib/wsl/lib/libd3d12.so", + "/usr/lib/wsl/lib/libdxcore.so", + ]: + raise ToolchainError("runtime.wslg.required_host_libraries are incomplete") + if wslg["fixed_environment"] != { + "DISPLAY": "", + "GALLIUM_DRIVER": "d3d12", + "LD_LIBRARY_PATH": "/usr/lib/wsl/lib", + "VK_DRIVER_FILES": "/usr/share/vulkan/icd.d/dzn_icd.x86_64.json", + "WAYLAND_DISPLAY": "wayland-0", + "XDG_RUNTIME_DIR": "/mnt/wslg/runtime-dir", + }: + raise ToolchainError("runtime.wslg.fixed_environment is incomplete") + if wslg["consumer_environment"] != ["MESA_D3D12_DEFAULT_ADAPTER_NAME"]: + raise ToolchainError("runtime.wslg.consumer_environment must remain consumer supplied") + if wslg["probe"] != ["vulkaninfo", "--summary"]: + raise ToolchainError("runtime.wslg.probe is unexpected") + for index, mount in enumerate(wslg["required_mounts"]): + safe_absolute_path(mount, f"runtime.wslg.required_mounts[{index}]") + for index, library in enumerate(wslg["required_host_libraries"]): + safe_absolute_path(library, f"runtime.wslg.required_host_libraries[{index}]") + return value + + +def download(url: str, expected: str, cache: Path) -> Path: + cache.mkdir(parents=True, exist_ok=True) + destination = cache / f"{expected}.tar.xz" + if destination.is_symlink(): + raise ToolchainError(f"Vulkan toolchain cache entry is a symlink: {destination}") + if destination.is_file() and destination.stat().st_size <= MAX_ARCHIVE_BYTES: + if sha256(destination) == expected: + return destination + elif destination.exists(): + raise ToolchainError(f"invalid Vulkan toolchain cache entry: {destination}") + + request = urllib.request.Request(url, headers={"User-Agent": "Atrinik Vulkan toolchain/1"}) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{expected}.", dir=cache) + temporary = Path(temporary_name) + try: + total = 0 + with os.fdopen(descriptor, "wb") as output, urllib.request.urlopen( + request, timeout=120 + ) as response: + final = urllib.parse.urlsplit(response.geturl()) + if final.scheme != "https": + raise ToolchainError("Vulkan toolchain download left HTTPS") + while block := response.read(1024 * 1024): + total += len(block) + if total > MAX_ARCHIVE_BYTES: + raise ToolchainError("Vulkan toolchain archive exceeds size limit") + output.write(block) + if sha256(temporary) != expected: + raise ToolchainError("Vulkan toolchain archive digest mismatch") + temporary.replace(destination) + except Exception: + temporary.unlink(missing_ok=True) + raise + return destination + + +def safe_archive_path(name: str) -> PurePosixPath: + if not name or "\x00" in name or "\\" in name: + raise ToolchainError(f"unsafe Mesa archive member: {name}") + path = PurePosixPath(name) + if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts): + raise ToolchainError(f"unsafe Mesa archive member: {name}") + return path + + +def normalize_link_target(member: PurePosixPath, linkname: str, root: str) -> str: + if not linkname or "\x00" in linkname or "\\" in linkname: + raise ToolchainError(f"unsafe Mesa archive link target: {linkname}") + target = PurePosixPath(linkname) + if target.is_absolute(): + raise ToolchainError(f"unsafe Mesa archive link target: {linkname}") + stack = list(member.parent.parts) + for part in target.parts: + if part in {"", "."}: + continue + if part == "..": + if not stack: + raise ToolchainError(f"Mesa archive link escapes source root: {linkname}") + stack.pop() + else: + stack.append(part) + if not stack or stack[0] != root: + raise ToolchainError(f"Mesa archive link escapes source root: {linkname}") + return "/".join(stack[1:]) + + +def copy_member(source: BinaryIO, destination: Path, size: int) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open("xb") as output: + remaining = size + while remaining: + block = source.read(min(1024 * 1024, remaining)) + if not block: + raise ToolchainError(f"truncated Mesa archive member: {destination}") + output.write(block) + remaining -= len(block) + + +def extract_source(archive_path: Path, destination: Path, entry: dict[str, object]) -> Path: + root = str(entry["archive_root"]) + records: set[str] = set() + expanded = 0 + count = 0 + with tarfile.open(archive_path, mode="r:xz") as archive: + for member in archive: + count += 1 + if count > MAX_MEMBERS: + raise ToolchainError("Mesa archive has too many members") + path = safe_archive_path(member.name) + if path.parts[0] != root: + raise ToolchainError(f"Mesa archive has an unexpected root: {member.name}") + key = path.as_posix().casefold() + if key in records: + raise ToolchainError(f"duplicate Mesa archive path: {member.name}") + if member.isdir(): + pass + elif member.isfile(): + if member.size > MAX_FILE_BYTES: + raise ToolchainError(f"Mesa archive member is too large: {member.name}") + expanded += member.size + if expanded > MAX_EXPANDED_BYTES: + raise ToolchainError("Mesa archive exceeds expanded size limit") + elif member.issym(): + normalize_link_target(path, member.linkname, root) + else: + raise ToolchainError(f"unsupported Mesa archive member: {member.name}") + records.add(key) + + source_root = destination + source_root.mkdir(parents=True, exist_ok=True) + directories: list[PurePosixPath] = [] + symlinks: list[tuple[PurePosixPath, str]] = [] + with tarfile.open(archive_path, mode="r:xz") as archive: + for member in archive: + path = safe_archive_path(member.name) + relative = PurePosixPath(*path.parts[1:]) + if member.isdir(): + if relative.parts: + directories.append(relative) + elif member.issym(): + symlinks.append((relative, member.linkname)) + + for relative in sorted(directories, key=lambda item: len(item.parts)): + (source_root / relative).mkdir(parents=True, exist_ok=False) + for member in archive.getmembers(): + if not member.isfile(): + continue + relative = PurePosixPath(*safe_archive_path(member.name).parts[1:]) + output = source_root / relative + source = archive.extractfile(member) + if source is None: + raise ToolchainError(f"cannot read Mesa archive member: {member.name}") + with source: + copy_member(source, output, member.size) + output.chmod(stat.S_IMODE(member.mode) & 0o755 or 0o644) + for relative, linkname in symlinks: + output = source_root / relative + output.parent.mkdir(parents=True, exist_ok=True) + os.symlink(linkname, output) + + version_path = source_root / "VERSION" + if version_path.is_symlink() or not version_path.is_file(): + raise ToolchainError("Mesa archive has no regular VERSION file") + version = version_path.read_text(encoding="utf-8").strip() + if version != str(entry["version"]): + raise ToolchainError(f"Mesa source VERSION is {version!r}, expected {entry['version']!r}") + return source_root + + +def option_value(value: object) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, list) and all(isinstance(item, str) for item in value): + return ",".join(value) + if isinstance(value, str): + return value + raise ToolchainError(f"unsupported Meson option value: {value!r}") + + +def run_build( + source: Path, + build: Path, + staging: Path, + manifest: dict[str, object], + jobs: int, +) -> None: + build_contract = manifest["build"] + assert isinstance(build_contract, dict) + configure = build_contract["configure"] + options = build_contract["options"] + assert isinstance(configure, dict) and isinstance(options, dict) + meson = shutil.which("meson") + if meson is None: + raise ToolchainError("meson is unavailable") + version = subprocess.check_output([meson, "--version"], text=True).strip() + if version != build_contract["version"]: + raise ToolchainError(f"Meson is {version}, expected {build_contract['version']}") + option_args = [ + f"-D{name}={option_value(options[name])}" for name in sorted(options) + ] + subprocess.run( + [ + meson, + "setup", + str(build), + str(source), + f"--buildtype={configure['buildtype']}", + f"--prefix={configure['prefix']}", + f"--libdir={configure['libdir']}", + f"--wrap-mode={configure['wrap_mode']}", + *option_args, + ], + check=True, + ) + subprocess.run([meson, "compile", "-C", str(build), "--jobs", str(jobs)], check=True) + subprocess.run( + [meson, "install", "-C", str(build), "--destdir", str(staging)], + check=True, + ) + + +def install_file(source: Path, destination: Path, executable: bool) -> None: + if source.is_symlink() or not source.is_file(): + raise ToolchainError(f"Mesa staged artifact is not a regular file: {source}") + if destination.is_symlink(): + raise ToolchainError(f"Mesa destination is a symlink: {destination}") + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(f".{destination.name}.tmp") + temporary.unlink(missing_ok=True) + shutil.copyfile(source, temporary) + temporary.chmod(0o755 if executable else 0o644) + temporary.replace(destination) + + +def staged_path(staging: Path, absolute: str) -> Path: + return staging / PurePosixPath(absolute).relative_to("/") + + +def install_runtime_files(manifest: dict[str, object], staging: Path, root: Path = Path("/")) -> None: + install = manifest["install"] + assert isinstance(install, dict) + library = str(install["library"]) + icd = str(install["icd"]) + destination_library = root / PurePosixPath(library).relative_to("/") + destination_icd = root / PurePosixPath(icd).relative_to("/") + install_file(staged_path(staging, library), destination_library, True) + install_file(staged_path(staging, icd), destination_icd, False) + + +def validate_installed(manifest: dict[str, object], root: Path = Path("/")) -> None: + install = manifest["install"] + assert isinstance(install, dict) + library_path = root / PurePosixPath(str(install["library"])).relative_to("/") + icd_path = root / PurePosixPath(str(install["icd"])).relative_to("/") + if library_path.is_symlink() or not library_path.is_file(): + raise ToolchainError(f"installed Dozen library is missing: {library_path}") + value = load_json(icd_path, "installed Dozen ICD") + if not isinstance(value, dict): + raise ToolchainError("installed Dozen ICD root must be an object") + require_keys(value, {"ICD", "file_format_version"}, "installed Dozen ICD") + icd = value["ICD"] + if not isinstance(icd, dict): + raise ToolchainError("installed Dozen ICD entry must be an object") + require_keys(icd, {"api_version", "library_arch", "library_path"}, "installed Dozen ICD entry") + if value["file_format_version"] != "1.0.1": + raise ToolchainError("installed Dozen ICD format is unexpected") + if icd["api_version"] != install["icd_api_version"]: + raise ToolchainError("installed Dozen ICD API version is unexpected") + if icd["library_arch"] != "64" or icd["library_path"] != install["library"]: + raise ToolchainError("installed Dozen ICD does not point to libvulkan_dzn.so") + + +def install(manifest_path: Path, cache: Path, prefix: Path, jobs: int) -> None: + manifest = load_manifest(manifest_path.resolve(strict=True)) + if platform.system() != "Linux" or platform.machine() not in {"x86_64", "AMD64"}: + raise ToolchainError("the governed Vulkan toolchain targets x86-64 Linux") + if jobs < 1 or jobs > 256: + raise ToolchainError("jobs must be between 1 and 256") + if prefix.resolve() != Path("/usr"): + raise ToolchainError("the governed Dozen install prefix is /usr") + cache = cache.resolve() + if cache == Path("/"): + raise ToolchainError("Vulkan toolchain cache must not be the filesystem root") + source = manifest["source"] + assert isinstance(source, dict) + archive = download(str(source["url"]), str(source["sha256"]), cache) + staging_work = Path(tempfile.mkdtemp(prefix=".classic-vulkan-toolchain-", dir=cache.parent)) + try: + source_root = extract_source(archive, staging_work / "source", source) + build = staging_work / "build" + staging = staging_work / "stage" + run_build(source_root, build, staging, manifest, jobs) + install_runtime_files(manifest, staging) + validate_installed(manifest) + finally: + shutil.rmtree(staging_work, ignore_errors=True) + + +def main(argv: Iterable[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--cache", type=Path, required=True) + parser.add_argument("--prefix", type=Path, required=True) + parser.add_argument("--jobs", type=int, required=True) + arguments = parser.parse_args(argv) + try: + install(arguments.manifest, arguments.cache, arguments.prefix, arguments.jobs) + except (OSError, subprocess.CalledProcessError, ToolchainError) as error: + parser.error(str(error)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/tests/test_install_classic_vulkan_toolchain.py b/tools/tests/test_install_classic_vulkan_toolchain.py new file mode 100644 index 0000000..a74386c --- /dev/null +++ b/tools/tests/test_install_classic_vulkan_toolchain.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import importlib.util +import io +import json +from pathlib import Path +import tarfile +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "install_classic_vulkan_toolchain", + ROOT / "tools" / "install_classic_vulkan_toolchain.py", +) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def add_file(archive: tarfile.TarFile, name: str, content: bytes, mode: int = 0o644) -> None: + info = tarfile.TarInfo(name) + info.size = len(content) + info.mode = mode + archive.addfile(info, io.BytesIO(content)) + + +class ClassicVulkanToolchainTests(unittest.TestCase): + def test_loads_the_qualified_manifest(self) -> None: + manifest = MODULE.load_manifest(ROOT / "classic-vulkan-toolchain.json") + self.assertEqual( + manifest["source"]["commit"], # type: ignore[index] + "60e95b787857afbc9a00b693b91c0d9c8923a430", + ) + self.assertEqual( # type: ignore[index] + manifest["install"]["icd"], + "/usr/share/vulkan/icd.d/dzn_icd.x86_64.json", + ) + + def test_rejects_a_different_mesa_archive_digest(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "manifest.json" + value = (ROOT / "classic-vulkan-toolchain.json").read_text(encoding="utf-8") + path.write_text( + value.replace( + "caf1c0061a68e88dfa74967a7e780c0e85d65b6c4e334cd69095a5dc54ad78bc", + "f" * 64, + 1, + ), + encoding="utf-8", + ) + with self.assertRaisesRegex(MODULE.ToolchainError, "archive digest"): + MODULE.load_manifest(path) + + def test_rejects_a_different_build_package_lock(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "manifest.json" + value = (ROOT / "classic-vulkan-toolchain.json").read_text(encoding="utf-8") + path.write_text( + value.replace("classic-vulkan-packages.lock", "other-packages.lock", 1), + encoding="utf-8", + ) + with self.assertRaisesRegex(MODULE.ToolchainError, "package lock"): + MODULE.load_manifest(path) + + def test_rejects_duplicate_json_keys(self) -> None: + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as stream: + stream.write('{"schema_version": 1, "schema_version": 1}\n') + stream.flush() + with self.assertRaisesRegex(MODULE.ToolchainError, "duplicate"): + MODULE.load_manifest(Path(stream.name)) + + def test_rejects_unsafe_archive_paths_and_links(self) -> None: + for value in ("/etc/passwd", "root/../outside", "root\\outside", "root/\x00bad"): + with self.subTest(value=value), self.assertRaisesRegex( + MODULE.ToolchainError, "unsafe Mesa archive member" + ): + MODULE.safe_archive_path(value) + with self.assertRaisesRegex(MODULE.ToolchainError, "escapes source root"): + MODULE.normalize_link_target( + MODULE.safe_archive_path("mesa-26.0.8/link"), "../../outside", "mesa-26.0.8" + ) + + def test_extracts_the_official_archive_shape_with_safe_links(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + archive_path = Path(temporary) / "mesa.tar.xz" + with tarfile.open(archive_path, "w:xz") as archive: + root = tarfile.TarInfo("mesa-26.0.8") + root.type = tarfile.DIRTYPE + archive.addfile(root) + directory = tarfile.TarInfo("mesa-26.0.8/src") + directory.type = tarfile.DIRTYPE + archive.addfile(directory) + add_file(archive, "mesa-26.0.8/VERSION", b"26.0.8\n") + add_file(archive, "mesa-26.0.8/src/real.c", b"int main(void) {}\n") + link = tarfile.TarInfo("mesa-26.0.8/src/alias.c") + link.type = tarfile.SYMTYPE + link.linkname = "real.c" + archive.addfile(link) + add_file(archive, "mesa-26.0.8/ignored.txt", b"ignored") + destination = Path(temporary) / "source" + MODULE.extract_source( + archive_path, + destination, + {"archive_root": "mesa-26.0.8", "version": "26.0.8"}, + ) + self.assertEqual((destination / "VERSION").read_text(), "26.0.8\n") + self.assertTrue((destination / "src/alias.c").is_symlink()) + self.assertEqual((destination / "src/alias.c").read_text(), "int main(void) {}\n") + + def test_installs_and_validates_only_declared_runtime_files(self) -> None: + manifest = MODULE.load_manifest(ROOT / "classic-vulkan-toolchain.json") + install = manifest["install"] + assert isinstance(install, dict) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "root" + stage = Path(temporary) / "stage" + library = stage / "usr/lib/x86_64-linux-gnu/libvulkan_dzn.so" + icd = stage / "usr/share/vulkan/icd.d/dzn_icd.x86_64.json" + library.parent.mkdir(parents=True) + icd.parent.mkdir(parents=True) + library.write_bytes(b"driver") + icd.write_text( + json.dumps( + { + "ICD": { + "api_version": install["icd_api_version"], + "library_arch": "64", + "library_path": install["library"], + }, + "file_format_version": "1.0.1", + } + ), + encoding="utf-8", + ) + MODULE.install_runtime_files(manifest, stage, root) + MODULE.validate_installed(manifest, root) + self.assertEqual( + (root / "usr/lib/x86_64-linux-gnu/libvulkan_dzn.so").read_bytes(), + b"driver", + ) + self.assertFalse((root / "usr/bin/spirv2dxil").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/validate-vulkan-toolchain.sh b/tools/validate-vulkan-toolchain.sh new file mode 100755 index 0000000..6c65b9c --- /dev/null +++ b/tools/validate-vulkan-toolchain.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 4 && $# -ne 5 ]]; then + echo "usage: $0 EXPECTED INSTALLED SPDX_EXPECTED SPDX_INSTALLED [PACKAGE_LOCK]" >&2 + exit 2 +fi + +expected=$1 +installed=$2 +spdx_expected=$3 +spdx_installed=$4 +package_lock=${5:-} + +for command in cmp jq readelf sha256sum vulkaninfo xvfb-run; do + command -v "${command}" >/dev/null || { + echo "required command is unavailable: ${command}" >&2 + exit 1 + } +done + +cmp --silent "${expected}" "${installed}" +cmp --silent "${spdx_expected}" "${spdx_installed}" + +jq -e ' + .schema_version == 1 + and .platform == "linux/amd64" + and .target == "classic-final" + and .image == "ghcr.io/atrinik/classic-build" + and .base == { + "image": "ubuntu:26.04", + "digest": "sha256:678c6550cc43645e08669028bc177f50be4e7c5b8cca677067b1914d4afc7a03", + "apt_snapshot": "20260810T000000Z" + } + and .source.repository == "mesa/mesa" + and .source.tag == "mesa-26.0.8" + and .source.version == "26.0.8" + and .source.commit == "60e95b787857afbc9a00b693b91c0d9c8923a430" + and .source.url == "https://archive.mesa3d.org/mesa-26.0.8.tar.xz" + and .source.sha256 == "caf1c0061a68e88dfa74967a7e780c0e85d65b6c4e334cd69095a5dc54ad78bc" + and .source.archive_root == "mesa-26.0.8" + and .build.system == "meson" + and .build.version == "1.10.1" + and .build.package_lock == "classic-vulkan-packages.lock" + and .build.configure == { + "buildtype": "release", + "libdir": "lib/x86_64-linux-gnu", + "prefix": "/usr", + "wrap_mode": "nodownload" + } + and .build.options.platforms == ["wayland"] + and .build.options["gallium-drivers"] == ["d3d12"] + and .build.options["vulkan-drivers"] == ["microsoft-experimental"] + and .build.options["gallium-d3d12-graphics"] == "enabled" + and .build.options["gallium-d3d12-video"] == "disabled" + and .build.options.opengl == false + and .build.options.llvm == "disabled" + and .build.options["spirv-tools"] == "disabled" + and .build.options["vulkan-manifest-per-architecture"] == true + and .install == { + "library": "/usr/lib/x86_64-linux-gnu/libvulkan_dzn.so", + "icd": "/usr/share/vulkan/icd.d/dzn_icd.x86_64.json", + "icd_library": "libvulkan_dzn.so", + "icd_api_version": "1.1.335" + } + and .runtime.packages == [ + {"name": "libdrm2", "version": "2.4.131-1"}, + {"name": "libudev1", "version": "259.5-0ubuntu3.3"}, + {"name": "libvulkan1", "version": "1.4.341.0-1"}, + {"name": "libwayland-client0", "version": "1.24.0-2"}, + {"name": "mesa-vulkan-drivers", "version": "26.0.3-1ubuntu1"}, + {"name": "vulkan-tools", "version": "1.4.341.0+dfsg1-1"}, + {"name": "zlib1g", "version": "1:1.3.dfsg+really1.3.1-1ubuntu3"} + ] + and .runtime.shared_library_sonames == [ + "libc.so.6", "libdrm.so.2", "libgcc_s.so.1", "libm.so.6", + "libudev.so.1", "libwayland-client.so.0", "libz.so.1" + ] + and .runtime.headless == { + "vulkan_icd": "/usr/share/vulkan/icd.d/lvp_icd.json", + "display_server": "Xvfb", + "commands": ["vulkaninfo", "Xvfb", "xvfb-run"] + } + and .runtime.wslg.vulkan_icd == .install.icd + and .runtime.wslg.required_mounts == [ + "/dev/dxg", "/mnt/wslg/runtime-dir", "/usr/lib/wsl" + ] + and .runtime.wslg.required_host_libraries == [ + "/usr/lib/wsl/lib/libd3d12.so", "/usr/lib/wsl/lib/libdxcore.so" + ] + and .runtime.wslg.consumer_environment == ["MESA_D3D12_DEFAULT_ADAPTER_NAME"] + and .runtime.wslg.probe == ["vulkaninfo", "--summary"] +' "${expected}" >/dev/null + +jq -e --slurpfile manifest "${expected}" ' + .spdxVersion == "SPDX-2.3" + and .dataLicense == "CC0-1.0" + and .SPDXID == "SPDXRef-DOCUMENT" + and ([.packages[].name] == ["Mesa"]) + and ([.packages[].SPDXID] | unique | length) == 1 + and .packages[0].filesAnalyzed == false + and .packages[0].versionInfo == $manifest[0].source.version + and .packages[0].downloadLocation == $manifest[0].source.url + and .packages[0].checksums == [{ + "algorithm": "SHA256", + "checksumValue": $manifest[0].source.sha256 + }] + and .packages[0].licenseConcluded == "NOASSERTION" + and .packages[0].licenseDeclared == "NOASSERTION" + and .relationships == [{ + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-Package-Mesa" + }] +' "${spdx_expected}" >/dev/null + +library=$(jq -er '.install.library' "${expected}") +icd=$(jq -er '.install.icd' "${expected}") +headless_icd=$(jq -er '.runtime.headless.vulkan_icd' "${expected}") +test -f "${library}" +test ! -L "${library}" +test -f "${icd}" +test -f "${headless_icd}" + +jq -e --arg library "${library}" \ + --arg api_version "$(jq -er '.install.icd_api_version' "${expected}")" ' + .file_format_version == "1.0.1" + and .ICD.api_version == $api_version + and .ICD.library_arch == "64" + and .ICD.library_path == $library +' "${icd}" >/dev/null + +while IFS= read -r soname; do + readelf -d "${library}" | grep -Fq "Shared library: [${soname}]" +done < <(jq -er '.runtime.shared_library_sonames[]' "${expected}") + +while IFS=$'\t' read -r package version; do + test "$(dpkg-query --show --showformat='${Version}' "${package}")" = "${version}" +done < <(jq -er '.runtime.packages[] | [.name, .version] | @tsv' "${expected}") + +if [[ -n ${package_lock} ]]; then + while IFS='=' read -r package version; do + if [[ -z ${package} || -z ${version} ]]; then + echo "invalid package lock entry" >&2 + exit 1 + fi + test "$(dpkg-query --show --showformat='${Version}' "${package}")" = "${version}" + done < "${package_lock}" +fi + +while IFS= read -r command; do + command -v "${command}" >/dev/null +done < <(jq -er '.runtime.headless.commands[]' "${expected}") + +VK_DRIVER_FILES="${headless_icd}" xvfb-run -a vulkaninfo --summary >/dev/null + +wslg_ready=true +while IFS= read -r path; do + if [[ ! -e ${path} ]]; then + wslg_ready=false + fi +done < <(jq -er '.runtime.wslg.required_mounts[]' "${expected}") +while IFS= read -r path; do + if [[ ! -e ${path} ]]; then + wslg_ready=false + fi +done < <(jq -er '.runtime.wslg.required_host_libraries[]' "${expected}") + +if [[ ${wslg_ready} == true && -n ${MESA_D3D12_DEFAULT_ADAPTER_NAME:-} ]]; then + display=$(jq -er '.runtime.wslg.fixed_environment.DISPLAY' "${expected}") + wayland_display=$(jq -er '.runtime.wslg.fixed_environment.WAYLAND_DISPLAY' "${expected}") + runtime_dir=$(jq -er '.runtime.wslg.fixed_environment.XDG_RUNTIME_DIR' "${expected}") + library_path=$(jq -er '.runtime.wslg.fixed_environment.LD_LIBRARY_PATH' "${expected}") + gallium_driver=$(jq -er '.runtime.wslg.fixed_environment.GALLIUM_DRIVER' "${expected}") + wslg_icd=$(jq -er '.runtime.wslg.vulkan_icd' "${expected}") + probe_output=$(mktemp) + trap 'rm -f -- "${probe_output}"' EXIT + DISPLAY="${display}" \ + WAYLAND_DISPLAY="${wayland_display}" \ + XDG_RUNTIME_DIR="${runtime_dir}" \ + LD_LIBRARY_PATH="${library_path}" \ + GALLIUM_DRIVER="${gallium_driver}" \ + MESA_D3D12_DEFAULT_ADAPTER_NAME="${MESA_D3D12_DEFAULT_ADAPTER_NAME}" \ + VK_DRIVER_FILES="${wslg_icd}" \ + vulkaninfo --summary | tee "${probe_output}" + grep -Fq "Microsoft Direct3D12" "${probe_output}" + grep -Fq "Mesa Dozen" "${probe_output}" +else + echo "WSLg Dozen probe skipped: WSLg mounts/libraries or consumer adapter selection are unavailable" >&2 +fi From 5e544ece97dbbe490f387c5a100711837e8ba2c0 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 31 Aug 2026 22:46:04 +0000 Subject: [PATCH 2/3] fix(devcontainer): retry pinned Ubuntu snapshot fetches --- linux/Dockerfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/linux/Dockerfile b/linux/Dockerfile index 5d08b54..b2e07b1 100644 --- a/linux/Dockerfile +++ b/linux/Dockerfile @@ -14,6 +14,8 @@ COPY classic-vulkan-packages.lock /tmp/classic-vulkan-packages.lock # signed snapshot, then repeat the update with normal HTTPS verification. RUN test "${TARGETARCH}" = amd64 \ && test "$(dpkg --print-architecture)" = amd64 \ + && printf '%s\n' 'Acquire::Retries "5";' \ + > /etc/apt/apt.conf.d/80-snapshot-retries \ && sed -i \ -e "s|http://archive.ubuntu.com/ubuntu/|https://snapshot.ubuntu.com/ubuntu/${UBUNTU_SNAPSHOT}/|" \ -e "s|http://security.ubuntu.com/ubuntu/|https://snapshot.ubuntu.com/ubuntu/${UBUNTU_SNAPSHOT}/|" \ @@ -35,6 +37,7 @@ RUN test "${TARGETARCH}" = amd64 \ && while IFS='=' read -r package version; do \ test "$(dpkg-query --show --showformat='${Version}' "${package}")" = "${version}"; \ done < /tmp/classic-vulkan-packages.lock \ + && rm -f /etc/apt/apt.conf.d/80-snapshot-retries \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*.deb COPY classic-vulkan-toolchain.json /tmp/classic-vulkan-toolchain.json @@ -71,6 +74,9 @@ RUN sed -i \ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ + printf '%s\n' 'Acquire::Retries "5";' \ + > /etc/apt/apt.conf.d/80-snapshot-retries \ + && \ apt-get -o Acquire::https::Verify-Peer=false update \ && apt-get -o Acquire::https::Verify-Peer=false install -y --no-install-recommends \ ca-certificates \ @@ -143,7 +149,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ xz-utils \ zip \ zlib1g=1:1.3.dfsg+really1.3.1-1ubuntu3 \ - zlib1g-dev=1:1.3.dfsg+really1.3.1-1ubuntu3 + zlib1g-dev=1:1.3.dfsg+really1.3.1-1ubuntu3 \ + && rm -f /etc/apt/apt.conf.d/80-snapshot-retries COPY --from=mesa-dozen \ /usr/lib/x86_64-linux-gnu/libvulkan_dzn.so \ @@ -466,6 +473,8 @@ COPY classic-packages.lock /tmp/classic-packages.lock # peer verification is temporarily unavailable in the minimal base. RUN test "${TARGETARCH}" = amd64 \ && test "$(dpkg --print-architecture)" = amd64 \ + && printf '%s\n' 'Acquire::Retries "5";' \ + > /etc/apt/apt.conf.d/80-snapshot-retries \ && sed -i \ -e "s|http://archive.ubuntu.com/ubuntu/|https://snapshot.ubuntu.com/ubuntu/${UBUNTU_SNAPSHOT}/|" \ -e "s|http://security.ubuntu.com/ubuntu/|https://snapshot.ubuntu.com/ubuntu/${UBUNTU_SNAPSHOT}/|" \ @@ -484,6 +493,7 @@ RUN test "${TARGETARCH}" = amd64 \ && apt-get update \ && xargs -r apt-get install -y --no-install-recommends \ < /tmp/classic-packages.lock \ + && rm -f /etc/apt/apt.conf.d/80-snapshot-retries \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*.deb COPY --from=mesa-dozen \ From ab617672ce05fcb5f772fe9c7d81e0a8cc5ed844 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Tue, 1 Sep 2026 00:25:06 +0000 Subject: [PATCH 3/3] ci(devcontainer): tolerate pinned snapshot slowness --- .github/workflows/publish-linux.yml | 2 +- .github/workflows/validate.yml | 4 +- linux/Dockerfile | 84 ++++++++++++++++++++++++----- 3 files changed, 73 insertions(+), 17 deletions(-) diff --git a/.github/workflows/publish-linux.yml b/.github/workflows/publish-linux.yml index 65aab9c..b2738aa 100644 --- a/.github/workflows/publish-linux.yml +++ b/.github/workflows/publish-linux.yml @@ -23,7 +23,7 @@ concurrency: jobs: publish: runs-on: ubuntu-26.04 - timeout-minutes: 60 + timeout-minutes: 180 steps: - name: Check out image sources uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 59e837f..1afe64a 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -117,7 +117,7 @@ jobs: needs: changes if: needs.changes.outputs.linux == 'true' runs-on: ubuntu-26.04 - timeout-minutes: 60 + timeout-minutes: 180 permissions: attestations: read contents: read @@ -350,7 +350,7 @@ jobs: needs: changes if: needs.changes.outputs.classic == 'true' runs-on: ubuntu-26.04 - timeout-minutes: 60 + timeout-minutes: 180 steps: - name: Check out image sources uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/linux/Dockerfile b/linux/Dockerfile index b2e07b1..9a38be8 100644 --- a/linux/Dockerfile +++ b/linux/Dockerfile @@ -12,8 +12,23 @@ ENV DEBIAN_FRONTEND=noninteractive \ COPY classic-vulkan-packages.lock /tmp/classic-vulkan-packages.lock # The minimal base has no CA bundle. Bootstrap the exact TLS closure over the # signed snapshot, then repeat the update with normal HTTPS verification. +# Snapshot package downloads can still fail after per-request retries. Repeat +# the transaction so successfully fetched archives can be reused. RUN test "${TARGETARCH}" = amd64 \ && test "$(dpkg --print-architecture)" = amd64 \ + && retry_command() { \ + attempt=1; \ + while test "${attempt}" -le 2; do \ + if "$@"; then \ + return 0; \ + fi; \ + if test "${attempt}" -eq 2; then \ + return 1; \ + fi; \ + attempt=$((attempt + 1)); \ + sleep 5; \ + done; \ + } \ && printf '%s\n' 'Acquire::Retries "5";' \ > /etc/apt/apt.conf.d/80-snapshot-retries \ && sed -i \ @@ -22,18 +37,23 @@ RUN test "${TARGETARCH}" = amd64 \ /etc/apt/sources.list.d/ubuntu.sources \ && sed -i '/^Signed-By:/a Check-Valid-Until: no' \ /etc/apt/sources.list.d/ubuntu.sources \ - && apt-get -o Acquire::https::Verify-Peer=false update \ + && retry_command apt-get -o Acquire::https::Verify-Peer=false update \ && awk -F= '$1 == "ca-certificates" \ || $1 == "libssl3t64" \ || $1 == "openssl" \ || $1 == "openssl-provider-legacy" { print }' \ /tmp/classic-vulkan-packages.lock \ - | xargs -r apt-get -o Acquire::https::Verify-Peer=false \ + > /tmp/classic-vulkan-bootstrap.lock \ + && retry_command sh -c \ + 'xargs -r apt-get -o Acquire::https::Verify-Peer=false \ install -y --no-install-recommends \ + < /tmp/classic-vulkan-bootstrap.lock' \ + && rm -f /tmp/classic-vulkan-bootstrap.lock \ && rm -rf /var/lib/apt/lists/* \ - && apt-get update \ - && xargs -r apt-get install -y --no-install-recommends \ - < /tmp/classic-vulkan-packages.lock \ + && retry_command apt-get update \ + && retry_command sh -c \ + 'xargs -r apt-get install -y --no-install-recommends \ + < /tmp/classic-vulkan-packages.lock' \ && while IFS='=' read -r package version; do \ test "$(dpkg-query --show --showformat='${Version}' "${package}")" = "${version}"; \ done < /tmp/classic-vulkan-packages.lock \ @@ -72,19 +92,35 @@ RUN sed -i \ && sed -i '/^Signed-By:/a Check-Valid-Until: no' \ /etc/apt/sources.list.d/ubuntu.sources +# Snapshot package downloads can still fail after per-request retries. Repeat +# the transaction so successfully fetched archives can be reused. RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ + retry_command() { \ + attempt=1; \ + while test "${attempt}" -le 2; do \ + if "$@"; then \ + return 0; \ + fi; \ + if test "${attempt}" -eq 2; then \ + return 1; \ + fi; \ + attempt=$((attempt + 1)); \ + sleep 5; \ + done; \ + } \ + && \ printf '%s\n' 'Acquire::Retries "5";' \ > /etc/apt/apt.conf.d/80-snapshot-retries \ && \ - apt-get -o Acquire::https::Verify-Peer=false update \ - && apt-get -o Acquire::https::Verify-Peer=false install -y --no-install-recommends \ + retry_command apt-get -o Acquire::https::Verify-Peer=false update \ + && retry_command apt-get -o Acquire::https::Verify-Peer=false install -y --no-install-recommends \ ca-certificates \ libssl3t64 \ openssl \ openssl-provider-legacy \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ + && retry_command apt-get update \ + && retry_command apt-get install -y --no-install-recommends \ alsa-utils \ build-essential \ ca-certificates \ @@ -471,8 +507,23 @@ ENV DEBIAN_FRONTEND=noninteractive \ COPY classic-packages.lock /tmp/classic-packages.lock # APT still verifies the signed Release metadata and package hashes. Only TLS # peer verification is temporarily unavailable in the minimal base. +# Snapshot package downloads can still fail after per-request retries. Repeat +# the transaction so successfully fetched archives can be reused. RUN test "${TARGETARCH}" = amd64 \ && test "$(dpkg --print-architecture)" = amd64 \ + && retry_command() { \ + attempt=1; \ + while test "${attempt}" -le 2; do \ + if "$@"; then \ + return 0; \ + fi; \ + if test "${attempt}" -eq 2; then \ + return 1; \ + fi; \ + attempt=$((attempt + 1)); \ + sleep 5; \ + done; \ + } \ && printf '%s\n' 'Acquire::Retries "5";' \ > /etc/apt/apt.conf.d/80-snapshot-retries \ && sed -i \ @@ -481,18 +532,23 @@ RUN test "${TARGETARCH}" = amd64 \ /etc/apt/sources.list.d/ubuntu.sources \ && sed -i '/^Signed-By:/a Check-Valid-Until: no' \ /etc/apt/sources.list.d/ubuntu.sources \ - && apt-get -o Acquire::https::Verify-Peer=false update \ + && retry_command apt-get -o Acquire::https::Verify-Peer=false update \ && awk -F= '$1 == "ca-certificates" \ || $1 == "libssl3t64" \ || $1 == "openssl" \ || $1 == "openssl-provider-legacy" { print }' \ /tmp/classic-packages.lock \ - | xargs -r apt-get -o Acquire::https::Verify-Peer=false \ + > /tmp/classic-bootstrap.lock \ + && retry_command sh -c \ + 'xargs -r apt-get -o Acquire::https::Verify-Peer=false \ install -y --no-install-recommends \ + < /tmp/classic-bootstrap.lock' \ + && rm -f /tmp/classic-bootstrap.lock \ && rm -rf /var/lib/apt/lists/* \ - && apt-get update \ - && xargs -r apt-get install -y --no-install-recommends \ - < /tmp/classic-packages.lock \ + && retry_command apt-get update \ + && retry_command sh -c \ + 'xargs -r apt-get install -y --no-install-recommends \ + < /tmp/classic-packages.lock' \ && rm -f /etc/apt/apt.conf.d/80-snapshot-retries \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*.deb