build.sh: translate workspace bind mounts for a shared Docker daemon (opt-in)

.github/workflows/build.yml

@@ -74,15 +74,89 @@ jobs:
           echo "Found valid targets: $TARGETS_OUTPUT"
           echo "Found valid versions: $VERSIONS_OUTPUT"
 
+  # Build the kernel_builder image exactly once per run, push it to Harbor
+  # under an immutable per-commit tag, and let every matrix build job pull it.
+  #
+  # Why a dedicated job (instead of building inside each matrix job):
+  #   * The image is now target-agnostic (TARGET=latest bundles every
+  #     cross-toolchain), so all 13 build jobs would otherwise rebuild the SAME
+  #     multi-GB image. setup-buildx-action's docker-container driver keeps its
+  #     own cache store, separate from the shared per-node Docker daemon, so the
+  #     daemon's warm-image reuse does NOT cover a buildx `FROM` base -- every
+  #     job re-pulled the full all-arch base. Building once here fixes that.
+  #   * A single writer to the `:all_cache` registry cache means the cache
+  #     actually accumulates run-to-run instead of 13 jobs racing to overwrite
+  #     one ref.
+  #   * An immutable `:<sha>` tag (vs a mutable `:latest`) is safe on the shared
+  #     daemon's single image store -- concurrent jobs/PRs can't clobber each
+  #     other's tag (same lesson as rehosting/penguin #893).
+  build-image:
+    runs-on: rehosting-arc
+    if: github.event.pull_request.draft == false
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Trust Harbor's self-signed certificate
+        run: |
+          echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
+          openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
+          sudo update-ca-certificates
+
+      - name: Log in to Rehosting Arc Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{secrets.REHOSTING_ARC_REGISTRY}}
+          username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
+          password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          # See the note in the (removed) per-job build step / penguin c35bedc5:
+          # don't pin moby/buildkit:master -- it regressed on the kernel-5.4
+          # runners (can't mask /proc/acpi). Default pinned-stable buildkit + the
+          # insecure Harbor registry config.
+          driver-opts: |
+            network=host
+          buildkitd-config-inline: |
+            [registry."${{ secrets.REHOSTING_ARC_REGISTRY }}"]
+              insecure = true
+              http = true
+
+      - name: Build and push kernel_builder image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          # Immutable per-commit tag -- matrix jobs pull this exact ref.
+          tags: |
+            ${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/linux_builder:${{ github.sha }}
+          # TARGET=latest builds FROM the all-arch embedded-toolchains:latest so
+          # the single built image carries every cross-toolchain and one image
+          # serves every matrix target.
+          build-args: |
+            REGISTRY=${{ secrets.REHOSTING_ARC_REGISTRY }}/proxy
+            TARGET=latest
+          # Single shared build cache, written by this one job (no matrix race).
+          cache-from: |
+            type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/rehosting/linux_builder:all_cache,mode=max
+          cache-to: |
+            type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/rehosting/linux_builder:all_cache,mode=max
+
   build:
-    needs: prebuild
+    needs: [prebuild, build-image]
     runs-on: rehosting-arc
     if: github.event.pull_request.draft == false
 
     strategy:
       matrix:
         target_version: ${{ fromJSON(needs.prebuild.outputs.targets) }}
 
+    env:
+      # The image built once by build-image; pulled (warm on the shared daemon
+      # after the first job lands on a node) instead of rebuilt per target.
+      KERNEL_BUILDER_IMAGE: ${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/linux_builder:${{ github.sha }}
+
     steps:
       - uses: actions/checkout@v4
         with:
@@ -170,38 +244,15 @@ jobs:
           username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
           password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          # Do NOT pin image=moby/buildkit:master here. A recent master
-          # regressed on the self-hosted runners (kernel 5.4):
-          #   runc run failed: ... can't mask dir "/proc/acpi": mount ...
-          #   MS_RDONLY ... invalid argument
-          # which fails the first RUN that needs container init. Letting buildx
-          # use its default pinned-stable buildkit avoids it. network=host and
-          # the registry config are kept. (Mirrors rehosting/penguin c35bedc5.)
-          driver-opts: |
-            network=host
-          buildkitd-config-inline: |
-            [registry."${{ secrets.REHOSTING_ARC_REGISTRY }}"]
-              insecure = true
-              http = true
+      - name: Pull kernel_builder image
+        run: |
+          set -eux
+          # Built once by the build-image job. On the shared per-node Docker
+          # daemon the first matrix job to land on a node pulls the layers; every
+          # later job on that node finds them already present (warm reuse). No
+          # per-job buildx build anymore.
+          docker pull "$KERNEL_BUILDER_IMAGE"
 
-      - name: Build kernel_builder docker image
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: false
-          tags: |
-            rehosting/linux_builder:latest
-          build-args: |
-            REGISTRY=${{ secrets.REHOSTING_ARC_REGISTRY }}/proxy
-            TARGET=${{ matrix.target_version }}
-          cache-from: |
-            type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/rehosting/linux_builder:${{ matrix.target_version }}_cache,mode=max
-          cache-to: |
-            type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/rehosting/linux_builder:${{ matrix.target_version }}_cache,mode=max
-          outputs: type=docker
       - name: Build Kernel for ${{ matrix.target_version }}
         run: |
             set -eux
@@ -218,8 +269,21 @@ jobs:
               VERSIONS=$(echo "$VERSIONS_JSON" | jq -r '.[]' | xargs)
             fi
 
-            # Mount the stable source directory instead of the run-specific one
-            ./build.sh --targets "$TARGET" ${VERSIONS:+--versions "$VERSIONS"} --extra-docker-opts "-v $SOURCES_DIR:/app/linux"
+            # Node-shared, persistent compiler cache. Lives under the same
+            # /home/runner/_shared mount the kernel sources use (mirrored into the
+            # shared Docker daemon by kube#15), so mounting it into the build
+            # container needs no extra plumbing. One dir shared by every
+            # target/version on the node -- ccache is content-keyed and dedups, so
+            # a warm cache turns the ~15-20 min from-scratch compile into a mostly-
+            # cache-hit rebuild. _in_container_build.sh enables ccache iff CCACHE_DIR
+            # is set (no-op otherwise).
+            CCACHE_HOST_DIR="/home/runner/_shared/linux_builder_ccache"
+            mkdir -p "$CCACHE_HOST_DIR"
+
+            # Use the prebuilt image (--image) and mount the stable source
+            # directory instead of the run-specific one, plus the shared ccache.
+            ./build.sh --image "$KERNEL_BUILDER_IMAGE" --targets "$TARGET" ${VERSIONS:+--versions "$VERSIONS"} \
+              --extra-docker-opts "-v $SOURCES_DIR:/app/linux -v $CCACHE_HOST_DIR:/ccache -e CCACHE_DIR=/ccache -e CCACHE_MAXSIZE=15G"
 
             # Stage per-target outputs in the workspace; they are handed to the
             # aggregate job via workflow artifacts (below) instead of a shared

_in_container_build.sh

@@ -6,15 +6,22 @@ set -eu
 # root-owned on the host. Runs as root inside the container via an EXIT trap so
 # it covers every exit path (including the early kernel-devel exit). Gated on a
 # non-root HOST_UID, so CI (which may run as root / not set these) is unaffected.
-fix_ownership() {
+on_exit() {
+  # ccache summary on every exit path (there are several early exits:
+  # config-only, menuconfig, diffdefconfig, kernel-devel). No-op unless ccache
+  # is enabled.
+  if [ -n "${CCACHE_DIR:-}" ]; then
+    echo "ccache stats after build:"
+    ccache -s 2>/dev/null | sed 's/^/  ccache(end): /' || true
+  fi
   if [ -n "${HOST_UID:-}" ] && [ "${HOST_UID}" != "0" ]; then
     chown -R "${HOST_UID}:${HOST_GID:-$HOST_UID}" \
       /tmp/build \
       /app/kernels-latest.tar.gz \
       /app/kernel-devel-all.tar.gz 2>/dev/null || true
   fi
 }
-trap fix_ownership EXIT
+trap on_exit EXIT
 
 # We want to build linux for each of our targets and versions using the config files. Linux is in /app/linux/[version]
 # while our configs are at configs/[version]/[arch]. We need to set the ARCH and CROSS_COMPILE variables
@@ -36,6 +43,29 @@ echo "No strip: $NO_STRIP"
 echo "menuconfig: $MENU_CONFIG"
 echo "diffdefconfig: $DIFFDEFCONFIG"
 
+# Optional compiler cache. Enabled purely by the presence of CCACHE_DIR (set by
+# the caller, e.g. CI mounts a node-shared persistent dir). When unset, CC_PREFIX
+# stays empty and every `CC=` below expands to exactly the kernel default
+# ($(CROSS_COMPILE)gcc), so behaviour is identical to before. ccache is content-
+# keyed, so it survives `mrproper`/clean and stays correct across config and
+# source bumps; concurrent build jobs sharing one CCACHE_DIR are safe (ccache
+# locks internally). The container paths ($O=/tmp/build/..., src /app/linux) are
+# stable across runs, which is what lets objects hit.
+CC_PREFIX=""
+if [ -n "${CCACHE_DIR:-}" ]; then
+    mkdir -p "$CCACHE_DIR"
+    export CCACHE_DIR
+    export CCACHE_MAXSIZE="${CCACHE_MAXSIZE:-15G}"
+    export CCACHE_COMPRESS=1
+    # __DATE__/__TIME__ and header mtimes churn across a fresh checkout; treat
+    # them as sloppy so unchanged translation units still hit.
+    export CCACHE_SLOPPINESS="${CCACHE_SLOPPINESS:-time_macros,include_file_mtime,include_file_ctime,file_stat_matches}"
+    ccache -M "$CCACHE_MAXSIZE" >/dev/null 2>&1 || true
+    CC_PREFIX="ccache "
+    echo "ccache enabled: dir=$CCACHE_DIR maxsize=$CCACHE_MAXSIZE"
+    ccache -s 2>/dev/null | sed 's/^/  ccache(start): /' || true
+fi
+
 # Array to keep track of child processes
 declare -a pids
 
@@ -159,13 +189,13 @@ for TARGET in $TARGETS; do
         /app/linux/${VERSION}/scripts/diffconfig /tmp/original_config /tmp/build/${VERSION}/${TARGET}/.config
         exit
       fi
-      make -C /app/linux/$VERSION ARCH=${short_arch} CROSS_COMPILE=$(get_cc $TARGET $VERSION) O=/tmp/build/${VERSION}/${TARGET}/ $BUILD_TARGETS -j$(nproc)
+      make -C /app/linux/$VERSION ARCH=${short_arch} CROSS_COMPILE=$(get_cc $TARGET $VERSION) CC="${CC_PREFIX}$(get_cc $TARGET $VERSION)gcc" O=/tmp/build/${VERSION}/${TARGET}/ $BUILD_TARGETS -j$(nproc)
 
       # Always run modules_prepare to ensure headers and Module.symvers are generated
-      make -C /app/linux/$VERSION ARCH=${short_arch} CROSS_COMPILE=$(get_cc $TARGET $VERSION) O=/tmp/build/${VERSION}/${TARGET}/ modules_prepare
+      make -C /app/linux/$VERSION ARCH=${short_arch} CROSS_COMPILE=$(get_cc $TARGET $VERSION) CC="${CC_PREFIX}$(get_cc $TARGET $VERSION)gcc" O=/tmp/build/${VERSION}/${TARGET}/ modules_prepare
 
       # Build modules to ensure Module.symvers is generated
-      make -C /app/linux/$VERSION ARCH=${short_arch} CROSS_COMPILE=$(get_cc $TARGET $VERSION) O=/tmp/build/${VERSION}/${TARGET}/ modules -j$(nproc)
+      make -C /app/linux/$VERSION ARCH=${short_arch} CROSS_COMPILE=$(get_cc $TARGET $VERSION) CC="${CC_PREFIX}$(get_cc $TARGET $VERSION)gcc" O=/tmp/build/${VERSION}/${TARGET}/ modules -j$(nproc)
 
       # Prepare and completely clean the output directory for perf
       PERF_OUTDIR="/tmp/build/${VERSION}/${TARGET}/tools/perf/"
@@ -184,6 +214,7 @@ for TARGET in $TARGETS; do
       make -C /app/linux/$VERSION/tools/perf \
           ARCH=${short_arch} \
           CROSS_COMPILE=$(get_cc $TARGET $VERSION) \
+          CC="${CC_PREFIX}$(get_cc $TARGET $VERSION)gcc" \
           LD="$PERF_LD" \
           OUTPUT="$PERF_OUTDIR" \
           LDFLAGS="-static" \

build.sh

@@ -127,8 +127,38 @@ else
     CACHE_HOST_DIR="$CACHE_DIR"
 fi
 
+# Bind-mount source translation for a shared Docker daemon (opt-in, pure
+# no-op otherwise).
+#
+# Normally dockerd shares this script's mount namespace, so a bind-mount
+# source path ("$PWD", the cache dir) means the same thing to the daemon as
+# to us. Under a shared per-node daemon (e.g. rehosting CI's shared-docker
+# runners) it does NOT: the daemon can't see this runner's per-pod workspace
+# under /home/runner/_work, so "-v $PWD:/app" would mount an empty dir and the
+# build would fail with "/app/_in_container_build.sh: No such file". The runner
+# exports PENGUIN_HOST_MOUNT_FROM / PENGUIN_HOST_MOUNT_TO (the same mechanism
+# penguin's wrapper uses) giving the daemon-visible location of that workspace.
+#
+# rewrite_mount() rewrites a bind source ONLY when BOTH env vars are set and
+# the path is under _FROM. When they're unset — every local build and every
+# GitHub-hosted runner — it returns the path unchanged, so behaviour is
+# identical to before. (Sources outside _FROM, e.g. the kernel-source cache
+# under /home/runner/_shared, are left as-is; the shared daemon mirrors that
+# node-shared path directly.)
+rewrite_mount() {
+    local path="$1"
+    if [[ -n "$PENGUIN_HOST_MOUNT_FROM" && -n "$PENGUIN_HOST_MOUNT_TO" \
+          && "$path" == "$PENGUIN_HOST_MOUNT_FROM"* ]]; then
+        printf '%s' "${PENGUIN_HOST_MOUNT_TO}${path#"$PENGUIN_HOST_MOUNT_FROM"}"
+    else
+        printf '%s' "$path"
+    fi
+}
+CACHE_MOUNT_SRC="$(rewrite_mount "$CACHE_HOST_DIR")"
+APP_MOUNT_SRC="$(rewrite_mount "$PWD")"
+
 if $CLEAR_CACHE; then
-    docker run --rm -v "$CACHE_HOST_DIR":/tmp/build -v "$PWD":/app pandare/kernel_builder /bin/bash -c "rm -rf /tmp/build/*"
+    docker run --rm -v "$CACHE_MOUNT_SRC":/tmp/build -v "$APP_MOUNT_SRC":/app pandare/kernel_builder /bin/bash -c "rm -rf /tmp/build/*"
     exit
 fi
 
@@ -141,8 +171,8 @@ fi
 mkdir -p "$CACHE_HOST_DIR"
 
 docker run $INTERACTIVE \
-    --rm -v "$CACHE_HOST_DIR":/tmp/build \
-    -v "$PWD":/app \
+    --rm -v "$CACHE_MOUNT_SRC":/tmp/build \
+    -v "$APP_MOUNT_SRC":/app \
     -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
     $EXTRA_DOCKER_OPTS \
     "$IMAGE" \