diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0930a95..bfa55af 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,8 +74,77 @@ jobs: echo "Found valid targets: $TARGETS_OUTPUT" echo "Found valid versions: $VERSIONS_OUTPUT" + # Build the kernel_builder image exactly once per run, push it to Harbor + # under an immutable per-commit tag, and let every matrix build job pull it. + # + # Why a dedicated job (instead of building inside each matrix job): + # * The image is now target-agnostic (TARGET=latest bundles every + # cross-toolchain), so all 13 build jobs would otherwise rebuild the SAME + # multi-GB image. setup-buildx-action's docker-container driver keeps its + # own cache store, separate from the shared per-node Docker daemon, so the + # daemon's warm-image reuse does NOT cover a buildx `FROM` base -- every + # job re-pulled the full all-arch base. Building once here fixes that. + # * A single writer to the `:all_cache` registry cache means the cache + # actually accumulates run-to-run instead of 13 jobs racing to overwrite + # one ref. + # * An immutable `:` tag (vs a mutable `:latest`) is safe on the shared + # daemon's single image store -- concurrent jobs/PRs can't clobber each + # other's tag (same lesson as rehosting/penguin #893). + build-image: + runs-on: rehosting-arc + if: github.event.pull_request.draft == false + steps: + - uses: actions/checkout@v4 + + - name: Trust Harbor's self-signed certificate + run: | + echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}" + openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null + sudo update-ca-certificates + + - name: Log in to Rehosting Arc Registry + uses: docker/login-action@v3 + with: + registry: ${{secrets.REHOSTING_ARC_REGISTRY}} + username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }} + password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + # See the note in the (removed) per-job build step / penguin c35bedc5: + # don't pin moby/buildkit:master -- it regressed on the kernel-5.4 + # runners (can't mask /proc/acpi). Default pinned-stable buildkit + the + # insecure Harbor registry config. + driver-opts: | + network=host + buildkitd-config-inline: | + [registry."${{ secrets.REHOSTING_ARC_REGISTRY }}"] + insecure = true + http = true + + - name: Build and push kernel_builder image + uses: docker/build-push-action@v6 + with: + context: . + push: true + # Immutable per-commit tag -- matrix jobs pull this exact ref. + tags: | + ${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/linux_builder:${{ github.sha }} + # TARGET=latest builds FROM the all-arch embedded-toolchains:latest so + # the single built image carries every cross-toolchain and one image + # serves every matrix target. + build-args: | + REGISTRY=${{ secrets.REHOSTING_ARC_REGISTRY }}/proxy + TARGET=latest + # Single shared build cache, written by this one job (no matrix race). + cache-from: | + type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/rehosting/linux_builder:all_cache,mode=max + cache-to: | + type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/rehosting/linux_builder:all_cache,mode=max + build: - needs: prebuild + needs: [prebuild, build-image] runs-on: rehosting-arc if: github.event.pull_request.draft == false @@ -83,6 +152,11 @@ jobs: matrix: target_version: ${{ fromJSON(needs.prebuild.outputs.targets) }} + env: + # The image built once by build-image; pulled (warm on the shared daemon + # after the first job lands on a node) instead of rebuilt per target. + KERNEL_BUILDER_IMAGE: ${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/linux_builder:${{ github.sha }} + steps: - uses: actions/checkout@v4 with: @@ -170,38 +244,15 @@ jobs: username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }} password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - # Do NOT pin image=moby/buildkit:master here. A recent master - # regressed on the self-hosted runners (kernel 5.4): - # runc run failed: ... can't mask dir "/proc/acpi": mount ... - # MS_RDONLY ... invalid argument - # which fails the first RUN that needs container init. Letting buildx - # use its default pinned-stable buildkit avoids it. network=host and - # the registry config are kept. (Mirrors rehosting/penguin c35bedc5.) - driver-opts: | - network=host - buildkitd-config-inline: | - [registry."${{ secrets.REHOSTING_ARC_REGISTRY }}"] - insecure = true - http = true + - name: Pull kernel_builder image + run: | + set -eux + # Built once by the build-image job. On the shared per-node Docker + # daemon the first matrix job to land on a node pulls the layers; every + # later job on that node finds them already present (warm reuse). No + # per-job buildx build anymore. + docker pull "$KERNEL_BUILDER_IMAGE" - - name: Build kernel_builder docker image - uses: docker/build-push-action@v6 - with: - context: . - push: false - tags: | - rehosting/linux_builder:latest - build-args: | - REGISTRY=${{ secrets.REHOSTING_ARC_REGISTRY }}/proxy - TARGET=${{ matrix.target_version }} - cache-from: | - type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/rehosting/linux_builder:${{ matrix.target_version }}_cache,mode=max - cache-to: | - type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/rehosting/linux_builder:${{ matrix.target_version }}_cache,mode=max - outputs: type=docker - name: Build Kernel for ${{ matrix.target_version }} run: | set -eux @@ -218,8 +269,21 @@ jobs: VERSIONS=$(echo "$VERSIONS_JSON" | jq -r '.[]' | xargs) fi - # Mount the stable source directory instead of the run-specific one - ./build.sh --targets "$TARGET" ${VERSIONS:+--versions "$VERSIONS"} --extra-docker-opts "-v $SOURCES_DIR:/app/linux" + # Node-shared, persistent compiler cache. Lives under the same + # /home/runner/_shared mount the kernel sources use (mirrored into the + # shared Docker daemon by kube#15), so mounting it into the build + # container needs no extra plumbing. One dir shared by every + # target/version on the node -- ccache is content-keyed and dedups, so + # a warm cache turns the ~15-20 min from-scratch compile into a mostly- + # cache-hit rebuild. _in_container_build.sh enables ccache iff CCACHE_DIR + # is set (no-op otherwise). + CCACHE_HOST_DIR="/home/runner/_shared/linux_builder_ccache" + mkdir -p "$CCACHE_HOST_DIR" + + # Use the prebuilt image (--image) and mount the stable source + # directory instead of the run-specific one, plus the shared ccache. + ./build.sh --image "$KERNEL_BUILDER_IMAGE" --targets "$TARGET" ${VERSIONS:+--versions "$VERSIONS"} \ + --extra-docker-opts "-v $SOURCES_DIR:/app/linux -v $CCACHE_HOST_DIR:/ccache -e CCACHE_DIR=/ccache -e CCACHE_MAXSIZE=15G" # Stage per-target outputs in the workspace; they are handed to the # aggregate job via workflow artifacts (below) instead of a shared diff --git a/_in_container_build.sh b/_in_container_build.sh index 426f29e..759c0ad 100755 --- a/_in_container_build.sh +++ b/_in_container_build.sh @@ -6,7 +6,14 @@ set -eu # root-owned on the host. Runs as root inside the container via an EXIT trap so # it covers every exit path (including the early kernel-devel exit). Gated on a # non-root HOST_UID, so CI (which may run as root / not set these) is unaffected. -fix_ownership() { +on_exit() { + # ccache summary on every exit path (there are several early exits: + # config-only, menuconfig, diffdefconfig, kernel-devel). No-op unless ccache + # is enabled. + if [ -n "${CCACHE_DIR:-}" ]; then + echo "ccache stats after build:" + ccache -s 2>/dev/null | sed 's/^/ ccache(end): /' || true + fi if [ -n "${HOST_UID:-}" ] && [ "${HOST_UID}" != "0" ]; then chown -R "${HOST_UID}:${HOST_GID:-$HOST_UID}" \ /tmp/build \ @@ -14,7 +21,7 @@ fix_ownership() { /app/kernel-devel-all.tar.gz 2>/dev/null || true fi } -trap fix_ownership EXIT +trap on_exit EXIT # We want to build linux for each of our targets and versions using the config files. Linux is in /app/linux/[version] # while our configs are at configs/[version]/[arch]. We need to set the ARCH and CROSS_COMPILE variables @@ -36,6 +43,29 @@ echo "No strip: $NO_STRIP" echo "menuconfig: $MENU_CONFIG" echo "diffdefconfig: $DIFFDEFCONFIG" +# Optional compiler cache. Enabled purely by the presence of CCACHE_DIR (set by +# the caller, e.g. CI mounts a node-shared persistent dir). When unset, CC_PREFIX +# stays empty and every `CC=` below expands to exactly the kernel default +# ($(CROSS_COMPILE)gcc), so behaviour is identical to before. ccache is content- +# keyed, so it survives `mrproper`/clean and stays correct across config and +# source bumps; concurrent build jobs sharing one CCACHE_DIR are safe (ccache +# locks internally). The container paths ($O=/tmp/build/..., src /app/linux) are +# stable across runs, which is what lets objects hit. +CC_PREFIX="" +if [ -n "${CCACHE_DIR:-}" ]; then + mkdir -p "$CCACHE_DIR" + export CCACHE_DIR + export CCACHE_MAXSIZE="${CCACHE_MAXSIZE:-15G}" + export CCACHE_COMPRESS=1 + # __DATE__/__TIME__ and header mtimes churn across a fresh checkout; treat + # them as sloppy so unchanged translation units still hit. + export CCACHE_SLOPPINESS="${CCACHE_SLOPPINESS:-time_macros,include_file_mtime,include_file_ctime,file_stat_matches}" + ccache -M "$CCACHE_MAXSIZE" >/dev/null 2>&1 || true + CC_PREFIX="ccache " + echo "ccache enabled: dir=$CCACHE_DIR maxsize=$CCACHE_MAXSIZE" + ccache -s 2>/dev/null | sed 's/^/ ccache(start): /' || true +fi + # Array to keep track of child processes declare -a pids @@ -159,13 +189,13 @@ for TARGET in $TARGETS; do /app/linux/${VERSION}/scripts/diffconfig /tmp/original_config /tmp/build/${VERSION}/${TARGET}/.config exit fi - make -C /app/linux/$VERSION ARCH=${short_arch} CROSS_COMPILE=$(get_cc $TARGET $VERSION) O=/tmp/build/${VERSION}/${TARGET}/ $BUILD_TARGETS -j$(nproc) + make -C /app/linux/$VERSION ARCH=${short_arch} CROSS_COMPILE=$(get_cc $TARGET $VERSION) CC="${CC_PREFIX}$(get_cc $TARGET $VERSION)gcc" O=/tmp/build/${VERSION}/${TARGET}/ $BUILD_TARGETS -j$(nproc) # Always run modules_prepare to ensure headers and Module.symvers are generated - make -C /app/linux/$VERSION ARCH=${short_arch} CROSS_COMPILE=$(get_cc $TARGET $VERSION) O=/tmp/build/${VERSION}/${TARGET}/ modules_prepare + make -C /app/linux/$VERSION ARCH=${short_arch} CROSS_COMPILE=$(get_cc $TARGET $VERSION) CC="${CC_PREFIX}$(get_cc $TARGET $VERSION)gcc" O=/tmp/build/${VERSION}/${TARGET}/ modules_prepare # Build modules to ensure Module.symvers is generated - make -C /app/linux/$VERSION ARCH=${short_arch} CROSS_COMPILE=$(get_cc $TARGET $VERSION) O=/tmp/build/${VERSION}/${TARGET}/ modules -j$(nproc) + make -C /app/linux/$VERSION ARCH=${short_arch} CROSS_COMPILE=$(get_cc $TARGET $VERSION) CC="${CC_PREFIX}$(get_cc $TARGET $VERSION)gcc" O=/tmp/build/${VERSION}/${TARGET}/ modules -j$(nproc) # Prepare and completely clean the output directory for perf PERF_OUTDIR="/tmp/build/${VERSION}/${TARGET}/tools/perf/" @@ -184,6 +214,7 @@ for TARGET in $TARGETS; do make -C /app/linux/$VERSION/tools/perf \ ARCH=${short_arch} \ CROSS_COMPILE=$(get_cc $TARGET $VERSION) \ + CC="${CC_PREFIX}$(get_cc $TARGET $VERSION)gcc" \ LD="$PERF_LD" \ OUTPUT="$PERF_OUTDIR" \ LDFLAGS="-static" \ diff --git a/build.sh b/build.sh index 6ae8e11..7987966 100755 --- a/build.sh +++ b/build.sh @@ -127,8 +127,38 @@ else CACHE_HOST_DIR="$CACHE_DIR" fi +# Bind-mount source translation for a shared Docker daemon (opt-in, pure +# no-op otherwise). +# +# Normally dockerd shares this script's mount namespace, so a bind-mount +# source path ("$PWD", the cache dir) means the same thing to the daemon as +# to us. Under a shared per-node daemon (e.g. rehosting CI's shared-docker +# runners) it does NOT: the daemon can't see this runner's per-pod workspace +# under /home/runner/_work, so "-v $PWD:/app" would mount an empty dir and the +# build would fail with "/app/_in_container_build.sh: No such file". The runner +# exports PENGUIN_HOST_MOUNT_FROM / PENGUIN_HOST_MOUNT_TO (the same mechanism +# penguin's wrapper uses) giving the daemon-visible location of that workspace. +# +# rewrite_mount() rewrites a bind source ONLY when BOTH env vars are set and +# the path is under _FROM. When they're unset — every local build and every +# GitHub-hosted runner — it returns the path unchanged, so behaviour is +# identical to before. (Sources outside _FROM, e.g. the kernel-source cache +# under /home/runner/_shared, are left as-is; the shared daemon mirrors that +# node-shared path directly.) +rewrite_mount() { + local path="$1" + if [[ -n "$PENGUIN_HOST_MOUNT_FROM" && -n "$PENGUIN_HOST_MOUNT_TO" \ + && "$path" == "$PENGUIN_HOST_MOUNT_FROM"* ]]; then + printf '%s' "${PENGUIN_HOST_MOUNT_TO}${path#"$PENGUIN_HOST_MOUNT_FROM"}" + else + printf '%s' "$path" + fi +} +CACHE_MOUNT_SRC="$(rewrite_mount "$CACHE_HOST_DIR")" +APP_MOUNT_SRC="$(rewrite_mount "$PWD")" + if $CLEAR_CACHE; then - docker run --rm -v "$CACHE_HOST_DIR":/tmp/build -v "$PWD":/app pandare/kernel_builder /bin/bash -c "rm -rf /tmp/build/*" + docker run --rm -v "$CACHE_MOUNT_SRC":/tmp/build -v "$APP_MOUNT_SRC":/app pandare/kernel_builder /bin/bash -c "rm -rf /tmp/build/*" exit fi @@ -141,8 +171,8 @@ fi mkdir -p "$CACHE_HOST_DIR" docker run $INTERACTIVE \ - --rm -v "$CACHE_HOST_DIR":/tmp/build \ - -v "$PWD":/app \ + --rm -v "$CACHE_MOUNT_SRC":/tmp/build \ + -v "$APP_MOUNT_SRC":/app \ -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ $EXTRA_DOCKER_OPTS \ "$IMAGE" \