From 15c4459fbed67cb49a682b3527195b016a1dc392 Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Tue, 21 Jul 2026 17:36:41 +0200 Subject: [PATCH] ci(build): split allTests into parallel per-target jobs to end OOM flakes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single build-job ran `assemble allTests` in one ubuntu-latest job. On a 7 GB runner the combined peak memory (Gradle JVM + Kotlin daemon + per-target test forks + Node for JS/Wasm + native toolchain, all at once) intermittently exceeded physical RAM. The runner was OOM-killed and GitHub reported "The operation was canceled" after ~15 min with no BUILD FAILED — the flaky ~20-minute red builds (seen on PRs #850, #852). Changes: - Split the test run into a matrix of three legs (jvm / js-wasm / native), each running one target family so no single job holds the whole footprint. Legs run in parallel, so wall-clock is the slowest leg instead of the sum. - Run `assemble` as its own memory-light job, preserving the build/package guarantee for all targets. - Add Gradle dependency/wrapper caching to each job (same pattern as docs.yml) so parallel legs don't cold-download the dependency graph. - Capture the memory snapshot on `cancelled()` as well as `failure()` — an OOM-killed runner reports cancelled, not failed, so the existing diagnostic never fired on the runs that needed it. - Keep an aggregate `build-job` gate with the same name so the existing required status check keeps working without re-pointing branch protection. No production code changed; allTests passes locally. --- .github/workflows/build.yml | 138 +++++++++++++++++++++++++++++------- 1 file changed, 113 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36042179..265b05f2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,10 +11,31 @@ concurrency: cancel-in-progress: true jobs: - build-job: + # `allTests` used to run in a single job alongside `assemble`. On a 7 GB + # ubuntu-latest runner the combined peak RSS (Gradle JVM + Kotlin daemon + + # per-target test forks + Node for JS/Wasm + the native toolchain, all at + # once) intermittently exceeded physical memory. The runner was then + # OOM-killed and GitHub reported "The operation was canceled" after ~15 min + # with no BUILD FAILED — the flaky ~20-minute reds. + # + # Splitting the test matrix so each leg runs one target family keeps every + # job's memory well under the runner limit, and the legs run in parallel so + # wall-clock is the slowest single leg rather than the sum. `assemble` runs + # as its own (memory-light) job. + test: + name: test (${{ matrix.name }}) runs-on: ubuntu-latest - timeout-minutes: 60 - + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + include: + - name: jvm + tasks: jvmTest + - name: js-wasm + tasks: jsTest wasmJsTest wasmWasiTest + - name: native + tasks: linuxX64Test steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -33,45 +54,112 @@ jobs: with: node-version: '20' - - name: Print git commit variables - run: | - echo "TAG: $CURRENT_TAG" + # Warm the Gradle dependency/wrapper cache so each parallel leg does not + # cold-download the full dependency graph. Matches docs.yml. + - name: Cache Gradle + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: gradle-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties', '**/libs.versions.toml') }} + restore-keys: | + gradle-${{ runner.os }}- - name: Disk space (observability) run: df -h - - name: Build and Test (PR) - if: github.event_name == 'pull_request' - env: - GRADLE_OPTS: -Dorg.gradle.jvmargs="-Xmx4g -Dfile.encoding=UTF-8" - run: | - ./gradlew --no-daemon --stacktrace --info \ - -Dorg.gradle.caching=true \ - -Dorg.gradle.configuration-cache=true \ - clean assemble allTests - - - name: Build and Test (push) - if: github.event_name != 'pull_request' + - name: Test (${{ matrix.name }}) env: GRADLE_OPTS: -Dorg.gradle.jvmargs="-Xmx4g -Dfile.encoding=UTF-8" run: | - ./gradlew --no-daemon --stacktrace --info \ + ./gradlew --no-daemon --stacktrace \ -Dorg.gradle.caching=true \ -Dorg.gradle.configuration-cache=true \ - assemble allTests + ${{ matrix.tasks }} - - name: Memory info (on failure) - if: failure() + # `cancelled()` matters: an OOM-killed runner reports the job as + # cancelled, not failed, so `if: failure()` alone never captured the + # memory snapshot on exactly the runs we needed it for. + - name: Memory info (on failure or cancellation) + if: failure() || cancelled() run: | free -h || true - cat /proc/meminfo | head -n 50 || true + head -n 50 /proc/meminfo || true - - name: Upload all test reports + - name: Upload test reports (${{ matrix.name }}) if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: test-reports + name: test-reports-${{ matrix.name }} path: | **/build/reports/tests/** **/build/test-results/** retention-days: 14 + + assemble: + name: assemble + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Copy CI gradle.properties + run: mkdir -p ~/.gradle ; cp .github/ci-gradle.properties ~/.gradle/gradle.properties + + - name: Set up JDK 25 + uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 + with: + distribution: 'zulu' + java-version: 25 + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '20' + + - name: Cache Gradle + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: gradle-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties', '**/libs.versions.toml') }} + restore-keys: | + gradle-${{ runner.os }}- + + - name: Assemble + env: + GRADLE_OPTS: -Dorg.gradle.jvmargs="-Xmx4g -Dfile.encoding=UTF-8" + run: | + ./gradlew --no-daemon --stacktrace \ + -Dorg.gradle.caching=true \ + -Dorg.gradle.configuration-cache=true \ + assemble + + - name: Memory info (on failure or cancellation) + if: failure() || cancelled() + run: | + free -h || true + head -n 50 /proc/meminfo || true + + # Aggregate gate. Branch protection references the single "build-job" check; + # keeping a job with that exact name (now backed by the parallel test matrix + # + assemble) preserves the required-status-check contract without needing to + # re-point branch protection. Fails if any dependency did not succeed. + build-job: + name: build-job + needs: [ test, assemble ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Verify test matrix and assemble succeeded + run: | + echo "test result: ${{ needs.test.result }}" + echo "assemble result: ${{ needs.assemble.result }}" + if [ "${{ needs.test.result }}" != "success" ] || [ "${{ needs.assemble.result }}" != "success" ]; then + echo "::error::One or more build jobs did not succeed (see the test/assemble jobs above)." + exit 1 + fi + echo "All build jobs succeeded."