Improve CI Docker cache reuse

.dockerignore

@@ -1,4 +1,5 @@
 /target
+.git/
 sqlpage.db
 docs/
 README.md
@@ -11,4 +12,4 @@ configuration.md
 examples/
 mssql/
 tests/
-.idea/
+.idea/

.github/workflows/ci.yml

@@ -14,6 +14,15 @@ on:
   pull_request:
     branches:
       - "main"
+    paths-ignore:
+      - "docs/**"
+      - "README.md"
+      - ".github/workflows/release.yml"
+      - ".github/workflows/official-site.yml"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
   CARGO_TERM_COLOR: always
@@ -36,7 +45,28 @@ jobs:
           save-if: ${{ github.ref == 'refs/heads/main' }}
       - run: cargo fmt --all -- --check
       - run: cargo clippy --all-targets --all-features -- -D warnings
-      - run: cargo test --features odbc-static
+      # The database matrix below runs the same Rust test harnesses against
+      # different DATABASE_URL values. Package the Linux test executables once
+      # here so those jobs do not recompile SQLPage or its dependencies.
+      - name: Package Linux Rust test binaries
+        run: |
+          set -euo pipefail
+          rm -rf target/sqlpage-test-binaries
+          mkdir -p target/sqlpage-test-binaries
+          cargo test --features odbc-static --no-run --message-format=json \
+            | jq -r 'select(.profile.test == true and .executable != null) | .executable' \
+            | while IFS= read -r test_binary; do
+                cp -- "$test_binary" target/sqlpage-test-binaries/
+              done
+          test -n "$(find target/sqlpage-test-binaries -maxdepth 1 -type f -print -quit)"
+          tar -C target/sqlpage-test-binaries -czf target/sqlpage-linux-test-binaries.tar.gz .
+      - name: Build Linux binary
+        run: cargo build --features odbc-static
+      - name: Upload Linux Rust test binaries
+        uses: actions/upload-artifact@v7
+        with:
+          name: sqlpage-linux-test-binaries
+          path: "target/sqlpage-linux-test-binaries.tar.gz"
       - name: Upload Linux binary
         uses: actions/upload-artifact@v7
         with:
@@ -45,9 +75,13 @@ jobs:
 
   test:
     runs-on: ubuntu-latest
+    needs: compile_and_lint
     strategy:
       matrix:
         include:
+          - database: sqlite
+            container: ""
+            db_url: "sqlite::memory:"
           - database: postgres
             container: postgres
             db_url: "postgres://root:Password123!@127.0.0.1/sqlpage"
@@ -66,13 +100,18 @@ jobs:
             db_url: "Driver=Oracle 21 ODBC driver;Dbq=//127.0.0.1:1521/FREEPDB1;Uid=root;Pwd=Password123!"
     steps:
       - uses: actions/checkout@v6
-      - name: Set up cargo cache
-        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
-        env:
-          NODE_OPTIONS: --no-deprecation
+      # Reuse the exact Linux test harnesses produced by compile_and_lint.
+      # This keeps the DB matrix focused on database behavior instead of
+      # compiling the same Rust crate five more times.
+      - name: Download Linux Rust test binaries
+        uses: actions/download-artifact@v8
         with:
-          shared-key: rust-sqlpage-proj-test
-          save-if: false
+          name: sqlpage-linux-test-binaries
+          path: target
+      - name: Extract Linux Rust test binaries
+        run: |
+          mkdir -p target/sqlpage-test-binaries
+          tar -xzf target/sqlpage-linux-test-binaries.tar.gz -C target/sqlpage-test-binaries
       - name: Install PostgreSQL ODBC driver
         if: matrix.setup_odbc
         run: sudo apt-get install -y odbc-postgresql
@@ -86,15 +125,28 @@ jobs:
           sudo alien -i oracle-instantclient-odbc-21.21.0.0.0-1.el8.x86_64.rpm
           sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/libaio.so.1
           sudo /usr/lib/oracle/21/client64/bin/odbc_update_ini.sh / /usr/lib/oracle/21/client64/lib
-          echo "LD_LIBRARY_PATH=/usr/lib/oracle/21/client64/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=/usr/lib/oracle/21/client64/lib:$LD_LIBRARY_PATH" >> "$GITHUB_ENV"
       - name: Start database container
+        if: matrix.container != ''
         run: docker compose up --wait ${{ matrix.container }}
       - name: Show container logs
-        if: failure()
+        if: failure() && matrix.container != ''
         run: docker compose logs ${{ matrix.container }}
       - name: Run tests against ${{ matrix.database }}
         timeout-minutes: 5
-        run: cargo test --features odbc-static
+        run: |
+          set -euo pipefail
+          shopt -s nullglob
+          test_binaries=(target/sqlpage-test-binaries/*)
+          if ((${#test_binaries[@]} == 0)); then
+            echo "No test binaries were found in target/sqlpage-test-binaries" >&2
+            exit 1
+          fi
+          for test_binary in "${test_binaries[@]}"; do
+            echo "::group::$(basename "$test_binary")"
+            "$test_binary"
+            echo "::endgroup::"
+          done
         env:
           DATABASE_URL: ${{ matrix.db_url }}
           MALLOC_CHECK_: 3
@@ -108,17 +160,71 @@ jobs:
         uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
         env:
           NODE_OPTIONS: --no-deprecation
-      - name: Check port usage
-        run: netstat -bano
+      - name: Set up Windows incremental cache
+        uses: actions/cache@v5
+        with:
+          path: |
+            target/debug/.fingerprint/sqlpage-*
+            target/debug/build/sqlpage-*
+            target/debug/deps/libsqlpage-*.rlib
+            target/debug/deps/libsqlpage-*.rmeta
+            target/debug/deps/sqlpage-*.d
+            target/debug/incremental/sqlpage-*
+          key: windows-incremental-${{ github.event.pull_request.number || github.ref_name }}-${{ github.sha }}
+          restore-keys: |
+            windows-incremental-${{ github.event.pull_request.number || github.ref_name }}-
       - run: cargo test
         env:
+          CARGO_INCREMENTAL: 1
           RUST_BACKTRACE: 1
       - name: Upload Windows binary
         uses: actions/upload-artifact@v7
         with:
           name: sqlpage-windows-debug
           path: "target/debug/sqlpage.exe"
 
+  playwright:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs: compile_and_lint
+    defaults:
+      run:
+        working-directory: ./tests/end-to-end
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v6
+        with:
+          node-version: lts/*
+          cache: 'npm'
+          cache-dependency-path: ./tests/end-to-end/package-lock.json
+      - run: sudo apt-get update && sudo apt-get install -y unixodbc-dev
+      - run: npm ci && npx playwright install --with-deps chromium
+      # The browser tests exercise the official site, but they do not need a
+      # separate Rust build. Reuse the binary compiled and tested above.
+      - name: Download Linux binary
+        uses: actions/download-artifact@v8
+        with:
+          name: sqlpage-linux-debug
+          path: ${{ runner.temp }}/sqlpage-bin
+      - name: Start official site and wait for it to be ready
+        timeout-minutes: 1
+        run: |
+          chmod +x "${{ runner.temp }}/sqlpage-bin/sqlpage"
+          "${{ runner.temp }}/sqlpage-bin/sqlpage" 2>/tmp/stderrlog &
+          tail -f /tmp/stderrlog | grep -q "started successfully"
+        working-directory: ./examples/official-site
+      - name: Run Playwright tests
+        run: npx playwright test
+      - name: show server logs
+        if: failure()
+        run: cat /tmp/stderrlog
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: playwright-report
+          path: ./tests/end-to-end/playwright-report/
+          retention-days: 30
+
   docker_build:
     runs-on: ubuntu-latest
     strategy:
@@ -149,6 +255,19 @@ jobs:
             suffix="${suffix}-${{ matrix.variant }}"
           fi
           echo "suffix=${suffix}" >> "$GITHUB_OUTPUT"
+      - id: cache-scope
+        name: Docker cache scope
+        run: |
+          ref_scope="main"
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            ref_scope="pr-${{ github.event.pull_request.number }}"
+          fi
+
+          recipe_hash="${{ hashFiles('Dockerfile', '.cargo/**', 'Cargo.toml', 'Cargo.lock', 'build.rs', 'scripts/**', 'sqlpage/**') }}"
+          {
+            echo "current=sqlpage-${ref_scope}${{ steps.suffix.outputs.suffix }}-${recipe_hash}"
+            echo "main=sqlpage-main${{ steps.suffix.outputs.suffix }}-${recipe_hash}"
+          } >> "$GITHUB_OUTPUT"
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v6
@@ -175,13 +294,11 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
-          cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:main${{ steps.suffix.outputs.suffix }}
-          # don't save cache on prs
-          cache-to: >
-            ${{ github.event_name != 'pull_request'
-              && format('type=registry,ref={0}:main{1},compression=zstd,mode=max', env.REGISTRY_IMAGE, steps.suffix.outputs.suffix)
-              || ''
-            }}
+          cache-from: |
+            type=gha,scope=${{ steps.cache-scope.outputs.current }}
+            type=gha,scope=${{ steps.cache-scope.outputs.main }}
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:main${{ steps.suffix.outputs.suffix }}
+          cache-to: type=gha,scope=${{ steps.cache-scope.outputs.current }},mode=max
       - name: Export digest
         if: github.event_name != 'pull_request'
         run: |
@@ -211,6 +328,20 @@ jobs:
           flavor: suffix=-linux-amd64
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
+      - id: cache-scope
+        name: Docker cache scope
+        run: |
+          ref_scope="main"
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            ref_scope="pr-${{ github.event.pull_request.number }}"
+          fi
+
+          recipe_hash="${{ hashFiles('Dockerfile', '.cargo/**', 'Cargo.toml', 'Cargo.lock', 'build.rs', 'scripts/**', 'sqlpage/**') }}"
+          {
+            echo "artifact=sqlpage-${ref_scope}-linux-amd64-hurl-${recipe_hash}"
+            echo "current=sqlpage-${ref_scope}-linux-amd64-${recipe_hash}"
+            echo "main=sqlpage-main-linux-amd64-${recipe_hash}"
+          } >> "$GITHUB_OUTPUT"
       - name: Login to Docker Hub
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v4
@@ -228,7 +359,12 @@ jobs:
             ${{ steps.meta.outputs.tags }}
             ${{ env.REGISTRY_IMAGE }}:main
           outputs: type=docker,dest=${{ runner.temp }}/sqlpage.tar
-          cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:main-linux-amd64
+          cache-from: |
+            type=gha,scope=${{ steps.cache-scope.outputs.artifact }}
+            type=gha,scope=${{ steps.cache-scope.outputs.current }}
+            type=gha,scope=${{ steps.cache-scope.outputs.main }}
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:main-linux-amd64
+          cache-to: type=gha,scope=${{ steps.cache-scope.outputs.artifact }},mode=max
       - name: Upload SQLPage image
         uses: actions/upload-artifact@v7
         with:
@@ -247,8 +383,11 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-          cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:main-linux-amd64
-          cache-to: type=registry,ref=${{ env.REGISTRY_IMAGE }}:main-linux-amd64,compression=zstd,mode=max
+          cache-from: |
+            type=gha,scope=${{ steps.cache-scope.outputs.current }}
+            type=gha,scope=${{ steps.cache-scope.outputs.main }}
+            type=registry,ref=${{ env.REGISTRY_IMAGE }}:main-linux-amd64
+          cache-to: type=gha,scope=${{ steps.cache-scope.outputs.current }},mode=max
       - name: Export digest
         if: github.event_name != 'pull_request'
         run: |

Dockerfile

@@ -11,7 +11,9 @@ RUN cargo init .
 
 RUN /usr/local/bin/setup-cross-compilation.sh "$TARGETARCH" "$BUILDARCH"
 
-COPY Cargo.toml Cargo.lock ./
+COPY .cargo/ .cargo/
+COPY Cargo.toml Cargo.lock build.rs ./
+COPY sqlpage/ sqlpage/
 RUN /usr/local/bin/build-dependencies.sh
 
 COPY . .