diff --git a/.dockerignore b/.dockerignore index be375585..5d17eabe 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,5 @@ /target +.git/ sqlpage.db docs/ README.md @@ -11,4 +12,4 @@ configuration.md examples/ mssql/ tests/ -.idea/ \ No newline at end of file +.idea/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e173d927..be312d63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,15 @@ on: pull_request: branches: - "main" + paths-ignore: + - "docs/**" + - "README.md" + - ".github/workflows/release.yml" + - ".github/workflows/official-site.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: CARGO_TERM_COLOR: always @@ -36,7 +45,28 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - run: cargo fmt --all -- --check - run: cargo clippy --all-targets --all-features -- -D warnings - - run: cargo test --features odbc-static + # The database matrix below runs the same Rust test harnesses against + # different DATABASE_URL values. Package the Linux test executables once + # here so those jobs do not recompile SQLPage or its dependencies. + - name: Package Linux Rust test binaries + run: | + set -euo pipefail + rm -rf target/sqlpage-test-binaries + mkdir -p target/sqlpage-test-binaries + cargo test --features odbc-static --no-run --message-format=json \ + | jq -r 'select(.profile.test == true and .executable != null) | .executable' \ + | while IFS= read -r test_binary; do + cp -- "$test_binary" target/sqlpage-test-binaries/ + done + test -n "$(find target/sqlpage-test-binaries -maxdepth 1 -type f -print -quit)" + tar -C target/sqlpage-test-binaries -czf target/sqlpage-linux-test-binaries.tar.gz . + - name: Build Linux binary + run: cargo build --features odbc-static + - name: Upload Linux Rust test binaries + uses: actions/upload-artifact@v7 + with: + name: sqlpage-linux-test-binaries + path: "target/sqlpage-linux-test-binaries.tar.gz" - name: Upload Linux binary uses: actions/upload-artifact@v7 with: @@ -45,9 +75,13 @@ jobs: test: runs-on: ubuntu-latest + needs: compile_and_lint strategy: matrix: include: + - database: sqlite + container: "" + db_url: "sqlite::memory:" - database: postgres container: postgres db_url: "postgres://root:Password123!@127.0.0.1/sqlpage" @@ -66,13 +100,18 @@ jobs: db_url: "Driver=Oracle 21 ODBC driver;Dbq=//127.0.0.1:1521/FREEPDB1;Uid=root;Pwd=Password123!" steps: - uses: actions/checkout@v6 - - name: Set up cargo cache - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 - env: - NODE_OPTIONS: --no-deprecation + # Reuse the exact Linux test harnesses produced by compile_and_lint. + # This keeps the DB matrix focused on database behavior instead of + # compiling the same Rust crate five more times. + - name: Download Linux Rust test binaries + uses: actions/download-artifact@v8 with: - shared-key: rust-sqlpage-proj-test - save-if: false + name: sqlpage-linux-test-binaries + path: target + - name: Extract Linux Rust test binaries + run: | + mkdir -p target/sqlpage-test-binaries + tar -xzf target/sqlpage-linux-test-binaries.tar.gz -C target/sqlpage-test-binaries - name: Install PostgreSQL ODBC driver if: matrix.setup_odbc run: sudo apt-get install -y odbc-postgresql @@ -86,15 +125,28 @@ jobs: sudo alien -i oracle-instantclient-odbc-21.21.0.0.0-1.el8.x86_64.rpm sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/libaio.so.1 sudo /usr/lib/oracle/21/client64/bin/odbc_update_ini.sh / /usr/lib/oracle/21/client64/lib - echo "LD_LIBRARY_PATH=/usr/lib/oracle/21/client64/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=/usr/lib/oracle/21/client64/lib:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" - name: Start database container + if: matrix.container != '' run: docker compose up --wait ${{ matrix.container }} - name: Show container logs - if: failure() + if: failure() && matrix.container != '' run: docker compose logs ${{ matrix.container }} - name: Run tests against ${{ matrix.database }} timeout-minutes: 5 - run: cargo test --features odbc-static + run: | + set -euo pipefail + shopt -s nullglob + test_binaries=(target/sqlpage-test-binaries/*) + if ((${#test_binaries[@]} == 0)); then + echo "No test binaries were found in target/sqlpage-test-binaries" >&2 + exit 1 + fi + for test_binary in "${test_binaries[@]}"; do + echo "::group::$(basename "$test_binary")" + "$test_binary" + echo "::endgroup::" + done env: DATABASE_URL: ${{ matrix.db_url }} MALLOC_CHECK_: 3 @@ -108,10 +160,22 @@ jobs: uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 env: NODE_OPTIONS: --no-deprecation - - name: Check port usage - run: netstat -bano + - name: Set up Windows incremental cache + uses: actions/cache@v5 + with: + path: | + target/debug/.fingerprint/sqlpage-* + target/debug/build/sqlpage-* + target/debug/deps/libsqlpage-*.rlib + target/debug/deps/libsqlpage-*.rmeta + target/debug/deps/sqlpage-*.d + target/debug/incremental/sqlpage-* + key: windows-incremental-${{ github.event.pull_request.number || github.ref_name }}-${{ github.sha }} + restore-keys: | + windows-incremental-${{ github.event.pull_request.number || github.ref_name }}- - run: cargo test env: + CARGO_INCREMENTAL: 1 RUST_BACKTRACE: 1 - name: Upload Windows binary uses: actions/upload-artifact@v7 @@ -119,6 +183,48 @@ jobs: name: sqlpage-windows-debug path: "target/debug/sqlpage.exe" + playwright: + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: compile_and_lint + defaults: + run: + working-directory: ./tests/end-to-end + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: lts/* + cache: 'npm' + cache-dependency-path: ./tests/end-to-end/package-lock.json + - run: sudo apt-get update && sudo apt-get install -y unixodbc-dev + - run: npm ci && npx playwright install --with-deps chromium + # The browser tests exercise the official site, but they do not need a + # separate Rust build. Reuse the binary compiled and tested above. + - name: Download Linux binary + uses: actions/download-artifact@v8 + with: + name: sqlpage-linux-debug + path: ${{ runner.temp }}/sqlpage-bin + - name: Start official site and wait for it to be ready + timeout-minutes: 1 + run: | + chmod +x "${{ runner.temp }}/sqlpage-bin/sqlpage" + "${{ runner.temp }}/sqlpage-bin/sqlpage" 2>/tmp/stderrlog & + tail -f /tmp/stderrlog | grep -q "started successfully" + working-directory: ./examples/official-site + - name: Run Playwright tests + run: npx playwright test + - name: show server logs + if: failure() + run: cat /tmp/stderrlog + - uses: actions/upload-artifact@v7 + if: always() + with: + name: playwright-report + path: ./tests/end-to-end/playwright-report/ + retention-days: 30 + docker_build: runs-on: ubuntu-latest strategy: @@ -149,12 +255,27 @@ jobs: suffix="${suffix}-${{ matrix.variant }}" fi echo "suffix=${suffix}" >> "$GITHUB_OUTPUT" + - id: cache-scope + name: Docker cache scope + run: | + ref_scope="main" + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + ref_scope="pr-${{ github.event.pull_request.number }}" + fi + + recipe_hash="${{ hashFiles('Dockerfile', '.cargo/**', 'Cargo.toml', 'Cargo.lock', 'build.rs', 'scripts/**', 'sqlpage/**') }}" + { + echo "current=sqlpage-${ref_scope}${{ steps.suffix.outputs.suffix }}-${recipe_hash}" + echo "main=sqlpage-main${{ steps.suffix.outputs.suffix }}-${recipe_hash}" + } >> "$GITHUB_OUTPUT" - name: Docker meta id: meta uses: docker/metadata-action@v6 with: images: ${{ env.REGISTRY_IMAGE }} flavor: suffix=${{ steps.suffix.outputs.suffix }} + labels: | + org.opencontainers.image.created=1970-01-01T00:00:00Z - name: Set up QEMU uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx @@ -169,19 +290,20 @@ jobs: id: build uses: docker/build-push-action@v7 with: - context: . + # Use BuildKit's Git context instead of the mutable runner workspace. + # The dependency cache should be keyed by committed source, not by a + # per-job local context stream. + context: "{{defaultContext}}" platforms: ${{ matrix.platform }} target: ${{ matrix.variant }} labels: ${{ steps.meta.outputs.labels }} push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} - cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:main${{ steps.suffix.outputs.suffix }} - # don't save cache on prs - cache-to: > - ${{ github.event_name != 'pull_request' - && format('type=registry,ref={0}:main{1},compression=zstd,mode=max', env.REGISTRY_IMAGE, steps.suffix.outputs.suffix) - || '' - }} + cache-from: | + type=gha,scope=${{ steps.cache-scope.outputs.current }} + type=gha,scope=${{ steps.cache-scope.outputs.main }} + type=registry,ref=${{ env.REGISTRY_IMAGE }}:main${{ steps.suffix.outputs.suffix }} + cache-to: type=gha,scope=${{ steps.cache-scope.outputs.current }},mode=max - name: Export digest if: github.event_name != 'pull_request' run: | @@ -209,8 +331,24 @@ jobs: with: images: ${{ env.REGISTRY_IMAGE }} flavor: suffix=-linux-amd64 + labels: | + org.opencontainers.image.created=1970-01-01T00:00:00Z - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 + - id: cache-scope + name: Docker cache scope + run: | + ref_scope="main" + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + ref_scope="pr-${{ github.event.pull_request.number }}" + fi + + recipe_hash="${{ hashFiles('Dockerfile', '.cargo/**', 'Cargo.toml', 'Cargo.lock', 'build.rs', 'scripts/**', 'sqlpage/**') }}" + { + echo "artifact=sqlpage-${ref_scope}-linux-amd64-hurl-${recipe_hash}" + echo "current=sqlpage-${ref_scope}-linux-amd64-${recipe_hash}" + echo "main=sqlpage-main-linux-amd64-${recipe_hash}" + } >> "$GITHUB_OUTPUT" - name: Login to Docker Hub if: github.event_name != 'pull_request' uses: docker/login-action@v4 @@ -220,7 +358,10 @@ jobs: - name: Build image for Hurl examples uses: docker/build-push-action@v7 with: - context: . + # Use BuildKit's Git context instead of the mutable runner workspace. + # The dependency cache should be keyed by committed source, not by a + # per-job local context stream. + context: "{{defaultContext}}" platforms: linux/amd64 target: minimal labels: ${{ steps.meta.outputs.labels }} @@ -228,7 +369,12 @@ jobs: ${{ steps.meta.outputs.tags }} ${{ env.REGISTRY_IMAGE }}:main outputs: type=docker,dest=${{ runner.temp }}/sqlpage.tar - cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:main-linux-amd64 + cache-from: | + type=gha,scope=${{ steps.cache-scope.outputs.artifact }} + type=gha,scope=${{ steps.cache-scope.outputs.current }} + type=gha,scope=${{ steps.cache-scope.outputs.main }} + type=registry,ref=${{ env.REGISTRY_IMAGE }}:main-linux-amd64 + cache-to: type=gha,scope=${{ steps.cache-scope.outputs.artifact }},mode=max - name: Upload SQLPage image uses: actions/upload-artifact@v7 with: @@ -241,14 +387,20 @@ jobs: if: github.event_name != 'pull_request' uses: docker/build-push-action@v7 with: - context: . + # Use BuildKit's Git context instead of the mutable runner workspace. + # The dependency cache should be keyed by committed source, not by a + # per-job local context stream. + context: "{{defaultContext}}" platforms: linux/amd64 target: minimal labels: ${{ steps.meta.outputs.labels }} push: true tags: ${{ steps.meta.outputs.tags }} - cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:main-linux-amd64 - cache-to: type=registry,ref=${{ env.REGISTRY_IMAGE }}:main-linux-amd64,compression=zstd,mode=max + cache-from: | + type=gha,scope=${{ steps.cache-scope.outputs.current }} + type=gha,scope=${{ steps.cache-scope.outputs.main }} + type=registry,ref=${{ env.REGISTRY_IMAGE }}:main-linux-amd64 + cache-to: type=gha,scope=${{ steps.cache-scope.outputs.current }},mode=max - name: Export digest if: github.event_name != 'pull_request' run: | diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml deleted file mode 100644 index 5b7cd3b9..00000000 --- a/.github/workflows/playwright.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: End to end Tests -on: - push: - branches: [ main, master ] - pull_request: - branches: [ main, master ] -jobs: - test: - timeout-minutes: 10 - defaults: - run: - working-directory: ./tests/end-to-end - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Set up cargo cache - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 - env: - NODE_OPTIONS: --no-deprecation - - uses: actions/setup-node@v6 - with: - node-version: lts/* - cache: 'npm' - cache-dependency-path: ./tests/end-to-end/package-lock.json - - run: sudo apt-get update && sudo apt-get install -y unixodbc-dev - - run: npm ci && npx playwright install --with-deps chromium - - name: build sqlpage - run: cargo build - working-directory: ./examples/official-site - - name: start official site and wait for it to be ready - timeout-minutes: 1 - run: | - cargo run 2>/tmp/stderrlog & - tail -f /tmp/stderrlog | grep -q "started successfully" - working-directory: ./examples/official-site - - name: Run Playwright tests - run: npx playwright test - - name: show server logs - if: failure() - run: cat /tmp/stderrlog - - uses: actions/upload-artifact@v7 - if: always() - with: - name: playwright-report - path: ./tests/end-to-end/playwright-report/ - retention-days: 30 diff --git a/Dockerfile b/Dockerfile index 443d4538..7684eab7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,9 @@ RUN cargo init . RUN /usr/local/bin/setup-cross-compilation.sh "$TARGETARCH" "$BUILDARCH" -COPY Cargo.toml Cargo.lock ./ +COPY .cargo/ .cargo/ +COPY Cargo.toml Cargo.lock build.rs ./ +COPY sqlpage/ sqlpage/ RUN /usr/local/bin/build-dependencies.sh COPY . .