diff --git a/.github/actions/sccache/action.yml b/.github/actions/sccache/action.yml index feee3831ee22..a327d042a9c8 100644 --- a/.github/actions/sccache/action.yml +++ b/.github/actions/sccache/action.yml @@ -5,10 +5,6 @@ inputs: description: 'SCCACHE_BASE_DIR for path normalization (default: $GITHUB_WORKSPACE)' required: false default: '' - turbo-token: - description: 'Fallback TURBO_TOKEN if not set in environment (e.g. from secrets.TURBO_TOKEN)' - required: false - default: '' runs: using: 'node20' main: 'main.js' diff --git a/.github/actions/sccache/start.sh b/.github/actions/sccache/start.sh index 02a47d445518..c117421b272b 100644 --- a/.github/actions/sccache/start.sh +++ b/.github/actions/sccache/start.sh @@ -1,28 +1,23 @@ #!/usr/bin/env bash set -euo pipefail -# Normalize TURBO_API and TURBO_TOKEN: prefer environment values and -# fall back to Vercel's public API and the secret passed as an action input. +# Default TURBO_API to Vercel's public API when the workflow does not set it. if [ -z "${TURBO_API:-}" ]; then export TURBO_API="https://api.vercel.com" echo "TURBO_API=${TURBO_API}" >> "$GITHUB_ENV" fi -if [ -z "${TURBO_TOKEN:-}" ]; then - if [ -n "${INPUT_TURBO_TOKEN:-}" ]; then - export TURBO_TOKEN="${INPUT_TURBO_TOKEN}" - echo "TURBO_TOKEN=${TURBO_TOKEN}" >> "$GITHUB_ENV" - else - echo "WARNING: no TURBO_TOKEN available" - fi -fi +# Exported by `vercel/setup-turborepo-remote-cache-action`, and unset entirely +# when that step was skipped. Bind them so the substring expansions below don't +# trip `set -u`, which rejects an unset name even in `${var:0:3}`. +TURBO_TOKEN="${TURBO_TOKEN:-}" +TURBO_TEAM="${TURBO_TEAM:-}" -if [ -z "${TURBO_TEAM:-}" ]; then - export TURBO_TEAM="vtest314-next-adapter-e2e-tests" - echo "TURBO_TEAM=${TURBO_TEAM}" >> "$GITHUB_ENV" +if [ -z "$TURBO_TOKEN" ]; then + echo "WARNING: no TURBO_TOKEN available" fi -echo "::add-mask::${TURBO_TOKEN:-}" +echo "::add-mask::${TURBO_TOKEN}" echo "Cache endpoint: ${TURBO_API:0:9}..." echo "TURBO_TOKEN: ${TURBO_TOKEN:0:3}..." echo "TURBO_TEAM: ${TURBO_TEAM}" diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 14fb997221d1..df6ad3fb49d5 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -30,11 +30,10 @@ env: # --env-mode loose is a breaking change required with turbo 2.x since Strict mode is now the default # TODO: we should add the relevant envs later to to switch to strict mode TURBO_ARGS: '-v --env-mode loose --remote-cache-timeout 90 --summarize --log-order stream' - TURBO_TEAM: 'vtest314-next-adapter-e2e-tests' + TURBO_TEAM: ${{ vars.TURBO_TEAM }} # Prefer shared remote cache across runs, but keep local cache enabled so jobs # degrade gracefully if the remote cache or token is unavailable. TURBO_CACHE: 'local:rw,remote:rw' - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} # Without this environment variable, rust-lld will fail because some dependencies defaults to newer version of macOS by default. # # See https://doc.rust-lang.org/rustc/platform-support/apple-darwin.html#os-version for more details @@ -136,12 +135,22 @@ jobs: runs-on: ubuntu-latest env: NEXT_TELEMETRY_DISABLED: 1 + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 25 persist-credentials: false + - name: Set up Turborepo remote cache + if: ${{ vars.TURBO_TEAM != '' }} + continue-on-error: true + uses: vercel/setup-turborepo-remote-cache-action@135046f5efcadb17337d9b29b8d5b4035ae64b10 # v1.0.0 + with: + team: ${{ vars.TURBO_TEAM }} + - name: Setup node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -247,6 +256,9 @@ jobs: env: # Disable all build caches for production/staging/force-preview deploys NEXT_SKIP_BUILD_CACHE: ${{ contains(fromJSON('["production","staging","force-preview"]'), needs.deploy-target.outputs.value) && '1' || '' }} + permissions: + contents: read + id-token: write steps: # Enable long paths on Windows to avoid MAX_PATH (260 char) errors # with deeply nested node_modules/.pnpm paths @@ -263,6 +275,13 @@ jobs: fetch-depth: 100 persist-credentials: false + - name: Set up Turborepo remote cache + if: ${{ vars.TURBO_TEAM != '' }} + continue-on-error: true + uses: vercel/setup-turborepo-remote-cache-action@135046f5efcadb17337d9b29b8d5b4035ae64b10 # v1.0.0 + with: + team: ${{ vars.TURBO_TEAM }} + - name: Setup node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 if: ${{ !matrix.docker }} @@ -480,11 +499,21 @@ jobs: runs-on: ubuntu-latest-16-core-oss + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Set up Turborepo remote cache + if: ${{ vars.TURBO_TEAM != '' }} + continue-on-error: true + uses: vercel/setup-turborepo-remote-cache-action@135046f5efcadb17337d9b29b8d5b4035ae64b10 # v1.0.0 + with: + team: ${{ vars.TURBO_TEAM }} + - name: Setup node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index e1b128361e42..003bcc291089 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -127,6 +127,9 @@ jobs: build-native: name: build-native + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml needs: ['changes'] if: ${{ needs.changes.outputs.docs-only == 'false' }} @@ -138,6 +141,9 @@ jobs: build-native-windows: name: build-native-windows + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml needs: ['changes'] if: ${{ needs.changes.outputs.docs-only == 'false' }} @@ -151,6 +157,9 @@ jobs: build-next: name: build-next + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: needsPlaywright: 'no' @@ -226,6 +235,9 @@ jobs: lint: name: lint needs: ['build-next'] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: needsPlaywright: 'no' @@ -266,6 +278,9 @@ jobs: name: types and precompiled needs: ['changes', 'build-native', 'build-next'] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: needsPlaywright: 'no' @@ -278,6 +293,9 @@ jobs: needs: ['changes', 'build-next'] if: ${{ needs.changes.outputs.docs-only == 'false' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: needsRust: 'yes' @@ -300,6 +318,9 @@ jobs: needs: ['optimize-ci', 'changes', 'build-next'] if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/test-turbopack-rust-bench-test.yml secrets: inherit @@ -308,6 +329,9 @@ jobs: needs: ['changes', 'build-next'] if: ${{ needs.changes.outputs.docs-only == 'false' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: needsRust: 'yes' @@ -322,6 +346,9 @@ jobs: needs: ['changes', 'build-next'] if: ${{ needs.changes.outputs.docs-only == 'false' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: needsRust: 'yes' @@ -360,6 +387,9 @@ jobs: - '--scenario=heavy-npm-deps-dev --page=homepage' - '--scenario=heavy-npm-deps-build --page=homepage' - '--scenario=heavy-npm-deps-build-turbo-cache-enabled --page=homepage' + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: afterBuild: | @@ -374,6 +404,9 @@ jobs: name: test devlow package needs: ['optimize-ci', 'changes'] if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: skipNativeBuild: 'yes' @@ -403,6 +436,9 @@ jobs: group: [1/7, 2/7, 3/7, 4/7, 5/7, 6/7, 7/7] # Empty value uses default react: ['', '18.3.1'] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: afterBuild: | @@ -444,6 +480,9 @@ jobs: group: [1/7, 2/7, 3/7, 4/7, 5/7, 6/7, 7/7] # Empty value uses default react: ['', '18.3.1'] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: nodeVersion: 20.9.0 @@ -481,6 +520,9 @@ jobs: group: [1/5, 2/5, 3/5, 4/5, 5/5] # Empty value uses default react: ['', '18.3.1'] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: nodeVersion: 20.19.x @@ -527,6 +569,9 @@ jobs: group: [1/7, 2/7, 3/7, 4/7, 5/7, 6/7, 7/7] # Empty value uses default react: ['', '18.3.1'] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: nodeVersion: 20.19.x @@ -554,6 +599,9 @@ jobs: needs: ['optimize-ci', 'changes', 'build-next'] if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: skipNativeBuild: 'yes' @@ -583,6 +631,9 @@ jobs: if: false # if: ${{ needs.changes.outputs.docs-only == 'false' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: skipNativeBuild: 'yes' @@ -603,6 +654,9 @@ jobs: matrix: node: [20, 22] # TODO: use env var like [env.NODE_MAINTENANCE_VERSION, env.NODE_LTS_VERSION] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: needsPlaywright: 'no' @@ -626,6 +680,9 @@ jobs: # https://github.com/microsoft/playwright/issues/40724 node: [22, '24.15.0'] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: nodeVersion: ${{ matrix.node }} @@ -650,6 +707,9 @@ jobs: # pin once we update to playwright 1.60.0 or newer. node: [22, '24.15.0'] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: nodeVersion: ${{ matrix.node }} @@ -671,6 +731,9 @@ jobs: matrix: node: [20, 22] # TODO: use env var like [env.NODE_MAINTENANCE_VERSION, env.NODE_LTS_VERSION] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: needsPlaywright: 'no' @@ -693,6 +756,9 @@ jobs: matrix: group: [1/5, 2/5, 3/5, 4/5, 5/5] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: afterBuild: | @@ -719,6 +785,9 @@ jobs: matrix: group: [1/5, 2/5, 3/5, 4/5, 5/5] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: afterBuild: | @@ -866,6 +935,9 @@ jobs: group: [1/10, 2/10, 3/10, 4/10, 5/10, 6/10, 7/10, 8/10, 9/10, 10/10] # Empty value uses default react: ['', '18.3.1'] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: afterBuild: | @@ -889,6 +961,9 @@ jobs: needs: ['optimize-ci', 'changes', 'build-native-windows', 'build-next'] if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: afterBuild: | @@ -911,6 +986,9 @@ jobs: needs: ['optimize-ci', 'changes', 'build-native-windows', 'build-next'] if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: nodeVersion: 20.9.0 @@ -936,6 +1014,9 @@ jobs: needs: ['optimize-ci', 'changes', 'build-native-windows', 'build-next'] if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: afterBuild: | @@ -975,6 +1056,9 @@ jobs: group: [1/10, 2/10, 3/10, 4/10, 5/10, 6/10, 7/10, 8/10, 9/10, 10/10] # Empty value uses default react: ['', '18.3.1'] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: afterBuild: | @@ -994,6 +1078,9 @@ jobs: needs: ['optimize-ci', 'changes', 'build-native', 'build-next'] if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: browser: 'firefox webkit' @@ -1040,6 +1127,9 @@ jobs: fail-fast: false matrix: group: [1/6, 2/6, 3/6, 4/6, 5/6, 6/6] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: afterBuild: | @@ -1076,6 +1166,9 @@ jobs: fail-fast: false matrix: group: [1/7, 2/7, 3/7, 4/7, 5/7, 6/7, 7/7] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: afterBuild: | diff --git a/.github/workflows/build_reusable.yml b/.github/workflows/build_reusable.yml index 93d46d5fd1c3..24f5336aab47 100644 --- a/.github/workflows/build_reusable.yml +++ b/.github/workflows/build_reusable.yml @@ -121,11 +121,10 @@ env: # disable backtrace for test snapshots RUST_BACKTRACE: 0 - TURBO_TEAM: 'vtest314-next-adapter-e2e-tests' + TURBO_TEAM: ${{ vars.TURBO_TEAM }} # Prefer shared remote cache across runs, but keep local cache enabled so jobs # degrade gracefully if the remote cache or token is unavailable. TURBO_CACHE: 'local:rw,remote:rw' - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} NEXT_TELEMETRY_DISABLED: 1 # `skipNativeInstall: 'no'` must force the download even though CI otherwise @@ -185,6 +184,13 @@ jobs: fetch-depth: 25 persist-credentials: false + - name: Set up Turborepo remote cache + if: ${{ vars.TURBO_TEAM != '' }} + continue-on-error: true + uses: vercel/setup-turborepo-remote-cache-action@135046f5efcadb17337d9b29b8d5b4035ae64b10 # v1.0.0 + with: + team: ${{ vars.TURBO_TEAM }} + # Depends on rust-toolchain.toml in the checkout - name: Install Rust id: install-rust @@ -288,8 +294,6 @@ jobs: - name: Start sccache uses: ./.github/actions/sccache if: ${{ runner.os != 'Windows' && (inputs.uploadNativeArtifact || inputs.needsNextest == 'yes' || inputs.needsRust == 'yes') }} - with: - turbo-token: ${{ secrets.TURBO_TOKEN }} # Infer the Rust target triple from the runner's OS/arch. napi would build # for the host target if `--target` were omitted, but Turborepo does not diff --git a/.github/workflows/integration_tests_reusable.yml b/.github/workflows/integration_tests_reusable.yml index 824801707226..15648ae1ac6b 100644 --- a/.github/workflows/integration_tests_reusable.yml +++ b/.github/workflows/integration_tests_reusable.yml @@ -45,6 +45,9 @@ jobs: # First, build Next.js to execute across tests. build-next: name: build-next + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: nodeVersion: ${{ inputs.nodeVersion }} @@ -54,6 +57,9 @@ jobs: build-native: name: build-native + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: nodeVersion: ${{ inputs.nodeVersion }} @@ -92,6 +98,9 @@ jobs: fail-fast: false matrix: group: ${{ fromJSON(needs.generate-matrices.outputs.e2e) }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: nodeVersion: ${{ inputs.nodeVersion }} @@ -124,6 +133,9 @@ jobs: fail-fast: false matrix: group: ${{ fromJSON(needs.generate-matrices.outputs.integration) }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml with: nodeVersion: ${{ inputs.nodeVersion }} diff --git a/.github/workflows/pull_request_stats.yml b/.github/workflows/pull_request_stats.yml index f523bea33af7..92e85e87a89a 100644 --- a/.github/workflows/pull_request_stats.yml +++ b/.github/workflows/pull_request_stats.yml @@ -17,11 +17,10 @@ env: TURBO_VERSION: 2.9.4 TEST_CONCURRENCY: 6 - TURBO_TEAM: 'vtest314-next-adapter-e2e-tests' + TURBO_TEAM: ${{ vars.TURBO_TEAM }} # Prefer shared remote cache across runs, but keep local cache enabled so jobs # degrade gracefully if the remote cache or token is unavailable. TURBO_CACHE: 'local:rw,remote:rw' - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} NEXT_TELEMETRY_DISABLED: 1 # Vercel KV Store for test timings KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }} @@ -75,6 +74,9 @@ jobs: retention-days: 1 build: + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml secrets: inherit with: @@ -94,6 +96,9 @@ jobs: # Keep in sync with STATS_EXPECTED_BUNDLERS bundler: [webpack, turbopack] runs-on: ubuntu-latest-16-core-arm-oss + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -113,13 +118,20 @@ jobs: - run: cp -r packages/next-swc/native .github/actions/next-stats-action/native if: ${{ steps.docs-change.outputs.DOCS_CHANGE == 'nope' }} + - name: Set up Turborepo remote cache + if: ${{ vars.TURBO_TEAM != '' }} + continue-on-error: true + uses: vercel/setup-turborepo-remote-cache-action@135046f5efcadb17337d9b29b8d5b4035ae64b10 # v1.0.0 + with: + team: ${{ vars.TURBO_TEAM }} + + # `TURBO_TOKEN` reaches this Docker action through the job environment. - uses: ./.github/actions/next-stats-action if: ${{ steps.docs-change.outputs.DOCS_CHANGE == 'nope' }} with: bundler: ${{ matrix.bundler }} env: TURBO_TEAM: ${{ env.TURBO_TEAM }} - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_CACHE: ${{ env.TURBO_CACHE }} PREVIEW_BUILDS_BASE_URL: ${{ vars.PREVIEW_BUILDS_BASE_URL }} diff --git a/.github/workflows/rspack-nextjs-build-integration-tests.yml b/.github/workflows/rspack-nextjs-build-integration-tests.yml index 5c453e9a8307..361305ca2518 100644 --- a/.github/workflows/rspack-nextjs-build-integration-tests.yml +++ b/.github/workflows/rspack-nextjs-build-integration-tests.yml @@ -9,6 +9,9 @@ on: jobs: test-dev: name: Rspack integration tests + permissions: + contents: read + id-token: write uses: ./.github/workflows/integration_tests_reusable.yml with: name: rspack-production diff --git a/.github/workflows/rspack-nextjs-dev-integration-tests.yml b/.github/workflows/rspack-nextjs-dev-integration-tests.yml index a9b9cf7d20eb..50b3dfce9b0a 100644 --- a/.github/workflows/rspack-nextjs-dev-integration-tests.yml +++ b/.github/workflows/rspack-nextjs-dev-integration-tests.yml @@ -9,6 +9,9 @@ on: jobs: test-dev: name: Rspack integration tests + permissions: + contents: read + id-token: write uses: ./.github/workflows/integration_tests_reusable.yml with: name: rspack-development diff --git a/.github/workflows/test-turbopack-rust-bench-test.yml b/.github/workflows/test-turbopack-rust-bench-test.yml index f4c96595eeb3..66a8b01a48d8 100644 --- a/.github/workflows/test-turbopack-rust-bench-test.yml +++ b/.github/workflows/test-turbopack-rust-bench-test.yml @@ -15,8 +15,7 @@ on: env: TURBOPACK_BENCH_COUNTS: '100' TURBOPACK_BENCH_PROGRESS: '1' - TURBO_TEAM: 'vtest314-next-adapter-e2e-tests' - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} jobs: test: @@ -49,10 +48,15 @@ jobs: - run: pnpm install working-directory: turbopack/benchmark-apps + - name: Set up Turborepo remote cache + if: ${{ vars.TURBO_TEAM != '' }} + continue-on-error: true + uses: vercel/setup-turborepo-remote-cache-action@135046f5efcadb17337d9b29b8d5b4035ae64b10 # v1.0.0 + with: + team: ${{ vars.TURBO_TEAM }} + - name: Start sccache uses: ./.github/actions/sccache - with: - turbo-token: ${{ secrets.TURBO_TOKEN }} - name: Build benchmarks for tests timeout-minutes: 120 diff --git a/.github/workflows/test_e2e_deploy_release.yml b/.github/workflows/test_e2e_deploy_release.yml index a1572222f843..749ee9887af7 100644 --- a/.github/workflows/test_e2e_deploy_release.yml +++ b/.github/workflows/test_e2e_deploy_release.yml @@ -174,6 +174,9 @@ jobs: name: Run Deploy Tests (Webpack) needs: setup if: ${{ github.event.inputs.deployScriptPath == '' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml secrets: inherit strategy: @@ -208,6 +211,9 @@ jobs: test-deploy-turbopack: name: Run Deploy Tests (Turbopack) needs: [setup] + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml secrets: inherit strategy: @@ -258,6 +264,9 @@ jobs: name: Run Deploy Adapter Tests (Turbopack) needs: setup if: ${{ github.event.inputs.deployScriptPath == '' }} + permissions: + contents: read + id-token: write uses: ./.github/workflows/build_reusable.yml secrets: inherit strategy: diff --git a/.github/workflows/test_e2e_project_reset_cron.yml b/.github/workflows/test_e2e_project_reset_cron.yml index 350e933d2107..ede112f5cb8f 100644 --- a/.github/workflows/test_e2e_project_reset_cron.yml +++ b/.github/workflows/test_e2e_project_reset_cron.yml @@ -14,12 +14,6 @@ env: VERCEL_ADAPTER_TEST_TOKEN: ${{ secrets.VERCEL_ADAPTER_TEST_TOKEN }} VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbo-e2e-tests VERCEL_TURBOPACK_TEST_TOKEN: ${{ secrets.VERCEL_TURBOPACK_TEST_TOKEN }} - # Unrelated to the teams above: this is the Turborepo remote cache team. - TURBO_TEAM: 'vtest314-next-adapter-e2e-tests' - # Prefer shared remote cache across runs, but keep local cache enabled so jobs - # degrade gracefully if the remote cache or token is unavailable. - TURBO_CACHE: 'local:rw,remote:rw' - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} run-name: test-e2e-project-reset (scheduled) diff --git a/.github/workflows/turbopack-benchmark.yml b/.github/workflows/turbopack-benchmark.yml index deda33be2d23..421163ff3a6b 100644 --- a/.github/workflows/turbopack-benchmark.yml +++ b/.github/workflows/turbopack-benchmark.yml @@ -20,11 +20,10 @@ concurrency: env: CI: 1 RUST_LOG: 'off' - TURBO_TEAM: 'vtest314-next-adapter-e2e-tests' + TURBO_TEAM: ${{ vars.TURBO_TEAM }} # Prefer shared remote cache across runs, but keep local cache enabled so jobs # degrade gracefully if the remote cache or token is unavailable. TURBO_CACHE: 'local:rw,remote:rw' - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} permissions: contents: read @@ -45,10 +44,15 @@ jobs: - name: Install cargo-codspeed run: cargo binstall --locked --no-confirm cargo-codspeed@3.0.5 + - name: Set up Turborepo remote cache + if: ${{ vars.TURBO_TEAM != '' }} + continue-on-error: true + uses: vercel/setup-turborepo-remote-cache-action@135046f5efcadb17337d9b29b8d5b4035ae64b10 # v1.0.0 + with: + team: ${{ vars.TURBO_TEAM }} + - name: Start sccache uses: ./.github/actions/sccache - with: - turbo-token: ${{ secrets.TURBO_TOKEN }} - name: Install pnpm dependencies working-directory: turbopack/benchmark-apps @@ -82,10 +86,15 @@ jobs: - name: Install cargo-codspeed run: cargo binstall --locked --no-confirm cargo-codspeed@3.0.5 + - name: Set up Turborepo remote cache + if: ${{ vars.TURBO_TEAM != '' }} + continue-on-error: true + uses: vercel/setup-turborepo-remote-cache-action@135046f5efcadb17337d9b29b8d5b4035ae64b10 # v1.0.0 + with: + team: ${{ vars.TURBO_TEAM }} + - name: Start sccache uses: ./.github/actions/sccache - with: - turbo-token: ${{ secrets.TURBO_TOKEN }} - name: Install pnpm dependencies working-directory: turbopack/benchmark-apps @@ -120,10 +129,15 @@ jobs: - name: Install cargo-codspeed run: cargo binstall --locked --no-confirm cargo-codspeed@3.0.5 + - name: Set up Turborepo remote cache + if: ${{ vars.TURBO_TEAM != '' }} + continue-on-error: true + uses: vercel/setup-turborepo-remote-cache-action@135046f5efcadb17337d9b29b8d5b4035ae64b10 # v1.0.0 + with: + team: ${{ vars.TURBO_TEAM }} + - name: Start sccache uses: ./.github/actions/sccache - with: - turbo-token: ${{ secrets.TURBO_TOKEN }} - name: Build the benchmark target(s) env: diff --git a/.github/workflows/turbopack-nextjs-build-integration-tests.yml b/.github/workflows/turbopack-nextjs-build-integration-tests.yml index 8b1b20fba60c..f8227abbd09c 100644 --- a/.github/workflows/turbopack-nextjs-build-integration-tests.yml +++ b/.github/workflows/turbopack-nextjs-build-integration-tests.yml @@ -8,6 +8,9 @@ on: jobs: test-dev: name: Next.js integration tests + permissions: + contents: read + id-token: write uses: ./.github/workflows/integration_tests_reusable.yml with: name: turbopack-production diff --git a/.github/workflows/turbopack-nextjs-dev-integration-tests.yml b/.github/workflows/turbopack-nextjs-dev-integration-tests.yml index 6a07bbcf8443..86adfa738e76 100644 --- a/.github/workflows/turbopack-nextjs-dev-integration-tests.yml +++ b/.github/workflows/turbopack-nextjs-dev-integration-tests.yml @@ -8,6 +8,9 @@ on: jobs: test-dev: name: Next.js integration tests + permissions: + contents: read + id-token: write uses: ./.github/workflows/integration_tests_reusable.yml with: name: turbopack-development diff --git a/jest.config.js b/jest.config.js index 3a01443419bf..19ab0ab0485f 100644 --- a/jest.config.js +++ b/jest.config.js @@ -29,11 +29,20 @@ const customJestConfig = { // Jest does not normalize/resolve paths in modulePathIgnorePatterns so we can't // prefix with /../ like we do in roots. 'packages/next/src/compiled/', + '/development/app-dir/non-context-aware-addon/bindings', + '/development/app-dir/non-context-aware-addon/single-context-addon', '/development/app-dir/ssr-in-rsc/internal-pkg/', '/e2e/app-dir/self-importing-package/internal-pkg', '/e2e/app-dir/self-importing-package-monorepo/internal-pkg', '/e2e/app-dir/server-source-maps/fixtures/default/internal-pkg', + '/e2e/app-dir/turbopack-reports/bindings', + '/e2e/app-dir/turbopack-reports/native-addon', + '/e2e/prerender-native-module/bindings', + '/e2e/prerender-native-module/native-addon', + '/e2e/prerender-native-module/native-addon-wrapper', '/e2e/transpile-packages-typescript-foreign/pkg', + '/production/prerender-worker-threads/bindings', + '/production/prerender-worker-threads/single-context-addon', '/production/standalone-mode/tracing-side-effects-false/foo', '/production/standalone-mode/tracing-static-files/foo', '/production/standalone-mode/tracing-unparsable/foo', diff --git a/packages/create-next-app/helpers/examples.ts b/packages/create-next-app/helpers/examples.ts index b52859205ab3..71e0328923b5 100644 --- a/packages/create-next-app/helpers/examples.ts +++ b/packages/create-next-app/helpers/examples.ts @@ -67,10 +67,13 @@ export function hasRepo({ branch, filePath, }: RepoInfo): Promise { - const contentsUrl = `https://api.github.com/repos/${username}/${name}/contents` - const packagePath = `${filePath ? `/${filePath}` : ''}/package.json` + // Avoid GitHub's low unauthenticated API rate limit. create-next-app only + // needs to verify that the example's package.json can be downloaded. + const packagePath = `${filePath ? `${filePath}/` : ''}package.json` - return isUrlOk(contentsUrl + packagePath + `?ref=${branch}`) + return isUrlOk( + `https://raw.githubusercontent.com/${username}/${name}/${branch}/${packagePath}` + ) } export function existsInRepo(nameOrUrl: string): Promise { diff --git a/skills/next-cache-components-adoption/SKILL.md b/skills/next-cache-components-adoption/SKILL.md index b5cf8f2a3793..91bb76403ff6 100644 --- a/skills/next-cache-components-adoption/SKILL.md +++ b/skills/next-cache-components-adoption/SKILL.md @@ -55,7 +55,7 @@ In both, the per-route success bar is the same: **dev loop reports no errors AND Three classes of blocker come up, usually in this order: 1. **Request-time reads** (`cookies()`, `headers()`, `await params`, `await searchParams`). All four block when awaited at the top of a page or layout. `params` and `searchParams` often get missed because they're not framed as "request data" the way cookies and headers are. The fix is to push the read into a ``-wrapped child — and for `params`/`searchParams`, forward the promise into the child and await it there; don't `await` at the page top. -2. **Sync-IO at module/render time** (`new Date()`, `Date.now()`, `Math.random()`, `crypto.randomUUID()`). These fail the build even with `instant = false` — the opt-out doesn't suppress them. If they're in a shared layout, they block every route under it. The codemod can't fix them; you have to translate each one by hand before the build can pass (see the [incremental pre-step](#incremental)). Grep the whole repo for these calls before running anything else. +2. **Sync-IO at module/render time** (`new Date()`, `Date.now()`, `Math.random()`, `crypto.randomUUID()`). These fail the build even with `instant = false` — the opt-out doesn't suppress them. If they're in a shared layout, they block every route under it. The codemod can't fix them; after running it, use the build errors to identify which calls to translate by hand (see the [incremental pre-step](#incremental)). 3. **`"use cache"` files that read request data.** A file with a top-level `"use cache"` directive can't export `instant`; combining the two errors with `Only async functions are allowed to be exported in a "use cache" file.`, which means the directive was wrong for that route. Remove it before running the codemod. ## working surfaces @@ -106,17 +106,9 @@ If there's no user to ask, default to **Incremental** and document the choice. ### incremental -Before invoking the codemod, fix the two classes of blocker it can't. +Before invoking the codemod, fix the blocker that does not require build feedback. -1. **Sync-IO at module/render time.** Grep the whole repo for `new Date()`, `Date.now()`, `Math.random()`, and `crypto.randomUUID()` (not only `app/**/layout.{js,jsx,ts,tsx}` — the read might live in any component imported by a layout). Unblock each match with the `await connection()` + `` fix from its `blocking-prerender-*` error card: it defers the value to request time, exactly as it behaved before the migration, so it needs no product decision. Add this exact comment on the line above the `await connection()`: - - ```tsx - // TODO: Cache Components adoption. Added to unblock the build: remove this connection() to re-trigger the error and review the fix options. - ``` - - It shares the `TODO: Cache Components adoption` prefix with the comments the codemod writes, so the check-in grep finds both. Removing the `await connection()` makes the error fire again with its fix cards — the same motion as removing an opt-out in the loop. - -2. **Incompatible segment configs.** Grep for `^export const (revalidate|dynamic|fetchCache)` across the app directory and translate per the `requires` note above. The codemod does not touch them; leaving them in place fails the build after the codemod. +1. **Incompatible segment configs.** Grep for `^export const (revalidate|dynamic|fetchCache)` across the app directory and translate per the `requires` note above. The codemod does not touch them; leaving them in place fails the build after the codemod. The codemod refuses to run on a dirty working tree. Commit or stash unrelated work first, or pass `--force` to let its edits land alongside your WIP. Common false positive: if you recently upgraded Next.js, `package.json` and the lockfile will already be dirty — commit those first. @@ -140,7 +132,17 @@ The codemod opts every segment out, not only the root, on purpose. Resolution is Because the highest opt-out wins, remove them top-down (root layout first, then descend). Removing a leaf's opt-out does nothing while an ancestor still holds one. -Confirm the pre-step with `next build`. The build is the proof, not the codemod run — a shared layout that calls `new Date()` / `Math.random()` directly still fails regardless of the opt-out (see [background](#background)). +Next, run `next build` to surface blockers the codemod could not handle. The build is the proof, not the codemod run — a shared layout that calls `new Date()` / `Math.random()` directly still fails regardless of the opt-out (see [background](#background)). If the normal build reports a sync-IO error without locating the call, rerun that route with `next build --debug-prerender --debug-build-paths="app/path/to/page.tsx"`. For each sync-IO error it reports: + +1. **Sync-IO at module/render time.** Use the route, originating file and line, and `/docs/messages/` link in the build output to locate the error. If needed, grep the whole repo for `new Date()`, `Date.now()`, `Math.random()`, and `crypto.randomUUID()` (not only `app/**/layout.{js,jsx,ts,tsx}` — the read might live in any component imported by a layout). Do not change unreported matches. Unblock the reported call with the `await connection()` + `` fix from its `blocking-prerender-*` error card: it defers the value to request time, exactly as it behaved before the migration, so it needs no product decision. Add this exact comment on the line above the `await connection()`: + + ```tsx + // TODO: Cache Components adoption. Added to unblock the build: remove this connection() to re-trigger the error and review the fix options. + ``` + + It shares the `TODO: Cache Components adoption` prefix with the comments the codemod writes, so the check-in grep finds both. Removing the `await connection()` makes the error fire again with its fix cards — the same motion as removing an opt-out in the loop. + +After each fix, rerun the scoped build when available, then run `next build` again to find the next blocker. Repeat until the normal build passes. After the build passes, confirm the root layout got an opt-out (`grep -n "export const instant" /layout.*`). The root layout renders every route, including framework routes like `/_not-found`, so if it was missed, add `export const instant = false` to it by hand. diff --git a/test/.gitignore b/test/.gitignore index 5782cc142201..df8657ce4030 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -10,3 +10,9 @@ development/**/tsconfig.json rspack-test-junit-report/ test-junit-report/ turbopack-test-junit-report/ + +# node-gyp output from the compiled native addon fixtures. The addons are built +# inside each test's throwaway install directory, so this only catches builds +# someone runs in place while iterating on a fixture. +**/native-addon/build/ +**/single-context-addon/build/ diff --git a/test/development/app-dir/non-context-aware-addon/app/layout.tsx b/test/development/app-dir/non-context-aware-addon/app/layout.tsx new file mode 100644 index 000000000000..e7077399c03c --- /dev/null +++ b/test/development/app-dir/non-context-aware-addon/app/layout.tsx @@ -0,0 +1,7 @@ +export default function Root({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ) +} diff --git a/test/development/app-dir/non-context-aware-addon/app/page.tsx b/test/development/app-dir/non-context-aware-addon/app/page.tsx new file mode 100644 index 000000000000..30d51b919970 --- /dev/null +++ b/test/development/app-dir/non-context-aware-addon/app/page.tsx @@ -0,0 +1,7 @@ +import addon from 'single-context-addon' + +// `CONTEXT_AWARE` comes from the compiled binary, so rendering it proves the addon +// really loaded rather than resolving to an empty module. +export default function Page() { + return

{String(addon.CONTEXT_AWARE)}

+} diff --git a/test/development/app-dir/non-context-aware-addon/bindings/index.js b/test/development/app-dir/non-context-aware-addon/bindings/index.js new file mode 100644 index 000000000000..fd4a2e414af0 --- /dev/null +++ b/test/development/app-dir/non-context-aware-addon/bindings/index.js @@ -0,0 +1,62 @@ +// A minimal stand-in for the `bindings` npm package, reproducing only the part +// both bundlers model: locate the calling package's root by walking up for a +// `package.json`, then load the compiled binary from `build/Release`. +// +// Neither bundler evaluates this file. `@vercel/nft` maps the `bindings` +// specifier to its own bundled copy, and Turbopack maps it to +// `WellKnownFunctionKind::NodeBindings`; both then resolve the binary's path +// themselves. `build/Release/` is node-gyp's default output location and +// is on both of their candidate lists, so this agrees with what they trace. + +const fs = require('fs') +const path = require('path') + +function getCallerFile() { + const { prepareStackTrace } = Error + try { + Error.prepareStackTrace = (_error, stack) => stack + for (const frame of new Error().stack) { + const fileName = frame.getFileName() + if (typeof fileName === 'string' && fileName !== __filename) { + return fileName + } + } + } finally { + Error.prepareStackTrace = prepareStackTrace + } + throw new Error('bindings: could not determine the calling file') +} + +function getPackageRoot(file) { + let dir = path.dirname(file) + while (!fs.existsSync(path.join(dir, 'package.json'))) { + const parent = path.dirname(dir) + if (parent === dir) { + throw new Error(`bindings: found no package.json above ${file}`) + } + dir = parent + } + return dir +} + +module.exports = function bindings(name) { + const binary = path.join( + getPackageRoot(getCallerFile()), + 'build', + 'Release', + name + ) + + if (!fs.existsSync(binary)) { + throw new Error( + `bindings: ${binary} does not exist. The fixture addon is compiled by ` + + `node-gyp during install, which requires the package to be listed in ` + + `the test's pnpm.onlyBuiltDependencies.` + ) + } + + // Deliberately not wrapped. A non-context-aware addon fails on this line with + // Node's own `ERR_DLOPEN_FAILED` / "Module did not self-register", and the + // worker-thread tests assert on that message. + return require(binary) +} diff --git a/test/development/app-dir/non-context-aware-addon/bindings/package.json b/test/development/app-dir/non-context-aware-addon/bindings/package.json new file mode 100644 index 000000000000..a942bce5029c --- /dev/null +++ b/test/development/app-dir/non-context-aware-addon/bindings/package.json @@ -0,0 +1,6 @@ +{ + "name": "bindings", + "version": "1.0.0", + "description": "Minimal stand-in for the bindings npm package, for tests", + "main": "index.js" +} diff --git a/test/development/app-dir/non-context-aware-addon/next.config.js b/test/development/app-dir/non-context-aware-addon/next.config.js new file mode 100644 index 000000000000..463d2c176b5d --- /dev/null +++ b/test/development/app-dir/non-context-aware-addon/next.config.js @@ -0,0 +1,10 @@ +/** + * @type {import('next').NextConfig} + */ +const nextConfig = { + // The addon resolves and loads a compiled binary at runtime, so it has to stay + // external rather than being bundled into the server build. + serverExternalPackages: ['single-context-addon'], +} + +module.exports = nextConfig diff --git a/test/development/app-dir/non-context-aware-addon/non-context-aware-addon.test.ts b/test/development/app-dir/non-context-aware-addon/non-context-aware-addon.test.ts new file mode 100644 index 000000000000..264217e21555 --- /dev/null +++ b/test/development/app-dir/non-context-aware-addon/non-context-aware-addon.test.ts @@ -0,0 +1,33 @@ +import { nextTestSetup } from 'e2e-utils' + +// A native addon declared with `NODE_MODULE` rather than `NODE_MODULE_INIT` can +// only be loaded once per process, so loading it on a second thread of that +// process fails with `ERR_DLOPEN_FAILED` / "Module did not self-register". +// +// `next dev` runs some work on worker threads regardless of +// `experimental.workerThreads`: the dev validation pool hardcodes +// `enableWorkerThreads: true`. This asserts that evaluating a route still does not +// put such an addon on one of those threads, so it would catch a change that moved +// route evaluation onto that pool. +// +// The expectation is the same with and without Cache Components. Both were checked +// by logging `isMainThread` from the page's module scope: the route is evaluated on +// the main thread either way, so there is nothing to fork on here. +describe('non-context-aware addon in development', () => { + const { next } = nextTestSetup({ + files: __dirname, + // `bindings` is a direct dependency rather than one of the addon, and both use + // `file:` so pnpm copies them into the virtual store. See the comments in + // test/production/prerender-worker-threads for why neither is `link:`. + dependencies: require('./package.json').dependencies, + }) + + it('should render a route that loads a non-context-aware addon', async () => { + const $ = await next.render$('/') + + // `false` is the value the non-context-aware binary exports, so reading it back + // confirms the addon was loaded rather than stubbed out. + expect($('#context-aware').text()).toBe('false') + expect(next.cliOutput).not.toContain('Module did not self-register') + }) +}) diff --git a/test/development/app-dir/non-context-aware-addon/package.json b/test/development/app-dir/non-context-aware-addon/package.json new file mode 100644 index 000000000000..e434d0f3de1e --- /dev/null +++ b/test/development/app-dir/non-context-aware-addon/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "single-context-addon": "file:single-context-addon", + "bindings": "file:bindings" + } +} diff --git a/test/development/app-dir/non-context-aware-addon/pnpm-workspace.yaml b/test/development/app-dir/non-context-aware-addon/pnpm-workspace.yaml new file mode 100644 index 000000000000..4f9ffe8d637b --- /dev/null +++ b/test/development/app-dir/non-context-aware-addon/pnpm-workspace.yaml @@ -0,0 +1,5 @@ +# `single-context-addon` is compiled by node-gyp during install, and pnpm does +# not run a dependency's build scripts unless they are approved here. The repo +# root uses the same key for the one package it allows to build. +allowBuilds: + single-context-addon: true diff --git a/test/development/app-dir/non-context-aware-addon/single-context-addon/addon.cc b/test/development/app-dir/non-context-aware-addon/single-context-addon/addon.cc new file mode 100644 index 000000000000..8ca5eb993a36 --- /dev/null +++ b/test/development/app-dir/non-context-aware-addon/single-context-addon/addon.cc @@ -0,0 +1,29 @@ +// A deliberately non-context-aware addon. `NODE_MODULE` registers once per +// process, so loading it on the main thread and then again on a worker thread +// fails: the second `dlopen` returns the cached handle, the module constructor +// does not run again, and Node reports +// +// Error [ERR_DLOPEN_FAILED]: Module did not self-register: ''. +// +// which is the failure PR #9199 and PR #25063 both worked around by keeping +// worker threads off by default. Do not port this to Node-API or +// `NODE_MODULE_INIT`: either would make it context-aware and the tests that +// depend on it would silently start passing for the wrong reason. + +#include + +namespace { + +void Init(v8::Local exports) { + v8::Isolate* isolate = v8::Isolate::GetCurrent(); + v8::Local context = isolate->GetCurrentContext(); + exports + ->Set(context, + v8::String::NewFromUtf8(isolate, "CONTEXT_AWARE").ToLocalChecked(), + v8::Boolean::New(isolate, false)) + .Check(); +} + +} // namespace + +NODE_MODULE(NODE_GYP_MODULE_NAME, Init) diff --git a/test/development/app-dir/non-context-aware-addon/single-context-addon/binding.gyp b/test/development/app-dir/non-context-aware-addon/single-context-addon/binding.gyp new file mode 100644 index 000000000000..94ec15c12cef --- /dev/null +++ b/test/development/app-dir/non-context-aware-addon/single-context-addon/binding.gyp @@ -0,0 +1,8 @@ +{ + "targets": [ + { + "target_name": "single_context_addon", + "sources": ["addon.cc"] + } + ] +} diff --git a/test/development/app-dir/non-context-aware-addon/single-context-addon/index.js b/test/development/app-dir/non-context-aware-addon/single-context-addon/index.js new file mode 100644 index 000000000000..69da152bd5ef --- /dev/null +++ b/test/development/app-dir/non-context-aware-addon/single-context-addon/index.js @@ -0,0 +1 @@ +module.exports = require('bindings')('single_context_addon.node') diff --git a/test/development/app-dir/non-context-aware-addon/single-context-addon/package.json b/test/development/app-dir/non-context-aware-addon/single-context-addon/package.json new file mode 100644 index 000000000000..6a4de0e146a9 --- /dev/null +++ b/test/development/app-dir/non-context-aware-addon/single-context-addon/package.json @@ -0,0 +1,6 @@ +{ + "name": "single-context-addon", + "version": "1.0.0", + "description": "Non-context-aware native addon fixture, compiled by node-gyp", + "main": "index.js" +} diff --git a/test/e2e/app-dir/server-components-externals/app/predefined/page.tsx b/test/e2e/app-dir/server-components-externals/app/predefined/page.tsx index 3ceffa035ef1..0566832ffd31 100644 --- a/test/e2e/app-dir/server-components-externals/app/predefined/page.tsx +++ b/test/e2e/app-dir/server-components-externals/app/predefined/page.tsx @@ -1,4 +1,4 @@ -import { dir } from 'sqlite3' +import { dir } from 'keyv' export default function Predefined() { return
{dir}
diff --git a/test/e2e/app-dir/server-components-externals/index.test.ts b/test/e2e/app-dir/server-components-externals/index.test.ts index b26d2802774e..ab6e6cf6b075 100644 --- a/test/e2e/app-dir/server-components-externals/index.test.ts +++ b/test/e2e/app-dir/server-components-externals/index.test.ts @@ -26,8 +26,13 @@ describe('app-dir - server components externals', () => { it('uses externals for predefined list in server-external-packages.json', async () => { const $ = await next.render$('/predefined') + // `keyv` is on the built-in list in + // packages/next/src/lib/server-external-packages.jsonc. Resolving to the + // package's own directory is what proves it stayed external: a bundled copy + // would report the chunk's directory instead. The package here is a stub, so + // the assertion is about the list rather than about anything `keyv` does. const text = $('#directory').text() - expect(text).toBe(path.join(next.testDir, 'node_modules', 'sqlite3')) + expect(text).toBe(path.join(next.testDir, 'node_modules', 'keyv')) }) // Inspect webpack server bundles diff --git a/test/e2e/app-dir/server-components-externals/node_modules/sqlite3/index.js b/test/e2e/app-dir/server-components-externals/node_modules/keyv/index.js similarity index 100% rename from test/e2e/app-dir/server-components-externals/node_modules/sqlite3/index.js rename to test/e2e/app-dir/server-components-externals/node_modules/keyv/index.js diff --git a/test/e2e/app-dir/server-components-externals/node_modules/keyv/package.json b/test/e2e/app-dir/server-components-externals/node_modules/keyv/package.json new file mode 100644 index 000000000000..20bd78c8498c --- /dev/null +++ b/test/e2e/app-dir/server-components-externals/node_modules/keyv/package.json @@ -0,0 +1,6 @@ +{ + "name": "keyv", + "version": "1.0.0", + "description": "keyv stub", + "main": "index.js" +} diff --git a/test/e2e/app-dir/server-components-externals/node_modules/sqlite3/package.json b/test/e2e/app-dir/server-components-externals/node_modules/sqlite3/package.json deleted file mode 100644 index 176d352c31c0..000000000000 --- a/test/e2e/app-dir/server-components-externals/node_modules/sqlite3/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "sqlite3", - "version": "1.0.0", - "description": "sqlite3 stub", - "main": "index.js" -} diff --git a/test/e2e/app-dir/turbopack-reports/.npmrc b/test/e2e/app-dir/turbopack-reports/.npmrc new file mode 100644 index 000000000000..951a2b6e795f --- /dev/null +++ b/test/e2e/app-dir/turbopack-reports/.npmrc @@ -0,0 +1,13 @@ +# Deploy builds install with npm, which symlinks a `file:` directory dependency by +# default. A symlink resolves back into the app, so the server build stops treating +# the package as external and bundles it, and `bindings` cannot survive that: it +# locates its binary from the calling file's path, which becomes a path inside +# `.next` once bundled. +# +# `install-links` makes npm pack the dependency into `node_modules` instead. It only +# takes effect when the dependency is specified without a leading `./`, which is why +# `package.json` says `file:native-addon` rather than `file:./native-addon`. +# +# pnpm ignores this setting and copies the directory into its virtual store either +# way, so the local install is unaffected. +install-links=true diff --git a/test/e2e/app-dir/turbopack-reports/app/native-addon-import-5913/page.tsx b/test/e2e/app-dir/turbopack-reports/app/native-addon-import-5913/page.tsx new file mode 100644 index 000000000000..f0ca00a0b554 --- /dev/null +++ b/test/e2e/app-dir/turbopack-reports/app/native-addon-import-5913/page.tsx @@ -0,0 +1,16 @@ +import nativeAddon from 'native-addon' + +// Regression test for turbopack issue 5913: a package that locates and loads its +// compiled binary through `require('bindings')(...)` used to break the build. +// +// `CONTEXT_AWARE` comes from the compiled addon itself, so rendering it proves the +// binary was really loaded. A resolution that produced an empty module would +// render nothing here rather than passing quietly. +export default function Page() { + return ( + <> +

Hello World

+

{String(nativeAddon.CONTEXT_AWARE)}

+ + ) +} diff --git a/test/e2e/app-dir/turbopack-reports/app/sqlite-import-5913/page.tsx b/test/e2e/app-dir/turbopack-reports/app/sqlite-import-5913/page.tsx deleted file mode 100644 index 1ef4e689d05b..000000000000 --- a/test/e2e/app-dir/turbopack-reports/app/sqlite-import-5913/page.tsx +++ /dev/null @@ -1,6 +0,0 @@ -import sqlite3 from 'sqlite3' - -export default function Page() { - console.log(sqlite3.READONLY) - return

Hello World

-} diff --git a/test/e2e/app-dir/turbopack-reports/bindings/index.js b/test/e2e/app-dir/turbopack-reports/bindings/index.js new file mode 100644 index 000000000000..fd4a2e414af0 --- /dev/null +++ b/test/e2e/app-dir/turbopack-reports/bindings/index.js @@ -0,0 +1,62 @@ +// A minimal stand-in for the `bindings` npm package, reproducing only the part +// both bundlers model: locate the calling package's root by walking up for a +// `package.json`, then load the compiled binary from `build/Release`. +// +// Neither bundler evaluates this file. `@vercel/nft` maps the `bindings` +// specifier to its own bundled copy, and Turbopack maps it to +// `WellKnownFunctionKind::NodeBindings`; both then resolve the binary's path +// themselves. `build/Release/` is node-gyp's default output location and +// is on both of their candidate lists, so this agrees with what they trace. + +const fs = require('fs') +const path = require('path') + +function getCallerFile() { + const { prepareStackTrace } = Error + try { + Error.prepareStackTrace = (_error, stack) => stack + for (const frame of new Error().stack) { + const fileName = frame.getFileName() + if (typeof fileName === 'string' && fileName !== __filename) { + return fileName + } + } + } finally { + Error.prepareStackTrace = prepareStackTrace + } + throw new Error('bindings: could not determine the calling file') +} + +function getPackageRoot(file) { + let dir = path.dirname(file) + while (!fs.existsSync(path.join(dir, 'package.json'))) { + const parent = path.dirname(dir) + if (parent === dir) { + throw new Error(`bindings: found no package.json above ${file}`) + } + dir = parent + } + return dir +} + +module.exports = function bindings(name) { + const binary = path.join( + getPackageRoot(getCallerFile()), + 'build', + 'Release', + name + ) + + if (!fs.existsSync(binary)) { + throw new Error( + `bindings: ${binary} does not exist. The fixture addon is compiled by ` + + `node-gyp during install, which requires the package to be listed in ` + + `the test's pnpm.onlyBuiltDependencies.` + ) + } + + // Deliberately not wrapped. A non-context-aware addon fails on this line with + // Node's own `ERR_DLOPEN_FAILED` / "Module did not self-register", and the + // worker-thread tests assert on that message. + return require(binary) +} diff --git a/test/e2e/app-dir/turbopack-reports/bindings/package.json b/test/e2e/app-dir/turbopack-reports/bindings/package.json new file mode 100644 index 000000000000..a942bce5029c --- /dev/null +++ b/test/e2e/app-dir/turbopack-reports/bindings/package.json @@ -0,0 +1,6 @@ +{ + "name": "bindings", + "version": "1.0.0", + "description": "Minimal stand-in for the bindings npm package, for tests", + "main": "index.js" +} diff --git a/test/e2e/app-dir/turbopack-reports/native-addon/addon.cc b/test/e2e/app-dir/turbopack-reports/native-addon/addon.cc new file mode 100644 index 000000000000..2436d072feb1 --- /dev/null +++ b/test/e2e/app-dir/turbopack-reports/native-addon/addon.cc @@ -0,0 +1,14 @@ +// A context-aware addon. `NODE_MODULE_INIT` registers per-context, so this can +// be loaded from the main thread and from a worker thread in the same process. +// See https://nodejs.org/api/addons.html#context-aware-addons + +#include + +NODE_MODULE_INIT(/* exports, module, context */) { + v8::Isolate* isolate = context->GetIsolate(); + exports + ->Set(context, + v8::String::NewFromUtf8(isolate, "CONTEXT_AWARE").ToLocalChecked(), + v8::Boolean::New(isolate, true)) + .Check(); +} diff --git a/test/e2e/app-dir/turbopack-reports/native-addon/binding.gyp b/test/e2e/app-dir/turbopack-reports/native-addon/binding.gyp new file mode 100644 index 000000000000..9c1110672259 --- /dev/null +++ b/test/e2e/app-dir/turbopack-reports/native-addon/binding.gyp @@ -0,0 +1,8 @@ +{ + "targets": [ + { + "target_name": "native_addon", + "sources": ["addon.cc"] + } + ] +} diff --git a/test/e2e/app-dir/turbopack-reports/native-addon/index.js b/test/e2e/app-dir/turbopack-reports/native-addon/index.js new file mode 100644 index 000000000000..52a1e8bf48a9 --- /dev/null +++ b/test/e2e/app-dir/turbopack-reports/native-addon/index.js @@ -0,0 +1,3 @@ +// Mirrors how `sqlite3` loads its binary, which is the shape both bundlers +// special-case: a single `require('bindings')('.node')` call. +module.exports = require('bindings')('native_addon.node') diff --git a/test/e2e/app-dir/turbopack-reports/native-addon/package.json b/test/e2e/app-dir/turbopack-reports/native-addon/package.json new file mode 100644 index 000000000000..37c6888cf9a9 --- /dev/null +++ b/test/e2e/app-dir/turbopack-reports/native-addon/package.json @@ -0,0 +1,6 @@ +{ + "name": "native-addon", + "version": "1.0.0", + "description": "Context-aware native addon fixture, compiled by node-gyp", + "main": "index.js" +} diff --git a/test/e2e/app-dir/turbopack-reports/next.config.js b/test/e2e/app-dir/turbopack-reports/next.config.js index 807126e4cf0b..b3e90b5c7541 100644 --- a/test/e2e/app-dir/turbopack-reports/next.config.js +++ b/test/e2e/app-dir/turbopack-reports/next.config.js @@ -1,6 +1,13 @@ /** * @type {import('next').NextConfig} */ -const nextConfig = {} +const nextConfig = { + // `native-addon` resolves and loads a compiled binary at runtime, so it has to + // stay external instead of being bundled. Real native packages get this for + // free: `sqlite3`, which this fixture replaced, is on the built-in + // `serverExternalPackages` list in + // packages/next/src/lib/server-external-packages.jsonc. + serverExternalPackages: ['native-addon'], +} module.exports = nextConfig diff --git a/test/e2e/app-dir/turbopack-reports/package.json b/test/e2e/app-dir/turbopack-reports/package.json new file mode 100644 index 000000000000..611949394244 --- /dev/null +++ b/test/e2e/app-dir/turbopack-reports/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "native-addon": "file:native-addon", + "bindings": "file:bindings" + } +} diff --git a/test/e2e/app-dir/turbopack-reports/pnpm-workspace.yaml b/test/e2e/app-dir/turbopack-reports/pnpm-workspace.yaml new file mode 100644 index 000000000000..68e8462f6590 --- /dev/null +++ b/test/e2e/app-dir/turbopack-reports/pnpm-workspace.yaml @@ -0,0 +1,5 @@ +# `native-addon` is compiled by node-gyp during install, and pnpm does +# not run a dependency's build scripts unless they are approved here. The repo +# root uses the same key for the one package it allows to build. +allowBuilds: + native-addon: true diff --git a/test/e2e/app-dir/turbopack-reports/turbopack-reports.test.ts b/test/e2e/app-dir/turbopack-reports/turbopack-reports.test.ts index 7f9da9853bc9..513ff1f5c538 100644 --- a/test/e2e/app-dir/turbopack-reports/turbopack-reports.test.ts +++ b/test/e2e/app-dir/turbopack-reports/turbopack-reports.test.ts @@ -3,18 +3,22 @@ import { nextTestSetup } from 'e2e-utils' describe('turbopack-reports', () => { const { next } = nextTestSetup({ files: __dirname, - dependencies: { - sqlite3: '5.1.7', - }, - packageJson: { - pnpm: { - onlyBuiltDependencies: ['sqlite3'], - }, - }, + // `bindings` is a direct dependency rather than one of `native-addon`. The + // addon resolves it by walking up from the virtual store, and declaring it on + // the addon would make it a non-registry transitive dependency, which + // `blockExoticSubdeps` rejects. + // + // Both use `file:` rather than `link:` so pnpm copies them into the virtual + // store. A linked package resolves to a path inside the app, which the bundler + // treats as app code and tries to bundle, and `bindings` contains a `require` + // it cannot resolve statically. + dependencies: require('./package.json').dependencies, }) - it('should render page importing sqlite3', async () => { - const $ = await next.render$('/sqlite-import-5913') + it('should render page importing a package that loads a native binding', async () => { + const $ = await next.render$('/native-addon-import-5913') expect($('#message').text()).toBe('Hello World') + // Read off the compiled binary, so a module that resolved to nothing fails here. + expect($('#context-aware').text()).toBe('true') }) }) diff --git a/test/e2e/prerender-native-module.test.ts b/test/e2e/prerender-native-module.test.ts index cc28a3f040ae..8823bc01e497 100644 --- a/test/e2e/prerender-native-module.test.ts +++ b/test/e2e/prerender-native-module.test.ts @@ -1,23 +1,27 @@ import path from 'path' -import { FileRef, isReact18, nextTestSetup } from 'e2e-utils' +import { isReact18, nextTestSetup } from 'e2e-utils' + +const fixture = path.join(__dirname, 'prerender-native-module') + +const expectedUsers = [ + { id: 1, first_name: 'john', last_name: 'deux' }, + { id: 2, first_name: 'zeit', last_name: 'geist' }, +] describe('prerender native module', () => { const { next } = nextTestSetup({ - files: { - pages: new FileRef(path.join(__dirname, 'prerender-native-module/pages')), - 'data.sqlite': new FileRef( - path.join(__dirname, 'prerender-native-module/data.sqlite') - ), - }, - dependencies: { - sqlite: '4.0.22', - sqlite3: '5.0.2', - }, - packageJson: { - pnpm: { - onlyBuiltDependencies: ['sqlite3'], - }, - }, + files: fixture, + // `bindings` is a direct dependency rather than one of `native-addon`. The + // addon resolves it by walking up from the virtual store, and declaring it on + // the addon would make it a non-registry transitive dependency, which + // `blockExoticSubdeps` rejects. + // + // These use `file:` rather than `link:` so pnpm copies them into the virtual + // store. A linked package resolves to a path inside the app, which the bundler + // treats as app code and tries to bundle, and `bindings` contains a `require` + // it cannot resolve statically. + dependencies: require('./prerender-native-module/package.json') + .dependencies, }) it('should render index correctly', async () => { @@ -35,10 +39,8 @@ describe('prerender native module', () => { expect(JSON.parse(await browser.elementByCss('#props').text())).toEqual({ params: { slug: 'first' }, blog: true, - users: [ - { id: 1, first_name: 'john', last_name: 'deux' }, - { id: 2, first_name: 'zeit', last_name: 'geist' }, - ], + contextAware: true, + users: expectedUsers, }) }) @@ -50,10 +52,8 @@ describe('prerender native module', () => { expect(JSON.parse(await browser.elementByCss('#props').text())).toEqual({ params: { slug: 'second' }, blog: true, - users: [ - { id: 1, first_name: 'john', last_name: 'deux' }, - { id: 2, first_name: 'zeit', last_name: 'geist' }, - ], + contextAware: true, + users: expectedUsers, }) }) @@ -70,7 +70,6 @@ describe('prerender native module', () => { ? /node_modules\/react\/cjs\/react\.production\.min\.js/ : /node_modules\/react\/cjs\/react\.production\.js/, ], - notTests: [], }, { page: '/blog/[slug]', @@ -81,13 +80,16 @@ describe('prerender native module', () => { isReact18 ? /node_modules\/react\/cjs\/react\.production\.min\.js/ : /node_modules\/react\/cjs\/react\.production\.js/, - /node_modules\/sqlite3\/.*?\.js/, - /node_modules\/sqlite3\/.*?\.node/, - /node_modules\/sqlite\/.*?\.js/, + // The addon's JS entry, and the binary it locates through + // `require('bindings')(...)`. + /node_modules\/native-addon\/.*?\.js/, + /node_modules\/native-addon\/.*?\.node/, + // The pure-JS wrapper layered on top of it. + /node_modules\/native-addon-wrapper\/.*?\.js/, /node_modules\/next/, - /\/data\.sqlite/, + // Reached only by statically evaluating `path.join(process.cwd(), ...)`. + /\/users\.json/, ], - notTests: [], }, ] @@ -97,14 +99,11 @@ describe('prerender native module', () => { ) const { version, files } = JSON.parse(contents) expect(version).toBe(1) - expect( - check.tests.every((item) => files.some((file) => item.test(file))) - ).toBe(true) - if (path.sep === '/') { - expect( - check.notTests.some((item) => files.some((file) => item.test(file))) - ).toBe(false) + for (const test of check.tests) { + expect(files).toEqual( + expect.arrayContaining([expect.stringMatching(test)]) + ) } } }) diff --git a/test/e2e/prerender-native-module/.npmrc b/test/e2e/prerender-native-module/.npmrc new file mode 100644 index 000000000000..951a2b6e795f --- /dev/null +++ b/test/e2e/prerender-native-module/.npmrc @@ -0,0 +1,13 @@ +# Deploy builds install with npm, which symlinks a `file:` directory dependency by +# default. A symlink resolves back into the app, so the server build stops treating +# the package as external and bundles it, and `bindings` cannot survive that: it +# locates its binary from the calling file's path, which becomes a path inside +# `.next` once bundled. +# +# `install-links` makes npm pack the dependency into `node_modules` instead. It only +# takes effect when the dependency is specified without a leading `./`, which is why +# `package.json` says `file:native-addon` rather than `file:./native-addon`. +# +# pnpm ignores this setting and copies the directory into its virtual store either +# way, so the local install is unaffected. +install-links=true diff --git a/test/e2e/prerender-native-module/bindings/index.js b/test/e2e/prerender-native-module/bindings/index.js new file mode 100644 index 000000000000..fd4a2e414af0 --- /dev/null +++ b/test/e2e/prerender-native-module/bindings/index.js @@ -0,0 +1,62 @@ +// A minimal stand-in for the `bindings` npm package, reproducing only the part +// both bundlers model: locate the calling package's root by walking up for a +// `package.json`, then load the compiled binary from `build/Release`. +// +// Neither bundler evaluates this file. `@vercel/nft` maps the `bindings` +// specifier to its own bundled copy, and Turbopack maps it to +// `WellKnownFunctionKind::NodeBindings`; both then resolve the binary's path +// themselves. `build/Release/` is node-gyp's default output location and +// is on both of their candidate lists, so this agrees with what they trace. + +const fs = require('fs') +const path = require('path') + +function getCallerFile() { + const { prepareStackTrace } = Error + try { + Error.prepareStackTrace = (_error, stack) => stack + for (const frame of new Error().stack) { + const fileName = frame.getFileName() + if (typeof fileName === 'string' && fileName !== __filename) { + return fileName + } + } + } finally { + Error.prepareStackTrace = prepareStackTrace + } + throw new Error('bindings: could not determine the calling file') +} + +function getPackageRoot(file) { + let dir = path.dirname(file) + while (!fs.existsSync(path.join(dir, 'package.json'))) { + const parent = path.dirname(dir) + if (parent === dir) { + throw new Error(`bindings: found no package.json above ${file}`) + } + dir = parent + } + return dir +} + +module.exports = function bindings(name) { + const binary = path.join( + getPackageRoot(getCallerFile()), + 'build', + 'Release', + name + ) + + if (!fs.existsSync(binary)) { + throw new Error( + `bindings: ${binary} does not exist. The fixture addon is compiled by ` + + `node-gyp during install, which requires the package to be listed in ` + + `the test's pnpm.onlyBuiltDependencies.` + ) + } + + // Deliberately not wrapped. A non-context-aware addon fails on this line with + // Node's own `ERR_DLOPEN_FAILED` / "Module did not self-register", and the + // worker-thread tests assert on that message. + return require(binary) +} diff --git a/test/e2e/prerender-native-module/bindings/package.json b/test/e2e/prerender-native-module/bindings/package.json new file mode 100644 index 000000000000..a942bce5029c --- /dev/null +++ b/test/e2e/prerender-native-module/bindings/package.json @@ -0,0 +1,6 @@ +{ + "name": "bindings", + "version": "1.0.0", + "description": "Minimal stand-in for the bindings npm package, for tests", + "main": "index.js" +} diff --git a/test/e2e/prerender-native-module/data.sqlite b/test/e2e/prerender-native-module/data.sqlite deleted file mode 100644 index 658fe04154f7..000000000000 Binary files a/test/e2e/prerender-native-module/data.sqlite and /dev/null differ diff --git a/test/e2e/prerender-native-module/native-addon-wrapper/index.js b/test/e2e/prerender-native-module/native-addon-wrapper/index.js new file mode 100644 index 000000000000..3c4a13563407 --- /dev/null +++ b/test/e2e/prerender-native-module/native-addon-wrapper/index.js @@ -0,0 +1,19 @@ +// Stands in for a pure-JS convenience package layered over a native one, the +// role `sqlite` played over `sqlite3`. It has no dependencies of its own and +// resolves `native-addon` positionally, so it does not need a dependency edge. +// +// The caller passes an already-resolved `filename`. Keep it that way: output +// file tracing only follows a `process.cwd()`-derived path when the expression +// appears in the app's own code, since Turbopack gates that on the source not +// being inside `node_modules`. + +const fs = require('fs/promises') + +exports.open = async function open({ filename, driver }) { + const rows = JSON.parse(await fs.readFile(filename, 'utf8')) + + return { + contextAware: driver.CONTEXT_AWARE, + all: async () => rows, + } +} diff --git a/test/e2e/prerender-native-module/native-addon-wrapper/package.json b/test/e2e/prerender-native-module/native-addon-wrapper/package.json new file mode 100644 index 000000000000..be8bcc6a8708 --- /dev/null +++ b/test/e2e/prerender-native-module/native-addon-wrapper/package.json @@ -0,0 +1,6 @@ +{ + "name": "native-addon-wrapper", + "version": "1.0.0", + "description": "Dependency-free JS wrapper over native-addon, for tests", + "main": "index.js" +} diff --git a/test/e2e/prerender-native-module/native-addon/addon.cc b/test/e2e/prerender-native-module/native-addon/addon.cc new file mode 100644 index 000000000000..2436d072feb1 --- /dev/null +++ b/test/e2e/prerender-native-module/native-addon/addon.cc @@ -0,0 +1,14 @@ +// A context-aware addon. `NODE_MODULE_INIT` registers per-context, so this can +// be loaded from the main thread and from a worker thread in the same process. +// See https://nodejs.org/api/addons.html#context-aware-addons + +#include + +NODE_MODULE_INIT(/* exports, module, context */) { + v8::Isolate* isolate = context->GetIsolate(); + exports + ->Set(context, + v8::String::NewFromUtf8(isolate, "CONTEXT_AWARE").ToLocalChecked(), + v8::Boolean::New(isolate, true)) + .Check(); +} diff --git a/test/e2e/prerender-native-module/native-addon/binding.gyp b/test/e2e/prerender-native-module/native-addon/binding.gyp new file mode 100644 index 000000000000..9c1110672259 --- /dev/null +++ b/test/e2e/prerender-native-module/native-addon/binding.gyp @@ -0,0 +1,8 @@ +{ + "targets": [ + { + "target_name": "native_addon", + "sources": ["addon.cc"] + } + ] +} diff --git a/test/e2e/prerender-native-module/native-addon/index.js b/test/e2e/prerender-native-module/native-addon/index.js new file mode 100644 index 000000000000..52a1e8bf48a9 --- /dev/null +++ b/test/e2e/prerender-native-module/native-addon/index.js @@ -0,0 +1,3 @@ +// Mirrors how `sqlite3` loads its binary, which is the shape both bundlers +// special-case: a single `require('bindings')('.node')` call. +module.exports = require('bindings')('native_addon.node') diff --git a/test/e2e/prerender-native-module/native-addon/package.json b/test/e2e/prerender-native-module/native-addon/package.json new file mode 100644 index 000000000000..37c6888cf9a9 --- /dev/null +++ b/test/e2e/prerender-native-module/native-addon/package.json @@ -0,0 +1,6 @@ +{ + "name": "native-addon", + "version": "1.0.0", + "description": "Context-aware native addon fixture, compiled by node-gyp", + "main": "index.js" +} diff --git a/test/e2e/prerender-native-module/package.json b/test/e2e/prerender-native-module/package.json new file mode 100644 index 000000000000..22e1cf3cb203 --- /dev/null +++ b/test/e2e/prerender-native-module/package.json @@ -0,0 +1,7 @@ +{ + "dependencies": { + "native-addon": "file:native-addon", + "native-addon-wrapper": "file:native-addon-wrapper", + "bindings": "file:bindings" + } +} diff --git a/test/e2e/prerender-native-module/pages/blog/[slug].js b/test/e2e/prerender-native-module/pages/blog/[slug].js index f181d158ad37..fc0337f79e7b 100644 --- a/test/e2e/prerender-native-module/pages/blog/[slug].js +++ b/test/e2e/prerender-native-module/pages/blog/[slug].js @@ -1,22 +1,28 @@ import path from 'path' -import { open } from 'sqlite' -import sqlite3 from 'sqlite3' +import { open } from 'native-addon-wrapper' +import nativeAddon from 'native-addon' import { useRouter } from 'next/router' export const getStaticProps = async ({ params }) => { - const dbPath = path.join(process.cwd(), 'data.sqlite') - console.log('using db', dbPath) + // The `process.cwd()` join stays in the page on purpose. Output file tracing + // only follows it into the trace from the app's own code, so moving it into + // `native-addon-wrapper` would stop `users.json` being traced. + const dataPath = path.join(process.cwd(), 'users.json') + console.log('using data', dataPath) const db = await open({ - filename: dbPath, - driver: sqlite3.Database, + filename: dataPath, + driver: nativeAddon, }) - const users = await db.all(`SELECT * FROM users`) + const users = await db.all() return { props: { users, + // Read off the compiled binary, so a native module that failed to load + // shows up here rather than passing quietly. + contextAware: db.contextAware, blog: true, params: params || null, }, diff --git a/test/e2e/prerender-native-module/pnpm-workspace.yaml b/test/e2e/prerender-native-module/pnpm-workspace.yaml new file mode 100644 index 000000000000..68e8462f6590 --- /dev/null +++ b/test/e2e/prerender-native-module/pnpm-workspace.yaml @@ -0,0 +1,5 @@ +# `native-addon` is compiled by node-gyp during install, and pnpm does +# not run a dependency's build scripts unless they are approved here. The repo +# root uses the same key for the one package it allows to build. +allowBuilds: + native-addon: true diff --git a/test/e2e/prerender-native-module/users.json b/test/e2e/prerender-native-module/users.json new file mode 100644 index 000000000000..f683b73c8321 --- /dev/null +++ b/test/e2e/prerender-native-module/users.json @@ -0,0 +1,4 @@ +[ + { "id": 1, "first_name": "john", "last_name": "deux" }, + { "id": 2, "first_name": "zeit", "last_name": "geist" } +] diff --git a/test/lib/next-modes/next-deploy.ts b/test/lib/next-modes/next-deploy.ts index 50a5bf74ee7f..0471076d9a22 100644 --- a/test/lib/next-modes/next-deploy.ts +++ b/test/lib/next-modes/next-deploy.ts @@ -563,9 +563,17 @@ export class NextDeployInstance extends NextInstance { require('console').log( `Writing .npmrc for preview-builds mirror: ${registryKey}` ) + // Appended rather than written, because a fixture may ship its own `.npmrc` + // (several do) and overwriting it silently drops that configuration. + const npmrcPath = path.join(this.testDir, '.npmrc') + const existing = (await fs.pathExists(npmrcPath)) + ? await fs.readFile(npmrcPath, 'utf8') + : '' + const separator = existing === '' || existing.endsWith('\n') ? '' : '\n' + await fs.writeFile( - path.join(this.testDir, '.npmrc'), - `${registryKey}:_authToken=\${VERCEL_OIDC_TOKEN}\n` + npmrcPath, + `${existing}${separator}${registryKey}:_authToken=\${VERCEL_OIDC_TOKEN}\n` ) } diff --git a/test/production/firebase-grpc/firebase-grpc.test.ts b/test/production/firebase-grpc/firebase-grpc.test.ts deleted file mode 100644 index 1e995c40c4fa..000000000000 --- a/test/production/firebase-grpc/firebase-grpc.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { nextTestSetup } from 'e2e-utils' - -describe('Building Firebase', () => { - const { next } = nextTestSetup({ - files: __dirname, - skipStart: true, - dependencies: { - firebase: 'latest', - }, - }) - - // TODO: investigate re-enabling this test in node 12 environment - it.skip('Throws an error when building with firebase dependency with worker_threads', async () => { - await next.patchFile( - 'next.config.js', - `module.exports = { experimental: { workerThreads: true } }` - ) - await next.build() - expect(next.cliOutput).toMatch(/Build error occurred/) - expect(next.cliOutput).toMatch( - /grpc_node\.node\. Module did not self-register\./ - ) - }) - - it('Throws no error when building with firebase dependency without worker_threads', async () => { - await next.build() - expect(next.cliOutput).not.toMatch(/Build error occurred/) - expect(next.cliOutput).not.toMatch( - /grpc_node\.node\. Module did not self-register\./ - ) - }) -}) diff --git a/test/production/firebase-grpc/pages/page-1.js b/test/production/firebase-grpc/pages/page-1.js deleted file mode 100644 index dd9c28611eea..000000000000 --- a/test/production/firebase-grpc/pages/page-1.js +++ /dev/null @@ -1,2 +0,0 @@ -import 'firebase/firestore' -export default () =>
Firebase
diff --git a/test/production/firebase-grpc/pages/page-2.js b/test/production/firebase-grpc/pages/page-2.js deleted file mode 100644 index dd9c28611eea..000000000000 --- a/test/production/firebase-grpc/pages/page-2.js +++ /dev/null @@ -1,2 +0,0 @@ -import 'firebase/firestore' -export default () =>
Firebase
diff --git a/test/production/prerender-worker-threads/bindings/index.js b/test/production/prerender-worker-threads/bindings/index.js new file mode 100644 index 000000000000..fd4a2e414af0 --- /dev/null +++ b/test/production/prerender-worker-threads/bindings/index.js @@ -0,0 +1,62 @@ +// A minimal stand-in for the `bindings` npm package, reproducing only the part +// both bundlers model: locate the calling package's root by walking up for a +// `package.json`, then load the compiled binary from `build/Release`. +// +// Neither bundler evaluates this file. `@vercel/nft` maps the `bindings` +// specifier to its own bundled copy, and Turbopack maps it to +// `WellKnownFunctionKind::NodeBindings`; both then resolve the binary's path +// themselves. `build/Release/` is node-gyp's default output location and +// is on both of their candidate lists, so this agrees with what they trace. + +const fs = require('fs') +const path = require('path') + +function getCallerFile() { + const { prepareStackTrace } = Error + try { + Error.prepareStackTrace = (_error, stack) => stack + for (const frame of new Error().stack) { + const fileName = frame.getFileName() + if (typeof fileName === 'string' && fileName !== __filename) { + return fileName + } + } + } finally { + Error.prepareStackTrace = prepareStackTrace + } + throw new Error('bindings: could not determine the calling file') +} + +function getPackageRoot(file) { + let dir = path.dirname(file) + while (!fs.existsSync(path.join(dir, 'package.json'))) { + const parent = path.dirname(dir) + if (parent === dir) { + throw new Error(`bindings: found no package.json above ${file}`) + } + dir = parent + } + return dir +} + +module.exports = function bindings(name) { + const binary = path.join( + getPackageRoot(getCallerFile()), + 'build', + 'Release', + name + ) + + if (!fs.existsSync(binary)) { + throw new Error( + `bindings: ${binary} does not exist. The fixture addon is compiled by ` + + `node-gyp during install, which requires the package to be listed in ` + + `the test's pnpm.onlyBuiltDependencies.` + ) + } + + // Deliberately not wrapped. A non-context-aware addon fails on this line with + // Node's own `ERR_DLOPEN_FAILED` / "Module did not self-register", and the + // worker-thread tests assert on that message. + return require(binary) +} diff --git a/test/production/prerender-worker-threads/bindings/package.json b/test/production/prerender-worker-threads/bindings/package.json new file mode 100644 index 000000000000..a942bce5029c --- /dev/null +++ b/test/production/prerender-worker-threads/bindings/package.json @@ -0,0 +1,6 @@ +{ + "name": "bindings", + "version": "1.0.0", + "description": "Minimal stand-in for the bindings npm package, for tests", + "main": "index.js" +} diff --git a/test/production/prerender-worker-threads/next.config.js b/test/production/prerender-worker-threads/next.config.js new file mode 100644 index 000000000000..259857180a32 --- /dev/null +++ b/test/production/prerender-worker-threads/next.config.js @@ -0,0 +1,17 @@ +const { isMainThread } = require('node:worker_threads') + +// Loading the addon here puts it in the main build process, so the load the page +// performs inside the static generation worker is the *second* one. Both matter: +// "Module did not self-register" only happens on a second `dlopen` of the same +// file in one process, so an addon only ever loaded inside the worker would +// register there and the build would succeed. +// +// The `isMainThread` guard keeps the first two cases pointed at the static +// generation worker. `next build` also runs Turbopack in a worker thread that +// re-evaluates this file, and loading the addon there too fails the build before +// static generation is reached. The last case in the suite covers that separately. +if (isMainThread) { + require('single-context-addon') +} + +module.exports = {} diff --git a/test/production/prerender-worker-threads/package.json b/test/production/prerender-worker-threads/package.json new file mode 100644 index 000000000000..e434d0f3de1e --- /dev/null +++ b/test/production/prerender-worker-threads/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "single-context-addon": "file:single-context-addon", + "bindings": "file:bindings" + } +} diff --git a/test/production/prerender-worker-threads/pages/index.js b/test/production/prerender-worker-threads/pages/index.js new file mode 100644 index 000000000000..24e1d651477a --- /dev/null +++ b/test/production/prerender-worker-threads/pages/index.js @@ -0,0 +1,15 @@ +import addon from 'single-context-addon' + +export const getStaticProps = () => { + // Runs inside the static generation worker, which is the second place the addon + // gets loaded. `next.config.js` already loaded it in the main process. + return { + props: { + contextAware: addon.CONTEXT_AWARE, + }, + } +} + +export default function Page(props) { + return

{JSON.stringify(props)}

+} diff --git a/test/production/prerender-worker-threads/pnpm-workspace.yaml b/test/production/prerender-worker-threads/pnpm-workspace.yaml new file mode 100644 index 000000000000..4f9ffe8d637b --- /dev/null +++ b/test/production/prerender-worker-threads/pnpm-workspace.yaml @@ -0,0 +1,5 @@ +# `single-context-addon` is compiled by node-gyp during install, and pnpm does +# not run a dependency's build scripts unless they are approved here. The repo +# root uses the same key for the one package it allows to build. +allowBuilds: + single-context-addon: true diff --git a/test/production/prerender-worker-threads/prerender-worker-threads.test.ts b/test/production/prerender-worker-threads/prerender-worker-threads.test.ts new file mode 100644 index 000000000000..945b9598a319 --- /dev/null +++ b/test/production/prerender-worker-threads/prerender-worker-threads.test.ts @@ -0,0 +1,79 @@ +import { nextTestSetup } from 'e2e-utils' + +const UNGUARDED_CONFIG = `require('single-context-addon') +module.exports = {} +` + +// A native addon declared with `NODE_MODULE` rather than `NODE_MODULE_INIT` can +// only be loaded once per process, so a second `dlopen` on another thread of the +// same process fails with `ERR_DLOPEN_FAILED` / "Module did not self-register". +// +// `experimental.workerThreads` decides whether static generation runs in real +// worker threads or in forked child processes. Forks get a fresh process and load +// the addon cleanly; threads share the build process and do not. That is why the +// flag defaults to false (PR #9199) and why the static export worker was fixed to +// respect it instead of hardcoding threads on (PR #25063). +describe('prerender worker threads', () => { + const { next, isTurbopack } = nextTestSetup({ + files: __dirname, + skipStart: true, + // `bindings` is a direct dependency rather than one of the addon. The addon + // resolves it by walking up from the virtual store, and declaring it on the + // addon would make it a non-registry transitive dependency, which + // `blockExoticSubdeps` rejects. + // + // These use `file:` rather than `link:` so pnpm copies them into the virtual + // store. A linked package resolves to a path inside the app, which the bundler + // treats as app code and tries to bundle, and `bindings` contains a `require` + // it cannot resolve statically. + dependencies: require('./package.json').dependencies, + }) + + it('should prerender a page using a non-context-aware addon by default', async () => { + const { exitCode, cliOutput } = await next.build() + + expect(cliOutput).not.toContain('Module did not self-register') + expect(exitCode).toBe(0) + }) + + it('should fail to prerender that page when worker threads are enabled', async () => { + await next.patchFile( + 'next.config.js', + `const { isMainThread } = require('node:worker_threads') +if (isMainThread) { require('single-context-addon') } +module.exports = { experimental: { workerThreads: true } } +` + ) + + const { exitCode, cliOutput } = await next.build() + + expect(cliOutput).toContain('Module did not self-register') + expect(cliOutput).toContain('single_context_addon.node') + expect(exitCode).not.toBe(0) + }) + + // This documents a bug rather than intended behavior. `next build` runs Turbopack + // in a worker thread (`enableWorkerThreads: true` is hardcoded in + // packages/next/src/build/turbopack-build/index.ts) and that worker re-evaluates + // `next.config.js`, so requiring a non-context-aware addon from the config breaks + // the build even though `experimental.workerThreads` is off. It is the same class + // of failure PR #9199 and PR #25063 fixed, in a worker those PRs did not touch. + // + // Webpack is unaffected, because its build worker is a forked child process. + // + // When Turbopack stops evaluating the config on a worker thread, this test will + // fail: drop the branch and assert the webpack outcome for both bundlers. + it('should currently fail under Turbopack when the config loads the addon', async () => { + await next.patchFile('next.config.js', UNGUARDED_CONFIG) + + const { exitCode, cliOutput } = await next.build() + + if (isTurbopack) { + expect(cliOutput).toContain('Module did not self-register') + expect(exitCode).not.toBe(0) + } else { + expect(cliOutput).not.toContain('Module did not self-register') + expect(exitCode).toBe(0) + } + }) +}) diff --git a/test/production/prerender-worker-threads/single-context-addon/addon.cc b/test/production/prerender-worker-threads/single-context-addon/addon.cc new file mode 100644 index 000000000000..8ca5eb993a36 --- /dev/null +++ b/test/production/prerender-worker-threads/single-context-addon/addon.cc @@ -0,0 +1,29 @@ +// A deliberately non-context-aware addon. `NODE_MODULE` registers once per +// process, so loading it on the main thread and then again on a worker thread +// fails: the second `dlopen` returns the cached handle, the module constructor +// does not run again, and Node reports +// +// Error [ERR_DLOPEN_FAILED]: Module did not self-register: ''. +// +// which is the failure PR #9199 and PR #25063 both worked around by keeping +// worker threads off by default. Do not port this to Node-API or +// `NODE_MODULE_INIT`: either would make it context-aware and the tests that +// depend on it would silently start passing for the wrong reason. + +#include + +namespace { + +void Init(v8::Local exports) { + v8::Isolate* isolate = v8::Isolate::GetCurrent(); + v8::Local context = isolate->GetCurrentContext(); + exports + ->Set(context, + v8::String::NewFromUtf8(isolate, "CONTEXT_AWARE").ToLocalChecked(), + v8::Boolean::New(isolate, false)) + .Check(); +} + +} // namespace + +NODE_MODULE(NODE_GYP_MODULE_NAME, Init) diff --git a/test/production/prerender-worker-threads/single-context-addon/binding.gyp b/test/production/prerender-worker-threads/single-context-addon/binding.gyp new file mode 100644 index 000000000000..94ec15c12cef --- /dev/null +++ b/test/production/prerender-worker-threads/single-context-addon/binding.gyp @@ -0,0 +1,8 @@ +{ + "targets": [ + { + "target_name": "single_context_addon", + "sources": ["addon.cc"] + } + ] +} diff --git a/test/production/prerender-worker-threads/single-context-addon/index.js b/test/production/prerender-worker-threads/single-context-addon/index.js new file mode 100644 index 000000000000..69da152bd5ef --- /dev/null +++ b/test/production/prerender-worker-threads/single-context-addon/index.js @@ -0,0 +1 @@ +module.exports = require('bindings')('single_context_addon.node') diff --git a/test/production/prerender-worker-threads/single-context-addon/package.json b/test/production/prerender-worker-threads/single-context-addon/package.json new file mode 100644 index 000000000000..6a4de0e146a9 --- /dev/null +++ b/test/production/prerender-worker-threads/single-context-addon/package.json @@ -0,0 +1,6 @@ +{ + "name": "single-context-addon", + "version": "1.0.0", + "description": "Non-context-aware native addon fixture, compiled by node-gyp", + "main": "index.js" +} diff --git a/test/production/sharp-basic/sharp-basic.test.ts b/test/production/sharp-basic/sharp-basic.test.ts index 1ec52aca77ed..beac13b0a425 100644 --- a/test/production/sharp-basic/sharp-basic.test.ts +++ b/test/production/sharp-basic/sharp-basic.test.ts @@ -6,11 +6,6 @@ describe('sharp support with hasNextSupport', () => { dependencies: { sharp: 'latest', }, - packageJson: { - pnpm: { - onlyBuiltDependencies: ['sqlite3'], - }, - }, env: { NOW_BUILDER: '1', }, diff --git a/test/rspack-build-tests-manifest.json b/test/rspack-build-tests-manifest.json index a61b3209669f..6aa085877570 100644 --- a/test/rspack-build-tests-manifest.json +++ b/test/rspack-build-tests-manifest.json @@ -8031,7 +8031,9 @@ "runtimeError": false }, "test/e2e/app-dir/turbopack-reports/turbopack-reports.test.ts": { - "passed": ["turbopack-reports should render page importing sqlite3"], + "passed": [ + "turbopack-reports should render page importing a package that loads a native binding" + ], "failed": [], "pending": [], "flakey": [], @@ -16101,17 +16103,6 @@ "flakey": [], "runtimeError": false }, - "test/integration/firebase-grpc/test/index.test.ts": { - "passed": [ - "Building Firebase production mode Throws no error when building with firebase dependency without worker_threads" - ], - "failed": [], - "pending": [ - "Building Firebase production mode Throws an error when building with firebase dependency with worker_threads" - ], - "flakey": [], - "runtimeError": false - }, "test/integration/future/test/index.test.ts": { "passed": [ "excludeDefaultMomentLocales production mode should load momentjs" diff --git a/test/rspack-dev-tests-manifest.json b/test/rspack-dev-tests-manifest.json index 30eeb68c962a..084ffe76c99a 100644 --- a/test/rspack-dev-tests-manifest.json +++ b/test/rspack-dev-tests-manifest.json @@ -10484,7 +10484,9 @@ "runtimeError": false }, "test/e2e/app-dir/turbopack-reports/turbopack-reports.test.ts": { - "passed": ["turbopack-reports should render page importing sqlite3"], + "passed": [ + "turbopack-reports should render page importing a package that loads a native binding" + ], "failed": [], "pending": [], "flakey": [], @@ -18555,16 +18557,6 @@ "flakey": [], "runtimeError": false }, - "test/integration/firebase-grpc/test/index.test.ts": { - "passed": [], - "failed": [], - "pending": [ - "Building Firebase production mode Throws an error when building with firebase dependency with worker_threads", - "Building Firebase production mode Throws no error when building with firebase dependency without worker_threads" - ], - "flakey": [], - "runtimeError": false - }, "test/integration/future/test/index.test.ts": { "passed": [], "failed": [],