diff --git a/crates/next-api/src/project.rs b/crates/next-api/src/project.rs index ce5db197ca4f..5cfc1be77210 100644 --- a/crates/next-api/src/project.rs +++ b/crates/next-api/src/project.rs @@ -93,7 +93,7 @@ use turbopack_node::child_process_backend; use turbopack_node::execution_context::ExecutionContext; #[cfg(feature = "worker_pool")] use turbopack_node::worker_threads_backend; -use turbopack_nodejs::NodeJsChunkingContext; +use turbopack_nodejs::{NodeJsChunkingContext, fs::NodeModulesPathMatcher}; use crate::{ aggregate_hmr::{AggregateHmrVersion, ChunkListUpdateBuilder, DiffResult, diff_chunks_against}, @@ -1094,10 +1094,13 @@ impl Project { *self.root_path, vec![denied_path, denied_profiles_path], DiskWatcherConfig { - recursive_mode: None, poll_interval: self.watch.poll_interval, // the dev server reports these to the user report_invalidation_reason: true, + extended_batch_delay_matcher: Some(ResolvedVc::upcast( + NodeModulesPathMatcher.resolved_cell(), + )), + ..Default::default() }, )) } diff --git a/docs/01-app/01-getting-started/04-linking-and-navigating.mdx b/docs/01-app/01-getting-started/04-linking-and-navigating.mdx index de0e4c1b73bb..08a04ec62454 100644 --- a/docs/01-app/01-getting-started/04-linking-and-navigating.mdx +++ b/docs/01-app/01-getting-started/04-linking-and-navigating.mdx @@ -158,7 +158,7 @@ Next.js avoids this with client-side transitions using the `` component. I - Keeping any shared layouts and UI. - Replacing the current page with the prefetched loading state or a new page if available. -Client-side transitions are what makes a server-rendered apps _feel_ like client-rendered apps. And when paired with [prefetching](#prefetching) and [streaming](#streaming), it enables fast transitions, even for dynamic routes. +Client-side transitions make server-rendered apps _feel_ like client-rendered apps. And when paired with [prefetching](#prefetching) and [streaming](#streaming), they enable fast transitions, even for dynamic routes. Next.js also handles [scrolling to the top of the page](/docs/app/api-reference/components/link#scroll) during client-side transitions. If content scrolls behind a sticky or fixed header after navigation, you can fix this with CSS [`scroll-padding-top`](/docs/app/api-reference/components/link#scroll-offset-with-sticky-headers). diff --git a/docs/01-app/02-guides/redirecting.mdx b/docs/01-app/02-guides/redirecting.mdx index 46ab94d0ca4e..cf7bda8b9e13 100644 --- a/docs/01-app/02-guides/redirecting.mdx +++ b/docs/01-app/02-guides/redirecting.mdx @@ -376,11 +376,11 @@ Consider the following data structure: } ``` -In [Proxy](/docs/app/api-reference/file-conventions/proxy), you can read from a database such as Vercel's [Edge Config](https://vercel.com/docs/global-config/get-started) or [Redis](https://vercel.com/docs/redis), and redirect the user based on the incoming request: +In [Proxy](/docs/app/api-reference/file-conventions/proxy), you can read from a database such as Vercel's [Global Config](https://vercel.com/docs/global-config/get-started) or [Redis](https://vercel.com/docs/redis), and redirect the user based on the incoming request: ```ts filename="proxy.ts" switcher import { NextResponse, NextRequest } from 'next/server' -import { get } from '@vercel/edge-config' +import { get } from '@vercel/global-config' type RedirectEntry = { destination: string @@ -404,7 +404,7 @@ export async function proxy(request: NextRequest) { ```js filename="proxy.js" switcher import { NextResponse } from 'next/server' -import { get } from '@vercel/edge-config' +import { get } from '@vercel/global-config' export async function proxy(request) { const pathname = request.nextUrl.pathname diff --git a/docs/01-app/03-api-reference/04-functions/generate-metadata.mdx b/docs/01-app/03-api-reference/04-functions/generate-metadata.mdx index 6eeebd8151a1..064e5dd77c7a 100644 --- a/docs/01-app/03-api-reference/04-functions/generate-metadata.mdx +++ b/docs/01-app/03-api-reference/04-functions/generate-metadata.mdx @@ -923,6 +923,24 @@ export const metadata = { ``` +### `pagination` + +Describes the previous and next pages in a paginated sequence. + +```jsx filename="layout.js | page.js" +export const metadata = { + pagination: { + previous: 'https://nextjs.org/blog?page=1', + next: 'https://nextjs.org/blog?page=3', + }, +} +``` + +```html filename=" output" hideLineNumbers + + +``` + ### `category` ```jsx filename="layout.js | page.js" diff --git a/examples/with-apivideo/pages/videos/[videoId].tsx b/examples/with-apivideo/pages/videos/[videoId].tsx index 43d35ec6bdee..b2d81ccc73e3 100644 --- a/examples/with-apivideo/pages/videos/[videoId].tsx +++ b/examples/with-apivideo/pages/videos/[videoId].tsx @@ -68,7 +68,7 @@ const VideoView: NextPage = ({
- + = ({ />
- + = ({
{ setHideControls(e.currentTarget.checked); }} /> - +
( } .submit { display: flex; - justify-content: flex-end; align-items: center; justify-content: space-between; } diff --git a/examples/with-magic/pages/login.js b/examples/with-magic/pages/login.js index 8b93533b4356..69fcca222528 100644 --- a/examples/with-magic/pages/login.js +++ b/examples/with-magic/pages/login.js @@ -39,7 +39,7 @@ const Login = () => { throw new Error(await res.text()); } } catch (error) { - console.error("An unexpected error happened occurred:", error); + console.error("An unexpected error occurred:", error); setErrorMsg(error.message); } } diff --git a/examples/with-segment-analytics-pages-router/README.md b/examples/with-segment-analytics-pages-router/README.md index 65cc4b3b7d56..c51a3993a8b4 100644 --- a/examples/with-segment-analytics-pages-router/README.md +++ b/examples/with-segment-analytics-pages-router/README.md @@ -1,6 +1,6 @@ # With Segment Analytics (Pages Router) -This example shows how to use Next.js along with [Segment Analytics](https://segment.com) using [segmentio/analytics-next](https://github.com/segmentio/analytics-next). The custom app [component](https://github.com/vercel/next.js/blob/canary/examples/with-segment-analytics-pages-router/pages/_app.tsx) includes a component (analytics.tsx)[(https://github.com/vercel/next.js/blob/canary/examples/with-segment-analytics-pages-router/components/analytics.tsx)] which loads Segment and also exports the `analytics` object which can be imported and used to call the [Track API](https://segment.com/docs/connections/spec/track/) on user actions (Refer to [`contact.tsx`](https://github.com/vercel/next.js/blob/canary/examples/with-segment-analytics-pages-router/pages/contact.tsx)). +This example shows how to use Next.js along with [Segment Analytics](https://segment.com) using [segmentio/analytics-next](https://github.com/segmentio/analytics-next). The custom app [component](https://github.com/vercel/next.js/blob/canary/examples/with-segment-analytics-pages-router/pages/_app.tsx) includes a component [`analytics.tsx`](https://github.com/vercel/next.js/blob/canary/examples/with-segment-analytics-pages-router/components/analytics.tsx) which loads Segment and also exports the `analytics` object which can be imported and used to call the [Track API](https://segment.com/docs/connections/spec/track/) on user actions (Refer to [`contact.tsx`](https://github.com/vercel/next.js/blob/canary/examples/with-segment-analytics-pages-router/pages/contact.tsx)). ## Deploy your own diff --git a/examples/with-segment-analytics/README.md b/examples/with-segment-analytics/README.md index 3a12203fedbf..db782187cb64 100644 --- a/examples/with-segment-analytics/README.md +++ b/examples/with-segment-analytics/README.md @@ -1,6 +1,6 @@ # With Segment Analytics -This example shows how to use Next.js along with [Segment Analytics](https://segment.com) using [segmentio/analytics-next](https://github.com/segmentio/analytics-next). The main app [layout](https://github.com/vercel/next.js/blob/canary/examples/with-segment-analytics/app/layout.tsx) includes a Client Component (analytics.tsx)[(https://github.com/vercel/next.js/blob/canary/examples/with-segment-analytics/components/analytics.tsx)] which loads Segment and also exports the `analytics` object which can be imported and used to call the [Track API](https://segment.com/docs/connections/spec/track/) on user actions (Refer to [`contact.tsx`](https://github.com/vercel/next.js/blob/canary/examples/with-segment-analytics/app/contact/page.tsx)). +This example shows how to use Next.js along with [Segment Analytics](https://segment.com) using [segmentio/analytics-next](https://github.com/segmentio/analytics-next). The main app [layout](https://github.com/vercel/next.js/blob/canary/examples/with-segment-analytics/app/layout.tsx) includes a Client Component [`analytics.tsx`](https://github.com/vercel/next.js/blob/canary/examples/with-segment-analytics/components/analytics.tsx) which loads Segment and also exports the `analytics` object which can be imported and used to call the [Track API](https://segment.com/docs/connections/spec/track/) on user actions (Refer to [`contact.tsx`](https://github.com/vercel/next.js/blob/canary/examples/with-segment-analytics/app/contact/page.tsx)). ## Deploy your own diff --git a/lerna.json b/lerna.json index e09d41f80fa6..ac12ebd57423 100644 --- a/lerna.json +++ b/lerna.json @@ -15,5 +15,5 @@ "registry": "https://registry.npmjs.org/" } }, - "version": "16.3.1-canary.23" + "version": "16.3.1-canary.24" } \ No newline at end of file diff --git a/packages/create-next-app/package.json b/packages/create-next-app/package.json index 327d14118d19..f8950dd7f932 100644 --- a/packages/create-next-app/package.json +++ b/packages/create-next-app/package.json @@ -1,6 +1,6 @@ { "name": "create-next-app", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "keywords": [ "react", "next", diff --git a/packages/devlow-bench/package.json b/packages/devlow-bench/package.json index fed4a057cc93..2f66ad728027 100644 --- a/packages/devlow-bench/package.json +++ b/packages/devlow-bench/package.json @@ -1,7 +1,7 @@ { "name": "@vercel/devlow-bench", "private": true, - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "description": "Benchmarking tool for the developer workflow", "repository": { "type": "git", diff --git a/packages/eslint-config-next/package.json b/packages/eslint-config-next/package.json index 24ef5919c477..c1a8f834aa70 100644 --- a/packages/eslint-config-next/package.json +++ b/packages/eslint-config-next/package.json @@ -1,6 +1,6 @@ { "name": "eslint-config-next", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "description": "ESLint configuration used by Next.js.", "license": "MIT", "repository": { @@ -12,7 +12,7 @@ "dist" ], "dependencies": { - "@next/eslint-plugin-next": "16.3.1-canary.23", + "@next/eslint-plugin-next": "16.3.1-canary.24", "eslint-import-resolver-node": "^0.3.6", "eslint-import-resolver-typescript": "^3.5.2", "eslint-plugin-import": "^2.32.0", diff --git a/packages/eslint-plugin-internal/package.json b/packages/eslint-plugin-internal/package.json index c7c67cb1fe3a..88d9d9c200e3 100644 --- a/packages/eslint-plugin-internal/package.json +++ b/packages/eslint-plugin-internal/package.json @@ -1,7 +1,7 @@ { "name": "@next/eslint-plugin-internal", "private": true, - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "description": "ESLint plugin for working on Next.js.", "exports": { ".": "./src/eslint-plugin-internal.js" diff --git a/packages/eslint-plugin-next/package.json b/packages/eslint-plugin-next/package.json index 5a6837ea0c94..fb2f835e9167 100644 --- a/packages/eslint-plugin-next/package.json +++ b/packages/eslint-plugin-next/package.json @@ -1,6 +1,6 @@ { "name": "@next/eslint-plugin-next", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "description": "ESLint plugin for Next.js.", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/font/package.json b/packages/font/package.json index cbb994b7d2d0..54d2387933f8 100644 --- a/packages/font/package.json +++ b/packages/font/package.json @@ -1,7 +1,7 @@ { "name": "@next/font", "private": true, - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "repository": { "url": "vercel/next.js", "directory": "packages/font" diff --git a/packages/next-bundle-analyzer/package.json b/packages/next-bundle-analyzer/package.json index 1dc42e416810..e5d460d26ee8 100644 --- a/packages/next-bundle-analyzer/package.json +++ b/packages/next-bundle-analyzer/package.json @@ -1,6 +1,6 @@ { "name": "@next/bundle-analyzer", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "main": "index.js", "types": "index.d.ts", "license": "MIT", diff --git a/packages/next-codemod/package.json b/packages/next-codemod/package.json index 3656ee61be7e..ab88dc94269f 100644 --- a/packages/next-codemod/package.json +++ b/packages/next-codemod/package.json @@ -1,6 +1,6 @@ { "name": "@next/codemod", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "license": "MIT", "repository": { "type": "git", diff --git a/packages/next-env/package.json b/packages/next-env/package.json index 955448078f9d..2d4f6977552d 100644 --- a/packages/next-env/package.json +++ b/packages/next-env/package.json @@ -1,6 +1,6 @@ { "name": "@next/env", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "keywords": [ "react", "next", diff --git a/packages/next-mdx/package.json b/packages/next-mdx/package.json index 97513eb23987..f7afa0de78e3 100644 --- a/packages/next-mdx/package.json +++ b/packages/next-mdx/package.json @@ -1,6 +1,6 @@ { "name": "@next/mdx", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "main": "index.js", "license": "MIT", "repository": { diff --git a/packages/next-playwright/package.json b/packages/next-playwright/package.json index 2d76a3617656..4e9fdb072a48 100644 --- a/packages/next-playwright/package.json +++ b/packages/next-playwright/package.json @@ -1,6 +1,6 @@ { "name": "@next/playwright", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "repository": { "url": "vercel/next.js", "directory": "packages/next-playwright" diff --git a/packages/next-plugin-storybook/package.json b/packages/next-plugin-storybook/package.json index 8db48e9bd0cf..496cb796844c 100644 --- a/packages/next-plugin-storybook/package.json +++ b/packages/next-plugin-storybook/package.json @@ -1,6 +1,6 @@ { "name": "@next/plugin-storybook", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "repository": { "url": "vercel/next.js", "directory": "packages/next-plugin-storybook" diff --git a/packages/next-polyfill-module/package.json b/packages/next-polyfill-module/package.json index f31f28bc1206..86aa388a3957 100644 --- a/packages/next-polyfill-module/package.json +++ b/packages/next-polyfill-module/package.json @@ -1,6 +1,6 @@ { "name": "@next/polyfill-module", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "description": "A standard library polyfill for ES Modules supporting browsers (Edge 16+, Firefox 60+, Chrome 61+, Safari 10.1+)", "main": "dist/polyfill-module.js", "license": "MIT", diff --git a/packages/next-polyfill-nomodule/package.json b/packages/next-polyfill-nomodule/package.json index 32031cc3a355..a96e118e0124 100644 --- a/packages/next-polyfill-nomodule/package.json +++ b/packages/next-polyfill-nomodule/package.json @@ -1,6 +1,6 @@ { "name": "@next/polyfill-nomodule", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "description": "A polyfill for non-dead, nomodule browsers.", "main": "dist/polyfill-nomodule.js", "license": "MIT", diff --git a/packages/next-routing/package.json b/packages/next-routing/package.json index b5dea8319a57..d8f710b67c24 100644 --- a/packages/next-routing/package.json +++ b/packages/next-routing/package.json @@ -1,6 +1,6 @@ { "name": "@next/routing", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "keywords": [ "react", "next", diff --git a/packages/next-rspack/package.json b/packages/next-rspack/package.json index a85695747375..cc42df58b3d7 100644 --- a/packages/next-rspack/package.json +++ b/packages/next-rspack/package.json @@ -1,6 +1,6 @@ { "name": "next-rspack", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "repository": { "url": "vercel/next.js", "directory": "packages/next-rspack" diff --git a/packages/next-swc/package.json b/packages/next-swc/package.json index d12f378f521b..794b151969fe 100644 --- a/packages/next-swc/package.json +++ b/packages/next-swc/package.json @@ -1,6 +1,6 @@ { "name": "@next/swc", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "private": true, "files": [ "native/" diff --git a/packages/next/package.json b/packages/next/package.json index ec4013ff932c..1e40f227e73e 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -1,6 +1,6 @@ { "name": "next", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "description": "The React Framework", "main": "./dist/server/next.js", "license": "MIT", @@ -100,7 +100,7 @@ ] }, "dependencies": { - "@next/env": "16.3.1-canary.23", + "@next/env": "16.3.1-canary.24", "@swc/helpers": "0.5.23", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", @@ -164,11 +164,11 @@ "@modelcontextprotocol/sdk": "1.18.1", "@mswjs/interceptors": "0.42.0", "@napi-rs/triples": "1.2.0", - "@next/font": "16.3.1-canary.23", - "@next/polyfill-module": "16.3.1-canary.23", - "@next/polyfill-nomodule": "16.3.1-canary.23", - "@next/react-refresh-utils": "16.3.1-canary.23", - "@next/swc": "16.3.1-canary.23", + "@next/font": "16.3.1-canary.24", + "@next/polyfill-module": "16.3.1-canary.24", + "@next/polyfill-nomodule": "16.3.1-canary.24", + "@next/react-refresh-utils": "16.3.1-canary.24", + "@next/swc": "16.3.1-canary.24", "@opentelemetry/api": "1.6.0", "@playwright/test": "1.61.0", "@rspack/core": "1.6.7", diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 9b297077e69b..b09eb6fca543 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -138,7 +138,11 @@ import { pageToRoute, } from './utils' import type { DynamicManifestRoute, PageInfo, PageInfos } from './utils' -import type { FallbackRouteParam, PrerenderedRoute } from './static-paths/types' +import type { + FallbackRouteParam, + PrerenderRouteMatcher, + PrerenderedRoute, +} from './static-paths/types' import type { AppSegmentConfig } from './segment-config/app/app-segment-config' import { writeBuildId } from './write-build-id' import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path' @@ -2168,6 +2172,7 @@ export default async function build( const serverPropsPages = new Set() const additionalPaths = new Map() const staticPaths = new Map() + const prerenderRouteMatchers = new Map() const appNormalizedPaths = new Map() const fallbackModes = new Map() const appDefaultConfigs = new Map() @@ -2569,6 +2574,13 @@ export default async function build( isSSG = true } + if (workerResult.prerenderRouteMatchers) { + prerenderRouteMatchers.set( + originalAppPath, + workerResult.prerenderRouteMatchers + ) + } + const appConfig = workerResult.appConfig || {} if (appConfig.revalidate !== 0) { const hasGenerateStaticParams = @@ -3226,29 +3238,30 @@ export default async function build( // If there was no result, there's nothing more to do. if (!exportResult) return - const getFallbackMode = (route: PrerenderedRoute) => { - const hasEmptyStaticShell = exportResult.byPath.get( - route.pathname - )?.hasEmptyStaticShell - + const resolveFallbackMode = ( + matcher: PrerenderRouteMatcher, + prerenderCandidate: PrerenderedRoute | undefined, + hasEmptyStaticShell: boolean | undefined + ) => { // If the route has an empty static shell and is not configured to // throw on empty static shell, then we should use the blocking // static render mode. if ( + prerenderCandidate && hasEmptyStaticShell && - !route.throwOnEmptyStaticShell && - route.fallbackMode === FallbackMode.PRERENDER + !prerenderCandidate.throwOnEmptyStaticShell && + matcher.fallbackMode === FallbackMode.PRERENDER ) { return FallbackMode.BLOCKING_STATIC_RENDER } // If the route has no fallback mode, then we should use the // `NOT_FOUND` fallback mode. - if (!route.fallbackMode) { + if (!matcher.fallbackMode) { return FallbackMode.NOT_FOUND } - return route.fallbackMode + return matcher.fallbackMode } const getCacheControl = ( @@ -3311,6 +3324,15 @@ export default async function build( if (!appConfig) throw new InvariantError('App config not found') const ssgPageRoutesSet = new Set(pageInfos.get(page)?.ssgPageRoutes) + // Preserve the specificity order that unknown prerender routes had + // before matchers were modeled separately. Some metadata, such as + // prefetch hints, is collected using first-writer-wins semantics. + const dynamicRouteMatchers = [ + ...sortPageObjects( + prerenderRouteMatchers.get(originalAppPath) ?? [], + (route) => route.pathname + ), + ] let hasRevalidateZero = appConfig.revalidate === 0 || @@ -3367,13 +3389,10 @@ export default async function build( : []), ] - // We should collect all the dynamic routes into a single array for - // this page. Including the full fallback route (the original - // route), any routes that were generated with unknown route params - // should be collected and included in the dynamic routes part - // of the manifest instead. - const staticPrerenderedRoutes: PrerenderedRoute[] = [] - const dynamicPrerenderedRoutes: PrerenderedRoute[] = [] + // Candidates without unknown params can become concrete static + // outputs. Candidates with unknown params are finalized alongside + // the logical matcher directives collected above. + const concretePrerenderCandidates: PrerenderedRoute[] = [] // Sort the outputted routes to ensure consistent output. Any route // though that has unknown route params will be pulled and sorted @@ -3439,18 +3458,17 @@ export default async function build( prerenderedRoute.fallbackRouteParams && prerenderedRoute.fallbackRouteParams.length > 0 ) { - // If the route has unknown params, then we need to add it to - // the list of dynamic routes. - dynamicPrerenderedRoutes.push(prerenderedRoute) + // Partial candidates have a corresponding matcher directive + // and are finalized below after inspecting their render. } else { // If the route doesn't have unknown params, then we need to // add it to the list of static routes. - staticPrerenderedRoutes.push(prerenderedRoute) + concretePrerenderCandidates.push(prerenderedRoute) } } // Handle all the static routes. - for (const route of staticPrerenderedRoutes) { + for (const route of concretePrerenderCandidates) { if (isDynamicRoute(page) && route.pathname === page) continue const pageInfo = pageInfos.get(page) as PageInfo @@ -3602,20 +3620,59 @@ export default async function build( // they are enabled, then it'll already be included in the // prerendered routes. if (!isRoutePPREnabled) { - dynamicPrerenderedRoutes.push({ - params: {}, + dynamicRouteMatchers.push({ pathname: page, - encodedPathname: page, fallbackRouteParams: [], fallbackMode: fallbackModes.get(originalAppPath) ?? FallbackMode.NOT_FOUND, fallbackRootParams: [], - throwOnEmptyStaticShell: true, }) } - for (const route of dynamicPrerenderedRoutes) { + // A logical matcher can have zero or more render candidates. + // Today generateStaticParams produces at most one candidate per + // pathname. Variants can multiply that into several artifacts + // without changing the logical matcher, so retain every + // candidate instead of letting pathname select whichever one was + // inserted last. + const prerenderCandidatesByPathname = new Map< + string, + PrerenderedRoute[] + >() + for (const candidate of prerenderedRoutes) { + const candidates = prerenderCandidatesByPathname.get( + candidate.pathname + ) + if (candidates) { + candidates.push(candidate) + } else { + prerenderCandidatesByPathname.set(candidate.pathname, [ + candidate, + ]) + } + } + + const dynamicRouteEntries: Array<{ + matcher: PrerenderRouteMatcher + prerenderCandidate: PrerenderedRoute | undefined + }> = [] + for (const matcher of dynamicRouteMatchers) { + const candidates = prerenderCandidatesByPathname.get( + matcher.pathname + ) ?? [undefined] + for (const prerenderCandidate of candidates) { + dynamicRouteEntries.push({ + matcher, + prerenderCandidate, + }) + } + } + + for (const { + matcher: route, + prerenderCandidate, + } of dynamicRouteEntries) { // Static metadata files are rewritten above into the known // static bucket under their `-`-placeholder pathname, so any // entry that slips through here (e.g. an unexpected fallback @@ -3626,13 +3683,24 @@ export default async function build( continue } - const normalizedRoute = normalizePagePath(route.pathname) + // This is the artifact associated with this matcher entry. It + // currently has the same pathname as the logical matcher, but + // that is not an invariant: variants can write several + // artifacts for one matcher under distinct output paths. + const prerenderOutputPathname = + prerenderCandidate?.pathname ?? route.pathname + + const normalizedRoute = normalizePagePath( + prerenderOutputPathname + ) const parentPageInfo = pageInfos.get(page) as PageInfo - const routeResult = exportResult.byPath.get(route.pathname) + const routeResult = exportResult.byPath.get( + prerenderOutputPathname + ) const metadata = routeResult?.metadata - const cacheControl = getCacheControl(route.pathname) + const cacheControl = getCacheControl(prerenderOutputPathname) let dataRoute: string | null = null if (!isAppRouteHandler) { @@ -3720,10 +3788,10 @@ export default async function build( if (route.pathname === page) { // The route pattern entry (for example `/blog/[slug]`) is - // also present in `dynamicPrerenderedRoutes`. Keep updating - // the parent entry in place so it retains its `ssgPageRoutes` - // subtree; if we rewrote it like a concrete child route we - // would lose the generated child paths from the build output. + // also present in `dynamicRouteMatchers`. Keep updating the + // parent entry in place so it retains its `ssgPageRoutes` + // subtree; rewriting it like a concrete child route would + // lose the generated child paths from the build output. pageInfos.set(page, { ...(pageInfos.get(page) as PageInfo), initialCacheControl: cacheControl, @@ -3751,7 +3819,11 @@ export default async function build( }) } - const fallbackMode = getFallbackMode(route) + const fallbackMode = resolveFallbackMode( + route, + prerenderCandidate, + routeResult?.hasEmptyStaticShell + ) // When the route is configured to serve a prerender, we should // use the cache control from the export result. If it can't be @@ -3795,7 +3867,7 @@ export default async function build( } } - prerenderManifest.dynamicRoutes[route.pathname] = { + prerenderManifest.dynamicRoutes[prerenderOutputPathname] = { experimentalPPR: isRoutePPREnabled, remainingPrerenderableParams: route.remainingPrerenderableParams, @@ -3807,7 +3879,7 @@ export default async function build( ...classification, experimentalBypassFor: bypassFor, routeRegex: normalizeRouteRegex( - getNamedRouteRegex(route.pathname, { + getNamedRouteRegex(prerenderOutputPathname, { prefixRouteKeys: false, }).re.source ), diff --git a/packages/next/src/build/static-paths/app.ts b/packages/next/src/build/static-paths/app.ts index 57dec791abf0..5ad4c6e12c5a 100644 --- a/packages/next/src/build/static-paths/app.ts +++ b/packages/next/src/build/static-paths/app.ts @@ -3,6 +3,7 @@ import type { AppPageModule } from '../../server/route-modules/app-page/module' import type { AppSegment } from '../segment-config/app/app-segments' import type { FallbackRouteParam, + PrerenderRouteMatcher, PrerenderedRoute, StaticPathsResult, } from './types' @@ -1175,5 +1176,28 @@ export async function buildAppStaticPaths({ assignStaticShellMetadata(prerenderedRoutes, prerenderablePathSegments) } - return { fallbackMode, prerenderedRoutes } + const prerenderRouteMatchersByPathname = new Map< + string, + PrerenderRouteMatcher + >() + if (prerenderedRoutes && isRoutePPREnabled) { + for (const prerenderCandidate of prerenderedRoutes) { + if (!prerenderCandidate.fallbackRouteParams?.length) continue + prerenderRouteMatchersByPathname.set(prerenderCandidate.pathname, { + pathname: prerenderCandidate.pathname, + fallbackRouteParams: prerenderCandidate.fallbackRouteParams, + fallbackMode: prerenderCandidate.fallbackMode, + fallbackRootParams: prerenderCandidate.fallbackRootParams, + remainingPrerenderableParams: + prerenderCandidate.remainingPrerenderableParams, + }) + } + } + + const prerenderRouteMatchers = + prerenderRouteMatchersByPathname.size > 0 + ? [...prerenderRouteMatchersByPathname.values()] + : undefined + + return { fallbackMode, prerenderedRoutes, prerenderRouteMatchers } } diff --git a/packages/next/src/build/static-paths/types.ts b/packages/next/src/build/static-paths/types.ts index 93e2d8881c81..2c324d12648b 100644 --- a/packages/next/src/build/static-paths/types.ts +++ b/packages/next/src/build/static-paths/types.ts @@ -52,9 +52,40 @@ type FallbackPrerenderedRoute = { throwOnEmptyStaticShell: boolean } +/** + * A route the build plans to prerender. Rendering decides whether the result + * becomes a published output: for example, an allowed empty fallback shell is + * discarded and its matcher becomes blocking instead. + * + * The historical name is retained because this type is used throughout static + * path generation, but values of this type are prerender candidates rather + * than guaranteed outputs. + */ export type PrerenderedRoute = StaticPrerenderedRoute | FallbackPrerenderedRoute +/** + * Describes how a dynamic pathname is matched when no concrete build-time + * output matches it. It describes the logical route independently of any + * artifacts produced for it, and is not itself something to render. + * + * Zero or more prerender candidates may share this pathname. In particular, + * variants can produce several artifacts for one logical matcher, so consumers + * must not assume pathname identifies a single candidate or render result. + */ +export type PrerenderRouteMatcher = { + readonly pathname: string + readonly fallbackRouteParams: readonly FallbackRouteParam[] + readonly fallbackMode: FallbackMode | undefined + readonly fallbackRootParams: readonly string[] + readonly remainingPrerenderableParams?: readonly FallbackRouteParam[] +} + export type StaticPathsResult = { fallbackMode: FallbackMode | undefined + + /** Planned renders, some of which may be discarded after rendering. */ prerenderedRoutes: PrerenderedRoute[] | undefined + + /** Logical request matchers, independent of the artifacts rendered for them. */ + prerenderRouteMatchers?: PrerenderRouteMatcher[] } diff --git a/packages/next/src/build/utils.ts b/packages/next/src/build/utils.ts index cff981644a6b..cb850f904967 100644 --- a/packages/next/src/build/utils.ts +++ b/packages/next/src/build/utils.ts @@ -69,7 +69,10 @@ import { createIncrementalCache } from '../export/helpers/create-incremental-cac import { collectRootParamKeys } from './segment-config/app/collect-root-param-keys' import { buildAppStaticPaths } from './static-paths/app' import { buildPagesStaticPaths } from './static-paths/pages' -import type { PrerenderedRoute } from './static-paths/types' +import type { + PrerenderRouteMatcher, + PrerenderedRoute, +} from './static-paths/types' import type { CacheControl } from '../server/lib/cache-control' import { formatExpire, formatRevalidate } from './output/format' import type { @@ -672,6 +675,7 @@ type PageIsStaticResult = { hasServerProps?: boolean hasStaticProps?: boolean prerenderedRoutes: PrerenderedRoute[] | undefined + prerenderRouteMatchers: PrerenderRouteMatcher[] | undefined prerenderFallbackMode: FallbackMode | undefined rootParamKeys: readonly string[] | undefined isNextImageImported?: boolean @@ -742,6 +746,7 @@ export async function isPageStatic({ isRoutePPREnabled: false, prerenderFallbackMode: undefined, prerenderedRoutes: undefined, + prerenderRouteMatchers: undefined, rootParamKeys: undefined, hasStaticProps: false, hasServerProps: false, @@ -768,6 +773,7 @@ export async function isPageStatic({ let componentsResult: LoadComponentsReturnType let prerenderedRoutes: PrerenderedRoute[] | undefined + let prerenderRouteMatchers: PrerenderRouteMatcher[] | undefined let prerenderFallbackMode: FallbackMode | undefined let appConfig: AppSegmentConfig = {} let rootParamKeys: readonly string[] | undefined @@ -887,29 +893,32 @@ export async function isPageStatic({ ;({ prerenderedRoutes, fallbackMode: prerenderFallbackMode } = buildStaticMetadataStaticPaths(page)) } else { - ;({ prerenderedRoutes, fallbackMode: prerenderFallbackMode } = - await buildAppStaticPaths({ - dir, - page, - route, - cacheComponents, - authInterrupts, - useCacheTimeout, - staticPageGenerationTimeout, - segments, - distDir, - requestHeaders: {}, - isrFlushToDisk, - cacheMaxMemorySize, - cacheHandler, - cacheLifeProfiles, - ComponentMod, - nextConfigOutput, - isRoutePPREnabled, - buildId, - deploymentId, - rootParamKeys, - })) + ;({ + prerenderedRoutes, + prerenderRouteMatchers, + fallbackMode: prerenderFallbackMode, + } = await buildAppStaticPaths({ + dir, + page, + route, + cacheComponents, + authInterrupts, + useCacheTimeout, + staticPageGenerationTimeout, + segments, + distDir, + requestHeaders: {}, + isrFlushToDisk, + cacheMaxMemorySize, + cacheHandler, + cacheLifeProfiles, + ComponentMod, + nextConfigOutput, + isRoutePPREnabled, + buildId, + deploymentId, + rootParamKeys, + })) } } } else { @@ -982,6 +991,7 @@ export async function isPageStatic({ isRoutePPREnabled, prerenderFallbackMode, prerenderedRoutes, + prerenderRouteMatchers, rootParamKeys, hasStaticProps, hasServerProps, diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index ae393a3fe587..d0fc70d1cc6a 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -496,6 +496,7 @@ export async function createHotReloaderTurbopack( 'StartupCacheInvalidationEvent', 'TimingEvent', 'SlowFilesystemEvent', + 'FilesystemSettlingEvent', 'TraceEvent', ], parentSpan: hotReloaderSpan, diff --git a/packages/react-refresh-utils/package.json b/packages/react-refresh-utils/package.json index c07c6a9e857a..bbfa30468eff 100644 --- a/packages/react-refresh-utils/package.json +++ b/packages/react-refresh-utils/package.json @@ -1,6 +1,6 @@ { "name": "@next/react-refresh-utils", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "description": "An experimental package providing utilities for React Refresh.", "repository": { "url": "vercel/next.js", diff --git a/packages/third-parties/package.json b/packages/third-parties/package.json index 95b1a958da05..274a8bf5f91a 100644 --- a/packages/third-parties/package.json +++ b/packages/third-parties/package.json @@ -1,6 +1,6 @@ { "name": "@next/third-parties", - "version": "16.3.1-canary.23", + "version": "16.3.1-canary.24", "repository": { "url": "vercel/next.js", "directory": "packages/third-parties" @@ -26,7 +26,7 @@ "third-party-capital": "1.0.20" }, "devDependencies": { - "next": "16.3.1-canary.23", + "next": "16.3.1-canary.24", "outdent": "0.8.0", "prettier": "2.5.1", "typescript": "6.0.2" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index acc28398c22c..c0eb8bda2eda 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1024,7 +1024,7 @@ importers: packages/eslint-config-next: dependencies: '@next/eslint-plugin-next': - specifier: 16.3.1-canary.23 + specifier: 16.3.1-canary.24 version: link:../eslint-plugin-next eslint: specifier: '>=9.0.0' @@ -1107,7 +1107,7 @@ importers: packages/next: dependencies: '@next/env': - specifier: 16.3.1-canary.23 + specifier: 16.3.1-canary.24 version: link:../next-env '@swc/helpers': specifier: 0.5.23 @@ -1228,19 +1228,19 @@ importers: specifier: 1.2.0 version: 1.2.0 '@next/font': - specifier: 16.3.1-canary.23 + specifier: 16.3.1-canary.24 version: link:../font '@next/polyfill-module': - specifier: 16.3.1-canary.23 + specifier: 16.3.1-canary.24 version: link:../next-polyfill-module '@next/polyfill-nomodule': - specifier: 16.3.1-canary.23 + specifier: 16.3.1-canary.24 version: link:../next-polyfill-nomodule '@next/react-refresh-utils': - specifier: 16.3.1-canary.23 + specifier: 16.3.1-canary.24 version: link:../react-refresh-utils '@next/swc': - specifier: 16.3.1-canary.23 + specifier: 16.3.1-canary.24 version: link:../next-swc '@opentelemetry/api': specifier: 1.6.0 @@ -1983,7 +1983,7 @@ importers: version: 1.0.20 devDependencies: next: - specifier: 16.3.1-canary.23 + specifier: 16.3.1-canary.24 version: link:../next outdent: specifier: 0.8.0 diff --git a/test/development/fs-settling-event/app/layout.tsx b/test/development/fs-settling-event/app/layout.tsx new file mode 100644 index 000000000000..08eaa94fdc88 --- /dev/null +++ b/test/development/fs-settling-event/app/layout.tsx @@ -0,0 +1,11 @@ +export default function RootLayout({ + children, +}: { + children: React.ReactNode +}) { + return ( + + {children} + + ) +} diff --git a/test/development/fs-settling-event/app/page.tsx b/test/development/fs-settling-event/app/page.tsx new file mode 100644 index 000000000000..3301daa998be --- /dev/null +++ b/test/development/fs-settling-event/app/page.tsx @@ -0,0 +1,9 @@ +// Importing this package makes Turbopack read (and therefore watch) the file +// inside `node_modules`, so the writes the test performs generate watcher +// events. This matters on Linux, where the watcher is non-recursive and only +// watches directories it has been asked to read. +import counter from 'fs-settling-fixture-pkg' + +export default function Page() { + return

counter: {counter}

+} diff --git a/test/development/fs-settling-event/fs-settling-event.test.ts b/test/development/fs-settling-event/fs-settling-event.test.ts new file mode 100644 index 000000000000..1b9ade87ad46 --- /dev/null +++ b/test/development/fs-settling-event/fs-settling-event.test.ts @@ -0,0 +1,46 @@ +import { nextTestSetup } from 'e2e-utils' +import { retry } from 'next-test-utils' +import stripAnsi from 'strip-ansi' +import fs from 'fs' +import path from 'path' + +// The `FilesystemSettlingEvent` compilation event is Turbopack-only. +;(process.env.IS_TURBOPACK_TEST ? describe : describe.skip)( + 'fs-settling-event', + () => { + const { next } = nextTestSetup({ files: __dirname }) + + it('logs a settling event during sustained node_modules churn', async () => { + // Compile the page first so the imported `node_modules` file is watched. + await next.render('/') + + const pkgFile = path.join( + next.testDir, + 'node_modules/fs-settling-fixture-pkg/index.js' + ) + const outputIndex = next.cliOutput.length + + // Rewrite the imported module every 20ms. Since 20ms is well below the + // extended `node_modules` batch delay (200ms), the watcher keeps a single + // batch of events open, which triggers the settling event after ~5s. + let i = 0 + const interval = setInterval(() => { + fs.writeFileSync(pkgFile, `export default ${i++}\n`) + }, 20) + + try { + await retry( + () => { + const output = stripAnsi(next.cliOutput.slice(outputIndex)) + expect(output).toContain('waiting for the filesystem to settle') + }, + // The event fires after ~5s; allow a generous window to avoid flakes. + 15000, + 500 + ) + } finally { + clearInterval(interval) + } + }) + } +) diff --git a/test/development/fs-settling-event/node_modules/fs-settling-fixture-pkg/index.js b/test/development/fs-settling-event/node_modules/fs-settling-fixture-pkg/index.js new file mode 100644 index 000000000000..029f788d6d4c --- /dev/null +++ b/test/development/fs-settling-event/node_modules/fs-settling-fixture-pkg/index.js @@ -0,0 +1 @@ +export default 0 diff --git a/test/development/fs-settling-event/node_modules/fs-settling-fixture-pkg/package.json b/test/development/fs-settling-event/node_modules/fs-settling-fixture-pkg/package.json new file mode 100644 index 000000000000..37ba57905675 --- /dev/null +++ b/test/development/fs-settling-event/node_modules/fs-settling-fixture-pkg/package.json @@ -0,0 +1,5 @@ +{ + "name": "fs-settling-fixture-pkg", + "version": "1.0.0", + "main": "index.js" +} diff --git a/test/e2e/app-dir/segment-cache/prefetch-inlining/app/test-dynamic-partial/[top]/[bottom]/page.tsx b/test/e2e/app-dir/segment-cache/prefetch-inlining/app/test-dynamic-partial/[top]/[bottom]/page.tsx new file mode 100644 index 000000000000..d1b09a9676c2 --- /dev/null +++ b/test/e2e/app-dir/segment-cache/prefetch-inlining/app/test-dynamic-partial/[top]/[bottom]/page.tsx @@ -0,0 +1,8 @@ +export default async function Page({ + params, +}: { + params: Promise<{ top: string; bottom: string }> +}) { + const { top, bottom } = await params + return

{`Dynamic page: ${top}/${bottom}`}

+} diff --git a/test/e2e/app-dir/segment-cache/prefetch-inlining/app/test-dynamic-partial/[top]/layout.tsx b/test/e2e/app-dir/segment-cache/prefetch-inlining/app/test-dynamic-partial/[top]/layout.tsx new file mode 100644 index 000000000000..f8a898b8189e --- /dev/null +++ b/test/e2e/app-dir/segment-cache/prefetch-inlining/app/test-dynamic-partial/[top]/layout.tsx @@ -0,0 +1,23 @@ +import { Suspense, type ReactNode } from 'react' +import { NoInline } from '../../../components/no-inline' + +export function generateStaticParams() { + return [{ top: 't1' }] +} + +export default async function Layout({ + children, + params, +}: { + children: ReactNode + params: Promise<{ top: string }> +}) { + const { top } = await params + return ( +
+ +

{`Top: ${top}`}

+ Loading bottom...

}>{children}
+
+ ) +} diff --git a/test/e2e/app-dir/segment-cache/prefetch-inlining/app/test-dynamic-partial/layout.tsx b/test/e2e/app-dir/segment-cache/prefetch-inlining/app/test-dynamic-partial/layout.tsx new file mode 100644 index 000000000000..a1e138ba8b34 --- /dev/null +++ b/test/e2e/app-dir/segment-cache/prefetch-inlining/app/test-dynamic-partial/layout.tsx @@ -0,0 +1,10 @@ +import type { ReactNode } from 'react' + +export default function Layout({ children }: { children: ReactNode }) { + return ( +
+

Static parent

+ {children} +
+ ) +} diff --git a/test/e2e/app-dir/segment-cache/prefetch-inlining/prefetch-inlining.test.ts b/test/e2e/app-dir/segment-cache/prefetch-inlining/prefetch-inlining.test.ts index 64959a338c81..bd0549b6a750 100644 --- a/test/e2e/app-dir/segment-cache/prefetch-inlining/prefetch-inlining.test.ts +++ b/test/e2e/app-dir/segment-cache/prefetch-inlining/prefetch-inlining.test.ts @@ -185,7 +185,7 @@ async function getRouteTreeFromHistory( } describe('prefetch inlining', () => { - const { next, isNextDev, isTurbopack } = nextTestSetup({ + const { next, isNextDev, isNextStart, isTurbopack } = nextTestSetup({ files: __dirname, }) @@ -533,6 +533,40 @@ describe('prefetch inlining', () => { ) }) + if (isNextStart) { + it('partially generated dynamic route: build hints use the most specific shell', async () => { + const hints = await next.readJSON('.next/server/prefetch-hints.json') + + expect(hints['/test-dynamic-partial/[top]/[bottom]']) + .toMatchInlineSnapshot(` + { + "hints": 64, + "slots": { + "children": { + "hints": 96, + "slots": { + "children": { + "hints": 32, + "slots": { + "children": { + "hints": 64, + "slots": { + "children": { + "hints": 160, + "slots": null, + }, + }, + }, + }, + }, + }, + }, + }, + } + `) + }) + } + // TODO: Add a test for stale hints (InliningHintsStale). The stale hints // mechanism expires the route cache entry so the next prefetch re-fetches // the correct tree. This is hard to test reliably with act() because the diff --git a/turbopack/crates/turbo-tasks-fs/src/lib.rs b/turbopack/crates/turbo-tasks-fs/src/lib.rs index d0773b250a9a..93e748ee4502 100644 --- a/turbopack/crates/turbo-tasks-fs/src/lib.rs +++ b/turbopack/crates/turbo-tasks-fs/src/lib.rs @@ -60,7 +60,7 @@ pub use crate::{ path::{FileSystemPath, FileSystemPathOption, RealPathResult, RealPathResultError, rebase}, read_glob::ReadGlobResult, virtual_fs::VirtualFileSystem, - watcher::{DiskWatcherConfig, DiskWatcherRecursiveMode}, + watcher::{DiskWatcherConfig, DiskWatcherPathMatcher, DiskWatcherRecursiveMode}, windows::to_verbatim_with_case_folded_disk, }; diff --git a/turbopack/crates/turbo-tasks-fs/src/watcher/batch_schedule.rs b/turbopack/crates/turbo-tasks-fs/src/watcher/batch_schedule.rs new file mode 100644 index 000000000000..e9edab618384 --- /dev/null +++ b/turbopack/crates/turbo-tasks-fs/src/watcher/batch_schedule.rs @@ -0,0 +1,171 @@ +use std::{ + sync::{ + Arc, + mpsc::{Receiver, RecvTimeoutError}, + }, + time::{Duration, Instant}, +}; + +use serde::Serialize; +use turbo_tasks::message_queue::{CompilationEvent, Severity}; + +use crate::{DiskWatcherConfig, watcher::fs_api::DiskFileSystemWatcherApi}; + +/// Decides how long a batch of watcher events stays open, and emits a repeated +/// [`FilesystemSettlingEvent`] for as long as it does. +pub struct BatchSchedule { + settling_event_initial_delay: Duration, + settling_event_max_delay: Duration, + pending: Option, +} + +/// A batch that has at least one event in it and hasn't been flushed yet. +struct PendingBatch { + started: Instant, + /// The batch is flushed once this passes without any further events. + deadline: Instant, + /// When to emit the next [`FilesystemSettlingEvent`]. + settling_event_next_at: Instant, + /// Grows exponentially (up to [`BatchSchedule::settling_event_max_delay`]) so that a writer + /// holding a batch open for minutes doesn't flood the compilation event queue. + event_interval: Duration, +} + +impl BatchSchedule { + pub fn new(config: &DiskWatcherConfig) -> Self { + Self { + settling_event_initial_delay: config.settling_event_initial_delay, + settling_event_max_delay: config.settling_event_max_delay, + pending: None, + } + } + + /// Keeps the batch open for at least `delay` from now, opening a new batch if there isn't one. + pub fn extend(&mut self, delay: Duration) { + let now = Instant::now(); + let deadline = now.checked_add(delay).unwrap_or_else(far_future); + match &mut self.pending { + Some(pending) => pending.deadline = pending.deadline.max(deadline), + None => { + self.pending = Some(PendingBatch { + started: now, + deadline, + settling_event_next_at: now + .checked_add(self.settling_event_initial_delay) + .unwrap_or_else(far_future), + event_interval: self.settling_event_initial_delay, + }) + } + } + } + + /// Waits for the next watcher event, emitting [`FilesystemSettlingEvent`]s while the pending + /// batch keeps growing. If no batch is pending, this blocks until an event arrives. + /// + /// [`RecvTimeoutError::Timeout`] means the pending batch's deadline has passed *and* nothing + /// more is queued, so the batch is complete and should be flushed. + pub fn recv_event( + &mut self, + rx: &Receiver>, + fs: &FsApi, + ) -> Result, RecvTimeoutError> { + let max_event_delay = self.settling_event_max_delay; + loop { + let Some(pending) = &mut self.pending else { + // no pending batch: wait indefinitely + return rx.recv().map_err(|_| RecvTimeoutError::Disconnected); + }; + + let now = Instant::now(); + if now >= pending.settling_event_next_at { + pending.emit_settling_event(fs, now, max_event_delay); + } + + let timeout = pending + .deadline + .min(pending.settling_event_next_at) + .saturating_duration_since(now); + + match rx.recv_timeout(timeout) { + Ok(event) => { + return Ok(event); + } + Err(RecvTimeoutError::Timeout) => { + if Instant::now() >= pending.deadline { + self.pending = None; + return Err(RecvTimeoutError::Timeout); + } + continue; + } + Err(err) => return Err(err), + } + } + } + + /// Closes the pending batch, used when a rescan happens. + pub fn reset(&mut self) { + self.pending = None; + } +} + +impl PendingBatch { + fn emit_settling_event( + &mut self, + fs: &FsApi, + now: Instant, + max_event_delay: Duration, + ) { + let _guard = fs.tokio_handle().enter(); + if let Some(turbo_tasks) = fs.turbo_tasks() { + turbo_tasks.send_compilation_event(Arc::new(FilesystemSettlingEvent { + elapsed_secs: (now - self.started).as_secs_f64(), + })); + } + self.event_interval = self.event_interval.saturating_mul(2).min(max_event_delay); + // Schedule from "now" instead of accumulating intervals, so that emitting late (e.g. under + // heavy load) doesn't produce a catch-up burst of events. + self.settling_event_next_at = now + .checked_add(self.event_interval) + .unwrap_or_else(far_future); + } +} + +/// Emitted when frequent filesystem updates cause us to keep a batch open for an extended period of +/// time. Informing the user when this happens may help them understand what's happening, and that +/// Turbopack is not stalled. +#[derive(Debug, Clone, Serialize)] +pub struct FilesystemSettlingEvent { + /// How long the current batch has been held open, in seconds. + pub elapsed_secs: f64, +} + +impl CompilationEvent for FilesystemSettlingEvent { + fn type_name(&self) -> &'static str { + "FilesystemSettlingEvent" + } + + fn severity(&self) -> Severity { + Severity::Info + } + + fn message(&self) -> String { + format!( + "Turbopack has seen frequent file updates and is waiting for the filesystem to settle \ + ({:.1}s elapsed so far).", + self.elapsed_secs + ) + } + + fn to_json(&self) -> String { + serde_json::to_string(self).unwrap() + } +} + +// from https://github.com/tokio-rs/tokio/blob/29cd6ec1ec6f90a7ee1ad641c03e0e00badbcb0e/tokio/src/time/instant.rs#L57-L63 +fn far_future() -> Instant { + // Roughly 30 years from now. + // API does not provide a way to obtain max `Instant` + // or convert specific date in the future to instant. + // 1000 years overflows on macOS, 100 years overflows on FreeBSD. + Instant::now() + Duration::from_secs(86400 * 365 * 30) +} diff --git a/turbopack/crates/turbo-tasks-fs/src/watcher/mod.rs b/turbopack/crates/turbo-tasks-fs/src/watcher/mod.rs index 40f994033331..192284d21fde 100644 --- a/turbopack/crates/turbo-tasks-fs/src/watcher/mod.rs +++ b/turbopack/crates/turbo-tasks-fs/src/watcher/mod.rs @@ -1,3 +1,4 @@ +mod batch_schedule; mod fs_api; #[cfg(test)] mod mock_fs_api; @@ -11,7 +12,7 @@ use std::{ Arc, LazyLock, mpsc::{Receiver, RecvTimeoutError, channel}, }, - time::{Duration, Instant}, + time::Duration, }; use anyhow::{Context, Result}; @@ -31,7 +32,7 @@ use tokio::sync::{RwLock, RwLockWriteGuard}; use tracing::instrument; use turbo_rcstr::RcStr; use turbo_tasks::{ - FxIndexSet, InvalidationReason, InvalidationReasonKind, Invalidator, NonLocalValue, TaskInput, + FxIndexSet, InvalidationReason, InvalidationReasonKind, Invalidator, ResolvedVc, TraitRef, TurboTasksApi, spawn_thread, trace::TraceRawVcs, util::StaticOrArc, }; @@ -40,7 +41,7 @@ use crate::{ invalidation::{WatchChange, WatchStart}, invalidator_map::InvalidatorMap, path_map::OrderedPathMapExt, - watcher::fs_api::DiskFileSystemWatcherApi, + watcher::{batch_schedule::BatchSchedule, fs_api::DiskFileSystemWatcherApi}, }; /// Overrides [`DiskWatcherConfig::recursive_mode`]. Users shouldn't need to set this, this is @@ -59,9 +60,8 @@ static FORCED_WATCH_RECURSIVE_MODE: LazyLock> = }, ); -#[derive( - Clone, Copy, Debug, Default, Eq, PartialEq, Hash, TraceRawVcs, NonLocalValue, Encode, Decode, -)] +#[turbo_tasks::task_input] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, TraceRawVcs, Encode, Decode)] pub struct DiskWatcherConfig { /// Whether to let the [`notify::Watcher`] recurse into subdirectories itself, or to track and /// watch each directory we care about ourselves. @@ -85,14 +85,54 @@ pub struct DiskWatcherConfig { /// This costs an extra allocation per invalidated path, so it's only worth enabling when /// something actually consumes the reasons. pub report_invalidation_reason: bool, + + /// How long to keep a batch of filesystem events open, waiting for more events, before + /// flushing invalidations. Batching coalesces bursts (e.g. a `git checkout`) into a single + /// invalidation pass and avoids reading half-written files. + /// + /// If set too low (<10ms), this is known to cause partial file reads on Linux where `inotify` + /// has very low latency. + pub batch_delay: Duration, + /// When [`DiskWatcherPathMatcher::match_path`] returns `true`, we will extend the batch by + /// [`Self::extended_batch_delay_duration`]. + pub extended_batch_delay_matcher: Option>>, + /// The idle period required to close a batch once [`Self::extended_batch_delay_matcher`] has + /// matched. Unused when there is no matcher. + pub extended_batch_delay_duration: Duration, + + /// If a single batch stays open at least this long, emit a `FilesystemSettlingEvent` + /// compilation event so the user knows why work has stalled. Repeated events within the same + /// batch back off exponentially, up to [`Self::settling_event_max_delay`]. + pub settling_event_initial_delay: Duration, + /// Upper bound for the exponentially increasing interval between repeated + /// `FilesystemSettlingEvent`s within a single batch. + pub settling_event_max_delay: Duration, } -impl TaskInput for DiskWatcherConfig { - fn is_transient(&self) -> bool { - false +impl Default for DiskWatcherConfig { + fn default() -> Self { + Self { + recursive_mode: None, + poll_interval: None, + report_invalidation_reason: false, + batch_delay: Duration::from_millis(10), + extended_batch_delay_matcher: None, + extended_batch_delay_duration: Duration::from_millis(200), + settling_event_initial_delay: Duration::from_millis(500), + settling_event_max_delay: Duration::from_secs(60), + } } } +/// Matches absolute paths reported by the filesystem watcher. See +/// [`DiskWatcherConfig::extended_batch_delay_matcher`]. +#[turbo_tasks::value_trait] +pub trait DiskWatcherPathMatcher { + /// Called on the watcher thread once per path of every incoming event, so this should be + /// cheap and must not block. + fn match_path(&self, path: &Path) -> bool; +} + /// Equivalent to [`notify::RecursiveMode`], but implements traits needed by [`turbo_tasks`]. /// /// When using [`Self::Recursive`], [`notify::Watcher`] will recursively track all contents @@ -101,7 +141,8 @@ impl TaskInput for DiskWatcherConfig { /// /// When using [`Self::NonRecursive`], we only track previously read files and their parent /// directories. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, TraceRawVcs, NonLocalValue, Encode, Decode)] +#[turbo_tasks::task_input] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, TraceRawVcs, Encode, Decode)] pub enum DiskWatcherRecursiveMode { Recursive, NonRecursive, @@ -147,15 +188,6 @@ impl DiskWatcherConfig { } } -/// How long to extend an invalidation batch by when receiving new events, before flushing. This -/// reduces invalidations if the same file or directory is modified many times. -/// -/// Linux watching is too fast, so we need a longer delay there to avoid reading wip files. -#[cfg(target_os = "linux")] -const BATCH_DELAY: Duration = Duration::from_millis(10); -#[cfg(not(target_os = "linux"))] -const BATCH_DELAY: Duration = Duration::from_millis(1); - pub(crate) struct DiskWatcher { state: State, config: DiskWatcherConfig, @@ -451,6 +483,10 @@ mod non_recursive_helpers { impl DiskWatcher { pub fn new(config: DiskWatcherConfig) -> Self { + assert!( + config.extended_batch_delay_duration >= config.batch_delay, + "extended_batch_delay_duration must be at least batch_delay" + ); Self { state: State::new_stopped(config.resolve_recursive_mode()), config, @@ -459,6 +495,13 @@ impl DiskWatcher { pub async fn start_watching(fs: Arc) -> Result<()> { let watcher: &Self = fs.watcher(); + + // read in the turbo-task context and before acquiring the lock + let extended_batch_delay_matcher = match watcher.config.extended_batch_delay_matcher { + Some(matcher) => Some(matcher.into_trait_ref().await?), + None => None, + }; + let state_guard = watcher.state.write().await; // bail out if we're already watching @@ -513,7 +556,7 @@ impl DiskWatcher { spawn_thread({ let fs = fs.clone(); - move || Self::watch_thread(fs, rx) + move || Self::watch_thread(fs, rx, extended_batch_delay_matcher) }); // Updating `self.state` is done last. If we panic while setting up the watcher, it'll @@ -551,24 +594,20 @@ impl DiskWatcher { fn watch_thread( fs: Arc, rx: Receiver>, + extended_batch_delay_matcher: Option>>, ) { let watcher: &Self = fs.watcher(); - let report_invalidation_reason = watcher.config.report_invalidation_reason; + let config = &watcher.config; + let report_invalidation_reason = config.report_invalidation_reason; let mut batch = BatchedInvalidations::new( watcher.state.recursive_mode(), - watcher.config.poll_interval.is_some(), + config.poll_interval.is_some(), ); + let mut schedule = BatchSchedule::new(config); 'outer: loop { - let mut deadline: Option = None; loop { - let event_result = match deadline { - None => rx.recv().map_err(|_| RecvTimeoutError::Disconnected), - Some(deadline) => { - rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) - } - }; - match event_result { + match schedule.recv_event(&rx, &*fs) { Ok(Ok(event)) => { // TODO: We might benefit from some user-facing diagnostics if it rescans // occur frequently (i.e. more than X times in Y minutes) @@ -613,13 +652,23 @@ impl DiskWatcher { // no need to process the rest of the batch as we just // invalidated everything batch.clear(); + schedule.reset(); break; } - // Only an event that contributes to the batch keeps it open for another - // `BATCH_DELAY`. + // Any event that contributes to the batch keeps it open for another + // `batch_delay`. A path matching `extended_batch_delay_matcher` (e.g. a + // package-manager install target) keeps it open for + // `extended_batch_delay_duration` instead. + let mut delay = config.batch_delay; + if let Some(matcher) = &extended_batch_delay_matcher + && event.paths.iter().any(|path| matcher.match_path(path)) + { + delay = delay.max(config.extended_batch_delay_duration); + } + if batch.add_event(event) { - deadline = Some(Instant::now() + BATCH_DELAY); + schedule.extend(delay); } } // Error raised by notify watcher itself @@ -629,16 +678,16 @@ impl DiskWatcher { let flags = InvalidationFlags::PATH_AND_CHILDREN | InvalidationFlags::PATH_AND_CHILDREN_DIR; if paths.is_empty() { - batch.mark(fs.root_path().into(), flags); + batch.mark(Box::from(fs.root_path()), flags); } else { for path in paths { batch.mark(path.into_boxed_path(), flags); } } - deadline = Some(Instant::now() + BATCH_DELAY); + schedule.extend(config.batch_delay); } Err(RecvTimeoutError::Timeout) => { - // The batch is complete: break out to invalidate the collected paths. + // the batch is complete: break out to invalidate the collected paths. break; } Err(RecvTimeoutError::Disconnected) => { @@ -1016,7 +1065,10 @@ impl InvalidationReasonKind for InvalidateRescanKind { #[cfg(test)] mod tests { - use std::{fs, time::SystemTime}; + use std::{ + fs, + time::{Instant, SystemTime}, + }; use rstest::rstest; use turbo_tasks::TurboTasks; @@ -1078,6 +1130,7 @@ mod tests { recursive_mode: Some(recursive_mode), poll_interval, report_invalidation_reason: true, + ..Default::default() }); let sub_dir = fs.root_path.join("sub"); let file_path = sub_dir.join("file.txt"); diff --git a/turbopack/crates/turbopack-nodejs/src/fs.rs b/turbopack/crates/turbopack-nodejs/src/fs.rs new file mode 100644 index 000000000000..dd9c6cee85d4 --- /dev/null +++ b/turbopack/crates/turbopack-nodejs/src/fs.rs @@ -0,0 +1,21 @@ +use std::{ + ffi::OsStr, + path::{Component, Path}, +}; + +use turbo_tasks_fs::DiskWatcherPathMatcher; + +/// Matches anything inside of a `node_modules` directory. +/// +/// Package managers churn `node_modules` heavily while the dev server is running. More aggressively +/// batching these may reduce system load during an installation. +#[turbo_tasks::value(shared)] +pub struct NodeModulesPathMatcher; + +#[turbo_tasks::value_impl] +impl DiskWatcherPathMatcher for NodeModulesPathMatcher { + fn match_path(&self, path: &Path) -> bool { + path.components() + .any(|component| component == Component::Normal(OsStr::new("node_modules"))) + } +} diff --git a/turbopack/crates/turbopack-nodejs/src/lib.rs b/turbopack/crates/turbopack-nodejs/src/lib.rs index 33660a5d5dd5..aee5487f7895 100644 --- a/turbopack/crates/turbopack-nodejs/src/lib.rs +++ b/turbopack/crates/turbopack-nodejs/src/lib.rs @@ -3,5 +3,6 @@ pub(crate) mod chunking_context; pub mod ecmascript; +pub mod fs; pub use chunking_context::{NodeJsChunkingContext, NodeJsChunkingContextBuilder};