diff --git a/.github/scripts/check-prerender-cache.mjs b/.github/scripts/check-prerender-cache.mjs new file mode 100644 index 0000000..8ab81af --- /dev/null +++ b/.github/scripts/check-prerender-cache.mjs @@ -0,0 +1,325 @@ +#!/usr/bin/env node +/** + * Refuses to publish a docs Worker whose prerender cache is missing entries. + * + * ## The failure this exists to catch + * + * Every page on this site lives under `app/[lang]/`, so every page route is a + * DYNAMIC route prerendered through `generateStaticParams()`. OpenNext runs + * Next in minimal mode, where Next does not read prerendered HTML off a + * filesystem — it asks the configured incremental cache. `apps/docs` uses + * `staticAssetsIncrementalCache`, and `opennextjs-cloudflare deploy` copies + * `.open-next/cache` into `.open-next/assets` just before uploading. + * + * If those entries are absent, the Worker deploys with the cache CONFIGURED and + * EMPTY. Every lookup misses, `dynamicParams = false` refuses the on-demand + * render, Next raises `NoFallbackError`, and the request is answered by the + * prerendered `_not-found` route: the page 404s. Every page, every locale, + * every request — and the deploy step still exits 0, because the upload + * succeeded. That is exactly how 2026-09-04 went (#261), by a different route. + * + * A missing cache is not hypothetical. `.open-next/cache` is produced by a + * separate build invocation from the one a pull request runs, travels to the + * deploy job as a CI artifact, and is copied again by the deploy command. Three + * places it can be lost, none of which make any step go red on their own. + * + * ## What it asserts, and against what + * + * The population it checks is Next's own `prerender-manifest.json` — every + * route Next says it prerendered — not a number anyone wrote down. So a page + * added to the corpus is covered the day it is added, and this cannot pass by + * comparing a stale expectation to itself. + * + * Run it in the deploy job AFTER the artifact is downloaded and BEFORE the + * deploy step. A check that reports a bad artifact once it is already serving + * is a post-mortem, not a gate. + * + * ## Usage + * + * node .github/scripts/check-prerender-cache.mjs # apps/docs + * node .github/scripts/check-prerender-cache.mjs --dir PATH # elsewhere + * node .github/scripts/check-prerender-cache.mjs --self-test # the rules + * + * Exit 0 only when every prerendered route has a cache entry. + */ + +import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; + +/** Every rule this script enforces. The self-test asserts each one can fire. */ +const RULES = [ + 'no-open-next', + 'no-build-id', + 'no-prerender-manifest', + 'unreadable-prerender-manifest', + 'no-cache-dir', + 'no-routes', + 'missing-entries', +]; + +/** Where `.open-next` lives when nobody says otherwise. */ +const DEFAULT_DIR = 'apps/docs/.open-next'; + +/** + * The prerender manifest, as packaged inside the server function. + * + * Read from the bundle rather than from `apps/docs/.next/` on purpose: the + * deploy job downloads an artifact and never runs a Next build, so `.next/` is + * not there. Checking the copy that travels with the bundle is also the only + * way to be sure the manifest and the cache describe the same build. + */ +const MANIFEST_IN_BUNDLE = + 'server-functions/default/apps/docs/.next/prerender-manifest.json'; + +/** + * The cache file a route's entry is written to. + * + * `/` is stored as `index.cache`; every other route keeps its path. Mirrors + * `staticAssetsIncrementalCache.getAssetUrl`, which builds + * `CACHE_DIR/BUILD_ID/KEY.cache` from the same key. + */ +function entryPathFor(route, root) { + const key = route === '/' ? '/index' : route; + return join(root, `${key.slice(1)}.cache`); +} + +/** + * Judge one `.open-next` directory. Pure enough to drive from fixtures: it + * touches only the filesystem under `dir`. + */ +export function evaluate(dir) { + const findings = []; + const add = (rule, detail) => findings.push({ rule, detail }); + const measured = { dir }; + + if (!existsSync(dir)) { + add('no-open-next', `${dir} does not exist`); + return { findings, measured }; + } + + const buildIdPath = join(dir, 'assets/BUILD_ID'); + if (!existsSync(buildIdPath)) { + add('no-build-id', `${buildIdPath} does not exist`); + return { findings, measured }; + } + const buildId = readFileSync(buildIdPath, 'utf8').trim(); + measured.buildId = buildId; + + const manifestPath = join(dir, MANIFEST_IN_BUNDLE); + if (!existsSync(manifestPath)) { + add('no-prerender-manifest', `${manifestPath} does not exist`); + return { findings, measured }; + } + + let routes; + try { + routes = Object.keys(JSON.parse(readFileSync(manifestPath, 'utf8')).routes ?? {}); + } catch (error) { + add('unreadable-prerender-manifest', `${manifestPath}: ${error?.message ?? error}`); + return { findings, measured }; + } + measured.routes = routes.length; + + if (routes.length === 0) { + // A manifest with no prerendered routes would make every other rule below + // vacuous: zero routes, zero missing, a green that proves nothing. + add('no-routes', `${manifestPath} lists no prerendered routes`); + return { findings, measured }; + } + + const cacheRoot = join(dir, 'cache', buildId); + if (!existsSync(cacheRoot)) { + add('no-cache-dir', `${cacheRoot} does not exist — the Worker would 404 every page`); + measured.present = 0; + measured.missing = routes.length; + return { findings, measured }; + } + + const missing = routes.filter((route) => !existsSync(entryPathFor(route, cacheRoot))); + measured.present = routes.length - missing.length; + measured.missing = missing.length; + + if (missing.length > 0) { + const shown = missing.slice(0, 10).join(', '); + add( + 'missing-entries', + `${missing.length} of ${routes.length} prerendered route(s) have no cache entry ` + + `under ${cacheRoot} — they would 404 in production: ${shown}` + + (missing.length > 10 ? `, and ${missing.length - 10} more` : ''), + ); + } + + return { findings, measured }; +} + +/* ------------------------------------------------------------------ gate -- */ + +function gate(dir) { + const { findings, measured } = evaluate(dir); + + console.log(`prerender cache: ${measured.dir}`); + if (measured.buildId) console.log(` build id : ${measured.buildId}`); + if (measured.routes !== undefined) { + console.log( + ` routes : ${measured.routes} prerendered, ` + + `${measured.present ?? 0} servable, ${measured.missing ?? '?'} missing`, + ); + } + + if (findings.length === 0) { + console.log( + `\n✓ every one of the ${measured.routes} prerendered route(s) has a cache entry, ` + + 'so the Worker about to be uploaded can serve them', + ); + return 0; + } + + for (const f of findings) console.error(` [${f.rule}] ${f.detail}`); + console.error( + `\n✗ this bundle would publish a Worker that cannot serve its own pages — refusing to deploy`, + ); + return 1; +} + +/* ------------------------------------------------------------- self-test -- */ + +/** + * Fixtures, one per rule. The runner asserts that the set of rules a fixture + * trips is exactly the set declared here, AND that every rule in `RULES` has a + * fixture able to trip it — so weakening a rule fails this, which is what keeps + * a green from being decoration. + */ +const FIXTURES = [ + { + name: 'a complete bundle passes', + build: () => ({ routes: ['/', '/en/docs', '/zh-Hans/docs'], entries: ['/', '/en/docs', '/zh-Hans/docs'] }), + expect: [], + }, + { + name: 'no .open-next at all', + build: () => ({ absent: true }), + expect: ['no-open-next'], + }, + { + name: 'no BUILD_ID', + build: () => ({ routes: ['/en/docs'], entries: ['/en/docs'], noBuildId: true }), + expect: ['no-build-id'], + }, + { + name: 'no prerender manifest in the bundle', + build: () => ({ noManifest: true }), + expect: ['no-prerender-manifest'], + }, + { + name: 'a manifest that is not JSON', + build: () => ({ badManifest: true }), + expect: ['unreadable-prerender-manifest'], + }, + { + name: 'a manifest with no prerendered routes', + build: () => ({ routes: [], entries: [] }), + expect: ['no-routes'], + }, + { + name: 'the cache directory is missing entirely', + build: () => ({ routes: ['/en/docs'], entries: null }), + expect: ['no-cache-dir'], + }, + { + name: 'one route lost its cache entry', + build: () => ({ routes: ['/', '/en/docs', '/zh-Hans/docs'], entries: ['/', '/zh-Hans/docs'] }), + expect: ['missing-entries'], + }, +]; + +function materialise(spec) { + const dir = join(mkdtempSync(join(tmpdir(), 'os-prerender-cache-')), '.open-next'); + if (spec.absent) return dir; + + mkdirSync(join(dir, 'assets'), { recursive: true }); + const buildId = 'TESTBUILDID000000000'; + if (!spec.noBuildId) writeFileSync(join(dir, 'assets/BUILD_ID'), `${buildId}\n`); + + const manifestPath = join(dir, MANIFEST_IN_BUNDLE); + mkdirSync(dirname(manifestPath), { recursive: true }); + if (spec.badManifest) { + writeFileSync(manifestPath, 'not json {'); + return dir; + } + if (!spec.noManifest) { + const routes = Object.fromEntries((spec.routes ?? []).map((r) => [r, {}])); + writeFileSync(manifestPath, JSON.stringify({ routes })); + } else { + return dir; + } + + if (spec.entries === null) return dir; + const root = join(dir, 'cache', buildId); + for (const route of spec.entries ?? []) { + const p = entryPathFor(route, root); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, '{}'); + } + return dir; +} + +function selfTest() { + let failed = 0; + const fired = new Set(); + + for (const fixture of FIXTURES) { + const spec = fixture.build(); + const dir = materialise(spec); + let rules; + try { + rules = [...new Set(evaluate(dir).findings.map((f) => f.rule))].sort(); + } finally { + rmSync(resolve(dir, '..'), { recursive: true, force: true }); + } + for (const r of rules) fired.add(r); + + const want = [...fixture.expect].sort(); + if (rules.join('|') === want.join('|')) { + console.log(`✓ ${fixture.name.padEnd(42)} fired [${rules.join(' ') || 'nothing'}]`); + } else { + console.error( + `✗ ${fixture.name}\n expected [${want.join(' ') || 'nothing'}], got [${rules.join(' ') || 'nothing'}]`, + ); + failed += 1; + } + } + + const undemonstrated = RULES.filter((r) => !fired.has(r)); + if (undemonstrated.length) { + console.error( + `\n✗ ${undemonstrated.length} rule(s) have no fixture able to make them fire: ${undemonstrated.join(', ')}`, + ); + failed += 1; + } + + console.log(''); + if (failed) { + console.error(`✗ self-test: ${failed} failure(s)`); + return 1; + } + console.log( + `✓ self-test: ${FIXTURES.length} fixture(s) — all ${RULES.length} rules demonstrated able to fail`, + ); + return 0; +} + +/* -------------------------------------------------------------- dispatch -- */ + +const argv = process.argv.slice(2); +if (argv.includes('--self-test')) { + process.exit(selfTest()); +} else { + const at = argv.indexOf('--dir'); + const dir = at === -1 ? DEFAULT_DIR : argv[at + 1]; + if (!dir) { + console.error('--dir needs a path'); + process.exit(1); + } + process.exit(gate(resolve(dir))); +} diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index d40b182..488c383 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -117,6 +117,33 @@ jobs: name: ${{ inputs.artifact_name }} path: apps/docs/.open-next + # #261: the second defect that card turned up, and the one with no + # symptom until it is live. `apps/docs` serves every page from a + # prerendered entry in `.open-next/cache`, which + # `opennextjs-cloudflare deploy` copies into the uploaded assets. If those + # entries are absent, the Worker publishes with its cache CONFIGURED and + # EMPTY: every lookup misses, `dynamicParams = false` refuses the + # on-demand render, and every page 404s — while THIS JOB STILL EXITS 0, + # because the upload itself succeeded. `check-deploy-version.mjs` would + # not catch it either: a new version really would be serving. + # + # The cache is produced by a build invocation no pull request runs, + # travels here as an artifact, and is copied again by the deploy command. + # Three places to lose it, none of which turn a step red on their own. + # + # Placed BEFORE the deploy, deliberately: a check that reports a bad + # bundle once it is already serving is a post-mortem, not a gate. It + # asserts against Next's own `prerender-manifest.json` inside the bundle, + # so a page added to the corpus is covered the day it is added. + # + # `shell: bash` for the same reason as the verdict step below. + - name: Refuse a bundle that cannot serve its own pages + shell: bash + run: | + node .github/scripts/check-prerender-cache.mjs \ + --dir apps/docs/.open-next \ + | tee -a "$GITHUB_STEP_SUMMARY" + # Taken BEFORE the deploy and required to succeed: it is both half of the # "did anything actually change" comparison and the rollback target. A # deploy with no known-good version to fall back to is not one this diff --git a/apps/docs/app/[lang]/docs/[[...slug]]/page.tsx b/apps/docs/app/[lang]/docs/[[...slug]]/page.tsx index 3246679..8108eb5 100644 --- a/apps/docs/app/[lang]/docs/[[...slug]]/page.tsx +++ b/apps/docs/app/[lang]/docs/[[...slug]]/page.tsx @@ -240,7 +240,15 @@ export default async function Page(props: { const page = source.getPage(params.slug ?? [], params.lang); if (!page) notFound(); - const MDX = page.data.body; + // `async: true` on the docs collection makes the compiled body and toc load + // on demand instead of being statically imported, so they are awaited here. + // Frontmatter (`title`, `description`, `full`) stays eager. + // + // Under `dynamicParams = false` this await runs at BUILD time, in Node, on + // every one of the 1139 prerendered paths — the Worker serves the prerendered + // result out of the static-assets incremental cache and does not re-render. + const loaded = await page.data.load(); + const MDX = loaded.body; // Resolved once and handed to both controls, so they cannot drift apart and // so a third control added below inherits the locale-independent URL instead @@ -283,7 +291,7 @@ export default async function Page(props: { dangerouslySetInnerHTML={{ __html: jsonLdHtml(item) }} /> ))} - + {page.data.title} {page.data.description}
diff --git a/apps/docs/open-next.config.ts b/apps/docs/open-next.config.ts index 7a3d171..315674e 100644 --- a/apps/docs/open-next.config.ts +++ b/apps/docs/open-next.config.ts @@ -1,3 +1,72 @@ import { defineCloudflareConfig } from '@opennextjs/cloudflare'; +import staticAssetsIncrementalCache from '@opennextjs/cloudflare/overrides/incremental-cache/static-assets-incremental-cache'; -export default defineCloudflareConfig(); +/** + * ## Why this app needs an incremental cache, and what happens without one + * + * Every page this site serves lives under `app/[lang]/`, so every page route is + * a **dynamic** route as far as Next is concerned, prerendered through + * `generateStaticParams()` — 1139 paths in the prerender manifest. + * + * OpenNext runs Next in *minimal mode*: Next does not read prerendered HTML off + * a filesystem, it asks the configured incremental cache for it. And + * `defineCloudflareConfig()` with no arguments resolves `incrementalCache` to + * `"dummy"`, whose `get()` throws on every call by design. So the lookup for a + * prerendered page always misses. + * + * What happens next depends on one route-segment flag: + * + * - `dynamicParams` unset (Next's default, `true`): the miss falls through to + * an on-demand render. Pages are re-rendered on every request, wastefully + * but correctly, and the site works. + * - `dynamicParams = false`: Next refuses the on-demand render and raises + * `NoFallbackError`, which OpenNext answers with the prerendered + * `_not-found` route. **The page 404s. Every page, every locale.** + * + * `content/docs/` is never re-read at runtime and nothing here revalidates, so + * the on-demand render was pure waste — but it was load-bearing waste, and + * nothing recorded that. + * + * ## The outage this comment exists to stop repeating + * + * `export const dynamicParams = false` was added to `app/[lang]/layout.tsx`, + * `app/[lang]/docs/[[...slug]]/page.tsx` and `app/og/docs/[...slug]/route.tsx` + * on 2026-08-26, across five separate PRs about 404 semantics, each correct in + * itself. The last deploy Cloudflare accepted was 2026-08-25 — the Worker went + * over the 64 MiB limit that evening and every upload after it was rejected, so + * the flag sat on `main` for nine days without ever reaching production. + * + * On 2026-09-04 the size fix landed (PR #263, `async: true` in + * `source.config.ts`), the upload was accepted for the first time in nine days, + * and the site 404'd. `async: true` was blamed, reverted, and is innocent: + * measured on this tree under real workerd, `main` WITHOUT it fails the + * repository's own `smoke-docs.mjs` with 21 findings — `/`, `/en/docs`, + * `/docs/quickstart` and `/docs/build/interface/views` all 404 — and `main` + * WITH it fails with the same 21. The size fix published a defect that was + * already merged; it did not introduce one. + * + * ## Why the static-assets cache specifically + * + * It reads prerendered entries straight out of the Workers static assets this + * Worker already binds as `ASSETS` (under `cdn-cgi/_next_cache`, a prefix only + * the Worker can reach). No R2 bucket, no KV namespace, no new binding, no + * spend — `opennextjs-cloudflare deploy` copies `.open-next/cache` into + * `.open-next/assets` before uploading, and `preview` does the same locally. + * + * Its one documented restriction — read-only, for apps that "do NOT want + * revalidation and ONLY want to serve prerendered data" — is exactly this app: + * `revalidate = false` on every route handler, no ISR anywhere, no on-demand + * revalidation, and content that only changes when the site is rebuilt. + * + * ⚠️ If a future page ever needs real revalidation, this override is the wrong + * one and its `set()` will log an error rather than cache anything. Move to + * `r2IncrementalCache` then — do not remove this line and go back to no cache + * at all, because that is the configuration that 404s every page. + * + * Measured after this change, under real workerd (`opennextjs-cloudflare + * preview`): `smoke-docs.mjs` passes all four pages with its negative control + * still going red. Before it: 21 findings. + */ +export default defineCloudflareConfig({ + incrementalCache: staticAssetsIncrementalCache, +}); diff --git a/apps/docs/source.config.ts b/apps/docs/source.config.ts index a46822e..b57875b 100644 --- a/apps/docs/source.config.ts +++ b/apps/docs/source.config.ts @@ -5,6 +5,39 @@ import path from 'node:path'; export const docs = defineDocs({ dir: path.resolve(process.cwd(), '../../content/docs'), docs: { + /** + * Load each page's compiled body on demand instead of statically importing + * all of them into every server entrypoint. + * + * Without this, `fumadocs-mdx:collections/server` eagerly imports all 397 + * `.mdx` files, so every route that touches `source` — the docs page, but + * also `/llms.txt`, `/llms-full.txt`, `/llms.mdx/*`, `/og/*`, `/api/search` + * and `/sitemap.xml` — pulls the entire corpus into its own chunk, and the + * bundler then inlined the whole set five times over into one Worker. + * 2.50 MiB of authored MDX became a ~100 MiB `handler.mjs`, and Cloudflare + * rejects any Worker over 64 MiB uncompressed (`code: 10027`). + * + * The multiplier, not the corpus, is the problem: one probe sentence from a + * single English page appeared 15 times in the bundle before this flag and 6 + * times after. + * + * The cost is that `page.data.body` and `page.data.toc` become + * `page.data.load()`. Frontmatter stays eager, so `title`, `description`, + * `seoTitle` and `full` are unaffected, and `getText('processed')` — what + * the llms.txt routes call — is still a method on the entry. + * + * ## This flag did NOT break the site on 2026-09-04, and the record matters + * + * It shipped once (PR #263), the upload was accepted, the site 404'd, and it + * was reverted (PR #268) on the reasonable assumption that the new thing was + * the cause. It was not. Every page route on `main` was already unservable + * for an unrelated reason — see the long comment in `open-next.config.ts` — + * and had been since 2026-08-26, invisibly, because no deploy had been + * accepted since 2026-08-25 to publish it. Measured on this tree: base + * `main` with this flag ABSENT 404s on `/`, `/en/docs`, `/docs/quickstart` + * and `/docs/build/interface/views` under real workerd, identically. + */ + async: true, schema: pageSchema.extend({ /** * Optional SEO title: what the `` tag should say, when that is not diff --git a/tools/ci-scripts/run-self-tests.mjs b/tools/ci-scripts/run-self-tests.mjs index ee0d4cc..dad69c4 100644 --- a/tools/ci-scripts/run-self-tests.mjs +++ b/tools/ci-scripts/run-self-tests.mjs @@ -69,6 +69,13 @@ * Cloudflare, so on `main` they may not run for real either. What keeps them * from being decoration in the meantime is exactly this: their fixtures assert * that every rule they enforce still produces a red, and that runs on every PR. + * + * `check-prerender-cache.mjs` joins them on identical footing (#261). Its gate + * mode reads a `.open-next` bundle that only exists in the deploy job, so on a + * pull request it can never run for real; its fixtures are what prove it can + * still refuse a bundle whose prerender cache is incomplete — the bundle shape + * that publishes a Worker returning 404 for every page while the deploy step + * exits 0. */ import { readFileSync, readdirSync, existsSync } from 'node:fs'; @@ -88,6 +95,7 @@ const SELF_TESTED = [ 'check-locale-surface.mjs', 'check-deploy-version.mjs', 'smoke-docs.mjs', + 'check-prerender-cache.mjs', ]; /**