diff --git a/docs/deployments.md b/docs/deployments.md index 40466b2b..46de09a9 100644 --- a/docs/deployments.md +++ b/docs/deployments.md @@ -22,7 +22,7 @@ Deployments ship an app's built output addressed by the commit that produced it. `hash = first 32 hex chars of sha256(utf8(app_id) || raw file bytes)` — see `hashAsset()` in `src/core/site/manifest.ts`. The app-id salt is a cache-poisoning defense: a tenant can only produce hash collisions with its own files. -The output directory is walked with `globby` (`**/*`, dotfiles included, symlinks not followed). `.assetsignore` at the root is honored via globby's `ignoreFiles`, which parses it with the `ignore` package — the same library wrangler uses — so it gets real gitignore semantics: anchoring, directory patterns, `**`, literal braces/extglobs, and **negation** (`!.dev.vars.example` after `.dev.vars*`). Do not translate the patterns by hand, and do not pass globby's `ignore` option alongside `ignoreFiles`: globby globs for ignore files using that option, so it would then find none and silently apply no patterns at all. `.assetsignore` itself, `wrangler.json`, and `.dev.vars` are dropped from the results by name instead. Files over 25 MiB fail with a per-file error; total file count is capped at 100,000. Manifest keys are `/`-prefixed forward-slash paths. +The output directory is walked with `globby` (`**/*`, dotfiles included, symlinks not followed). `.assetsignore` at the root is honored via globby's `ignoreFiles`, which parses it with the `ignore` package — the same library wrangler uses — so it gets real gitignore semantics: anchoring, directory patterns, `**`, literal braces/extglobs, and **negation** (`!.dev.vars.example` after `.dev.vars*`). Do not translate the patterns by hand, and do not pass globby's `ignore` option alongside `ignoreFiles`: globby globs for ignore files using that option, so it would then find none and silently apply no patterns at all. `.assetsignore` itself, `wrangler.json`, and `.dev.vars` are dropped from the results by name instead. There is no per-file size limit — files are hashed by streaming them in chunks, so an asset is never held whole in memory at manifest time; total file count is capped at 100,000. Manifest keys are `/`-prefixed forward-slash paths. Content types are deliberately **not** derived client-side: the server decides each asset's Content-Type, signs it into the presigned URL, and the CLI echoes it verbatim — deriving our own value would 403 on any mapping difference. diff --git a/packages/cli/src/core/site/manifest.ts b/packages/cli/src/core/site/manifest.ts index 4df01884..ea0ece87 100644 --- a/packages/cli/src/core/site/manifest.ts +++ b/packages/cli/src/core/site/manifest.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; -import { readFile, stat } from "node:fs/promises"; +import { createReadStream } from "node:fs"; +import { stat } from "node:fs/promises"; import { basename, join } from "node:path"; import { globby } from "globby"; import { InvalidInputError } from "@/core/errors.js"; @@ -9,7 +10,6 @@ import type { AssetManifestResult, } from "./schema.js"; -const MAX_ASSET_SIZE_BYTES = 25 * 1024 * 1024; // 25 MiB const MAX_ASSET_COUNT = 100_000; const ASSETS_IGNORE_FILE = ".assetsignore"; @@ -34,6 +34,21 @@ export function hashAsset(appId: string, content: Buffer): string { .slice(0, 32); } +/** + * {@link hashAsset} over a file, read in chunks in order to support large files + * without reading them entirely to memory. + */ +async function hashAssetFile( + appId: string, + absolutePath: string, +): Promise { + const hash = createHash("sha256").update(Buffer.from(appId, "utf8")); + for await (const chunk of createReadStream(absolutePath)) { + hash.update(chunk); + } + return hash.digest("hex").slice(0, 32); +} + /** * Walk the assets directory and build the deployment asset manifest. Honors * `.assetsignore` at the assets root with full gitignore semantics, negation @@ -69,16 +84,8 @@ export async function buildAssetManifest( for (const relativePath of relativeFilePaths.sort()) { const absolutePath = join(assetsDir, ...relativePath.split("/")); - // Stat before read so an oversized file is never pulled into memory. const { size } = await stat(absolutePath); - if (size > MAX_ASSET_SIZE_BYTES) { - throw new InvalidInputError( - `Static asset "${relativePath}" is ${size} bytes, which exceeds the 25 MiB per-file limit.`, - ); - } - - const content = await readFile(absolutePath); - const hash = hashAsset(appId, content); + const hash = await hashAssetFile(appId, absolutePath); manifest[`/${relativePath}`] = { hash, size }; if (!filesByHash.has(hash)) { diff --git a/packages/cli/tests/core/site-manifest.spec.ts b/packages/cli/tests/core/site-manifest.spec.ts index 5974ed2b..f72c226f 100644 --- a/packages/cli/tests/core/site-manifest.spec.ts +++ b/packages/cli/tests/core/site-manifest.spec.ts @@ -166,13 +166,25 @@ describe("buildAssetManifest", () => { expect(Object.keys(manifest)).toEqual(["/index.html"]); }); - it("rejects files larger than 25 MiB with a per-file error", async () => { + it("has no per-file size limit — assets go straight to storage", async () => { const bigFile = join(assetsDir, "big.bin"); await writeFile(bigFile, ""); await truncate(bigFile, 25 * 1024 * 1024 + 1); - await expect(buildAssetManifest(assetsDir, "test-app-id")).rejects.toThrow( - /"big\.bin".*exceeds the 25 MiB per-file limit/, + const { manifest } = await buildAssetManifest(assetsDir, "test-app-id"); + + expect(manifest["/big.bin"].size).toBe(25 * 1024 * 1024 + 1); + }); + + it("hashes a file in chunks to the same digest as the whole buffer", async () => { + // Larger than one read-stream chunk, so a chunk-boundary bug would show. + const content = Buffer.alloc(200 * 1024, "ab"); + await writeFile(join(assetsDir, "chunky.bin"), content); + + const { manifest } = await buildAssetManifest(assetsDir, "test-app-id"); + + expect(manifest["/chunky.bin"].hash).toBe( + hashAsset("test-app-id", content), ); });