From bded4033cfd090c357cc93e149b7ef1b54e89325 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 31 Aug 2026 08:43:12 +0000 Subject: [PATCH 1/3] Drop the per-file asset size limit from the static manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildAssetManifest is used only by deployStaticSite — the S3 arm, where assets go straight to storage by presigned PUT and never pass through our servers. There is no per-file ceiling to mirror: the python upload driver this arm replaces has none, so the 25 MiB check was a limit the other arm never had, and it turned any app with a large file in dist/ into a permanently failing publish once the flag caught it. Seen in prod at 10%: one app retried a 109 MB build artifact in dist/ every 1-3 minutes, 16 failures in an hour. The check was also the thing keeping whole-file reads safe, so hashing now streams: hashAssetFile updates the digest chunk by chunk, making peak memory one chunk instead of the file. The buffer form stays for callers that already hold the bytes, and a test pins the two to the same digest across a chunk boundary. Note the upload side still buffers a whole file per worker (uploadPresignedAsset -> readFile, MAX_UPLOAD_CONCURRENCY documents it), so peak memory there is concurrency x largest asset. Streaming that too needs a per-attempt stream, since a consumed stream cannot be replayed across the three upload retries — left as a follow-up rather than folded in here. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01G1E31AZvYBa86zpcYJFD1g --- packages/cli/src/core/site/manifest.ts | 31 ++++++++++++------- packages/cli/tests/core/site-manifest.spec.ts | 18 ++++++++--- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/packages/cli/src/core/site/manifest.ts b/packages/cli/src/core/site/manifest.ts index 4df01884..6602c9a4 100644 --- a/packages/cli/src/core/site/manifest.ts +++ b/packages/cli/src/core/site/manifest.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; -import { readFile, stat } from "node:fs/promises"; +import { createReadStream } from "node:fs"; +import { stat } from "node:fs/promises"; import { basename, join } from "node:path"; import { globby } from "globby"; import { InvalidInputError } from "@/core/errors.js"; @@ -9,7 +10,6 @@ import type { AssetManifestResult, } from "./schema.js"; -const MAX_ASSET_SIZE_BYTES = 25 * 1024 * 1024; // 25 MiB const MAX_ASSET_COUNT = 100_000; const ASSETS_IGNORE_FILE = ".assetsignore"; @@ -34,6 +34,23 @@ export function hashAsset(appId: string, content: Buffer): string { .slice(0, 32); } +/** + * {@link hashAsset} over a file, read in chunks. Peak memory is one chunk + * rather than the whole file, which is what lets an asset be arbitrarily large: + * assets go straight to storage by presigned PUT, so the only ceiling here is + * this process's heap. + */ +export async function hashAssetFile( + appId: string, + absolutePath: string, +): Promise { + const hash = createHash("sha256").update(Buffer.from(appId, "utf8")); + for await (const chunk of createReadStream(absolutePath)) { + hash.update(chunk); + } + return hash.digest("hex").slice(0, 32); +} + /** * Walk the assets directory and build the deployment asset manifest. Honors * `.assetsignore` at the assets root with full gitignore semantics, negation @@ -69,16 +86,8 @@ export async function buildAssetManifest( for (const relativePath of relativeFilePaths.sort()) { const absolutePath = join(assetsDir, ...relativePath.split("/")); - // Stat before read so an oversized file is never pulled into memory. const { size } = await stat(absolutePath); - if (size > MAX_ASSET_SIZE_BYTES) { - throw new InvalidInputError( - `Static asset "${relativePath}" is ${size} bytes, which exceeds the 25 MiB per-file limit.`, - ); - } - - const content = await readFile(absolutePath); - const hash = hashAsset(appId, content); + const hash = await hashAssetFile(appId, absolutePath); manifest[`/${relativePath}`] = { hash, size }; if (!filesByHash.has(hash)) { diff --git a/packages/cli/tests/core/site-manifest.spec.ts b/packages/cli/tests/core/site-manifest.spec.ts index 5974ed2b..601b93a4 100644 --- a/packages/cli/tests/core/site-manifest.spec.ts +++ b/packages/cli/tests/core/site-manifest.spec.ts @@ -166,14 +166,24 @@ describe("buildAssetManifest", () => { expect(Object.keys(manifest)).toEqual(["/index.html"]); }); - it("rejects files larger than 25 MiB with a per-file error", async () => { + it("has no per-file size limit — assets go straight to storage", async () => { const bigFile = join(assetsDir, "big.bin"); await writeFile(bigFile, ""); await truncate(bigFile, 25 * 1024 * 1024 + 1); - await expect(buildAssetManifest(assetsDir, "test-app-id")).rejects.toThrow( - /"big\.bin".*exceeds the 25 MiB per-file limit/, - ); + const { manifest } = await buildAssetManifest(assetsDir, "test-app-id"); + + expect(manifest["/big.bin"].size).toBe(25 * 1024 * 1024 + 1); + }); + + it("hashes a file in chunks to the same digest as the whole buffer", async () => { + // Larger than one read-stream chunk, so a chunk-boundary bug would show. + const content = Buffer.alloc(200 * 1024, "ab"); + await writeFile(join(assetsDir, "chunky.bin"), content); + + const { manifest } = await buildAssetManifest(assetsDir, "test-app-id"); + + expect(manifest["/chunky.bin"].hash).toBe(hashAsset("test-app-id", content)); }); it("dedupes identical files by hash in filesByHash", async () => { From 3b0d406f6cdc5395526a04c0cb344aa25daf0ad9 Mon Sep 17 00:00:00 2001 From: Netanel Gilad Date: Mon, 31 Aug 2026 12:33:48 +0300 Subject: [PATCH 2/3] Update manifest.ts --- packages/cli/src/core/site/manifest.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/core/site/manifest.ts b/packages/cli/src/core/site/manifest.ts index 6602c9a4..a1c58dd6 100644 --- a/packages/cli/src/core/site/manifest.ts +++ b/packages/cli/src/core/site/manifest.ts @@ -35,10 +35,8 @@ export function hashAsset(appId: string, content: Buffer): string { } /** - * {@link hashAsset} over a file, read in chunks. Peak memory is one chunk - * rather than the whole file, which is what lets an asset be arbitrarily large: - * assets go straight to storage by presigned PUT, so the only ceiling here is - * this process's heap. + * {@link hashAsset} over a file, read in chunks in order to support large files + * without reading them entirely to memory. */ export async function hashAssetFile( appId: string, From ad78df869be9cfa399e448f52cad2a43859c545a Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:42:49 +0000 Subject: [PATCH 3/3] fix: satisfy lint and knip after dropping the asset size limit - Format the chunked-hash assertion in site-manifest.spec.ts (Biome). - Stop exporting hashAssetFile; it is only used inside manifest.ts, and the unused export failed Knip. - Drop the now-stale "files over 25 MiB fail" sentence from docs/deployments.md. Co-authored-by: Netanel Gilad <3474905+netanelgilad@users.noreply.github.com> --- docs/deployments.md | 2 +- packages/cli/src/core/site/manifest.ts | 2 +- packages/cli/tests/core/site-manifest.spec.ts | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/deployments.md b/docs/deployments.md index 40466b2b..46de09a9 100644 --- a/docs/deployments.md +++ b/docs/deployments.md @@ -22,7 +22,7 @@ Deployments ship an app's built output addressed by the commit that produced it. `hash = first 32 hex chars of sha256(utf8(app_id) || raw file bytes)` — see `hashAsset()` in `src/core/site/manifest.ts`. The app-id salt is a cache-poisoning defense: a tenant can only produce hash collisions with its own files. -The output directory is walked with `globby` (`**/*`, dotfiles included, symlinks not followed). `.assetsignore` at the root is honored via globby's `ignoreFiles`, which parses it with the `ignore` package — the same library wrangler uses — so it gets real gitignore semantics: anchoring, directory patterns, `**`, literal braces/extglobs, and **negation** (`!.dev.vars.example` after `.dev.vars*`). Do not translate the patterns by hand, and do not pass globby's `ignore` option alongside `ignoreFiles`: globby globs for ignore files using that option, so it would then find none and silently apply no patterns at all. `.assetsignore` itself, `wrangler.json`, and `.dev.vars` are dropped from the results by name instead. Files over 25 MiB fail with a per-file error; total file count is capped at 100,000. Manifest keys are `/`-prefixed forward-slash paths. +The output directory is walked with `globby` (`**/*`, dotfiles included, symlinks not followed). `.assetsignore` at the root is honored via globby's `ignoreFiles`, which parses it with the `ignore` package — the same library wrangler uses — so it gets real gitignore semantics: anchoring, directory patterns, `**`, literal braces/extglobs, and **negation** (`!.dev.vars.example` after `.dev.vars*`). Do not translate the patterns by hand, and do not pass globby's `ignore` option alongside `ignoreFiles`: globby globs for ignore files using that option, so it would then find none and silently apply no patterns at all. `.assetsignore` itself, `wrangler.json`, and `.dev.vars` are dropped from the results by name instead. There is no per-file size limit — files are hashed by streaming them in chunks, so an asset is never held whole in memory at manifest time; total file count is capped at 100,000. Manifest keys are `/`-prefixed forward-slash paths. Content types are deliberately **not** derived client-side: the server decides each asset's Content-Type, signs it into the presigned URL, and the CLI echoes it verbatim — deriving our own value would 403 on any mapping difference. diff --git a/packages/cli/src/core/site/manifest.ts b/packages/cli/src/core/site/manifest.ts index a1c58dd6..ea0ece87 100644 --- a/packages/cli/src/core/site/manifest.ts +++ b/packages/cli/src/core/site/manifest.ts @@ -38,7 +38,7 @@ export function hashAsset(appId: string, content: Buffer): string { * {@link hashAsset} over a file, read in chunks in order to support large files * without reading them entirely to memory. */ -export async function hashAssetFile( +async function hashAssetFile( appId: string, absolutePath: string, ): Promise { diff --git a/packages/cli/tests/core/site-manifest.spec.ts b/packages/cli/tests/core/site-manifest.spec.ts index 601b93a4..f72c226f 100644 --- a/packages/cli/tests/core/site-manifest.spec.ts +++ b/packages/cli/tests/core/site-manifest.spec.ts @@ -183,7 +183,9 @@ describe("buildAssetManifest", () => { const { manifest } = await buildAssetManifest(assetsDir, "test-app-id"); - expect(manifest["/chunky.bin"].hash).toBe(hashAsset("test-app-id", content)); + expect(manifest["/chunky.bin"].hash).toBe( + hashAsset("test-app-id", content), + ); }); it("dedupes identical files by hash in filesByHash", async () => {