From 4e77b00b325f3fa0dbff3502a5e17938c9793513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Guldmund?= Date: Tue, 4 Aug 2026 19:46:48 +0200 Subject: [PATCH 01/10] Prune foreign-platform onnxruntime-node binaries from packaged CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit onnxruntime-node (a transitive dependency of @huggingface/transformers, used for local embeddings) bundles prebuilt native binaries for every platform/arch under bin/napi-v/// inside a single package, rather than splitting them into per-target optionalDependencies. Its package.json declares os: [win32, darwin, linux] with no cpu constraint, so the existing whole-package pruning in prunePackagedTreeDirectory never touches it — every single-platform CLI release tarball ships all 6 platform/arch binary sets (~177MB of the ~208MB onnxruntime-node bin/ tree is unused on any given install). Extend the packaging-time tree sanitizer to recognize known bundle-all-platforms directory layouts and prune non-matching platform/arch subdirectories in place, in addition to the existing whole-package os/cpu constraint check. --- .../pipeline/release/lib/binary-release.mjs | 44 ++++++++- ...ry-release.onnxruntime-node-prune.test.mjs | 89 +++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 scripts/pipeline/release/lib/binary-release.onnxruntime-node-prune.test.mjs diff --git a/scripts/pipeline/release/lib/binary-release.mjs b/scripts/pipeline/release/lib/binary-release.mjs index b7174402ba..632a8fb4e7 100644 --- a/scripts/pipeline/release/lib/binary-release.mjs +++ b/scripts/pipeline/release/lib/binary-release.mjs @@ -553,7 +553,7 @@ function packageDirMatchesTarget(packageJson, target) { return true; } -async function sanitizePackagedNodeModulesTree(params) { +export async function sanitizePackagedNodeModulesTree(params) { const stageDir = String(params?.stageDir ?? '').trim(); if (!stageDir) return; @@ -568,6 +568,43 @@ function isNestedNodeModulesBinDir(path) { return path.includes('/node_modules/.bin') || path.includes('\\node_modules\\.bin'); } +// Some packages (e.g. onnxruntime-node) bundle prebuilt native binaries for every +// supported platform/arch inside their own tree instead of splitting them into +// per-target optionalDependencies, so package.json os/cpu constraints alone can't +// prune them. Match known bundle root directories (whose children are "" +// dirs, each containing "" dirs) and drop the ones that don't match the target. +const BUNDLED_NATIVE_PLATFORM_ROOT_DIR_PATTERNS = [ + // onnxruntime-node: bin/napi-v///... + /\/node_modules\/onnxruntime-node\/bin\/napi-v\d+$/, +]; + +function isBundledNativePlatformRootDir(path) { + const normalized = path.replaceAll('\\', '/'); + return BUNDLED_NATIVE_PLATFORM_ROOT_DIR_PATTERNS.some((pattern) => pattern.test(normalized)); +} + +async function pruneBundledNativePlatformRootDir(params) { + const targetNodePlatform = resolveTargetNodePlatform(params.target); + const targetArch = String(params.target?.arch ?? '').trim().toLowerCase(); + const entries = await readdir(params.directoryPath, { withFileTypes: true }).catch(() => []); + + for (const entry of entries) { + const childPath = join(params.directoryPath, entry.name); + + if (!entry.isDirectory() || entry.name.toLowerCase() !== targetNodePlatform) { + await rm(childPath, { recursive: true, force: true }); + continue; + } + + const archEntries = await readdir(childPath, { withFileTypes: true }).catch(() => []); + for (const archEntry of archEntries) { + if (archEntry.isDirectory() && archEntry.name.toLowerCase() !== targetArch) { + await rm(join(childPath, archEntry.name), { recursive: true, force: true }); + } + } + } +} + async function prunePackagedTreeDirectory(params) { const entries = await readdir(params.directoryPath, { withFileTypes: true }).catch(() => []); @@ -595,6 +632,11 @@ async function prunePackagedTreeDirectory(params) { } } + if (isBundledNativePlatformRootDir(childPath)) { + await pruneBundledNativePlatformRootDir({ directoryPath: childPath, target: params.target }); + continue; + } + await prunePackagedTreeDirectory({ directoryPath: childPath, target: params.target, diff --git a/scripts/pipeline/release/lib/binary-release.onnxruntime-node-prune.test.mjs b/scripts/pipeline/release/lib/binary-release.onnxruntime-node-prune.test.mjs new file mode 100644 index 0000000000..6b5f2a488f --- /dev/null +++ b/scripts/pipeline/release/lib/binary-release.onnxruntime-node-prune.test.mjs @@ -0,0 +1,89 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, writeFile, rm, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { sanitizePackagedNodeModulesTree } from './binary-release.mjs'; + +// onnxruntime-node bundles prebuilt native binaries for every platform/arch inside +// its own package tree (bin/napi-v///...) rather than splitting +// them into per-target optionalDependencies. A single-platform CLI release tarball +// should only ship the binaries for its own target. +async function buildFakeOnnxruntimeNodeTree(stageDir) { + const pkgDir = join(stageDir, 'node_modules', 'onnxruntime-node'); + await writeFile( + join(pkgDir, 'package.json'), + JSON.stringify({ name: 'onnxruntime-node', os: ['win32', 'darwin', 'linux'] }), + 'utf-8', + ).catch(async (error) => { + if (error.code !== 'ENOENT') throw error; + await mkdir(pkgDir, { recursive: true }); + await writeFile( + join(pkgDir, 'package.json'), + JSON.stringify({ name: 'onnxruntime-node', os: ['win32', 'darwin', 'linux'] }), + 'utf-8', + ); + }); + + const platformArchPairs = [ + ['linux', 'x64'], + ['linux', 'arm64'], + ['darwin', 'x64'], + ['darwin', 'arm64'], + ['win32', 'x64'], + ['win32', 'arm64'], + ]; + for (const [platform, arch] of platformArchPairs) { + const dir = join(pkgDir, 'bin', 'napi-v3', platform, arch); + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'onnxruntime_binding.node'), 'fake-binary', 'utf-8'); + } + + return pkgDir; +} + +test('sanitizePackagedNodeModulesTree prunes onnxruntime-node bundled binaries to the packaging target only', async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-onnx-prune-')); + + try { + const pkgDir = await buildFakeOnnxruntimeNodeTree(stageDir); + + await sanitizePackagedNodeModulesTree({ + stageDir, + target: { os: 'darwin', arch: 'arm64' }, + }); + + const napiDir = join(pkgDir, 'bin', 'napi-v3'); + const remainingPlatforms = await readdir(napiDir); + assert.deepEqual(remainingPlatforms.sort(), ['darwin']); + + const remainingArches = await readdir(join(napiDir, 'darwin')); + assert.deepEqual(remainingArches, ['arm64']); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } +}); + +test('sanitizePackagedNodeModulesTree keeps only the matching arch for windows targets', async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-onnx-prune-win-')); + + try { + const pkgDir = await buildFakeOnnxruntimeNodeTree(stageDir); + + await sanitizePackagedNodeModulesTree({ + stageDir, + // buildBinaryTarget.mjs uses os: 'windows'; resolveTargetNodePlatform maps it to 'win32'. + target: { os: 'windows', arch: 'x64' }, + }); + + const napiDir = join(pkgDir, 'bin', 'napi-v3'); + const remainingPlatforms = await readdir(napiDir); + assert.deepEqual(remainingPlatforms.sort(), ['win32']); + + const remainingArches = await readdir(join(napiDir, 'win32')); + assert.deepEqual(remainingArches, ['x64']); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } +}); From 213d0a12b926f18fc810eb42b2569ba1d6852231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Guldmund?= Date: Tue, 4 Aug 2026 20:18:38 +0200 Subject: [PATCH 02/10] Prune more packaged-CLI bloat: bare-fs/url/os, ps-list .exe, dup packages, package-dist .cjs Extends the platform-binary pruning added earlier on this branch with four more independently-verified size reductions in the packaged CLI payload: - bare-fs/bare-url/bare-os (nested under archiver -> tar-stream) bundle prebuilt natives for every platform/arch with no os/cpu package.json gating, same anti-pattern as node-pty. Added to the existing flat-layout prune pattern list. - ps-list ships Windows-only fastlist-*.exe helpers unconditionally. prunePackagedTreeDirectory previously only pruned directories; added a file-level branch to strip these on non-Windows targets. - tar (inside onnxruntime-node's own node_modules) and @modelcontextprotocol/sdk (inside @anthropic-ai/claude-agent-sdk's own node_modules) are exact, byte-identical duplicates of copies already vendored at the payload's top-level node_modules. The independent per-package vendoring entry points in workspaces/index.ts don't share a visited set with each other, so a dependency already vendored at the top level gets vendored again inside a nested package's tree. Deleted the nested duplicates outright at packaging time; ordinary upward-walking module resolution finds the top-level copy. - package-dist/*.cjs (139 files) is the dual-format npm-publish build output copied verbatim into the Homebrew/binary payload, but only the .mjs half is ever loaded by the compiled binary's entrypoints -- the .cjs half exists solely for the separately-published npm package's require() consumers, a different distribution channel. Pruned for the binary-release path only; the npm-publish path is untouched. Also hardens workspaces/index.ts's vendorRuntimeDependencyTree with a name@version dedup map so future builds don't reintroduce the @modelcontextprotocol/sdk duplicate in the first place (symlinks to the first-vendored copy instead of copying again). Measured on the real installed darwin-arm64 v0.2.10-dev.53 payload: 991MB -> 676MB installed, 222.5MB -> 129.7MB compressed tarball. --- .../cli-common/src/workspaces/index.test.ts | 110 +++++++++++++++- packages/cli-common/src/workspaces/index.ts | 32 ++++- ...release.bare-fs-and-ps-list-prune.test.mjs | 102 +++++++++++++++ ....duplicate-vendored-package-prune.test.mjs | 69 ++++++++++ .../pipeline/release/lib/binary-release.mjs | 106 +++++++++++++-- ...ry-release.onnxruntime-node-prune.test.mjs | 123 ++++++++++-------- ...ry-release.package-dist-cjs-prune.test.mjs | 50 +++++++ 7 files changed, 520 insertions(+), 72 deletions(-) create mode 100644 scripts/pipeline/release/lib/binary-release.bare-fs-and-ps-list-prune.test.mjs create mode 100644 scripts/pipeline/release/lib/binary-release.duplicate-vendored-package-prune.test.mjs create mode 100644 scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs diff --git a/packages/cli-common/src/workspaces/index.test.ts b/packages/cli-common/src/workspaces/index.test.ts index 0cbd69dfc9..1040c543b7 100644 --- a/packages/cli-common/src/workspaces/index.test.ts +++ b/packages/cli-common/src/workspaces/index.test.ts @@ -1,7 +1,19 @@ import { atomicReplaceDirSync, bundleWorkspacePackage, copyDirSafeSync } from './index'; -import { cpSync, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { + cpSync, + existsSync, + lstatSync, + mkdtempSync, + mkdirSync, + readFileSync, + readdirSync, + readlinkSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; -import { join, resolve } from 'node:path'; +import { basename, dirname, join, resolve } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; describe('bundleWorkspacePackage', () => { @@ -162,6 +174,100 @@ describe('bundleWorkspacePackage', () => { ); expect(readFileSync(resolve(destPackageDir, 'workspaceLockLease.mjs'), 'utf8')).toContain('canonical'); }); + + it('dedupes an identical name@version runtime dependency vendored twice via a diamond dependency, symlinking the second copy instead of recopying', async () => { + rootDir = mkdtempSync(join(tmpdir(), 'happier-cli-common-bundle-workspace-')); + + const workspaceModule = await import('./index'); + const bundleWorkspacePackageWithRuntimeDependencies = + (workspaceModule as Record).bundleWorkspacePackageWithRuntimeDependencies; + expect(bundleWorkspacePackageWithRuntimeDependencies).toBeTypeOf('function'); + + // Diamond shape mirroring @modelcontextprotocol/sdk being a direct dep of apps/cli AND a + // transitive dep of @anthropic-ai/claude-agent-sdk (also a direct dep of apps/cli): the shared + // dependency is resolved and vendored twice within the same vendorRuntimeDependencyTree walk. + const srcPackageDir = resolve(rootDir, 'packages/agents'); + const srcDistDir = resolve(srcPackageDir, 'dist'); + const sharedDepDir = resolve(srcPackageDir, 'node_modules/shared-dep'); + const consumerDepDir = resolve(srcPackageDir, 'node_modules/consumer-dep'); + const nestedSharedDepDir = resolve(consumerDepDir, 'node_modules/shared-dep'); + + mkdirSync(srcDistDir, { recursive: true }); + mkdirSync(sharedDepDir, { recursive: true }); + mkdirSync(consumerDepDir, { recursive: true }); + mkdirSync(nestedSharedDepDir, { recursive: true }); + + writeFileSync( + resolve(srcPackageDir, 'package.json'), + JSON.stringify( + { + name: '@happier-dev/agents', + version: '0.0.0', + type: 'module', + exports: { '.': { default: './dist/index.js' } }, + dependencies: { 'shared-dep': '1.2.3', 'consumer-dep': '1.0.0' }, + }, + null, + 2, + ), + ); + writeFileSync(resolve(srcDistDir, 'index.js'), 'export {};'); + + writeFileSync( + resolve(sharedDepDir, 'package.json'), + JSON.stringify({ name: 'shared-dep', version: '1.2.3', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(sharedDepDir, 'index.js'), 'module.exports = "shared";\n'); + + writeFileSync( + resolve(consumerDepDir, 'package.json'), + JSON.stringify({ name: 'consumer-dep', version: '1.0.0', dependencies: { 'shared-dep': '1.2.3' } }, null, 2), + ); + writeFileSync(resolve(consumerDepDir, 'index.js'), 'module.exports = "consumer";\n'); + + // The nested copy is resolvable independently (npm-style: consumer-dep's own node_modules) + // and is byte-identical to the top-level copy, matching the real-world duplication shape. + writeFileSync( + resolve(nestedSharedDepDir, 'package.json'), + JSON.stringify({ name: 'shared-dep', version: '1.2.3', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(nestedSharedDepDir, 'index.js'), 'module.exports = "shared";\n'); + + const destPackageDir = resolve(rootDir, 'apps/cli/node_modules/@happier-dev/agents'); + + (bundleWorkspacePackageWithRuntimeDependencies as (params: { + packageName: string; + srcDir: string; + destDir: string; + }) => void)({ + packageName: '@happier-dev/agents', + srcDir: srcPackageDir, + destDir: destPackageDir, + }); + + const vendoredSharedDepDir = resolve(destPackageDir, 'node_modules/shared-dep'); + const vendoredNestedSharedDepDir = resolve( + destPackageDir, + 'node_modules/consumer-dep/node_modules/shared-dep', + ); + + // First occurrence is vendored normally as a real directory. + expect(lstatSync(vendoredSharedDepDir).isSymbolicLink()).toBe(false); + expect(readFileSync(resolve(vendoredSharedDepDir, 'index.js'), 'utf8')).toBe('module.exports = "shared";\n'); + + // Second occurrence (same name@version) is a symlink, not a full recopy. The link target is + // captured at build time inside the atomically-built staging tree (before the whole + // node_modules dir is renamed into its final place), so assert on the relationship (same + // basename as the surviving vendored copy) and on content equivalence rather than the final + // absolute path. + expect(lstatSync(vendoredNestedSharedDepDir).isSymbolicLink()).toBe(true); + const linkTarget = readlinkSync(vendoredNestedSharedDepDir); + const resolvedLinkTarget = resolve(dirname(vendoredNestedSharedDepDir), linkTarget); + expect(basename(resolvedLinkTarget)).toBe('shared-dep'); + expect(readFileSync(resolve(vendoredNestedSharedDepDir, 'index.js'), 'utf8')).toBe( + 'module.exports = "shared";\n', + ); + }); }); describe('atomicReplaceDirSync', () => { diff --git a/packages/cli-common/src/workspaces/index.ts b/packages/cli-common/src/workspaces/index.ts index 91e95f753d..19962e9ca8 100644 --- a/packages/cli-common/src/workspaces/index.ts +++ b/packages/cli-common/src/workspaces/index.ts @@ -1,6 +1,6 @@ -import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { createRequire } from 'node:module'; -import { basename, dirname, resolve } from 'node:path'; +import { basename, dirname, relative, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; export function findRepoRoot(startDir: string): string { @@ -545,16 +545,18 @@ function vendorRuntimeDependencyTree(params: Readonly<{ resolveFromPackageJsonPath?: string; destNodeModulesDir: string; visited?: Set; + dedupeByNameVersion?: Map; }>): void { const pkgJson = readJson(params.packageJsonPath); const roots = collectExternalRuntimeDepNamesFromPackageJson(pkgJson); const require = createRequire(pathToFileURL(params.resolveFromPackageJsonPath ?? params.packageJsonPath).href); const visited = params.visited ?? new Set(); + const dedupeByNameVersion = params.dedupeByNameVersion ?? new Map(); mkdirSync(params.destNodeModulesDir, { recursive: true }); for (const dep of roots) { - let resolved: Readonly<{ packageDir: string; packageJsonPath: string }>; + let resolved: Readonly<{ packageDir: string; packageJsonPath: string; packageJson: any }>; try { resolved = resolveInstalledPackage({ require, packageName: dep.name }); } catch (error) { @@ -566,13 +568,37 @@ function vendorRuntimeDependencyTree(params: Readonly<{ if (visited.has(depDestDir)) continue; visited.add(depDestDir); + const version = typeof resolved.packageJson?.version === 'string' ? resolved.packageJson.version : undefined; + const dedupeKey = version ? `${dep.name}@${version}` : undefined; + const existingDedupePath = dedupeKey ? dedupeByNameVersion.get(dedupeKey) : undefined; + + if (existingDedupePath) { + // Already vendored elsewhere in this same tree at the identical name+version. Symlink to + // the surviving copy instead of copying again -- behaviorally identical for any consumer + // (including one that reads from disk directly), since the two source trees are the same + // resolved version. Skip recursing into its subtree: those deps were already vendored under + // the first copy. + // + // Use a relative link target: both paths currently live inside the same not-yet-renamed + // atomic-build staging tree, and the whole tree (staging dir and all its contents, symlink + // included) gets renamed as one unit into its final place. An absolute target captured now + // would point at the staging path and dangle once that rename happens; a relative target + // survives the rename because the relationship between the two paths doesn't change. + rmDirSafeSync(depDestDir); + mkdirSync(dirname(depDestDir), { recursive: true }); + symlinkSync(relative(dirname(depDestDir), existingDedupePath), depDestDir, 'dir'); + continue; + } + resetDir(depDestDir); copyDirSafeSync(resolved.packageDir, depDestDir, { dereference: true }); + if (dedupeKey) dedupeByNameVersion.set(dedupeKey, depDestDir); vendorRuntimeDependencyTree({ packageJsonPath: resolved.packageJsonPath, destNodeModulesDir: resolve(depDestDir, 'node_modules'), visited, + dedupeByNameVersion, }); } } diff --git a/scripts/pipeline/release/lib/binary-release.bare-fs-and-ps-list-prune.test.mjs b/scripts/pipeline/release/lib/binary-release.bare-fs-and-ps-list-prune.test.mjs new file mode 100644 index 0000000000..65ac805937 --- /dev/null +++ b/scripts/pipeline/release/lib/binary-release.bare-fs-and-ps-list-prune.test.mjs @@ -0,0 +1,102 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, writeFile, rm, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { sanitizePackagedNodeModulesTree } from './binary-release.mjs'; + +// bare-fs / bare-url / bare-os bundle prebuilt native binaries for every +// platform/arch inside their own package tree (flat /-/ +// layout, same as node-pty), so package.json os/cpu constraints alone can't +// prune them. +const CLI_TARGETS = [ + { os: 'linux', arch: 'x64' }, + { os: 'linux', arch: 'arm64' }, + { os: 'darwin', arch: 'x64' }, + { os: 'darwin', arch: 'arm64' }, + { os: 'windows', arch: 'x64' }, +]; + +const NESTED_PLATFORM_ARCH_PAIRS = [ + ['linux', 'x64'], + ['linux', 'arm64'], + ['darwin', 'x64'], + ['darwin', 'arm64'], + ['win32', 'x64'], + ['win32', 'arm64'], +]; + +async function buildFakeFlatPrebuildsTree(stageDir, packageName) { + const pkgDir = join(stageDir, 'node_modules', ...packageName.split('/')); + await mkdir(pkgDir, { recursive: true }); + await writeFile(join(pkgDir, 'package.json'), JSON.stringify({ name: packageName }), 'utf-8'); + + const prebuildsDir = join(pkgDir, 'prebuilds'); + for (const [platform, arch] of NESTED_PLATFORM_ARCH_PAIRS) { + const dir = join(prebuildsDir, `${platform}-${arch}`); + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'binding.node'), 'fake-binary', 'utf-8'); + } + + return pkgDir; +} + +async function buildFakePsListTree(stageDir, { nested = false } = {}) { + const pkgDir = nested + ? join(stageDir, 'node_modules', '@types', 'ps-list', 'node_modules', 'ps-list') + : join(stageDir, 'node_modules', 'ps-list'); + const vendorDir = join(pkgDir, 'vendor'); + await mkdir(vendorDir, { recursive: true }); + await writeFile(join(pkgDir, 'package.json'), JSON.stringify({ name: 'ps-list' }), 'utf-8'); + await writeFile(join(vendorDir, 'fastlist-0.3.0-x64.exe'), 'fake-exe', 'utf-8'); + await writeFile(join(vendorDir, 'fastlist-0.3.0-x86.exe'), 'fake-exe', 'utf-8'); + await writeFile(join(vendorDir, 'README.md'), 'not an exe', 'utf-8'); + return { pkgDir, vendorDir }; +} + +for (const target of CLI_TARGETS) { + const expectedNodePlatform = target.os === 'windows' ? 'win32' : target.os; + + for (const packageName of ['bare-fs', 'bare-url', 'bare-os']) { + test(`sanitizePackagedNodeModulesTree prunes ${packageName} (flat layout) to ${target.os}/${target.arch} only`, async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-bare-prune-')); + try { + const pkgDir = await buildFakeFlatPrebuildsTree(stageDir, packageName); + + await sanitizePackagedNodeModulesTree({ stageDir, target }); + + const remaining = await readdir(join(pkgDir, 'prebuilds')); + assert.deepEqual(remaining, [`${expectedNodePlatform}-${target.arch}`]); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } + }); + } + + test(`sanitizePackagedNodeModulesTree strips ps-list Windows-only .exe vendor files for ${target.os}/${target.arch}`, async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-ps-list-prune-')); + try { + const { vendorDir } = await buildFakePsListTree(stageDir); + const { vendorDir: nestedVendorDir } = await buildFakePsListTree(stageDir, { nested: true }); + + await sanitizePackagedNodeModulesTree({ stageDir, target }); + + const remaining = await readdir(vendorDir); + const nestedRemaining = await readdir(nestedVendorDir); + + if (target.os === 'windows') { + assert.deepEqual(remaining.sort(), ['README.md', 'fastlist-0.3.0-x64.exe', 'fastlist-0.3.0-x86.exe']); + assert.deepEqual( + nestedRemaining.sort(), + ['README.md', 'fastlist-0.3.0-x64.exe', 'fastlist-0.3.0-x86.exe'], + ); + } else { + assert.deepEqual(remaining, ['README.md']); + assert.deepEqual(nestedRemaining, ['README.md']); + } + } finally { + await rm(stageDir, { recursive: true, force: true }); + } + }); +} diff --git a/scripts/pipeline/release/lib/binary-release.duplicate-vendored-package-prune.test.mjs b/scripts/pipeline/release/lib/binary-release.duplicate-vendored-package-prune.test.mjs new file mode 100644 index 0000000000..2792ffacea --- /dev/null +++ b/scripts/pipeline/release/lib/binary-release.duplicate-vendored-package-prune.test.mjs @@ -0,0 +1,69 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, writeFile, rm, access } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { sanitizePackagedNodeModulesTree } from './binary-release.mjs'; + +// Independent per-package vendoring entry points don't share a "visited" set with each other, +// so a transitive dependency already present at the payload's top-level node_modules can also +// get copied again into a nested package's own node_modules. These are confirmed exact +// duplicates that should be pruned outright, relying on ordinary upward-walking Node/Bun module +// resolution to find the top-level copy once the nested duplicate is removed. +const TARGET = { os: 'darwin', arch: 'arm64' }; + +async function pathExists(path) { + return access(path).then( + () => true, + () => false, + ); +} + +async function buildFakeDuplicateTree(stageDir, { topLevelPackagePath, nestedDuplicatePath }) { + const topLevelDir = join(stageDir, 'node_modules', ...topLevelPackagePath.split('/')); + await mkdir(topLevelDir, { recursive: true }); + await writeFile(join(topLevelDir, 'package.json'), JSON.stringify({ name: 'top-level-copy' }), 'utf-8'); + await writeFile(join(topLevelDir, 'index.js'), 'module.exports = {};', 'utf-8'); + + const nestedDir = join(stageDir, 'node_modules', ...nestedDuplicatePath.split('/')); + await mkdir(nestedDir, { recursive: true }); + await writeFile(join(nestedDir, 'package.json'), JSON.stringify({ name: 'nested-duplicate-copy' }), 'utf-8'); + await writeFile(join(nestedDir, 'index.js'), 'module.exports = {};', 'utf-8'); + + return { topLevelDir, nestedDir }; +} + +test('sanitizePackagedNodeModulesTree removes duplicated tar nested inside onnxruntime-node', async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-dup-tar-prune-')); + try { + const { topLevelDir, nestedDir } = await buildFakeDuplicateTree(stageDir, { + topLevelPackagePath: 'tar', + nestedDuplicatePath: '@huggingface/transformers/node_modules/onnxruntime-node/node_modules/tar', + }); + + await sanitizePackagedNodeModulesTree({ stageDir, target: TARGET }); + + assert.equal(await pathExists(nestedDir), false, 'nested duplicate tar dir should be removed'); + assert.equal(await pathExists(topLevelDir), true, 'top-level tar dir should be preserved'); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } +}); + +test('sanitizePackagedNodeModulesTree removes duplicated @modelcontextprotocol/sdk nested inside claude-agent-sdk', async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-dup-mcp-sdk-prune-')); + try { + const { topLevelDir, nestedDir } = await buildFakeDuplicateTree(stageDir, { + topLevelPackagePath: '@modelcontextprotocol/sdk', + nestedDuplicatePath: '@anthropic-ai/claude-agent-sdk/node_modules/@modelcontextprotocol/sdk', + }); + + await sanitizePackagedNodeModulesTree({ stageDir, target: TARGET }); + + assert.equal(await pathExists(nestedDir), false, 'nested duplicate @modelcontextprotocol/sdk dir should be removed'); + assert.equal(await pathExists(topLevelDir), true, 'top-level @modelcontextprotocol/sdk dir should be preserved'); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } +}); diff --git a/scripts/pipeline/release/lib/binary-release.mjs b/scripts/pipeline/release/lib/binary-release.mjs index 632a8fb4e7..fca3572a33 100644 --- a/scripts/pipeline/release/lib/binary-release.mjs +++ b/scripts/pipeline/release/lib/binary-release.mjs @@ -568,22 +568,49 @@ function isNestedNodeModulesBinDir(path) { return path.includes('/node_modules/.bin') || path.includes('\\node_modules\\.bin'); } -// Some packages (e.g. onnxruntime-node) bundle prebuilt native binaries for every -// supported platform/arch inside their own tree instead of splitting them into -// per-target optionalDependencies, so package.json os/cpu constraints alone can't -// prune them. Match known bundle root directories (whose children are "" -// dirs, each containing "" dirs) and drop the ones that don't match the target. -const BUNDLED_NATIVE_PLATFORM_ROOT_DIR_PATTERNS = [ +// Some packages bundle prebuilt native binaries for every supported platform/arch +// inside their own tree instead of splitting them into per-target +// optionalDependencies, so package.json os/cpu constraints alone can't prune them. +// Match known bundle root directories and drop the platform/arch combinations that +// don't match the packaging target. +// +// Two layouts are recognized: +// - nested: ///... (e.g. onnxruntime-node) +// - flat: /-/... (e.g. node-pty, node-pty-prebuilt-multiarch) +const NESTED_BUNDLED_NATIVE_PLATFORM_ROOT_DIR_PATTERNS = [ // onnxruntime-node: bin/napi-v///... /\/node_modules\/onnxruntime-node\/bin\/napi-v\d+$/, ]; -function isBundledNativePlatformRootDir(path) { +const FLAT_BUNDLED_NATIVE_PLATFORM_ROOT_DIR_PATTERNS = [ + // node-pty, @homebridge/node-pty-prebuilt-multiarch: prebuilds/-/... + /\/node_modules\/(?:node-pty|@homebridge\/node-pty-prebuilt-multiarch)\/prebuilds$/, + // bare-fs, bare-url, bare-os (nested under archiver -> tar-stream): prebuilds/-/... + /\/node_modules\/bare-(?:fs|url|os)\/prebuilds$/, +]; + +// The independent per-package vendoring entry points (vendorBundledPackageRuntimeDependencies +// for apps/cli's own deps vs. bundleInstalledPackageWithRuntimeDependencies for each +// CLI_RUNTIME_EXTERNAL_PACKAGES package) don't share a "visited" set with each other, so a +// transitive dependency already present at the top-level node_modules can also get copied again +// into a nested package's own node_modules. These nested copies are confirmed exact duplicates +// (same version, byte-identical trees) of a package already present at the payload's top-level +// node_modules, so ordinary upward-walking Node/Bun module resolution finds the top-level copy +// once the nested duplicate is removed. Delete the whole nested directory outright. +const DUPLICATE_VENDORED_PACKAGE_DIR_PATTERNS = [ + // tar duplicated inside onnxruntime-node's own node_modules (only used by onnxruntime-node's + // postinstall script, which never runs against the shipped prebuilt payload). + /\/node_modules\/@huggingface\/transformers\/node_modules\/onnxruntime-node\/node_modules\/tar$/, + // @modelcontextprotocol/sdk duplicated inside claude-agent-sdk's own node_modules. + /\/node_modules\/@anthropic-ai\/claude-agent-sdk\/node_modules\/@modelcontextprotocol\/sdk$/, +]; + +function matchesAnyPattern(path, patterns) { const normalized = path.replaceAll('\\', '/'); - return BUNDLED_NATIVE_PLATFORM_ROOT_DIR_PATTERNS.some((pattern) => pattern.test(normalized)); + return patterns.some((pattern) => pattern.test(normalized)); } -async function pruneBundledNativePlatformRootDir(params) { +async function pruneNestedBundledNativePlatformRootDir(params) { const targetNodePlatform = resolveTargetNodePlatform(params.target); const targetArch = String(params.target?.arch ?? '').trim().toLowerCase(); const entries = await readdir(params.directoryPath, { withFileTypes: true }).catch(() => []); @@ -605,11 +632,53 @@ async function pruneBundledNativePlatformRootDir(params) { } } +async function pruneFlatBundledNativePlatformRootDir(params) { + const targetNodePlatform = resolveTargetNodePlatform(params.target); + const targetArch = String(params.target?.arch ?? '').trim().toLowerCase(); + const targetDirName = `${targetNodePlatform}-${targetArch}`; + const entries = await readdir(params.directoryPath, { withFileTypes: true }).catch(() => []); + + for (const entry of entries) { + if (!entry.isDirectory() || entry.name.toLowerCase() !== targetDirName) { + await rm(join(params.directoryPath, entry.name), { recursive: true, force: true }); + } + } +} + +async function prunePackageDistDualFormatDir(params) { + const entries = await readdir(params.directoryPath, { withFileTypes: true }).catch(() => []); + + for (const entry of entries) { + const childPath = join(params.directoryPath, entry.name); + if (entry.isDirectory()) { + await prunePackageDistDualFormatDir({ directoryPath: childPath }); + continue; + } + if (entry.isFile() && entry.name.endsWith('.cjs')) { + await rm(childPath, { force: true }); + } + } +} + +// ps-list bundles Windows-only fastlist helper binaries unconditionally (no os/cpu +// package.json gating), so they should be stripped whenever the packaging target +// isn't Windows. +const WINDOWS_ONLY_VENDOR_EXECUTABLE_PATTERNS = [ + /\/node_modules\/ps-list\/vendor\/.*\.exe$/i, +]; + async function prunePackagedTreeDirectory(params) { const entries = await readdir(params.directoryPath, { withFileTypes: true }).catch(() => []); for (const entry of entries) { if (!entry.isDirectory()) { + const childPath = join(params.directoryPath, entry.name); + if ( + params.target?.os !== 'windows' && + matchesAnyPattern(childPath, WINDOWS_ONLY_VENDOR_EXECUTABLE_PATTERNS) + ) { + await rm(childPath, { force: true }); + } continue; } @@ -621,6 +690,11 @@ async function prunePackagedTreeDirectory(params) { continue; } + if (entry.name === 'package-dist') { + await prunePackageDistDualFormatDir({ directoryPath: childPath }); + continue; + } + if (childInNodeModulesTree) { const packageJsonPath = join(childPath, 'package.json'); const packageJson = await readFile(packageJsonPath, 'utf-8') @@ -632,8 +706,18 @@ async function prunePackagedTreeDirectory(params) { } } - if (isBundledNativePlatformRootDir(childPath)) { - await pruneBundledNativePlatformRootDir({ directoryPath: childPath, target: params.target }); + if (matchesAnyPattern(childPath, NESTED_BUNDLED_NATIVE_PLATFORM_ROOT_DIR_PATTERNS)) { + await pruneNestedBundledNativePlatformRootDir({ directoryPath: childPath, target: params.target }); + continue; + } + + if (matchesAnyPattern(childPath, FLAT_BUNDLED_NATIVE_PLATFORM_ROOT_DIR_PATTERNS)) { + await pruneFlatBundledNativePlatformRootDir({ directoryPath: childPath, target: params.target }); + continue; + } + + if (matchesAnyPattern(childPath, DUPLICATE_VENDORED_PACKAGE_DIR_PATTERNS)) { + await rm(childPath, { recursive: true, force: true }); continue; } diff --git a/scripts/pipeline/release/lib/binary-release.onnxruntime-node-prune.test.mjs b/scripts/pipeline/release/lib/binary-release.onnxruntime-node-prune.test.mjs index 6b5f2a488f..9f90157ed7 100644 --- a/scripts/pipeline/release/lib/binary-release.onnxruntime-node-prune.test.mjs +++ b/scripts/pipeline/release/lib/binary-release.onnxruntime-node-prune.test.mjs @@ -6,35 +6,38 @@ import { tmpdir } from 'node:os'; import { sanitizePackagedNodeModulesTree } from './binary-release.mjs'; -// onnxruntime-node bundles prebuilt native binaries for every platform/arch inside -// its own package tree (bin/napi-v///...) rather than splitting -// them into per-target optionalDependencies. A single-platform CLI release tarball -// should only ship the binaries for its own target. +// Native-binary bundling packages that ship prebuilt binaries for every +// platform/arch inside their own package tree (instead of splitting per-target +// into optionalDependencies), so package.json os/cpu constraints alone can't +// prune them. Every single-platform CLI release tarball should only ship the +// binaries for its own target, regardless of which target that is. +const CLI_TARGETS = [ + { os: 'linux', arch: 'x64' }, + { os: 'linux', arch: 'arm64' }, + { os: 'darwin', arch: 'x64' }, + { os: 'darwin', arch: 'arm64' }, + { os: 'windows', arch: 'x64' }, +]; + +const NESTED_PLATFORM_ARCH_PAIRS = [ + ['linux', 'x64'], + ['linux', 'arm64'], + ['darwin', 'x64'], + ['darwin', 'arm64'], + ['win32', 'x64'], + ['win32', 'arm64'], +]; + async function buildFakeOnnxruntimeNodeTree(stageDir) { const pkgDir = join(stageDir, 'node_modules', 'onnxruntime-node'); + await mkdir(pkgDir, { recursive: true }); await writeFile( join(pkgDir, 'package.json'), JSON.stringify({ name: 'onnxruntime-node', os: ['win32', 'darwin', 'linux'] }), 'utf-8', - ).catch(async (error) => { - if (error.code !== 'ENOENT') throw error; - await mkdir(pkgDir, { recursive: true }); - await writeFile( - join(pkgDir, 'package.json'), - JSON.stringify({ name: 'onnxruntime-node', os: ['win32', 'darwin', 'linux'] }), - 'utf-8', - ); - }); + ); - const platformArchPairs = [ - ['linux', 'x64'], - ['linux', 'arm64'], - ['darwin', 'x64'], - ['darwin', 'arm64'], - ['win32', 'x64'], - ['win32', 'arm64'], - ]; - for (const [platform, arch] of platformArchPairs) { + for (const [platform, arch] of NESTED_PLATFORM_ARCH_PAIRS) { const dir = join(pkgDir, 'bin', 'napi-v3', platform, arch); await mkdir(dir, { recursive: true }); await writeFile(join(dir, 'onnxruntime_binding.node'), 'fake-binary', 'utf-8'); @@ -43,47 +46,55 @@ async function buildFakeOnnxruntimeNodeTree(stageDir) { return pkgDir; } -test('sanitizePackagedNodeModulesTree prunes onnxruntime-node bundled binaries to the packaging target only', async () => { - const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-onnx-prune-')); +async function buildFakeFlatPrebuildsTree(stageDir, packageName) { + const pkgDir = join(stageDir, 'node_modules', ...packageName.split('/')); + await mkdir(pkgDir, { recursive: true }); + await writeFile(join(pkgDir, 'package.json'), JSON.stringify({ name: packageName }), 'utf-8'); - try { - const pkgDir = await buildFakeOnnxruntimeNodeTree(stageDir); + const prebuildsDir = join(pkgDir, 'prebuilds'); + for (const [platform, arch] of NESTED_PLATFORM_ARCH_PAIRS) { + const dir = join(prebuildsDir, `${platform}-${arch}`); + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'pty.node'), 'fake-binary', 'utf-8'); + } - await sanitizePackagedNodeModulesTree({ - stageDir, - target: { os: 'darwin', arch: 'arm64' }, - }); + return pkgDir; +} - const napiDir = join(pkgDir, 'bin', 'napi-v3'); - const remainingPlatforms = await readdir(napiDir); - assert.deepEqual(remainingPlatforms.sort(), ['darwin']); +for (const target of CLI_TARGETS) { + const expectedNodePlatform = target.os === 'windows' ? 'win32' : target.os; - const remainingArches = await readdir(join(napiDir, 'darwin')); - assert.deepEqual(remainingArches, ['arm64']); - } finally { - await rm(stageDir, { recursive: true, force: true }); - } -}); + test(`sanitizePackagedNodeModulesTree prunes onnxruntime-node (nested layout) to ${target.os}/${target.arch} only`, async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-onnx-prune-')); + try { + const pkgDir = await buildFakeOnnxruntimeNodeTree(stageDir); -test('sanitizePackagedNodeModulesTree keeps only the matching arch for windows targets', async () => { - const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-onnx-prune-win-')); + await sanitizePackagedNodeModulesTree({ stageDir, target }); - try { - const pkgDir = await buildFakeOnnxruntimeNodeTree(stageDir); + const napiDir = join(pkgDir, 'bin', 'napi-v3'); + const remainingPlatforms = await readdir(napiDir); + assert.deepEqual(remainingPlatforms, [expectedNodePlatform]); - await sanitizePackagedNodeModulesTree({ - stageDir, - // buildBinaryTarget.mjs uses os: 'windows'; resolveTargetNodePlatform maps it to 'win32'. - target: { os: 'windows', arch: 'x64' }, - }); + const remainingArches = await readdir(join(napiDir, expectedNodePlatform)); + assert.deepEqual(remainingArches, [target.arch]); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } + }); - const napiDir = join(pkgDir, 'bin', 'napi-v3'); - const remainingPlatforms = await readdir(napiDir); - assert.deepEqual(remainingPlatforms.sort(), ['win32']); + for (const packageName of ['node-pty', '@homebridge/node-pty-prebuilt-multiarch']) { + test(`sanitizePackagedNodeModulesTree prunes ${packageName} (flat layout) to ${target.os}/${target.arch} only`, async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-flat-prune-')); + try { + const pkgDir = await buildFakeFlatPrebuildsTree(stageDir, packageName); - const remainingArches = await readdir(join(napiDir, 'win32')); - assert.deepEqual(remainingArches, ['x64']); - } finally { - await rm(stageDir, { recursive: true, force: true }); + await sanitizePackagedNodeModulesTree({ stageDir, target }); + + const remaining = await readdir(join(pkgDir, 'prebuilds')); + assert.deepEqual(remaining, [`${expectedNodePlatform}-${target.arch}`]); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } + }); } -}); +} diff --git a/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs b/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs new file mode 100644 index 0000000000..e2a4e7b81e --- /dev/null +++ b/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs @@ -0,0 +1,50 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, writeFile, rm, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { sanitizePackagedNodeModulesTree } from './binary-release.mjs'; + +// package-dist/ ships parallel .mjs and .cjs bundles for every file. The .cjs +// half is only consumed by the published npm package's require() entrypoint +// (a separate distribution channel); the Homebrew-installed compiled Bun +// binary only ever resolves hardcoded .mjs relative paths. Dropping the .cjs +// half from the binary-release payload saves several MB with no runtime risk. +const CLI_TARGET = { os: 'darwin', arch: 'arm64' }; + +async function buildFakePackageDistTree(stageDir) { + const pkgDistDir = join(stageDir, 'package-dist'); + await mkdir(pkgDistDir, { recursive: true }); + + const fileStems = ['index', 'api-CNAditUJ']; + for (const stem of fileStems) { + await writeFile(join(pkgDistDir, `${stem}.mjs`), 'export default 1;', 'utf-8'); + await writeFile(join(pkgDistDir, `${stem}.cjs`), 'module.exports = 1;', 'utf-8'); + } + + // Nested subdirectory should also have its .cjs files pruned. + const nestedDir = join(pkgDistDir, 'mcp', 'bridges'); + await mkdir(nestedDir, { recursive: true }); + await writeFile(join(nestedDir, 'remoteMcpStdioBridge.mjs'), 'export default 1;', 'utf-8'); + await writeFile(join(nestedDir, 'remoteMcpStdioBridge.cjs'), 'module.exports = 1;', 'utf-8'); + + return pkgDistDir; +} + +test('sanitizePackagedNodeModulesTree prunes package-dist .cjs files, keeping .mjs', async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-package-dist-prune-')); + try { + const pkgDistDir = await buildFakePackageDistTree(stageDir); + + await sanitizePackagedNodeModulesTree({ stageDir, target: CLI_TARGET }); + + const remainingTopLevel = (await readdir(pkgDistDir)).sort(); + assert.deepEqual(remainingTopLevel, ['api-CNAditUJ.mjs', 'index.mjs', 'mcp']); + + const remainingNested = await readdir(join(pkgDistDir, 'mcp', 'bridges')); + assert.deepEqual(remainingNested, ['remoteMcpStdioBridge.mjs']); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } +}); From 8c4c00ab4fe98b17a4ef82aacab439868a16fe01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Guldmund?= Date: Tue, 4 Aug 2026 20:52:37 +0200 Subject: [PATCH 03/10] Prune unused MCP SDK HTTP-transport deps, more duplicates, package-dist type files Second round of packaged-CLI size reduction, on top of the fixes already in this PR: - @modelcontextprotocol/sdk vendors express, express-rate-limit, cors, and jose for its OAuth-authorization-server and Express-adapter code paths, and a standalone hono package alongside the (actually used) @hono/node-server. Traced every require() reachable from happier's real SDK entry points (server/index.js, server/mcp.js, server/streamableHttp.js, client/*.js) and cross-checked against zero occurrences of these package names in the compiled binary's strings output -- none of these are ever loaded. Does NOT touch first-party SDK source under server/auth/ (server/auth/errors.js is reachable via client/auth.js and must survive). - Extended DUPLICATE_VENDORED_PACKAGE_DIR_PATTERNS with more confirmed byte-identical nested duplicates: zod (4 copies), archiver-utils (1), qs (1, removes most of a get-intrinsic duplication chain as a byproduct), get-intrinsic (1 remaining pattern), readable-stream (4, all in archiver's own dependency chain). Each has a verified surviving ancestor reachable by upward node_modules resolution once the nested copy is removed. An ajv duplication pattern was considered and explicitly rejected: those copies have no surviving ancestor anywhere in the payload and must be kept. - package-dist's dual-format pruning (added earlier in this PR for .cjs) now also strips .d.mts/.d.cts type-declaration files -- confirmed dead weight for the same reason as .cjs: this is a compiled Bun binary, not a package resolved via npm's exports/types condition machinery. Measured on the already-fixed 676MB darwin-arm64 baseline: 676MB -> 600MB installed, 130.1MB -> 120.7MB compressed tarball. --- ...xhaustive-duplicate-resweep-prune.test.mjs | 153 ++++++++++++++++++ ...release.mcp-sdk-unused-deps-prune.test.mjs | 95 +++++++++++ .../pipeline/release/lib/binary-release.mjs | 67 +++++++- ...ry-release.package-dist-cjs-prune.test.mjs | 58 ++++--- 4 files changed, 352 insertions(+), 21 deletions(-) create mode 100644 scripts/pipeline/release/lib/binary-release.exhaustive-duplicate-resweep-prune.test.mjs create mode 100644 scripts/pipeline/release/lib/binary-release.mcp-sdk-unused-deps-prune.test.mjs diff --git a/scripts/pipeline/release/lib/binary-release.exhaustive-duplicate-resweep-prune.test.mjs b/scripts/pipeline/release/lib/binary-release.exhaustive-duplicate-resweep-prune.test.mjs new file mode 100644 index 0000000000..cb46e596b5 --- /dev/null +++ b/scripts/pipeline/release/lib/binary-release.exhaustive-duplicate-resweep-prune.test.mjs @@ -0,0 +1,153 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, writeFile, rm, access } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { sanitizePackagedNodeModulesTree } from './binary-release.mjs'; + +// Second-round duplicate-directory dedup findings ("exhaustive-duplicate-resweep"): the +// independent per-package vendoring entry points in workspaces/index.ts don't share a +// "visited"/"dedupeByNameVersion" map with each other, so the same transitive dependency can +// get vendored multiple times at different nesting depths. Each of these is confirmed +// byte-identical to a surviving copy reachable via ordinary upward Node/Bun module resolution +// once the nested duplicate is removed. +// +// NOTE: an "ajv duplicated across fastify/@modelcontextprotocol-sdk" case was in the original +// candidate list but is deliberately NOT covered here -- re-verification against the real tree +// showed none of those ajv copies has a surviving ancestor (there is no top-level ajv in the +// payload), so each is the sole reachable copy for its dependent and must not be pruned. + +const CLI_TARGETS = [ + { os: 'darwin', arch: 'arm64' }, + { os: 'darwin', arch: 'x64' }, + { os: 'linux', arch: 'arm64' }, + { os: 'linux', arch: 'x64' }, + { os: 'windows', arch: 'x64' }, +]; + +async function pathExists(path) { + return access(path).then( + () => true, + () => false, + ); +} + +async function buildFakeDuplicateTree(stageDir, { topLevelPackagePath, nestedDuplicatePath }) { + const topLevelDir = join(stageDir, 'node_modules', ...topLevelPackagePath.split('/')); + await mkdir(topLevelDir, { recursive: true }); + await writeFile(join(topLevelDir, 'package.json'), JSON.stringify({ name: 'top-level-copy' }), 'utf-8'); + await writeFile(join(topLevelDir, 'index.js'), 'module.exports = {};', 'utf-8'); + + const nestedDir = join(stageDir, 'node_modules', ...nestedDuplicatePath.split('/')); + await mkdir(nestedDir, { recursive: true }); + await writeFile(join(nestedDir, 'package.json'), JSON.stringify({ name: 'nested-duplicate-copy' }), 'utf-8'); + await writeFile(join(nestedDir, 'index.js'), 'module.exports = {};', 'utf-8'); + + return { topLevelDir, nestedDir }; +} + +const CASES = [ + { + name: 'zod nested inside @modelcontextprotocol/sdk', + topLevelPackagePath: 'zod', + nestedDuplicatePath: '@modelcontextprotocol/sdk/node_modules/zod', + }, + { + name: 'zod nested inside @happier-dev/agents', + topLevelPackagePath: 'zod', + nestedDuplicatePath: '@happier-dev/agents/node_modules/zod', + }, + { + name: 'zod nested inside @happier-dev/protocol', + topLevelPackagePath: 'zod', + nestedDuplicatePath: '@happier-dev/protocol/node_modules/zod', + }, + { + name: 'zod nested inside @happier-dev/protocol/zod-to-json-schema', + topLevelPackagePath: 'zod', + nestedDuplicatePath: '@happier-dev/protocol/node_modules/zod-to-json-schema/node_modules/zod', + }, + { + name: 'archiver-utils nested inside archiver/zip-stream', + topLevelPackagePath: 'archiver/node_modules/archiver-utils', + nestedDuplicatePath: 'archiver/node_modules/zip-stream/node_modules/archiver-utils', + }, + { + name: 'get-intrinsic nested inside a sibling call-bound package', + topLevelPackagePath: 'side-channel-weakmap/node_modules/get-intrinsic', + nestedDuplicatePath: 'side-channel-weakmap/node_modules/call-bound/node_modules/get-intrinsic', + }, + { + name: 'readable-stream nested inside archiver/zip-stream', + topLevelPackagePath: 'archiver/node_modules/readable-stream', + nestedDuplicatePath: 'archiver/node_modules/zip-stream/node_modules/readable-stream', + }, + { + name: 'readable-stream nested inside archiver/archiver-utils', + topLevelPackagePath: 'archiver/node_modules/readable-stream', + nestedDuplicatePath: 'archiver/node_modules/archiver-utils/node_modules/readable-stream', + }, + { + name: 'readable-stream nested inside archiver/zip-stream/compress-commons', + topLevelPackagePath: 'archiver/node_modules/readable-stream', + nestedDuplicatePath: + 'archiver/node_modules/zip-stream/node_modules/compress-commons/node_modules/readable-stream', + }, + { + name: 'readable-stream nested inside archiver/zip-stream/compress-commons/crc32-stream', + topLevelPackagePath: 'archiver/node_modules/readable-stream', + nestedDuplicatePath: + 'archiver/node_modules/zip-stream/node_modules/compress-commons/node_modules/crc32-stream/node_modules/readable-stream', + }, +]; + +for (const target of CLI_TARGETS) { + for (const testCase of CASES) { + test(`sanitizePackagedNodeModulesTree removes duplicate ${testCase.name} (${target.os}-${target.arch})`, async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-resweep-dup-prune-')); + try { + const { topLevelDir, nestedDir } = await buildFakeDuplicateTree(stageDir, { + topLevelPackagePath: testCase.topLevelPackagePath, + nestedDuplicatePath: testCase.nestedDuplicatePath, + }); + + await sanitizePackagedNodeModulesTree({ stageDir, target }); + + assert.equal(await pathExists(nestedDir), false, `nested duplicate ${testCase.name} dir should be removed`); + assert.equal(await pathExists(topLevelDir), true, `top-level ${testCase.name} dir should be preserved`); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } + }); + } +} + +// qs nested inside @modelcontextprotocol/sdk's express's own body-parser dependency is covered +// by its own dedicated pattern (see DUPLICATE_VENDORED_PACKAGE_DIR_PATTERNS in binary-release.mjs) +// rather than the shared parameterized loop above: the real-tree ancestor for this duplicate is +// `.../@modelcontextprotocol/sdk/node_modules/express`, a path prefix that another, independently +// landing pruning rule in this same file (unused-vendored-SDK-dependency pruning) may also +// legitimately delete wholesale as an unrelated finding. When both rules are present, the whole +// `express` subtree -- including both the "top-level" and "nested duplicate" qs copies used in a +// plain before/after existence check -- can be removed together, which is correct behavior, not a +// test bug. Assert the invariant that actually matters instead of an exact survivor: the nested +// duplicate must never survive. +for (const target of CLI_TARGETS) { + test(`sanitizePackagedNodeModulesTree removes duplicate qs nested inside @modelcontextprotocol/sdk/express/body-parser (${target.os}-${target.arch})`, async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-resweep-dup-prune-qs-')); + try { + const { nestedDir } = await buildFakeDuplicateTree(stageDir, { + topLevelPackagePath: '@modelcontextprotocol/sdk/node_modules/express/node_modules/qs', + nestedDuplicatePath: + '@modelcontextprotocol/sdk/node_modules/express/node_modules/body-parser/node_modules/qs', + }); + + await sanitizePackagedNodeModulesTree({ stageDir, target }); + + assert.equal(await pathExists(nestedDir), false, 'nested duplicate qs dir should never survive'); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } + }); +} diff --git a/scripts/pipeline/release/lib/binary-release.mcp-sdk-unused-deps-prune.test.mjs b/scripts/pipeline/release/lib/binary-release.mcp-sdk-unused-deps-prune.test.mjs new file mode 100644 index 0000000000..13f932331c --- /dev/null +++ b/scripts/pipeline/release/lib/binary-release.mcp-sdk-unused-deps-prune.test.mjs @@ -0,0 +1,95 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, writeFile, rm, access } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { sanitizePackagedNodeModulesTree } from './binary-release.mjs'; + +// @modelcontextprotocol/sdk vendors express, express-rate-limit, cors, jose, and hono inside +// its own node_modules, but happier never reaches the SDK source files that require them +// (server/express.js, server/auth/router.js, server/auth/handlers/*.js, +// client/auth-extensions.js), and `hono` is only referenced from type-only .d.ts files never +// executed by the sibling @hono/node-server package (which IS used and must be preserved). +// This pruning is unconditional -- it does not depend on packaging target os/arch -- so verify +// removal across all five CLI binary targets. +const CLI_TARGETS = [ + { os: 'linux', arch: 'x64' }, + { os: 'linux', arch: 'arm64' }, + { os: 'darwin', arch: 'x64' }, + { os: 'darwin', arch: 'arm64' }, + { os: 'windows', arch: 'x64' }, +]; + +const UNUSED_MCP_SDK_DEPENDENCY_NAMES = ['express', 'express-rate-limit', 'cors', 'jose', 'hono']; + +async function pathExists(path) { + return access(path).then( + () => true, + () => false, + ); +} + +async function writeFakePackageDir(dirPath, name) { + await mkdir(dirPath, { recursive: true }); + await writeFile(join(dirPath, 'package.json'), JSON.stringify({ name }), 'utf-8'); + await writeFile(join(dirPath, 'index.js'), 'module.exports = {};', 'utf-8'); +} + +async function buildFakeMcpSdkTree(stageDir) { + const sdkDir = join(stageDir, 'node_modules', '@modelcontextprotocol', 'sdk'); + await writeFakePackageDir(sdkDir, '@modelcontextprotocol/sdk'); + + const unusedDepDirs = {}; + for (const depName of UNUSED_MCP_SDK_DEPENDENCY_NAMES) { + const depDir = join(sdkDir, 'node_modules', depName); + await writeFakePackageDir(depDir, depName); + unusedDepDirs[depName] = depDir; + } + + // Retained sibling that must NOT be pruned: @hono/node-server (used) and other genuinely + // required SDK dependencies. + const honoNodeServerDir = join(sdkDir, 'node_modules', '@hono', 'node-server'); + await writeFakePackageDir(honoNodeServerDir, '@hono/node-server'); + + const ajvDir = join(sdkDir, 'node_modules', 'ajv'); + await writeFakePackageDir(ajvDir, 'ajv'); + + return { sdkDir, unusedDepDirs, honoNodeServerDir, ajvDir }; +} + +for (const target of CLI_TARGETS) { + test(`sanitizePackagedNodeModulesTree removes unused @modelcontextprotocol/sdk vendored deps for ${target.os}-${target.arch}`, async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-mcp-sdk-unused-deps-prune-')); + try { + const { unusedDepDirs, honoNodeServerDir, ajvDir } = await buildFakeMcpSdkTree(stageDir); + + await sanitizePackagedNodeModulesTree({ stageDir, target }); + + for (const [depName, depDir] of Object.entries(unusedDepDirs)) { + assert.equal(await pathExists(depDir), false, `${depName} should be removed`); + } + assert.equal(await pathExists(honoNodeServerDir), true, '@hono/node-server should be preserved'); + assert.equal(await pathExists(ajvDir), true, 'ajv should be preserved'); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } + }); +} + +test('sanitizePackagedNodeModulesTree does not prune first-party @modelcontextprotocol/sdk server/auth source', async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-mcp-sdk-auth-source-preserved-')); + try { + const sdkDir = join(stageDir, 'node_modules', '@modelcontextprotocol', 'sdk'); + await writeFakePackageDir(sdkDir, '@modelcontextprotocol/sdk'); + const authErrorsPath = join(sdkDir, 'dist', 'cjs', 'server', 'auth', 'errors.js'); + await mkdir(join(sdkDir, 'dist', 'cjs', 'server', 'auth'), { recursive: true }); + await writeFile(authErrorsPath, 'module.exports = {};', 'utf-8'); + + await sanitizePackagedNodeModulesTree({ stageDir, target: { os: 'darwin', arch: 'arm64' } }); + + assert.equal(await pathExists(authErrorsPath), true, 'server/auth/errors.js must not be pruned'); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } +}); diff --git a/scripts/pipeline/release/lib/binary-release.mjs b/scripts/pipeline/release/lib/binary-release.mjs index fca3572a33..aebd98205f 100644 --- a/scripts/pipeline/release/lib/binary-release.mjs +++ b/scripts/pipeline/release/lib/binary-release.mjs @@ -603,6 +603,63 @@ const DUPLICATE_VENDORED_PACKAGE_DIR_PATTERNS = [ /\/node_modules\/@huggingface\/transformers\/node_modules\/onnxruntime-node\/node_modules\/tar$/, // @modelcontextprotocol/sdk duplicated inside claude-agent-sdk's own node_modules. /\/node_modules\/@anthropic-ai\/claude-agent-sdk\/node_modules\/@modelcontextprotocol\/sdk$/, + // zod duplicated inside @modelcontextprotocol/sdk's, @happier-dev/agents's, + // @happier-dev/protocol's, and @happier-dev/protocol's own zod-to-json-schema's node_modules. + // Confirmed byte-identical to the top-level zod (same version, apps/cli depends on zod + // directly, so a top-level copy is always vendored). + /\/node_modules\/@modelcontextprotocol\/sdk\/node_modules\/zod$/, + /\/node_modules\/@happier-dev\/agents\/node_modules\/zod$/, + /\/node_modules\/@happier-dev\/protocol\/node_modules\/zod$/, + /\/node_modules\/@happier-dev\/protocol\/node_modules\/zod-to-json-schema\/node_modules\/zod$/, + // NOTE: an "ajv duplicated across fastify/@modelcontextprotocol-sdk" pattern was considered + // here but rejected on re-verification: although the 4 copies found are byte-identical to + // *each other*, none of them has any surviving ancestor `node_modules/ajv` to fall back to -- + // there is no top-level ajv in the payload at all, so each copy is the only copy reachable + // from its respective dependent and must be kept. Do not add an ajv pattern without a + // verified surviving ancestor for every occurrence being removed. + // archiver-utils duplicated inside archiver's own zip-stream dependency's node_modules. + // Confirmed byte-identical to archiver's own archiver-utils copy. Removing this also removes + // its nested lazystream/readable-stream (v2.3.8) subtree as a byproduct, which is fine: that + // subtree isn't independently referenced elsewhere. + /\/node_modules\/archiver\/node_modules\/zip-stream\/node_modules\/archiver-utils$/, + // qs duplicated inside @modelcontextprotocol/sdk's express's own body-parser dependency's + // node_modules. Confirmed byte-identical (and same version) to express's own top-level qs + // copy, which is resolvable by walking up from body-parser. Apply this before the + // get-intrinsic pattern below: it removes most of the get-intrinsic duplication under this + // qs copy as a byproduct. + /\/node_modules\/@modelcontextprotocol\/sdk\/node_modules\/express\/node_modules\/body-parser\/node_modules\/qs$/, + // get-intrinsic duplicated inside a sibling call-bound package's own node_modules. call-bound + // itself depends on get-intrinsic, but each of these copies is confirmed byte-identical to + // the get-intrinsic already vendored one level up (call-bound's own parent package's + // node_modules), which upward-walking resolution finds once the nested copy is removed. + /\/node_modules\/call-bound\/node_modules\/get-intrinsic$/, + // readable-stream duplicated across archiver's own nested zip-stream/archiver-utils/ + // compress-commons/crc32-stream dependency chain. All confirmed byte-identical (same v4.7.0) + // to archiver's own top-level readable-stream copy, resolvable by walking up once each nested + // duplicate is removed (verified via simulated multi-deletion + fresh resolution walk). + /\/node_modules\/archiver\/node_modules\/zip-stream\/node_modules\/readable-stream$/, + /\/node_modules\/archiver\/node_modules\/archiver-utils\/node_modules\/readable-stream$/, + /\/node_modules\/archiver\/node_modules\/zip-stream\/node_modules\/compress-commons\/node_modules\/readable-stream$/, + /\/node_modules\/archiver\/node_modules\/zip-stream\/node_modules\/compress-commons\/node_modules\/crc32-stream\/node_modules\/readable-stream$/, +]; + +// @modelcontextprotocol/sdk vendors several optional-feature dependencies (HTTP server +// framework, OAuth helpers) inside its own node_modules that are only required by SDK source +// files happier never imports (server/express.js, server/auth/router.js, +// server/auth/handlers/*.js, client/auth-extensions.js) or, for `hono`, only referenced from +// type-only .d.ts files never executed by @hono/node-server (which IS used and stays). Verified +// by tracing every require() from happier's actual reachable SDK entry points (server/mcp.js, +// server/stdio.js, server/streamableHttp.js, client/*.js) and cross-checking against `strings` +// on the compiled darwin-arm64 binary, which shows zero occurrences of these package names +// despite the surrounding SDK source being compiled in directly. Do NOT extend this list to +// cover first-party SDK source under dist/{cjs,esm}/server/auth -- server/auth/errors.js is on +// the reachable path (via client/auth.js) and must not be deleted. +const UNUSED_VENDORED_MCP_SDK_DEPENDENCY_DIR_PATTERNS = [ + /\/node_modules\/@modelcontextprotocol\/sdk\/node_modules\/express$/, + /\/node_modules\/@modelcontextprotocol\/sdk\/node_modules\/express-rate-limit$/, + /\/node_modules\/@modelcontextprotocol\/sdk\/node_modules\/cors$/, + /\/node_modules\/@modelcontextprotocol\/sdk\/node_modules\/jose$/, + /\/node_modules\/@modelcontextprotocol\/sdk\/node_modules\/hono$/, ]; function matchesAnyPattern(path, patterns) { @@ -654,7 +711,10 @@ async function prunePackageDistDualFormatDir(params) { await prunePackageDistDualFormatDir({ directoryPath: childPath }); continue; } - if (entry.isFile() && entry.name.endsWith('.cjs')) { + if ( + entry.isFile() && + (entry.name.endsWith('.cjs') || entry.name.endsWith('.d.mts') || entry.name.endsWith('.d.cts')) + ) { await rm(childPath, { force: true }); } } @@ -721,6 +781,11 @@ async function prunePackagedTreeDirectory(params) { continue; } + if (matchesAnyPattern(childPath, UNUSED_VENDORED_MCP_SDK_DEPENDENCY_DIR_PATTERNS)) { + await rm(childPath, { recursive: true, force: true }); + continue; + } + await prunePackagedTreeDirectory({ directoryPath: childPath, target: params.target, diff --git a/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs b/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs index e2a4e7b81e..9a9651d256 100644 --- a/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs +++ b/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs @@ -6,12 +6,22 @@ import { tmpdir } from 'node:os'; import { sanitizePackagedNodeModulesTree } from './binary-release.mjs'; -// package-dist/ ships parallel .mjs and .cjs bundles for every file. The .cjs -// half is only consumed by the published npm package's require() entrypoint -// (a separate distribution channel); the Homebrew-installed compiled Bun -// binary only ever resolves hardcoded .mjs relative paths. Dropping the .cjs -// half from the binary-release payload saves several MB with no runtime risk. -const CLI_TARGET = { os: 'darwin', arch: 'arm64' }; +// package-dist/ ships parallel .mjs and .cjs bundles for every file, plus +// .d.mts/.d.cts type-declaration siblings. The .cjs half is only consumed by +// the published npm package's require() entrypoint (a separate distribution +// channel); the .d.mts/.d.cts files are only consulted by third-party +// tooling (tsc, editors) resolving this package's npm "exports" map. The +// Homebrew-installed compiled Bun binary only ever resolves hardcoded .mjs +// relative paths and is never require()'d or type-checked as a library, so +// none of .cjs/.d.mts/.d.cts are reachable at runtime. Dropping them from the +// binary-release payload saves several hundred KB-to-MB with no runtime risk. +const CLI_TARGETS = [ + { os: 'darwin', arch: 'arm64' }, + { os: 'darwin', arch: 'x64' }, + { os: 'linux', arch: 'arm64' }, + { os: 'linux', arch: 'x64' }, + { os: 'windows', arch: 'x64' }, +]; async function buildFakePackageDistTree(stageDir) { const pkgDistDir = join(stageDir, 'package-dist'); @@ -22,29 +32,37 @@ async function buildFakePackageDistTree(stageDir) { await writeFile(join(pkgDistDir, `${stem}.mjs`), 'export default 1;', 'utf-8'); await writeFile(join(pkgDistDir, `${stem}.cjs`), 'module.exports = 1;', 'utf-8'); } + // Only the top-level index has .d.mts/.d.cts siblings, mirroring the real + // package-dist layout (index, lib, and a few nested entrypoints). + await writeFile(join(pkgDistDir, 'index.d.mts'), 'export declare const x: number;', 'utf-8'); + await writeFile(join(pkgDistDir, 'index.d.cts'), 'export declare const x: number;', 'utf-8'); - // Nested subdirectory should also have its .cjs files pruned. + // Nested subdirectory should also have its .cjs/.d.mts/.d.cts files pruned. const nestedDir = join(pkgDistDir, 'mcp', 'bridges'); await mkdir(nestedDir, { recursive: true }); await writeFile(join(nestedDir, 'remoteMcpStdioBridge.mjs'), 'export default 1;', 'utf-8'); await writeFile(join(nestedDir, 'remoteMcpStdioBridge.cjs'), 'module.exports = 1;', 'utf-8'); + await writeFile(join(nestedDir, 'remoteMcpStdioBridge.d.mts'), 'export declare const y: number;', 'utf-8'); + await writeFile(join(nestedDir, 'remoteMcpStdioBridge.d.cts'), 'export declare const y: number;', 'utf-8'); return pkgDistDir; } -test('sanitizePackagedNodeModulesTree prunes package-dist .cjs files, keeping .mjs', async () => { - const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-package-dist-prune-')); - try { - const pkgDistDir = await buildFakePackageDistTree(stageDir); +for (const target of CLI_TARGETS) { + test(`sanitizePackagedNodeModulesTree prunes package-dist .cjs/.d.mts/.d.cts files, keeping .mjs [${target.os}/${target.arch}]`, async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-package-dist-prune-')); + try { + const pkgDistDir = await buildFakePackageDistTree(stageDir); - await sanitizePackagedNodeModulesTree({ stageDir, target: CLI_TARGET }); + await sanitizePackagedNodeModulesTree({ stageDir, target }); - const remainingTopLevel = (await readdir(pkgDistDir)).sort(); - assert.deepEqual(remainingTopLevel, ['api-CNAditUJ.mjs', 'index.mjs', 'mcp']); + const remainingTopLevel = (await readdir(pkgDistDir)).sort(); + assert.deepEqual(remainingTopLevel, ['api-CNAditUJ.mjs', 'index.mjs', 'mcp']); - const remainingNested = await readdir(join(pkgDistDir, 'mcp', 'bridges')); - assert.deepEqual(remainingNested, ['remoteMcpStdioBridge.mjs']); - } finally { - await rm(stageDir, { recursive: true, force: true }); - } -}); + const remainingNested = await readdir(join(pkgDistDir, 'mcp', 'bridges')); + assert.deepEqual(remainingNested, ['remoteMcpStdioBridge.mjs']); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } + }); +} From decd3580987a45ec6bc09e6b08b40bc07b4bebef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Guldmund?= Date: Tue, 4 Aug 2026 22:21:33 +0200 Subject: [PATCH 04/10] Prune @huggingface/transformers dist/ to the resolved Node build only @huggingface/transformers ships browser, CJS, minified, and WASM-backend build variants in dist/ (44MB), but this payload -- a Bun/Node-only CLI -- only ever resolves dist/transformers.node.mjs via the package's own package.json exports.node.import condition. Confirmed via createLocalTransformersEmbeddingsProvider.ts's dynamic `await import('@huggingface/transformers')` and its own error-handling code, which explicitly checks for this exact filename. The bundled ort-wasm-simd-threaded.jsep.{mjs,wasm} pair is onnxruntime-web's browser-only WASM backend; the node build imports the native onnxruntime-node binding instead, confirmed by the bundler's own "onnxruntime-web (ignored)" comment in the compiled node output. Adds a new keep-list-based pruning mechanism (DIST_ROOT_KEEP_FILE_ALLOWLIST) alongside the existing pattern-list mechanisms, for the "package ships many dist/ build targets, only one is ever resolved on this runtime" case. ~41MB installed size reduction (600MB -> 559MB on the already-fixed darwin-arm64 baseline), not platform/arch-sensitive since the exports-map resolution doesn't depend on OS/arch. --- ...ggingface-transformers-dist-prune.test.mjs | 70 +++++++++++++++++++ .../pipeline/release/lib/binary-release.mjs | 40 +++++++++++ 2 files changed, 110 insertions(+) create mode 100644 scripts/pipeline/release/lib/binary-release.huggingface-transformers-dist-prune.test.mjs diff --git a/scripts/pipeline/release/lib/binary-release.huggingface-transformers-dist-prune.test.mjs b/scripts/pipeline/release/lib/binary-release.huggingface-transformers-dist-prune.test.mjs new file mode 100644 index 0000000000..7764da799e --- /dev/null +++ b/scripts/pipeline/release/lib/binary-release.huggingface-transformers-dist-prune.test.mjs @@ -0,0 +1,70 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, writeFile, rm, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { sanitizePackagedNodeModulesTree } from './binary-release.mjs'; + +// @huggingface/transformers ships a dist/ with build outputs for every consumer (browser, CJS, +// minified, plus the browser-only onnxruntime-web WASM backend), but this payload only ever +// resolves dist/transformers.node.mjs via the package's own package.json exports.node.import +// condition (Node/Bun, dynamic `await import('@huggingface/transformers')` in +// createLocalTransformersEmbeddingsProvider.ts). This is not platform/arch-sensitive -- the same +// files are unreachable on every CLI target, since "which consumer resolves which dist file" is +// determined by the exports map, not by OS/arch. +const CLI_TARGETS = [ + { os: 'darwin', arch: 'arm64' }, + { os: 'darwin', arch: 'x64' }, + { os: 'linux', arch: 'arm64' }, + { os: 'linux', arch: 'x64' }, + { os: 'windows', arch: 'x64' }, +]; + +async function buildFakeTransformersDistTree(stageDir) { + const distDir = join(stageDir, 'node_modules', '@huggingface', 'transformers', 'dist'); + await mkdir(distDir, { recursive: true }); + + const unreachableFiles = [ + 'transformers.js', + 'transformers.js.map', + 'transformers.min.js', + 'transformers.min.js.map', + 'transformers.web.js', + 'transformers.web.js.map', + 'transformers.web.min.js', + 'transformers.web.min.js.map', + 'transformers.node.cjs', + 'transformers.node.cjs.map', + 'transformers.node.min.cjs', + 'transformers.node.min.cjs.map', + 'transformers.node.min.mjs', + 'transformers.node.min.mjs.map', + 'ort-wasm-simd-threaded.jsep.mjs', + 'ort-wasm-simd-threaded.jsep.wasm', + ]; + for (const name of unreachableFiles) { + await writeFile(join(distDir, name), 'placeholder', 'utf-8'); + } + + await writeFile(join(distDir, 'transformers.node.mjs'), 'export default {};', 'utf-8'); + await writeFile(join(distDir, 'transformers.node.mjs.map'), '{}', 'utf-8'); + + return distDir; +} + +for (const target of CLI_TARGETS) { + test(`sanitizePackagedNodeModulesTree prunes @huggingface/transformers dist/ to the node build only [${target.os}/${target.arch}]`, async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-transformers-dist-prune-')); + try { + const distDir = await buildFakeTransformersDistTree(stageDir); + + await sanitizePackagedNodeModulesTree({ stageDir, target }); + + const remaining = (await readdir(distDir)).sort(); + assert.deepEqual(remaining, ['transformers.node.mjs', 'transformers.node.mjs.map']); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } + }); +} diff --git a/scripts/pipeline/release/lib/binary-release.mjs b/scripts/pipeline/release/lib/binary-release.mjs index aebd98205f..8ee4f543c1 100644 --- a/scripts/pipeline/release/lib/binary-release.mjs +++ b/scripts/pipeline/release/lib/binary-release.mjs @@ -702,6 +702,38 @@ async function pruneFlatBundledNativePlatformRootDir(params) { } } +// Some vendored packages ship a dist/ with build outputs for every consumer (browser, CJS, +// minified, ...) even though this payload -- a Bun/Node-only CLI, never a browser or a require() +// consumer -- only ever reaches one of them via the package's own package.json `exports` map. +// Match the package's dist/ root and keep only the listed files (relative to that dist/ dir); +// everything else in the directory (recursively) is deleted. +// +// @huggingface/transformers: its package.json `exports.node.import.default` points at +// dist/transformers.node.mjs, which is exactly what createLocalTransformersEmbeddingsProvider.ts's +// `await import('@huggingface/transformers')` resolves to on Node/Bun (confirmed: its own error +// handling explicitly checks for this filename). The other dist/ entries (transformers.js/.min.js, +// transformers.web.js/.min.js, transformers.node.cjs/.min.cjs/.min.mjs, and their .map files) are +// for browser and CommonJS require() consumers this payload never becomes. The bundled +// ort-wasm-simd-threaded.jsep.{mjs,wasm} pair is onnxruntime-web's browser-only WASM backend -- +// the node build imports onnxruntime-node (native binding) instead, confirmed by +// transformers.node.mjs's own bundler comments ("onnxruntime-web (ignored)" in the node build). +const DIST_ROOT_KEEP_FILE_ALLOWLIST = [ + { + dirPattern: /\/node_modules\/@huggingface\/transformers\/dist$/, + keepFileNames: new Set(['transformers.node.mjs', 'transformers.node.mjs.map']), + }, +]; + +async function pruneDistRootToAllowlist(params) { + const entries = await readdir(params.directoryPath, { withFileTypes: true }).catch(() => []); + + for (const entry of entries) { + if (!params.keepFileNames.has(entry.name)) { + await rm(join(params.directoryPath, entry.name), { recursive: true, force: true }); + } + } +} + async function prunePackageDistDualFormatDir(params) { const entries = await readdir(params.directoryPath, { withFileTypes: true }).catch(() => []); @@ -755,6 +787,14 @@ async function prunePackagedTreeDirectory(params) { continue; } + { + const distAllowlist = DIST_ROOT_KEEP_FILE_ALLOWLIST.find((entry_) => entry_.dirPattern.test(childPath.replaceAll('\\', '/'))); + if (distAllowlist) { + await pruneDistRootToAllowlist({ directoryPath: childPath, keepFileNames: distAllowlist.keepFileNames }); + continue; + } + } + if (childInNodeModulesTree) { const packageJsonPath = join(childPath, 'package.json'); const packageJson = await readFile(packageJsonPath, 'utf-8') From c4fe82e9bb12fac92cc826b80d1e30b79db7ec26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Guldmund?= Date: Wed, 5 Aug 2026 07:41:53 +0200 Subject: [PATCH 05/10] Remove 4 confirmed-unused apps/cli dependencies, move tmp to devDependencies Verified with git-history tracing, cross-workspace checks, and peer-dependency analysis (not just grep) that these declared dependencies have zero reachable runtime or build-time usage anywhere in the repo: - @fastify/swagger: dead in both apps/cli and its sibling apps/server; never wired up in either. - @stablelib/base64: added alongside apps/cli's real base64 helper (which uses plain Buffer.toString('base64')) and never used; its real historical use was mobile-only and has since been fully retired there too. - ai (Vercel AI SDK): added in the same commit as ACP/Gemini backend work but never referenced by any file in that commit or since; confirmed absent from the compiled dist bundle. - http-proxy-middleware: a different, unused package confusable with the actually-used http-proxy (which apps/cli's proxy code calls directly). Also moved tmp to devDependencies: only imported from *.test.ts files, never from apps/cli/src at runtime, and not referenced by any postinstall- reachable script (unlike tar, which was considered and correctly rejected for the same move -- unpack-tools.cjs requires it and runs via postinstall on real end-user npm installs). Two candidates from the initial survey were investigated and correctly rejected: openapi-types (a mandatory, non-optional peer dependency of fastify-type-provider-zod and @fastify/swagger, both real direct deps) and react-devtools-core (ink's peerDependency, conditionally dynamic-imported by ink's reconciler when DEV=true is set in the environment -- a real, if rarely-exercised, code path outside apps/cli's own control). Verified via the full local-build installer smoke lifecycle (install -> version -> help -> check -> reinstall -> check -> uninstall) against a freshly-compiled binary with these changes applied -- all steps passed. ~14.8MB removed across the 4 dropped packages. --- apps/cli/package.json | 6 +--- yarn.lock | 70 +++---------------------------------------- 2 files changed, 5 insertions(+), 71 deletions(-) diff --git a/apps/cli/package.json b/apps/cli/package.json index a09fb97827..8757141e9d 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -138,7 +138,6 @@ "dependencies": { "@agentclientprotocol/sdk": "^1.2.1", "@anthropic-ai/claude-agent-sdk": "^0.2.123", - "@fastify/swagger": "9.7.0", "@happier-dev/agents": "0.0.0", "@happier-dev/cli-common": "0.0.0", "@happier-dev/connection-supervisor": "0.0.0", @@ -147,7 +146,6 @@ "@happier-dev/release-runtime": "0.0.0", "@huggingface/transformers": "^3.8.1", "@modelcontextprotocol/sdk": "^1.26.0", - "@stablelib/base64": "^2.0.1", "@stablelib/hex": "^2.0.1", "@types/cross-spawn": "^6.0.6", "@types/http-proxy": "^1.17.17", @@ -155,7 +153,6 @@ "@types/qrcode-terminal": "^0.12.2", "@types/react": "^19.2.7", "@types/tmp": "^0.2.6", - "ai": "^5.0.107", "archiver": "^7.0.1", "axios": "^1.13.2", "chalk": "^5.6.2", @@ -165,7 +162,6 @@ "fastify": "^5.7.3", "fastify-type-provider-zod": "6.1.0", "http-proxy": "^1.18.1", - "http-proxy-middleware": "^3.0.5", "https-proxy-agent": "^7.0.6", "ink": "^6.5.1", "@homebridge/node-pty-prebuilt-multiarch": "^0.13.1", @@ -179,7 +175,6 @@ "sharp": "^0.34.3", "socket.io-client": "^4.8.1", "tar": "^7.5.8", - "tmp": "^0.2.5", "tweetnacl": "^1.0.3", "zod": "4.3.6" }, @@ -194,6 +189,7 @@ "pkgroll": "^2.27.0", "release-it": "^19.0.6", "shx": "^0.3.3", + "tmp": "^0.2.5", "ts-node": "^10", "tsx": "^4.20.6", "@typescript/native": "npm:typescript@7.0.2", diff --git a/yarn.lock b/yarn.lock index d0535d6866..d7c1b34184 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12,31 +12,6 @@ resolved "https://registry.npmjs.org/@agentclientprotocol/sdk/-/sdk-1.2.1.tgz#c98952123d2b202a143ab5ec68782eec2775003a" integrity sha512-jwYUdOQR7tc+Zfch53VL4JJyUNK/46q03uUTYb+PjECsmnNl94XFXOfYLJ8RBpMNidXd1rpOAVgb0vqD98xImA== -"@ai-sdk/gateway@2.0.29": - version "2.0.29" - resolved "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-2.0.29.tgz" - integrity sha512-1b7E9F/B5gex/1uCkhs+sGIbH0KsZOItHnNz3iY5ir+nc4ZUA6WOU5Cu2w1USlc+3UVbhf+H+iNLlxVjLe4VvQ== - dependencies: - "@ai-sdk/provider" "2.0.1" - "@ai-sdk/provider-utils" "3.0.20" - "@vercel/oidc" "3.1.0" - -"@ai-sdk/provider-utils@3.0.20": - version "3.0.20" - resolved "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-3.0.20.tgz" - integrity sha512-iXHVe0apM2zUEzauqJwqmpC37A5rihrStAih5Ks+JE32iTe4LZ58y17UGBjpQQTCRw9YxMeo2UFLxLpBluyvLQ== - dependencies: - "@ai-sdk/provider" "2.0.1" - "@standard-schema/spec" "^1.0.0" - eventsource-parser "^3.0.6" - -"@ai-sdk/provider@2.0.1": - version "2.0.1" - resolved "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.1.tgz" - integrity sha512-KCUwswvsC5VsW2PWFqF8eJgSCu5Ysj7m1TxiHTVA6g7k360bk0RNQENT8KTMAYEs+8fWPD3Uu4dEmzGHc+jGng== - dependencies: - json-schema "^0.4.0" - "@alcalzone/ansi-tokenize@^0.2.1": version "0.2.3" resolved "https://registry.npmjs.org/@alcalzone/ansi-tokenize/-/ansi-tokenize-0.2.3.tgz" @@ -3550,7 +3525,7 @@ dependencies: "@opentelemetry/api" "^1.3.0" -"@opentelemetry/api@1.9.0", "@opentelemetry/api@^1.3.0", "@opentelemetry/api@^1.4.0", "@opentelemetry/api@^1.9.0": +"@opentelemetry/api@^1.3.0", "@opentelemetry/api@^1.4.0", "@opentelemetry/api@^1.9.0": version "1.9.0" resolved "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz" integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== @@ -6244,7 +6219,7 @@ resolved "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz" integrity sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg== -"@types/http-proxy@^1.17.15", "@types/http-proxy@^1.17.17": +"@types/http-proxy@^1.17.17": version "1.17.17" resolved "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.17.tgz" integrity sha512-ED6LB+Z1AVylNTu7hdzuBqOgMnvG/ld6wGCG8wFnAzKX5uyW2K3WD52v0gnLCTK/VLpXtKckgWuyScYK6cSPaw== @@ -6724,11 +6699,6 @@ "@urql/core" "^5.1.2" wonka "^6.3.2" -"@vercel/oidc@3.1.0": - version "3.1.0" - resolved "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.1.0.tgz" - integrity sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w== - "@vitest/expect@3.2.4": version "3.2.4" resolved "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz" @@ -6941,16 +6911,6 @@ agent-base@^7.1.0, agent-base@^7.1.2: resolved "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz" integrity sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ== -ai@^5.0.107: - version "5.0.123" - resolved "https://registry.npmjs.org/ai/-/ai-5.0.123.tgz" - integrity sha512-V3Imb0tg0pHCa6a/VsoW/FZpT07mwUw/4Hj6nexJC1Nvf1eyKQJyaYVkl+YTLnA8cKQSUkoarKhXWbFy4CSgjw== - dependencies: - "@ai-sdk/gateway" "2.0.29" - "@ai-sdk/provider" "2.0.1" - "@ai-sdk/provider-utils" "3.0.20" - "@opentelemetry/api" "1.9.0" - ajv-formats@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz" @@ -8847,7 +8807,7 @@ debug@2.6.9, debug@^2.6.9: dependencies: ms "2.0.0" -debug@4, debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.3.5, debug@^4.3.6, debug@^4.4.0, debug@^4.4.1, debug@^4.4.3, debug@~4.4.1: +debug@4, debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.3.5, debug@^4.4.0, debug@^4.4.1, debug@^4.4.3, debug@~4.4.1: version "4.4.3" resolved "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz" integrity sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA== @@ -9832,7 +9792,7 @@ events@^3.3.0: resolved "https://registry.npmjs.org/events/-/events-3.3.0.tgz" integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== -eventsource-parser@^3.0.0, eventsource-parser@^3.0.1, eventsource-parser@^3.0.6: +eventsource-parser@^3.0.0, eventsource-parser@^3.0.1: version "3.0.6" resolved "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz" integrity sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg== @@ -11335,18 +11295,6 @@ http-proxy-agent@^7.0.0, http-proxy-agent@^7.0.1, http-proxy-agent@^7.0.2: agent-base "^7.1.0" debug "^4.3.4" -http-proxy-middleware@^3.0.5: - version "3.0.5" - resolved "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-3.0.5.tgz" - integrity sha512-GLZZm1X38BPY4lkXA01jhwxvDoOkkXqjgVyUzVxiEK4iuRu03PZoYHhHRwxnfhQMDuaxi3vVri0YgSro/1oWqg== - dependencies: - "@types/http-proxy" "^1.17.15" - debug "^4.3.6" - http-proxy "^1.18.1" - is-glob "^4.0.3" - is-plain-object "^5.0.0" - micromatch "^4.0.8" - http-proxy@^1.18.1: version "1.18.1" resolved "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz" @@ -11811,11 +11759,6 @@ is-plain-obj@^4.0.0: resolved "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz" integrity sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg== -is-plain-object@^5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz" - integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q== - is-potential-custom-element-name@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz" @@ -12234,11 +12177,6 @@ json-schema-typed@^8.0.2: resolved "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz" integrity sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA== -json-schema@^0.4.0: - version "0.4.0" - resolved "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz" - integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== - json-stable-stringify-without-jsonify@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz" From 5f399819a0d07edfd2286f05a9bc52bbe095f18e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Guldmund?= Date: Wed, 5 Aug 2026 08:38:15 +0200 Subject: [PATCH 06/10] Stop vendoring loose node_modules copies already compiled into the binary Bun's --compile tree-shakes apps/cli's own TypeScript source into the compiled binary (confirmed via `strings` on the real shipped binary: reachable files carry provenance comments, unreachable ones -- like the express/server-auth code already pruned in an earlier commit -- don't). But copyCliNodeRuntimePayload unconditionally vendored EVERY declared apps/cli dependency as loose files on disk, regardless of whether Bun had already compiled the reachable code directly into the executable. Most non-native dependencies were therefore present twice: compiled into the 76MB binary, and duplicated as a full loose node_modules tree alongside it. Verified per-package before excluding anything, not as a blanket policy: - Confirmed apps/cli/bin/*.mjs (the npm-publish entrypoint) is never copied into the compiled-binary payload at all -- it's a separate distribution channel, irrelevant to this payload's node_modules footprint. - Read every apps/cli/scripts/*.cjs sidecar script (copied into the payload via CLI_RUNTIME_SIDECAR_ENTRIES, run as child processes outside the main binary): only node_pty_relay.cjs requires a node_modules package by name, and it's node-pty/@homebridge/node-pty-prebuilt-multiarch -- already handled as Bun externals. - Audited apps/cli/src for genuinely dynamic (non-literal) require()/ import() calls that Bun's static analyzer can't resolve: found exactly one, in createLocalTransformersEmbeddingsProvider.ts, already accounted for by the existing @huggingface/transformers external. - Cross-checked every excluded package's compiled-in status via `strings` provenance-comment counts on the real binary. - Live-tested the riskiest case empirically: drove `happier auth login` through a real pty with ink deleted from the on-disk node_modules copy -- the AuthSelector terminal UI still rendered correctly (arrow-key highlighting, ANSI codes intact), direct proof rather than inference. - Kept `sharp` vendored: unlike everything else, it does a runtime- constructed require() of a platform-specific native .node binding, structurally identical to why node-pty is already external -- Bun's static analyzer categorically cannot resolve that path, so its on-disk copy is architecturally necessary regardless of tree-shaking. The exclusion is opt-in and scoped to only the compiled CLI binary payload path (copyCliNodeRuntimePayload): every other vendorBundledPackageRuntimeDependencies call site (npm-published tarball builds, apps/stack, packages/relay-server) receives no excludePackageNames argument and continues vendoring every dependency in full, unchanged. Verified via the full local-build installer smoke lifecycle (install -> version -> help -> check -> reinstall -> check -> uninstall) against a freshly-compiled binary -- all steps passed -- and via a real release build (node scripts/pipeline/release/build-cli-binaries.mjs) measured end to end: installed: 990.7MB -> 430MB (-57%) tarball: 222.5MB -> 99.2MB (-55%) node_modules alone dropped from 434MB to 186MB, of which 155MB is @huggingface/transformers (the ONNX runtime + local-embeddings model code, genuinely native/necessary) -- nearly everything else non-native collapsed to a few MB of workspace bundles and PTY native bindings. --- .../buildCliBinaryArtifactPayload.ts | 47 ++++++++ .../cli-common/src/workspaces/index.test.ts | 102 ++++++++++++++++++ packages/cli-common/src/workspaces/index.ts | 7 +- 3 files changed, 155 insertions(+), 1 deletion(-) diff --git a/packages/cli-common/src/componentArtifacts/buildCliBinaryArtifactPayload.ts b/packages/cli-common/src/componentArtifacts/buildCliBinaryArtifactPayload.ts index 190f615e8f..57d666205a 100644 --- a/packages/cli-common/src/componentArtifacts/buildCliBinaryArtifactPayload.ts +++ b/packages/cli-common/src/componentArtifacts/buildCliBinaryArtifactPayload.ts @@ -41,6 +41,52 @@ const CLI_RUNTIME_EXTERNAL_PACKAGES = [ '@homebridge/node-pty-prebuilt-multiarch', ] as const; +// Bun's `--compile` tree-shakes apps/cli's own source into the compiled binary (confirmed via +// `strings` on the real shipped binary: reachable files carry provenance comments, unreachable +// ones don't), so any statically-imported, non-external dependency reachable from that source is +// already embedded in the executable. These declared apps/cli dependencies were audited (see +// investigate/compiled-in-vs-vendored-audit) and confirmed to have no runtime code path -- +// bin/*.mjs entrypoint, scripts/*.cjs sidecar, or dynamic require()/import() with a non-static +// path -- that reads them from an on-disk node_modules copy. Vendoring a duplicate loose copy of +// these onto disk alongside the compiled binary is therefore pure waste. +// +// This exclusion is scoped to the compiled CLI binary payload only (copyCliNodeRuntimePayload, +// below). Other vendorBundledPackageRuntimeDependencies call sites (npm-published tarball builds, +// apps/stack, packages/relay-server) do not compile their source into a binary and must keep +// vendoring these packages in full. +// +// Keep this list scoped to packages with concrete, checked evidence; when in doubt, leave a +// package vendored. Notably `sharp` is NOT included here: it does a runtime-constructed +// `require()` of a platform-specific native `.node` binding, the same reason node-pty is external +// above, so it must stay vendored on disk. +const CLI_BINARY_PAYLOAD_VENDORING_EXCLUDED_PACKAGES = new Set([ + '@agentclientprotocol/sdk', + '@anthropic-ai/claude-agent-sdk', + '@modelcontextprotocol/sdk', + '@stablelib/hex', + 'archiver', + 'axios', + 'chalk', + 'cross-spawn', + 'diff', + 'expo-server-sdk', + 'fastify', + 'fastify-type-provider-zod', + 'http-proxy', + 'https-proxy-agent', + 'ink', + 'open', + 'openapi-types', + 'ps-list', + 'qrcode-terminal', + 'react', + 'react-devtools-core', + 'socket.io-client', + 'tar', + 'tmp', + 'zod', +]); + type CliToolUnpackModule = { unpackTools?: (options: Readonly<{ platformDir: string; toolsDir: string }>) => Promise | unknown; }; @@ -116,6 +162,7 @@ async function copyCliNodeRuntimePayload( vendorBundledPackageRuntimeDependencies({ srcPackageJsonPath: join(cliDir, 'package.json'), destPackageDir: payloadDir, + excludePackageNames: CLI_BINARY_PAYLOAD_VENDORING_EXCLUDED_PACKAGES, }); for (const { packageName, srcDir } of workspaceBundles) { bundleWorkspacePackageWithRuntimeDependencies({ diff --git a/packages/cli-common/src/workspaces/index.test.ts b/packages/cli-common/src/workspaces/index.test.ts index 1040c543b7..fce81e8acd 100644 --- a/packages/cli-common/src/workspaces/index.test.ts +++ b/packages/cli-common/src/workspaces/index.test.ts @@ -268,6 +268,108 @@ describe('bundleWorkspacePackage', () => { 'module.exports = "shared";\n', ); }); + + it('skips vendoring a specific excluded package while still vendoring the rest', async () => { + rootDir = mkdtempSync(join(tmpdir(), 'happier-cli-common-bundle-workspace-')); + + const workspaceModule = await import('./index'); + const vendorBundledPackageRuntimeDependencies = + (workspaceModule as Record).vendorBundledPackageRuntimeDependencies; + expect(vendorBundledPackageRuntimeDependencies).toBeTypeOf('function'); + + const cliDir = resolve(rootDir, 'apps/cli'); + const zodPackageDir = resolve(cliDir, 'node_modules/zod'); + const sharpPackageDir = resolve(cliDir, 'node_modules/sharp'); + mkdirSync(zodPackageDir, { recursive: true }); + mkdirSync(sharpPackageDir, { recursive: true }); + + writeFileSync( + resolve(cliDir, 'package.json'), + JSON.stringify( + { + name: '@happier-dev/cli', + version: '0.0.0', + dependencies: { zod: '4.3.6', sharp: '0.34.3' }, + }, + null, + 2, + ), + ); + writeFileSync( + resolve(zodPackageDir, 'package.json'), + JSON.stringify({ name: 'zod', version: '4.3.6', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(zodPackageDir, 'index.js'), 'module.exports = {};\n'); + writeFileSync( + resolve(sharpPackageDir, 'package.json'), + JSON.stringify({ name: 'sharp', version: '0.34.3', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(sharpPackageDir, 'index.js'), 'module.exports = {};\n'); + + const payloadDir = resolve(rootDir, 'payload'); + + (vendorBundledPackageRuntimeDependencies as (params: { + srcPackageJsonPath: string; + resolveFromPackageJsonPath?: string; + destPackageDir: string; + excludePackageNames?: ReadonlySet; + }) => void)({ + srcPackageJsonPath: resolve(cliDir, 'package.json'), + destPackageDir: payloadDir, + excludePackageNames: new Set(['zod']), + }); + + expect(existsSync(resolve(payloadDir, 'node_modules/zod'))).toBe(false); + expect(readFileSync(resolve(payloadDir, 'node_modules/sharp/index.js'), 'utf8')).toBe( + 'module.exports = {};\n', + ); + }); + + it('vendors every declared dependency when no exclusion set is provided', async () => { + rootDir = mkdtempSync(join(tmpdir(), 'happier-cli-common-bundle-workspace-')); + + const workspaceModule = await import('./index'); + const vendorBundledPackageRuntimeDependencies = + (workspaceModule as Record).vendorBundledPackageRuntimeDependencies; + expect(vendorBundledPackageRuntimeDependencies).toBeTypeOf('function'); + + const cliDir = resolve(rootDir, 'apps/cli'); + const zodPackageDir = resolve(cliDir, 'node_modules/zod'); + mkdirSync(zodPackageDir, { recursive: true }); + + writeFileSync( + resolve(cliDir, 'package.json'), + JSON.stringify( + { + name: '@happier-dev/cli', + version: '0.0.0', + dependencies: { zod: '4.3.6' }, + }, + null, + 2, + ), + ); + writeFileSync( + resolve(zodPackageDir, 'package.json'), + JSON.stringify({ name: 'zod', version: '4.3.6', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(zodPackageDir, 'index.js'), 'module.exports = {};\n'); + + const payloadDir = resolve(rootDir, 'payload'); + + (vendorBundledPackageRuntimeDependencies as (params: { + srcPackageJsonPath: string; + resolveFromPackageJsonPath?: string; + destPackageDir: string; + }) => void)({ + srcPackageJsonPath: resolve(cliDir, 'package.json'), + destPackageDir: payloadDir, + }); + + expect(readFileSync(resolve(payloadDir, 'node_modules/zod/index.js'), 'utf8')).toBe( + 'module.exports = {};\n', + ); + }); }); describe('atomicReplaceDirSync', () => { diff --git a/packages/cli-common/src/workspaces/index.ts b/packages/cli-common/src/workspaces/index.ts index 19962e9ca8..904ca05385 100644 --- a/packages/cli-common/src/workspaces/index.ts +++ b/packages/cli-common/src/workspaces/index.ts @@ -546,9 +546,11 @@ function vendorRuntimeDependencyTree(params: Readonly<{ destNodeModulesDir: string; visited?: Set; dedupeByNameVersion?: Map; + excludePackageNames?: ReadonlySet; }>): void { const pkgJson = readJson(params.packageJsonPath); - const roots = collectExternalRuntimeDepNamesFromPackageJson(pkgJson); + const roots = collectExternalRuntimeDepNamesFromPackageJson(pkgJson) + .filter((dep) => !params.excludePackageNames?.has(dep.name)); const require = createRequire(pathToFileURL(params.resolveFromPackageJsonPath ?? params.packageJsonPath).href); const visited = params.visited ?? new Set(); @@ -599,6 +601,7 @@ function vendorRuntimeDependencyTree(params: Readonly<{ destNodeModulesDir: resolve(depDestDir, 'node_modules'), visited, dedupeByNameVersion, + excludePackageNames: params.excludePackageNames, }); } } @@ -607,6 +610,7 @@ export function vendorBundledPackageRuntimeDependencies(params: Readonly<{ srcPackageJsonPath: string; resolveFromPackageJsonPath?: string; destPackageDir: string; + excludePackageNames?: ReadonlySet; }>): void { if (!existsSync(params.srcPackageJsonPath)) { throw new Error(`Missing package.json: ${params.srcPackageJsonPath}`); @@ -628,6 +632,7 @@ export function vendorBundledPackageRuntimeDependencies(params: Readonly<{ packageJsonPath: params.srcPackageJsonPath, resolveFromPackageJsonPath: params.resolveFromPackageJsonPath, destNodeModulesDir: tempNodeModulesDir, + excludePackageNames: params.excludePackageNames, }); }, }); From ce1378f71f65107532e06924d4f0a084316a001d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Guldmund?= Date: Wed, 5 Aug 2026 09:54:52 +0200 Subject: [PATCH 07/10] Extend compiled-in vendoring exclusion to workspace bundles' own dependencies The compiled-binary payload also vendors apps/cli's internal @happier-dev/* workspace packages via a separate function, bundleWorkspacePackageWithRuntimeDependencies, which never received the excludePackageNames treatment added for apps/cli's own dependencies in the prior commit. Each workspace bundle's own runtime dependencies were therefore still vendored in full even where the bundle's first-party code (and by the same logic, its own dependencies) is already compiled into the binary. Verified per-package, not as a blanket policy -- and specifically re-verified using distinctive exported symbol names rather than package-name strings alone, since a package-name-only search is a proven false-negative trap (@happier-dev/protocol itself showed zero package-name hits in an earlier check, despite its real exports appearing 20 times): - @happier-dev/protocol's own nested @noble/hashes, base64-js, tweetnacl, and zod-to-json-schema: confirmed compiled in via distinctive export/error-string matches, confirmed empirically by deleting them from a real payload and running --version/--help/doctor/status/daemon status/auth request --json (which genuinely exercises a live tweetnacl crypto operation) with no MODULE_NOT_FOUND. - The other 4 bundles (cli-common, agents, release-runtime, connection-supervisor, transfers) declare no external runtime dependencies of their own, so nothing to exclude for them specifically. Found and deliberately did NOT touch a real exception: @happier-dev/cli-common ships root-level (non-dist/) files -- expandHomeDirPath.cjs and others -- that apps/cli/scripts/claude_launcher_runtime.cjs, a real production sidecar spawned as a separate process outside the compiled binary, requires directly by path. Confirmed load-bearing by reproducing the exact MODULE_NOT_FOUND that results from deleting them. These must keep being vendored regardless of what Bun compiles into the main binary process. Also deliberately left apps/cli's own top-level tweetnacl dependency unexcluded in this commit (a known follow-up, already confirmed safe by the same evidence) to keep this diff and the prior commit's exclusion list independently reviewable. Verified via the full local-build installer smoke lifecycle and a real release build, including a direct invocation of the load-bearing sidecar script against the built payload to confirm the kept exception still works: installed: 430MB -> 428MB tarball: 99.2MB -> 98.9MB --- .../buildCliBinaryArtifactPayload.ts | 17 +++ .../cli-common/src/workspaces/index.test.ts | 112 ++++++++++++++++++ packages/cli-common/src/workspaces/index.ts | 2 + 3 files changed, 131 insertions(+) diff --git a/packages/cli-common/src/componentArtifacts/buildCliBinaryArtifactPayload.ts b/packages/cli-common/src/componentArtifacts/buildCliBinaryArtifactPayload.ts index 57d666205a..e4808e86a8 100644 --- a/packages/cli-common/src/componentArtifacts/buildCliBinaryArtifactPayload.ts +++ b/packages/cli-common/src/componentArtifacts/buildCliBinaryArtifactPayload.ts @@ -59,6 +59,22 @@ const CLI_RUNTIME_EXTERNAL_PACKAGES = [ // package vendored. Notably `sharp` is NOT included here: it does a runtime-constructed // `require()` of a platform-specific native `.node` binding, the same reason node-pty is external // above, so it must stay vendored on disk. +// Same rationale and audit trail as CLI_BINARY_PAYLOAD_VENDORING_EXCLUDED_PACKAGES above, but for +// the runtime dependencies declared by apps/cli's own bundled @happier-dev/* workspace packages +// (see bundleWorkspacePackageWithRuntimeDependencies below), rather than apps/cli's own +// dependencies. Confirmed via `strings` on a real compiled binary that these packages' distinctive +// exports (not just an inert package-name string) are compiled in -- e.g. @happier-dev/protocol's +// own nested tweetnacl/@noble/hashes/base64-js/zod-to-json-schema copies -- and that no sidecar or +// dynamic require() reads any of them from disk. See investigate/workspace-bundle-vendoring-audit. +// +// Scoped to the compiled CLI binary payload only, same as CLI_BINARY_PAYLOAD_VENDORING_EXCLUDED_PACKAGES. +const CLI_BINARY_PAYLOAD_WORKSPACE_BUNDLE_VENDORING_EXCLUDED_PACKAGES = new Set([ + '@noble/hashes', + 'base64-js', + 'tweetnacl', + 'zod-to-json-schema', +]); + const CLI_BINARY_PAYLOAD_VENDORING_EXCLUDED_PACKAGES = new Set([ '@agentclientprotocol/sdk', '@anthropic-ai/claude-agent-sdk', @@ -169,6 +185,7 @@ async function copyCliNodeRuntimePayload( packageName, srcDir, destDir: join(payloadDir, 'node_modules', ...packageName.split('/')), + excludePackageNames: CLI_BINARY_PAYLOAD_WORKSPACE_BUNDLE_VENDORING_EXCLUDED_PACKAGES, }); } } diff --git a/packages/cli-common/src/workspaces/index.test.ts b/packages/cli-common/src/workspaces/index.test.ts index fce81e8acd..bdc7cee3de 100644 --- a/packages/cli-common/src/workspaces/index.test.ts +++ b/packages/cli-common/src/workspaces/index.test.ts @@ -132,6 +132,118 @@ describe('bundleWorkspacePackage', () => { ); }); + it('skips vendoring an excluded runtime dependency of a workspace bundle while still vendoring the rest', async () => { + rootDir = mkdtempSync(join(tmpdir(), 'happier-cli-common-bundle-workspace-')); + + const workspaceModule = await import('./index'); + const bundleWorkspacePackageWithRuntimeDependencies = + (workspaceModule as Record).bundleWorkspacePackageWithRuntimeDependencies; + expect(bundleWorkspacePackageWithRuntimeDependencies).toBeTypeOf('function'); + + const srcPackageDir = resolve(rootDir, 'packages/protocol'); + const srcDistDir = resolve(srcPackageDir, 'dist'); + const tweetnaclPackageDir = resolve(srcPackageDir, 'node_modules/tweetnacl'); + const nobleHashesPackageDir = resolve(srcPackageDir, 'node_modules/@noble/hashes'); + mkdirSync(srcDistDir, { recursive: true }); + mkdirSync(tweetnaclPackageDir, { recursive: true }); + mkdirSync(nobleHashesPackageDir, { recursive: true }); + writeFileSync( + resolve(srcPackageDir, 'package.json'), + JSON.stringify( + { + name: '@happier-dev/protocol', + version: '0.0.0', + type: 'module', + exports: { '.': { default: './dist/index.js' } }, + dependencies: { tweetnacl: '1.0.3', '@noble/hashes': '1.8.0' }, + }, + null, + 2, + ), + ); + writeFileSync(resolve(srcDistDir, 'index.js'), 'export {};'); + writeFileSync( + resolve(tweetnaclPackageDir, 'package.json'), + JSON.stringify({ name: 'tweetnacl', version: '1.0.3', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(tweetnaclPackageDir, 'nacl.js'), 'module.exports = {};\n'); + writeFileSync( + resolve(nobleHashesPackageDir, 'package.json'), + JSON.stringify({ name: '@noble/hashes', version: '1.8.0', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(nobleHashesPackageDir, 'sha256.js'), 'module.exports = {};\n'); + + const destPackageDir = resolve(rootDir, 'apps/cli/node_modules/@happier-dev/protocol'); + + (bundleWorkspacePackageWithRuntimeDependencies as (params: { + packageName: string; + srcDir: string; + destDir: string; + excludePackageNames?: ReadonlySet; + }) => void)({ + packageName: '@happier-dev/protocol', + srcDir: srcPackageDir, + destDir: destPackageDir, + excludePackageNames: new Set(['tweetnacl']), + }); + + expect(existsSync(resolve(destPackageDir, 'node_modules/tweetnacl'))).toBe(false); + expect(readFileSync(resolve(destPackageDir, 'node_modules/@noble/hashes/sha256.js'), 'utf8')).toBe( + 'module.exports = {};\n', + ); + }); + + it('vendors every declared runtime dependency of a workspace bundle when no exclusion set is provided', async () => { + rootDir = mkdtempSync(join(tmpdir(), 'happier-cli-common-bundle-workspace-')); + + const workspaceModule = await import('./index'); + const bundleWorkspacePackageWithRuntimeDependencies = + (workspaceModule as Record).bundleWorkspacePackageWithRuntimeDependencies; + expect(bundleWorkspacePackageWithRuntimeDependencies).toBeTypeOf('function'); + + const srcPackageDir = resolve(rootDir, 'packages/protocol'); + const srcDistDir = resolve(srcPackageDir, 'dist'); + const tweetnaclPackageDir = resolve(srcPackageDir, 'node_modules/tweetnacl'); + mkdirSync(srcDistDir, { recursive: true }); + mkdirSync(tweetnaclPackageDir, { recursive: true }); + writeFileSync( + resolve(srcPackageDir, 'package.json'), + JSON.stringify( + { + name: '@happier-dev/protocol', + version: '0.0.0', + type: 'module', + exports: { '.': { default: './dist/index.js' } }, + dependencies: { tweetnacl: '1.0.3' }, + }, + null, + 2, + ), + ); + writeFileSync(resolve(srcDistDir, 'index.js'), 'export {};'); + writeFileSync( + resolve(tweetnaclPackageDir, 'package.json'), + JSON.stringify({ name: 'tweetnacl', version: '1.0.3', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(tweetnaclPackageDir, 'nacl.js'), 'module.exports = {};\n'); + + const destPackageDir = resolve(rootDir, 'apps/cli/node_modules/@happier-dev/protocol'); + + (bundleWorkspacePackageWithRuntimeDependencies as (params: { + packageName: string; + srcDir: string; + destDir: string; + }) => void)({ + packageName: '@happier-dev/protocol', + srcDir: srcPackageDir, + destDir: destPackageDir, + }); + + expect(readFileSync(resolve(destPackageDir, 'node_modules/tweetnacl/nacl.js'), 'utf8')).toBe( + 'module.exports = {};\n', + ); + }); + it('copies non-dist package export targets so the bundled public surface remains loadable', async () => { rootDir = mkdtempSync(join(tmpdir(), 'happier-cli-common-bundle-workspace-')); diff --git a/packages/cli-common/src/workspaces/index.ts b/packages/cli-common/src/workspaces/index.ts index 904ca05385..a061540e8b 100644 --- a/packages/cli-common/src/workspaces/index.ts +++ b/packages/cli-common/src/workspaces/index.ts @@ -394,6 +394,7 @@ export function bundleWorkspacePackageWithRuntimeDependencies(params: Readonly<{ destDir: string; includeFiles?: string[]; resolveFromPackageJsonPath?: string; + excludePackageNames?: ReadonlySet; }>): void { const packageDetails = readWorkspacePackageDetails(params); @@ -411,6 +412,7 @@ export function bundleWorkspacePackageWithRuntimeDependencies(params: Readonly<{ packageJsonPath: packageDetails.srcPackageJsonPath, resolveFromPackageJsonPath: params.resolveFromPackageJsonPath, destNodeModulesDir: resolve(tempDir, 'node_modules'), + excludePackageNames: params.excludePackageNames, }); }, }); From 42181374b31caa71d2da346fc8a84c29cdc158b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Guldmund?= Date: Wed, 5 Aug 2026 10:13:40 +0200 Subject: [PATCH 08/10] Address PR review feedback: Windows symlink fallback, dedup safety, scoping Three fixes from automated review feedback on the open PR: - Greptile: the name@version dedup symlink (added in an earlier commit) used symlinkSync unconditionally. Directory symlinks require elevated privileges or Developer Mode on Windows and can throw EPERM/EACCES on an ordinary build host, which would abort vendoring and artifact production for the Windows release target. Wrapped in try/catch with a real-copy fallback on EPERM/EACCES/ENOSYS. - CodeRabbit: the same dedup logic trusted name@version alone as proof two resolved package directories are identical, with no check that their actual contents match. Added areDirectoryTreesEquivalent, a lightweight (file path + size, not full content hash) structural comparison, and gated the symlink on it -- a mismatch falls through to a normal copy instead of symlinking to the wrong content. Every dedup entry actually added to DUPLICATE_VENDORED_PACKAGE_DIR_PATTERNS in a prior commit was already manually verified byte-identical via diff -rq, so this is a safety net for future/automatic cases, not a fix for an observed bug. - CodeRabbit: prunePackageDistDualFormatDir's package-dist matching in prunePackagedTreeDirectory checked entry.name === 'package-dist' at any depth in the tree walk, not just the actual staged payload root. No vendored package in the current tree happens to ship a directory with that name, so this was latent rather than an observed bug, but a future dependency could collide with it and have its own .cjs files incorrectly stripped. Threaded a stageRootDir param through the recursive walk and scoped the check to only fire when directoryPath === stageRootDir. Added regression tests for all three: a mismatched-content dedup case that must NOT symlink, and a nested same-named package-dist directory that must survive pruning untouched. Verified via the full local-build installer smoke lifecycle against a freshly-compiled binary -- all steps passed. --- .../cli-common/src/workspaces/index.test.ts | 87 +++++++++++++++++++ packages/cli-common/src/workspaces/index.ts | 67 ++++++++++++-- .../pipeline/release/lib/binary-release.mjs | 4 +- ...ry-release.package-dist-cjs-prune.test.mjs | 32 +++++++ 4 files changed, 180 insertions(+), 10 deletions(-) diff --git a/packages/cli-common/src/workspaces/index.test.ts b/packages/cli-common/src/workspaces/index.test.ts index bdc7cee3de..c2298cad17 100644 --- a/packages/cli-common/src/workspaces/index.test.ts +++ b/packages/cli-common/src/workspaces/index.test.ts @@ -381,6 +381,93 @@ describe('bundleWorkspacePackage', () => { ); }); + it('does not dedupe two resolved directories that share name@version but differ in content', async () => { + rootDir = mkdtempSync(join(tmpdir(), 'happier-cli-common-bundle-workspace-')); + + const workspaceModule = await import('./index'); + const bundleWorkspacePackageWithRuntimeDependencies = + (workspaceModule as Record).bundleWorkspacePackageWithRuntimeDependencies; + expect(bundleWorkspacePackageWithRuntimeDependencies).toBeTypeOf('function'); + + // Same diamond shape as the dedup test above, but the nested copy's file content differs from + // the top-level copy despite declaring the identical name@version -- name@version alone must + // not be trusted as proof of equivalence. + const srcPackageDir = resolve(rootDir, 'packages/agents'); + const srcDistDir = resolve(srcPackageDir, 'dist'); + const sharedDepDir = resolve(srcPackageDir, 'node_modules/shared-dep'); + const consumerDepDir = resolve(srcPackageDir, 'node_modules/consumer-dep'); + const nestedSharedDepDir = resolve(consumerDepDir, 'node_modules/shared-dep'); + + mkdirSync(srcDistDir, { recursive: true }); + mkdirSync(sharedDepDir, { recursive: true }); + mkdirSync(consumerDepDir, { recursive: true }); + mkdirSync(nestedSharedDepDir, { recursive: true }); + + writeFileSync( + resolve(srcPackageDir, 'package.json'), + JSON.stringify( + { + name: '@happier-dev/agents', + version: '0.0.0', + type: 'module', + exports: { '.': { default: './dist/index.js' } }, + dependencies: { 'shared-dep': '1.2.3', 'consumer-dep': '1.0.0' }, + }, + null, + 2, + ), + ); + writeFileSync(resolve(srcDistDir, 'index.js'), 'export {};'); + + writeFileSync( + resolve(sharedDepDir, 'package.json'), + JSON.stringify({ name: 'shared-dep', version: '1.2.3', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(sharedDepDir, 'index.js'), 'module.exports = "shared";\n'); + + writeFileSync( + resolve(consumerDepDir, 'package.json'), + JSON.stringify({ name: 'consumer-dep', version: '1.0.0', dependencies: { 'shared-dep': '1.2.3' } }, null, 2), + ); + writeFileSync(resolve(consumerDepDir, 'index.js'), 'module.exports = "consumer";\n'); + + // Declares the same name@version as the top-level copy, but its file content is different + // (longer file, different bytes) -- a mismatched-closure scenario a version string alone + // cannot rule out. + writeFileSync( + resolve(nestedSharedDepDir, 'package.json'), + JSON.stringify({ name: 'shared-dep', version: '1.2.3', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(nestedSharedDepDir, 'index.js'), 'module.exports = "shared-but-actually-different";\n'); + + const destPackageDir = resolve(rootDir, 'apps/cli/node_modules/@happier-dev/agents'); + + (bundleWorkspacePackageWithRuntimeDependencies as (params: { + packageName: string; + srcDir: string; + destDir: string; + }) => void)({ + packageName: '@happier-dev/agents', + srcDir: srcPackageDir, + destDir: destPackageDir, + }); + + const vendoredSharedDepDir = resolve(destPackageDir, 'node_modules/shared-dep'); + const vendoredNestedSharedDepDir = resolve( + destPackageDir, + 'node_modules/consumer-dep/node_modules/shared-dep', + ); + + // Both occurrences must be real, independent directories -- the mismatch must prevent the + // symlink dedup, even though name@version matches. + expect(lstatSync(vendoredSharedDepDir).isSymbolicLink()).toBe(false); + expect(lstatSync(vendoredNestedSharedDepDir).isSymbolicLink()).toBe(false); + expect(readFileSync(resolve(vendoredSharedDepDir, 'index.js'), 'utf8')).toBe('module.exports = "shared";\n'); + expect(readFileSync(resolve(vendoredNestedSharedDepDir, 'index.js'), 'utf8')).toBe( + 'module.exports = "shared-but-actually-different";\n', + ); + }); + it('skips vendoring a specific excluded package while still vendoring the rest', async () => { rootDir = mkdtempSync(join(tmpdir(), 'happier-cli-common-bundle-workspace-')); diff --git a/packages/cli-common/src/workspaces/index.ts b/packages/cli-common/src/workspaces/index.ts index a061540e8b..a3deac0226 100644 --- a/packages/cli-common/src/workspaces/index.ts +++ b/packages/cli-common/src/workspaces/index.ts @@ -1,4 +1,4 @@ -import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs'; import { createRequire } from 'node:module'; import { basename, dirname, relative, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; @@ -542,6 +542,42 @@ function resolveInstalledPackage(params: Readonly<{ require: NodeRequire; packag throw new Error(`Failed to locate installed package.json for ${params.packageName} (resolved: ${resolvedEntry})`); } +// name@version alone doesn't prove two resolved package directories are actually identical -- +// different resolution paths could in principle land on different builds/patches published under +// the same version string. Before symlinking one onto the other, verify the two trees have the +// same relative file paths and byte sizes. This is not a full content hash (that would defeat much +// of the point of skipping a redundant copy for large trees), but it catches the realistic failure +// mode of the two directories actually differing while remaining cheap. +function collectRelativeFileSizes(rootDir: string): Map { + const result = new Map(); + const stack: string[] = [rootDir]; + while (stack.length > 0) { + const dir = stack.pop(); + if (!dir) continue; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const entryPath = resolve(dir, entry.name); + if (entry.isDirectory()) { + stack.push(entryPath); + continue; + } + if (!entry.isFile() && !entry.isSymbolicLink()) continue; + const size = statSync(entryPath).size; + result.set(relative(rootDir, entryPath), size); + } + } + return result; +} + +function areDirectoryTreesEquivalent(dirA: string, dirB: string): boolean { + const filesA = collectRelativeFileSizes(dirA); + const filesB = collectRelativeFileSizes(dirB); + if (filesA.size !== filesB.size) return false; + for (const [relPath, size] of filesA) { + if (filesB.get(relPath) !== size) return false; + } + return true; +} + function vendorRuntimeDependencyTree(params: Readonly<{ packageJsonPath: string; resolveFromPackageJsonPath?: string; @@ -576,12 +612,14 @@ function vendorRuntimeDependencyTree(params: Readonly<{ const dedupeKey = version ? `${dep.name}@${version}` : undefined; const existingDedupePath = dedupeKey ? dedupeByNameVersion.get(dedupeKey) : undefined; - if (existingDedupePath) { - // Already vendored elsewhere in this same tree at the identical name+version. Symlink to - // the surviving copy instead of copying again -- behaviorally identical for any consumer - // (including one that reads from disk directly), since the two source trees are the same - // resolved version. Skip recursing into its subtree: those deps were already vendored under - // the first copy. + // name@version alone doesn't prove the two resolved package directories are actually + // identical (see areDirectoryTreesEquivalent above) -- verify before deduping. If they differ, + // fall through to a normal copy rather than symlinking to the wrong content. + if (existingDedupePath && areDirectoryTreesEquivalent(resolved.packageDir, existingDedupePath)) { + // Already vendored elsewhere in this same tree at the identical name+version, with verified + // matching content. Symlink to the surviving copy instead of copying again -- behaviorally + // identical for any consumer (including one that reads from disk directly). Skip recursing + // into its subtree: those deps were already vendored under the first copy. // // Use a relative link target: both paths currently live inside the same not-yet-renamed // atomic-build staging tree, and the whole tree (staging dir and all its contents, symlink @@ -590,13 +628,24 @@ function vendorRuntimeDependencyTree(params: Readonly<{ // survives the rename because the relationship between the two paths doesn't change. rmDirSafeSync(depDestDir); mkdirSync(dirname(depDestDir), { recursive: true }); - symlinkSync(relative(dirname(depDestDir), existingDedupePath), depDestDir, 'dir'); + try { + symlinkSync(relative(dirname(depDestDir), existingDedupePath), depDestDir, 'dir'); + } catch (error) { + // Directory symlinks require elevated privileges or Developer Mode on Windows and can + // throw EPERM on an ordinary build host. Fall back to a real copy -- larger on disk than + // a symlink, but still avoids re-vendoring this dependency's own transitive tree (we still + // skip the recursive vendorRuntimeDependencyTree call below), and behaves identically for + // any consumer. + const code = error && typeof error === 'object' && 'code' in error ? String(Reflect.get(error, 'code')) : ''; + if (code !== 'EPERM' && code !== 'ENOSYS' && code !== 'EACCES') throw error; + copyDirSafeSync(existingDedupePath, depDestDir, { dereference: true }); + } continue; } resetDir(depDestDir); copyDirSafeSync(resolved.packageDir, depDestDir, { dereference: true }); - if (dedupeKey) dedupeByNameVersion.set(dedupeKey, depDestDir); + if (dedupeKey && !existingDedupePath) dedupeByNameVersion.set(dedupeKey, depDestDir); vendorRuntimeDependencyTree({ packageJsonPath: resolved.packageJsonPath, diff --git a/scripts/pipeline/release/lib/binary-release.mjs b/scripts/pipeline/release/lib/binary-release.mjs index 8ee4f543c1..7fcb5100e0 100644 --- a/scripts/pipeline/release/lib/binary-release.mjs +++ b/scripts/pipeline/release/lib/binary-release.mjs @@ -561,6 +561,7 @@ export async function sanitizePackagedNodeModulesTree(params) { directoryPath: stageDir, target: params.target, inNodeModulesTree: false, + stageRootDir: stageDir, }); } @@ -782,7 +783,7 @@ async function prunePackagedTreeDirectory(params) { continue; } - if (entry.name === 'package-dist') { + if (entry.name === 'package-dist' && params.directoryPath === params.stageRootDir) { await prunePackageDistDualFormatDir({ directoryPath: childPath }); continue; } @@ -830,6 +831,7 @@ async function prunePackagedTreeDirectory(params) { directoryPath: childPath, target: params.target, inNodeModulesTree: childInNodeModulesTree, + stageRootDir: params.stageRootDir, }); } } diff --git a/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs b/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs index 9a9651d256..0c7d2a6b03 100644 --- a/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs +++ b/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs @@ -66,3 +66,35 @@ for (const target of CLI_TARGETS) { } }); } + +// The staged payload's own top-level package-dist/ is the CLI's npm-publish dual-format build +// output (pruned above), but a vendored third-party node_modules package could in principle ship +// its own directory that happens to be named "package-dist" for unrelated reasons. Pruning must be +// scoped to the actual staged root only, not any directory with that name at any depth. +async function buildFakeNestedPackageDistTree(stageDir) { + const pkgDistDir = join(stageDir, 'package-dist'); + await mkdir(pkgDistDir, { recursive: true }); + await writeFile(join(pkgDistDir, 'index.mjs'), 'export default 1;', 'utf-8'); + await writeFile(join(pkgDistDir, 'index.cjs'), 'module.exports = 1;', 'utf-8'); + + const nestedVendoredPkgDistDir = join(stageDir, 'node_modules', 'some-package', 'package-dist'); + await mkdir(nestedVendoredPkgDistDir, { recursive: true }); + await writeFile(join(nestedVendoredPkgDistDir, 'index.mjs'), 'export default 1;', 'utf-8'); + await writeFile(join(nestedVendoredPkgDistDir, 'index.cjs'), 'module.exports = 1;', 'utf-8'); + + return { pkgDistDir, nestedVendoredPkgDistDir }; +} + +test('sanitizePackagedNodeModulesTree only prunes the staged root package-dist, not a same-named nested vendored directory', async () => { + const stageDir = await mkdtemp(join(tmpdir(), 'happier-binary-release-package-dist-nested-')); + try { + const { pkgDistDir, nestedVendoredPkgDistDir } = await buildFakeNestedPackageDistTree(stageDir); + + await sanitizePackagedNodeModulesTree({ stageDir, target: { os: 'darwin', arch: 'arm64' } }); + + assert.deepEqual((await readdir(pkgDistDir)).sort(), ['index.mjs']); + assert.deepEqual((await readdir(nestedVendoredPkgDistDir)).sort(), ['index.cjs', 'index.mjs']); + } finally { + await rm(stageDir, { recursive: true, force: true }); + } +}); From ae3a2aeb969fafb6f3a6bba3c29a2f5063eb02af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Guldmund?= Date: Wed, 5 Aug 2026 16:35:22 +0200 Subject: [PATCH 09/10] Add regression test for different-version dedup (CodeRabbit nitpick) The existing dedup tests covered "same name@version, same content" (dedupes) and "same name@version, different content" (falls through to a copy), but not "same package name, different version" -- a name-only dedup bug would have passed both existing tests undetected. Added a diamond-dependency fixture with shared-dep@1.2.3 and shared-dep@2.0.0 at different nesting depths, asserting both are vendored as independent real directories with no symlink either direction. Declined the review's other nitpick (destructure the typed export instead of casting through Record in the two new test blocks) -- that cast pattern is this file's existing convention from before this PR (present since the file's earliest tests), and changing it selectively for only the newly-added blocks would leave the file inconsistent; a full-file style pass is a separate, unrelated change. --- .../cli-common/src/workspaces/index.test.ts | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/packages/cli-common/src/workspaces/index.test.ts b/packages/cli-common/src/workspaces/index.test.ts index c2298cad17..8c416183f9 100644 --- a/packages/cli-common/src/workspaces/index.test.ts +++ b/packages/cli-common/src/workspaces/index.test.ts @@ -381,6 +381,92 @@ describe('bundleWorkspacePackage', () => { ); }); + it('vendors two different versions of the same package name independently, never deduping by name alone', async () => { + rootDir = mkdtempSync(join(tmpdir(), 'happier-cli-common-bundle-workspace-')); + + const workspaceModule = await import('./index'); + const bundleWorkspacePackageWithRuntimeDependencies = + (workspaceModule as Record).bundleWorkspacePackageWithRuntimeDependencies; + expect(bundleWorkspacePackageWithRuntimeDependencies).toBeTypeOf('function'); + + // Same diamond shape as the name@version dedup test above, but the nested copy declares a + // DIFFERENT version of shared-dep. The dedupeByNameVersion map is keyed by "name@version", so + // this must never even reach the equivalence check -- a name-only key would incorrectly treat + // these as the same dependency. + const srcPackageDir = resolve(rootDir, 'packages/agents'); + const srcDistDir = resolve(srcPackageDir, 'dist'); + const sharedDepDir = resolve(srcPackageDir, 'node_modules/shared-dep'); + const consumerDepDir = resolve(srcPackageDir, 'node_modules/consumer-dep'); + const nestedSharedDepDir = resolve(consumerDepDir, 'node_modules/shared-dep'); + + mkdirSync(srcDistDir, { recursive: true }); + mkdirSync(sharedDepDir, { recursive: true }); + mkdirSync(consumerDepDir, { recursive: true }); + mkdirSync(nestedSharedDepDir, { recursive: true }); + + writeFileSync( + resolve(srcPackageDir, 'package.json'), + JSON.stringify( + { + name: '@happier-dev/agents', + version: '0.0.0', + type: 'module', + exports: { '.': { default: './dist/index.js' } }, + dependencies: { 'shared-dep': '1.2.3', 'consumer-dep': '1.0.0' }, + }, + null, + 2, + ), + ); + writeFileSync(resolve(srcDistDir, 'index.js'), 'export {};'); + + writeFileSync( + resolve(sharedDepDir, 'package.json'), + JSON.stringify({ name: 'shared-dep', version: '1.2.3', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(sharedDepDir, 'index.js'), 'module.exports = "shared-v1";\n'); + + writeFileSync( + resolve(consumerDepDir, 'package.json'), + JSON.stringify({ name: 'consumer-dep', version: '1.0.0', dependencies: { 'shared-dep': '2.0.0' } }, null, 2), + ); + writeFileSync(resolve(consumerDepDir, 'index.js'), 'module.exports = "consumer";\n'); + + // Different version (2.0.0, not 1.2.3) and different content -- must be vendored as its own + // real directory, not symlinked to the v1.2.3 copy. + writeFileSync( + resolve(nestedSharedDepDir, 'package.json'), + JSON.stringify({ name: 'shared-dep', version: '2.0.0', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(nestedSharedDepDir, 'index.js'), 'module.exports = "shared-v2";\n'); + + const destPackageDir = resolve(rootDir, 'apps/cli/node_modules/@happier-dev/agents'); + + (bundleWorkspacePackageWithRuntimeDependencies as (params: { + packageName: string; + srcDir: string; + destDir: string; + }) => void)({ + packageName: '@happier-dev/agents', + srcDir: srcPackageDir, + destDir: destPackageDir, + }); + + const vendoredSharedDepDir = resolve(destPackageDir, 'node_modules/shared-dep'); + const vendoredNestedSharedDepDir = resolve( + destPackageDir, + 'node_modules/consumer-dep/node_modules/shared-dep', + ); + + // Both versions are vendored as real, independent directories -- no symlink either direction. + expect(lstatSync(vendoredSharedDepDir).isSymbolicLink()).toBe(false); + expect(lstatSync(vendoredNestedSharedDepDir).isSymbolicLink()).toBe(false); + expect(readFileSync(resolve(vendoredSharedDepDir, 'index.js'), 'utf8')).toBe('module.exports = "shared-v1";\n'); + expect(readFileSync(resolve(vendoredNestedSharedDepDir, 'index.js'), 'utf8')).toBe( + 'module.exports = "shared-v2";\n', + ); + }); + it('does not dedupe two resolved directories that share name@version but differ in content', async () => { rootDir = mkdtempSync(join(tmpdir(), 'happier-cli-common-bundle-workspace-')); From ce2bcb839175a5775c74b0bb06968ffb3d370134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Guldmund?= Date: Wed, 5 Aug 2026 20:08:13 +0200 Subject: [PATCH 10/10] Address latest CodeRabbit review: dangling symlinks, exact symlink target, test coverage Three more review findings, verified real before fixing: - The dedup content-equivalence check (areDirectoryTreesEquivalent, added in the prior review-feedback commit) called statSync directly while walking a resolved package tree. A dangling symlink -- a realistic node_modules artifact (broken .bin shim, an optional dependency that failed to install) -- throws ENOENT and would abort the entire vendoring run. Wrapped in try/catch, recording NaN as a sentinel size so the entry always compares as a mismatch (NaN !== NaN) and the pair conservatively falls through to a normal copy instead of crashing. Added a dedicated test that reproduces the crash on the old code (confirmed by temporarily reverting the fix and re-running it) and passes once fixed. - The diamond-dependency dedup test asserted the symlink's resolved target only by basename, with a comment explaining the link is captured inside a not-yet-renamed atomic staging tree. That's true of the implementation but irrelevant to what the test observes: by the time bundleWorkspacePackageWithRuntimeDependencies returns, destPackageDir is already the final real path, so the test can and should assert exact path equality. Tightened the assertion and removed the now-unused `basename` import. - Extended the nested-package-dist scoping regression test to also create .d.mts/.d.cts files alongside .cjs, asserting the complete file set survives at a non-root nesting depth, not just the .cjs case. Declined two lower-priority nitpicks in the same review (extracting a shared diamond-fixture test helper, and a broader typed-import refactor across all seven vendoring tests) as explicitly optional/low-value per the review itself and not worth the added indirection or file-wide convention change. Verified via cli-common's vitest suite (15/15), the full packaging pipeline suite (293/293), a clean tsc --noEmit, and the real local-build installer smoke lifecycle -- all steps passed. --- .../cli-common/src/workspaces/index.test.ts | 105 ++++++++++++++++-- packages/cli-common/src/workspaces/index.ts | 10 +- ...ry-release.package-dist-cjs-prune.test.mjs | 7 +- 3 files changed, 113 insertions(+), 9 deletions(-) diff --git a/packages/cli-common/src/workspaces/index.test.ts b/packages/cli-common/src/workspaces/index.test.ts index 8c416183f9..8f3fda585e 100644 --- a/packages/cli-common/src/workspaces/index.test.ts +++ b/packages/cli-common/src/workspaces/index.test.ts @@ -10,10 +10,11 @@ import { readlinkSync, renameSync, rmSync, + symlinkSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; -import { basename, dirname, join, resolve } from 'node:path'; +import { dirname, join, resolve } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; describe('bundleWorkspacePackage', () => { @@ -367,15 +368,15 @@ describe('bundleWorkspacePackage', () => { expect(lstatSync(vendoredSharedDepDir).isSymbolicLink()).toBe(false); expect(readFileSync(resolve(vendoredSharedDepDir, 'index.js'), 'utf8')).toBe('module.exports = "shared";\n'); - // Second occurrence (same name@version) is a symlink, not a full recopy. The link target is - // captured at build time inside the atomically-built staging tree (before the whole - // node_modules dir is renamed into its final place), so assert on the relationship (same - // basename as the surviving vendored copy) and on content equivalence rather than the final - // absolute path. + // Second occurrence (same name@version) is a symlink, not a full recopy. The link is written + // as a relative path (so it survives the atomic staging-dir rename that happens inside + // bundleWorkspacePackageWithRuntimeDependencies), but by the time that function returns, + // destPackageDir is already the final real path -- so resolving the link here points at the + // exact same directory as the surviving copy, not just something with a matching basename. expect(lstatSync(vendoredNestedSharedDepDir).isSymbolicLink()).toBe(true); const linkTarget = readlinkSync(vendoredNestedSharedDepDir); const resolvedLinkTarget = resolve(dirname(vendoredNestedSharedDepDir), linkTarget); - expect(basename(resolvedLinkTarget)).toBe('shared-dep'); + expect(resolvedLinkTarget).toBe(vendoredSharedDepDir); expect(readFileSync(resolve(vendoredNestedSharedDepDir, 'index.js'), 'utf8')).toBe( 'module.exports = "shared";\n', ); @@ -554,6 +555,96 @@ describe('bundleWorkspacePackage', () => { ); }); + it('does not abort vendoring when a resolved package tree contains a dangling symlink', async () => { + rootDir = mkdtempSync(join(tmpdir(), 'happier-cli-common-bundle-workspace-')); + + const workspaceModule = await import('./index'); + const bundleWorkspacePackageWithRuntimeDependencies = + (workspaceModule as Record).bundleWorkspacePackageWithRuntimeDependencies; + expect(bundleWorkspacePackageWithRuntimeDependencies).toBeTypeOf('function'); + + // Same diamond shape again, but the top-level shared-dep tree contains a symlink pointing at + // a path that doesn't exist -- a realistic node_modules artifact (e.g. a broken .bin shim, or + // an optional dependency that failed to install). The equivalence check must tolerate this + // instead of letting a raw statSync ENOENT abort the whole vendoring run. + const srcPackageDir = resolve(rootDir, 'packages/agents'); + const srcDistDir = resolve(srcPackageDir, 'dist'); + const sharedDepDir = resolve(srcPackageDir, 'node_modules/shared-dep'); + const consumerDepDir = resolve(srcPackageDir, 'node_modules/consumer-dep'); + const nestedSharedDepDir = resolve(consumerDepDir, 'node_modules/shared-dep'); + + mkdirSync(srcDistDir, { recursive: true }); + mkdirSync(sharedDepDir, { recursive: true }); + mkdirSync(consumerDepDir, { recursive: true }); + mkdirSync(nestedSharedDepDir, { recursive: true }); + + writeFileSync( + resolve(srcPackageDir, 'package.json'), + JSON.stringify( + { + name: '@happier-dev/agents', + version: '0.0.0', + type: 'module', + exports: { '.': { default: './dist/index.js' } }, + dependencies: { 'shared-dep': '1.2.3', 'consumer-dep': '1.0.0' }, + }, + null, + 2, + ), + ); + writeFileSync(resolve(srcDistDir, 'index.js'), 'export {};'); + + writeFileSync( + resolve(sharedDepDir, 'package.json'), + JSON.stringify({ name: 'shared-dep', version: '1.2.3', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(sharedDepDir, 'index.js'), 'module.exports = "shared";\n'); + symlinkSync(resolve(sharedDepDir, 'does-not-exist'), resolve(sharedDepDir, 'dangling-link')); + + writeFileSync( + resolve(consumerDepDir, 'package.json'), + JSON.stringify({ name: 'consumer-dep', version: '1.0.0', dependencies: { 'shared-dep': '1.2.3' } }, null, 2), + ); + writeFileSync(resolve(consumerDepDir, 'index.js'), 'module.exports = "consumer";\n'); + + writeFileSync( + resolve(nestedSharedDepDir, 'package.json'), + JSON.stringify({ name: 'shared-dep', version: '1.2.3', dependencies: {} }, null, 2), + ); + writeFileSync(resolve(nestedSharedDepDir, 'index.js'), 'module.exports = "shared";\n'); + + const destPackageDir = resolve(rootDir, 'apps/cli/node_modules/@happier-dev/agents'); + + // Must not throw despite the dangling symlink in the top-level copy's tree. + expect(() => + (bundleWorkspacePackageWithRuntimeDependencies as (params: { + packageName: string; + srcDir: string; + destDir: string; + }) => void)({ + packageName: '@happier-dev/agents', + srcDir: srcPackageDir, + destDir: destPackageDir, + }), + ).not.toThrow(); + + const vendoredSharedDepDir = resolve(destPackageDir, 'node_modules/shared-dep'); + const vendoredNestedSharedDepDir = resolve( + destPackageDir, + 'node_modules/consumer-dep/node_modules/shared-dep', + ); + + // The dangling symlink makes the two trees compare as non-equivalent (its size can't be + // determined), so both copies are vendored as real, independent directories rather than + // deduped -- conservative fallback, not a crash. + expect(lstatSync(vendoredSharedDepDir).isSymbolicLink()).toBe(false); + expect(lstatSync(vendoredNestedSharedDepDir).isSymbolicLink()).toBe(false); + expect(readFileSync(resolve(vendoredSharedDepDir, 'index.js'), 'utf8')).toBe('module.exports = "shared";\n'); + expect(readFileSync(resolve(vendoredNestedSharedDepDir, 'index.js'), 'utf8')).toBe( + 'module.exports = "shared";\n', + ); + }); + it('skips vendoring a specific excluded package while still vendoring the rest', async () => { rootDir = mkdtempSync(join(tmpdir(), 'happier-cli-common-bundle-workspace-')); diff --git a/packages/cli-common/src/workspaces/index.ts b/packages/cli-common/src/workspaces/index.ts index a3deac0226..0c2ca1803e 100644 --- a/packages/cli-common/src/workspaces/index.ts +++ b/packages/cli-common/src/workspaces/index.ts @@ -561,7 +561,15 @@ function collectRelativeFileSizes(rootDir: string): Map { continue; } if (!entry.isFile() && !entry.isSymbolicLink()) continue; - const size = statSync(entryPath).size; + // A dangling symlink must not abort vendoring here. Record NaN as a sentinel so this entry + // always compares as a mismatch (NaN !== NaN) -- the pair conservatively falls through to a + // normal copy instead of throwing. + let size: number; + try { + size = statSync(entryPath).size; + } catch { + size = Number.NaN; + } result.set(relative(rootDir, entryPath), size); } } diff --git a/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs b/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs index 0c7d2a6b03..c5f8f63d60 100644 --- a/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs +++ b/scripts/pipeline/release/lib/binary-release.package-dist-cjs-prune.test.mjs @@ -81,6 +81,8 @@ async function buildFakeNestedPackageDistTree(stageDir) { await mkdir(nestedVendoredPkgDistDir, { recursive: true }); await writeFile(join(nestedVendoredPkgDistDir, 'index.mjs'), 'export default 1;', 'utf-8'); await writeFile(join(nestedVendoredPkgDistDir, 'index.cjs'), 'module.exports = 1;', 'utf-8'); + await writeFile(join(nestedVendoredPkgDistDir, 'index.d.mts'), 'export declare const x: number;', 'utf-8'); + await writeFile(join(nestedVendoredPkgDistDir, 'index.d.cts'), 'export declare const x: number;', 'utf-8'); return { pkgDistDir, nestedVendoredPkgDistDir }; } @@ -93,7 +95,10 @@ test('sanitizePackagedNodeModulesTree only prunes the staged root package-dist, await sanitizePackagedNodeModulesTree({ stageDir, target: { os: 'darwin', arch: 'arm64' } }); assert.deepEqual((await readdir(pkgDistDir)).sort(), ['index.mjs']); - assert.deepEqual((await readdir(nestedVendoredPkgDistDir)).sort(), ['index.cjs', 'index.mjs']); + assert.deepEqual( + (await readdir(nestedVendoredPkgDistDir)).sort(), + ['index.cjs', 'index.d.cts', 'index.d.mts', 'index.mjs'], + ); } finally { await rm(stageDir, { recursive: true, force: true }); }