diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ccdb7a18..2903a0563 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,12 @@ jobs: echo "sdk_playground=true" >> "$GITHUB_OUTPUT" fi + if git diff --quiet "${{ github.event.pull_request.base.sha }}" HEAD -- packages/chunking; then + echo "chunking=false" >> "$GITHUB_OUTPUT" + else + echo "chunking=true" >> "$GITHUB_OUTPUT" + fi + - name: Setup Python for SDK Playground if: steps.sdk-changes.outputs.sdk_playground == 'true' uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -90,6 +96,14 @@ jobs: if: steps.sdk-changes.outputs.ai_sdk == 'true' || steps.sdk-changes.outputs.sdk_playground == 'true' run: bun run --cwd packages/ai-sdk build + - name: Run Chunking unit tests + if: steps.sdk-changes.outputs.chunking == 'true' + run: bun run --cwd packages/chunking test + + - name: Run Chunking type checking + if: steps.sdk-changes.outputs.chunking == 'true' + run: bun run --cwd packages/chunking check-types + - name: Run SDK Playground type checking if: steps.sdk-changes.outputs.sdk_playground == 'true' run: bun run --cwd apps/sdk-playground check-types:app diff --git a/bun.lock b/bun.lock index ab8d515d7..9e0565b3f 100644 --- a/bun.lock +++ b/bun.lock @@ -297,6 +297,15 @@ "vitest": "^3.2.4", }, }, + "packages/chunking": { + "name": "@repo/chunking", + "version": "0.0.0", + "devDependencies": { + "@total-typescript/tsconfig": "^1.0.4", + "typescript": "^5.9.2", + "vitest": "^3.2.4", + }, + }, "packages/docs-test": { "name": "docs-test", "version": "1.0.0", @@ -1476,6 +1485,8 @@ "@remix-run/node-fetch-server": ["@remix-run/node-fetch-server@0.13.0", "", {}, "sha512-1EsNo0ZpgXu/90AWoRZf/oE3RVTUS80tiTUpt+hv5pjtAkw7icN4WskDwz/KdAw5ARbJLMhZBrO1NqThmy/McA=="], + "@repo/chunking": ["@repo/chunking@workspace:packages/chunking"], + "@repo/docs": ["@repo/docs@workspace:apps/docs"], "@repo/hooks": ["@repo/hooks@workspace:packages/hooks"], @@ -2978,7 +2989,7 @@ "estree-util-visit": ["estree-util-visit@2.0.0", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/unist": "^3.0.0" } }, "sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww=="], - "estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="], + "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], "esutils": ["esutils@2.0.3", "", {}, "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g=="], @@ -3490,7 +3501,7 @@ "js-tiktoken": ["js-tiktoken@1.0.21", "", { "dependencies": { "base64-js": "^1.5.1" } }, "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g=="], - "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + "js-tokens": ["js-tokens@9.0.1", "", {}, "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ=="], "js-yaml": ["js-yaml@4.1.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA=="], @@ -4710,7 +4721,7 @@ "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], - "tinyexec": ["tinyexec@1.0.2", "", {}, "sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg=="], + "tinyexec": ["tinyexec@0.3.2", "", {}, "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA=="], "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], @@ -5172,6 +5183,8 @@ "@alcalzone/ansi-tokenize/is-fullwidth-code-point": ["is-fullwidth-code-point@5.1.0", "", { "dependencies": { "get-east-asian-width": "^1.3.1" } }, "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ=="], + "@antfu/install-pkg/tinyexec": ["tinyexec@1.0.2", "", {}, "sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg=="], + "@asyncapi/parser/ajv-formats": ["ajv-formats@2.1.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA=="], "@asyncapi/parser/node-fetch": ["node-fetch@2.6.7", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ=="], @@ -5190,6 +5203,8 @@ "@aws-sdk/xml-builder/fast-xml-parser": ["fast-xml-parser@5.4.1", "", { "dependencies": { "fast-xml-builder": "^1.0.0", "strnum": "^2.1.2" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-BQ30U1mKkvXQXXkAGcuyUA/GA26oEB7NzOtsxCDtyu62sjGw5QraKFhx2Em3WQNjPw9PG6MQ9yuIIgkSDfGu5A=="], + "@babel/code-frame/js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + "@babel/core/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], "@babel/generator/@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], @@ -5292,8 +5307,6 @@ "@mastra/core/p-retry": ["p-retry@7.1.1", "", { "dependencies": { "is-network-error": "^1.1.0" } }, "sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w=="], - "@mdx-js/mdx/estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], - "@mdx-js/mdx/remark-rehype": ["remark-rehype@11.1.2", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "mdast-util-to-hast": "^13.0.0", "unified": "^11.0.0", "vfile": "^6.0.0" } }, "sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw=="], "@mdx-js/mdx/source-map": ["source-map@0.7.6", "", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="], @@ -5312,8 +5325,6 @@ "@mintlify/common/acorn": ["acorn@8.11.2", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w=="], - "@mintlify/common/estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], - "@mintlify/common/lodash": ["lodash@4.17.21", "", {}, "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="], "@mintlify/common/mdast-util-gfm": ["mdast-util-gfm@3.0.0", "", { "dependencies": { "mdast-util-from-markdown": "^2.0.0", "mdast-util-gfm-autolink-literal": "^2.0.0", "mdast-util-gfm-footnote": "^2.0.0", "mdast-util-gfm-strikethrough": "^2.0.0", "mdast-util-gfm-table": "^2.0.0", "mdast-util-gfm-task-list-item": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-dgQEX5Amaq+DuUqf26jJqSK9qgixgd6rYDHAv4aTBuA92cTknZlKpPfa86Z/s8Dj8xsAQpFfBmPUHWJBWqS4Bw=="], @@ -5492,6 +5503,8 @@ "@react-router/dev/semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], + "@repo/chunking/typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + "@repo/docs/typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], "@repo/lib/better-auth": ["better-auth@1.3.3", "", { "dependencies": { "@better-auth/utils": "0.2.5", "@better-fetch/fetch": "^1.1.18", "@noble/ciphers": "^0.6.0", "@noble/hashes": "^1.8.0", "@simplewebauthn/browser": "^13.0.0", "@simplewebauthn/server": "^13.0.0", "better-call": "^1.0.12", "defu": "^6.1.4", "jose": "^5.9.6", "kysely": "^0.28.1", "nanostores": "^0.11.3", "zod": "^4.0.5" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-q1aD2nNpGfEI2ckYu+pBjN+23CIRctOpmREkWyJDJdoYW1q9EPs1Xdb+KhFztg2rMmsoUN8I9Xm5mUWMxiWuLw=="], @@ -5504,6 +5517,10 @@ "@rolldown/plugin-babel/picomatch": ["picomatch@4.0.5", "", {}, "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A=="], + "@rollup/plugin-commonjs/estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="], + + "@rollup/pluginutils/estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="], + "@sap-ai-sdk/prompt-registry/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], "@sap-cloud-sdk/connectivity/axios": ["axios@1.13.6", "", { "dependencies": { "follow-redirects": "^1.15.11", "form-data": "^4.0.5", "proxy-from-env": "^1.1.0" } }, "sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ=="], @@ -5592,8 +5609,6 @@ "@vanilla-extract/css/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], - "@vitest/mocker/estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], - "@voltagent/core/@ai-sdk/anthropic": ["@ai-sdk/anthropic@3.0.58", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.19" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-/53SACgmVukO4bkms4dpxpRlYhW8Ct6QZRe6sj1Pi5H00hYhxIrqfiLbZBGxkdRvjsBQeP/4TVGsXgH5rQeb8Q=="], "@voltagent/core/@ai-sdk/cerebras": ["@ai-sdk/cerebras@2.0.41", "", { "dependencies": { "@ai-sdk/openai-compatible": "2.0.37", "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.21" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-kDMEpjaRdRXIUi1EH8WHwLRahyDTYv9SAJnP6VCCeq8X+tVqZbMLCqqxSG5dRknrI65ucjvzQt+FiDKTAa7AHg=="], @@ -5756,8 +5771,6 @@ "eslint/minimatch": ["minimatch@3.1.5", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w=="], - "estree-util-build-jsx/estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], - "estree-util-to-js/source-map": ["source-map@0.7.6", "", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="], "express/cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="], @@ -5838,6 +5851,8 @@ "local-pkg/quansync": ["quansync@0.2.11", "", {}, "sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA=="], + "loose-envify/js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + "markdown-it/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], "mdast-util-find-and-replace/escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="], @@ -5886,6 +5901,8 @@ "npm-run-path/path-key": ["path-key@4.0.0", "", {}, "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ=="], + "nypm/tinyexec": ["tinyexec@1.0.2", "", {}, "sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg=="], + "ollama-ai-provider-v2/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@3.0.22", "", { "dependencies": { "@ai-sdk/provider": "2.0.1", "@standard-schema/spec": "^1.0.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-fFT1KfUUKktfAFm5mClJhS1oux9tP2qgzmEZVl5UdwltQ1LO/s8hd7znVrgKzivwv1s1FIPza0s9OpJaNB/vHw=="], "openai/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="], @@ -6028,8 +6045,6 @@ "strip-ansi-cjs/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], - "strip-literal/js-tokens": ["js-tokens@9.0.1", "", {}, "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ=="], - "sucrase/commander": ["commander@4.1.1", "", {}, "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA=="], "sucrase/lines-and-columns": ["lines-and-columns@1.2.4", "", {}, "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="], @@ -6062,14 +6077,14 @@ "topojson-client/commander": ["commander@2.20.3", "", {}, "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="], + "tsdown/tinyexec": ["tinyexec@1.0.2", "", {}, "sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg=="], + "tsx/esbuild": ["esbuild@0.27.3", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.27.3", "@esbuild/android-arm": "0.27.3", "@esbuild/android-arm64": "0.27.3", "@esbuild/android-x64": "0.27.3", "@esbuild/darwin-arm64": "0.27.3", "@esbuild/darwin-x64": "0.27.3", "@esbuild/freebsd-arm64": "0.27.3", "@esbuild/freebsd-x64": "0.27.3", "@esbuild/linux-arm": "0.27.3", "@esbuild/linux-arm64": "0.27.3", "@esbuild/linux-ia32": "0.27.3", "@esbuild/linux-loong64": "0.27.3", "@esbuild/linux-mips64el": "0.27.3", "@esbuild/linux-ppc64": "0.27.3", "@esbuild/linux-riscv64": "0.27.3", "@esbuild/linux-s390x": "0.27.3", "@esbuild/linux-x64": "0.27.3", "@esbuild/netbsd-arm64": "0.27.3", "@esbuild/netbsd-x64": "0.27.3", "@esbuild/openbsd-arm64": "0.27.3", "@esbuild/openbsd-x64": "0.27.3", "@esbuild/openharmony-arm64": "0.27.3", "@esbuild/sunos-x64": "0.27.3", "@esbuild/win32-arm64": "0.27.3", "@esbuild/win32-ia32": "0.27.3", "@esbuild/win32-x64": "0.27.3" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg=="], "tsyringe/tslib": ["tslib@1.14.1", "", {}, "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="], "unimport/escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="], - "unimport/estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], - "update-notifier/is-in-ci": ["is-in-ci@1.0.0", "", { "bin": { "is-in-ci": "cli.js" } }, "sha512-eUuAjybVTHMYWm/U+vBO1sY/JOCgoPCXRxzdju0K+K0BiGW0SChEL1MLC0PoCIR1OlPo5YAp8HuQoUlsWEICwg=="], "update-notifier/semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], @@ -6080,8 +6095,6 @@ "vite/esbuild": ["esbuild@0.27.3", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.27.3", "@esbuild/android-arm": "0.27.3", "@esbuild/android-arm64": "0.27.3", "@esbuild/android-x64": "0.27.3", "@esbuild/darwin-arm64": "0.27.3", "@esbuild/darwin-x64": "0.27.3", "@esbuild/freebsd-arm64": "0.27.3", "@esbuild/freebsd-x64": "0.27.3", "@esbuild/linux-arm": "0.27.3", "@esbuild/linux-arm64": "0.27.3", "@esbuild/linux-ia32": "0.27.3", "@esbuild/linux-loong64": "0.27.3", "@esbuild/linux-mips64el": "0.27.3", "@esbuild/linux-ppc64": "0.27.3", "@esbuild/linux-riscv64": "0.27.3", "@esbuild/linux-s390x": "0.27.3", "@esbuild/linux-x64": "0.27.3", "@esbuild/netbsd-arm64": "0.27.3", "@esbuild/netbsd-x64": "0.27.3", "@esbuild/openbsd-arm64": "0.27.3", "@esbuild/openbsd-x64": "0.27.3", "@esbuild/openharmony-arm64": "0.27.3", "@esbuild/sunos-x64": "0.27.3", "@esbuild/win32-arm64": "0.27.3", "@esbuild/win32-ia32": "0.27.3", "@esbuild/win32-x64": "0.27.3" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg=="], - "vitest/tinyexec": ["tinyexec@0.3.2", "", {}, "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA=="], - "vscode-languageserver-protocol/vscode-jsonrpc": ["vscode-jsonrpc@8.2.0", "", {}, "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA=="], "web-ext-run/@babel/runtime": ["@babel/runtime@7.28.2", "", {}, "sha512-KHp2IflsnGywDjBWDkR9iEqiWSpc8GIi0lgTT3mOElT0PP1tG26P4tmFI2YvAdzgq9RGyoHZQEIEdZy6Ec5xCA=="], @@ -7264,8 +7277,6 @@ "@mintlify/link-rot/@mintlify/scraping/@mintlify/common/acorn": ["acorn@8.11.2", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w=="], - "@mintlify/link-rot/@mintlify/scraping/@mintlify/common/estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], - "@mintlify/link-rot/@mintlify/scraping/@mintlify/common/lodash": ["lodash@4.17.21", "", {}, "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="], "@mintlify/link-rot/@mintlify/scraping/@mintlify/common/mdast-util-gfm": ["mdast-util-gfm@3.0.0", "", { "dependencies": { "mdast-util-from-markdown": "^2.0.0", "mdast-util-gfm-autolink-literal": "^2.0.0", "mdast-util-gfm-footnote": "^2.0.0", "mdast-util-gfm-strikethrough": "^2.0.0", "mdast-util-gfm-table": "^2.0.0", "mdast-util-gfm-task-list-item": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-dgQEX5Amaq+DuUqf26jJqSK9qgixgd6rYDHAv4aTBuA92cTknZlKpPfa86Z/s8Dj8xsAQpFfBmPUHWJBWqS4Bw=="], diff --git a/packages/chunking/package.json b/packages/chunking/package.json new file mode 100644 index 000000000..50b9d423e --- /dev/null +++ b/packages/chunking/package.json @@ -0,0 +1,20 @@ +{ + "name": "@repo/chunking", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Stable, content-defined text chunking plus content-hash chunk reuse planning for living documents", + "exports": { + "./*": "./*", + "./package.json": "./package.json" + }, + "scripts": { + "check-types": "tsc --noEmit", + "test": "vitest run" + }, + "devDependencies": { + "@total-typescript/tsconfig": "^1.0.4", + "typescript": "^5.9.2", + "vitest": "^3.2.4" + } +} diff --git a/packages/chunking/src/index.ts b/packages/chunking/src/index.ts new file mode 100644 index 000000000..9b9a94295 --- /dev/null +++ b/packages/chunking/src/index.ts @@ -0,0 +1,12 @@ +export { + hashContent, + planChunkReuse, + stableChunkText, +} from "./stable-chunking" +export type { + ChunkLike, + ChunkReuse, + ReusePlan, + StableChunk, + StableChunkOptions, +} from "./stable-chunking" diff --git a/packages/chunking/src/stable-chunking.test.ts b/packages/chunking/src/stable-chunking.test.ts new file mode 100644 index 000000000..09754d38a --- /dev/null +++ b/packages/chunking/src/stable-chunking.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, it } from "vitest" +import { hashContent, planChunkReuse, stableChunkText } from "./stable-chunking" + +/** Deterministic PRNG so the corpus is identical on every run. */ +function mulberry32(seed: number): () => number { + let state = seed >>> 0 + return () => { + state = (state + 0x6d2b79f5) >>> 0 + let t = state + t = Math.imul(t ^ (t >>> 15), t | 1) + t ^= t + Math.imul(t ^ (t >>> 7), t | 61) + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +const WORDS = ( + "amber beacon cedar delta ember frost grove harbor inlet juniper ridge " + + "meadow north ocean prairie quartz river stone timber under vale wheat " + + "yield zephyr ledger motif novel orbit pixel quota relay surge token unity " + + "vivid wander xenon yearn anchor bloom crest dune eddy field glide heath " + + "ivory jolt knoll lumen marsh notch opal plume quill rocky solar tango umber" +).split(" ") + +function pickWord(rng: () => number): string { + return WORDS[Math.floor(rng() * WORDS.length)] ?? "word" +} + +function makeSentence(rng: () => number): string { + const count = 8 + Math.floor(rng() * 9) + const words: string[] = [] + for (let i = 0; i < count; i++) { + words.push(pickWord(rng)) + } + const first = words[0] ?? "word" + const sentence = `${first.charAt(0).toUpperCase()}${first.slice(1)} ${words.slice(1).join(" ")}` + const ender = rng() + return sentence + (ender < 0.85 ? "." : ender < 0.95 ? "?" : "!") +} + +function makeParagraph(rng: () => number, sentences: number): string { + const parts: string[] = [] + for (let i = 0; i < sentences; i++) { + parts.push(makeSentence(rng)) + } + return parts.join(" ") +} + +const MID_FILE_MARKER = + "The quick brown fox marker sentence for mid-file edit testing purposes." + +function buildDocument(): string { + const rng = mulberry32(1649) + const sections: string[] = [`# Reference manual\n\n${makeParagraph(rng, 4)}`] + for (let s = 0; s < 30; s++) { + const paragraphs: string[] = [] + for (let p = 0; p < 5; p++) { + paragraphs.push(makeParagraph(rng, 4)) + } + if (s === 14) { + // Known sentence in the middle of the document for edit tests. + paragraphs[2] = `${makeSentence(rng)} ${MID_FILE_MARKER} ${makeSentence(rng)}` + } + if (s === 20) { + paragraphs.push( + "```ts\nconst alpha = 1\nconst beta = alpha + 2\nconsole.log(beta)\n```", + ) + } + sections.push( + `## Section ${s + 1} operations\n\n${paragraphs.join("\n\n")}`, + ) + } + return `${sections.join("\n\n")}\n` +} + +/** Positional fixed-size chunking: the behavior reported in the issue. */ +function naiveFixedChunks(text: string, size = 500): string[] { + const chunks: string[] = [] + for (let at = 0; at < text.length; at += size) { + chunks.push(text.slice(at, at + size)) + } + return chunks +} + +describe("stableChunkText", () => { + it("is deterministic", () => { + const doc = buildDocument() + expect(stableChunkText(doc)).toEqual(stableChunkText(doc)) + }) + + it("assigns sequential positions and content hashes", () => { + const chunks = stableChunkText(buildDocument()) + expect(chunks.length).toBeGreaterThan(20) + chunks.forEach((chunk, index) => { + expect(chunk.position).toBe(index) + expect(chunk.hash).toBe(hashContent(chunk.content)) + }) + }) + + it("handles edge cases without throwing", () => { + expect(stableChunkText("")).toEqual([]) + expect(stableChunkText(" \n\n ")).toEqual([]) + expect(stableChunkText("Single sentence here.")).toHaveLength(1) + }) + + it("keeps fenced code lines intact", () => { + const chunks = stableChunkText(buildDocument()) + for (const line of ["const alpha = 1", "const beta = alpha + 2"]) { + expect(chunks.some((chunk) => chunk.content.includes(line))).toBe(true) + } + }) + + it("cuts prose on sentence boundaries", () => { + const prose = `${makeParagraph(mulberry32(7), 40)}\n\n${makeParagraph(mulberry32(8), 40)}` + for (const chunk of stableChunkText(prose)) { + expect(chunk.content).toMatch(/[.!?…]["'”’)\]]?$/) + } + }) +}) + +describe("issue #1649: mid-file edits", () => { + it("only invalidates local chunks for a mid-file sentence replacement", () => { + const doc = buildDocument() + const before = stableChunkText(doc) + const edited = doc.replace( + MID_FILE_MARKER, + "A completely rewritten sentence about zebra migration patterns across the southern valley region.", + ) + const after = stableChunkText(edited) + const plan = planChunkReuse(before, after) + + expect(plan.reusedCount).toBeGreaterThanOrEqual(before.length - 4) + expect(plan.embedCount).toBeLessThanOrEqual(6) + expect(plan.reusedCount / before.length).toBeGreaterThan(0.9) + }) + + it("only invalidates local chunks for a mid-file paragraph insertion", () => { + const doc = buildDocument() + const before = stableChunkText(doc) + const insertion = `\n\n${makeParagraph(mulberry32(99), 4)}\n\n` + const edited = doc.replace( + MID_FILE_MARKER, + `${MID_FILE_MARKER}${insertion}`, + ) + const after = stableChunkText(edited) + const plan = planChunkReuse(before, after) + + // New content must embed, but every untouched old chunk is reused. + expect(plan.reusedCount).toBeGreaterThanOrEqual(before.length - 4) + expect(plan.embedCount).toBeLessThanOrEqual(8) + }) + + it("reproduces the bug with naive fixed-size chunking", () => { + const doc = buildDocument() + const edited = doc.replace( + MID_FILE_MARKER, + "A completely rewritten sentence about zebra migration patterns across the southern valley region.", + ) + const before = naiveFixedChunks(doc) + const after = naiveFixedChunks(edited) + const plan = planChunkReuse(before, after) + + // The downstream boundary shift defeats hash reuse for most chunks. + expect(plan.reusedCount).toBeLessThan(before.length * 0.5) + expect(plan.embedCount).toBeGreaterThan(before.length * 0.5) + }) + + it("reuses everything on append-only edits", () => { + const doc = buildDocument() + const before = stableChunkText(doc) + const appended = `${doc}\n\n## Section 31 appendix\n\n${makeParagraph(mulberry32(100), 6)}\n` + const after = stableChunkText(appended) + const plan = planChunkReuse(before, after) + + expect(plan.reusedCount).toBe(before.length) + expect(plan.embedCount).toBeLessThanOrEqual(5) + }) + + it("embeds nothing new when a section is deleted", () => { + const doc = buildDocument() + const before = stableChunkText(doc) + const sectionStart = doc.indexOf("## Section 15 operations") + const sectionEnd = doc.indexOf("## Section 16 operations") + const edited = doc.slice(0, sectionStart) + doc.slice(sectionEnd) + const after = stableChunkText(edited) + const plan = planChunkReuse(before, after) + + expect(plan.embedCount).toBe(0) + expect(plan.reusedCount).toBe(after.length) + }) +}) + +describe("planChunkReuse", () => { + it("matches by content hash, not position", () => { + const plan = planChunkReuse(["a", "b", "c"], ["x", "b", "c", "a"]) + expect(plan.embedIndices).toEqual([0]) + expect(plan.reusedCount).toBe(3) + }) + + it("pairs duplicates FIFO so each embedding is reused once", () => { + const plan = planChunkReuse(["a", "b", "a"], ["a", "a", "b"]) + expect(plan.embedCount).toBe(0) + expect(plan.reusedCount).toBe(3) + const reusedOld = plan.reused.map((entry) => entry.oldIndex).sort() + expect(reusedOld).toEqual([0, 1, 2]) + }) + + it("accepts chunk objects with precomputed hashes", () => { + const oldChunks = stableChunkText(buildDocument()) + const plan = planChunkReuse(oldChunks, oldChunks) + expect(plan.embedCount).toBe(0) + expect(plan.reusedCount).toBe(oldChunks.length) + }) +}) + +describe("hashContent", () => { + it("is stable, 16 hex chars, and sensitive to changes", () => { + expect(hashContent("hello")).toBe(hashContent("hello")) + expect(hashContent("hello")).toMatch(/^[0-9a-f]{16}$/) + expect(hashContent("hello")).not.toBe(hashContent("hello!")) + }) +}) diff --git a/packages/chunking/src/stable-chunking.ts b/packages/chunking/src/stable-chunking.ts new file mode 100644 index 000000000..6fe2c5d25 --- /dev/null +++ b/packages/chunking/src/stable-chunking.ts @@ -0,0 +1,445 @@ +/** + * Stable, content-defined chunking for living documents. + * + * Motivation (supermemoryai/supermemory#1649): fixed-position chunking turns a + * one-paragraph mid-file edit into a full-document re-embed, because every + * downstream chunk boundary shifts and per-chunk content-hash reuse never hits. + * This module chunks so that an edit only invalidates the chunk(s) it touches: + * + * - Structural resync points: markdown headings and fenced code blocks are + * never merged across, so an edit in one section cannot shift boundaries in + * another section. + * - Content-defined cuts: within a section, boundaries fall on sentence (or + * code-line) units whose cut decision depends on the unit's own content hash + * (FastCDC-style anchoring), so boundaries re-synchronize a few units after + * an edit instead of shifting to the end of the document. + * + * Pair with {@link planChunkReuse} on the document upsert path: compare the + * new chunk list against the stored chunks by content hash (not by position) + * and embed only {@link ReusePlan.embedIndices}. + * + * Zero dependencies and pure TypeScript so it runs in Node, Bun, and + * Cloudflare Workers (no `node:crypto`, no async hashing). + */ + +export interface StableChunkOptions { + /** + * Cut after a unit whose hash satisfies `hash % anchorModulo === 0`. + * Lower values produce smaller, more numerous chunks. Defaults to 8 + * (averages ~8 sentences per chunk). + */ + anchorModulo?: number + /** Minimum sentences (or code lines) per chunk. Defaults to 2. */ + minSentences?: number + /** Maximum sentences (or code lines) per chunk. Defaults to 16. */ + maxSentences?: number + /** Hard cap on chunk characters; oversized sentences are word-split. Defaults to 4000. */ + maxChars?: number + /** + * Trailing sentences of the previous chunk repeated for context. + * Matches the documented 2-sentence overlap. Overlap is restricted to the + * same section so sections stay independent. Defaults to 2. + */ + overlapSentences?: number +} + +export interface StableChunk { + /** Final chunk text, prefixed with its section heading when present. */ + content: string + /** 64-bit FNV-1a hex digest of `content`; the reuse/identity key. */ + hash: string + /** Markdown heading this chunk belongs to, or null for lead-in content. */ + heading: string | null + /** Sequential position within the document. */ + position: number +} + +const DEFAULT_ANCHOR_MODULO = 8 +const DEFAULT_MIN_SENTENCES = 2 +const DEFAULT_MAX_SENTENCES = 16 +const DEFAULT_MAX_CHARS = 4000 +const DEFAULT_OVERLAP_SENTENCES = 2 + +/** 64-bit FNV-1a via two 32-bit lanes. Non-cryptographic; for change detection only. */ +export function hashContent(text: string): string { + let h1 = 0x811c9dc5 + let h2 = 0x811c9dc5 ^ 0x9e3779b9 + for (let i = 0; i < text.length; i++) { + const code = text.charCodeAt(i) + h1 = Math.imul(h1 ^ code, 0x01000193) + h2 = Math.imul(h2 ^ code, 0x01000193) + } + return ( + (h1 >>> 0).toString(16).padStart(8, "0") + + (h2 >>> 0).toString(16).padStart(8, "0") + ) +} + +function fnv1a32(text: string): number { + let h = 0x811c9dc5 + for (let i = 0; i < text.length; i++) { + h = Math.imul(h ^ text.charCodeAt(i), 0x01000193) + } + return h >>> 0 +} + +interface Section { + heading: string | null + body: string + code: boolean +} + +function isFenceStart(line: string): boolean { + const trimmed = line.trimStart() + return trimmed.startsWith("```") || trimmed.startsWith("~~~") +} + +function isHeading(line: string): boolean { + return /^#{1,6}(?:\s|$)/.test(line.trimStart()) +} + +/** + * Split markdown into independent sections. Headings open a new section and + * fenced code blocks become their own sections so an edit inside one region + * can never shift chunk boundaries in another. + */ +function splitSections(markdown: string): Section[] { + const sections: Section[] = [] + const lines = markdown.replace(/\r\n?/g, "\n").split("\n") + let heading: string | null = null + let prose: string[] = [] + let fence: string[] | null = null + + const flushProse = () => { + const body = prose.join("\n").trim() + prose = [] + if (body.length > 0) { + sections.push({ heading, body, code: false }) + } + } + + for (const line of lines) { + if (fence !== null) { + fence.push(line) + if (isFenceStart(line)) { + sections.push({ heading, body: fence.join("\n"), code: true }) + fence = null + } + continue + } + if (isFenceStart(line)) { + flushProse() + fence = [line] + continue + } + if (isHeading(line)) { + flushProse() + heading = line.trim() + continue + } + prose.push(line) + } + if (fence !== null) { + // Unclosed fence: keep the lines rather than dropping content. + sections.push({ heading, body: fence.join("\n"), code: true }) + } + flushProse() + return sections +} + +function isDigit(char: string | undefined): boolean { + return char !== undefined && char >= "0" && char <= "9" +} + +const CLOSERS = new Set(['"', "'", "”", "’", ")", "]"]) + +/** + * Split a paragraph into sentences. Tuned for determinism and stability (same + * input always yields the same splits; an edit only affects nearby splits), + * not linguistic perfection. + */ +function splitSentences(paragraph: string): string[] { + const clean = paragraph.replace(/\s+/g, " ").trim() + if (clean.length === 0) { + return [] + } + const sentences: string[] = [] + let start = 0 + let i = 0 + while (i < clean.length) { + const char = clean[i] + if (char !== "." && char !== "!" && char !== "?" && char !== "…") { + i++ + continue + } + const prev = i > 0 ? clean[i - 1] : undefined + const next = i + 1 < clean.length ? clean[i + 1] : undefined + // Skip decimals ("3.14") and ellipsis runs ("..."). + if (char === "." && isDigit(prev) && isDigit(next)) { + i++ + continue + } + if (char === "." && next === ".") { + i++ + continue + } + // Consume closing quotes/brackets, then require whitespace or end. + let end = i + 1 + while (end < clean.length && CLOSERS.has(clean[end] ?? "")) { + end++ + } + if (end < clean.length && clean[end] !== " ") { + i++ + continue + } + const sentence = clean.slice(start, end).trim() + if (sentence.length > 0) { + sentences.push(sentence) + } + let nextStart = end + while (nextStart < clean.length && clean[nextStart] === " ") { + nextStart++ + } + start = nextStart + i = nextStart + } + const tail = clean.slice(start).trim() + if (tail.length > 0) { + sentences.push(tail) + } + return sentences +} + +/** Word-split an oversized unit so no chunk exceeds `maxChars`. */ +function splitLongText(text: string, maxChars: number): string[] { + if (text.length <= maxChars) { + return [text] + } + const pieces: string[] = [] + let current = "" + for (const word of text.split(/\s+/)) { + if (word.length === 0) { + continue + } + if (word.length > maxChars) { + if (current.length > 0) { + pieces.push(current) + current = "" + } + for (let at = 0; at < word.length; at += maxChars) { + pieces.push(word.slice(at, at + maxChars)) + } + continue + } + const candidate = current.length === 0 ? word : `${current} ${word}` + if (candidate.length > maxChars) { + pieces.push(current) + current = word + } else { + current = candidate + } + } + if (current.length > 0) { + pieces.push(current) + } + return pieces.length > 0 ? pieces : [text] +} + +interface Unit { + text: string + spaceBefore: string +} + +function unitsForSection(section: Section, maxChars: number): Unit[] { + const units: Unit[] = [] + const push = (text: string, spaceBefore: string) => { + for (const piece of splitLongText(text, maxChars)) { + units.push({ + text: piece, + spaceBefore: units.length === 0 ? "" : spaceBefore, + }) + } + } + if (section.code) { + for (const line of section.body.split("\n")) { + const text = line.trimEnd() + if (text.trim().length === 0) { + continue + } + push(text, "\n") + } + return units + } + const paragraphs = section.body + .split(/\n\s*\n/) + .map((paragraph) => paragraph.trim()) + .filter((paragraph) => paragraph.length > 0) + for (const paragraph of paragraphs) { + let firstInParagraph = true + for (const sentence of splitSentences(paragraph)) { + push(sentence, firstInParagraph ? "\n\n" : " ") + firstInParagraph = false + } + } + return units +} + +function renderChunk( + heading: string | null, + overlap: Unit[], + units: Unit[], +): string { + let content = heading !== null ? `${heading}\n\n` : "" + const all = [...overlap, ...units] + all.forEach((unit, index) => { + content += `${index === 0 ? "" : unit.spaceBefore}${unit.text}` + }) + return content +} + +/** + * Chunk markdown into stable, content-defined pieces. + * + * Boundaries always fall on sentence (prose) or line (code) edges and never + * cross a heading or fenced code block, so a mid-file edit only changes the + * chunk(s) covering the edit. Re-chunking an edited document yields + * byte-identical content for every untouched region, letting + * {@link planChunkReuse} skip re-embedding them. + */ +export function stableChunkText( + markdown: string, + options: StableChunkOptions = {}, +): StableChunk[] { + const anchorModulo = Math.max( + 1, + options.anchorModulo ?? DEFAULT_ANCHOR_MODULO, + ) + const minSentences = Math.max( + 1, + options.minSentences ?? DEFAULT_MIN_SENTENCES, + ) + const maxSentences = Math.max( + minSentences, + options.maxSentences ?? DEFAULT_MAX_SENTENCES, + ) + const maxChars = Math.max(256, options.maxChars ?? DEFAULT_MAX_CHARS) + const overlapSentences = Math.max( + 0, + options.overlapSentences ?? DEFAULT_OVERLAP_SENTENCES, + ) + + const chunks: StableChunk[] = [] + for (const section of splitSections(markdown)) { + const units = unitsForSection(section, maxChars) + if (units.length === 0) { + continue + } + const runs: Unit[][] = [] + let run: Unit[] = [] + let runChars = 0 + let unitIndex = 0 + for (const unit of units) { + const isLast = unitIndex === units.length - 1 + unitIndex++ + run.push(unit) + runChars += unit.text.length + const anchor = fnv1a32(unit.text) % anchorModulo === 0 + if ( + !isLast && + run.length >= minSentences && + (run.length >= maxSentences || runChars >= maxChars || anchor) + ) { + runs.push(run) + run = [] + runChars = 0 + } + } + if (run.length > 0) { + runs.push(run) + } + let previousRun: Unit[] = [] + for (const currentRun of runs) { + const overlap = + previousRun.length === 0 ? [] : previousRun.slice(-overlapSentences) + const content = renderChunk(section.heading, overlap, currentRun) + chunks.push({ + content, + hash: hashContent(content), + heading: section.heading, + position: chunks.length, + }) + previousRun = currentRun + } + } + return chunks +} + +export type ChunkLike = + | string + | { content: string; hash?: string } + | { hash: string } + +function chunkHash(chunk: ChunkLike): string { + if (typeof chunk === "string") { + return hashContent(chunk) + } + if ("hash" in chunk && typeof chunk.hash === "string") { + return chunk.hash + } + return hashContent((chunk as { content: string }).content) +} + +export interface ChunkReuse { + newIndex: number + oldIndex: number +} + +export interface ReusePlan { + /** New chunks whose content already exists; carry over the old embedding. */ + reused: ChunkReuse[] + /** Indices into the NEW chunk list that actually need embedding. */ + embedIndices: number[] + reusedCount: number + embedCount: number +} + +/** + * Anchor-based resync between an old and a new chunk list. + * + * Matches by content hash instead of position, so a mid-file insertion or + * deletion no longer invalidates every downstream chunk: unchanged chunks hit + * regardless of where they moved. Duplicate contents are paired FIFO so each + * stored embedding is reused at most once. Runs in O(old + new). + */ +export function planChunkReuse( + oldChunks: ChunkLike[], + newChunks: ChunkLike[], +): ReusePlan { + const available = new Map() + oldChunks.forEach((chunk, oldIndex) => { + const hash = chunkHash(chunk) + const queue = available.get(hash) + if (queue !== undefined) { + queue.push(oldIndex) + } else { + available.set(hash, [oldIndex]) + } + }) + const reused: ChunkReuse[] = [] + const embedIndices: number[] = [] + newChunks.forEach((chunk, newIndex) => { + const queue = available.get(chunkHash(chunk)) + const oldIndex = queue?.shift() + if (oldIndex === undefined) { + embedIndices.push(newIndex) + } else { + reused.push({ newIndex, oldIndex }) + } + }) + return { + reused, + embedIndices, + reusedCount: reused.length, + embedCount: embedIndices.length, + } +} diff --git a/packages/chunking/tsconfig.json b/packages/chunking/tsconfig.json new file mode 100644 index 000000000..d0c8742bb --- /dev/null +++ b/packages/chunking/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "@total-typescript/tsconfig/bundler/dom/library-monorepo", + "compilerOptions": { + "baseUrl": "." + } +}