diff --git a/.changeset/deterministic-threads.md b/.changeset/deterministic-threads.md new file mode 100644 index 0000000..9acd3a2 --- /dev/null +++ b/.changeset/deterministic-threads.md @@ -0,0 +1,25 @@ +--- +'@platforma-open/milaboratories.sequence-properties.software': patch +'@platforma-open/milaboratories.sequence-properties.workflow': patch +--- + +Parallelize the compute step and size its resources to the input, keeping output byte-identical. + +Memory now scales with the input: `memFormula(clamp(max(lineCount * perRow, size * 64), 2GiB, 64GiB))` replaces the flat `mem("12GiB")`. The two terms cover two cost sources — a row-scaled term for per-clone structures (count matrix, output columns, aa_fraction; measured ~3 KiB/row peptide, ~4.3 KiB/row antibody), and a residue-scaled term (`size` ≈ total residues) for the O(residues) cleaning transients. The size term keeps long-sequence/low-row inputs (amplicon variants) off the 2 GiB floor, where a `lineCount`-only formula would OOM them. Large datasets get the RAM they need; small ones stop over-reserving. A `fallback: "12GiB"` preserves the old behavior on backends that cannot evaluate formulas. + +CPU scales with the input too: both modes request up to 4 cores (`cpuFormula`), matching their measured thread speedups (peptide ~1.7x, antibody ~1.5x). `POLARS_MAX_THREADS` — which sizes both the polars pool and the numpy pI-bisection workers — takes the cores the backend actually grants, via the `{system.cpu}` command expression, so a sub-cap allocation never oversubscribes its quota. A static `env` covers backends without command expressions. + +Determinism holds regardless of core count. Polars parallelizes only order-stable work (CSV read, chain reconstruction, aa_fraction reshape, the unique-key sort, CSV write), and `main.py` pins the BLAS/OpenMP intra-op threads to 1 so the one order-sensitive step — the `counts @ weights` reduction — never splits across threads. The emitted bytes, and the resource CID, stay identical. + +Compute-engine optimizations, all output-preserving: + +- `_charge_raw` computes `10**ph` once and reuses it via scalar factors instead of a per-amino-acid `10**(ph±pk)` — ~2.4x faster on the charge/pI path (FP drift ~1e-15, absorbed by the 3-dp quantization). +- Peptide mode cleans its column once, sharing the intermediate between the count substrate and the instability index instead of cleaning twice. +- The count and instability scatters use `np.bincount` instead of the slower unbuffered `np.add.at` — bit-identical, ~3x faster. +- The per-chain median-CDR3-length stat uses a vectorized polars `count_matches` instead of one Python `effective_length` call per row. +- Peptide `aa_fraction` uses a polars unpivot instead of a 20×N Python-list explosion, cutting peak memory on the peptide path. +- The pI bisection (`isoelectric_point` / `fv_isoelectric_point`) runs row-parallel across threads — contiguous, data-independent blocks, each depending only on its own rows, so the result is bit-identical regardless of worker count. + +Together these cut peptide compute ~2.3x and make antibody scale from ~1.1x to ~1.5x on 4 cores (10.0s → 4.6s at 1M rows). + +Output stays within the quantized-equal contract, verified byte-for-byte between 1 and 4 threads by an extended determinism test and against the pre-change implementation (peptide and antibody, including edge cases). diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 89676eb..442ba20 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -25,26 +25,26 @@ catalogs: specifier: 1.8.2 version: 1.8.2 '@platforma-sdk/block-tools': - specifier: 2.11.6 - version: 2.11.6 + specifier: 2.11.9 + version: 2.11.9 '@platforma-sdk/eslint-config': specifier: 1.2.0 version: 1.2.0 '@platforma-sdk/model': - specifier: 1.79.20 - version: 1.79.20 + specifier: 1.79.24 + version: 1.79.24 '@platforma-sdk/package-builder': specifier: 3.14.0 version: 3.14.0 '@platforma-sdk/tengo-builder': - specifier: 4.0.11 - version: 4.0.11 + specifier: 4.0.14 + version: 4.0.14 '@platforma-sdk/test': - specifier: 1.79.23 - version: 1.79.23 + specifier: 1.79.26 + version: 1.79.26 '@platforma-sdk/ui-vue': - specifier: 1.79.20 - version: 1.79.20 + specifier: 1.79.24 + version: 1.79.24 '@platforma-sdk/workflow-tengo': specifier: 6.6.5 version: 6.6.5 @@ -86,7 +86,7 @@ importers: version: 1.6.0(@types/node@25.3.2)(rollup@4.53.3)(vue@3.5.25(typescript@5.9.3))(yaml@2.8.1) '@platforma-sdk/block-tools': specifier: 'catalog:' - version: 2.11.6(@types/node@25.3.2) + version: 2.11.9(@types/node@25.3.2) shx: specifier: 'catalog:' version: 0.4.0 @@ -107,23 +107,23 @@ importers: version: link:../workflow '@platforma-sdk/model': specifier: 'catalog:' - version: 1.79.20 + version: 1.79.24 devDependencies: '@platforma-sdk/block-tools': specifier: 'catalog:' - version: 2.11.6(@types/node@25.3.2) + version: 2.11.9(@types/node@25.3.2) model: dependencies: '@milaboratories/graph-maker': specifier: 'catalog:' - version: 1.4.8(@milaboratories/pl-model-common@1.46.2)(@platforma-sdk/model@1.79.20)(@platforma-sdk/ui-vue@1.79.20(@bytecodealliance/preview2-shim@0.17.8)(typescript@5.6.3))(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0)(typescript@5.6.3) + version: 1.4.8(@milaboratories/pl-model-common@1.46.3)(@platforma-sdk/model@1.79.24)(@platforma-sdk/ui-vue@1.79.24(@bytecodealliance/preview2-shim@0.17.8)(typescript@5.6.3))(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0)(typescript@5.6.3) '@milaboratories/helpers': specifier: 'catalog:' version: 1.14.2 '@platforma-sdk/model': specifier: 'catalog:' - version: 1.79.20 + version: 1.79.24 '@types/node': specifier: '*' version: 25.3.2 @@ -139,7 +139,7 @@ importers: version: 1.3.0 '@platforma-sdk/block-tools': specifier: 'catalog:' - version: 2.11.6(@types/node@25.3.2) + version: 2.11.9(@types/node@25.3.2) vitest: specifier: 'catalog:' version: 4.1.4(@types/node@25.3.2)(@vitest/coverage-istanbul@4.1.4)(vite@8.0.8(@types/node@25.3.2)(yaml@2.8.1)) @@ -175,7 +175,7 @@ importers: version: 1.0.4 '@platforma-sdk/model': specifier: 'catalog:' - version: 1.79.20 + version: 1.79.24 this-block: specifier: workspace:@platforma-open/milaboratories.sequence-properties@* version: link:../block @@ -191,7 +191,7 @@ importers: version: 1.2.0(@eslint/js@9.39.4)(@stylistic/eslint-plugin@2.13.0(eslint@9.39.4)(typescript@5.6.3))(eslint-plugin-n@17.24.0(eslint@9.39.4)(typescript@5.6.3))(eslint-plugin-vue@9.33.0(eslint@9.39.4))(eslint@9.39.4)(globals@15.15.0)(typescript-eslint@8.59.1(eslint@9.39.4)(typescript@5.6.3))(typescript@5.6.3) '@platforma-sdk/test': specifier: 'catalog:' - version: 1.79.23(@bytecodealliance/preview2-shim@0.17.8)(@types/node@25.3.2)(vite@8.0.8(@types/node@25.3.2)(yaml@2.8.1)) + version: 1.79.26(@bytecodealliance/preview2-shim@0.17.8)(@types/node@25.3.2)(vite@8.0.8(@types/node@25.3.2)(yaml@2.8.1)) eslint: specifier: 'catalog:' version: 9.39.4 @@ -206,16 +206,16 @@ importers: dependencies: '@milaboratories/graph-maker': specifier: 'catalog:' - version: 1.4.8(@milaboratories/pl-model-common@1.46.2)(@platforma-sdk/model@1.79.20)(@platforma-sdk/ui-vue@1.79.20(@bytecodealliance/preview2-shim@0.17.8)(typescript@5.6.3))(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0)(typescript@5.6.3) + version: 1.4.8(@milaboratories/pl-model-common@1.46.3)(@platforma-sdk/model@1.79.24)(@platforma-sdk/ui-vue@1.79.24(@bytecodealliance/preview2-shim@0.17.8)(typescript@5.6.3))(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0)(typescript@5.6.3) '@platforma-open/milaboratories.sequence-properties.model': specifier: workspace:* version: link:../model '@platforma-sdk/model': specifier: 'catalog:' - version: 1.79.20 + version: 1.79.24 '@platforma-sdk/ui-vue': specifier: 'catalog:' - version: 1.79.20(@bytecodealliance/preview2-shim@0.17.8)(typescript@5.6.3) + version: 1.79.24(@bytecodealliance/preview2-shim@0.17.8)(typescript@5.6.3) typescript: specifier: '*' version: 5.6.3 @@ -250,10 +250,10 @@ importers: devDependencies: '@platforma-sdk/tengo-builder': specifier: 'catalog:' - version: 4.0.11 + version: 4.0.14 '@platforma-sdk/test': specifier: 'catalog:' - version: 1.79.23(@bytecodealliance/preview2-shim@0.17.8)(@types/node@25.3.2)(vite@8.0.8(@types/node@25.3.2)(yaml@2.8.1)) + version: 1.79.26(@bytecodealliance/preview2-shim@0.17.8)(@types/node@25.3.2)(vite@8.0.8(@types/node@25.3.2)(yaml@2.8.1)) packages: @@ -1154,8 +1154,8 @@ packages: '@milaboratories/miplots4@1.2.3': resolution: {integrity: sha512-rhoF7iRcJWKUqJMQq8y84vlkqZn4Yk7tOV8Zef+W+GqVt9UYgV1XLJu/+Wp1GcsUVtPP8Xs5CRgfA/YfV+a4Hg==} - '@milaboratories/pf-driver@1.7.14': - resolution: {integrity: sha512-JRuMmuhObyD27MtpCcimGklr2yMJ2PobcYJ2y5k+UuhQWXKllhEDlUAyv2Xw/I7Xxym8Nn2npcVdOaJvTP3bPA==} + '@milaboratories/pf-driver@1.7.15': + resolution: {integrity: sha512-55/8fs5q+hmqW1qQIlE4MYujDh84YaAIbTxPhG0NTn5dsOk/t9vCJuDgL3ztdlUbX5pDhI+4sg+AppVg1CgJng==} engines: {node: '>=22.19.0'} '@milaboratories/pf-plots@1.4.6': @@ -1167,8 +1167,8 @@ packages: '@milaboratories/pf-spec-driver@1.2.4': resolution: {integrity: sha512-bJdC9NLNTqbN84cnaUqZifMNboMHfiqIp9AOQ+mrBgBjHysm/fopMg9OAt3zHCY+xlnlB6z+0es/PiYLsG9kyQ==} - '@milaboratories/pf-spec-driver@1.4.16': - resolution: {integrity: sha512-x6SQVUU3fl+1fFavCSzc1DNMB08HKoo0PT5dpXJWAi6oEw77yueoe1StZJVY0Kgj1BpVhlmuuC87PJu8s4eFRw==} + '@milaboratories/pf-spec-driver@1.4.17': + resolution: {integrity: sha512-FK/OK7hHDM4NNX7jm68DYMK2KZg6bq6JubG7WMFs+EzHUDbEnMwVbsa0Ch1SmwNoKwsVVcO1E6bMFuBGuW4Wjg==} '@milaboratories/pframes-rs-node@1.1.52': resolution: {integrity: sha512-Uu4KsQx0GiKvFASI+Cm0nMCMieUVo35mpz4j8mYUYImr7S+bVDDcHweCWrkLkSeImnOQnYHVo9RycViG6QwUQw==} @@ -1194,19 +1194,19 @@ packages: '@milaboratories/pl-model-common': 1.46.2 '@milaboratories/pl-model-middle-layer': 1.30.7 - '@milaboratories/pl-client@3.12.0': - resolution: {integrity: sha512-uca6tbW7hy7H/Sf010/9HNQMFKwf6bJ/JONyNgvQB+hGdJZb//vjw0xoBQDun0wSAkPm+9H2I+e0q6RUfcNZng==} + '@milaboratories/pl-client@3.13.1': + resolution: {integrity: sha512-WbA8+dwFw7fs/oNPHDsVap65zAMMnbCvs8C/RNO1ZlhIwNL9EayTiSrVz4NxwZeFI7C1GNMzYL2SOHZlDfmFgg==} engines: {node: '>=22.19.0'} '@milaboratories/pl-config@1.8.2': resolution: {integrity: sha512-FG9rlAKDf1rwlg+3G9OG2bCKISNp6ajM27W7ODDSsWE4MSCVxhYB172UusbohY2RDmaD9GI5pDKO5O9uoHQCCA==} - '@milaboratories/pl-deployments@3.0.8': - resolution: {integrity: sha512-7WvA761yI7eLCdJE19uNa/a6fLxytVrKJ/utBJ6GelZQFuA/LT12VkG6r0uAFtU8afY9Sr89078oLzZSZJR/oA==} + '@milaboratories/pl-deployments@3.0.9': + resolution: {integrity: sha512-oIxUc9PjtdyjQMzR37Cmb5YEK83cxBAq6kPC8KM/gTDjmiJR3idLSYsmTeA7Rmcz1OSvrYRHuJhmLgWMMUhm7A==} engines: {node: '>=22.19.0'} - '@milaboratories/pl-drivers@1.16.3': - resolution: {integrity: sha512-/HObfG0uuFDYUd8G3qxlhrvhvTo8T5MH42+Vpr8snN+YnSHGyMXo6rw6oKRNO+h/GF+kbI5DTr9kNsQVEwyFng==} + '@milaboratories/pl-drivers@1.16.6': + resolution: {integrity: sha512-3Pjusfv1MKCJ7NwTFlYmd6qNr2iQfoqJK7kgUACic8wh/5PLeZ4fVeghZHOBYWpi5PSuncAMaiImEfciJyKNKg==} engines: {node: '>=22'} '@milaboratories/pl-error-like@1.12.10': @@ -1215,8 +1215,8 @@ packages: '@milaboratories/pl-error-like@1.12.9': resolution: {integrity: sha512-9qlfawLFO4qG656ayvVSRholhhwlfXUvKKllXpHadqbnf2zO2oCd+5J76bPaFXDz/A3T+1eokCnCX74JsqCYSw==} - '@milaboratories/pl-errors@1.4.24': - resolution: {integrity: sha512-pV0oF0zO6bwAo/llQlu4Q00uI0guPQ9Wn+coK3pwLrateKidVRH/2XhwUgTwgS8vDECBNCgtYR6E/Im+nl+FMw==} + '@milaboratories/pl-errors@1.4.27': + resolution: {integrity: sha512-TBmiuNXyo3MufCbUdCcohX7hc7i26hXZOh8b1CN+uloPb39NGxhtaCUfhpXD0vHPeJXEdY9bPCKS8oEJdduGJQ==} '@milaboratories/pl-healthcheck@1.0.1': resolution: {integrity: sha512-eiYd/TFm18SW4EY8vkI3c+fcPnjUu3Hd1GnPLWCN8U+dNmp/tnrzfeCgaY4xO/s0vkHNX9E7IrsQiMg2Ioz4rw==} @@ -1225,12 +1225,12 @@ packages: '@milaboratories/pl-http@1.2.4': resolution: {integrity: sha512-QKmhx+WEvJCV9dUy/SBdQk/ApaJ5ewBFgm/b+XPlS10SusAdqUUTGvK5+hq8YSuUMXlHb/dk++UtI5YlDuDl2Q==} - '@milaboratories/pl-middle-layer@1.64.41': - resolution: {integrity: sha512-6xXwmTaaImd6HQ2PZgUWGf1gXsfgEPz3zWQsmaUaPMyMusjPVynPCX2hnThqrPVf54mEMlvc3HTGaC+dofDF0g==} + '@milaboratories/pl-middle-layer@1.65.1': + resolution: {integrity: sha512-dOcyutS2XJoBxmHm9f0zposPzGSSlcv0qxrw0Kb7hFH7VbsNqoEGhPsxmkp+OLfYeCZipckALazuWGLRrIgR4w==} engines: {node: '>=22.19.0'} - '@milaboratories/pl-model-backend@1.4.9': - resolution: {integrity: sha512-84+Pv2RhonE/ZwR5bXipOGK4cC5V3C4vmhzIgc8TWpgLxUNsjCjoodSo3bzPcpA+URShTf4+3h4C28O/wO/MCw==} + '@milaboratories/pl-model-backend@1.4.12': + resolution: {integrity: sha512-cRztgCkY8f4JPEWk4/1zNcdx83crvc7byLwfG+fTh8jOja+4CqfQ73fFMAt/kuTuezMWIvO2cLvxiI2Cc2ztAg==} '@milaboratories/pl-model-common@1.31.1': resolution: {integrity: sha512-MLQvhXXFOykABZr8aVgzt5x0htT7ye4cvvnVy0TgOITXpct+lEmWvaVj29zirK/0VFw7wnTXDGnLwusr77NZFA==} @@ -1241,24 +1241,27 @@ packages: '@milaboratories/pl-model-common@1.46.2': resolution: {integrity: sha512-VEeauisApYScvCS8lnK3zpFJ520xuTAodKJmjR8ulHcMrWMyWMfHEdGb7j5OMD0mM/OwTgmQrrJ5eB7Xd+xoOQ==} + '@milaboratories/pl-model-common@1.46.3': + resolution: {integrity: sha512-fMYwVmkGrgzIBo7WxWW911MYSaUwIoSwNFjgtMsus7Xa/SAX44EK7DGq3T9J/cHLK2j6aIRzIL27OUF5xiEk9A==} + '@milaboratories/pl-model-middle-layer@1.16.3': resolution: {integrity: sha512-XLZEC3POgUMNNjXr3Pudh7gx2CXZygeskxuQCyPfj4c8Fi35+kAfH9VBpNZD1koiT2n+x5QZyS3RpzcAddQTnQ==} '@milaboratories/pl-model-middle-layer@1.16.4': resolution: {integrity: sha512-uFTZHEjRmfRvY97tRbBuTvNzQnTYIj58S8HBzyMsFkzxbUI8vTBPtvEqbTU2IbQ31IEGvv8T0BBl9b7/vdud0A==} + '@milaboratories/pl-model-middle-layer@1.30.10': + resolution: {integrity: sha512-SSUl8iHC0XMFOwj4IOxWwtgU/v80gF/qi1BXIGOOV8ZAgAoiY8+Ahox5UMKi+M57KpPQtDcR1uCDtJk5eaQ5SA==} + '@milaboratories/pl-model-middle-layer@1.30.7': resolution: {integrity: sha512-rs9x3Ron4ujR/UOdEgB8WUB1SvZ8ZAScT1Av/e4or+iiQ/CzhmK9nqtYamHVwKV+JgwIFbXQwxGvIHDVz955dQ==} - '@milaboratories/pl-model-middle-layer@1.30.9': - resolution: {integrity: sha512-fnrCjWPR0HeytWC3rnHkY3bpNk3UOuED+lwmauhYBKRaF46cV1df46clJSkGNZHS9g439SYwclW4no2X/ALaFQ==} - - '@milaboratories/pl-tree@1.12.14': - resolution: {integrity: sha512-G6uUG+3b4rH0eEQP6B31unFJtxsutETJckvapqXd9u+Uo906mK4w5u6DnXcNWfhWhKx3/hp8v89CsYSDO3b7Qw==} + '@milaboratories/pl-tree@1.12.17': + resolution: {integrity: sha512-60oB2IN0yj+4W7pi62yfpQH/CUyCwCMBeIw7CI/1tCsKEo46AMYCu9O0ATKxEFpELBtUrzbfc8TrDFm//WYEew==} engines: {node: '>=22.19.0'} - '@milaboratories/ptabler-expression-js@1.2.31': - resolution: {integrity: sha512-VB3fxuVqc0Beb+3/ge9uZ39uU8YqAhck9pTIxTpNZ5hOfMtPGJEI98YxoJQ8BuYhyyx5SM5qMU04NNs/vdk46A==} + '@milaboratories/ptabler-expression-js@1.2.32': + resolution: {integrity: sha512-hpg3hWqZ63bOAUNNVt6yKVjvX611TegVkgI3Thk/svv8ZSwDKWX5sEHIq+3E0x/+MWbuTce+CvXtDB5rR1atXg==} '@milaboratories/ptabler-expression-js@1.2.5': resolution: {integrity: sha512-pxBFYycBAVc4pVCo+qDzi2mQDQiar5D2tOnkWjSLgo13OAY+KVDYQJeD4KUt2DCocMy0Uo+lREZpU1bvVBqH3A==} @@ -1291,8 +1294,8 @@ packages: '@milaboratories/uikit@2.11.7': resolution: {integrity: sha512-qCgxfOHUhp3E1EfuoKtY314ZvdmFa+xHVQzMVC0cFj5YLJFPlgRc3ZiiEHQGLnrUsoICZxqtICia99fcHoyJrw==} - '@milaboratories/uikit@2.15.11': - resolution: {integrity: sha512-oBQiPXpHEwg1dLptfCQNtpRrJ3stunjrTWiP5U90rAXl4c6Z/Ks7YU/fVvoqdBsoSBQ6nEcAG0Sk4sV8DVRlJQ==} + '@milaboratories/uikit@2.15.12': + resolution: {integrity: sha512-aJgFQroRdGiW5+Z7coopYAftyk065XXYC6/z31gI/9pr9JQ/edAqMEREK20TUBKG/K3EQExvhc/QR8CMJZWw9A==} '@napi-rs/wasm-runtime@1.1.4': resolution: {integrity: sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==} @@ -1778,8 +1781,8 @@ packages: '@platforma-open/milaboratories.software-ptabler.schema@1.14.8': resolution: {integrity: sha512-wSrsHJB8zgeglndBJBUXnIrJjaeUNKbMhj9uZAFg8OOec8rZXSdoPN6vJcJr6Qf9m4dwsLCM7pplnyO6VzCfxg==} - '@platforma-open/milaboratories.software-ptabler.schema@1.15.15': - resolution: {integrity: sha512-zvPuRZgQyxjED+txJVgWHUOPeRp4TG1jOrJOtuw9WnwxFiHNjQHIB+Xe1j8f6skgMqZgRSQvEbhRvqbWjGmhKQ==} + '@platforma-open/milaboratories.software-ptabler.schema@1.15.16': + resolution: {integrity: sha512-zgEFbdH7E1uAPzX7BI36Rx0z/T5526K95lH+aDrEp0eEi0kraGpB2Sg5mRAasgItKL4EvUn2cVHbdbPs+BFctA==} '@platforma-open/milaboratories.software-ptabler@1.15.0': resolution: {integrity: sha512-7hbuOsIU3WsmAsBwWoVfP1UyZPBQj/Ec8KGoUBoaAq5VNjVQ7oQR3Nx4co+dIJJ9+cEB3c633Xx1DNxebb5uNQ==} @@ -1817,8 +1820,8 @@ packages: '@platforma-open/milaboratories.xsv-import.model@1.0.4': resolution: {integrity: sha512-9aa7+TPpEjBKLWu7bklN89dxIKenPMKZTU7Q648hLRO9GFQ7DhIq2zJGidabPY41jYqk3swTE6Y5zoQuIynhJQ==} - '@platforma-sdk/block-tools@2.11.6': - resolution: {integrity: sha512-CyekNfqEildNeTrkjAeaWC7ah+skGMb3d/crSZSgCWACPIVQwg2oRZiJliwmJU/ylcTONPDAef7DYOtmfXsgtw==} + '@platforma-sdk/block-tools@2.11.9': + resolution: {integrity: sha512-rwvX3P1gyFPK8alFW+UvkO1T2gnENLPT1Wzpm+LQ2PkAjCMSl57zYnobOnozlHMTA0+isCudYSYCF7SpIifedQ==} hasBin: true '@platforma-sdk/blocks-deps-updater@2.2.0': @@ -1843,8 +1846,8 @@ packages: '@platforma-sdk/model@1.65.4': resolution: {integrity: sha512-OVZBOeN9LCQt1AuOd6a4kBrXqTRD+tUukDQxqcsqHdG1lG2y9kJ2RxqUWXVbzIEv2qD0m8owg99KNWEVG35nAQ==} - '@platforma-sdk/model@1.79.20': - resolution: {integrity: sha512-0ULETzwgo4E5rIddn/PoCAY5gSv9luDYAaSp66b8u3u1XTh43k7fgOpobu/rrfHJMiY9IGfzALDlo6jtvMLUyQ==} + '@platforma-sdk/model@1.79.24': + resolution: {integrity: sha512-ZIoRLpSY7kONOwncIuvZmgFYn5hUiXnKouAHINb0Np0SeYuDnDp2rwfZc4CAkxQB9GNBBqttwWqEIZ6EwhO8dg==} '@platforma-sdk/package-builder-lib@1.1.0': resolution: {integrity: sha512-h/Hx4Fg+kGnX2sVPWJa48JHbSEofTnaOescTKBP1Py5wl3A68BjqKaI6yybLWSR5Ojp3tHNNlvFSAnoF2QYMMg==} @@ -1853,19 +1856,19 @@ packages: resolution: {integrity: sha512-7bRducsc9NLc1zo0GRfz3RHSTejxFfmFvr21EH8NBndGaCj5KvHt2+0jmQsSI1Sl7ZCaj+zQKWWukRlsi+b/uA==} hasBin: true - '@platforma-sdk/tengo-builder@4.0.11': - resolution: {integrity: sha512-c0LuiR7vtclto853Q1O+j8g109ooIZKtEl5tqRHG6dN/kk6S+J7DZHCpAjFuZ9pt3x6jF95sNaJEtP8uFs1uXw==} + '@platforma-sdk/tengo-builder@4.0.14': + resolution: {integrity: sha512-cbuaNSQnHcCNdaN/uYJ68PyLoFOc9ZNwOexjA9Dx2ArWoaVCDQYvjAeWLXSsg6/KT8q8Q+Q5+ocpsyFtRz6bSQ==} engines: {node: '>=22'} hasBin: true - '@platforma-sdk/test@1.79.23': - resolution: {integrity: sha512-vb5K4Rtjy5ogom3fBYpOxvKL+Nb50iURUdd1vjzN3pcQGx47omBnDr9YvNvUsRC3INhZibLzy44dUKD2la/IMw==} + '@platforma-sdk/test@1.79.26': + resolution: {integrity: sha512-hQPUZB50OsXUPGN6OVJv1jR8+77RHgK1IU0gZA34E33Pma8vvIGaNxkcUcn/YthhH77w253CwAfUKlKQmWW0Lw==} '@platforma-sdk/ui-vue@1.63.8': resolution: {integrity: sha512-Nv/QhhTlGE8vfw3es+FNiHxrvRBn7M4xP5JPta0qIuDgCV7s0y/yK4hJQkkYuSVmigOGLQuPJGqgAV+wJVQK+g==} - '@platforma-sdk/ui-vue@1.79.20': - resolution: {integrity: sha512-VwBG8MKV1M2KMR2PgW+o6uxJJRCFGihaRsnkzLL8HGKM/Vr9k3FY/V929B1XAhU5y9wb1g7fKNJvy9JNEq5GiQ==} + '@platforma-sdk/ui-vue@1.79.24': + resolution: {integrity: sha512-yDJjxfYzKrMjuXWajLBylJQnV4t3m5rJ18e0VROnhd/BD9nbxbvDB4pPE1BrAfGF94sje1ci33JYRoA0cJfdXw==} '@platforma-sdk/workflow-tengo@5.12.0': resolution: {integrity: sha512-l3poLNfbIXTl3CYaTMPpBWV/mRaoEWEmc6/Nl7PNPkjLAy6oAfc/4xVw+Ib3Lly7MVeMEMvuVBTz5BbpwzFaIQ==} @@ -8337,14 +8340,14 @@ snapshots: '@types/node': 24.5.2 utility-types: 3.11.0 - '@milaboratories/graph-maker@1.4.8(@milaboratories/pl-model-common@1.46.2)(@platforma-sdk/model@1.79.20)(@platforma-sdk/ui-vue@1.79.20(@bytecodealliance/preview2-shim@0.17.8)(typescript@5.6.3))(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0)(typescript@5.6.3)': + '@milaboratories/graph-maker@1.4.8(@milaboratories/pl-model-common@1.46.3)(@platforma-sdk/model@1.79.24)(@platforma-sdk/ui-vue@1.79.24(@bytecodealliance/preview2-shim@0.17.8)(typescript@5.6.3))(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0)(typescript@5.6.3)': dependencies: '@ag-grid-community/core': 32.3.9 '@milaboratories/helpers': 1.14.2 '@milaboratories/miplots4': 1.2.3(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0) - '@milaboratories/pf-plots': 1.4.6(@milaboratories/pl-model-common@1.46.2)(@platforma-sdk/model@1.79.20) - '@platforma-sdk/model': 1.79.20 - '@platforma-sdk/ui-vue': 1.79.20(@bytecodealliance/preview2-shim@0.17.8)(typescript@5.6.3) + '@milaboratories/pf-plots': 1.4.6(@milaboratories/pl-model-common@1.46.3)(@platforma-sdk/model@1.79.24) + '@platforma-sdk/model': 1.79.24 + '@platforma-sdk/ui-vue': 1.79.24(@bytecodealliance/preview2-shim@0.17.8)(typescript@5.6.3) '@types/d3-hierarchy': 3.1.7 '@types/d3-scale': 4.0.9 '@vueuse/core': 13.8.0(vue@3.5.25(typescript@5.6.3)) @@ -8403,13 +8406,13 @@ snapshots: - d3-scale-chromatic - supports-color - '@milaboratories/pf-driver@1.7.14(@bytecodealliance/preview2-shim@0.17.8)': + '@milaboratories/pf-driver@1.7.15(@bytecodealliance/preview2-shim@0.17.8)': dependencies: '@milaboratories/helpers': 1.14.2 '@milaboratories/pframes-rs-node': 1.1.52 - '@milaboratories/pframes-rs-wasm': 1.1.52(@bytecodealliance/preview2-shim@0.17.8)(@milaboratories/pl-model-common@1.46.2)(@milaboratories/pl-model-middle-layer@1.30.9) - '@milaboratories/pl-model-common': 1.46.2 - '@milaboratories/pl-model-middle-layer': 1.30.9 + '@milaboratories/pframes-rs-wasm': 1.1.52(@bytecodealliance/preview2-shim@0.17.8)(@milaboratories/pl-model-common@1.46.3)(@milaboratories/pl-model-middle-layer@1.30.10) + '@milaboratories/pl-model-common': 1.46.3 + '@milaboratories/pl-model-middle-layer': 1.30.10 '@milaboratories/ts-helpers': 1.8.3 es-toolkit: 1.42.0 lru-cache: 11.2.4 @@ -8418,11 +8421,11 @@ snapshots: - encoding - supports-color - '@milaboratories/pf-plots@1.4.6(@milaboratories/pl-model-common@1.46.2)(@platforma-sdk/model@1.79.20)': + '@milaboratories/pf-plots@1.4.6(@milaboratories/pl-model-common@1.46.3)(@platforma-sdk/model@1.79.24)': dependencies: '@milaboratories/helpers': 1.14.2 - '@milaboratories/pl-model-common': 1.46.2 - '@platforma-sdk/model': 1.79.20 + '@milaboratories/pl-model-common': 1.46.3 + '@platforma-sdk/model': 1.79.24 canonicalize: 2.1.0 lodash: 4.17.23 @@ -8436,12 +8439,12 @@ snapshots: transitivePeerDependencies: - '@bytecodealliance/preview2-shim' - '@milaboratories/pf-spec-driver@1.4.16(@bytecodealliance/preview2-shim@0.17.8)': + '@milaboratories/pf-spec-driver@1.4.17(@bytecodealliance/preview2-shim@0.17.8)': dependencies: '@milaboratories/helpers': 1.14.2 - '@milaboratories/pframes-rs-wasm': 1.1.52(@bytecodealliance/preview2-shim@0.17.8)(@milaboratories/pl-model-common@1.46.2)(@milaboratories/pl-model-middle-layer@1.30.9) - '@milaboratories/pl-model-common': 1.46.2 - '@milaboratories/pl-model-middle-layer': 1.30.9 + '@milaboratories/pframes-rs-wasm': 1.1.52(@bytecodealliance/preview2-shim@0.17.8)(@milaboratories/pl-model-common@1.46.3)(@milaboratories/pl-model-middle-layer@1.30.10) + '@milaboratories/pl-model-common': 1.46.3 + '@milaboratories/pl-model-middle-layer': 1.30.10 '@noble/hashes': 2.2.0 transitivePeerDependencies: - '@bytecodealliance/preview2-shim' @@ -8473,18 +8476,18 @@ snapshots: '@milaboratories/pl-model-common': 1.31.1 '@milaboratories/pl-model-middle-layer': 1.16.3 - '@milaboratories/pframes-rs-wasm@1.1.52(@bytecodealliance/preview2-shim@0.17.8)(@milaboratories/pl-model-common@1.46.2)(@milaboratories/pl-model-middle-layer@1.30.9)': + '@milaboratories/pframes-rs-wasm@1.1.52(@bytecodealliance/preview2-shim@0.17.8)(@milaboratories/pl-model-common@1.46.3)(@milaboratories/pl-model-middle-layer@1.30.10)': dependencies: '@bytecodealliance/preview2-shim': 0.17.8 '@milaboratories/pframes-rs-wasip2': 1.1.52 - '@milaboratories/pl-model-common': 1.46.2 - '@milaboratories/pl-model-middle-layer': 1.30.9 + '@milaboratories/pl-model-common': 1.46.3 + '@milaboratories/pl-model-middle-layer': 1.30.10 - '@milaboratories/pl-client@3.12.0': + '@milaboratories/pl-client@3.13.1': dependencies: '@grpc/grpc-js': 1.13.4 '@milaboratories/pl-http': 1.2.4 - '@milaboratories/pl-model-common': 1.46.2 + '@milaboratories/pl-model-common': 1.46.3 '@milaboratories/ts-helpers': 1.8.3 '@protobuf-ts/grpc-transport': 2.11.1(@grpc/grpc-js@1.13.4) '@protobuf-ts/runtime': 2.11.1 @@ -8504,12 +8507,12 @@ snapshots: upath: 2.0.1 yaml: 2.8.1 - '@milaboratories/pl-deployments@3.0.8': + '@milaboratories/pl-deployments@3.0.9': dependencies: '@milaboratories/pl-config': 1.8.2 '@milaboratories/pl-healthcheck': 1.0.1 '@milaboratories/pl-http': 1.2.4 - '@milaboratories/pl-model-common': 1.46.2 + '@milaboratories/pl-model-common': 1.46.3 '@milaboratories/ts-helpers': 1.8.3 decompress: 4.2.1 ssh2: 1.16.0 @@ -8519,14 +8522,14 @@ snapshots: yaml: 2.8.1 zod: 3.25.76 - '@milaboratories/pl-drivers@1.16.3': + '@milaboratories/pl-drivers@1.16.6': dependencies: '@grpc/grpc-js': 1.13.4 '@milaboratories/computable': 2.9.5 '@milaboratories/helpers': 1.14.2 - '@milaboratories/pl-client': 3.12.0 - '@milaboratories/pl-model-common': 1.46.2 - '@milaboratories/pl-tree': 1.12.14 + '@milaboratories/pl-client': 3.13.1 + '@milaboratories/pl-model-common': 1.46.3 + '@milaboratories/pl-tree': 1.12.17 '@milaboratories/ts-helpers': 1.8.3 '@protobuf-ts/grpc-transport': 2.11.1(@grpc/grpc-js@1.13.4) '@protobuf-ts/plugin': 2.11.1 @@ -8552,9 +8555,9 @@ snapshots: json-stringify-safe: 5.0.1 zod: 3.23.8 - '@milaboratories/pl-errors@1.4.24': + '@milaboratories/pl-errors@1.4.27': dependencies: - '@milaboratories/pl-client': 3.12.0 + '@milaboratories/pl-client': 3.13.1 '@milaboratories/ts-helpers': 1.8.3 zod: 3.25.76 @@ -8570,27 +8573,27 @@ snapshots: dependencies: undici: 7.16.0 - '@milaboratories/pl-middle-layer@1.64.41(@bytecodealliance/preview2-shim@0.17.8)(@types/node@25.3.2)': + '@milaboratories/pl-middle-layer@1.65.1(@bytecodealliance/preview2-shim@0.17.8)(@types/node@25.3.2)': dependencies: '@milaboratories/computable': 2.9.5 '@milaboratories/helpers': 1.14.2 - '@milaboratories/pf-driver': 1.7.14(@bytecodealliance/preview2-shim@0.17.8) - '@milaboratories/pf-spec-driver': 1.4.16(@bytecodealliance/preview2-shim@0.17.8) + '@milaboratories/pf-driver': 1.7.15(@bytecodealliance/preview2-shim@0.17.8) + '@milaboratories/pf-spec-driver': 1.4.17(@bytecodealliance/preview2-shim@0.17.8) '@milaboratories/pframes-rs-node': 1.1.52 - '@milaboratories/pframes-rs-wasm': 1.1.52(@bytecodealliance/preview2-shim@0.17.8)(@milaboratories/pl-model-common@1.46.2)(@milaboratories/pl-model-middle-layer@1.30.9) - '@milaboratories/pl-client': 3.12.0 - '@milaboratories/pl-deployments': 3.0.8 - '@milaboratories/pl-drivers': 1.16.3 - '@milaboratories/pl-errors': 1.4.24 + '@milaboratories/pframes-rs-wasm': 1.1.52(@bytecodealliance/preview2-shim@0.17.8)(@milaboratories/pl-model-common@1.46.3)(@milaboratories/pl-model-middle-layer@1.30.10) + '@milaboratories/pl-client': 3.13.1 + '@milaboratories/pl-deployments': 3.0.9 + '@milaboratories/pl-drivers': 1.16.6 + '@milaboratories/pl-errors': 1.4.27 '@milaboratories/pl-http': 1.2.4 - '@milaboratories/pl-model-backend': 1.4.9 - '@milaboratories/pl-model-common': 1.46.2 - '@milaboratories/pl-model-middle-layer': 1.30.9 - '@milaboratories/pl-tree': 1.12.14 + '@milaboratories/pl-model-backend': 1.4.12 + '@milaboratories/pl-model-common': 1.46.3 + '@milaboratories/pl-model-middle-layer': 1.30.10 + '@milaboratories/pl-tree': 1.12.17 '@milaboratories/resolve-helper': 1.1.3 '@milaboratories/ts-helpers': 1.8.3 - '@platforma-sdk/block-tools': 2.11.6(@types/node@25.3.2) - '@platforma-sdk/model': 1.79.20 + '@platforma-sdk/block-tools': 2.11.9(@types/node@25.3.2) + '@platforma-sdk/model': 1.79.24 '@platforma-sdk/workflow-tengo': 6.6.5 canonicalize: 2.1.0 denque: 2.1.0 @@ -8609,9 +8612,9 @@ snapshots: - encoding - supports-color - '@milaboratories/pl-model-backend@1.4.9': + '@milaboratories/pl-model-backend@1.4.12': dependencies: - '@milaboratories/pl-client': 3.12.0 + '@milaboratories/pl-client': 3.13.1 canonicalize: 2.1.0 zod: 3.25.76 @@ -8636,6 +8639,13 @@ snapshots: canonicalize: 2.1.0 zod: 3.25.76 + '@milaboratories/pl-model-common@1.46.3': + dependencies: + '@milaboratories/helpers': 1.14.2 + '@milaboratories/pl-error-like': 1.12.10 + canonicalize: 2.1.0 + zod: 3.25.76 + '@milaboratories/pl-model-middle-layer@1.16.3': dependencies: '@milaboratories/helpers': 1.14.1 @@ -8652,15 +8662,15 @@ snapshots: utility-types: 3.11.0 zod: 3.25.76 - '@milaboratories/pl-model-middle-layer@1.30.7': + '@milaboratories/pl-model-middle-layer@1.30.10': dependencies: '@milaboratories/helpers': 1.14.2 - '@milaboratories/pl-model-common': 1.46.2 + '@milaboratories/pl-model-common': 1.46.3 es-toolkit: 1.42.0 utility-types: 3.11.0 zod: 3.25.76 - '@milaboratories/pl-model-middle-layer@1.30.9': + '@milaboratories/pl-model-middle-layer@1.30.7': dependencies: '@milaboratories/helpers': 1.14.2 '@milaboratories/pl-model-common': 1.46.2 @@ -8668,19 +8678,19 @@ snapshots: utility-types: 3.11.0 zod: 3.25.76 - '@milaboratories/pl-tree@1.12.14': + '@milaboratories/pl-tree@1.12.17': dependencies: '@milaboratories/computable': 2.9.5 - '@milaboratories/pl-client': 3.12.0 - '@milaboratories/pl-errors': 1.4.24 + '@milaboratories/pl-client': 3.13.1 + '@milaboratories/pl-errors': 1.4.27 '@milaboratories/ts-helpers': 1.8.3 denque: 2.1.0 utility-types: 3.11.0 zod: 3.25.76 - '@milaboratories/ptabler-expression-js@1.2.31': + '@milaboratories/ptabler-expression-js@1.2.32': dependencies: - '@platforma-open/milaboratories.software-ptabler.schema': 1.15.15 + '@platforma-open/milaboratories.software-ptabler.schema': 1.15.16 '@milaboratories/ptabler-expression-js@1.2.5': dependencies: @@ -8706,7 +8716,7 @@ snapshots: oxlint: 1.63.0 oxlint-plugin-eslint: 1.63.0 rolldown: 1.1.3 - rolldown-plugin-dts: 0.26.0(rolldown@1.1.3)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.6.3)) + rolldown-plugin-dts: 0.26.0(rolldown@1.1.3)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.9.3)) rollup-plugin-copy: 3.5.0 rollup-plugin-sourcemaps2: 0.5.6(@types/node@25.3.2)(rollup@4.53.3) typescript: 5.9.3 @@ -8747,7 +8757,7 @@ snapshots: oxlint: 1.63.0 oxlint-plugin-eslint: 1.63.0 rolldown: 1.1.3 - rolldown-plugin-dts: 0.26.0(rolldown@1.1.3)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.6.3)) + rolldown-plugin-dts: 0.26.0(rolldown@1.1.3)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.9.3)) rollup-plugin-copy: 3.5.0 rollup-plugin-sourcemaps2: 0.5.6(@types/node@25.3.2)(rollup@4.53.3) typescript: 5.9.3 @@ -8788,7 +8798,7 @@ snapshots: oxlint: 1.63.0 oxlint-plugin-eslint: 1.63.0 rolldown: 1.1.3 - rolldown-plugin-dts: 0.26.0(rolldown@1.1.3)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.6.3)) + rolldown-plugin-dts: 0.26.0(rolldown@1.1.3)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.9.3)) rollup-plugin-copy: 3.5.0 rollup-plugin-sourcemaps2: 0.5.6(@types/node@25.3.2)(rollup@4.53.3) typescript: 5.9.3 @@ -8861,10 +8871,10 @@ snapshots: - typescript - universal-cookie - '@milaboratories/uikit@2.15.11(typescript@5.6.3)': + '@milaboratories/uikit@2.15.12(typescript@5.6.3)': dependencies: '@milaboratories/helpers': 1.14.2 - '@platforma-sdk/model': 1.79.20 + '@platforma-sdk/model': 1.79.24 '@types/d3-array': 3.2.1 '@types/d3-axis': 3.0.6 '@types/d3-scale': 4.0.9 @@ -9311,9 +9321,9 @@ snapshots: dependencies: '@milaboratories/pl-model-common': 1.31.2 - '@platforma-open/milaboratories.software-ptabler.schema@1.15.15': + '@platforma-open/milaboratories.software-ptabler.schema@1.15.16': dependencies: - '@milaboratories/pl-model-common': 1.46.2 + '@milaboratories/pl-model-common': 1.46.3 '@platforma-open/milaboratories.software-ptabler@1.15.0': {} @@ -9350,17 +9360,17 @@ snapshots: '@platforma-open/milaboratories.xsv-import.model@1.0.4': dependencies: - '@platforma-sdk/model': 1.79.20 + '@platforma-sdk/model': 1.79.24 fast-json-patch: 3.1.1 - '@platforma-sdk/block-tools@2.11.6(@types/node@25.3.2)': + '@platforma-sdk/block-tools@2.11.9(@types/node@25.3.2)': dependencies: '@aws-sdk/client-s3': 3.859.0 '@inquirer/prompts': 7.10.1(@types/node@25.3.2) '@milaboratories/pl-http': 1.2.4 - '@milaboratories/pl-model-backend': 1.4.9 - '@milaboratories/pl-model-common': 1.46.2 - '@milaboratories/pl-model-middle-layer': 1.30.9 + '@milaboratories/pl-model-backend': 1.4.12 + '@milaboratories/pl-model-common': 1.46.3 + '@milaboratories/pl-model-middle-layer': 1.30.10 '@milaboratories/resolve-helper': 1.1.3 '@milaboratories/ts-helpers': 1.8.3 '@platforma-sdk/blocks-deps-updater': 2.2.0 @@ -9417,13 +9427,13 @@ snapshots: utility-types: 3.11.0 zod: 3.25.76 - '@platforma-sdk/model@1.79.20': + '@platforma-sdk/model@1.79.24': dependencies: '@milaboratories/helpers': 1.14.2 '@milaboratories/pl-error-like': 1.12.10 - '@milaboratories/pl-model-common': 1.46.2 - '@milaboratories/pl-model-middle-layer': 1.30.9 - '@milaboratories/ptabler-expression-js': 1.2.31 + '@milaboratories/pl-model-common': 1.46.3 + '@milaboratories/pl-model-middle-layer': 1.30.10 + '@milaboratories/ptabler-expression-js': 1.2.32 canonicalize: 2.1.0 es-toolkit: 1.42.0 fast-json-patch: 3.1.1 @@ -9451,22 +9461,22 @@ snapshots: transitivePeerDependencies: - aws-crt - '@platforma-sdk/tengo-builder@4.0.11': + '@platforma-sdk/tengo-builder@4.0.14': dependencies: - '@milaboratories/pl-model-backend': 1.4.9 + '@milaboratories/pl-model-backend': 1.4.12 '@milaboratories/resolve-helper': 1.1.3 '@milaboratories/tengo-tester': 1.6.4 '@milaboratories/ts-helpers': 1.8.3 commander: 15.0.0 winston: 3.17.0 - '@platforma-sdk/test@1.79.23(@bytecodealliance/preview2-shim@0.17.8)(@types/node@25.3.2)(vite@8.0.8(@types/node@25.3.2)(yaml@2.8.1))': + '@platforma-sdk/test@1.79.26(@bytecodealliance/preview2-shim@0.17.8)(@types/node@25.3.2)(vite@8.0.8(@types/node@25.3.2)(yaml@2.8.1))': dependencies: '@milaboratories/computable': 2.9.5 - '@milaboratories/pl-client': 3.12.0 - '@milaboratories/pl-middle-layer': 1.64.41(@bytecodealliance/preview2-shim@0.17.8)(@types/node@25.3.2) - '@milaboratories/pl-tree': 1.12.14 - '@platforma-sdk/model': 1.79.20 + '@milaboratories/pl-client': 3.13.1 + '@milaboratories/pl-middle-layer': 1.65.1(@bytecodealliance/preview2-shim@0.17.8)(@types/node@25.3.2) + '@milaboratories/pl-tree': 1.12.17 + '@platforma-sdk/model': 1.79.24 '@vitest/coverage-istanbul': 4.1.4(vitest@4.1.4) vitest: 4.1.4(@types/node@25.3.2)(@vitest/coverage-istanbul@4.1.4)(vite@8.0.8(@types/node@25.3.2)(yaml@2.8.1)) transitivePeerDependencies: @@ -9522,12 +9532,12 @@ snapshots: - typescript - universal-cookie - '@platforma-sdk/ui-vue@1.79.20(@bytecodealliance/preview2-shim@0.17.8)(typescript@5.6.3)': + '@platforma-sdk/ui-vue@1.79.24(@bytecodealliance/preview2-shim@0.17.8)(typescript@5.6.3)': dependencies: - '@milaboratories/pf-spec-driver': 1.4.16(@bytecodealliance/preview2-shim@0.17.8) - '@milaboratories/pl-model-common': 1.46.2 - '@milaboratories/uikit': 2.15.11(typescript@5.6.3) - '@platforma-sdk/model': 1.79.20 + '@milaboratories/pf-spec-driver': 1.4.17(@bytecodealliance/preview2-shim@0.17.8) + '@milaboratories/pl-model-common': 1.46.3 + '@milaboratories/uikit': 2.15.12(typescript@5.6.3) + '@platforma-sdk/model': 1.79.24 '@types/d3-format': 3.0.4 '@types/node': 24.5.2 '@types/semver': 7.7.0 @@ -12805,7 +12815,7 @@ snapshots: istanbul-lib-report: 3.0.1 istanbul-reports: 3.2.0 magicast: 0.5.2 - obug: 2.1.1 + obug: 2.1.3 tinyrainbow: 3.1.0 vitest: 4.1.4(@types/node@25.3.2)(@vitest/coverage-istanbul@4.1.4)(vite@8.0.8(@types/node@25.3.2)(yaml@2.8.1)) transitivePeerDependencies: @@ -14822,7 +14832,7 @@ snapshots: dependencies: glob: 10.4.5 - rolldown-plugin-dts@0.26.0(rolldown@1.1.3)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.6.3)): + rolldown-plugin-dts@0.26.0(rolldown@1.1.3)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.9.3)): dependencies: '@babel/generator': 8.0.0 '@babel/helper-validator-identifier': 8.0.2 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 024a117..7a325ef 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -11,12 +11,12 @@ catalog: "@milaboratories/ts-builder": 1.6.0 "@milaboratories/ts-configs": 1.3.0 "@platforma-sdk/workflow-tengo": 6.6.5 - "@platforma-sdk/block-tools": 2.11.6 - "@platforma-sdk/model": 1.79.20 - "@platforma-sdk/ui-vue": 1.79.20 - "@platforma-sdk/test": 1.79.23 + "@platforma-sdk/block-tools": 2.11.9 + "@platforma-sdk/model": 1.79.24 + "@platforma-sdk/ui-vue": 1.79.24 + "@platforma-sdk/test": 1.79.26 "@milaboratories/helpers": 1.14.2 - "@platforma-sdk/tengo-builder": 4.0.11 + "@platforma-sdk/tengo-builder": 4.0.14 "@platforma-sdk/package-builder": 3.14.0 "@platforma-sdk/blocks-deps-updater": 2.2.0 "@platforma-sdk/eslint-config": 1.2.0 diff --git a/software/src/main.py b/software/src/main.py index bf4ad94..acdde44 100644 --- a/software/src/main.py +++ b/software/src/main.py @@ -11,11 +11,34 @@ import os -# Pin the polars thread pool to 1 BEFORE polars (or anything importing it, -# including `pipeline`) is loaded — POLARS_MAX_THREADS is read once at import. -# The block reserves cpu(1), and single-threaded execution keeps output -# byte-stable (no thread-count-dependent reduction order). setdefault so an -# explicit override from the environment still wins. +# Thread configuration MUST be set before numpy / polars are imported — both +# read their thread-pool size once, at import time. +# +# BLAS / OpenMP intra-op threads -> 1 (forced). The vectorized engine's only +# order-sensitive step is the `counts @ weights` matvec (vectorized.py): if a +# single dot-product reduction is split across threads, the float summation +# order — and therefore the emitted TSV bytes and the resource CID — becomes +# thread-count dependent. Pinning every BLAS backend to one intra-op thread +# keeps that reduction bit-identical. This is the load-bearing half of the +# determinism contract, so it is forced, not setdefault. +for _thread_var in ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + "VECLIB_MAXIMUM_THREADS", +): + os.environ[_thread_var] = "1" + +# Polars parallelism is safe to scale across cores: every polars op the pipeline +# uses (CSV read, concat_str chain reconstruction, the aa_fraction explode, the +# unique-key output sort, CSV write) is deterministic regardless of thread count +# — none is a cross-row float reduction, and the sorts are total orders on unique +# keys. This value also sizes the numpy pI-bisection worker pool (see +# vectorized._n_workers). The workflow sets POLARS_MAX_THREADS to the cores the +# backend actually grants (the {system.cpu} command expression = the resolved +# cpuFormula value); setdefault to 1 keeps local / test runs single-threaded +# unless they opt in. os.environ.setdefault("POLARS_MAX_THREADS", "1") import argparse @@ -42,6 +65,10 @@ def _configure_logging() -> None: def main(argv: list[str] | None = None) -> int: _configure_logging() + logging.getLogger(__name__).info( + "thread config: POLARS_MAX_THREADS=%s, BLAS intra-op threads pinned to 1", + os.environ["POLARS_MAX_THREADS"], + ) parser = argparse.ArgumentParser(prog="compute-properties") parser.add_argument("--input", required=True, help="path to input entity TSV") parser.add_argument("--plan", required=True, help="path to plan JSON") diff --git a/software/src/pipeline.py b/software/src/pipeline.py index 4d7c12f..edf9c68 100644 --- a/software/src/pipeline.py +++ b/software/src/pipeline.py @@ -33,6 +33,12 @@ PH = 7.0 # All charge values computed at pH 7 (spec default). +# Regex character class matching a standard residue (either case), derived from +# the single source of truth STANDARD_AAS so it cannot drift from +# properties.effective_length / the count substrate. STANDARD_AAS is all letters, +# so it needs no regex escaping. +_STANDARD_AA_CLASS = "[" + STANDARD_AAS + STANDARD_AAS.lower() + "]" + # --------------------------------------------------------------------------- # CID quantization @@ -186,7 +192,13 @@ def run_peptide(reads: pl.DataFrame) -> dict[str, Any]: n = len(seqs) log.info("Computing peptide properties + AA fractions (%d sequences)", n) - sub = vec.build_counts(seqs) + # Clean the column ONCE and share the _Cleaned between the count substrate and + # the instability index — the same pattern the antibody full-chain path uses. + # The old code called build_counts(seqs) and instability_index(seqs) + # separately, each running a full _clean_vectorized pass (the dominant per-row + # cost), so the column was cleaned twice. + cleaned = vec._clean_vectorized(seqs) + sub = vec.counts_from_cleaned(cleaned) eox, ered = vec.extinction(sub) prop_arrays: dict[str, np.ndarray] = { @@ -197,7 +209,7 @@ def run_peptide(reads: pl.DataFrame) -> dict[str, Any]: "pi_peptide": vec.isoelectric_point(sub, IPC2_PEPTIDE, include_cys=True), "eox_peptide": eox, "ered_peptide": ered, - "instability_peptide": vec.instability_index(seqs), + "instability_peptide": vec.instability_from_cleaned(cleaned), "aliphatic_peptide": vec.aliphatic_index(sub), "aromaticity_peptide": vec.aromaticity(sub), } @@ -210,19 +222,27 @@ def run_peptide(reads: pl.DataFrame) -> dict[str, Any]: # AA-fraction long frame from the (N, 20) mole-fraction matrix: 20 rows per # entity (one per STANDARD_AAS), value NaN -> null for invalid entities so - # the 2-axis PColumn keeps a uniform shape across entities. The long-format - # entity axis needs Python-level repetition, so materialize keys here. + # the 2-axis PColumn keeps a uniform shape across entities. + # + # Built polars-native: assemble a wide frame (entity_key + one Float64 column + # per standard AA, in STANDARD_AAS order), then unpivot to the long + # (entity_key, aminoAcid, value) contract. This replaces the old + # `[k for k in keys for _ in STANDARD_AAS]` construction, which materialized + # two 20*N-element Python lists — the dominant peptide-mode memory transient — + # and lets polars parallelize the reshape. The unpivot reorders the rows, but + # write_output_tsv sorts by (entity_key, aminoAcid), so the emitted bytes are + # unchanged. np.ascontiguousarray keeps each column slice contiguous, so the + # pl.Series build never hits a strided-array path. fractions = vec.aa_fractions(sub) # (N, 20), NaN for invalid rows - keys = key_series.to_list() - aa_entity = [k for k in keys for _ in STANDARD_AAS] - aa_amino = list(STANDARD_AAS) * n - aa_value = fractions.reshape(-1) # row-major: entity 0's 20 AAs, then entity 1's, ... - aa_fraction = pl.DataFrame( - { - "entity_key": pl.Series(values=aa_entity, dtype=pl.Utf8), - "aminoAcid": pl.Series(values=aa_amino, dtype=pl.Utf8), - "value": _na_to_null(aa_value), - } + wide = pl.DataFrame( + {"entity_key": key_series} + | {aa: _na_to_null(np.ascontiguousarray(fractions[:, i])) for i, aa in enumerate(STANDARD_AAS)} + ) + aa_fraction = wide.unpivot( + index="entity_key", + on=list(STANDARD_AAS), + variable_name="aminoAcid", + value_name="value", ) # Quantize at the output boundary like the property columns, through the # shared CID-stability helper. aaFraction displays as .3f; round to 5 dp (one @@ -304,7 +324,21 @@ def _median_cdr3_length_by_chain(reads: pl.DataFrame, chains: list[str]) -> dict col = f"{ch}_CDR3" if col not in reads.columns: continue - lengths = [effective_length(s) for s in reads[col].to_list() if s] + # Effective length = count of standard-AA characters (case-insensitive), + # matching properties.effective_length, computed vectorized in polars over + # the whole column instead of one Python call per row (the dominant + # antibody-mode stats cost). Only non-empty CDR3 cells count (`if s` in the + # scalar form drops null and ""). The median formula below is unchanged, so + # the emitted stat is byte-identical. + lengths = ( + reads.select( + pl.col(col) + .filter(pl.col(col).is_not_null() & (pl.col(col) != "")) + .str.count_matches(_STANDARD_AA_CLASS) + ) + .to_series() + .to_list() + ) if not lengths: continue lengths.sort() diff --git a/software/src/vectorized.py b/software/src/vectorized.py index 03f2957..fd97202 100644 --- a/software/src/vectorized.py +++ b/software/src/vectorized.py @@ -5,6 +5,8 @@ from __future__ import annotations import math +import os +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass import numpy as np @@ -219,7 +221,7 @@ def _clean_vectorized(seqs: list[str | None]) -> _Cleaned: def counts_from_cleaned(cl: _Cleaned) -> Substrate: """Build the (N, 20) count substrate from an already-cleaned column. - Counts are scattered with a single `np.add.at` over the + Counts are scattered with a single flattened `np.bincount` over the (candidate-local-row, AA-index) pairs, then mapped back to the full N rows. A candidate is valid iff at least one standard residue remained (lengths > 0) — reproducing the scalar `clean_sequence(...) or None` rule. @@ -234,9 +236,14 @@ def counts_from_cleaned(cl: _Cleaned) -> Substrate: valid = np.zeros(n, dtype=bool) n_cand = len(cl.cand_rows) if n_cand: - cand_counts = np.zeros((n_cand, 20), dtype=np.int64) if cl.aa_flat.size: - np.add.at(cand_counts, (cl.row_of_res, cl.aa_flat.astype(np.intp)), 1) + # 2D count histogram via a single flattened bincount rather than + # np.add.at's unbuffered scatter (~3x faster, bit-identical for + # integer counts). Flat index = candidate-row * 20 + AA-index. + flat = cl.row_of_res.astype(np.int64) * 20 + cl.aa_flat + cand_counts = np.bincount(flat, minlength=n_cand * 20).reshape(n_cand, 20).astype(np.int64, copy=False) + else: + cand_counts = np.zeros((n_cand, 20), dtype=np.int64) counts[cl.cand_rows] = cand_counts valid[cl.cand_rows] = cl.lengths > 0 length = counts.sum(axis=1) @@ -339,28 +346,47 @@ def aa_fractions(sub: Substrate) -> np.ndarray: # --------------------------------------------------------------------------- # -def _charge_raw(sub: Substrate, ph: float, pka_set: PKaSet, include_cys: bool) -> np.ndarray: +def _charge_raw( + sub: Substrate, ph: float, pka_set: PKaSet, include_cys: bool, ten_ph: np.ndarray | None = None +) -> np.ndarray: """Vectorized BioPython charge_at_pH over ALL rows — finite for every row, invalid rows are NOT masked here. The public wrappers apply NaN via _mask. - Kept finite-for-all-rows on purpose: the pI bisection (Task 8) reuses this - over the same count matrix and needs a clean charge value per row. + Kept finite-for-all-rows on purpose: the pI bisection reuses this over the + same count matrix and needs a clean charge value per row. + + Computes 10**ph ONCE and reuses it for every pKa term via a scalar factor + (10**(ph-pk) = 10**ph * 10**(-pk); 10**(pk-ph) = 10**pk / 10**ph) instead of a + separate 10**(ph±pk) per amino acid. `ph` is an (N,) array during the pI + bisection, so this replaces ~7 array-wide transcendentals with one (~2.4x + faster on the charge path). Callers that evaluate charge for two substrates at + the SAME `ph` (the Fv pI path) can precompute `10**ph` once and pass it as + `ten_ph`, sharing that array transcendental across both chains. + + The FP drift vs the per-AA form is ~1e-15 — below the 1e-6 parity tolerance + and the 3-dp charge/pI quantization, so the output is unchanged within the + quantized-equal contract (verified byte-identical against the pre-change + engine on the test corpus). The one theoretical exception is a bisection + midpoint whose net charge lands within ~1e-15 of zero, which could flip a + branch — astronomically rare and never observed. """ c = sub.counts + if ten_ph is None: + ten_ph = 10.0**ph # single transcendental; ph may be scalar or (N,) # One N-terminus per sequence (count = 1), basic side chains K/R/H. - pos = 1.0 / (10 ** (ph - pka_set.n_terminus) + 1.0) + pos = 1.0 / (ten_ph * (10.0 ** (-pka_set.n_terminus)) + 1.0) for aa in ("K", "R", "H"): pk = pka_set.side_chain[aa] - pos = pos + c[:, _AA_INDEX[aa]] * (1.0 / (10 ** (ph - pk) + 1.0)) + pos = pos + c[:, _AA_INDEX[aa]] * (1.0 / (ten_ph * (10.0 ** (-pk)) + 1.0)) # One C-terminus per sequence (count = 1), acidic side chains D/E/Y (+C). - neg = 1.0 / (10 ** (pka_set.c_terminus - ph) + 1.0) + neg = 1.0 / ((10.0**pka_set.c_terminus) / ten_ph + 1.0) neg_aas = ["D", "E", "Y"] if include_cys and "C" in pka_set.side_chain: neg_aas.append("C") for aa in neg_aas: pk = pka_set.side_chain[aa] - neg = neg + c[:, _AA_INDEX[aa]] * (1.0 / (10 ** (pk - ph) + 1.0)) + neg = neg + c[:, _AA_INDEX[aa]] * (1.0 / ((10.0**pk) / ten_ph + 1.0)) return pos - neg @@ -446,6 +472,46 @@ def _bisect_charge_zero_vec( return pi +_PARALLEL_MIN_ROWS = 50_000 + + +def _n_workers() -> int: + """Row-parallel worker count = the allocated core count. The workflow sets + POLARS_MAX_THREADS to the cores the backend grants, so this stays in lockstep + with the CPU quota; defaults to 1 (serial) for local / test runs.""" + try: + return max(1, int(os.environ.get("POLARS_MAX_THREADS", "1"))) + except ValueError: + return 1 + + +def _bisect_rows_parallel(make_charge_fn, valid, lo=0.0, hi=14.0, tol=1e-3): + """Row-parallel wrapper around `_bisect_charge_zero_vec`. + + `make_charge_fn(a, b)` returns a charge function bound to rows [a, b). Rows + are split into contiguous, data-independent blocks and bisected on separate + threads (numpy's elementwise pow / where release the GIL), then concatenated + in row order. Each row's bisection depends only on that row's own counts, so + the result is bit-identical to the serial path regardless of the block + count — the byte-stability contract holds independent of worker count. Small + inputs run serial to avoid thread-dispatch overhead. + """ + n = len(valid) + workers = _n_workers() + if workers <= 1 or n < _PARALLEL_MIN_ROWS: + return _bisect_charge_zero_vec(make_charge_fn(0, n), valid, lo=lo, hi=hi, tol=tol) + edges = [(n * i) // workers for i in range(workers + 1)] + blocks = [(edges[i], edges[i + 1]) for i in range(workers) if edges[i] < edges[i + 1]] + + def run_block(ab): + a, b = ab + return _bisect_charge_zero_vec(make_charge_fn(a, b), valid[a:b], lo=lo, hi=hi, tol=tol) + + with ThreadPoolExecutor(max_workers=len(blocks)) as ex: + parts = list(ex.map(run_block, blocks)) + return np.concatenate(parts) + + def isoelectric_point( sub: Substrate, pka_set: PKaSet, @@ -454,20 +520,19 @@ def isoelectric_point( hi: float = 14.0, tol: float = 1e-3, ) -> np.ndarray: - """Vectorized pI: lockstep fixed-iteration bisection of `_charge_raw` over - all rows, bit-identical to the scalar `properties.isoelectric_point`. + """Vectorized pI: lockstep fixed-iteration bisection of `_charge_raw`, + bit-identical to the scalar `properties.isoelectric_point`. - Delegates the bisection to `_bisect_charge_zero_vec`; `_charge_raw` is - bit-identical to the scalar charge (charge parity = 0.0), so the pI is - bit-identical to the scalar pI for every valid sequence. + Row-parallel across `_n_workers()` threads (see `_bisect_rows_parallel`); the + per-row bisection is unchanged, so the pI is bit-identical to the scalar pI + for every valid sequence and independent of the worker count. """ - return _bisect_charge_zero_vec( - lambda ph: _charge_raw(sub, ph, pka_set, include_cys), - sub.valid, - lo=lo, - hi=hi, - tol=tol, - ) + + def make(a: int, b: int): + block = Substrate(counts=sub.counts[a:b], length=sub.length[a:b], valid=sub.valid[a:b]) + return lambda ph: _charge_raw(block, ph, pka_set, include_cys) + + return _bisect_rows_parallel(make, sub.valid, lo=lo, hi=hi, tol=tol) def fv_isoelectric_point( @@ -492,13 +557,20 @@ def fv_isoelectric_point( Returns float64, NaN where either chain is invalid or no zero crossing. """ valid = sub_vh.valid & sub_vl.valid - return _bisect_charge_zero_vec( - lambda ph: _charge_raw(sub_vh, ph, pka_set, include_cys) + _charge_raw(sub_vl, ph, pka_set, include_cys), - valid, - lo=lo, - hi=hi, - tol=tol, - ) + + def make(a: int, b: int): + vh = Substrate(counts=sub_vh.counts[a:b], length=sub_vh.length[a:b], valid=sub_vh.valid[a:b]) + vl = Substrate(counts=sub_vl.counts[a:b], length=sub_vl.length[a:b], valid=sub_vl.valid[a:b]) + + def charge_sum(ph): + # Both chains share the same ph, so compute the (N,) 10**ph transcendental + # once and pass it to both instead of recomputing it in each _charge_raw. + ten_ph = 10.0**ph + return _charge_raw(vh, ph, pka_set, include_cys, ten_ph) + _charge_raw(vl, ph, pka_set, include_cys, ten_ph) + + return charge_sum + + return _bisect_rows_parallel(make, valid, lo=lo, hi=hi, tol=tol) # --------------------------------------------------------------------------- # @@ -550,8 +622,11 @@ def instability_from_cleaned(cl: _Cleaned) -> np.ndarray: right = aa[1:][same] pair_row = rows[:-1][same] weights = _DIWV[left, right] - score = np.zeros(n_cand, dtype=np.float64) - np.add.at(score, pair_row, weights) + # Weighted per-candidate sum via bincount instead of np.add.at. Both + # accumulate weights[i] into bin pair_row[i] in array order, so the float + # summation order — and the result — is identical, while bincount is + # ~3x faster. Empty pair_row (no same-row adjacent pair) -> zeros. + score = np.bincount(pair_row, weights=weights, minlength=n_cand) else: score = np.zeros(n_cand, dtype=np.float64) diff --git a/software/tests/integration/test_determinism.py b/software/tests/integration/test_determinism.py index 2f20601..d4a615b 100644 --- a/software/tests/integration/test_determinism.py +++ b/software/tests/integration/test_determinism.py @@ -21,6 +21,8 @@ import hashlib import json +import os +import random import subprocess import sys from pathlib import Path @@ -53,8 +55,19 @@ def _run_cli_subprocess( out_tsv: Path, aa_tsv: Path, stats_json: Path, + polars_threads: int | None = None, ) -> None: - """Invoke main.py in a fresh subprocess. Independent hash seed per call.""" + """Invoke main.py in a fresh subprocess. Independent hash seed per call. + + `polars_threads`, when given, sets POLARS_MAX_THREADS for this run so callers + can prove the output is invariant to the polars thread-pool size. main.py + pins the BLAS intra-op threads to 1 regardless of environment, so the only + concurrency that varies across runs is the polars parallelism — exactly the + knob the workflow raises via cpu(N). + """ + env = None + if polars_threads is not None: + env = {**os.environ, "POLARS_MAX_THREADS": str(polars_threads)} result = subprocess.run( [ sys.executable, @@ -73,6 +86,7 @@ def _run_cli_subprocess( capture_output=True, text=True, check=False, + env=env, ) assert result.returncode == 0, f"main.py failed (rc={result.returncode}); stderr=\n{result.stderr}" @@ -169,6 +183,9 @@ def _assert_all_three_byte_identical(a: dict[str, Path], b: dict[str, Path]) -> # Two subprocess runs on the same input must produce byte-identical output # files. Catches future hash-order regressions (Polars group_by, set() iter, # etc.) that pass in-process tests but break across separate worker processes. +# The two runs also use DIFFERENT polars thread-pool sizes (1 vs 4): the output +# must be invariant to the core count the workflow allocates via cpu(N), not just +# to the process hash seed. @pytest.mark.parametrize( "rows, columns, plan", [ @@ -191,7 +208,73 @@ def test_outputs_byte_stable_across_subprocess_runs( a = _run_paths(tmp_path, "_a") b = _run_paths(tmp_path, "_b") - _run_cli_subprocess(input_tsv=in_tsv, plan_json=plan_json, **a) - _run_cli_subprocess(input_tsv=in_tsv, plan_json=plan_json, **b) + _run_cli_subprocess(input_tsv=in_tsv, plan_json=plan_json, polars_threads=1, **a) + _run_cli_subprocess(input_tsv=in_tsv, plan_json=plan_json, polars_threads=4, **b) + + _assert_all_three_byte_identical(a, b) + + +# Scale test: a few thousand peptides is enough for polars to actually engage its +# thread pool. Tiny inputs (the case above) run effectively single-threaded +# regardless of POLARS_MAX_THREADS, so they prove the contract but not that +# threads were exercised. Running the SAME generated input at 1 vs 4 threads and +# asserting byte-identical output proves the reshape / sort / write parallelism +# is order-stable at scale. Peptide mode is chosen because its aa_fraction +# long-format (now a polars unpivot) is the most thread-sensitive new path. +@pytest.mark.slow +def test_peptide_output_invariant_to_thread_count_at_scale(tmp_path: Path) -> None: + rng = random.Random(0) + aas = "ACDEFGHIKLMNPQRSTVWY" + rows = [ + {"entity_key": f"p{i}", "sequence": "".join(rng.choice(aas) for _ in range(rng.randint(5, 30)))} + for i in range(3000) + ] + in_tsv = tmp_path / "input.tsv" + plan_json = tmp_path / "plan.json" + _write_tsv(in_tsv, rows, _PEPTIDE_COLUMNS) + plan_json.write_text(json.dumps(_PEPTIDE_PLAN)) + + a = _run_paths(tmp_path, "_t1") + b = _run_paths(tmp_path, "_t4") + + _run_cli_subprocess(input_tsv=in_tsv, plan_json=plan_json, polars_threads=1, **a) + _run_cli_subprocess(input_tsv=in_tsv, plan_json=plan_json, polars_threads=4, **b) + + _assert_all_three_byte_identical(a, b) + + +# Scale test #2: the pI bisection runs row-parallel only at or above +# vectorized._PARALLEL_MIN_ROWS (smaller inputs bisect serially), so the cases +# above — under that threshold — never exercise it. This case crosses it and uses +# antibody mode, which drives BOTH parallelized bisections: isoelectric_point +# (full chains) and fv_isoelectric_point (Fv). Byte-identical output at 1 vs 4 +# threads proves the row-chunked bisection is independent of the worker count. +# The row count is derived from the threshold so raising it can't silently drop +# this coverage back to the serial path. +@pytest.mark.slow +def test_antibody_bisection_invariant_to_thread_count_at_scale(tmp_path: Path) -> None: + from vectorized import _PARALLEL_MIN_ROWS + + rng = random.Random(0) + aas = "ACDEFGHIKLMNPQRSTVWY" + regions = ("FR1", "CDR1", "FR2", "CDR2", "FR3", "CDR3", "FR4") + rows: list[dict[str, str]] = [] + for i in range(_PARALLEL_MIN_ROWS + 10_000): + row = {"entity_key": f"c{i}"} + for ch in ("A", "B"): + for r in regions: + row[f"{ch}_{r}"] = "".join(rng.choice(aas) for _ in range(rng.randint(5, 14))) + rows.append(row) + + in_tsv = tmp_path / "input.tsv" + plan_json = tmp_path / "plan.json" + _write_tsv(in_tsv, rows, _ANTIBODY_COLUMNS) + plan_json.write_text(json.dumps(_ANTIBODY_PLAN)) + + a = _run_paths(tmp_path, "_ab1") + b = _run_paths(tmp_path, "_ab4") + + _run_cli_subprocess(input_tsv=in_tsv, plan_json=plan_json, polars_threads=1, **a) + _run_cli_subprocess(input_tsv=in_tsv, plan_json=plan_json, polars_threads=4, **b) _assert_all_three_byte_identical(a, b) diff --git a/workflow/src/main.tpl.tengo b/workflow/src/main.tpl.tengo index 42ac142..2a5919f 100644 --- a/workflow/src/main.tpl.tengo +++ b/workflow/src/main.tpl.tengo @@ -17,6 +17,7 @@ pSpec := import("@platforma-sdk/workflow-tengo:pframes.spec") canonical := import("@platforma-sdk/workflow-tengo:canonical") maps := import("@platforma-sdk/workflow-tengo:maps") constants := import("@platforma-sdk/workflow-tengo:constants") +feats := import("@platforma-sdk/workflow-tengo:feats") messages := import(":messages") columnSpecs := import(":columns") @@ -383,13 +384,54 @@ wf.body(func(args) { // stats.json (dataset-level scalars consumed by the info layer — e.g. R11c // median CDR-H3 length per chain). soft := assets.importSoftware("@platforma-open/milaboratories.sequence-properties.software:compute-properties") - // 12GiB (up from the 4GiB default): the vectorized step materializes full-N - // count matrices, so peak memory scales with clone count. cpu(1) is - // deliberate — the engine is single-threaded for deterministic output. - pyRun := exec.builder(). + f := exec.formula + // Memory scales with the input instead of a flat allocation, as the max of two + // terms (each calibrated from measured peak RSS): + // * row-scaled — lineCount("input.tsv") * per-row bytes. Covers the per-clone + // structures (count matrix, output columns, aa_fraction). Measured peptide + // ~3.0 KiB/row, antibody ~4.3 KiB/row; kib(5)/kib(8) carry the headroom. + // * residue-scaled — size("input.tsv") * 64. Peak RSS from the cleaning + // transients (the DIWV weight gather, residue-row ids, byte buffers) is + // O(total residues), and input bytes ≈ total residues, so this term covers + // long sequences that lineCount alone misses (measured ~40 B RSS per input + // byte; 64x adds headroom). WITHOUT it, a low-row/long-sequence input (few + // amplicon variants, long sequences) sizes to the 2 GiB floor and OOMs + // where the old flat 12 GiB survived. + // max() lets each term dominate in its own regime (short-seq -> row term; + // long-seq -> residue term) without double-counting. Thread count barely moves + // peak RSS (+<0.1GiB at 8 threads), so no thread headroom is needed. + // + // Both modes parallelize on the compute engine (peptide ~1.7x, antibody ~1.5x + // on 4 cores: row-parallel pI bisection + vectorized CDR3-length stat), so CPU + // scales up to 4 with dataset size; small inputs get 1 core. main.py pins BLAS + // to one intra-op thread, so output is byte-identical regardless of core count. + // The memFormula fallback is the previous static 12GiB, for backends that + // cannot evaluate formulas (no getBlobSize). + memPerRow := f.kib(8) // antibody / TCR — heavier per row (2 chains + ~30 cols) + if wholeSeqMode { + memPerRow = f.kib(5) // peptide / amplicon + } + computeCpu := 4 + execBuilder := exec.builder(). software(soft). - mem("12GiB"). - cpu(1). + memFormula(f.clamp(f.max(f.mul(f.lineCount(), memPerRow), f.mul(f.size(), 64)), f.gib(2), f.gib(64)), { fallback: "12GiB" }). + cpu(computeCpu). + // Reserve ~1 core per 250k rows, capped at computeCpu; small inputs free + // cluster capacity. + cpuFormula(f.clamp(f.divCeil(f.lineCount(), 250000), 1, computeCpu)) + // POLARS_MAX_THREADS sizes BOTH the polars pool and the numpy pI-bisection worker + // count (vectorized._n_workers reads it), so it must equal the cores the cpuFormula + // actually GRANTS — not the static cap. Otherwise a sub-cap input (e.g. 1 core + // granted) would still spawn computeCpu threads and oversubscribe its quota. On + // backends with command expressions, {system.cpu} resolves at run time to the + // granted core count (the backend substitutes it before the process starts, so + // main.py reads the final integer); older backends fall back to the static cap. + if feats.commandExpressions { + execBuilder = execBuilder.envExpr("POLARS_MAX_THREADS", "{system.cpu}") + } else { + execBuilder = execBuilder.env("POLARS_MAX_THREADS", string(computeCpu)) + } + pyRun := execBuilder. addFile("input.tsv", seqTable). writeFile("plan.json", canonical.encode(plan)). arg("--input").arg("input.tsv").