From b13c9709918e54e546dd0ff0f023f75f5a543a91 Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Fri, 24 Jul 2026 17:53:22 +0300 Subject: [PATCH 01/14] chore: upgrade rapyer to >=1.3.4 and fix breaking changes Bump the rapyer cap to >=1.3.4,<1.4.0 across third-magic, mageflow-mcp and mage-voyance and re-lock every workspace. Adapt to 1.3.x changes: - instance apipeline now re-fetches on entry, so add_tasks uses ignore_redis_error=True (the swarm is not yet committed during the atomic create + add_tasks pipeline). - RedisConfig is now a pydantic model, so apply_ttl_config and the TTL test helpers use model_copy(update=...) instead of dataclasses.replace. Full unit suites green on 1.3.4 (third-magic 134, mageflow 263). Refs #136 Co-Authored-By: Claude Opus 4.8 (1M context) --- .DS_Store | Bin 0 -> 8196 bytes .codex/config.toml | 6 ++++++ libs/.DS_Store | Bin 0 -> 8196 bytes libs/mage-voyance/pyproject.toml | 2 +- libs/mage-voyance/uv.lock | 10 +++++----- libs/mageflow-e2e/uv.lock | 8 ++++---- libs/mageflow-mcp/pyproject.toml | 2 +- libs/mageflow-mcp/uv.lock | 10 +++++----- libs/mageflow/.DS_Store | Bin 0 -> 6148 bytes libs/mageflow/mageflow/config.py | 3 +-- libs/mageflow/tests/unit/test_remove_ttl.py | 8 ++++---- libs/mageflow/tests/unit/test_ttl_config.py | 6 ++---- libs/mageflow/uv.lock | 8 ++++---- libs/third-magic/.DS_Store | Bin 0 -> 6148 bytes libs/third-magic/pyproject.toml | 2 +- libs/third-magic/thirdmagic/swarm/model.py | 2 +- libs/third-magic/uv.lock | 8 ++++---- 17 files changed, 39 insertions(+), 36 deletions(-) create mode 100644 .DS_Store create mode 100644 .codex/config.toml create mode 100644 libs/.DS_Store create mode 100644 libs/mageflow/.DS_Store create mode 100644 libs/third-magic/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..80e6e1da223a18a77d1f7fbd7462cd90d081e6ad GIT binary patch literal 8196 zcmeHML2nyH6n^8lc%3wfle!_L0$KHiLr5J;Q7aBOPNEVhB#P6J&=#|6Z=7v5JJ#;n zj){VNhXdk_ICF**H!l1E4xEtS#tm`hRKA(nb=K?piV)qAX5Q?~_ukC=X6KD(O+>8H zZmbY35|M^1WNr$@Gm_{Rc_gKR7NN|bPt>LcnVUAR8}~8#N9nZ z%TyySC50a&*reM>Dx2MA+NMED4DcG!GR=o?$qGec1dJ<*m zGOf`&v`Qb)J=&oAv<10LC)60#V>1y3(qNTiBrk@8-$OW!4zW%g3<=MjxLx7x5PlZ; zHn456y7U^Juy%vz*k}yz$dGm#Yhqy)V>rRpvvEe(YZ~ZRCwdoo9E|Q?P~GnTRXL5`^4d& z<5fap!MR4$>UmIpf!e@vtKJD5V!wtC{g#^xUCK^QU7ntq&CShU$)@QxC%WZp`dzNo;=ireC$~S%UWYan17`y|z zw_2jh+cRAcvX+bK>>uc^t2<4z(tB*{hG}m(74G}|L?DGV__Xbs!jhauwWZfBr&)aa zFU4uGyd73vRh1kSRRwpUd{_%4EB^^(I_E?j(SetcNS6N7fYyL`LF%Ybe%3#eqRoAh( z(?pxVXE5R>#@)cCuF}Tv1{HyCXe(&gG8_Sw%K$YR$PoJ2XOWFRiQsS?Zhq+W@Ei-5 za#KfjW@K1u$w2S~8@i4St%BN!uAE;~H%mM4WR=$GeaLa4>CeavGYGJCjAda|KOPFi zVa1|o5SFBI3{0qwMREV%82|l$oO;rpFpx0tKVX2R*Q@K|M{sx^#eKpLEGi&>f-I82 x^_7$=CU|2uRNV;3u)T+$HC~dHV%6o1zia0nl6NJ{~MEM;LRkOD;@20{u#1*BFaEiJSJC%FVu=Pt6Hq@}8q z9R~gY{s3l}*jV@n7?_Y?V`XK6cONNHLN<^J+(~!8=X)Q%``OR-GXRimqdWyL0RR?u zfp#2+Uo@c?y{`@F8KETT4^ROgIJ9PcUh1GnA>;wkzoh_O*;JvT3 zW;6^K2L4M1#Ph+yE-<69tx!HXaF8Vc;s{R5hBDRx(#O%5(b!ffT`{Jr9*A}&y2K#L z9rcbdN6ct!D^%`4lsgc8W}+(;BF~O~jx-07QD{cPfMKAY0ae`Bz=bN%?)tsx^CFsy zbjo5s{arR0z=9t8Vk@j``!~h9?}br1{ecV(kL=pL$J%4Xt<~Hk-pWO8)CjX~a9{M&!kS-W{Ha_z5dHjs{pxsoe1=lr#=KHfZ;X8G zZ0wUg+SyucZ+zeW0|)KV#GzzjEHOTw96vm9qkp>}R!S3-{T6TlLWmTHpSoPf^ z_2*cYM-i_HOA!tCG#^kANiHK-cS|0trmlQd+@><~9pR)BY)B~q8Bca3jU@Hf`z+-= ze*YFPB!x#8QhRG;Nku&5Xxk_%5+hl<(k}9c_ z25FL(iZIl<_9t{i4Wx}i-?eY%0_Jm8MV;`ALj;e&g8<674wV2Nl*F&Du2gx2mffi` z!0TU0m;u`}X~Wrnk~RbGfS%;RtT1iIF)*xFG$HE$>x2LQ@3?iF1sMhm1HX>}GIA|< zO?>2czK?oCi(1>k{t&xp!mh1Qy5Jy8#JnuW5p_8(=NzreXlyH#uIM9O5mhDs$A1Kv R&*tCx*=*kb;%5E}+Yf-v5Ul_J literal 0 HcmV?d00001 diff --git a/libs/mage-voyance/pyproject.toml b/libs/mage-voyance/pyproject.toml index 80ede843..a09e718a 100644 --- a/libs/mage-voyance/pyproject.toml +++ b/libs/mage-voyance/pyproject.toml @@ -9,7 +9,7 @@ license = {text = "MIT"} requires-python = ">=3.10,<3.14" dependencies = [ "thirdmagic>=0.0.4,<0.1.0", - "rapyer>=1.2.5,<1.3.0", + "rapyer>=1.3.4,<1.4.0", "pydantic>=2.0.0,<3.0.0", "uvicorn>=0.34.0,<1.0.0", "fastapi>=0.115.0,<1.0.0", diff --git a/libs/mage-voyance/uv.lock b/libs/mage-voyance/uv.lock index 0a63bb1d..edf09d9e 100644 --- a/libs/mage-voyance/uv.lock +++ b/libs/mage-voyance/uv.lock @@ -154,7 +154,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.0.0,<3.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2,<10.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.2.0,<2.0.0" }, - { name = "rapyer", specifier = ">=1.2.5,<1.3.0" }, + { name = "rapyer", specifier = ">=1.3.4,<1.4.0" }, { name = "thirdmagic", directory = "../third-magic" }, { name = "uvicorn", specifier = ">=0.34.0,<1.0.0" }, ] @@ -326,15 +326,15 @@ wheels = [ [[package]] name = "rapyer" -version = "1.2.5" +version = "1.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "redis" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f1/1b/17510f5d8c33ccb495e015b359a0665f891023527750d87b13a5d6c5b5d8/rapyer-1.2.5.tar.gz", hash = "sha256:f3819a52ab58ea2afd7d3e17918112d7454573d75b03fb218017f71665b2e003", size = 3094493, upload-time = "2026-03-05T16:04:52.242Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/f8/9c5448f68ab88ac30037cacfd0cb8d17342133abf0289d942623d6b2f347/rapyer-1.3.4.tar.gz", hash = "sha256:cd61c9708c0b7568b473368fdea46671569d1a90d5d0b6c96ec280953f9b6de8", size = 3226823, upload-time = "2026-07-21T13:56:17.67Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/ea/b2898f92121bb3b8dccc4f77d26fc712b0166016c357a113fa9a7fb17db5/rapyer-1.2.5-py3-none-any.whl", hash = "sha256:d7e10c538f9e0d747e674848ec532b78a16a0bf2f5dcf78f8846c6624c8d8802", size = 44679, upload-time = "2026-03-05T16:04:50.761Z" }, + { url = "https://files.pythonhosted.org/packages/0c/14/c32b0f348c88f552d541b06910f28ea411b233158478b907a7157254422a/rapyer-1.3.4-py3-none-any.whl", hash = "sha256:229376bb8e2094a1f1e02e9980aa382fc9979196cd610c60ce07f4990ab8dcf3", size = 86601, upload-time = "2026-07-21T13:56:16.299Z" }, ] [[package]] @@ -380,7 +380,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.0.0,<3.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2,<10.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.2.0,<2.0.0" }, - { name = "rapyer", specifier = ">=1.2.5,<1.3.0" }, + { name = "rapyer", specifier = ">=1.3.4,<1.4.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.5" }, ] provides-extras = ["dev"] diff --git a/libs/mageflow-e2e/uv.lock b/libs/mageflow-e2e/uv.lock index b8116cbc..d0d15d8a 100644 --- a/libs/mageflow-e2e/uv.lock +++ b/libs/mageflow-e2e/uv.lock @@ -1112,15 +1112,15 @@ wheels = [ [[package]] name = "rapyer" -version = "1.2.5" +version = "1.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "redis" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f1/1b/17510f5d8c33ccb495e015b359a0665f891023527750d87b13a5d6c5b5d8/rapyer-1.2.5.tar.gz", hash = "sha256:f3819a52ab58ea2afd7d3e17918112d7454573d75b03fb218017f71665b2e003", size = 3094493, upload-time = "2026-03-05T16:04:52.242Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/f8/9c5448f68ab88ac30037cacfd0cb8d17342133abf0289d942623d6b2f347/rapyer-1.3.4.tar.gz", hash = "sha256:cd61c9708c0b7568b473368fdea46671569d1a90d5d0b6c96ec280953f9b6de8", size = 3226823, upload-time = "2026-07-21T13:56:17.67Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/ea/b2898f92121bb3b8dccc4f77d26fc712b0166016c357a113fa9a7fb17db5/rapyer-1.2.5-py3-none-any.whl", hash = "sha256:d7e10c538f9e0d747e674848ec532b78a16a0bf2f5dcf78f8846c6624c8d8802", size = 44679, upload-time = "2026-03-05T16:04:50.761Z" }, + { url = "https://files.pythonhosted.org/packages/0c/14/c32b0f348c88f552d541b06910f28ea411b233158478b907a7157254422a/rapyer-1.3.4-py3-none-any.whl", hash = "sha256:229376bb8e2094a1f1e02e9980aa382fc9979196cd610c60ce07f4990ab8dcf3", size = 86601, upload-time = "2026-07-21T13:56:16.299Z" }, ] [[package]] @@ -1225,7 +1225,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.0.0,<3.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2,<10.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.2.0,<2.0.0" }, - { name = "rapyer", specifier = ">=1.2.5,<1.3.0" }, + { name = "rapyer", specifier = ">=1.3.4,<1.4.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.5" }, ] provides-extras = ["dev"] diff --git a/libs/mageflow-mcp/pyproject.toml b/libs/mageflow-mcp/pyproject.toml index 70e0a159..c85bbec8 100644 --- a/libs/mageflow-mcp/pyproject.toml +++ b/libs/mageflow-mcp/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "pyjwt>=2.12.0", # TODO: remove when mcp updates past pyjwt 2.11.0 — pinned only for CVE-2024-53861 "pydantic>=2.11.0,<3.0.0", "thirdmagic>=0.0.1", - "rapyer>=1.2.3,<1.3.0", + "rapyer>=1.3.4,<1.4.0", "httpx>=0.27.1,<1.0.0", ] diff --git a/libs/mageflow-mcp/uv.lock b/libs/mageflow-mcp/uv.lock index 8a930db4..f2fe15dd 100644 --- a/libs/mageflow-mcp/uv.lock +++ b/libs/mageflow-mcp/uv.lock @@ -1004,7 +1004,7 @@ requires-dist = [ { name = "pyjwt", specifier = ">=2.12.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2,<10.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.2.0,<2.0.0" }, - { name = "rapyer", specifier = ">=1.2.3,<1.3.0" }, + { name = "rapyer", specifier = ">=1.3.4,<1.4.0" }, { name = "requests", marker = "extra == 'integration'", specifier = ">=2.33.0,<3.0.0" }, { name = "thirdmagic", directory = "../third-magic" }, ] @@ -1614,15 +1614,15 @@ wheels = [ [[package]] name = "rapyer" -version = "1.2.5" +version = "1.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "redis" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f1/1b/17510f5d8c33ccb495e015b359a0665f891023527750d87b13a5d6c5b5d8/rapyer-1.2.5.tar.gz", hash = "sha256:f3819a52ab58ea2afd7d3e17918112d7454573d75b03fb218017f71665b2e003", size = 3094493, upload-time = "2026-03-05T16:04:52.242Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/f8/9c5448f68ab88ac30037cacfd0cb8d17342133abf0289d942623d6b2f347/rapyer-1.3.4.tar.gz", hash = "sha256:cd61c9708c0b7568b473368fdea46671569d1a90d5d0b6c96ec280953f9b6de8", size = 3226823, upload-time = "2026-07-21T13:56:17.67Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/ea/b2898f92121bb3b8dccc4f77d26fc712b0166016c357a113fa9a7fb17db5/rapyer-1.2.5-py3-none-any.whl", hash = "sha256:d7e10c538f9e0d747e674848ec532b78a16a0bf2f5dcf78f8846c6624c8d8802", size = 44679, upload-time = "2026-03-05T16:04:50.761Z" }, + { url = "https://files.pythonhosted.org/packages/0c/14/c32b0f348c88f552d541b06910f28ea411b233158478b907a7157254422a/rapyer-1.3.4-py3-none-any.whl", hash = "sha256:229376bb8e2094a1f1e02e9980aa382fc9979196cd610c60ce07f4990ab8dcf3", size = 86601, upload-time = "2026-07-21T13:56:16.299Z" }, ] [[package]] @@ -1861,7 +1861,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.0.0,<3.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2,<10.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.2.0,<2.0.0" }, - { name = "rapyer", specifier = ">=1.2.5,<1.3.0" }, + { name = "rapyer", specifier = ">=1.3.4,<1.4.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.5" }, ] provides-extras = ["dev"] diff --git a/libs/mageflow/.DS_Store b/libs/mageflow/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..a65821c9a80a4c86a7b0e2d66e82373064f88d26 GIT binary patch literal 6148 zcmeHKyG{c^3>-s>lW0;>?k^yIU=@Xe67d5NUd^GaP}-~bF8&P04~gom&E z-8k)d=0EI;e*PZDl}rjq0VyB_q<|FojRM|#X~RjPq7;wR+`&WTBf#m#(9-E1|XSlrHhi*i^` zRFncz;820%+^)U!2`1 lJ0?au=EmFcX%uB$^EJ-s>h%_lF_YV+%usVf;nje595+DUdLeO8ucjeDu{175KXpm^oSh8o= z>$%NMagG7l@VdPOmH_5-N4$BMnxDH*?4&YAr1KeDJmC)ac*JH>{e8l@XL!K_InQ7D z*Sq~@z1y#O?mupzU%ZELC6fYDKnh3!DIf)YtAO`j+I*3yC*bB$R z_;fJD2tb@M9maLc62#^SVlNyMnW0%yiAl8@F)ZoKx2o%fV`9=_aWkJ&H(O087Pm9s zq8!!}6{Ua_I9A{^w@dH;SM(p||6`JNQa}p)D+O$}x>*f;rRuGdm-AlR=(lvQIndp> p4hloGV`8*pZoD0zM^V-_U-P^dj)_5MKIlaK8E{=>QsA!@_y9mf8@d1h literal 0 HcmV?d00001 diff --git a/libs/third-magic/pyproject.toml b/libs/third-magic/pyproject.toml index a76726cf..50c8484e 100644 --- a/libs/third-magic/pyproject.toml +++ b/libs/third-magic/pyproject.toml @@ -8,7 +8,7 @@ authors = [ license = {text = "MIT"} requires-python = ">=3.10,<3.14" dependencies = [ - "rapyer>=1.2.5,<1.3.0", + "rapyer>=1.3.4,<1.4.0", "pydantic>=2.0.0,<3.0.0", ] [project.optional-dependencies] diff --git a/libs/third-magic/thirdmagic/swarm/model.py b/libs/third-magic/thirdmagic/swarm/model.py index 6819f0f8..ced4568b 100644 --- a/libs/third-magic/thirdmagic/swarm/model.py +++ b/libs/third-magic/thirdmagic/swarm/model.py @@ -160,7 +160,7 @@ async def add_tasks( tasks = await resolve_signatures(tasks) task_keys = [task.key for task in tasks] - async with self.apipeline(use_existing_pipe=True): + async with self.apipeline(use_existing_pipe=True, ignore_redis_error=True): for task in tasks: task.signature_container_id = self.key self.tasks.extend(task_keys) diff --git a/libs/third-magic/uv.lock b/libs/third-magic/uv.lock index 4719c327..5f328b4e 100644 --- a/libs/third-magic/uv.lock +++ b/libs/third-magic/uv.lock @@ -1099,15 +1099,15 @@ wheels = [ [[package]] name = "rapyer" -version = "1.2.5" +version = "1.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "redis" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f1/1b/17510f5d8c33ccb495e015b359a0665f891023527750d87b13a5d6c5b5d8/rapyer-1.2.5.tar.gz", hash = "sha256:f3819a52ab58ea2afd7d3e17918112d7454573d75b03fb218017f71665b2e003", size = 3094493, upload-time = "2026-03-05T16:04:52.242Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/f8/9c5448f68ab88ac30037cacfd0cb8d17342133abf0289d942623d6b2f347/rapyer-1.3.4.tar.gz", hash = "sha256:cd61c9708c0b7568b473368fdea46671569d1a90d5d0b6c96ec280953f9b6de8", size = 3226823, upload-time = "2026-07-21T13:56:17.67Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/ea/b2898f92121bb3b8dccc4f77d26fc712b0166016c357a113fa9a7fb17db5/rapyer-1.2.5-py3-none-any.whl", hash = "sha256:d7e10c538f9e0d747e674848ec532b78a16a0bf2f5dcf78f8846c6624c8d8802", size = 44679, upload-time = "2026-03-05T16:04:50.761Z" }, + { url = "https://files.pythonhosted.org/packages/0c/14/c32b0f348c88f552d541b06910f28ea411b233158478b907a7157254422a/rapyer-1.3.4-py3-none-any.whl", hash = "sha256:229376bb8e2094a1f1e02e9980aa382fc9979196cd610c60ce07f4990ab8dcf3", size = 86601, upload-time = "2026-07-21T13:56:16.299Z" }, ] [[package]] @@ -1212,7 +1212,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.0.0,<3.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2,<10.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.2.0,<2.0.0" }, - { name = "rapyer", specifier = ">=1.2.5,<1.3.0" }, + { name = "rapyer", specifier = ">=1.3.4,<1.4.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.5" }, ] provides-extras = ["dev"] From 9769f10fefae5232e4ebc37830d765441f0b1ae7 Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Fri, 24 Jul 2026 18:28:33 +0300 Subject: [PATCH 02/14] feat: cascade TTL from containers to sub-tasks on write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Declare a sub_task_refs ForeignKey collection on SwarmTaskSignature and ChainTaskSignature, annotated with CascadeTTL(), mirroring the child tasks. Containers get a Meta with refresh_ttl scoped to write actions, so any write refreshes the container's TTL and cascades that refresh to its sub-task edges (real-Redis cascade; fakeredis re-arms root only). The existing tasks key-lists and their consumers are untouched — the FK field is an additional cascade edge, populated at chain creation and in swarm add_tasks. Refs #136 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/unit/test_cascade_ttl.py | 65 +++++++++++++++++++ libs/third-magic/thirdmagic/chain/creator.py | 2 + libs/third-magic/thirdmagic/chain/model.py | 13 +++- libs/third-magic/thirdmagic/container.py | 16 +++++ libs/third-magic/thirdmagic/swarm/model.py | 16 ++++- 5 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 libs/third-magic/tests/unit/test_cascade_ttl.py diff --git a/libs/third-magic/tests/unit/test_cascade_ttl.py b/libs/third-magic/tests/unit/test_cascade_ttl.py new file mode 100644 index 00000000..6d96acf8 --- /dev/null +++ b/libs/third-magic/tests/unit/test_cascade_ttl.py @@ -0,0 +1,65 @@ +import pytest +from rapyer.actions import ActionGroup +from rapyer.cascade import CascadeTTL +from rapyer.cascade.planner import build_cascade_plan + +import thirdmagic +from thirdmagic.chain.model import ChainTaskSignature +from thirdmagic.swarm.model import SwarmTaskSignature +from thirdmagic.task.model import TaskSignature + +WRITE_ACTIONS = ( + ActionGroup.CREATE + | ActionGroup.UPDATE + | ActionGroup.APPEND + | ActionGroup.ERASE + | ActionGroup.ARITHMETIC +) + + +@pytest.mark.parametrize("container_cls", [SwarmTaskSignature, ChainTaskSignature]) +def test_container_refreshes_and_cascades_on_write(container_cls): + # Assert: writes refresh TTL, and the sub_task_refs edge carries a cascade marker + assert container_cls.Meta.refresh_ttl == WRITE_ACTIONS + assert "sub_task_refs" in container_cls._contain_fk + metadata = container_cls.model_fields["sub_task_refs"].metadata + assert any(isinstance(marker, CascadeTTL) for marker in metadata) + + +def test_cascade_plan_has_container_to_task_edges(): + # Act + plan = build_cascade_plan([SwarmTaskSignature, ChainTaskSignature, TaskSignature]) + + # Assert + for name in ("SwarmTaskSignature", "ChainTaskSignature"): + edges = plan[name].fks + targets = {edge.target for edge in edges} + assert "TaskSignature" in targets + assert any(edge.path == "$.sub_task_refs" for edge in edges) + + +@pytest.mark.asyncio +async def test_chain_persists_sub_task_refs(mock_task_def): + # Arrange + tasks = [await thirdmagic.sign(f"chain_task_{i}") for i in range(3)] + chain = await thirdmagic.chain([task.key for task in tasks]) + + # Act + reloaded = await ChainTaskSignature.aget(chain.key) + + # Assert + assert [ref.target_key for ref in reloaded.sub_task_refs] == list(reloaded.tasks) + + +@pytest.mark.asyncio +async def test_swarm_persists_sub_task_refs_on_add(mock_task_def): + # Arrange + swarm = await thirdmagic.swarm(task_name="test_swarm") + tasks = [await thirdmagic.sign(f"test_task_{i}") for i in range(3)] + + # Act + await swarm.add_tasks(tasks) + reloaded = await SwarmTaskSignature.aget(swarm.key) + + # Assert + assert [ref.target_key for ref in reloaded.sub_task_refs] == list(reloaded.tasks) diff --git a/libs/third-magic/thirdmagic/chain/creator.py b/libs/third-magic/thirdmagic/chain/creator.py index 0c910ca3..a83035ab 100644 --- a/libs/third-magic/thirdmagic/chain/creator.py +++ b/libs/third-magic/thirdmagic/chain/creator.py @@ -1,4 +1,5 @@ import rapyer +from rapyer.types import Reference from thirdmagic.chain.model import ChainTaskSignature from thirdmagic.signature.model import TaskInputType @@ -34,6 +35,7 @@ async def chain( success_callbacks=[success] if success else [], error_callbacks=[error] if error else [], tasks=tasks, + sub_task_refs=[Reference(task) for task in tasks], kwargs=kwargs, ) async with rapyer.apipeline(use_existing_pipe=True): diff --git a/libs/third-magic/thirdmagic/chain/model.py b/libs/third-magic/thirdmagic/chain/model.py index 3ac387b9..6f67f2da 100644 --- a/libs/third-magic/thirdmagic/chain/model.py +++ b/libs/third-magic/thirdmagic/chain/model.py @@ -1,11 +1,14 @@ import asyncio -from typing import Any, cast +from typing import Annotated, Any, ClassVar, cast import rapyer from pydantic import BaseModel, Field, field_validator +from rapyer.cascade import CascadeTTL +from rapyer.config import RedisConfig from rapyer.fields import RapyerKey +from rapyer.types import Reference -from thirdmagic.container import ContainerTaskSignature +from thirdmagic.container import ContainerTaskSignature, container_ttl_cascade_meta from thirdmagic.errors import MissingSignatureError from thirdmagic.signature.status import ContainerStatus, SignatureStatus from thirdmagic.task.model import TaskSignature @@ -17,6 +20,12 @@ class ChainTaskSignature(ContainerTaskSignature): tasks: list[RapyerKey] = Field(default_factory=list) + # Cascade edge: writing to the chain refreshes and cascades TTL to its sub-tasks. + sub_task_refs: Annotated[list[Reference[TaskSignature]], CascadeTTL()] = Field( + default_factory=list + ) + + Meta: ClassVar[RedisConfig] = container_ttl_cascade_meta() @field_validator("tasks", mode="before") @classmethod diff --git a/libs/third-magic/thirdmagic/container.py b/libs/third-magic/thirdmagic/container.py index 0af80352..b7d6d01b 100644 --- a/libs/third-magic/thirdmagic/container.py +++ b/libs/third-magic/thirdmagic/container.py @@ -3,11 +3,27 @@ from abc import ABC from typing import Any +from rapyer.actions import ActionGroup +from rapyer.config import RedisConfig from rapyer.fields import RapyerKey from thirdmagic.signature import Signature from thirdmagic.signature.status import ContainerStatus +# Write actions that refresh (and cascade) a container's TTL; not READ/FETCH/DELETE. +CONTAINER_WRITE_ACTIONS = ( + ActionGroup.CREATE + | ActionGroup.UPDATE + | ActionGroup.APPEND + | ActionGroup.ERASE + | ActionGroup.ARITHMETIC +) + + +def container_ttl_cascade_meta() -> RedisConfig: + # cascade is declared per-field via CascadeTTL(); init resets Meta.cascade_ttl. + return RedisConfig(ttl=24 * 60 * 60, refresh_ttl=CONTAINER_WRITE_ACTIONS) + class ContainerTaskSignature(Signature, ABC): @property diff --git a/libs/third-magic/thirdmagic/swarm/model.py b/libs/third-magic/thirdmagic/swarm/model.py index ced4568b..dbbf858b 100644 --- a/libs/third-magic/thirdmagic/swarm/model.py +++ b/libs/third-magic/thirdmagic/swarm/model.py @@ -1,13 +1,15 @@ import asyncio -from typing import Any, Optional, Self, cast +from typing import Annotated, Any, ClassVar, Optional, Self, cast import rapyer from pydantic import BaseModel, Field, field_validator from rapyer import AtomicRedisModel +from rapyer.cascade import CascadeTTL +from rapyer.config import RedisConfig from rapyer.fields import RapyerKey -from rapyer.types import RedisInt, RedisList +from rapyer.types import RedisInt, RedisList, Reference -from thirdmagic.container import ContainerTaskSignature +from thirdmagic.container import ContainerTaskSignature, container_ttl_cascade_meta from thirdmagic.errors import ( SwarmIsCanceledError, TaskAndMsgsDontMatchForSwarmError, @@ -54,6 +56,12 @@ class SwarmTaskSignature(ContainerTaskSignature): current_running_tasks: RedisInt = 0 publishing_state_id: str config: SwarmConfig = Field(default_factory=SwarmConfig) + # Cascade edge: writing to the swarm refreshes and cascades TTL to its sub-tasks. + sub_task_refs: Annotated[list[Reference[TaskSignature]], CascadeTTL()] = Field( + default_factory=list + ) + + Meta: ClassVar[RedisConfig] = container_ttl_cascade_meta() @field_validator( "tasks", "tasks_left_to_run", "finished_tasks", "failed_tasks", mode="before" @@ -165,6 +173,8 @@ async def add_tasks( task.signature_container_id = self.key self.tasks.extend(task_keys) self.tasks_left_to_run.extend(task_keys) + self.sub_task_refs.extend([Reference(task) for task in tasks]) + await self.asave() if close_on_max_task and not self.config.can_add_task(self): # We dont activate check for finish the swarm, this check is done by the tasks that were added. From 81dd0347453422b2b3633413f618a9309f81ece4 Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Fri, 24 Jul 2026 18:33:49 +0300 Subject: [PATCH 03/14] test: real-Redis integration test for container TTL cascade Cascade edge-following needs Redis Functions + RedisJSON, which fakeredis cannot emulate, so add an integration suite that runs against a real Redis Stack (THIRDMAGIC_TEST_REDIS_URL, default redis://localhost:6379) and skips when unavailable or the RedisJSON module is missing. Verifies that a write to a swarm/chain extends its sub-tasks' TTL. Refs #136 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../third-magic/tests/integration/__init__.py | 0 .../third-magic/tests/integration/conftest.py | 49 ++++++++++++++++++ .../tests/integration/test_cascade_ttl.py | 50 +++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 libs/third-magic/tests/integration/__init__.py create mode 100644 libs/third-magic/tests/integration/conftest.py create mode 100644 libs/third-magic/tests/integration/test_cascade_ttl.py diff --git a/libs/third-magic/tests/integration/__init__.py b/libs/third-magic/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/third-magic/tests/integration/conftest.py b/libs/third-magic/tests/integration/conftest.py new file mode 100644 index 00000000..8a918f14 --- /dev/null +++ b/libs/third-magic/tests/integration/conftest.py @@ -0,0 +1,49 @@ +import os +from unittest.mock import MagicMock + +import pytest +import pytest_asyncio +import rapyer +import redis.asyncio as aioredis + +from thirdmagic.clients import BaseClientAdapter +from thirdmagic.signature import Signature + +# Cascade needs a real Redis Stack (Functions + RedisJSON); fakeredis cannot emulate it. +REDIS_URL = os.environ.get("THIRDMAGIC_TEST_REDIS_URL", "redis://localhost:6379") + + +async def _has_json_module(client: aioredis.Redis) -> bool: + modules = await client.execute_command("MODULE", "LIST") + names = { + (m[1].decode() if isinstance(m[1], bytes) else m[1]) + for m in modules + if len(m) > 1 + } + return "ReJSON" in names or "json" in names + + +@pytest_asyncio.fixture +async def real_redis(): + client = aioredis.Redis.from_url(REDIS_URL) + try: + await client.ping() + except Exception: + pytest.skip(f"No Redis reachable at {REDIS_URL}") + if not await _has_json_module(client): + await client.aclose() + pytest.skip(f"Redis at {REDIS_URL} lacks the RedisJSON module") + await client.flushall() + await rapyer.init_rapyer(client) + try: + yield client + finally: + await client.flushall() + await client.aclose() + + +@pytest.fixture +def mock_adapter(): + adapter = MagicMock(spec=BaseClientAdapter) + Signature.ClientAdapter = adapter + yield adapter diff --git a/libs/third-magic/tests/integration/test_cascade_ttl.py b/libs/third-magic/tests/integration/test_cascade_ttl.py new file mode 100644 index 00000000..4138f43b --- /dev/null +++ b/libs/third-magic/tests/integration/test_cascade_ttl.py @@ -0,0 +1,50 @@ +import pytest +from pydantic import BaseModel + +import thirdmagic + +SHORT_TTL = 100 +ACTIVE_TTL = 24 * 60 * 60 + + +class Msg(BaseModel): + x: int = 0 + + +@pytest.mark.asyncio +async def test_swarm_write_cascades_ttl_to_sub_tasks(real_redis, mock_adapter): + # Arrange + swarm = await thirdmagic.swarm(task_name="cascade_swarm") + tasks = [ + await thirdmagic.sign(f"cascade_task_{i}", model_validators=Msg) + for i in range(3) + ] + await swarm.add_tasks(tasks) + for task in tasks: + await real_redis.expire(task.key, SHORT_TTL) + + # Act: an ordinary write on the swarm refreshes and cascades TTL to its sub-tasks + await swarm.close_swarm(should_check_swarm=False) + + # Assert + for task in tasks: + assert await real_redis.ttl(task.key) > SHORT_TTL + + +@pytest.mark.asyncio +async def test_chain_write_cascades_ttl_to_sub_tasks(real_redis, mock_adapter): + # Arrange + tasks = [ + await thirdmagic.sign(f"cascade_chain_task_{i}", model_validators=Msg) + for i in range(3) + ] + chain = await thirdmagic.chain([task.key for task in tasks]) + for task in tasks: + await real_redis.expire(task.key, SHORT_TTL) + + # Act: an ordinary write on the chain refreshes and cascades TTL to its sub-tasks + await chain.aupdate(task_name=chain.task_name) + + # Assert + for task in tasks: + assert await real_redis.ttl(task.key) > SHORT_TTL From f9ee8563b7ae8e60696e218c6fd59bc2651b2906 Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Fri, 24 Jul 2026 18:34:58 +0300 Subject: [PATCH 04/14] docs: changelog for cascade TTL + rapyer 1.3.4 bump (#136) Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2c35117..f1b1e10a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### ✨ Added +- **Cascade TTL to container sub-tasks** (#136): Swarm and chain signatures now declare their sub-tasks as `Reference` foreign keys (`sub_task_refs`) with `CascadeTTL()`, and refresh their TTL on any write. Writing to a container therefore cascades a TTL refresh to all of its sub-tasks, keeping the whole subtree alive together (cascade edge-following requires a real Redis Stack). - **Container Status / Progress (`mageflow.astatus`)** (#134): Container signatures (swarms and chains) now expose an `astatus()` method returning a structured `ContainerStatus` (total / finished / failed / running / pending, terminal-state percentage, and completion flag). The new `mageflow.astatus(*ids)` loads several containers in a single Redis lookup and returns a `ContainersStatus` with an aggregate `overall_percentage`, raising on missing or non-container ids. - **Signing Hatchet Workflows (`MageWorkflow`)**: Native Hatchet `Workflow` objects can now be tracked by mageflow's signature lifecycle, enabling status callbacks (success/failure) without wrapping tasks in mageflow decorators. @@ -15,6 +16,7 @@ ### 🔄 Changed +- **Rapyer bumped to `>=1.3.4`** (#136): all workspaces now require rapyer 1.3.x for `Reference` foreign keys, `CascadeTTL`, and action-scoped `refresh_ttl`. Adapted to 1.3.x changes: instance `apipeline` re-fetches on entry (swarm `add_tasks` uses `ignore_redis_error=True`), and `RedisConfig` is now a pydantic model (`apply_ttl_config` uses `model_copy` instead of `dataclasses.replace`). - **App Renamed to Mage Voyance**: Product name changed from "Mageflow Viewer" to "Mage Voyance" across the Tauri config, tray tooltip, onboarding, splash screen, and Homebrew cask. ### 🔒 Security From f961eabe5fffd5cdc4d6ff97393da386cdc1bb0a Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Fri, 24 Jul 2026 18:46:29 +0300 Subject: [PATCH 05/14] test: integration cascade tests error without Redis instead of skipping Match the existing mageflow integration convention: the real_redis fixture connects to THIRDMAGIC_TEST_REDIS_URL and lets a missing/ unsuitable Redis surface as an error, rather than silently skipping. Refs #136 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../third-magic/tests/integration/conftest.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/libs/third-magic/tests/integration/conftest.py b/libs/third-magic/tests/integration/conftest.py index 8a918f14..93091df1 100644 --- a/libs/third-magic/tests/integration/conftest.py +++ b/libs/third-magic/tests/integration/conftest.py @@ -13,26 +13,9 @@ REDIS_URL = os.environ.get("THIRDMAGIC_TEST_REDIS_URL", "redis://localhost:6379") -async def _has_json_module(client: aioredis.Redis) -> bool: - modules = await client.execute_command("MODULE", "LIST") - names = { - (m[1].decode() if isinstance(m[1], bytes) else m[1]) - for m in modules - if len(m) > 1 - } - return "ReJSON" in names or "json" in names - - @pytest_asyncio.fixture async def real_redis(): - client = aioredis.Redis.from_url(REDIS_URL) - try: - await client.ping() - except Exception: - pytest.skip(f"No Redis reachable at {REDIS_URL}") - if not await _has_json_module(client): - await client.aclose() - pytest.skip(f"Redis at {REDIS_URL} lacks the RedisJSON module") + client = aioredis.Redis.from_url(REDIS_URL, decode_responses=True) await client.flushall() await rapyer.init_rapyer(client) try: From fe981b5fba27936fb6734a8b54908eb51a107145 Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Sun, 26 Jul 2026 02:27:52 +0300 Subject: [PATCH 06/14] refactor: make tasks a Reference list instead of a duplicate field Address review: don't add a parallel sub_task_refs field. Convert the existing chain/swarm tasks field itself to Annotated[list[Reference[TaskSignature]], CascadeTTL()] so it is both the child list and the cascade edge. task_ids returns the target keys, and internal/test consumers that treated tasks as keys now use task_ids. Refs #136 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/integration/hatchet/assertions.py | 2 +- .../hatchet/swarm/test_edge_cases.py | 2 +- .../hatchet/test_complex_scenarios.py | 8 ++--- .../test_chain_workflows_idempotent.py | 4 +-- .../test_fill_running_tasks_idempotent.py | 2 +- .../workflows/test_workflow_composition.py | 12 +++---- .../tests/unit/change_status/test_swarm.py | 2 +- .../tests/unit/creation/test_chain.py | 4 +-- .../unit/creation/test_chain_and_swarm.py | 2 +- .../unit/publish/test_aio_run_in_swarm.py | 10 +++--- .../tests/unit/test_cascade_ttl.py | 16 ++++----- libs/third-magic/thirdmagic/chain/creator.py | 2 -- libs/third-magic/thirdmagic/chain/model.py | 33 ++++++++----------- libs/third-magic/thirdmagic/swarm/model.py | 27 +++++++-------- 14 files changed, 58 insertions(+), 68 deletions(-) diff --git a/libs/mageflow/tests/integration/hatchet/assertions.py b/libs/mageflow/tests/integration/hatchet/assertions.py index 71d4f378..098f60a2 100644 --- a/libs/mageflow/tests/integration/hatchet/assertions.py +++ b/libs/mageflow/tests/integration/hatchet/assertions.py @@ -431,7 +431,7 @@ def assert_chain_done( chain_tasks = [task_map[task_id] for task_id in chain_signature.tasks] assert_tasks_in_order(wf_by_signature, chain_tasks) output_value = None - for chain_task_id in chain_signature.tasks: + for chain_task_id in chain_signature.task_ids: input_params = chain_kwargs.copy() task = task_map[chain_task_id] if output_value is not None: diff --git a/libs/mageflow/tests/integration/hatchet/swarm/test_edge_cases.py b/libs/mageflow/tests/integration/hatchet/swarm/test_edge_cases.py index 7f3ec09f..25ebbcdd 100644 --- a/libs/mageflow/tests/integration/hatchet/swarm/test_edge_cases.py +++ b/libs/mageflow/tests/integration/hatchet/swarm/test_edge_cases.py @@ -34,7 +34,7 @@ async def test__sub_task_is_cancelled__swarm_still_finish( for i in range(2): await swarm.aio_run_in_swarm(task1, regular_message, options=trigger_options) await swarm.close_swarm() - tasks = await TaskSignature.afind(*swarm.tasks) + tasks = await TaskSignature.afind(*swarm.task_ids) await asyncio.sleep(15) # Assert diff --git a/libs/mageflow/tests/integration/hatchet/test_complex_scenarios.py b/libs/mageflow/tests/integration/hatchet/test_complex_scenarios.py index b6d462a3..124945fa 100644 --- a/libs/mageflow/tests/integration/hatchet/test_complex_scenarios.py +++ b/libs/mageflow/tests/integration/hatchet/test_complex_scenarios.py @@ -90,11 +90,11 @@ async def test__swarm_with_swarms_and_chains__sanity( assert_chain_done(runs, chain, tasks, check_callbacks=False) # Check kwargs for an inner task was called - signed_task = tasks_map[chain.tasks[-1]] + signed_task = tasks_map[chain.task_ids[-1]] assert_signature_done(runs, signed_task, field_int=field_int_val) # Check the first task is called with msg params - first_task = tasks_map[chain.tasks[0]] + first_task = tasks_map[chain.task_ids[0]] assert_signature_done(runs, first_task, **msg_dump, **main_swarm_kwargs) # Check error was not called @@ -116,9 +116,9 @@ async def test__swarm_with_swarms_and_chains__sanity( # **main_swarm_kwargs ) # Assert swarms were called with params - first_task = tasks_map[base_swarm.tasks[0]] + first_task = tasks_map[base_swarm.task_ids[0]] assert_signature_done(runs, first_task, base_data=test_ctx) - second_task = tasks_map[base_swarm.tasks[1]] + second_task = tasks_map[base_swarm.task_ids[1]] assert_signature_done(runs, second_task, **msg_dump) # Check final success was called diff --git a/libs/mageflow/tests/unit/idempotency/test_chain_workflows_idempotent.py b/libs/mageflow/tests/unit/idempotency/test_chain_workflows_idempotent.py index 34189ebb..1f502800 100644 --- a/libs/mageflow/tests/unit/idempotency/test_chain_workflows_idempotent.py +++ b/libs/mageflow/tests/unit/idempotency/test_chain_workflows_idempotent.py @@ -110,7 +110,7 @@ async def test__chain_end_fail_on_remove_task__able_to_delete( setup.msg.chain_results, setup.lifecycle_manager, setup.logger ) - sub_tasks_exists = await redis_client.exists(*setup.chain_signature.tasks) + sub_tasks_exists = await redis_client.exists(*setup.chain_signature.task_ids) assert not sub_tasks_exists await chain_end_task(setup.msg.chain_results, setup.lifecycle_manager, setup.logger) chain_exists = await redis_client.exists(*setup.chain_signature.key) @@ -219,7 +219,7 @@ async def test__chain_error_fail_on_remove_task__able_to_delete( setup.logger, ) - sub_tasks_exists = await redis_client.exists(*setup.chain_signature.tasks) + sub_tasks_exists = await redis_client.exists(*setup.chain_signature.task_ids) assert not sub_tasks_exists await chain_end_task(setup.msg.chain_results, setup.lifecycle_manager, setup.logger) chain_exists = await redis_client.exists(*setup.chain_signature.key) diff --git a/libs/mageflow/tests/unit/idempotency/test_fill_running_tasks_idempotent.py b/libs/mageflow/tests/unit/idempotency/test_fill_running_tasks_idempotent.py index f49c54ae..72607874 100644 --- a/libs/mageflow/tests/unit/idempotency/test_fill_running_tasks_idempotent.py +++ b/libs/mageflow/tests/unit/idempotency/test_fill_running_tasks_idempotent.py @@ -90,7 +90,7 @@ async def test_two_consecutive_calls_ignore_second_call__no_concurrency_resource ) reloaded_swarm = await SwarmTaskSignature.aget(swarm_signature.key) - assert reloaded_swarm.tasks_left_to_run == swarm_signature.tasks[-2:] + assert reloaded_swarm.tasks_left_to_run == swarm_signature.task_ids[-2:] reloaded_publish_state = await PublishState.aget( swarm_signature.publishing_state_id ) diff --git a/libs/mageflow/tests/unit/workflows/test_workflow_composition.py b/libs/mageflow/tests/unit/workflows/test_workflow_composition.py index ec3d4570..f0b10b90 100644 --- a/libs/mageflow/tests/unit/workflows/test_workflow_composition.py +++ b/libs/mageflow/tests/unit/workflows/test_workflow_composition.py @@ -63,7 +63,7 @@ async def test_workflow_in_chain_creates_correct_subtasks(workflow, hatchet_adap assert len(chain_sig.tasks) == 2 # First sub-task should be the workflow-derived TaskSignature - wf_task = await rapyer.afind_one(chain_sig.tasks[0]) + wf_task = await rapyer.afind_one(chain_sig.task_ids[0]) assert wf_task is not None assert isinstance(wf_task, TaskSignature) assert wf_task.task_name == workflow.name @@ -79,7 +79,7 @@ async def test_workflow_in_chain_subtask_has_container_id(workflow, hatchet_adap chain_sig = await mageflow.achain([workflow, task_sig]) - wf_task = await rapyer.afind_one(chain_sig.tasks[0]) + wf_task = await rapyer.afind_one(chain_sig.task_ids[0]) assert wf_task is not None assert wf_task.signature_container_id == chain_sig.key @@ -101,8 +101,8 @@ async def test_workflow_in_chain_with_raw_object(workflow, hatchet_adapter): assert isinstance(chain_sig, ChainTaskSignature) assert len(chain_sig.tasks) == 2 - first_task = await rapyer.afind_one(chain_sig.tasks[0]) - second_task = await rapyer.afind_one(chain_sig.tasks[1]) + first_task = await rapyer.afind_one(chain_sig.task_ids[0]) + second_task = await rapyer.afind_one(chain_sig.task_ids[1]) assert first_task is not None assert second_task is not None @@ -133,7 +133,7 @@ async def test_workflow_in_swarm_tracked_as_single_unit(workflow, hatchet_adapte assert isinstance(swarm, SwarmTaskSignature) assert len(swarm.tasks) == 1 - assert swarm.tasks[0] == wf_sig.key + assert swarm.task_ids[0] == wf_sig.key @pytest.mark.asyncio @@ -176,7 +176,7 @@ async def test_workflow_raw_object_in_swarm_tracked_as_single_unit( assert isinstance(swarm, SwarmTaskSignature) assert len(swarm.tasks) == 1 - wf_task = await rapyer.afind_one(swarm.tasks[0]) + wf_task = await rapyer.afind_one(swarm.task_ids[0]) assert wf_task is not None assert isinstance(wf_task, TaskSignature) assert wf_task.task_name == workflow.name diff --git a/libs/third-magic/tests/unit/change_status/test_swarm.py b/libs/third-magic/tests/unit/change_status/test_swarm.py index c6d0994c..6abc1fc3 100644 --- a/libs/third-magic/tests/unit/change_status/test_swarm.py +++ b/libs/third-magic/tests/unit/change_status/test_swarm.py @@ -223,7 +223,7 @@ async def test_swarm_suspend_sanity(swarm_with_tasks): swarm_data.swarm_signature, SignatureStatus.SUSPENDED ) await assert_tasks_changed_status( - swarm_data.swarm_signature.tasks, SignatureStatus.SUSPENDED + swarm_data.swarm_signature.task_ids, SignatureStatus.SUSPENDED ) await assert_tasks_changed_status( swarm_data.task_signatures, SignatureStatus.SUSPENDED diff --git a/libs/third-magic/tests/unit/creation/test_chain.py b/libs/third-magic/tests/unit/creation/test_chain.py index 7037204c..e58efce7 100644 --- a/libs/third-magic/tests/unit/creation/test_chain.py +++ b/libs/third-magic/tests/unit/creation/test_chain.py @@ -96,7 +96,7 @@ async def test_chain_creation_with_various_task_types_loads_correctly_from_redis # Assert loaded_chain = await ChainTaskSignature.aget(chain_signature.key) assert isinstance(loaded_chain, ChainTaskSignature) - assert loaded_chain.tasks == [task.key for task in tasks] + assert loaded_chain.task_ids == [task.key for task in tasks] for task in tasks: loaded_task = await TaskSignature.aget(task.key) @@ -148,7 +148,7 @@ async def test_chain_success_callbacks_tasks_linked_via_container_sanity(): loaded_chain = await ChainTaskSignature.aget(chain_signature.key) assert isinstance(loaded_chain, ChainTaskSignature) - assert loaded_chain.tasks == [task1.key, task2.key, task3.key] + assert loaded_chain.task_ids == [task1.key, task2.key, task3.key] @pytest.mark.asyncio diff --git a/libs/third-magic/tests/unit/creation/test_chain_and_swarm.py b/libs/third-magic/tests/unit/creation/test_chain_and_swarm.py index dca38caa..a7e962b5 100644 --- a/libs/third-magic/tests/unit/creation/test_chain_and_swarm.py +++ b/libs/third-magic/tests/unit/creation/test_chain_and_swarm.py @@ -106,7 +106,7 @@ async def test_chain_with_mixed_task_types_loads_and_chains_correctly_sanity( loaded_chain = await assert_task_reloaded_as_type( chain_signature.key, ChainTaskSignature ) - assert loaded_chain.tasks == [simple_task.key, swarm_task.key, final_task.key] + assert loaded_chain.task_ids == [simple_task.key, swarm_task.key, final_task.key] @pytest.mark.asyncio diff --git a/libs/third-magic/tests/unit/publish/test_aio_run_in_swarm.py b/libs/third-magic/tests/unit/publish/test_aio_run_in_swarm.py index 95742ddf..2df80fed 100644 --- a/libs/third-magic/tests/unit/publish/test_aio_run_in_swarm.py +++ b/libs/third-magic/tests/unit/publish/test_aio_run_in_swarm.py @@ -43,8 +43,8 @@ async def test_aio_run_tasks_in_swarm_each_task_gets_own_message(mock_adapter): await swarm.aio_run_tasks_in_swarm([t1, t2], [msg1, msg2]) reloaded = await SwarmTaskSignature.aget(swarm.key) - sub1 = await TaskSignature.aget(reloaded.tasks[0]) - sub2 = await TaskSignature.aget(reloaded.tasks[1]) + sub1 = await TaskSignature.aget(reloaded.task_ids[0]) + sub2 = await TaskSignature.aget(reloaded.task_ids[1]) assert sub1.kwargs["base_data"] == {"a": 1} assert sub2.kwargs["base_data"] == {"b": 2} @@ -143,8 +143,8 @@ async def test_aio_run_in_swarm_list_tasks_all_get_same_kwargs(mock_adapter): await swarm.aio_run_in_swarm([t1, t2], msg) reloaded = await SwarmTaskSignature.aget(swarm.key) - sub1 = await TaskSignature.aget(reloaded.tasks[0]) - sub2 = await TaskSignature.aget(reloaded.tasks[1]) + sub1 = await TaskSignature.aget(reloaded.task_ids[0]) + sub2 = await TaskSignature.aget(reloaded.task_ids[1]) assert sub1.kwargs["base_data"] == {"shared": "data"} assert sub2.kwargs["base_data"] == {"shared": "data"} @@ -239,7 +239,7 @@ async def capture_redis_state_on_call(swarm_arg, **kwargs): reloaded = await SwarmTaskSignature.aget(swarm_arg.key) snapshots.append( { - "tasks": list(reloaded.tasks), + "tasks": list(reloaded.task_ids), "tasks_left_to_run": list(reloaded.tasks_left_to_run), } ) diff --git a/libs/third-magic/tests/unit/test_cascade_ttl.py b/libs/third-magic/tests/unit/test_cascade_ttl.py index 6d96acf8..227ab23e 100644 --- a/libs/third-magic/tests/unit/test_cascade_ttl.py +++ b/libs/third-magic/tests/unit/test_cascade_ttl.py @@ -19,10 +19,10 @@ @pytest.mark.parametrize("container_cls", [SwarmTaskSignature, ChainTaskSignature]) def test_container_refreshes_and_cascades_on_write(container_cls): - # Assert: writes refresh TTL, and the sub_task_refs edge carries a cascade marker + # Assert: writes refresh TTL, and the tasks edge carries a cascade marker assert container_cls.Meta.refresh_ttl == WRITE_ACTIONS - assert "sub_task_refs" in container_cls._contain_fk - metadata = container_cls.model_fields["sub_task_refs"].metadata + assert "tasks" in container_cls._contain_fk + metadata = container_cls.model_fields["tasks"].metadata assert any(isinstance(marker, CascadeTTL) for marker in metadata) @@ -35,11 +35,11 @@ def test_cascade_plan_has_container_to_task_edges(): edges = plan[name].fks targets = {edge.target for edge in edges} assert "TaskSignature" in targets - assert any(edge.path == "$.sub_task_refs" for edge in edges) + assert any(edge.path == "$.tasks" for edge in edges) @pytest.mark.asyncio -async def test_chain_persists_sub_task_refs(mock_task_def): +async def test_chain_tasks_persist_as_references(mock_task_def): # Arrange tasks = [await thirdmagic.sign(f"chain_task_{i}") for i in range(3)] chain = await thirdmagic.chain([task.key for task in tasks]) @@ -48,11 +48,11 @@ async def test_chain_persists_sub_task_refs(mock_task_def): reloaded = await ChainTaskSignature.aget(chain.key) # Assert - assert [ref.target_key for ref in reloaded.sub_task_refs] == list(reloaded.tasks) + assert reloaded.task_ids == [task.key for task in tasks] @pytest.mark.asyncio -async def test_swarm_persists_sub_task_refs_on_add(mock_task_def): +async def test_swarm_tasks_persist_as_references_on_add(mock_task_def): # Arrange swarm = await thirdmagic.swarm(task_name="test_swarm") tasks = [await thirdmagic.sign(f"test_task_{i}") for i in range(3)] @@ -62,4 +62,4 @@ async def test_swarm_persists_sub_task_refs_on_add(mock_task_def): reloaded = await SwarmTaskSignature.aget(swarm.key) # Assert - assert [ref.target_key for ref in reloaded.sub_task_refs] == list(reloaded.tasks) + assert reloaded.task_ids == [task.key for task in tasks] diff --git a/libs/third-magic/thirdmagic/chain/creator.py b/libs/third-magic/thirdmagic/chain/creator.py index a83035ab..0c910ca3 100644 --- a/libs/third-magic/thirdmagic/chain/creator.py +++ b/libs/third-magic/thirdmagic/chain/creator.py @@ -1,5 +1,4 @@ import rapyer -from rapyer.types import Reference from thirdmagic.chain.model import ChainTaskSignature from thirdmagic.signature.model import TaskInputType @@ -35,7 +34,6 @@ async def chain( success_callbacks=[success] if success else [], error_callbacks=[error] if error else [], tasks=tasks, - sub_task_refs=[Reference(task) for task in tasks], kwargs=kwargs, ) async with rapyer.apipeline(use_existing_pipe=True): diff --git a/libs/third-magic/thirdmagic/chain/model.py b/libs/third-magic/thirdmagic/chain/model.py index 6f67f2da..d35ccbc3 100644 --- a/libs/third-magic/thirdmagic/chain/model.py +++ b/libs/third-magic/thirdmagic/chain/model.py @@ -2,7 +2,7 @@ from typing import Annotated, Any, ClassVar, cast import rapyer -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field from rapyer.cascade import CascadeTTL from rapyer.config import RedisConfig from rapyer.fields import RapyerKey @@ -19,30 +19,25 @@ class ChainTaskSignature(ContainerTaskSignature): - tasks: list[RapyerKey] = Field(default_factory=list) - # Cascade edge: writing to the chain refreshes and cascades TTL to its sub-tasks. - sub_task_refs: Annotated[list[Reference[TaskSignature]], CascadeTTL()] = Field( + # Sub-tasks are ForeignKey edges so a write to the chain cascades TTL to them. + tasks: Annotated[list[Reference[TaskSignature]], CascadeTTL()] = Field( default_factory=list ) Meta: ClassVar[RedisConfig] = container_ttl_cascade_meta() - @field_validator("tasks", mode="before") - @classmethod - def validate_tasks(cls, v: list[TaskSignature]): - return [cls.validate_task_key(item) for item in v] - @property def task_ids(self) -> list[RapyerKey]: - return self.tasks + return [ref.target_key for ref in self.tasks] async def on_sub_task_done(self, sub_task: TaskSignature, results: Any): - sub_task_idx = self.tasks.index(sub_task.key) + task_ids = self.task_ids + sub_task_idx = task_ids.index(sub_task.key) # If this is the last task, activate chain success callbacks - if sub_task_idx == len(self.tasks) - 1: + if sub_task_idx == len(task_ids) - 1: await self.ClientAdapter.acall_chain_done(results, self) else: - next_task_key = self.tasks[sub_task_idx + 1] + next_task_key = task_ids[sub_task_idx + 1] next_task = await rapyer.aget(next_task_key) next_task = cast(TaskSignature, next_task) await next_task.acall(results, set_return_field=True, **self.kwargs) @@ -53,7 +48,7 @@ async def on_sub_task_error( await self.ClientAdapter.acall_chain_error(original_msg, error, self, sub_task) async def sub_tasks(self) -> list[TaskSignature]: - sub_tasks = await rapyer.afind(*self.tasks, skip_missing=True) + sub_tasks = await rapyer.afind(*self.task_ids, skip_missing=True) return cast(list[TaskSignature], sub_tasks) async def astatus(self) -> ContainerStatus: @@ -82,7 +77,7 @@ async def astatus(self) -> ContainerStatus: ) async def acall(self, msg: Any, set_return_field: bool = True, **kwargs): - first_task = await rapyer.afind_one(self.tasks[0]) + first_task = await rapyer.afind_one(self.task_ids[0]) if first_task is None: raise MissingSignatureError(f"First task from chain {self.key} not found") @@ -98,28 +93,28 @@ async def aio_run_no_wait( async def change_status(self, status: SignatureStatus): pause_chain_tasks = [ - TaskSignature.safe_change_status(task, status) for task in self.tasks + TaskSignature.safe_change_status(task, status) for task in self.task_ids ] pause_chain = super().change_status(status) await asyncio.gather(pause_chain, *pause_chain_tasks, return_exceptions=True) async def suspend(self): await asyncio.gather( - *[TaskSignature.suspend_from_key(task_id) for task_id in self.tasks], + *[TaskSignature.suspend_from_key(task_id) for task_id in self.task_ids], return_exceptions=True, ) await super().change_status(SignatureStatus.SUSPENDED) async def interrupt(self): await asyncio.gather( - *[TaskSignature.interrupt_from_key(task_id) for task_id in self.tasks], + *[TaskSignature.interrupt_from_key(task_id) for task_id in self.task_ids], return_exceptions=True, ) await super().change_status(SignatureStatus.INTERRUPTED) async def resume(self): await asyncio.gather( - *[TaskSignature.resume_from_key(task_key) for task_key in self.tasks], + *[TaskSignature.resume_from_key(task_key) for task_key in self.task_ids], return_exceptions=True, ) await super().change_status(self.task_status.last_status) diff --git a/libs/third-magic/thirdmagic/swarm/model.py b/libs/third-magic/thirdmagic/swarm/model.py index dbbf858b..0d1b9530 100644 --- a/libs/third-magic/thirdmagic/swarm/model.py +++ b/libs/third-magic/thirdmagic/swarm/model.py @@ -44,8 +44,10 @@ def can_add_n_tasks(self, swarm: "SwarmTaskSignature", n: int) -> bool: class SwarmTaskSignature(ContainerTaskSignature): - # TODO - TASKS list should be set once we enable this in rapyer - tasks: RedisList[RapyerKey] = Field(default_factory=list) + # Sub-tasks are ForeignKey edges so a write to the swarm cascades TTL to them. + tasks: Annotated[list[Reference[TaskSignature]], CascadeTTL()] = Field( + default_factory=list + ) tasks_left_to_run: RedisList[RapyerKey] = Field(default_factory=list) finished_tasks: RedisList[RapyerKey] = Field(default_factory=list) failed_tasks: RedisList[RapyerKey] = Field(default_factory=list) @@ -56,15 +58,11 @@ class SwarmTaskSignature(ContainerTaskSignature): current_running_tasks: RedisInt = 0 publishing_state_id: str config: SwarmConfig = Field(default_factory=SwarmConfig) - # Cascade edge: writing to the swarm refreshes and cascades TTL to its sub-tasks. - sub_task_refs: Annotated[list[Reference[TaskSignature]], CascadeTTL()] = Field( - default_factory=list - ) Meta: ClassVar[RedisConfig] = container_ttl_cascade_meta() @field_validator( - "tasks", "tasks_left_to_run", "finished_tasks", "failed_tasks", mode="before" + "tasks_left_to_run", "finished_tasks", "failed_tasks", mode="before" ) @classmethod def validate_tasks(cls, v): @@ -72,10 +70,10 @@ def validate_tasks(cls, v): @property def task_ids(self) -> list[RapyerKey]: - return self.tasks + return [ref.target_key for ref in self.tasks] async def sub_tasks(self) -> list[TaskSignature]: - tasks = await rapyer.afind(*self.tasks) + tasks = await rapyer.afind(*self.task_ids) return cast(list[TaskSignature], tasks) async def on_sub_task_done(self, sub_task: TaskSignature, results: Any): @@ -143,7 +141,7 @@ async def aio_run_tasks_in_swarm( async def change_status(self, status: SignatureStatus): paused_chain_tasks = [ - TaskSignature.safe_change_status(task, status) for task in self.tasks + TaskSignature.safe_change_status(task, status) for task in self.task_ids ] pause_chain = super().change_status(status) await asyncio.gather(pause_chain, *paused_chain_tasks, return_exceptions=True) @@ -171,9 +169,8 @@ async def add_tasks( async with self.apipeline(use_existing_pipe=True, ignore_redis_error=True): for task in tasks: task.signature_container_id = self.key - self.tasks.extend(task_keys) + self.tasks.extend([Reference(task) for task in tasks]) self.tasks_left_to_run.extend(task_keys) - self.sub_task_refs.extend([Reference(task) for task in tasks]) await self.asave() if close_on_max_task and not self.config.can_add_task(self): @@ -194,7 +191,7 @@ async def add_task( async def is_swarm_done(self): done_tasks = self.finished_tasks + self.failed_tasks - finished_all_tasks = set(done_tasks) == set(self.tasks) + finished_all_tasks = set(done_tasks) == set(self.task_ids) return self.is_swarm_closed and finished_all_tasks async def astatus(self) -> ContainerStatus: @@ -226,14 +223,14 @@ async def activate_success(self, msg): async def suspend(self): await asyncio.gather( - *[TaskSignature.suspend_from_key(swarm_id) for swarm_id in self.tasks], + *[TaskSignature.suspend_from_key(swarm_id) for swarm_id in self.task_ids], return_exceptions=True, ) await super().change_status(SignatureStatus.SUSPENDED) async def resume(self): await asyncio.gather( - *[TaskSignature.resume_from_key(task_id) for task_id in self.tasks], + *[TaskSignature.resume_from_key(task_id) for task_id in self.task_ids], return_exceptions=True, ) await super().change_status(self.task_status.last_status) From 84a92cd184779fb6776e87a9dfaba9dfdbe7948b Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Sun, 26 Jul 2026 12:11:58 +0300 Subject: [PATCH 07/14] perf: avoid materializing task_ids for single-task lookups task_ids builds the full key list; chain.acall and on_sub_task_done only need one element, so read the target_key directly off self.tasks instead of indexing into the materialized list. Refs #136 Co-Authored-By: Claude Opus 4.8 (1M context) --- libs/third-magic/thirdmagic/chain/model.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libs/third-magic/thirdmagic/chain/model.py b/libs/third-magic/thirdmagic/chain/model.py index d35ccbc3..3d290cd7 100644 --- a/libs/third-magic/thirdmagic/chain/model.py +++ b/libs/third-magic/thirdmagic/chain/model.py @@ -31,16 +31,16 @@ def task_ids(self) -> list[RapyerKey]: return [ref.target_key for ref in self.tasks] async def on_sub_task_done(self, sub_task: TaskSignature, results: Any): - task_ids = self.task_ids - sub_task_idx = task_ids.index(sub_task.key) # If this is the last task, activate chain success callbacks - if sub_task_idx == len(task_ids) - 1: + if self.tasks[-1].target_key == sub_task.key: await self.ClientAdapter.acall_chain_done(results, self) - else: - next_task_key = task_ids[sub_task_idx + 1] - next_task = await rapyer.aget(next_task_key) - next_task = cast(TaskSignature, next_task) - await next_task.acall(results, set_return_field=True, **self.kwargs) + return + for idx, ref in enumerate(self.tasks): + if ref.target_key == sub_task.key: + next_task = await rapyer.aget(self.tasks[idx + 1].target_key) + next_task = cast(TaskSignature, next_task) + await next_task.acall(results, set_return_field=True, **self.kwargs) + return async def on_sub_task_error( self, sub_task: TaskSignature, error: BaseException, original_msg: dict @@ -77,7 +77,7 @@ async def astatus(self) -> ContainerStatus: ) async def acall(self, msg: Any, set_return_field: bool = True, **kwargs): - first_task = await rapyer.afind_one(self.task_ids[0]) + first_task = await rapyer.afind_one(self.tasks[0].target_key) if first_task is None: raise MissingSignatureError(f"First task from chain {self.key} not found") From 85d8ab4060b238882e53dd403b0f85d005c94d2b Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Sun, 26 Jul 2026 12:27:39 +0300 Subject: [PATCH 08/14] =?UTF-8?q?refactor:=20address=20review=20=E2=80=94?= =?UTF-8?q?=20reference=20API,=20single=20iteration,=20simpler=20refresh?= =?UTF-8?q?=20actions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sub_tasks now resolves via the Reference API (ref.afetch) instead of rebuilding keys and calling afind. - Container fan-outs (change_status/suspend/interrupt/resume) iterate self.tasks once and read ref.target_key, instead of iterating the materialized task_ids list. - refresh_ttl is just CREATE | UPDATE (APPEND/ERASE/ARITHMETIC folded in). - chain acall / on_sub_task_done already read a single ref directly. - astatus test loads the container first, matching how mageflow.astatus invokes it (fresh, unresolved references). Refs #136 Co-Authored-By: Claude Opus 4.8 (1M context) --- libs/third-magic/tests/unit/test_cascade_ttl.py | 8 +------- .../tests/unit/test_container_astatus.py | 6 ++++-- libs/third-magic/thirdmagic/chain/model.py | 15 +++++++++------ libs/third-magic/thirdmagic/container.py | 10 ++-------- libs/third-magic/thirdmagic/swarm/model.py | 14 +++++++------- 5 files changed, 23 insertions(+), 30 deletions(-) diff --git a/libs/third-magic/tests/unit/test_cascade_ttl.py b/libs/third-magic/tests/unit/test_cascade_ttl.py index 227ab23e..e11f45b7 100644 --- a/libs/third-magic/tests/unit/test_cascade_ttl.py +++ b/libs/third-magic/tests/unit/test_cascade_ttl.py @@ -8,13 +8,7 @@ from thirdmagic.swarm.model import SwarmTaskSignature from thirdmagic.task.model import TaskSignature -WRITE_ACTIONS = ( - ActionGroup.CREATE - | ActionGroup.UPDATE - | ActionGroup.APPEND - | ActionGroup.ERASE - | ActionGroup.ARITHMETIC -) +WRITE_ACTIONS = ActionGroup.CREATE | ActionGroup.UPDATE @pytest.mark.parametrize("container_cls", [SwarmTaskSignature, ChainTaskSignature]) diff --git a/libs/third-magic/tests/unit/test_container_astatus.py b/libs/third-magic/tests/unit/test_container_astatus.py index 526bea0f..387688c6 100644 --- a/libs/third-magic/tests/unit/test_container_astatus.py +++ b/libs/third-magic/tests/unit/test_container_astatus.py @@ -2,6 +2,7 @@ import thirdmagic from thirdmagic import ContainerStatus +from thirdmagic.chain.model import ChainTaskSignature from thirdmagic.signature.status import SignatureStatus @@ -119,8 +120,9 @@ async def test_chain_astatus_classifies_children(mock_task_def): is_done=False, ) - # Act - status = await chain.astatus() + # Act: astatus runs on a freshly-loaded container (as mageflow.astatus does) + reloaded = await ChainTaskSignature.aget(chain.key) + status = await reloaded.astatus() # Assert assert status == expected diff --git a/libs/third-magic/thirdmagic/chain/model.py b/libs/third-magic/thirdmagic/chain/model.py index 3d290cd7..c6eb38aa 100644 --- a/libs/third-magic/thirdmagic/chain/model.py +++ b/libs/third-magic/thirdmagic/chain/model.py @@ -48,8 +48,10 @@ async def on_sub_task_error( await self.ClientAdapter.acall_chain_error(original_msg, error, self, sub_task) async def sub_tasks(self) -> list[TaskSignature]: - sub_tasks = await rapyer.afind(*self.task_ids, skip_missing=True) - return cast(list[TaskSignature], sub_tasks) + sub_tasks = await asyncio.gather( + *(ref.afetch() for ref in self.tasks), return_exceptions=True + ) + return [task for task in sub_tasks if isinstance(task, TaskSignature)] async def astatus(self) -> ContainerStatus: sub_tasks = await self.sub_tasks() @@ -93,28 +95,29 @@ async def aio_run_no_wait( async def change_status(self, status: SignatureStatus): pause_chain_tasks = [ - TaskSignature.safe_change_status(task, status) for task in self.task_ids + TaskSignature.safe_change_status(ref.target_key, status) + for ref in self.tasks ] pause_chain = super().change_status(status) await asyncio.gather(pause_chain, *pause_chain_tasks, return_exceptions=True) async def suspend(self): await asyncio.gather( - *[TaskSignature.suspend_from_key(task_id) for task_id in self.task_ids], + *[TaskSignature.suspend_from_key(ref.target_key) for ref in self.tasks], return_exceptions=True, ) await super().change_status(SignatureStatus.SUSPENDED) async def interrupt(self): await asyncio.gather( - *[TaskSignature.interrupt_from_key(task_id) for task_id in self.task_ids], + *[TaskSignature.interrupt_from_key(ref.target_key) for ref in self.tasks], return_exceptions=True, ) await super().change_status(SignatureStatus.INTERRUPTED) async def resume(self): await asyncio.gather( - *[TaskSignature.resume_from_key(task_key) for task_key in self.task_ids], + *[TaskSignature.resume_from_key(ref.target_key) for ref in self.tasks], return_exceptions=True, ) await super().change_status(self.task_status.last_status) diff --git a/libs/third-magic/thirdmagic/container.py b/libs/third-magic/thirdmagic/container.py index b7d6d01b..cb35b6a4 100644 --- a/libs/third-magic/thirdmagic/container.py +++ b/libs/third-magic/thirdmagic/container.py @@ -10,14 +10,8 @@ from thirdmagic.signature import Signature from thirdmagic.signature.status import ContainerStatus -# Write actions that refresh (and cascade) a container's TTL; not READ/FETCH/DELETE. -CONTAINER_WRITE_ACTIONS = ( - ActionGroup.CREATE - | ActionGroup.UPDATE - | ActionGroup.APPEND - | ActionGroup.ERASE - | ActionGroup.ARITHMETIC -) +# A container refreshes (and cascades) its TTL when it is created or updated. +CONTAINER_WRITE_ACTIONS = ActionGroup.CREATE | ActionGroup.UPDATE def container_ttl_cascade_meta() -> RedisConfig: diff --git a/libs/third-magic/thirdmagic/swarm/model.py b/libs/third-magic/thirdmagic/swarm/model.py index 0d1b9530..3ce3d152 100644 --- a/libs/third-magic/thirdmagic/swarm/model.py +++ b/libs/third-magic/thirdmagic/swarm/model.py @@ -1,7 +1,6 @@ import asyncio from typing import Annotated, Any, ClassVar, Optional, Self, cast -import rapyer from pydantic import BaseModel, Field, field_validator from rapyer import AtomicRedisModel from rapyer.cascade import CascadeTTL @@ -73,7 +72,7 @@ def task_ids(self) -> list[RapyerKey]: return [ref.target_key for ref in self.tasks] async def sub_tasks(self) -> list[TaskSignature]: - tasks = await rapyer.afind(*self.task_ids) + tasks = await asyncio.gather(*(ref.afetch() for ref in self.tasks)) return cast(list[TaskSignature], tasks) async def on_sub_task_done(self, sub_task: TaskSignature, results: Any): @@ -85,7 +84,7 @@ async def on_sub_task_error( await self.ClientAdapter.acall_swarm_item_error(error, self, sub_task) async def acall(self, msg: Any, set_return_field: bool = True, **kwargs): - # We update the kwargs that everyone are using, we also tell weather we should put this in the Return value or just in the message + # Update shared kwargs and record whether the swarm msg goes to the return value. async with self.apipeline(): self.kwargs.update(**{SWARM_MESSAGE_PARAM_NAME: msg}) self.config.send_swarm_message_to_return_field = set_return_field @@ -141,12 +140,13 @@ async def aio_run_tasks_in_swarm( async def change_status(self, status: SignatureStatus): paused_chain_tasks = [ - TaskSignature.safe_change_status(task, status) for task in self.task_ids + TaskSignature.safe_change_status(ref.target_key, status) + for ref in self.tasks ] pause_chain = super().change_status(status) await asyncio.gather(pause_chain, *paused_chain_tasks, return_exceptions=True) - # TODO - once there is if statements in rapyer we need to use them to add tasks only if swarm is not closed + # TODO - use rapyer if-statements to add tasks only when the swarm is not closed. async def add_tasks( self, tasks: list[TaskSignatureConvertible], close_on_max_task: bool = True ) -> list[Signature]: @@ -223,14 +223,14 @@ async def activate_success(self, msg): async def suspend(self): await asyncio.gather( - *[TaskSignature.suspend_from_key(swarm_id) for swarm_id in self.task_ids], + *[TaskSignature.suspend_from_key(ref.target_key) for ref in self.tasks], return_exceptions=True, ) await super().change_status(SignatureStatus.SUSPENDED) async def resume(self): await asyncio.gather( - *[TaskSignature.resume_from_key(task_id) for task_id in self.task_ids], + *[TaskSignature.resume_from_key(ref.target_key) for ref in self.tasks], return_exceptions=True, ) await super().change_status(self.task_status.last_status) From 3545cab4ae21b11a9af3dbf595ea9ddc523ca7a8 Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Sun, 26 Jul 2026 15:30:07 +0300 Subject: [PATCH 09/14] perf: O(1) chain next-task via stored index; single afind for sub_tasks - on_sub_task_done reads sub_task.chain_index (stored on each chain member at creation) and indexes the next task directly, instead of scanning the whole tasks list. chain_index lives on Signature so chain members of any type (task/swarm/chain) carry it. - sub_tasks goes back to a single batched rapyer.afind (one Redis round trip) over the references' target keys, rather than per-reference afetch. Refs #136 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/unit/test_container_astatus.py | 6 ++---- libs/third-magic/thirdmagic/chain/creator.py | 3 ++- libs/third-magic/thirdmagic/chain/model.py | 20 +++++++++---------- .../third-magic/thirdmagic/signature/model.py | 2 ++ libs/third-magic/thirdmagic/swarm/model.py | 3 ++- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/libs/third-magic/tests/unit/test_container_astatus.py b/libs/third-magic/tests/unit/test_container_astatus.py index 387688c6..526bea0f 100644 --- a/libs/third-magic/tests/unit/test_container_astatus.py +++ b/libs/third-magic/tests/unit/test_container_astatus.py @@ -2,7 +2,6 @@ import thirdmagic from thirdmagic import ContainerStatus -from thirdmagic.chain.model import ChainTaskSignature from thirdmagic.signature.status import SignatureStatus @@ -120,9 +119,8 @@ async def test_chain_astatus_classifies_children(mock_task_def): is_done=False, ) - # Act: astatus runs on a freshly-loaded container (as mageflow.astatus does) - reloaded = await ChainTaskSignature.aget(chain.key) - status = await reloaded.astatus() + # Act + status = await chain.astatus() # Assert assert status == expected diff --git a/libs/third-magic/thirdmagic/chain/creator.py b/libs/third-magic/thirdmagic/chain/creator.py index 0c910ca3..c3b6d53b 100644 --- a/libs/third-magic/thirdmagic/chain/creator.py +++ b/libs/third-magic/thirdmagic/chain/creator.py @@ -37,8 +37,9 @@ async def chain( kwargs=kwargs, ) async with rapyer.apipeline(use_existing_pipe=True): - for task in tasks: + for index, task in enumerate(tasks): task.signature_container_id = chain_task_signature.key + task.chain_index = index await chain_task_signature.asave() await cache_signature(chain_task_signature) diff --git a/libs/third-magic/thirdmagic/chain/model.py b/libs/third-magic/thirdmagic/chain/model.py index c6eb38aa..0d283f27 100644 --- a/libs/third-magic/thirdmagic/chain/model.py +++ b/libs/third-magic/thirdmagic/chain/model.py @@ -31,16 +31,14 @@ def task_ids(self) -> list[RapyerKey]: return [ref.target_key for ref in self.tasks] async def on_sub_task_done(self, sub_task: TaskSignature, results: Any): - # If this is the last task, activate chain success callbacks - if self.tasks[-1].target_key == sub_task.key: + next_idx = sub_task.chain_index + 1 + # If this was the last task, activate chain success callbacks + if next_idx >= len(self.tasks): await self.ClientAdapter.acall_chain_done(results, self) return - for idx, ref in enumerate(self.tasks): - if ref.target_key == sub_task.key: - next_task = await rapyer.aget(self.tasks[idx + 1].target_key) - next_task = cast(TaskSignature, next_task) - await next_task.acall(results, set_return_field=True, **self.kwargs) - return + next_task = await rapyer.aget(self.tasks[next_idx].target_key) + next_task = cast(TaskSignature, next_task) + await next_task.acall(results, set_return_field=True, **self.kwargs) async def on_sub_task_error( self, sub_task: TaskSignature, error: BaseException, original_msg: dict @@ -48,10 +46,10 @@ async def on_sub_task_error( await self.ClientAdapter.acall_chain_error(original_msg, error, self, sub_task) async def sub_tasks(self) -> list[TaskSignature]: - sub_tasks = await asyncio.gather( - *(ref.afetch() for ref in self.tasks), return_exceptions=True + sub_tasks = await rapyer.afind( + *(ref.target_key for ref in self.tasks), skip_missing=True ) - return [task for task in sub_tasks if isinstance(task, TaskSignature)] + return cast(list[TaskSignature], sub_tasks) async def astatus(self) -> ContainerStatus: sub_tasks = await self.sub_tasks() diff --git a/libs/third-magic/thirdmagic/signature/model.py b/libs/third-magic/thirdmagic/signature/model.py index 02adbf48..783b0bfa 100644 --- a/libs/third-magic/thirdmagic/signature/model.py +++ b/libs/third-magic/thirdmagic/signature/model.py @@ -34,6 +34,8 @@ class Signature(AtomicRedisModel, ABC): error_callbacks: RedisList[RapyerKey] = Field(default_factory=list) task_status: TaskStatus = Field(default_factory=TaskStatus) signature_container_id: Optional[RapyerKey] = None + # Position of this signature within its chain container (None outside a chain). + chain_index: Optional[int] = None Meta: ClassVar[RedisConfig] = RedisConfig(ttl=24 * 60 * 60, refresh_ttl=False) ClientAdapter: ClassVar[BaseClientAdapter] = DefaultClientAdapter() diff --git a/libs/third-magic/thirdmagic/swarm/model.py b/libs/third-magic/thirdmagic/swarm/model.py index 3ce3d152..6a9c0f2c 100644 --- a/libs/third-magic/thirdmagic/swarm/model.py +++ b/libs/third-magic/thirdmagic/swarm/model.py @@ -1,6 +1,7 @@ import asyncio from typing import Annotated, Any, ClassVar, Optional, Self, cast +import rapyer from pydantic import BaseModel, Field, field_validator from rapyer import AtomicRedisModel from rapyer.cascade import CascadeTTL @@ -72,7 +73,7 @@ def task_ids(self) -> list[RapyerKey]: return [ref.target_key for ref in self.tasks] async def sub_tasks(self) -> list[TaskSignature]: - tasks = await asyncio.gather(*(ref.afetch() for ref in self.tasks)) + tasks = await rapyer.afind(*(ref.target_key for ref in self.tasks)) return cast(list[TaskSignature], tasks) async def on_sub_task_done(self, sub_task: TaskSignature, results: Any): From 59d063bd9c1cbf33d52e7bdcea46bc6efb9efe09 Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Sun, 26 Jul 2026 15:40:13 +0300 Subject: [PATCH 10/14] refactor: chain progress pointer on the chain, not the task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the leaky chain_index-on-Signature with a current_index pointer on ChainTaskSignature. on_sub_task_done uses it as an O(1) fast path, and only falls back to a scan (re-syncing the pointer) when the pointer does not match the completed task — which also keeps it correct under retry double-fires. Tasks no longer carry any chain-position state. Refs #136 Co-Authored-By: Claude Opus 4.8 (1M context) --- libs/third-magic/thirdmagic/chain/creator.py | 3 +-- libs/third-magic/thirdmagic/chain/model.py | 16 +++++++++++++++- libs/third-magic/thirdmagic/signature/model.py | 2 -- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/libs/third-magic/thirdmagic/chain/creator.py b/libs/third-magic/thirdmagic/chain/creator.py index c3b6d53b..0c910ca3 100644 --- a/libs/third-magic/thirdmagic/chain/creator.py +++ b/libs/third-magic/thirdmagic/chain/creator.py @@ -37,9 +37,8 @@ async def chain( kwargs=kwargs, ) async with rapyer.apipeline(use_existing_pipe=True): - for index, task in enumerate(tasks): + for task in tasks: task.signature_container_id = chain_task_signature.key - task.chain_index = index await chain_task_signature.asave() await cache_signature(chain_task_signature) diff --git a/libs/third-magic/thirdmagic/chain/model.py b/libs/third-magic/thirdmagic/chain/model.py index 0d283f27..b87989c3 100644 --- a/libs/third-magic/thirdmagic/chain/model.py +++ b/libs/third-magic/thirdmagic/chain/model.py @@ -23,6 +23,8 @@ class ChainTaskSignature(ContainerTaskSignature): tasks: Annotated[list[Reference[TaskSignature]], CascadeTTL()] = Field( default_factory=list ) + # Index of the currently-running sub-task; a cached pointer, re-synced on miss. + current_index: int = 0 Meta: ClassVar[RedisConfig] = container_ttl_cascade_meta() @@ -31,11 +33,23 @@ def task_ids(self) -> list[RapyerKey]: return [ref.target_key for ref in self.tasks] async def on_sub_task_done(self, sub_task: TaskSignature, results: Any): - next_idx = sub_task.chain_index + 1 + idx = self.current_index + # Pointer stale (e.g. a retry) — re-locate the completed task by scanning. + if idx >= len(self.tasks) or self.tasks[idx].target_key != sub_task.key: + idx = next( + ( + i + for i, ref in enumerate(self.tasks) + if ref.target_key == sub_task.key + ), + len(self.tasks), + ) + next_idx = idx + 1 # If this was the last task, activate chain success callbacks if next_idx >= len(self.tasks): await self.ClientAdapter.acall_chain_done(results, self) return + await self.aupdate(current_index=next_idx) next_task = await rapyer.aget(self.tasks[next_idx].target_key) next_task = cast(TaskSignature, next_task) await next_task.acall(results, set_return_field=True, **self.kwargs) diff --git a/libs/third-magic/thirdmagic/signature/model.py b/libs/third-magic/thirdmagic/signature/model.py index 783b0bfa..02adbf48 100644 --- a/libs/third-magic/thirdmagic/signature/model.py +++ b/libs/third-magic/thirdmagic/signature/model.py @@ -34,8 +34,6 @@ class Signature(AtomicRedisModel, ABC): error_callbacks: RedisList[RapyerKey] = Field(default_factory=list) task_status: TaskStatus = Field(default_factory=TaskStatus) signature_container_id: Optional[RapyerKey] = None - # Position of this signature within its chain container (None outside a chain). - chain_index: Optional[int] = None Meta: ClassVar[RedisConfig] = RedisConfig(ttl=24 * 60 * 60, refresh_ttl=False) ClientAdapter: ClassVar[BaseClientAdapter] = DefaultClientAdapter() From 050f13fa0dd35dba3874645678ab14257867eb17 Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Sun, 26 Jul 2026 15:50:36 +0300 Subject: [PATCH 11/14] test: run thirdmagic cascade integration on testcontainers + CI; drop asave in add_tasks - add_tasks no longer calls asave(); reassigning self.tasks inside the apipeline persists safely without a blind full-document overwrite that could clobber a concurrent worker (review feedback). - Integration tests spin up a real Redis Stack via testcontainers (AsyncRedisContainer) instead of a fixed URL, session-scoped, and run in CI (new 'Run Integration Tests' step; ubuntu runners have Docker). - Add testcontainers[redis] to thirdmagic dev deps; ignore .DS_Store/.codex. Refs #136 Co-Authored-By: Claude Opus 4.8 (1M context) --- .DS_Store | Bin 8196 -> 0 bytes .codex/config.toml | 6 - .github/workflows/ci.yml | 4 + .gitignore | 5 +- libs/.DS_Store | Bin 8196 -> 0 bytes libs/mageflow/.DS_Store | Bin 6148 -> 0 bytes libs/third-magic/.DS_Store | Bin 6148 -> 0 bytes libs/third-magic/pyproject.toml | 1 + .../third-magic/tests/integration/conftest.py | 36 ++- .../tests/integration/test_cascade_ttl.py | 4 +- libs/third-magic/thirdmagic/swarm/model.py | 3 +- libs/third-magic/uv.lock | 205 ++++++++++++++++++ 12 files changed, 242 insertions(+), 22 deletions(-) delete mode 100644 .DS_Store delete mode 100644 .codex/config.toml delete mode 100644 libs/.DS_Store delete mode 100644 libs/mageflow/.DS_Store delete mode 100644 libs/third-magic/.DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 80e6e1da223a18a77d1f7fbd7462cd90d081e6ad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHML2nyH6n^8lc%3wfle!_L0$KHiLr5J;Q7aBOPNEVhB#P6J&=#|6Z=7v5JJ#;n zj){VNhXdk_ICF**H!l1E4xEtS#tm`hRKA(nb=K?piV)qAX5Q?~_ukC=X6KD(O+>8H zZmbY35|M^1WNr$@Gm_{Rc_gKR7NN|bPt>LcnVUAR8}~8#N9nZ z%TyySC50a&*reM>Dx2MA+NMED4DcG!GR=o?$qGec1dJ<*m zGOf`&v`Qb)J=&oAv<10LC)60#V>1y3(qNTiBrk@8-$OW!4zW%g3<=MjxLx7x5PlZ; zHn456y7U^Juy%vz*k}yz$dGm#Yhqy)V>rRpvvEe(YZ~ZRCwdoo9E|Q?P~GnTRXL5`^4d& z<5fap!MR4$>UmIpf!e@vtKJD5V!wtC{g#^xUCK^QU7ntq&CShU$)@QxC%WZp`dzNo;=ireC$~S%UWYan17`y|z zw_2jh+cRAcvX+bK>>uc^t2<4z(tB*{hG}m(74G}|L?DGV__Xbs!jhauwWZfBr&)aa zFU4uGyd73vRh1kSRRwpUd{_%4EB^^(I_E?j(SetcNS6N7fYyL`LF%Ybe%3#eqRoAh( z(?pxVXE5R>#@)cCuF}Tv1{HyCXe(&gG8_Sw%K$YR$PoJ2XOWFRiQsS?Zhq+W@Ei-5 za#KfjW@K1u$w2S~8@i4St%BN!uAE;~H%mM4WR=$GeaLa4>CeavGYGJCjAda|KOPFi zVa1|o5SFBI3{0qwMREV%82|l$oO;rpFpx0tKVX2R*Q@K|M{sx^#eKpLEGi&>f-I82 x^_7$=CU|2uRNV;3u)T+$HC~dHV%6o1zia0nl6NJ{~MEM;LRkOD;@20{u#1*BFaEiJSJC%FVu=Pt6Hq@}8q z9R~gY{s3l}*jV@n7?_Y?V`XK6cONNHLN<^J+(~!8=X)Q%``OR-GXRimqdWyL0RR?u zfp#2+Uo@c?y{`@F8KETT4^ROgIJ9PcUh1GnA>;wkzoh_O*;JvT3 zW;6^K2L4M1#Ph+yE-<69tx!HXaF8Vc;s{R5hBDRx(#O%5(b!ffT`{Jr9*A}&y2K#L z9rcbdN6ct!D^%`4lsgc8W}+(;BF~O~jx-07QD{cPfMKAY0ae`Bz=bN%?)tsx^CFsy zbjo5s{arR0z=9t8Vk@j``!~h9?}br1{ecV(kL=pL$J%4Xt<~Hk-pWO8)CjX~a9{M&!kS-W{Ha_z5dHjs{pxsoe1=lr#=KHfZ;X8G zZ0wUg+SyucZ+zeW0|)KV#GzzjEHOTw96vm9qkp>}R!S3-{T6TlLWmTHpSoPf^ z_2*cYM-i_HOA!tCG#^kANiHK-cS|0trmlQd+@><~9pR)BY)B~q8Bca3jU@Hf`z+-= ze*YFPB!x#8QhRG;Nku&5Xxk_%5+hl<(k}9c_ z25FL(iZIl<_9t{i4Wx}i-?eY%0_Jm8MV;`ALj;e&g8<674wV2Nl*F&Du2gx2mffi` z!0TU0m;u`}X~Wrnk~RbGfS%;RtT1iIF)*xFG$HE$>x2LQ@3?iF1sMhm1HX>}GIA|< zO?>2czK?oCi(1>k{t&xp!mh1Qy5Jy8#JnuW5p_8(=NzreXlyH#uIM9O5mhDs$A1Kv R&*tCx*=*kb;%5E}+Yf-v5Ul_J diff --git a/libs/mageflow/.DS_Store b/libs/mageflow/.DS_Store deleted file mode 100644 index a65821c9a80a4c86a7b0e2d66e82373064f88d26..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKyG{c^3>-s>lW0;>?k^yIU=@Xe67d5NUd^GaP}-~bF8&P04~gom&E z-8k)d=0EI;e*PZDl}rjq0VyB_q<|FojRM|#X~RjPq7;wR+`&WTBf#m#(9-E1|XSlrHhi*i^` zRFncz;820%+^)U!2`1 lJ0?au=EmFcX%uB$^EJ-s>h%_lF_YV+%usVf;nje595+DUdLeO8ucjeDu{175KXpm^oSh8o= z>$%NMagG7l@VdPOmH_5-N4$BMnxDH*?4&YAr1KeDJmC)ac*JH>{e8l@XL!K_InQ7D z*Sq~@z1y#O?mupzU%ZELC6fYDKnh3!DIf)YtAO`j+I*3yC*bB$R z_;fJD2tb@M9maLc62#^SVlNyMnW0%yiAl8@F)ZoKx2o%fV`9=_aWkJ&H(O087Pm9s zq8!!}6{Ua_I9A{^w@dH;SM(p||6`JNQa}p)D+O$}x>*f;rRuGdm-AlR=(lvQIndp> p4hloGV`8*pZoD0zM^V-_U-P^dj)_5MKIlaK8E{=>QsA!@_y9mf8@d1h diff --git a/libs/third-magic/pyproject.toml b/libs/third-magic/pyproject.toml index 50c8484e..1d921cc5 100644 --- a/libs/third-magic/pyproject.toml +++ b/libs/third-magic/pyproject.toml @@ -20,6 +20,7 @@ dev = [ "hatchet-sdk>=1.23.0", "black>=26.1.0", "ruff>=0.15.5", + "testcontainers[redis]>=4.14.0,<5.0.0", ] [build-system] diff --git a/libs/third-magic/tests/integration/conftest.py b/libs/third-magic/tests/integration/conftest.py index 93091df1..1c13d64f 100644 --- a/libs/third-magic/tests/integration/conftest.py +++ b/libs/third-magic/tests/integration/conftest.py @@ -1,30 +1,44 @@ -import os +import warnings from unittest.mock import MagicMock import pytest import pytest_asyncio import rapyer -import redis.asyncio as aioredis from thirdmagic.clients import BaseClientAdapter from thirdmagic.signature import Signature # Cascade needs a real Redis Stack (Functions + RedisJSON); fakeredis cannot emulate it. -REDIS_URL = os.environ.get("THIRDMAGIC_TEST_REDIS_URL", "redis://localhost:6379") +REDIS_STACK_IMAGE = "redis/redis-stack-server:7.2.0-v13" -@pytest_asyncio.fixture -async def real_redis(): - client = aioredis.Redis.from_url(REDIS_URL, decode_responses=True) - await client.flushall() - await rapyer.init_rapyer(client) - try: +@pytest_asyncio.fixture(scope="session", loop_scope="session") +async def _redis_container(): + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="The @wait_container_is_ready decorator is deprecated.*", + category=DeprecationWarning, + ) + from testcontainers.redis import AsyncRedisContainer + + with AsyncRedisContainer(image=REDIS_STACK_IMAGE) as container: + client = await container.get_async_client(decode_responses=True) yield client - finally: - await client.flushall() await client.aclose() +@pytest_asyncio.fixture(loop_scope="session") +async def real_redis(_redis_container): + await _redis_container.flushall() + await rapyer.init_rapyer(_redis_container) + try: + yield _redis_container + finally: + await rapyer.teardown_rapyer() + await _redis_container.flushall() + + @pytest.fixture def mock_adapter(): adapter = MagicMock(spec=BaseClientAdapter) diff --git a/libs/third-magic/tests/integration/test_cascade_ttl.py b/libs/third-magic/tests/integration/test_cascade_ttl.py index 4138f43b..5fdde3b0 100644 --- a/libs/third-magic/tests/integration/test_cascade_ttl.py +++ b/libs/third-magic/tests/integration/test_cascade_ttl.py @@ -11,7 +11,7 @@ class Msg(BaseModel): x: int = 0 -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="session") async def test_swarm_write_cascades_ttl_to_sub_tasks(real_redis, mock_adapter): # Arrange swarm = await thirdmagic.swarm(task_name="cascade_swarm") @@ -31,7 +31,7 @@ async def test_swarm_write_cascades_ttl_to_sub_tasks(real_redis, mock_adapter): assert await real_redis.ttl(task.key) > SHORT_TTL -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="session") async def test_chain_write_cascades_ttl_to_sub_tasks(real_redis, mock_adapter): # Arrange tasks = [ diff --git a/libs/third-magic/thirdmagic/swarm/model.py b/libs/third-magic/thirdmagic/swarm/model.py index 6a9c0f2c..87ff0606 100644 --- a/libs/third-magic/thirdmagic/swarm/model.py +++ b/libs/third-magic/thirdmagic/swarm/model.py @@ -170,9 +170,8 @@ async def add_tasks( async with self.apipeline(use_existing_pipe=True, ignore_redis_error=True): for task in tasks: task.signature_container_id = self.key - self.tasks.extend([Reference(task) for task in tasks]) + self.tasks = self.tasks + [Reference(task) for task in tasks] self.tasks_left_to_run.extend(task_keys) - await self.asave() if close_on_max_task and not self.config.can_add_task(self): # We dont activate check for finish the swarm, this check is done by the tasks that were added. diff --git a/libs/third-magic/uv.lock b/libs/third-magic/uv.lock index 5f328b4e..53670f17 100644 --- a/libs/third-magic/uv.lock +++ b/libs/third-magic/uv.lock @@ -195,6 +195,76 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/51/f975cae76d44274cc2868dc9040ac5d58d464784610234455b4e7b19c6ef/black-26.5.1-py3-none-any.whl", hash = "sha256:4ed7f7da04046d2e488437170797d3b4a4ad83906683bcb7dfc68b673bbce5e2", size = 213693, upload-time = "2026-05-18T16:53:33.964Z" }, ] +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/81/8e983840c6e5b93b33c2ba81aa3d52c2e42f0e9a690ce7607a2e61da4a5c/charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a", size = 322240, upload-time = "2026-07-07T14:32:36.236Z" }, + { url = "https://files.pythonhosted.org/packages/de/d1/b4319dc3229d8272fba305e206fc0a148e2de8d4087917ce62ae6382f359/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616", size = 216475, upload-time = "2026-07-07T14:32:38.142Z" }, + { url = "https://files.pythonhosted.org/packages/80/33/6c99c1b3e6b8bf730e1bc809b9a2608f224145069114c479a2e9e1494346/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209", size = 238670, upload-time = "2026-07-07T14:32:39.658Z" }, + { url = "https://files.pythonhosted.org/packages/7f/f4/ffbb83546e1f198ecc70ecd372b65cf2b50f9068b380abd67640f17a8e18/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99", size = 233476, upload-time = "2026-07-07T14:32:41.155Z" }, + { url = "https://files.pythonhosted.org/packages/e8/5f/b98b8da398637b551e427e7be922bdec19177dc54d6811dcdaa503f23aac/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8", size = 223817, upload-time = "2026-07-07T14:32:42.592Z" }, + { url = "https://files.pythonhosted.org/packages/36/31/a276bb2e66243072a3fd06fdcab9cbb61a305b02143d70d2bda21d888fa8/charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b", size = 207974, upload-time = "2026-07-07T14:32:44.258Z" }, + { url = "https://files.pythonhosted.org/packages/5e/be/7ee4453d7e88dfbc4104ccd34900b9f2c7c17dac22881865fe0e82424a25/charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2", size = 221655, upload-time = "2026-07-07T14:32:45.64Z" }, + { url = "https://files.pythonhosted.org/packages/1d/85/181c652953eb5276d198f375b1dd641047392050098100a3a02d6534f657/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9", size = 219229, upload-time = "2026-07-07T14:32:47.376Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e7/aaf6da33fc9f4691cda8f7efbc9f69179d3d39ec8a4799baf273ee1d8db0/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15", size = 209704, upload-time = "2026-07-07T14:32:48.855Z" }, + { url = "https://files.pythonhosted.org/packages/63/01/f2fb3bd3a73be48b173ee0c6aa8d2497af97d5663a8c4c4b491de4c62f7a/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d", size = 226243, upload-time = "2026-07-07T14:32:50.239Z" }, + { url = "https://files.pythonhosted.org/packages/c4/02/c57a22739fe05246b0b5783b3bfb6afaac4eebb46f3ececdfb2f048f780e/charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381", size = 150935, upload-time = "2026-07-07T14:32:51.676Z" }, + { url = "https://files.pythonhosted.org/packages/37/8d/ca39a7559a4797505530d084fd3a49a2c959efbbbff146302fb7be4e3b35/charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee", size = 162314, upload-time = "2026-07-07T14:32:53.193Z" }, + { url = "https://files.pythonhosted.org/packages/01/da/a44bd7a13d426e69e4894557106cd58669097bfad4a8681123b618fbfc5d/charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419", size = 153075, upload-time = "2026-07-07T14:32:54.554Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e3/85ec501f206fb049259288c1f3506e53876937fb00edb47009348e66756b/charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5", size = 317075, upload-time = "2026-07-07T14:32:56.021Z" }, + { url = "https://files.pythonhosted.org/packages/c3/69/2a5385192e67175f7d8bd5ce4f57c24bc956439adeae5c13a99aa28a53d1/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2", size = 213837, upload-time = "2026-07-07T14:32:57.78Z" }, + { url = "https://files.pythonhosted.org/packages/b3/46/03ddc7da576d814fe0a36dd1f0fd3258e95404b4b2e3c026b7923d7e133f/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a", size = 235503, upload-time = "2026-07-07T14:32:59.205Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6e/de0229a7ef40f6f9d28a837eebf4ec47bdca5dab4e900c84f22919af636a/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29", size = 229944, upload-time = "2026-07-07T14:33:00.803Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/49b9060e8418b14fb5cba9cf6bfb383111e2538a03a1fb18e66a95aeb3d5/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c", size = 221276, upload-time = "2026-07-07T14:33:02.199Z" }, + { url = "https://files.pythonhosted.org/packages/44/95/80282cce0fae9c3061203d723ee87da996aed79679e65d8935050ee7ca1f/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b", size = 205260, upload-time = "2026-07-07T14:33:03.698Z" }, + { url = "https://files.pythonhosted.org/packages/0c/74/2f62c8821b969ea3bd67cc2e6976834f48ca5d12664d2559ebcd9bcfbed7/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db", size = 217786, upload-time = "2026-07-07T14:33:05.12Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8d/feabb82cb49fcad14515b1d7d1ca4787b0da7fc723a212bf89bc9e0fac52/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993", size = 216798, upload-time = "2026-07-07T14:33:06.629Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ff/c946d63bc3786d5b84d960b0f7ab7e25b828486a946b5aa997625bcaf6a6/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da", size = 206429, upload-time = "2026-07-07T14:33:08.006Z" }, + { url = "https://files.pythonhosted.org/packages/af/ba/5e5007c370702f85d2ef75791fac7943ed41e080364a673b20142e430e3e/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3", size = 223066, upload-time = "2026-07-07T14:33:09.783Z" }, + { url = "https://files.pythonhosted.org/packages/83/d5/9096aa3cf532dfad237861544eb47a0f20d5adbf1039760fed8eaae935d9/charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d", size = 150456, upload-time = "2026-07-07T14:33:11.217Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a1/e29995109e455dc8eff8d0fac6ae509be39561318a7cfeac5d33ad029213/charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1", size = 161410, upload-time = "2026-07-07T14:33:12.743Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8d/1569f4d0032d6ba2a4fe4591c35bf87868c600c41a71eb5c2e1ffa8464c2/charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec", size = 152649, upload-time = "2026-07-07T14:33:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, + { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" }, + { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, + { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, + { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, + { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + [[package]] name = "click" version = "8.3.1" @@ -304,6 +374,20 @@ toml = [ { name = "tomli", marker = "python_full_version <= '3.11'" }, ] +[[package]] +name = "docker" +version = "7.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/7f/731ff914b0255d3d065f45fd4e626d4b8c95dbcbaada049f337a6ac16410/docker-7.2.0.tar.gz", hash = "sha256:cebb93773d334f778e023a7ee352a8d6e13ab1bd3b863a4d4a59dec897df43ac", size = 118731, upload-time = "2026-07-09T14:53:46.39Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/23/529140fe1aab80fc6992f93a706deec709140a6397439139a054e1515c45/docker-7.2.0-py3-none-any.whl", hash = "sha256:a3f45fdeb9165e2d25d9a1d02ddf3bc70fb572cf5ebbf9b58558c22caf29b71f", size = 148775, upload-time = "2026-07-09T14:53:45.224Z" }, +] + [[package]] name = "exceptiongroup" version = "1.3.1" @@ -1097,6 +1181,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, ] +[[package]] +name = "pywin32" +version = "312" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/1b/9cfdeac80ee45bebbbcb31f1b7b99a0d81a1c72de48d837be984e0e88b1d/pywin32-312-cp310-cp310-win32.whl", hash = "sha256:772235332b5d1024c696f11cea1ae4be7930f0a8b894bb43db14e3f435f1ff7e", size = 6361387, upload-time = "2026-06-04T07:49:14.329Z" }, + { url = "https://files.pythonhosted.org/packages/33/b1/7afc96d041d982c27bc2df6f853d43f01fd273e3d39d04be3647ddeb533d/pywin32-312-cp310-cp310-win_amd64.whl", hash = "sha256:5dbc35d2b5320dc07f25fa31269cfb767471002b17de5eb067d03da68c7cb2db", size = 6926780, upload-time = "2026-06-04T07:49:16.881Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/4140da9ad54108e517f4a16b2d83da3033e08662144623e1239587cb7db6/pywin32-312-cp310-cp310-win_arm64.whl", hash = "sha256:3020656e34f1cf7faeb7bccd2b84653a607c6ff0c55ada85e6487d61716deabd", size = 4307203, upload-time = "2026-06-04T07:49:18.993Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f5/10a6e845a00fc5e7afd0a988b744f403d4d57162a28d160a093c4d9322f0/pywin32-312-cp311-cp311-win32.whl", hash = "sha256:17948aeadbdb091f0ced6ef0841620794e68327b94ee415571c1203594b7215c", size = 6362659, upload-time = "2026-06-04T07:49:21.349Z" }, + { url = "https://files.pythonhosted.org/packages/35/c4/dcd2d62b5944b6d5db53413a5899016ccd57ffcb7278f3f81655d25d2027/pywin32-312-cp311-cp311-win_amd64.whl", hash = "sha256:d11417d84412f859b722fad0841b3614459ed0047f7542d8362e77884f6b6e8a", size = 6928825, upload-time = "2026-06-04T07:49:23.934Z" }, + { url = "https://files.pythonhosted.org/packages/b7/56/3cbb433fe4501cdba2eb9040f56a4e1a8243faa4186b25295564d1a7a79d/pywin32-312-cp311-cp311-win_arm64.whl", hash = "sha256:b2200a054ca6d6625c4842fc56a4976a4b47f96b73dbe5538c3f813a80359f47", size = 6721875, upload-time = "2026-06-04T07:49:26.416Z" }, + { url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", size = 6371877, upload-time = "2026-06-04T07:49:28.836Z" }, + { url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", size = 6914841, upload-time = "2026-06-04T07:49:31.85Z" }, + { url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", size = 6727901, upload-time = "2026-06-04T07:49:34.244Z" }, + { url = "https://files.pythonhosted.org/packages/2d/41/12fbfd7f36ed2146d8bc9de96c2741296bf0d490b98508496cff322e274c/pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c", size = 6370184, upload-time = "2026-06-04T07:49:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/ba/db/36a78e3403099d31d9746d13fdcde5accc43c1155f375a34d15983a479a7/pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9", size = 6914298, upload-time = "2026-06-04T07:49:38.876Z" }, + { url = "https://files.pythonhosted.org/packages/84/37/c1697194092b76de9ed47ca124323f02c57ffc8a45c06f88a3d5acaf01eb/pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831", size = 6727640, upload-time = "2026-06-04T07:49:41.083Z" }, +] + [[package]] name = "rapyer" version = "1.3.4" @@ -1122,6 +1225,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/97/9f22a33c475cda519f20aba6babb340fb2f2254a02fb947816960d1e669a/redis-7.0.1-py3-none-any.whl", hash = "sha256:4977af3c7d67f8f0eb8b6fec0dafc9605db9343142f634041fb0235f67c0588a", size = 339938, upload-time = "2025-10-27T14:33:58.553Z" }, ] +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + [[package]] name = "ruff" version = "0.15.8" @@ -1183,6 +1301,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, ] +[[package]] +name = "testcontainers" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docker" }, + { name = "python-dotenv" }, + { name = "typing-extensions" }, + { name = "urllib3" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/13/2cc466bddf26d0085f30a2b2bd56b7f8708b54a54db833eec97c5c69129b/testcontainers-4.15.0.tar.gz", hash = "sha256:085cde086337632e19002719460b7b80bbab2bdd51bb3ea04f77d0de96504706", size = 95340, upload-time = "2026-07-24T23:08:01.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/7e/424aac8b355597835deb333e757a0e94b5ccf38ad00f07fe6ed1f4e17c88/testcontainers-4.15.0-py3-none-any.whl", hash = "sha256:8796c14e76604031ad39cf0ed3b8e9806283a1fbf5270965c2b1c594caa31b74", size = 160771, upload-time = "2026-07-24T23:08:00.13Z" }, +] + +[package.optional-dependencies] +redis = [ + { name = "redis" }, +] + [[package]] name = "thirdmagic" version = "0.0.6" @@ -1201,6 +1340,7 @@ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "ruff" }, + { name = "testcontainers", extra = ["redis"] }, ] [package.metadata] @@ -1214,6 +1354,7 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.2.0,<2.0.0" }, { name = "rapyer", specifier = ">=1.3.4,<1.4.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.5" }, + { name = "testcontainers", extras = ["redis"], marker = "extra == 'dev'", specifier = ">=4.14.0,<5.0.0" }, ] provides-extras = ["dev"] @@ -1283,6 +1424,70 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] +[[package]] +name = "wrapt" +version = "2.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068, upload-time = "2026-06-20T23:49:44.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/8b/59781d0fe7b0adfbea37f600857de4be68921e454aeecf1a11bda35cdccc/wrapt-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:055e6fcfaa28e58c6a8c247d48b92be9d56f818b7068aa4f22b15b3343a09931", size = 80556, upload-time = "2026-06-20T23:47:28.473Z" }, + { url = "https://files.pythonhosted.org/packages/94/dc/66c61aca927230c9cf97a3cb005c803971a1076ff9f7d61085d035c20085/wrapt-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8374eb6b1a58809211e84ff835a182bb17ab2807a5bfef23204c8cff38178a00", size = 81648, upload-time = "2026-06-20T23:47:30.504Z" }, + { url = "https://files.pythonhosted.org/packages/23/1b/545eee1c18f3af4cf140bb5822b6ef81ebe569df0a63ac109973103a30a5/wrapt-2.2.2-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:656593bb3f5529f03d27af4136c4d7b11990e470bcbc6fefa5ef218695bece55", size = 152956, upload-time = "2026-06-20T23:47:31.867Z" }, + { url = "https://files.pythonhosted.org/packages/44/a7/6f42a3d03e44dc612a5dcff324e7366075a7857f0be2d49a8cb8a68279b8/wrapt-2.2.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfb00cb7bb22099e2f64b7340fb96113639aa7260c0972af3797ace2297b936c", size = 154771, upload-time = "2026-06-20T23:47:33.352Z" }, + { url = "https://files.pythonhosted.org/packages/bf/55/4d76175aaa97523c38f1d28f79d18ab41a1b116814158a818bc0eba00571/wrapt-2.2.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e7f10ee0bd53673bfd52b67cbce83336fe6cad90d2377b03baf66491d2bbfb91", size = 149460, upload-time = "2026-06-20T23:47:34.712Z" }, + { url = "https://files.pythonhosted.org/packages/84/9b/12e23264d8f4735e8483262f95c5a6b03c3665fd2a84bdf99a45b6a2f4ec/wrapt-2.2.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4402f57c5f0d0579599858ffbdd9bf4e3f0972f51096f2bd6cc7dab6b76ee49e", size = 153648, upload-time = "2026-06-20T23:47:36.092Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a3/bcd5ec37289dcd85ecd4d15395a6a6063d60bc45ff94a9d77814e1e54d64/wrapt-2.2.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:3a4eb7964ff4643d333c84f880bcf554652b2a1050aebc54ae696327f61acfaf", size = 148502, upload-time = "2026-06-20T23:47:37.623Z" }, + { url = "https://files.pythonhosted.org/packages/f2/be/716d708f607fa70f8a6eb47dff8ee945d5278dfc89ffeeff33039d052e63/wrapt-2.2.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e542b7c5af91e2123a8aabf19894319d5ec4268d2a9ffd2f239386133fc47746", size = 152238, upload-time = "2026-06-20T23:47:39.118Z" }, + { url = "https://files.pythonhosted.org/packages/b5/c0/1a48e7e54501274f5d906f18372221b13183b0afbb5b8bb4c7ca0392c0b4/wrapt-2.2.2-cp310-cp310-win32.whl", hash = "sha256:6e7e45b43d3c774d244fe7264378f5a3f0f383bc55a54a9866434e524540110f", size = 77278, upload-time = "2026-06-20T23:47:40.476Z" }, + { url = "https://files.pythonhosted.org/packages/b0/82/9cd69a1af288fbdedf01a10e3c8a0b6890b08c7f3f96d36a213699dbcd94/wrapt-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:955f1d6e72a352e478de8d8b503abe301c5e139a141b62eb0923bd694995025f", size = 80131, upload-time = "2026-06-20T23:47:41.785Z" }, + { url = "https://files.pythonhosted.org/packages/7f/73/8db7e27daef37ae70a53ea62bef7fe80cc51a8b5e9e9181a8be6eb9a999c/wrapt-2.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:b89d8d73c82db2bb7e6090b3afd7973f980d24e905cc34394eab60b884b3bf67", size = 79615, upload-time = "2026-06-20T23:47:43.109Z" }, + { url = "https://files.pythonhosted.org/packages/27/15/0c2d55168707465abfc41f33c0b23d792a5fa9b65c26983606940900a120/wrapt-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f1a2ff355ece6a111ca7a20dc86df6659c9205d3fcee674ca34f2a2854fd4e73", size = 80782, upload-time = "2026-06-20T23:47:44.367Z" }, + { url = "https://files.pythonhosted.org/packages/7d/b5/5c0b093eb48f8a062ef6267d3cb36e9bb1b88440181f6545a383c60efdf8/wrapt-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55b9a899e6fff5444f229d30aa6e9ac92d2216d9d60f33c771b5d76a760d5f8e", size = 81678, upload-time = "2026-06-20T23:47:45.857Z" }, + { url = "https://files.pythonhosted.org/packages/34/f3/de70937472dd3e8a4e6811192f9c6075efdffd4a2cd9b4596bf160f89668/wrapt-2.2.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a2d78c363f97d8bd718ee40432c66395685e9e98528ccaa423c3355d1715a26d", size = 159671, upload-time = "2026-06-20T23:47:47.345Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ec/40aed2330e7f02ecf74386ffcfef9ccb7108c6a430f15b6a252b663b1bed/wrapt-2.2.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d619e1eed9bd4f6ed9f24cd61971aa086fa86505289628d464bcf8a2c2e3f328", size = 160785, upload-time = "2026-06-20T23:47:48.759Z" }, + { url = "https://files.pythonhosted.org/packages/45/04/aa5309beed5344b00220ae6b3b24055852192656194c27947bee1736306a/wrapt-2.2.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:518b0c5e323511ec56a38894802ddd5e1222626484e68efe63f201854ad788e5", size = 153699, upload-time = "2026-06-20T23:47:50.177Z" }, + { url = "https://files.pythonhosted.org/packages/01/df/2def7e99d1fe87eea413f95f671924cdddcb08823b1ffd212748dfa6d062/wrapt-2.2.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4bccea5cdecffa9dd70e343741f0e41e0a16619313d04b72f78bb525162ebcd0", size = 159695, upload-time = "2026-06-20T23:47:51.602Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f6/a906d01a2ce12157bad2404957b3e2140da354b8a70b2fa48bbf282871c0/wrapt-2.2.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:209112cafd963710a05d199aae431d79a28bc76eb8e6d1bbbb8ad24340722cae", size = 152813, upload-time = "2026-06-20T23:47:53.03Z" }, + { url = "https://files.pythonhosted.org/packages/02/49/bc0086292d239575b4c08f4cf8a4079fa58abbad58ec23abf84833a283ed/wrapt-2.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e5a5290e4bf2f332fc29ce72ffb9a2fff678aaac047e2e9f5f7165cd7792e099", size = 158809, upload-time = "2026-06-20T23:47:54.391Z" }, + { url = "https://files.pythonhosted.org/packages/55/83/8fbd034de1f3e907edaa18786d5dd8f6932874edee0826c7cecb5cab03a1/wrapt-2.2.2-cp311-cp311-win32.whl", hash = "sha256:5499236ad1dc116012e2a5dd943f3f31af12fce452128e2bbcbd55a7d3d4d14c", size = 77414, upload-time = "2026-06-20T23:47:55.882Z" }, + { url = "https://files.pythonhosted.org/packages/7e/9c/23695baa331c6de4e874c3d78b8e0bed92e1d2a274e665b29858f6841672/wrapt-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:8636809939152be6ae20a6cef0fed9fe60f411b47847d0426a826884b469e971", size = 80368, upload-time = "2026-06-20T23:47:57.237Z" }, + { url = "https://files.pythonhosted.org/packages/08/49/40cefc342bf89b234a4490d741290fce781774b831aefb39c25471da96c9/wrapt-2.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:5d0a142f7af07caeb5e5da87493162a7b8efa19ba919e550a746f7446e13fb30", size = 79489, upload-time = "2026-06-20T23:47:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/2a/85/180b40628b23772692a0c76e8030114e1c0ae068470ed531919f0a5f2a4a/wrapt-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8417fd3c674d3c8023d080292d29301531a12daf8bd938dd419710dd2f464f2b", size = 81484, upload-time = "2026-06-20T23:47:59.924Z" }, + { url = "https://files.pythonhosted.org/packages/94/f2/21c90f2a16689702e2aaff45795b11018dff2c9b1242bac10d225483f676/wrapt-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e7070c7472582e31af3dfc2622b2381a0df7435110a9388ed8db5ffbce67efb", size = 82151, upload-time = "2026-06-20T23:48:01.303Z" }, + { url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828, upload-time = "2026-06-20T23:48:02.719Z" }, + { url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544, upload-time = "2026-06-20T23:48:04.266Z" }, + { url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663, upload-time = "2026-06-20T23:48:05.708Z" }, + { url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387, upload-time = "2026-06-20T23:48:07.243Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849, upload-time = "2026-06-20T23:48:08.91Z" }, + { url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147, upload-time = "2026-06-20T23:48:10.374Z" }, + { url = "https://files.pythonhosted.org/packages/42/63/3eb25da41049d20ae18fcab2dd8b056e02387c4bfa626cbdfb7c3b872e4f/wrapt-2.2.2-cp312-cp312-win32.whl", hash = "sha256:ef2cce266b5b0b07e19fa82e59673b81142b7a3607c8ed1254113d048ed668da", size = 77734, upload-time = "2026-06-20T23:48:11.769Z" }, + { url = "https://files.pythonhosted.org/packages/da/09/0390e008a305360948fa9ce69507d041ac12cb2ee5d28e34467e2ee79391/wrapt-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:abf8c20a2d72ee69e16328b3c91342c446e723bfe48bfcc4dded3b9722ac027f", size = 80585, upload-time = "2026-06-20T23:48:13.117Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b3/84c445c66969f2d3457276b183a48c91097d59bbef9af6c075366b0f8c36/wrapt-2.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:c6c64c5d02578bc4c4bca4f0aef1504de933c1d5b4ac2710b9131111459506c8", size = 79553, upload-time = "2026-06-20T23:48:14.5Z" }, + { url = "https://files.pythonhosted.org/packages/43/fc/f32f4b22c6511173c11d9e541ab4e7d8467a0f1b3455acaf784115d31ff8/wrapt-2.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e8b648270c613720a202d9a45ebabc33261b22c3a839b115ac5bce8c0bb0d69", size = 81296, upload-time = "2026-06-20T23:48:15.881Z" }, + { url = "https://files.pythonhosted.org/packages/72/06/4d117d5d77a9344776c0248b24dae3d3dd2f58e5f765fa08cf887072e719/wrapt-2.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6fb7e94e8fe3e4c3067bb1653a91cce7c5e83acc119fdd41501b1bf74654617", size = 81841, upload-time = "2026-06-20T23:48:17.262Z" }, + { url = "https://files.pythonhosted.org/packages/15/ff/63ad96f98eb58a742b1a20d80f21da88924405910149950b912368150468/wrapt-2.2.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb18fc51e813df0d9c98049e3bf2298a5495a648602040e21fa3c7329371159e", size = 167882, upload-time = "2026-06-20T23:48:18.764Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/8bb62d8933df7acf3247194e6e9fc68edf9d2fa203252c89c94b319dd472/wrapt-2.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94b00b00f806eb3ef2abe9049ed45994a81ee9284884d96e6b8314927c6cea3d", size = 167411, upload-time = "2026-06-20T23:48:20.315Z" }, + { url = "https://files.pythonhosted.org/packages/17/09/8789dcb09ee1de715727db7521aabbb68ffa68dfade3a49468440cfced49/wrapt-2.2.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62415fd095bc590b842b6d092f2b5d9ccbaeb7e0b28535c03dcea2718b48636b", size = 158607, upload-time = "2026-06-20T23:48:21.728Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/66e02562d53ee67d841f175e38e3c993c2d78a3e104c576cad61c028b43c/wrapt-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a41e758d80dc0ab8c210f641ac892009d356cf1f955d97db544c8dd317b4d14c", size = 166367, upload-time = "2026-06-20T23:48:23.177Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a3/832ac4e41222fb263b3042d42c2f08d305db7d0f0c9b1d3a271a9eede8f6/wrapt-2.2.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b84cd4058001c9727b0e9980b7a9e66325b5ca748b1b578e822cade1bc6b304f", size = 157176, upload-time = "2026-06-20T23:48:24.711Z" }, + { url = "https://files.pythonhosted.org/packages/b7/01/1bd5e4d2df9c0178989ac8da9186543465388588ee2ef153e2591accebef/wrapt-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26fc73a1b15e0946d2942b9a4426d162b51676338327dc067ccd8d2d76385f94", size = 167025, upload-time = "2026-06-20T23:48:26.118Z" }, + { url = "https://files.pythonhosted.org/packages/1c/69/583ed25291ab53e1ec117135fb1c33425e2f46d2bc8f29c17f7a94cf4274/wrapt-2.2.2-cp313-cp313-win32.whl", hash = "sha256:3c4095803491f6ef72128914c28ec05bbad9758433bb35f6715a3e9c8e46fb2d", size = 77605, upload-time = "2026-06-20T23:48:27.643Z" }, + { url = "https://files.pythonhosted.org/packages/29/68/e69fc6d06e1523c68e0d00f95c9aed1158ce9908ee41603f7f2eae3d5db6/wrapt-2.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:2cb07f414fab25dbe6b5c7398e1491423a5c81a6209533639969a6c928d474a4", size = 80508, upload-time = "2026-06-20T23:48:29.013Z" }, + { url = "https://files.pythonhosted.org/packages/55/21/fe7a393d9e5dc0923bed8f5d857e9dcff210f1fa0888c02cc8f3ffaa55aa/wrapt-2.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:1fc7691f070220215cccb2a20836b9adbaecb8ff22ad47abe63de5f110994fac", size = 79565, upload-time = "2026-06-20T23:48:30.429Z" }, + { url = "https://files.pythonhosted.org/packages/b6/e5/c120d13bf5091164f68c3c1657e84f16f57e71d978421b626393ac5bd7eb/wrapt-2.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ec8f83949028366531383603139403cac7a826e4011955813cdd640017845ce5", size = 83264, upload-time = "2026-06-20T23:48:31.807Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b0/d4a1eb97e0e286625bdf21bc7f702637f9607787ffbbdb5ec14d50c79dbf/wrapt-2.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4b481fb0c40d9fd90a5809911208da700987d373a20a4709dc9e3944af7a6bec", size = 83791, upload-time = "2026-06-20T23:48:33.482Z" }, + { url = "https://files.pythonhosted.org/packages/18/1e/f060df47755e87b57684cee7bfc1362b204df55fac96ffebc0631b697b79/wrapt-2.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0065a3b657cec06813b4241d2462ccec287f6863103d7445b725fb3a889736f9", size = 203399, upload-time = "2026-06-20T23:48:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/c4/de/2316a757a1abb6453700b79d83e532146dcef2611348282d4d8889792161/wrapt-2.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f7424af5c5c345b7f26490e097f74a2ef45b3d08b664dc33571aee3bd3b56c", size = 210461, upload-time = "2026-06-20T23:48:36.569Z" }, + { url = "https://files.pythonhosted.org/packages/ed/29/d1160785ae18ca2495a6d82a21154103d74f656c9fd457fb35f6b11b965a/wrapt-2.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07fdcb012821859168641acf68afad61ef9783cf37100af85f152550e9677194", size = 195313, upload-time = "2026-06-20T23:48:38.175Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2d/7caa9598ae61a9cf0989cc501739cbeeb7d650ab3193cca1407b9af0c6ab/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f90038ab58fafb584801ca62d72384d7d5225d93c76f7b773c22fae545bd8066", size = 206116, upload-time = "2026-06-20T23:48:39.804Z" }, + { url = "https://files.pythonhosted.org/packages/ac/02/281ea1088b8650d865f311b35cf86fd21df89128e2909714f1161e01c9d0/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c5d7825491bfa2d08b97e9557768987952c7b9ae687d06c3320b40a37ccb7f20", size = 192668, upload-time = "2026-06-20T23:48:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/be/7d/976e2d5b4b5c5babda40974edd54d0a5585cb60132ed86b46f4b80239b16/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ad520e6daa9bbf136f14de735474dbec7dcc0891f718e1d274ce8dc92e645af", size = 198891, upload-time = "2026-06-20T23:48:43.056Z" }, + { url = "https://files.pythonhosted.org/packages/59/b7/e47651797c097f75a37e2ce86dcf04048ff576f3a674f7c558df7b5e9622/wrapt-2.2.2-cp313-cp313t-win32.whl", hash = "sha256:25904acb9475f46c24fe0423dbc8fda8cc5fbc282ab3dc6e72e919748c53f4e9", size = 78537, upload-time = "2026-06-20T23:48:44.509Z" }, + { url = "https://files.pythonhosted.org/packages/d1/6f/9fa5d59fb06d890defb5a8f727ce6a14d2932c8760153f96956628559fee/wrapt-2.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:305d4c247d61c4115794a169141823c62f719525ddb90b23aa332741c77d2c28", size = 82005, upload-time = "2026-06-20T23:48:46.391Z" }, + { url = "https://files.pythonhosted.org/packages/15/80/4c7bd9873d1f9f7d138d93556b500469dbe24f42710b877519c2b9eb380d/wrapt-2.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c20279cd1a29800815d7b2d6338b60a6c6e78263f9d6e62e0eda251ba9cae2d0", size = 80762, upload-time = "2026-06-20T23:48:47.964Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, +] + [[package]] name = "yarl" version = "1.23.0" From 790f0236e87227bee10d37852beb6fc5e6cab286 Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Sun, 26 Jul 2026 15:55:36 +0300 Subject: [PATCH 12/14] return todo --- libs/third-magic/thirdmagic/swarm/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/third-magic/thirdmagic/swarm/model.py b/libs/third-magic/thirdmagic/swarm/model.py index 87ff0606..8a31cf0e 100644 --- a/libs/third-magic/thirdmagic/swarm/model.py +++ b/libs/third-magic/thirdmagic/swarm/model.py @@ -44,7 +44,7 @@ def can_add_n_tasks(self, swarm: "SwarmTaskSignature", n: int) -> bool: class SwarmTaskSignature(ContainerTaskSignature): - # Sub-tasks are ForeignKey edges so a write to the swarm cascades TTL to them. + # TODO - TASKS list should be set once we enable this in rapyer tasks: Annotated[list[Reference[TaskSignature]], CascadeTTL()] = Field( default_factory=list ) From 141163f4eb59abe7cd88cb40e0f31fbb5a99fe2a Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Sun, 26 Jul 2026 16:59:14 +0300 Subject: [PATCH 13/14] fix: guard afind against empty key sets so it can't scan the whole DB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rapyer.afind() with no keys returns every model in the DB. Several call sites spread a possibly-empty iterable into it (task callbacks, sub_tasks), so an empty set silently triggered a full-DB fetch — worst on every task completion via activate_success/activate_error, where callbacks are usually empty. Add afind_keys_guarded(keys) in thirdmagic.utils that short-circuits to [] for an empty key set, and route the unguarded sites through it. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Ge235p3Le4aLDjmP1AubRT --- libs/third-magic/thirdmagic/chain/model.py | 6 +++--- libs/third-magic/thirdmagic/signature/model.py | 8 ++++---- libs/third-magic/thirdmagic/swarm/model.py | 5 ++--- libs/third-magic/thirdmagic/utils.py | 12 +++++++++++- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/libs/third-magic/thirdmagic/chain/model.py b/libs/third-magic/thirdmagic/chain/model.py index b87989c3..0c396c29 100644 --- a/libs/third-magic/thirdmagic/chain/model.py +++ b/libs/third-magic/thirdmagic/chain/model.py @@ -12,7 +12,7 @@ from thirdmagic.errors import MissingSignatureError from thirdmagic.signature.status import ContainerStatus, SignatureStatus from thirdmagic.task.model import TaskSignature -from thirdmagic.utils import HAS_HATCHET +from thirdmagic.utils import HAS_HATCHET, afind_keys_guarded if HAS_HATCHET: from hatchet_sdk.clients.admin import TriggerWorkflowOptions @@ -60,8 +60,8 @@ async def on_sub_task_error( await self.ClientAdapter.acall_chain_error(original_msg, error, self, sub_task) async def sub_tasks(self) -> list[TaskSignature]: - sub_tasks = await rapyer.afind( - *(ref.target_key for ref in self.tasks), skip_missing=True + sub_tasks = await afind_keys_guarded( + (ref.target_key for ref in self.tasks), skip_missing=True ) return cast(list[TaskSignature], sub_tasks) diff --git a/libs/third-magic/thirdmagic/signature/model.py b/libs/third-magic/thirdmagic/signature/model.py index 02adbf48..a6dc4112 100644 --- a/libs/third-magic/thirdmagic/signature/model.py +++ b/libs/third-magic/thirdmagic/signature/model.py @@ -15,7 +15,7 @@ from thirdmagic.clients import BaseClientAdapter, DefaultClientAdapter from thirdmagic.consts import REMOVED_TASK_TTL from thirdmagic.signature.status import PauseActionTypes, SignatureStatus, TaskStatus -from thirdmagic.utils import HAS_HATCHET +from thirdmagic.utils import HAS_HATCHET, afind_keys_guarded if HAS_HATCHET: from hatchet_sdk.clients.admin import TriggerWorkflowOptions @@ -74,12 +74,12 @@ async def on_cancel_signature(self, msg: dict): await self.remove() async def activate_success(self, msg): - success_signatures = await rapyer.afind(*self.success_callbacks) + success_signatures = await afind_keys_guarded(self.success_callbacks) success_signatures = cast(list[Signature], success_signatures) return await self.ClientAdapter.acall_signatures(success_signatures, msg, True) async def activate_error(self, msg): - error_signatures = await rapyer.afind(*self.error_callbacks) + error_signatures = await afind_keys_guarded(self.error_callbacks) error_signatures = cast(list[Signature], error_signatures) return await self.ClientAdapter.acall_signatures(error_signatures, msg, False) @@ -93,7 +93,7 @@ async def remove_branches(self, success: bool = True, errors: bool = True): if success: keys_to_remove.extend([success_id for success_id in self.success_callbacks]) - signatures = cast(list[Signature], await rapyer.afind(*keys_to_remove)) + signatures = cast(list[Signature], await afind_keys_guarded(keys_to_remove)) await asyncio.gather(*[signature.remove() for signature in signatures]) async def remove_references(self): diff --git a/libs/third-magic/thirdmagic/swarm/model.py b/libs/third-magic/thirdmagic/swarm/model.py index 8a31cf0e..c687684f 100644 --- a/libs/third-magic/thirdmagic/swarm/model.py +++ b/libs/third-magic/thirdmagic/swarm/model.py @@ -1,7 +1,6 @@ import asyncio from typing import Annotated, Any, ClassVar, Optional, Self, cast -import rapyer from pydantic import BaseModel, Field, field_validator from rapyer import AtomicRedisModel from rapyer.cascade import CascadeTTL @@ -21,7 +20,7 @@ from thirdmagic.swarm.state import PublishState from thirdmagic.task.creator import TaskSignatureConvertible, resolve_signatures from thirdmagic.task.model import TaskSignature -from thirdmagic.utils import HAS_HATCHET +from thirdmagic.utils import HAS_HATCHET, afind_keys_guarded if HAS_HATCHET: from hatchet_sdk.clients.admin import TriggerWorkflowOptions @@ -73,7 +72,7 @@ def task_ids(self) -> list[RapyerKey]: return [ref.target_key for ref in self.tasks] async def sub_tasks(self) -> list[TaskSignature]: - tasks = await rapyer.afind(*(ref.target_key for ref in self.tasks)) + tasks = await afind_keys_guarded(ref.target_key for ref in self.tasks) return cast(list[TaskSignature], tasks) async def on_sub_task_done(self, sub_task: TaskSignature, results: Any): diff --git a/libs/third-magic/thirdmagic/utils.py b/libs/third-magic/thirdmagic/utils.py index d9808462..300aea11 100644 --- a/libs/third-magic/thirdmagic/utils.py +++ b/libs/third-magic/thirdmagic/utils.py @@ -1,13 +1,23 @@ import dataclasses -from typing import Callable, Optional, TypeVar, get_type_hints +from typing import Any, Callable, Iterable, Optional, TypeVar, get_type_hints +import rapyer from pydantic import BaseModel +from rapyer.fields import RapyerKey from thirdmagic.message import DEFAULT_RESULT_NAME, ReturnValueAnnotation PropType = TypeVar("PropType", bound=dataclasses.dataclass) +async def afind_keys_guarded(keys: Iterable[RapyerKey], **kwargs) -> list[Any]: + # afind() with no keys scans the whole DB, so an empty key set returns []. + keys = list(keys) + if not keys: + return [] + return await rapyer.afind(*keys, **kwargs) + + def get_marked_fields( model: type[BaseModel], mark_type: type[PropType] ) -> list[tuple[PropType, str]]: From 033c9da13e83eae89fd16f55f270fc69b2ade5b9 Mon Sep 17 00:00:00 2001 From: YedidyaHKfir Date: Sun, 26 Jul 2026 17:02:40 +0300 Subject: [PATCH 14/14] refactor: use atomic list.extend for swarm add_tasks The `tasks` field annotation is a list[Reference] but rapyer converts it to a RedisList at runtime, so `.extend()` is available and issues an atomic append. Replace the read-modify-write reassignment (self.tasks = self.tasks + [...]), which could clobber a concurrent worker's additions, with a race-safe extend. Verified: thirdmagic unit (139) and the swarm cascade integration test pass, so TTL cascade still fires on the extended list. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Ge235p3Le4aLDjmP1AubRT --- libs/third-magic/thirdmagic/swarm/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/third-magic/thirdmagic/swarm/model.py b/libs/third-magic/thirdmagic/swarm/model.py index c687684f..138df3c1 100644 --- a/libs/third-magic/thirdmagic/swarm/model.py +++ b/libs/third-magic/thirdmagic/swarm/model.py @@ -43,7 +43,7 @@ def can_add_n_tasks(self, swarm: "SwarmTaskSignature", n: int) -> bool: class SwarmTaskSignature(ContainerTaskSignature): - # TODO - TASKS list should be set once we enable this in rapyer + # Sub-tasks are ForeignKey edges so a write to the swarm cascades TTL to them. tasks: Annotated[list[Reference[TaskSignature]], CascadeTTL()] = Field( default_factory=list ) @@ -169,7 +169,7 @@ async def add_tasks( async with self.apipeline(use_existing_pipe=True, ignore_redis_error=True): for task in tasks: task.signature_container_id = self.key - self.tasks = self.tasks + [Reference(task) for task in tasks] + self.tasks.extend([Reference(task) for task in tasks]) self.tasks_left_to_run.extend(task_keys) if close_on_max_task and not self.config.can_add_task(self):