diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml new file mode 100644 index 00000000..ea4bdb1f --- /dev/null +++ b/.github/workflows/bandit.yml @@ -0,0 +1,56 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# Bandit is a security linter designed to find common security issues in Python code. +# This action will run Bandit on your codebase. +# The results of the scan will be found under the Security tab of your repository. + +# https://github.com/marketplace/actions/bandit-scan is ISC licensed, by abirismyname +# https://pypi.org/project/bandit/ is Apache v2.0 licensed, by PyCQA + +name: Bandit +on: + push: + branches: [ "main" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + schedule: + - cron: '38 4 * * 1' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + bandit: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Bandit Scan + uses: shundor/python-bandit-scan@ab1d87dfccc5a0ffab88be3aaac6ffe35c10d6cd + with: # optional arguments + # exit with 0, even with results found + exit_zero: true # optional, default is DEFAULT + # Github token of the repository (automatically created by Github) + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information. + # File or directory to run bandit on + # path: # optional, default is . + # Report only issues of a given severity level or higher. Can be LOW, MEDIUM or HIGH. Default is UNDEFINED (everything) + # level: # optional, default is UNDEFINED + # Report only issues of a given confidence level or higher. Can be LOW, MEDIUM or HIGH. Default is UNDEFINED (everything) + # confidence: # optional, default is UNDEFINED + # comma-separated list of paths (glob patterns supported) to exclude from scan (note that these are in addition to the excluded paths provided in the config file) (default: .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg) + # excluded_paths: # optional, default is DEFAULT + # comma-separated list of test IDs to skip + # skips: # optional, default is DEFAULT + # path to a .bandit file that supplies command line arguments + # ini_path: # optional, default is DEFAULT + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ad40947..ef486d56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,9 @@ -name: Python tests +name: CI on: push: branches: [main] pull_request: - workflow_dispatch: jobs: validate-branch-target: @@ -16,10 +15,10 @@ jobs: run: | SOURCE_BRANCH="${{ github.head_ref }}" TARGET_BRANCH="${{ github.base_ref }}" - + echo "Source branch: $SOURCE_BRANCH" echo "Target branch: $TARGET_BRANCH" - + # Rule 1: Only develop branch can merge to main if [[ "$TARGET_BRANCH" == "main" && "$SOURCE_BRANCH" != "develop" ]]; then echo "❌ Error: Only 'develop' branch can merge to 'main'" @@ -27,19 +26,53 @@ jobs: echo "Please target 'develop' branch instead" exit 1 fi - + echo "✅ Branch targeting rules validated successfully" - tests: - name: Run Tests + unit-tests: + name: Unit Tests (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest - needs: [ validate-branch-target ] + needs: [validate-branch-target] if: always() && (needs.validate-branch-target.result == 'success' || needs.validate-branch-target.result == 'skipped') strategy: + fail-fast: false matrix: - python-version: ['3.11', '3.12', '3.13'] + python-version: ['3.11', '3.12', '3.13'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install tox + + - name: Run Unit Tests + run: | + PYTHON_VER=$(echo "${{ matrix.python-version }}" | tr -d '.') + TOX_ENV="py${PYTHON_VER}-unit" + echo "Running tox environment: $TOX_ENV" + tox -e $TOX_ENV + + integration-tests: + name: Integration (Py${{ matrix.python-version }}/H${{ matrix.hatchet-version }}/G${{ matrix.group }}) + runs-on: ubuntu-latest + needs: [validate-branch-target] + if: always() && (needs.validate-branch-target.result == 'success' || needs.validate-branch-target.result == 'skipped') + + strategy: + fail-fast: false + matrix: + python-version: ['3.11', '3.12', '3.13'] hatchet-version: ['1.20', '1.21'] + group: [1, 2, 3] steps: - uses: actions/checkout@v4 @@ -55,51 +88,51 @@ jobs: pip install --upgrade pip pip install tox - - name: Deploy Hatch Lite + - name: Deploy Hatchet Lite run: | docker compose -f tests/integration/hatchet/docker-compose.hatchet.yml up -d - - name: Generate Env File + - name: Generate Hatchet API Key env: SERVER_AUTH_COOKIE_INSECURE: "t" - working-directory: . run: | sleep 10 HATCHET_API_KEY=$(docker compose -f tests/integration/hatchet/docker-compose.hatchet.yml exec hatchet-lite /hatchet-admin token create --config /config --tenant-id 707d0855-80ab-4e1f-a156-f1c4546cbf52) - echo "HATCHET_API_KEY=$HATCHET_API_KEY" - echo "HATCHET_API_KEY=$HATCHET_API_KEY" >> $GITHUB_ENV echo "DYNACONF_hatchet__api_key=$HATCHET_API_KEY" >> $GITHUB_ENV echo "DYNACONF_hatchet__tls_config__strategy=none" >> $GITHUB_ENV echo "DYNACONF_REDIS__URL=redis://localhost:6379/" >> $GITHUB_ENV echo "HATCHET_CLIENT_WORKER_HEALTHCHECK_ENABLED=True" >> $GITHUB_ENV - - name: Run Tests with Tox + - name: Run Integration Tests env: PYTHONHASHSEED: 0 + PYTEST_SPLITS: 3 + PYTEST_GROUP: ${{ matrix.group }} run: | - # Map Python and Hatchet versions to tox env PYTHON_VER=$(echo "${{ matrix.python-version }}" | tr -d '.') HATCHET_VER=$(echo "${{ matrix.hatchet-version }}" | tr -d '.') - TOX_ENV="py${PYTHON_VER}-hatchet${HATCHET_VER}" - echo "Running tox environment: $TOX_ENV" + TOX_ENV="py${PYTHON_VER}-hatchet${HATCHET_VER}-integration" + echo "Running tox environment: $TOX_ENV (Group ${{ matrix.group }} of 3)" tox -e $TOX_ENV - - name: Clean up credentials + - name: Clean up if: always() run: | - rm -f ~/.git-credentials - docker stop hatchet-lite || true - docker rm hatchet-lite || true + docker compose -f tests/integration/hatchet/docker-compose.hatchet.yml down test-results: runs-on: ubuntu-22.04 - needs: [ tests ] + needs: [unit-tests, integration-tests] if: always() steps: - name: Check test results run: | - if [[ "${{ needs.tests.result }}" != "success" ]]; then - echo "Test job failed" + if [[ "${{ needs.unit-tests.result }}" != "success" ]]; then + echo "Unit tests failed" + exit 1 + fi + if [[ "${{ needs.integration-tests.result }}" != "success" ]]; then + echo "Integration tests failed" exit 1 fi - echo "All checks passed successfully!" \ No newline at end of file + echo "All checks passed successfully!" diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 86f20a98..83b72607 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -8,6 +8,10 @@ on: - main workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: coverage: environment: diff --git a/.github/workflows/pm.yml b/.github/workflows/pm.yml new file mode 100644 index 00000000..2839fed7 --- /dev/null +++ b/.github/workflows/pm.yml @@ -0,0 +1,19 @@ +name: Close issues related to a merged pull request based on master branch. + +on: + pull_request: + types: [closed] + branches: + - develop + - main + +jobs: + closeIssueOnPrMergeTrigger: + + runs-on: ubuntu-latest + + steps: + - name: Closes issues related to a merged pull request. + uses: ldez/gha-mjolnir@v1.4.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 16cc7802..38793fed 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -8,6 +8,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: publish: environment: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 170f721d..eeb1c6a6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,8 @@ on: jobs: create-github-release: - needs: check-tests + environment: + name: create-release runs-on: ubuntu-22.04 permissions: contents: write # Required for creating releases diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 00000000..2b95184f --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,89 @@ +name: Security Scan + +on: + push: + branches: [main, develop] + pull_request: + branches: [main, develop] + schedule: + # Run weekly on Monday at 9am UTC + - cron: '0 9 * * 1' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + security-events: write + +jobs: + pip-audit: + name: Dependency Vulnerability Scan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install tools + run: | + python -m pip install --upgrade pip pipx + pip install pip-audit + pipx install poetry + pipx inject poetry poetry-plugin-export + + - name: Export and audit dependencies + run: | + poetry export -f requirements.txt --without-hashes -o requirements-audit.txt + pip-audit --strict -r requirements-audit.txt + + + secrets-scan: + name: Secrets Detection + environment: + name: security + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run Gitleaks + uses: gitleaks/gitleaks-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }} + + summary: + name: Security Summary + runs-on: ubuntu-latest + needs: [pip-audit, secrets-scan] + if: always() + steps: + - name: Check results + run: | + echo "## Security Scan Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ needs.pip-audit.result }}" == "success" ]; then + echo "✅ pip-audit: Passed" >> $GITHUB_STEP_SUMMARY + else + echo "❌ pip-audit: Failed" >> $GITHUB_STEP_SUMMARY + fi + + if [ "${{ needs.secrets-scan.result }}" == "success" ]; then + echo "✅ Secrets scan: Passed" >> $GITHUB_STEP_SUMMARY + else + echo "❌ Secrets scan: Failed" >> $GITHUB_STEP_SUMMARY + fi + + - name: Fail if any check failed + if: | + needs.pip-audit.result == 'failure' || + needs.secrets-scan.result == 'failure' + run: exit 1 \ No newline at end of file diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml new file mode 100644 index 00000000..8149ea04 --- /dev/null +++ b/.github/workflows/semgrep.yml @@ -0,0 +1,53 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# This workflow file requires a free account on Semgrep.dev to +# manage rules, file ignores, notifications, and more. +# +# See https://semgrep.dev/docs + +name: Semgrep + +on: + push: + branches: [ "main", "develop" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + schedule: + - cron: '29 23 * * 6' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + semgrep: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + name: Scan + runs-on: ubuntu-latest + steps: + # Checkout project source + - uses: actions/checkout@v4 + + # Scan code using project's configuration on https://semgrep.dev/manage + - uses: returntocorp/semgrep-action@fcd5ab7459e8d91cb1777481980d1b18b4fc6735 + with: + publishToken: ${{ secrets.SEMGREP_APP_TOKEN }} + publishDeployment: ${{ secrets.SEMGREP_DEPLOYMENT_ID }} + generateSarif: "1" + + # Upload SARIF file generated in previous step + - name: Upload SARIF file + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: semgrep.sarif + if: always() diff --git a/docs/documentation/root-task.md b/docs/documentation/root-task.md new file mode 100644 index 00000000..dfd3b685 --- /dev/null +++ b/docs/documentation/root-task.md @@ -0,0 +1,241 @@ +# Root Tasks + +
+ 🧪 Beta Feature
+ Root tasks are currently experimental. The API may change in future releases based on feedback. +
+ +Root tasks in MageFlow provide a convenient way to manage multiple parallel task executions by automatically creating and managing an internal swarm. When you need a task that spawns other tasks dynamically and wants to wait for all of them to complete before triggering callbacks, root tasks simplify this pattern significantly. + +## What is a Root Task? + +A root task creates an **internal swarm** that: +- Captures all tasks published within the root task body +- Runs them in parallel with managed concurrency +- Waits for ALL tasks to finish before triggering callbacks +- Uses the same configuration options as a regular swarm (SwarmConfig) + +This is particularly useful for "orchestrator" tasks that spawn multiple child tasks and need to track their completion as a group. + +## Creating a Root Task + +Use the `@hatchet.root_task()` decorator alongside `@hatchet.task()`: + +```python +from mageflow import Mageflow + +hatchet = Mageflow(hatchet_client, redis) + +@hatchet.task() +@hatchet.root_task(max_concurrency=4, stop_after_n_failures=2) +async def process_batch(msg: BatchMessage): + # All tasks published here are added to the internal swarm + for item in msg.items: + task_sig = await mageflow.sign(process_item) + await task_sig.aio_run_no_wait(ItemMessage(data=item)) + + return {"processed": len(msg.items)} +``` + +!!! info "Alternative Client Usage" + You can also use the global `mageflow` module for signing tasks inside the root task: + + ```python + import mageflow + + @hatchet.task() + @hatchet.root_task(max_concurrency=10) + async def my_root_task(msg: InputMessage): + task_sig = await mageflow.sign(child_task) + await task_sig.aio_run_no_wait(msg) + ``` + +### Parameters (SwarmConfig) + +The `@hatchet.root_task()` decorator accepts the same configuration options as SwarmConfig: + +- `max_concurrency`: Maximum number of child tasks that can run in parallel (default: 30) +- `stop_after_n_failures`: Stop the internal swarm after N task failures (optional, defaults to None - meaning continue despite failures) + +These parameters are passed directly to the internal swarm that gets created when the root task starts. + +## How Root Tasks Work + +Understanding the lifecycle of a root task helps you use it effectively: + +1. **Task Start**: When the root task begins execution, an internal swarm is created with the name `root-swarm:{task_name}` +2. **Task Capture**: Any task published via `aio_run()` or `aio_run_no_wait()` within the root task body is automatically added to this internal swarm +3. **Concurrency Management**: The internal swarm manages how many child tasks run in parallel based on `max_concurrency` +4. **Task Completion**: When the root task function returns, the internal swarm is closed (no more tasks can be added) +5. **Callback Execution**: Success/error callbacks attached to the root task only trigger after ALL child tasks in the internal swarm finish + +```python +@hatchet.task() +@hatchet.root_task(max_concurrency=5) +async def orchestrator(msg: OrchestratorMessage): + # Create and run multiple child tasks + for i in range(10): + sig = await mageflow.sign(worker_task) + await sig.aio_run_no_wait(WorkerMessage(index=i)) + + # Root task returns, but callbacks wait for all 10 tasks + return {"spawned": 10} + +# Callbacks only execute after all 10 worker tasks complete +root_sig = await mageflow.sign( + orchestrator, + success_callbacks=[notify_completion], + error_callbacks=[handle_failure] +) +await root_sig.aio_run_no_wait(msg) +``` + +## Root Tasks with Chains and Swarms + +Root tasks can contain chains and explicit swarms inside them. These are also captured by the root task's internal swarm: + +```python +@hatchet.task() +@hatchet.root_task(max_concurrency=10) +async def complex_orchestrator(msg: InputMessage): + # Create a chain - will be tracked by root swarm + chain_sig = await hatchet.chain([task1, task2, task3]) + await chain_sig.aio_run_no_wait(msg) + + # Create an explicit swarm - also tracked by root swarm + swarm_sig = await hatchet.swarm( + tasks=[worker1, worker2], + is_swarm_closed=True + ) + await swarm_sig.aio_run_no_wait(msg) + + # Individual tasks + single_sig = await mageflow.sign(single_task) + await single_sig.aio_run_no_wait(msg) + + return {"completed": True} +``` + +## Excluding Tasks from the Root Swarm + +Sometimes you need to publish a task from within a root task without adding it to the internal swarm. Use the `without_root_swarm` context manager for this: + +```python +from mageflow.root.context import without_root_swarm + +@hatchet.task() +@hatchet.root_task(max_concurrency=5) +async def orchestrator(msg: OrchestratorMessage): + # This task IS added to the root swarm + sig1 = await mageflow.sign(tracked_task) + await sig1.aio_run_no_wait(msg) + + # This task is NOT added to the root swarm + with without_root_swarm(): + sig2 = await mageflow.sign(independent_task) + await sig2.aio_run_no_wait(msg) + + # Back to normal - this IS added to the root swarm + sig3 = await mageflow.sign(another_tracked_task) + await sig3.aio_run_no_wait(msg) + + return {"done": True} +``` + +Tasks published inside the `without_root_swarm()` context: +- Run independently of the root task's internal swarm +- Are not subject to the root task's `max_concurrency` limit +- Do not block the root task's callbacks from executing +- Are useful for "fire and forget" tasks that shouldn't affect the root task completion + +## Root Tasks vs Explicit Swarms + +| Aspect | Root Task | Explicit Swarm | +|--------|-----------|----------------| +| Creation | `@hatchet.root_task()` decorator | `mageflow.swarm()` | +| Task capture | Automatic (context-based) | Manual (`add_task()`) | +| Closing | Automatic when task returns | Manual (`close_swarm()`) | +| Callbacks | On decorator, wait for all children | On swarm creation | +| Use case | Dynamic task spawning within a task | Batch processing known tasks | + +## Example Use Cases + +### Dynamic Batch Processing + +Process items where you don't know the count upfront: + +```python +@hatchet.task() +@hatchet.root_task(max_concurrency=20, stop_after_n_failures=5) +async def process_query_results(msg: QueryMessage): + # Fetch items dynamically + items = await fetch_items_from_database(msg.query) + + # Spawn a task for each item + for item in items: + sig = await mageflow.sign(process_item) + await sig.aio_run_no_wait(ItemMessage(id=item.id)) + + return {"total_items": len(items)} + +# Run with callbacks +notify = await mageflow.sign(send_completion_notification) +root = await mageflow.sign( + process_query_results, + success_callbacks=[notify] +) +await root.aio_run_no_wait(QueryMessage(query="SELECT * FROM items")) +``` + +### Agent Task Spawning + +When an AI agent needs to spawn multiple sub-tasks: + +```python +@hatchet.task() +@hatchet.root_task(max_concurrency=5) +async def agent_executor(msg: AgentMessage): + # Agent decides what tasks to run + plan = await generate_execution_plan(msg.goal) + + for step in plan.steps: + task_func = get_task_for_step(step) + sig = await mageflow.sign(task_func) + await sig.aio_run_no_wait(StepMessage(step=step)) + + return {"steps_spawned": len(plan.steps)} +``` + +### Fan-out/Fan-in Pattern + +Distribute work and collect results: + +```python +@hatchet.task() +@hatchet.root_task(max_concurrency=10) +async def distributed_analysis(msg: AnalysisMessage): + # Fan-out: spawn analysis tasks for each data source + for source in msg.data_sources: + sig = await mageflow.sign(analyze_source) + await sig.aio_run_no_wait(SourceMessage(source=source)) + + return {"sources": len(msg.data_sources)} + +# Fan-in: aggregate results in callback +aggregate = await mageflow.sign(aggregate_results) +root = await mageflow.sign( + distributed_analysis, + success_callbacks=[aggregate] +) +``` + +## Why Use Root Tasks? + +Root tasks are ideal when: + +- **Dynamic Task Spawning**: You don't know at design time how many tasks will be spawned +- **Simplified Orchestration**: You want automatic tracking without explicit swarm management +- **Callback Coordination**: You need callbacks to wait for all dynamically spawned tasks +- **Clean Code**: You prefer a decorator-based approach over manual swarm lifecycle management + +The key benefit is that root tasks give you swarm-like behavior (parallel execution, concurrency control, failure handling) without the boilerplate of creating, managing, and closing a swarm explicitly. diff --git a/mageflow/__init__.py b/mageflow/__init__.py index a5b2b5ae..988f35d7 100644 --- a/mageflow/__init__.py +++ b/mageflow/__init__.py @@ -2,6 +2,7 @@ from mageflow.chain.creator import chain from mageflow.client import Mageflow from mageflow.init import init_mageflow_hatchet_tasks +from mageflow.root.model import RootTaskSignature from mageflow.signature.creator import ( sign, load_signature, @@ -27,4 +28,5 @@ "Mageflow", "chain", "swarm", + "RootTaskSignature", ] diff --git a/mageflow/callbacks.py b/mageflow/callbacks.py index 3d3b53c0..5ac64112 100644 --- a/mageflow/callbacks.py +++ b/mageflow/callbacks.py @@ -7,11 +7,10 @@ from hatchet_sdk import Context from hatchet_sdk.runnables.types import EmptyModel from hatchet_sdk.runnables.workflow import Standalone -from pydantic import BaseModel - from mageflow.invokers.hatchet import HatchetInvoker from mageflow.task.model import HatchetTaskModel from mageflow.utils.pythonic import flexible_call +from pydantic import BaseModel class AcceptParams(Enum): @@ -55,6 +54,7 @@ async def wrapper(message: EmptyModel, ctx: Context, *args, **kwargs): await invoker.remove_task(with_error=False) raise else: + await invoker.end_task() task_results = HatchetResult(hatchet_results=result) dumped_results = task_results.model_dump(mode="json") await invoker.run_success(dumped_results["hatchet_results"]) @@ -70,11 +70,15 @@ async def wrapper(message: EmptyModel, ctx: Context, *args, **kwargs): return task_decorator -def register_task(register_name: str): +def register_task( + register_name: str, + is_root_task: bool = False, + root_task_config=None, +): from mageflow.startup import REGISTERED_TASKS def decorator(func: Standalone): - REGISTERED_TASKS.append((func, register_name)) + REGISTERED_TASKS.append((func, register_name, is_root_task, root_task_config)) return func return decorator diff --git a/mageflow/chain/creator.py b/mageflow/chain/creator.py index 621257a1..baf8736c 100644 --- a/mageflow/chain/creator.py +++ b/mageflow/chain/creator.py @@ -3,15 +3,13 @@ from mageflow.chain.consts import ON_CHAIN_END, ON_CHAIN_ERROR from mageflow.chain.messages import ChainSuccessTaskCommandMessage from mageflow.chain.model import ChainTaskSignature -from mageflow.signature.creator import ( - TaskSignatureConvertible, - resolve_signature_key, -) from mageflow.signature.model import ( TaskIdentifierType, TaskSignature, TaskInputType, ) +from mageflow.signature.resolve import resolve_signature_key +from mageflow.signature.types import TaskSignatureConvertible async def chain( diff --git a/mageflow/chain/messages.py b/mageflow/chain/messages.py index 35efdccb..dbcfa1bb 100644 --- a/mageflow/chain/messages.py +++ b/mageflow/chain/messages.py @@ -1,8 +1,7 @@ from typing import Any, Annotated -from pydantic import BaseModel - from mageflow.models.message import ReturnValue +from pydantic import BaseModel class ChainSuccessTaskCommandMessage(BaseModel): diff --git a/mageflow/chain/model.py b/mageflow/chain/model.py index d7794961..48cc893b 100644 --- a/mageflow/chain/model.py +++ b/mageflow/chain/model.py @@ -1,11 +1,10 @@ import asyncio import rapyer -from pydantic import field_validator, Field - from mageflow.errors import MissingSignatureError from mageflow.signature.model import TaskSignature, TaskIdentifierType from mageflow.signature.status import SignatureStatus +from pydantic import field_validator, Field class ChainTaskSignature(TaskSignature): diff --git a/mageflow/chain/workflows.py b/mageflow/chain/workflows.py index b33e8f12..a20dfc50 100644 --- a/mageflow/chain/workflows.py +++ b/mageflow/chain/workflows.py @@ -2,7 +2,6 @@ from hatchet_sdk import Context from hatchet_sdk.runnables.types import EmptyModel - from mageflow.chain.consts import CHAIN_TASK_ID_NAME from mageflow.chain.messages import ChainSuccessTaskCommandMessage from mageflow.chain.model import ChainTaskSignature diff --git a/mageflow/client.py b/mageflow/client.py index 4eb0c277..29d1b155 100644 --- a/mageflow/client.py +++ b/mageflow/client.py @@ -10,15 +10,13 @@ from hatchet_sdk import Hatchet, Worker, Context from hatchet_sdk.runnables.workflow import BaseWorkflow from hatchet_sdk.worker.worker import LifespanFn -from redis.asyncio import Redis -from typing_extensions import override - from mageflow.callbacks import AcceptParams, register_task, handle_task_callback from mageflow.chain.creator import chain from mageflow.init import init_mageflow_hatchet_tasks -from mageflow.signature.creator import sign, TaskSignatureConvertible +from mageflow.root.consts import ROOT_TASK_MARKER, ROOT_TASK_CONFIG +from mageflow.signature.creator import sign from mageflow.signature.model import TaskSignature, TaskInputType -from mageflow.signature.types import HatchetTaskType +from mageflow.signature.types import HatchetTaskType, TaskSignatureConvertible from mageflow.startup import ( lifespan_initialize, mageflow_config, @@ -26,7 +24,10 @@ teardown_mageflow, ) from mageflow.swarm.creator import swarm, SignatureOptions +from mageflow.swarm.model import SwarmConfig from mageflow.utils.mageflow import does_task_wants_ctx +from redis.asyncio import Redis +from typing_extensions import override async def merge_lifespan(original_lifespan: LifespanFn): @@ -143,6 +144,24 @@ async def stagger_wrapper(message, ctx: Context, *args, **kwargs): return decorator + def root_task( + self, + *, + max_concurrency: int = 30, + stop_after_n_failures: int | None = None, + ): + swarm_config = SwarmConfig( + max_concurrency=max_concurrency, + stop_after_n_failures=stop_after_n_failures, + ) + + def decorator(func): + setattr(func, ROOT_TASK_MARKER, True) + setattr(func, ROOT_TASK_CONFIG, swarm_config) + return func + + return decorator + def task_decorator( func: Callable, @@ -154,12 +173,14 @@ def task_decorator( AcceptParams.ALL if does_task_wants_ctx(func) else mage_client.param_config ) send_signature = getattr(func, "__send_signature__", False) + is_root_task = getattr(func, ROOT_TASK_MARKER, False) + root_task_config = getattr(func, ROOT_TASK_CONFIG, None) handler_dec = handle_task_callback(param_config, send_signature=send_signature) func = handler_dec(func) wf = hatchet_task(func) task_name = hatchet_task_name or func.__name__ - register = register_task(task_name) + register = register_task(task_name, is_root_task, root_task_config) return register(wf) diff --git a/mageflow/init.py b/mageflow/init.py index ac8f99da..269b5cfc 100644 --- a/mageflow/init.py +++ b/mageflow/init.py @@ -1,7 +1,6 @@ from datetime import timedelta from hatchet_sdk import Hatchet - from mageflow.callbacks import register_task from mageflow.chain.consts import ON_CHAIN_END, ON_CHAIN_ERROR from mageflow.chain.messages import ChainSuccessTaskCommandMessage diff --git a/mageflow/invokers/hatchet.py b/mageflow/invokers/hatchet.py index c5e52d4e..d7ac7e68 100644 --- a/mageflow/invokers/hatchet.py +++ b/mageflow/invokers/hatchet.py @@ -1,15 +1,16 @@ import asyncio from typing import Any +import rapyer from hatchet_sdk import Context from hatchet_sdk.runnables.contextvars import ctx_additional_metadata -from pydantic import BaseModel - from mageflow.invokers.base import BaseInvoker from mageflow.signature.consts import TASK_ID_PARAM_NAME from mageflow.signature.model import TaskSignature from mageflow.signature.status import SignatureStatus +from mageflow.utils.mageflow import rapyer_aget_safe from mageflow.workflows import TASK_DATA_PARAM_NAME +from pydantic import BaseModel class HatchetInvoker(BaseInvoker): @@ -28,16 +29,27 @@ def task_ctx(self) -> dict: async def start_task(self) -> TaskSignature | None: task_id = self.task_data.get(TASK_ID_PARAM_NAME, None) if task_id: - async with TaskSignature.lock_from_key(task_id) as signature: + async with rapyer.alock_from_key(task_id) as signature: await signature.change_status(SignatureStatus.ACTIVE) await signature.task_status.aupdate(worker_task_id=self.workflow_id) + await signature.start_task() + return signature + return None + + async def end_task(self) -> TaskSignature | None: + task_id = self.task_data.get(TASK_ID_PARAM_NAME, None) + if task_id: + signature: TaskSignature = await rapyer_aget_safe(task_id) # noqa + if signature: + await signature.end_task(True) return signature + return None async def run_success(self, result: Any) -> bool: success_publish_tasks = [] task_id = self.task_data.get(TASK_ID_PARAM_NAME, None) if task_id: - current_task = await TaskSignature.get_safe(task_id) + current_task: TaskSignature = await rapyer_aget_safe(task_id) # noqa task_success_workflows = current_task.activate_success(result) success_publish_tasks.append(asyncio.create_task(task_success_workflows)) @@ -50,7 +62,8 @@ async def run_error(self) -> bool: error_publish_tasks = [] task_id = self.task_data.get(TASK_ID_PARAM_NAME, None) if task_id: - current_task = await TaskSignature.get_safe(task_id) + current_task: TaskSignature = await rapyer_aget_safe(task_id) # noqa + await current_task.end_task(False) task_error_workflows = current_task.activate_error(self.message) error_publish_tasks.append(asyncio.create_task(task_error_workflows)) @@ -64,14 +77,14 @@ async def remove_task( ) -> TaskSignature | None: task_id = self.task_data.get(TASK_ID_PARAM_NAME, None) if task_id: - signature = await TaskSignature.get_safe(task_id) + signature: TaskSignature = await rapyer_aget_safe(task_id) # noqa if signature: await signature.remove(with_error, with_success) async def should_run_task(self) -> bool: task_id = self.task_data.get(TASK_ID_PARAM_NAME, None) if task_id: - signature = await TaskSignature.get_safe(task_id) + signature: TaskSignature = await rapyer_aget_safe(task_id) # noqa if signature is None: return False should_task_run = await signature.should_run() diff --git a/mageflow/root/__init__.py b/mageflow/root/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mageflow/root/consts.py b/mageflow/root/consts.py new file mode 100644 index 00000000..696732a8 --- /dev/null +++ b/mageflow/root/consts.py @@ -0,0 +1,2 @@ +ROOT_TASK_MARKER = "__is_root_task__" +ROOT_TASK_CONFIG = "__root_task_config__" diff --git a/mageflow/root/context.py b/mageflow/root/context.py new file mode 100644 index 00000000..9b2cdfbf --- /dev/null +++ b/mageflow/root/context.py @@ -0,0 +1,19 @@ +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from mageflow.swarm.model import SwarmTaskSignature + +current_root_swarm: ContextVar[Optional["SwarmTaskSignature"]] = ContextVar( + "current_root_swarm", default=None +) + + +@contextmanager +def without_root_swarm(): + token = current_root_swarm.set(None) + try: + yield + finally: + current_root_swarm.reset(token) diff --git a/mageflow/root/model.py b/mageflow/root/model.py new file mode 100644 index 00000000..a05df201 --- /dev/null +++ b/mageflow/root/model.py @@ -0,0 +1,42 @@ +from typing import Optional, Self + +from mageflow.root.context import current_root_swarm +from mageflow.signature.model import TaskSignature +from mageflow.swarm.creator import swarm +from mageflow.swarm.model import SwarmTaskSignature, SwarmConfig +from pydantic import Field + + +class RootTaskSignature(TaskSignature): + swarm_id: Optional[str] = None + swarm_config: SwarmConfig = Field(default_factory=SwarmConfig) + + async def create_swarm(self) -> SwarmTaskSignature: + root_swarm = await swarm( + task_name=f"root-swarm:{self.task_name}", + config=self.swarm_config, + success_callbacks=list(self.success_callbacks), + error_callbacks=list(self.error_callbacks), + ) + await root_swarm.asave() + await self.aupdate( + swarm_id=root_swarm.key, success_callbacks=[], error_callbacks=[] + ) + return root_swarm + + async def start_task(self) -> Self: + await super().start_task() + root_swarm = await self.create_swarm() + current_root_swarm.set(root_swarm) + return self + + async def end_task(self, success: bool = True) -> Self: + current_root_swarm.set(None) + root_swarm = await SwarmTaskSignature.get_safe(self.swarm_id) + if root_swarm: + await root_swarm.close_swarm() + if not success: + # TODO - should be interrupt once implemented then set to delete + await root_swarm.suspend() + await self.aupdate(error_callbacks=root_swarm.error_callbacks) + return await super().end_task(success) diff --git a/mageflow/signature/creator.py b/mageflow/signature/creator.py index 2677d92b..919fad80 100644 --- a/mageflow/signature/creator.py +++ b/mageflow/signature/creator.py @@ -1,43 +1,11 @@ -from datetime import datetime -from typing import TypeAlias, TypedDict, Any, overload +from typing import Any, overload -from mageflow.signature.model import ( - TaskSignature, - TaskIdentifierType, - HatchetTaskType, -) -from mageflow.signature.status import TaskStatus - -TaskSignatureConvertible: TypeAlias = ( - TaskIdentifierType | TaskSignature | HatchetTaskType -) - - -async def resolve_signature_key(task: TaskSignatureConvertible) -> TaskSignature: - if isinstance(task, TaskSignature): - return task - elif isinstance(task, TaskIdentifierType): - return await TaskSignature.get_safe(task) - else: - return await TaskSignature.from_task(task) - - -try: - # Python 3.12+ - from typing import Unpack -except ImportError: - # Older Python versions - from typing_extensions import Unpack - - -class TaskSignatureOptions(TypedDict, total=False): - kwargs: dict - creation_time: datetime - model_validators: Any - success_callbacks: list[TaskIdentifierType] - error_callbacks: list[TaskIdentifierType] - task_status: TaskStatus - task_identifiers: dict +from mageflow.root.model import RootTaskSignature +from mageflow.signature.model import TaskSignature, HatchetTaskType +from mageflow.signature.types import TaskSignatureOptions +from mageflow.swarm.model import SwarmConfig +from mageflow.task.model import HatchetTaskModel +from rapyer.typing_support import Unpack @overload @@ -49,17 +17,27 @@ async def sign(task: str | HatchetTaskType, **options: Any) -> TaskSignature: .. async def sign(task: str | HatchetTaskType, **options: Any) -> TaskSignature: - model_fields = list(TaskSignature.model_fields.keys()) + task_name = task if isinstance(task, str) else task.name + + task_model = await HatchetTaskModel.safe_get(task_name) + is_root = task_model and task_model.is_root_task + + signature_class = RootTaskSignature if is_root else TaskSignature + + model_fields = list(signature_class.model_fields.keys()) kwargs = { field_name: options.pop(field_name) for field_name in model_fields if field_name in options } + if is_root and task_model.root_task_config: + kwargs["swarm_config"] = SwarmConfig(**task_model.root_task_config) + if isinstance(task, str): - return await TaskSignature.from_task_name(task, kwargs=options, **kwargs) + return await signature_class.from_task_name(task, kwargs=options, **kwargs) else: - return await TaskSignature.from_task(task, kwargs=options, **kwargs) + return await signature_class.from_task(task, kwargs=options, **kwargs) load_signature = TaskSignature.get_safe diff --git a/mageflow/signature/model.py b/mageflow/signature/model.py index 050f66ae..e0331b13 100644 --- a/mageflow/signature/model.py +++ b/mageflow/signature/model.py @@ -1,13 +1,13 @@ import asyncio -import contextlib from datetime import datetime -from typing import Optional, Self, Any, TypeAlias, AsyncGenerator, ClassVar +from typing import Optional, Self, Any, TypeAlias, ClassVar import rapyer from hatchet_sdk.runnables.types import EmptyModel from hatchet_sdk.runnables.workflow import Workflow from mageflow.errors import MissingSignatureError from mageflow.models.message import ReturnValue +from mageflow.root.context import current_root_swarm from mageflow.signature.consts import TASK_ID_PARAM_NAME from mageflow.signature.status import TaskStatus, SignatureStatus, PauseActionTypes from mageflow.signature.types import TaskIdentifierType, HatchetTaskType @@ -15,17 +15,11 @@ from mageflow.task.model import HatchetTaskModel from mageflow.utils.models import get_marked_fields from mageflow.workflows import MageflowWorkflow -from pydantic import ( - BaseModel, - field_validator, - Field, -) +from pydantic import BaseModel, field_validator, Field from rapyer import AtomicRedisModel from rapyer.config import RedisConfig from rapyer.errors.base import KeyNotFound from rapyer.types import RedisDict, RedisList, RedisDatetime -from rapyer.utils.redis import acquire_lock -from typing_extensions import deprecated class TaskSignature(AtomicRedisModel): @@ -142,9 +136,22 @@ def task_ctx(self) -> dict: return self.task_identifiers | {TASK_ID_PARAM_NAME: self.key} async def aio_run_no_wait(self, msg: BaseModel, **kwargs): + root_swarm = current_root_swarm.get() + if root_swarm: + batch_item = await root_swarm.add_task(self) + return await batch_item.aio_run_no_wait(msg, **kwargs) + workflow = await self.workflow(use_return_field=False) return await workflow.aio_run_no_wait(msg, **kwargs) + async def aio_run(self, msg: BaseModel, **kwargs): + root_swarm = current_root_swarm.get() + if root_swarm: + batch_item = await root_swarm.add_task(self) + return await batch_item.aio_run(msg, **kwargs) + workflow = await self.workflow(use_return_field=False) + return await workflow.aio_run(msg, **kwargs) + async def callback_workflows( self, with_success: bool = True, with_error: bool = True, **kwargs ) -> list[Workflow]: @@ -231,11 +238,17 @@ async def handle_inactive_task(self, msg: BaseModel): async def should_run(self): return self.task_status.should_run() - async def change_status(self, status: SignatureStatus) -> bool: + async def change_status(self, status: SignatureStatus): await self.task_status.aupdate( last_status=self.task_status.status, status=status ) + async def start_task(self) -> Self: + return self + + async def end_task(self, success: bool = True) -> Self: + return self + async def aupdate_real_task_kwargs(self, **kwargs): return await self.kwargs.aupdate(**kwargs) @@ -245,7 +258,7 @@ async def safe_change_status( cls, task_id: TaskIdentifierType, status: SignatureStatus ) -> bool: try: - async with lock_from_key(cls, task_id) as task: + async with rapyer.alock_from_key(task_id) as task: return await task.change_status(status) except Exception as e: return False @@ -258,7 +271,7 @@ async def on_cancel_signature(self, msg: BaseModel): @classmethod async def resume_from_key(cls, task_key: TaskIdentifierType): - async with lock_from_key(cls, task_key) as task: + async with rapyer.alock_from_key(task_key) as task: await task.resume() async def resume(self): @@ -271,7 +284,7 @@ async def resume(self): @classmethod async def suspend_from_key(cls, task_key: TaskIdentifierType): - async with lock_from_key(cls, task_key) as task: + async with rapyer.alock_from_key(task_key) as task: await task.suspend() async def suspend(self): @@ -282,7 +295,7 @@ async def suspend(self): @classmethod async def interrupt_from_key(cls, task_key: TaskIdentifierType): - async with lock_from_key(cls, task_key) as task: + async with rapyer.alock_from_key(task_key) as task: return task.interrupt() async def interrupt(self): @@ -297,7 +310,7 @@ async def pause_from_key( task_key: TaskIdentifierType, pause_type: PauseActionTypes = PauseActionTypes.SUSPEND, ): - async with lock_from_key(cls, task_key) as task: + async with rapyer.alock_from_key(task_key) as task: await task.pause_task(pause_type) async def pause_task(self, pause_type: PauseActionTypes = PauseActionTypes.SUSPEND): @@ -308,17 +321,5 @@ async def pause_task(self, pause_type: PauseActionTypes = PauseActionTypes.SUSPE raise NotImplementedError(f"Pause type {pause_type} not supported") -@contextlib.asynccontextmanager -@deprecated(f"You should switch to rapyer 1.1.1 with rapyer.lock_from_key") -async def lock_from_key( - cls, key: str, action: str = "default", save_at_end: bool = False -) -> AsyncGenerator[TaskSignature, None]: - async with acquire_lock(cls.Meta.redis, f"{key}/{action}"): - redis_model = await rapyer.aget(key) - yield redis_model - if save_at_end: - await redis_model.save() - - SIGNATURES_NAME_MAPPING: dict[str, type[TaskSignature]] = {} TaskInputType: TypeAlias = TaskIdentifierType | TaskSignature diff --git a/mageflow/signature/resolve.py b/mageflow/signature/resolve.py new file mode 100644 index 00000000..74aca7b1 --- /dev/null +++ b/mageflow/signature/resolve.py @@ -0,0 +1,11 @@ +from mageflow.signature.model import TaskSignature, TaskIdentifierType +from mageflow.signature.types import TaskSignatureConvertible + + +async def resolve_signature_key(task: TaskSignatureConvertible) -> TaskSignature: + if isinstance(task, TaskSignature): + return task + elif isinstance(task, TaskIdentifierType): + return await TaskSignature.get_safe(task) + else: + return await TaskSignature.from_task(task) diff --git a/mageflow/signature/types.py b/mageflow/signature/types.py index 994f9885..4a912c12 100644 --- a/mageflow/signature/types.py +++ b/mageflow/signature/types.py @@ -1,6 +1,25 @@ -from typing import Callable +from datetime import datetime +from typing import Callable, TYPE_CHECKING, TypeAlias, Any, TypedDict from hatchet_sdk.runnables.workflow import BaseWorkflow +from mageflow.signature.status import TaskStatus + +if TYPE_CHECKING: + from mageflow.signature.model import TaskSignature TaskIdentifierType = str HatchetTaskType = BaseWorkflow | Callable + +TaskSignatureConvertible: TypeAlias = ( + "TaskIdentifierType | TaskSignature | HatchetTaskType" +) + + +class TaskSignatureOptions(TypedDict, total=False): + kwargs: dict + creation_time: datetime + model_validators: Any + success_callbacks: list[TaskIdentifierType] + error_callbacks: list[TaskIdentifierType] + task_status: TaskStatus + task_identifiers: dict diff --git a/mageflow/startup.py b/mageflow/startup.py index a0d1c6a4..8fd9cbd3 100644 --- a/mageflow/startup.py +++ b/mageflow/startup.py @@ -1,12 +1,13 @@ +from typing import Optional, Any + import rapyer from hatchet_sdk import Hatchet from hatchet_sdk.runnables.workflow import Standalone +from mageflow.task.model import HatchetTaskModel from pydantic import BaseModel from redis.asyncio.client import Redis -from mageflow.task.model import HatchetTaskModel - -REGISTERED_TASKS: list[tuple[Standalone, str]] = [] +REGISTERED_TASKS: list[tuple[Standalone, str, bool, Optional[Any]]] = [] class ConfigModel(BaseModel): @@ -48,12 +49,17 @@ async def update_register_signature_models(): async def register_workflows(): for reg_task in REGISTERED_TASKS: - workflow, mageflow_task_name = reg_task + workflow, mageflow_task_name, is_root_task, root_task_config = reg_task + config_dict = ( + root_task_config.model_dump(mode="json") if root_task_config else None + ) hatchet_task = HatchetTaskModel( mageflow_task_name=mageflow_task_name, task_name=workflow.name, input_validator=workflow.input_validator, retries=workflow.tasks[0].retries, + is_root_task=is_root_task, + root_task_config=config_dict, ) await hatchet_task.save() diff --git a/mageflow/swarm/creator.py b/mageflow/swarm/creator.py index 3dc4f80e..cb875612 100644 --- a/mageflow/swarm/creator.py +++ b/mageflow/swarm/creator.py @@ -1,18 +1,9 @@ import asyncio import uuid -from mageflow.signature.creator import ( - TaskSignatureConvertible, - TaskSignatureOptions, -) +from mageflow.signature.types import TaskSignatureOptions, TaskSignatureConvertible from mageflow.swarm.model import SwarmTaskSignature, SwarmConfig - -try: - # Python 3.12+ - from typing import Unpack -except ImportError: - # Older Python versions - from typing_extensions import Unpack +from rapyer.typing_support import Unpack class SignatureOptions(TaskSignatureOptions): diff --git a/mageflow/swarm/model.py b/mageflow/swarm/model.py index a8ac113e..6640dbb8 100644 --- a/mageflow/swarm/model.py +++ b/mageflow/swarm/model.py @@ -8,13 +8,11 @@ TooManyTasksError, SwarmIsCanceledError, ) -from mageflow.signature.creator import ( - TaskSignatureConvertible, - resolve_signature_key, -) +from mageflow.root.context import without_root_swarm from mageflow.signature.model import TaskSignature +from mageflow.signature.resolve import resolve_signature_key from mageflow.signature.status import SignatureStatus -from mageflow.signature.types import TaskIdentifierType +from mageflow.signature.types import TaskIdentifierType, TaskSignatureConvertible from mageflow.swarm.consts import ( BATCH_TASK_NAME_INITIALS, SWARM_TASK_ID_PARAM_NAME, @@ -53,7 +51,8 @@ async def aio_run_no_wait(self, msg: BaseModel, **orig_task_kwargs): kwargs = deep_merge(kwargs, msg.model_dump(mode="json")) await original_task.aupdate_real_task_kwargs(**kwargs) if can_run_task: - return await original_task.aio_run_no_wait(msg, **orig_task_kwargs) + with without_root_swarm(): + return await original_task.aio_run_no_wait(msg, **orig_task_kwargs) return None @@ -115,8 +114,7 @@ def has_swarm_started(self): async def aio_run_no_wait(self, msg: BaseModel, **kwargs): await self.kwargs.aupdate(**msg.model_dump(mode="json")) - workflow = await self.workflow(use_return_field=False) - return await workflow.aio_run_no_wait(msg, **kwargs) + return await super().aio_run_no_wait(msg, **kwargs) async def workflow(self, use_return_field: bool = True, **task_additional_params): # Use on swarm start task name for wf @@ -160,7 +158,7 @@ async def change_status(self, status: SignatureStatus): await asyncio.gather(pause_chain, *paused_chain_tasks, return_exceptions=True) async def add_task( - self, task: TaskSignatureConvertible, close_on_max_task: bool = True + self, task: "TaskSignatureConvertible", close_on_max_task: bool = True ) -> BatchItemTaskSignature: """ task - task signature to add to swarm @@ -207,7 +205,7 @@ async def add_task( return batch_task - async def add_to_running_tasks(self, task: TaskSignatureConvertible) -> bool: + async def add_to_running_tasks(self, task: "TaskSignatureConvertible") -> bool: async with self.lock() as swarm_task: task = await resolve_signature_key(task) if self.current_running_tasks < self.config.max_concurrency: diff --git a/mageflow/swarm/workflows.py b/mageflow/swarm/workflows.py index bde46104..9bf3f77f 100644 --- a/mageflow/swarm/workflows.py +++ b/mageflow/swarm/workflows.py @@ -2,9 +2,6 @@ from hatchet_sdk import Context from hatchet_sdk.runnables.types import EmptyModel -from pydantic import BaseModel - -from mageflow.errors import MissingSwarmItemError from mageflow.invokers.hatchet import HatchetInvoker from mageflow.signature.consts import TASK_ID_PARAM_NAME from mageflow.signature.model import TaskSignature @@ -15,6 +12,7 @@ ) from mageflow.swarm.messages import SwarmResultsMessage from mageflow.swarm.model import SwarmTaskSignature +from pydantic import BaseModel async def swarm_start_tasks(msg: EmptyModel, ctx: Context): diff --git a/mageflow/task/model.py b/mageflow/task/model.py index 6071e89b..e062cf06 100644 --- a/mageflow/task/model.py +++ b/mageflow/task/model.py @@ -1,4 +1,4 @@ -from typing import Optional, Annotated, Self +from typing import Optional, Self from hatchet_sdk import NonRetryableException from pydantic import BaseModel @@ -8,10 +8,12 @@ class HatchetTaskModel(AtomicRedisModel): - mageflow_task_name: Annotated[str, Key()] + mageflow_task_name: Key[str] task_name: str input_validator: Optional[type[BaseModel]] = None retries: Optional[int] = None + is_root_task: bool = False + root_task_config: Optional[dict] = None @classmethod async def safe_get(cls, key: str) -> Self | None: diff --git a/mageflow/typing_support.py b/mageflow/typing_support.py index 09c06b12..c2ab0537 100644 --- a/mageflow/typing_support.py +++ b/mageflow/typing_support.py @@ -4,5 +4,11 @@ except ImportError: from typing_extensions import Self +try: + # Python 3.12+ + from typing import Unpack +except ImportError: + # Older Python versions + from typing_extensions import Unpack -__all__ = ["Self"] +__all__ = ["Self", "Unpack"] diff --git a/mageflow/utils/mageflow.py b/mageflow/utils/mageflow.py index f06365d6..ee6288b9 100644 --- a/mageflow/utils/mageflow.py +++ b/mageflow/utils/mageflow.py @@ -1,2 +1,18 @@ +from typing import Optional + +import rapyer +from rapyer import AtomicRedisModel +from rapyer.errors.base import KeyNotFound +from typing_extensions import deprecated + + def does_task_wants_ctx(func) -> bool: - return getattr(func, "__user_ctx__", False) \ No newline at end of file + return getattr(func, "__user_ctx__", False) + + +@deprecated("Use this untile rapyer provide safe option for aget") +async def rapyer_aget_safe(redis_key: str) -> Optional[AtomicRedisModel]: + try: + return await rapyer.aget(redis_key) + except KeyNotFound: + return None diff --git a/mageflow/visualizer/builder.py b/mageflow/visualizer/builder.py index e5613dbf..cda8a340 100644 --- a/mageflow/visualizer/builder.py +++ b/mageflow/visualizer/builder.py @@ -5,9 +5,6 @@ from typing import Generic, TypeVar, TypeAlias, Any, Union, Annotated, Literal from dash import html -from pydantic import GetCoreSchemaHandler, BaseModel, PrivateAttr, Field -from pydantic_core import core_schema - from mageflow.chain.consts import ON_CHAIN_ERROR, ON_CHAIN_END from mageflow.chain.model import ChainTaskSignature from mageflow.signature.consts import MAGEFLOW_TASK_INITIALS @@ -15,6 +12,8 @@ from mageflow.swarm.consts import ON_SWARM_ERROR, ON_SWARM_START, ON_SWARM_END from mageflow.swarm.model import SwarmTaskSignature, BatchItemTaskSignature from mageflow.typing_support import Self +from pydantic import GetCoreSchemaHandler, BaseModel, PrivateAttr, Field +from pydantic_core import core_schema T = TypeVar("T", bound=TaskSignature) INTERNAL_TASKS = [ diff --git a/mageflow/visualizer/data.py b/mageflow/visualizer/data.py index 13750804..dc21de46 100644 --- a/mageflow/visualizer/data.py +++ b/mageflow/visualizer/data.py @@ -1,8 +1,7 @@ import asyncio -import rapyer - import mageflow +import rapyer from mageflow.signature.model import TaskSignature diff --git a/mageflow/visualizer/utils.py b/mageflow/visualizer/utils.py index dc5cc280..0c594e1b 100644 --- a/mageflow/visualizer/utils.py +++ b/mageflow/visualizer/utils.py @@ -2,12 +2,11 @@ import inspect from typing import Any, Optional, Callable, get_origin, Tuple, get_args +from mageflow.utils.pythonic import flexible_call from pydantic import TypeAdapter from pydantic_core import ValidationError from rapyer.types.base import REDIS_DUMP_FLAG_NAME -from mageflow.utils.pythonic import flexible_call - def try_validate_json(validator: TypeAdapter, data: Any): if data is None: diff --git a/mageflow/workflows.py b/mageflow/workflows.py index 0ae7ddf4..e7bad338 100644 --- a/mageflow/workflows.py +++ b/mageflow/workflows.py @@ -5,9 +5,8 @@ from hatchet_sdk.runnables.types import TWorkflowInput, EmptyModel from hatchet_sdk.runnables.workflow import Workflow from hatchet_sdk.utils.typing import JSONSerializableMapping -from pydantic import BaseModel - from mageflow.utils.pythonic import deep_merge +from pydantic import BaseModel TASK_DATA_PARAM_NAME = "task_data" diff --git a/mkdocs.yml b/mkdocs.yml index e2386559..7f091600 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -66,6 +66,7 @@ nav: - Callbacks: documentation/callbacks.md - Chain: documentation/chain.md - Swarm: documentation/swarm.md + - Root Task: documentation/root-task.md - Task Lifecycle: documentation/task-lifecycle.md - API Reference: - Client: api/client.md diff --git a/poetry.lock b/poetry.lock index f6c9c270..2480c677 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -155,7 +155,7 @@ propcache = ">=0.2.0" yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns (>=3.3.0)", "backports.zstd", "brotlicffi"] +speedups = ["Brotli ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" @@ -211,7 +211,7 @@ version = "5.0.1" description = "Timeout context manager for asyncio programs" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main"] markers = "python_full_version < \"3.11.3\"" files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, @@ -235,10 +235,10 @@ files = [ name = "backports-asyncio-runner" version = "1.2.0" description = "Backport of asyncio.Runner, a context manager that controls event loop life cycle." -optional = false +optional = true python-versions = "<3.11,>=3.8" -groups = ["dev"] -markers = "python_version < \"3.11\"" +groups = ["main"] +markers = "extra == \"test\" and python_version == \"3.10\"" files = [ {file = "backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5"}, {file = "backports_asyncio_runner-1.2.0.tar.gz", hash = "sha256:a5aa7b2b7d8f8bfcaa2b57313f70792df84e32a2a746f585213373f900b42162"}, @@ -248,9 +248,10 @@ files = [ name = "black" version = "25.12.0" description = "The uncompromising code formatter." -optional = false +optional = true python-versions = ">=3.10" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"dev\"" files = [ {file = "black-25.12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f85ba1ad15d446756b4ab5f3044731bf68b777f8f9ac9cdabd2425b97cd9c4e8"}, {file = "black-25.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:546eecfe9a3a6b46f9d69d8a642585a6eaf348bcbbc4d87a19635570e02d9f4a"}, @@ -314,9 +315,10 @@ files = [ name = "cachetools" version = "6.2.4" description = "Extensible memoizing collections and decorators" -optional = false +optional = true python-versions = ">=3.9" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"dev\"" files = [ {file = "cachetools-6.2.4-py3-none-any.whl", hash = "sha256:69a7a52634fed8b8bf6e24a050fb60bff1c9bd8f6d24572b99c32d4e71e62a51"}, {file = "cachetools-6.2.4.tar.gz", hash = "sha256:82c5c05585e70b6ba2d3ae09ea60b79548872185d2f24ae1f2709d37299fd607"}, @@ -326,9 +328,10 @@ files = [ name = "certifi" version = "2025.11.12" description = "Python package for providing Mozilla's CA Bundle." -optional = false +optional = true python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main"] +markers = "extra == \"display\" or extra == \"integration\"" files = [ {file = "certifi-2025.11.12-py3-none-any.whl", hash = "sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b"}, {file = "certifi-2025.11.12.tar.gz", hash = "sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316"}, @@ -338,9 +341,10 @@ files = [ name = "chardet" version = "5.2.0" description = "Universal encoding detector for Python 3" -optional = false +optional = true python-versions = ">=3.7" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"dev\"" files = [ {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, @@ -350,9 +354,10 @@ files = [ name = "charset-normalizer" version = "3.4.4" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false +optional = true python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main"] +markers = "extra == \"display\" or extra == \"integration\"" files = [ {file = "charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d"}, {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8"}, @@ -473,14 +478,14 @@ files = [ name = "click" version = "8.3.1" description = "Composable command line interface toolkit" -optional = false +optional = true python-versions = ">=3.10" -groups = ["main", "dev"] +groups = ["main"] +markers = "extra == \"display\" or extra == \"dev\"" files = [ {file = "click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6"}, {file = "click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a"}, ] -markers = {main = "extra == \"display\""} [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -489,14 +494,123 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -optional = false +optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main", "dev"] +groups = ["main"] +markers = "(extra == \"display\" or extra == \"dev\") and platform_system == \"Windows\" or (extra == \"test\" or extra == \"integration\") and sys_platform == \"win32\" or extra == \"dev\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {main = "extra == \"display\" and platform_system == \"Windows\""} + +[[package]] +name = "coverage" +version = "7.13.1" +description = "Code coverage measurement for Python" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"coverage\"" +files = [ + {file = "coverage-7.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e1fa280b3ad78eea5be86f94f461c04943d942697e0dac889fa18fff8f5f9147"}, + {file = "coverage-7.13.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c3d8c679607220979434f494b139dfb00131ebf70bb406553d69c1ff01a5c33d"}, + {file = "coverage-7.13.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:339dc63b3eba969067b00f41f15ad161bf2946613156fb131266d8debc8e44d0"}, + {file = "coverage-7.13.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:db622b999ffe49cb891f2fff3b340cdc2f9797d01a0a202a0973ba2562501d90"}, + {file = "coverage-7.13.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1443ba9acbb593fa7c1c29e011d7c9761545fe35e7652e85ce7f51a16f7e08d"}, + {file = "coverage-7.13.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c832ec92c4499ac463186af72f9ed4d8daec15499b16f0a879b0d1c8e5cf4a3b"}, + {file = "coverage-7.13.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:562ec27dfa3f311e0db1ba243ec6e5f6ab96b1edfcfc6cf86f28038bc4961ce6"}, + {file = "coverage-7.13.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:4de84e71173d4dada2897e5a0e1b7877e5eefbfe0d6a44edee6ce31d9b8ec09e"}, + {file = "coverage-7.13.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:a5a68357f686f8c4d527a2dc04f52e669c2fc1cbde38f6f7eb6a0e58cbd17cae"}, + {file = "coverage-7.13.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:77cc258aeb29a3417062758975521eae60af6f79e930d6993555eeac6a8eac29"}, + {file = "coverage-7.13.1-cp310-cp310-win32.whl", hash = "sha256:bb4f8c3c9a9f34423dba193f241f617b08ffc63e27f67159f60ae6baf2dcfe0f"}, + {file = "coverage-7.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:c8e2706ceb622bc63bac98ebb10ef5da80ed70fbd8a7999a5076de3afaef0fb1"}, + {file = "coverage-7.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a55d509a1dc5a5b708b5dad3b5334e07a16ad4c2185e27b40e4dba796ab7f88"}, + {file = "coverage-7.13.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4d010d080c4888371033baab27e47c9df7d6fb28d0b7b7adf85a4a49be9298b3"}, + {file = "coverage-7.13.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d938b4a840fb1523b9dfbbb454f652967f18e197569c32266d4d13f37244c3d9"}, + {file = "coverage-7.13.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bf100a3288f9bb7f919b87eb84f87101e197535b9bd0e2c2b5b3179633324fee"}, + {file = "coverage-7.13.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef6688db9bf91ba111ae734ba6ef1a063304a881749726e0d3575f5c10a9facf"}, + {file = "coverage-7.13.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0b609fc9cdbd1f02e51f67f51e5aee60a841ef58a68d00d5ee2c0faf357481a3"}, + {file = "coverage-7.13.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c43257717611ff5e9a1d79dce8e47566235ebda63328718d9b65dd640bc832ef"}, + {file = "coverage-7.13.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e09fbecc007f7b6afdfb3b07ce5bd9f8494b6856dd4f577d26c66c391b829851"}, + {file = "coverage-7.13.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:a03a4f3a19a189919c7055098790285cc5c5b0b3976f8d227aea39dbf9f8bfdb"}, + {file = "coverage-7.13.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3820778ea1387c2b6a818caec01c63adc5b3750211af6447e8dcfb9b6f08dbba"}, + {file = "coverage-7.13.1-cp311-cp311-win32.whl", hash = "sha256:ff10896fa55167371960c5908150b434b71c876dfab97b69478f22c8b445ea19"}, + {file = "coverage-7.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:a998cc0aeeea4c6d5622a3754da5a493055d2d95186bad877b0a34ea6e6dbe0a"}, + {file = "coverage-7.13.1-cp311-cp311-win_arm64.whl", hash = "sha256:fea07c1a39a22614acb762e3fbbb4011f65eedafcb2948feeef641ac78b4ee5c"}, + {file = "coverage-7.13.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6f34591000f06e62085b1865c9bc5f7858df748834662a51edadfd2c3bfe0dd3"}, + {file = "coverage-7.13.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b67e47c5595b9224599016e333f5ec25392597a89d5744658f837d204e16c63e"}, + {file = "coverage-7.13.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3e7b8bd70c48ffb28461ebe092c2345536fb18bbbf19d287c8913699735f505c"}, + {file = "coverage-7.13.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c223d078112e90dc0e5c4e35b98b9584164bea9fbbd221c0b21c5241f6d51b62"}, + {file = "coverage-7.13.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:794f7c05af0763b1bbd1b9e6eff0e52ad068be3b12cd96c87de037b01390c968"}, + {file = "coverage-7.13.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0642eae483cc8c2902e4af7298bf886d605e80f26382124cddc3967c2a3df09e"}, + {file = "coverage-7.13.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f5e772ed5fef25b3de9f2008fe67b92d46831bd2bc5bdc5dd6bfd06b83b316f"}, + {file = "coverage-7.13.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:45980ea19277dc0a579e432aef6a504fe098ef3a9032ead15e446eb0f1191aee"}, + {file = "coverage-7.13.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f18eca6028ffa62adbd185a8f1e1dd242f2e68164dba5c2b74a5204850b4cf"}, + {file = "coverage-7.13.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8dca5590fec7a89ed6826fce625595279e586ead52e9e958d3237821fbc750c"}, + {file = "coverage-7.13.1-cp312-cp312-win32.whl", hash = "sha256:ff86d4e85188bba72cfb876df3e11fa243439882c55957184af44a35bd5880b7"}, + {file = "coverage-7.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:16cc1da46c04fb0fb128b4dc430b78fa2aba8a6c0c9f8eb391fd5103409a6ac6"}, + {file = "coverage-7.13.1-cp312-cp312-win_arm64.whl", hash = "sha256:8d9bc218650022a768f3775dd7fdac1886437325d8d295d923ebcfef4892ad5c"}, + {file = "coverage-7.13.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:cb237bfd0ef4d5eb6a19e29f9e528ac67ac3be932ea6b44fb6cc09b9f3ecff78"}, + {file = "coverage-7.13.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1dcb645d7e34dcbcc96cd7c132b1fc55c39263ca62eb961c064eb3928997363b"}, + {file = "coverage-7.13.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3d42df8201e00384736f0df9be2ced39324c3907607d17d50d50116c989d84cd"}, + {file = "coverage-7.13.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fa3edde1aa8807de1d05934982416cb3ec46d1d4d91e280bcce7cca01c507992"}, + {file = "coverage-7.13.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9edd0e01a343766add6817bc448408858ba6b489039eaaa2018474e4001651a4"}, + {file = "coverage-7.13.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:985b7836931d033570b94c94713c6dba5f9d3ff26045f72c3e5dbc5fe3361e5a"}, + {file = "coverage-7.13.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ffed1e4980889765c84a5d1a566159e363b71d6b6fbaf0bebc9d3c30bc016766"}, + {file = "coverage-7.13.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8842af7f175078456b8b17f1b73a0d16a65dcbdc653ecefeb00a56b3c8c298c4"}, + {file = "coverage-7.13.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ccd7a6fca48ca9c131d9b0a2972a581e28b13416fc313fb98b6d24a03ce9a398"}, + {file = "coverage-7.13.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0403f647055de2609be776965108447deb8e384fe4a553c119e3ff6bfbab4784"}, + {file = "coverage-7.13.1-cp313-cp313-win32.whl", hash = "sha256:549d195116a1ba1e1ae2f5ca143f9777800f6636eab917d4f02b5310d6d73461"}, + {file = "coverage-7.13.1-cp313-cp313-win_amd64.whl", hash = "sha256:5899d28b5276f536fcf840b18b61a9fce23cc3aec1d114c44c07fe94ebeaa500"}, + {file = "coverage-7.13.1-cp313-cp313-win_arm64.whl", hash = "sha256:868a2fae76dfb06e87291bcbd4dcbcc778a8500510b618d50496e520bd94d9b9"}, + {file = "coverage-7.13.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:67170979de0dacac3f3097d02b0ad188d8edcea44ccc44aaa0550af49150c7dc"}, + {file = "coverage-7.13.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f80e2bb21bfab56ed7405c2d79d34b5dc0bc96c2c1d2a067b643a09fb756c43a"}, + {file = "coverage-7.13.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f83351e0f7dcdb14d7326c3d8d8c4e915fa685cbfdc6281f9470d97a04e9dfe4"}, + {file = "coverage-7.13.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bb3f6562e89bad0110afbe64e485aac2462efdce6232cdec7862a095dc3412f6"}, + {file = "coverage-7.13.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77545b5dcda13b70f872c3b5974ac64c21d05e65b1590b441c8560115dc3a0d1"}, + {file = "coverage-7.13.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4d240d260a1aed814790bbe1f10a5ff31ce6c21bc78f0da4a1e8268d6c80dbd"}, + {file = "coverage-7.13.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d2287ac9360dec3837bfdad969963a5d073a09a85d898bd86bea82aa8876ef3c"}, + {file = "coverage-7.13.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:0d2c11f3ea4db66b5cbded23b20185c35066892c67d80ec4be4bab257b9ad1e0"}, + {file = "coverage-7.13.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:3fc6a169517ca0d7ca6846c3c5392ef2b9e38896f61d615cb75b9e7134d4ee1e"}, + {file = "coverage-7.13.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d10a2ed46386e850bb3de503a54f9fe8192e5917fcbb143bfef653a9355e9a53"}, + {file = "coverage-7.13.1-cp313-cp313t-win32.whl", hash = "sha256:75a6f4aa904301dab8022397a22c0039edc1f51e90b83dbd4464b8a38dc87842"}, + {file = "coverage-7.13.1-cp313-cp313t-win_amd64.whl", hash = "sha256:309ef5706e95e62578cda256b97f5e097916a2c26247c287bbe74794e7150df2"}, + {file = "coverage-7.13.1-cp313-cp313t-win_arm64.whl", hash = "sha256:92f980729e79b5d16d221038dbf2e8f9a9136afa072f9d5d6ed4cb984b126a09"}, + {file = "coverage-7.13.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:97ab3647280d458a1f9adb85244e81587505a43c0c7cff851f5116cd2814b894"}, + {file = "coverage-7.13.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8f572d989142e0908e6acf57ad1b9b86989ff057c006d13b76c146ec6a20216a"}, + {file = "coverage-7.13.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d72140ccf8a147e94274024ff6fd8fb7811354cf7ef88b1f0a988ebaa5bc774f"}, + {file = "coverage-7.13.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d3c9f051b028810f5a87c88e5d6e9af3c0ff32ef62763bf15d29f740453ca909"}, + {file = "coverage-7.13.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f398ba4df52d30b1763f62eed9de5620dcde96e6f491f4c62686736b155aa6e4"}, + {file = "coverage-7.13.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:132718176cc723026d201e347f800cd1a9e4b62ccd3f82476950834dad501c75"}, + {file = "coverage-7.13.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9e549d642426e3579b3f4b92d0431543b012dcb6e825c91619d4e93b7363c3f9"}, + {file = "coverage-7.13.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:90480b2134999301eea795b3a9dbf606c6fbab1b489150c501da84a959442465"}, + {file = "coverage-7.13.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e825dbb7f84dfa24663dd75835e7257f8882629fc11f03ecf77d84a75134b864"}, + {file = "coverage-7.13.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:623dcc6d7a7ba450bbdbeedbaa0c42b329bdae16491af2282f12a7e809be7eb9"}, + {file = "coverage-7.13.1-cp314-cp314-win32.whl", hash = "sha256:6e73ebb44dca5f708dc871fe0b90cf4cff1a13f9956f747cc87b535a840386f5"}, + {file = "coverage-7.13.1-cp314-cp314-win_amd64.whl", hash = "sha256:be753b225d159feb397bd0bf91ae86f689bad0da09d3b301478cd39b878ab31a"}, + {file = "coverage-7.13.1-cp314-cp314-win_arm64.whl", hash = "sha256:228b90f613b25ba0019361e4ab81520b343b622fc657daf7e501c4ed6a2366c0"}, + {file = "coverage-7.13.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:60cfb538fe9ef86e5b2ab0ca8fc8d62524777f6c611dcaf76dc16fbe9b8e698a"}, + {file = "coverage-7.13.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:57dfc8048c72ba48a8c45e188d811e5efd7e49b387effc8fb17e97936dde5bf6"}, + {file = "coverage-7.13.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3f2f725aa3e909b3c5fdb8192490bdd8e1495e85906af74fe6e34a2a77ba0673"}, + {file = "coverage-7.13.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ee68b21909686eeb21dfcba2c3b81fee70dcf38b140dcd5aa70680995fa3aa5"}, + {file = "coverage-7.13.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:724b1b270cb13ea2e6503476e34541a0b1f62280bc997eab443f87790202033d"}, + {file = "coverage-7.13.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:916abf1ac5cf7eb16bc540a5bf75c71c43a676f5c52fcb9fe75a2bd75fb944e8"}, + {file = "coverage-7.13.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:776483fd35b58d8afe3acbd9988d5de592ab6da2d2a865edfdbc9fdb43e7c486"}, + {file = "coverage-7.13.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:b6f3b96617e9852703f5b633ea01315ca45c77e879584f283c44127f0f1ec564"}, + {file = "coverage-7.13.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd63e7b74661fed317212fab774e2a648bc4bb09b35f25474f8e3325d2945cd7"}, + {file = "coverage-7.13.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:933082f161bbb3e9f90d00990dc956120f608cdbcaeea15c4d897f56ef4fe416"}, + {file = "coverage-7.13.1-cp314-cp314t-win32.whl", hash = "sha256:18be793c4c87de2965e1c0f060f03d9e5aff66cfeae8e1dbe6e5b88056ec153f"}, + {file = "coverage-7.13.1-cp314-cp314t-win_amd64.whl", hash = "sha256:0e42e0ec0cd3e0d851cb3c91f770c9301f48647cb2877cb78f74bdaa07639a79"}, + {file = "coverage-7.13.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eaecf47ef10c72ece9a2a92118257da87e460e113b83cc0d2905cbbe931792b4"}, + {file = "coverage-7.13.1-py3-none-any.whl", hash = "sha256:2016745cb3ba554469d02819d78958b571792bb68e31302610e898f80dd3a573"}, + {file = "coverage-7.13.1.tar.gz", hash = "sha256:b7593fe7eb5feaa3fbb461ac79aac9f9fc0387a5ca8080b0c6fe2ca27b091afd"}, +] + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "dash" @@ -527,7 +641,7 @@ Werkzeug = "<3.2" ag-grid = ["dash-ag-grid"] async = ["flask[async]"] celery = ["celery[redis] (>=5.1.2,<5.4.0)", "kombu (<5.4.0)", "redis (>=3.5.3,<=5.0.4)"] -ci = ["black (==22.3.0)", "flake8 (==7.0.0)", "flaky (==3.8.1)", "flask-talisman (==1.0.0)", "ipython (<9.0.0)", "jupyterlab (<4.0.0)", "mimesis (<=11.1.0)", "mock (==4.0.3)", "mypy (==1.15.0)", "numpy (<=1.26.3)", "openpyxl", "orjson (>=3.10.11)", "pandas (>=1.4.0)", "pyarrow", "pylint (==3.0.3)", "pyright (==1.1.398)", "pytest-mock", "pytest-rerunfailures", "pytest-sugar (==0.9.6)", "pyzmq (>=26.0.0)", "xlrd (>=2.0.1)"] +ci = ["black (==22.3.0)", "flake8 (==7.0.0)", "flaky (==3.8.1)", "flask-talisman (==1.0.0)", "ipython (<9.0.0)", "jupyterlab (<4.0.0)", "mimesis (<=11.1.0)", "mock (==4.0.3)", "mypy (==1.15.0) ; python_version >= \"3.12\"", "numpy (<=1.26.3)", "openpyxl", "orjson (>=3.10.11)", "pandas (>=1.4.0)", "pyarrow", "pylint (==3.0.3)", "pyright (==1.1.398) ; python_version >= \"3.7\"", "pytest-mock", "pytest-rerunfailures", "pytest-sugar (==0.9.6)", "pyzmq (>=26.0.0)", "xlrd (>=2.0.1)"] cloud = ["plotly-cloud"] compress = ["flask-compress"] dev = ["PyYAML (>=5.4.1)", "coloredlogs (>=15.0.1)", "fire (>=0.4.0)"] @@ -575,9 +689,10 @@ leaflet = ["dash-leaflet (>=1.0.16rc3)"] name = "distlib" version = "0.4.0" description = "Distribution utilities" -optional = false +optional = true python-versions = "*" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"dev\"" files = [ {file = "distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16"}, {file = "distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d"}, @@ -587,9 +702,10 @@ files = [ name = "dynaconf" version = "3.2.12" description = "The dynamic configurator for your Python Project" -optional = false +optional = true python-versions = ">=3.8" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"test\"" files = [ {file = "dynaconf-3.2.12-py2.py3-none-any.whl", hash = "sha256:eb2a11865917dff8810c6098cd736b8f4d2f4e39ad914500e2dfbe064b82c499"}, {file = "dynaconf-3.2.12.tar.gz", hash = "sha256:29cea583b007d890e6031fa89c0ac489b631c73dbee83bcd5e6f97602c26354e"}, @@ -609,10 +725,10 @@ yaml = ["ruamel.yaml"] name = "exceptiongroup" version = "1.3.1" description = "Backport of PEP 654 (exception groups)" -optional = false +optional = true python-versions = ">=3.7" -groups = ["dev"] -markers = "python_version < \"3.11\"" +groups = ["main"] +markers = "(extra == \"test\" or extra == \"integration\") and python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, @@ -628,9 +744,10 @@ test = ["pytest (>=6)"] name = "fakeredis" version = "2.33.0" description = "Python implementation of redis API, can be used for testing purposes." -optional = false +optional = true python-versions = ">=3.7" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"test\"" files = [ {file = "fakeredis-2.33.0-py3-none-any.whl", hash = "sha256:de535f3f9ccde1c56672ab2fdd6a8efbc4f2619fc2f1acc87b8737177d71c965"}, {file = "fakeredis-2.33.0.tar.gz", hash = "sha256:d7bc9a69d21df108a6451bbffee23b3eba432c21a654afc7ff2d295428ec5770"}, @@ -638,6 +755,7 @@ files = [ [package.dependencies] jsonpath-ng = {version = ">=1.6", optional = true, markers = "extra == \"json\""} +lupa = {version = ">=2.1", optional = true, markers = "extra == \"lua\""} redis = {version = ">=4.3", markers = "python_version > \"3.8\""} sortedcontainers = ">=2" typing-extensions = {version = ">=4.7,<5.0", markers = "python_version < \"3.11\""} @@ -648,15 +766,16 @@ cf = ["pyprobables (>=0.6)"] json = ["jsonpath-ng (>=1.6)"] lua = ["lupa (>=2.1)"] probabilistic = ["pyprobables (>=0.6)"] -valkey = ["valkey (>=6)"] +valkey = ["valkey (>=6) ; python_version >= \"3.8\""] [[package]] name = "filelock" version = "3.20.1" description = "A platform independent file lock." -optional = false +optional = true python-versions = ">=3.10" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"dev\"" files = [ {file = "filelock-3.20.1-py3-none-any.whl", hash = "sha256:15d9e9a67306188a44baa72f569d2bfd803076269365fdea0934385da4dc361a"}, {file = "filelock-3.20.1.tar.gz", hash = "sha256:b8360948b351b80f420878d8516519a2204b07aefcdcfd24912a5d33127f188c"}, @@ -1016,9 +1135,10 @@ otel = ["opentelemetry-api (>=1.28.0,<2.0.0)", "opentelemetry-distro (>=0.49b0)" name = "idna" version = "3.11" description = "Internationalized Domain Names in Applications (IDNA)" -optional = false +optional = true python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main"] +markers = "extra == \"hatchet\" or extra == \"display\" or extra == \"integration\"" files = [ {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, @@ -1044,21 +1164,22 @@ files = [ zipp = ">=3.20" [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=3.4)"] perf = ["ipython"] test = ["flufl.flake8", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] -type = ["mypy (<1.19)", "pytest-mypy (>=1.0.1)"] +type = ["mypy (<1.19) ; platform_python_implementation == \"PyPy\"", "pytest-mypy (>=1.0.1)"] [[package]] name = "iniconfig" version = "2.3.0" description = "brain-dead simple config-ini parsing" -optional = false +optional = true python-versions = ">=3.10" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"test\" or extra == \"integration\"" files = [ {file = "iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12"}, {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, @@ -1100,9 +1221,10 @@ i18n = ["Babel (>=2.7)"] name = "jsonpath-ng" version = "1.7.0" description = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." -optional = false +optional = true python-versions = "*" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"test\"" files = [ {file = "jsonpath-ng-1.7.0.tar.gz", hash = "sha256:f6f5f7fd4e5ff79c785f1573b394043b39849fb2bb47bcead935d12b00beab3c"}, {file = "jsonpath_ng-1.7.0-py2-none-any.whl", hash = "sha256:898c93fc173f0c336784a3fa63d7434297544b7198124a68f9a3ef9597b0ae6e"}, @@ -1112,6 +1234,105 @@ files = [ [package.dependencies] ply = "*" +[[package]] +name = "lupa" +version = "2.6" +description = "Python wrapper around Lua and LuaJIT" +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"test\"" +files = [ + {file = "lupa-2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6b3dabda836317e63c5ad052826e156610f356a04b3003dfa0dbe66b5d54d671"}, + {file = "lupa-2.6-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:8726d1c123bbe9fbb974ce29825e94121824e66003038ff4532c14cc2ed0c51c"}, + {file = "lupa-2.6-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:f4e159e7d814171199b246f9235ca8961f6461ea8c1165ab428afa13c9289a94"}, + {file = "lupa-2.6-cp310-cp310-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:202160e80dbfddfb79316692a563d843b767e0f6787bbd1c455f9d54052efa6c"}, + {file = "lupa-2.6-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5deede7c5b36ab64f869dae4831720428b67955b0bb186c8349cf6ea121c852b"}, + {file = "lupa-2.6-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:86f04901f920bbf7c0cac56807dc9597e42347123e6f1f3ca920f15f54188ce5"}, + {file = "lupa-2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6deef8f851d6afb965c84849aa5b8c38856942df54597a811ce0369ced678610"}, + {file = "lupa-2.6-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:21f2b5549681c2a13b1170a26159d30875d367d28f0247b81ca347222c755038"}, + {file = "lupa-2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:66eea57630eab5e6f49fdc5d7811c0a2a41f2011be4ea56a087ea76112011eb7"}, + {file = "lupa-2.6-cp310-cp310-win32.whl", hash = "sha256:60a403de8cab262a4fe813085dd77010effa6e2eb1886db2181df803140533b1"}, + {file = "lupa-2.6-cp310-cp310-win_amd64.whl", hash = "sha256:e4656a39d93dfa947cf3db56dc16c7916cb0cc8024acd3a952071263f675df64"}, + {file = "lupa-2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6d988c0f9331b9f2a5a55186701a25444ab10a1432a1021ee58011499ecbbdd5"}, + {file = "lupa-2.6-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:ebe1bbf48259382c72a6fe363dea61a0fd6fe19eab95e2ae881e20f3654587bf"}, + {file = "lupa-2.6-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:a8fcee258487cf77cdd41560046843bb38c2e18989cd19671dd1e2596f798306"}, + {file = "lupa-2.6-cp311-cp311-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:561a8e3be800827884e767a694727ed8482d066e0d6edfcbf423b05e63b05535"}, + {file = "lupa-2.6-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af880a62d47991cae78b8e9905c008cbfdc4a3a9723a66310c2634fc7644578c"}, + {file = "lupa-2.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80b22923aa4023c86c0097b235615f89d469a0c4eee0489699c494d3367c4c85"}, + {file = "lupa-2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:153d2cc6b643f7efb9cfc0c6bb55ec784d5bac1a3660cfc5b958a7b8f38f4a75"}, + {file = "lupa-2.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:3fa8777e16f3ded50b72967dc17e23f5a08e4f1e2c9456aff2ebdb57f5b2869f"}, + {file = "lupa-2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8dbdcbe818c02a2f56f5ab5ce2de374dab03e84b25266cfbaef237829bc09b3f"}, + {file = "lupa-2.6-cp311-cp311-win32.whl", hash = "sha256:defaf188fde8f7a1e5ce3a5e6d945e533b8b8d547c11e43b96c9b7fe527f56dc"}, + {file = "lupa-2.6-cp311-cp311-win_amd64.whl", hash = "sha256:9505ae600b5c14f3e17e70f87f88d333717f60411faca1ddc6f3e61dce85fa9e"}, + {file = "lupa-2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:47ce718817ef1cc0c40d87c3d5ae56a800d61af00fbc0fad1ca9be12df2f3b56"}, + {file = "lupa-2.6-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:7aba985b15b101495aa4b07112cdc08baa0c545390d560ad5cfde2e9e34f4d58"}, + {file = "lupa-2.6-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:b766f62f95b2739f2248977d29b0722e589dcf4f0ccfa827ccbd29f0148bd2e5"}, + {file = "lupa-2.6-cp312-cp312-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:00a934c23331f94cb51760097ebfab14b005d55a6b30a2b480e3c53dd2fa290d"}, + {file = "lupa-2.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21de9f38bd475303e34a042b7081aabdf50bd9bafd36ce4faea2f90fd9f15c31"}, + {file = "lupa-2.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf3bda96d3fc41237e964a69c23647d50d4e28421111360274d4799832c560e9"}, + {file = "lupa-2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5a76ead245da54801a81053794aa3975f213221f6542d14ec4b859ee2e7e0323"}, + {file = "lupa-2.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8dd0861741caa20886ddbda0a121d8e52fb9b5bb153d82fa9bba796962bf30e8"}, + {file = "lupa-2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:239e63948b0b23023f81d9a19a395e768ed3da6a299f84e7963b8f813f6e3f9c"}, + {file = "lupa-2.6-cp312-cp312-win32.whl", hash = "sha256:325894e1099499e7a6f9c351147661a2011887603c71086d36fe0f964d52d1ce"}, + {file = "lupa-2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c735a1ce8ee60edb0fe71d665f1e6b7c55c6021f1d340eb8c865952c602cd36f"}, + {file = "lupa-2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:663a6e58a0f60e7d212017d6678639ac8df0119bc13c2145029dcba084391310"}, + {file = "lupa-2.6-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:d1f5afda5c20b1f3217a80e9bc1b77037f8a6eb11612fd3ada19065303c8f380"}, + {file = "lupa-2.6-cp313-cp313-macosx_11_0_x86_64.whl", hash = "sha256:26f2b3c085fe76e9119e48c1013c1cccdc1f51585d456858290475aa38e7089e"}, + {file = "lupa-2.6-cp313-cp313-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:60d2f902c7b96fb8ab98493dcff315e7bb4d0b44dc9dd76eb37de575025d5685"}, + {file = "lupa-2.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a02d25dee3a3250967c36590128d9220ae02f2eda166a24279da0b481519cbff"}, + {file = "lupa-2.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6eae1ee16b886b8914ff292dbefbf2f48abfbdee94b33a88d1d5475e02423203"}, + {file = "lupa-2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0edd5073a4ee74ab36f74fe61450148e6044f3952b8d21248581f3c5d1a58be"}, + {file = "lupa-2.6-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0c53ee9f22a8a17e7d4266ad48e86f43771951797042dd51d1494aaa4f5f3f0a"}, + {file = "lupa-2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:de7c0f157a9064a400d828789191a96da7f4ce889969a588b87ec80de9b14772"}, + {file = "lupa-2.6-cp313-cp313-win32.whl", hash = "sha256:ee9523941ae0a87b5b703417720c5d78f72d2f5bc23883a2ea80a949a3ed9e75"}, + {file = "lupa-2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b1335a5835b0a25ebdbc75cf0bda195e54d133e4d994877ef025e218c2e59db9"}, + {file = "lupa-2.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dcb6d0a3264873e1653bc188499f48c1fb4b41a779e315eba45256cfe7bc33c1"}, + {file = "lupa-2.6-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:a37e01f2128f8c36106726cb9d360bac087d58c54b4522b033cc5691c584db18"}, + {file = "lupa-2.6-cp314-cp314-macosx_11_0_x86_64.whl", hash = "sha256:458bd7e9ff3c150b245b0fcfbb9bd2593d1152ea7f0a7b91c1d185846da033fe"}, + {file = "lupa-2.6-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:052ee82cac5206a02df77119c325339acbc09f5ce66967f66a2e12a0f3211cad"}, + {file = "lupa-2.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96594eca3c87dd07938009e95e591e43d554c1dbd0385be03c100367141db5a8"}, + {file = "lupa-2.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8faddd9d198688c8884091173a088a8e920ecc96cda2ffed576a23574c4b3f6"}, + {file = "lupa-2.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:daebb3a6b58095c917e76ba727ab37b27477fb926957c825205fbda431552134"}, + {file = "lupa-2.6-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f3154e68972befe0f81564e37d8142b5d5d79931a18309226a04ec92487d4ea3"}, + {file = "lupa-2.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e4dadf77b9fedc0bfa53417cc28dc2278a26d4cbd95c29f8927ad4d8fe0a7ef9"}, + {file = "lupa-2.6-cp314-cp314-win32.whl", hash = "sha256:cb34169c6fa3bab3e8ac58ca21b8a7102f6a94b6a5d08d3636312f3f02fafd8f"}, + {file = "lupa-2.6-cp314-cp314-win_amd64.whl", hash = "sha256:b74f944fe46c421e25d0f8692aef1e842192f6f7f68034201382ac440ef9ea67"}, + {file = "lupa-2.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0e21b716408a21ab65723f8841cf7f2f37a844b7a965eeabb785e27fca4099cf"}, + {file = "lupa-2.6-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:589db872a141bfff828340079bbdf3e9a31f2689f4ca0d88f97d9e8c2eae6142"}, + {file = "lupa-2.6-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:cd852a91a4a9d4dcbb9a58100f820a75a425703ec3e3f049055f60b8533b7953"}, + {file = "lupa-2.6-cp314-cp314t-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:0334753be028358922415ca97a64a3048e4ed155413fc4eaf87dd0a7e2752983"}, + {file = "lupa-2.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:661d895cd38c87658a34780fac54a690ec036ead743e41b74c3fb81a9e65a6aa"}, + {file = "lupa-2.6-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aa58454ccc13878cc177c62529a2056be734da16369e451987ff92784994ca7"}, + {file = "lupa-2.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1425017264e470c98022bba8cff5bd46d054a827f5df6b80274f9cc71dafd24f"}, + {file = "lupa-2.6-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:224af0532d216e3105f0a127410f12320f7c5f1aa0300bdf9646b8d9afb0048c"}, + {file = "lupa-2.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9abb98d5a8fd27c8285302e82199f0e56e463066f88f619d6594a450bf269d80"}, + {file = "lupa-2.6-cp314-cp314t-win32.whl", hash = "sha256:1849efeba7a8f6fb8aa2c13790bee988fd242ae404bd459509640eeea3d1e291"}, + {file = "lupa-2.6-cp314-cp314t-win_amd64.whl", hash = "sha256:fc1498d1a4fc028bc521c26d0fad4ca00ed63b952e32fb95949bda76a04bad52"}, + {file = "lupa-2.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9591700991e333b70dd92b48f152eb4731b8b24af671a9f6f721b74d68ed4499"}, + {file = "lupa-2.6-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:ef8dfa7fe08bc3f4591411b8945bbeb15af8512c3e7ad5e9b1e3a9036cdbbce7"}, + {file = "lupa-2.6-cp38-cp38-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:728c466e91174dad238f8a9c1cbdb8e69ffe559df85f87ee76edac3395300949"}, + {file = "lupa-2.6-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c781170bc7134704ae317a66204d30688b41d3e471e17e659987ea4947e11f20"}, + {file = "lupa-2.6-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241f4ddab33b9a686fc76667241bebc39a06b74ec40d79ec222f5add9000fe57"}, + {file = "lupa-2.6-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:c17f6b6193ced33cc7ca0b2b08b319a1b3501b014a3a3f9999c01cafc04c40f5"}, + {file = "lupa-2.6-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:fa6c1379e83d4104065c151736250a09f3a99e368423c7a20f9c59b15945e9fc"}, + {file = "lupa-2.6-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aef1a8bc10c50695e1a33a07dbef803b93eb97fc150fdb19858d704a603a67dd"}, + {file = "lupa-2.6-cp38-cp38-win32.whl", hash = "sha256:10c191bc1d5565e4360d884bea58320975ddb33270cdf9a9f55d1a1efe79aa03"}, + {file = "lupa-2.6-cp38-cp38-win_amd64.whl", hash = "sha256:05681f8ffb41f0c7fbb9ca859cc3a7e4006e9c6350d25358b535c5295c6a9928"}, + {file = "lupa-2.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8897dc6c3249786b2cdf2f83324febb436193d4581b6a71dea49f77bf8b19bb0"}, + {file = "lupa-2.6-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:4446396ca3830be0c106c70db4b4f622c37b2d447874c07952cafb9c57949a4a"}, + {file = "lupa-2.6-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:5826e687c89995a6eaafeae242071ba16448eec1a9ee8e17ed48551b5d1e21c2"}, + {file = "lupa-2.6-cp39-cp39-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:5871935cb36d1d22f9c04ac0db75c06751bd95edcfa0d9309f732de908e297a9"}, + {file = "lupa-2.6-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:43eb6e43ea8512d0d65b995d36dd9d77aa02598035e25b84c23a1b58700c9fb2"}, + {file = "lupa-2.6-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:559714053018d9885cc8c36a33c5b7eb9aad30fb6357719cac3ce4dc6b39157e"}, + {file = "lupa-2.6-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:57ac88a00ce59bd9d4ddcd4fca8e02564765725f5068786b011c9d1be3de20c5"}, + {file = "lupa-2.6-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:b683fbd867c2e54c44a686361b75eee7e7a790da55afdbe89f1f23b106de0274"}, + {file = "lupa-2.6-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:d2f656903a2ed2e074bf2b7d300968028dfa327a45b055be8e3b51ef0b82f9bf"}, + {file = "lupa-2.6-cp39-cp39-win32.whl", hash = "sha256:bf28f68ae231b72008523ab5ac23835ba0f76e0e99ec38b59766080a84eb596a"}, + {file = "lupa-2.6-cp39-cp39-win_amd64.whl", hash = "sha256:b4b2e9b3795a9897cf6cfcc58d08210fdc0d13ab47c9a0e13858c68932d8353c"}, + {file = "lupa-2.6.tar.gz", hash = "sha256:9a770a6e89576be3447668d7ced312cd6fd41d3c13c2462c9dc2c2ab570e45d9"}, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -1376,9 +1597,10 @@ typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} name = "mypy-extensions" version = "1.1.0" description = "Type system extensions for programs checked with the mypy type checker." -optional = false +optional = true python-versions = ">=3.8" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"dev\"" files = [ {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, @@ -1427,22 +1649,23 @@ files = [ name = "packaging" version = "25.0" description = "Core utilities for Python packages" -optional = false +optional = true python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main"] +markers = "extra == \"display\" or extra == \"test\" or extra == \"integration\" or extra == \"dev\"" files = [ {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] -markers = {main = "extra == \"display\""} [[package]] name = "pathspec" version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." -optional = false +optional = true python-versions = ">=3.8" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"dev\"" files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -1452,9 +1675,10 @@ files = [ name = "platformdirs" version = "4.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." -optional = false +optional = true python-versions = ">=3.10" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"dev\"" files = [ {file = "platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31"}, {file = "platformdirs-4.5.1.tar.gz", hash = "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda"}, @@ -1486,7 +1710,7 @@ packaging = "*" dev = ["plotly[dev-optional]"] dev-build = ["build", "jupyter", "plotly[dev-core]"] dev-core = ["pytest", "requests", "ruff (==0.11.12)"] -dev-optional = ["anywidget", "colorcet", "fiona (<=1.9.6)", "geopandas", "inflect", "numpy", "orjson", "pandas", "pdfrw", "pillow", "plotly-geo", "plotly[dev-build]", "plotly[kaleido]", "polars[timezone]", "pyarrow", "pyshp", "pytz", "scikit-image", "scipy", "shapely", "statsmodels", "vaex", "xarray"] +dev-optional = ["anywidget", "colorcet", "fiona (<=1.9.6) ; python_version <= \"3.8\"", "geopandas", "inflect", "numpy", "orjson", "pandas", "pdfrw", "pillow", "plotly-geo", "plotly[dev-build]", "plotly[kaleido]", "polars[timezone]", "pyarrow", "pyshp", "pytz", "scikit-image", "scipy", "shapely", "statsmodels", "vaex ; python_version <= \"3.9\"", "xarray"] express = ["numpy"] kaleido = ["kaleido (>=1.1.0)"] @@ -1494,9 +1718,10 @@ kaleido = ["kaleido (>=1.1.0)"] name = "pluggy" version = "1.6.0" description = "plugin and hook calling mechanisms for python" -optional = false +optional = true python-versions = ">=3.9" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"test\" or extra == \"integration\" or extra == \"dev\"" files = [ {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, @@ -1510,9 +1735,10 @@ testing = ["coverage", "pytest", "pytest-benchmark"] name = "ply" version = "3.11" description = "Python Lex & Yacc" -optional = false +optional = true python-versions = "*" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"test\"" files = [ {file = "ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce"}, {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, @@ -1692,9 +1918,10 @@ files = [ name = "psutil" version = "7.2.1" description = "Cross-platform lib for process and system monitoring." -optional = false +optional = true python-versions = ">=3.6" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"integration\"" files = [ {file = "psutil-7.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ba9f33bb525b14c3ea563b2fd521a84d2fa214ec59e3e6a2858f78d0844dd60d"}, {file = "psutil-7.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:81442dac7abfc2f4f4385ea9e12ddf5a796721c0f6133260687fec5c3780fa49"}, @@ -1743,7 +1970,7 @@ typing-inspection = ">=0.4.2" [package.extras] email = ["email-validator (>=2.0.0)"] -timezone = ["tzdata"] +timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""] [[package]] name = "pydantic-core" @@ -1908,9 +2135,10 @@ yaml = ["pyyaml (>=6.0.1)"] name = "pygments" version = "2.19.2" description = "Pygments is a syntax highlighting package written in Python." -optional = false +optional = true python-versions = ">=3.8" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"test\" or extra == \"integration\"" files = [ {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, @@ -1923,9 +2151,10 @@ windows-terminal = ["colorama (>=0.4.6)"] name = "pyproject-api" version = "1.10.0" description = "API to interact with the python pyproject.toml based projects" -optional = false +optional = true python-versions = ">=3.10" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"dev\"" files = [ {file = "pyproject_api-1.10.0-py3-none-any.whl", hash = "sha256:8757c41a79c0f4ab71b99abed52b97ecf66bd20b04fa59da43b5840bac105a09"}, {file = "pyproject_api-1.10.0.tar.gz", hash = "sha256:40c6f2d82eebdc4afee61c773ed208c04c19db4c4a60d97f8d7be3ebc0bbb330"}, @@ -1943,9 +2172,10 @@ testing = ["covdefaults (>=2.3)", "pytest (>=8.4.2)", "pytest-cov (>=7)", "pytes name = "pytest" version = "9.0.2" description = "pytest: simple powerful testing with Python" -optional = false +optional = true python-versions = ">=3.10" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"test\" or extra == \"integration\"" files = [ {file = "pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b"}, {file = "pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11"}, @@ -1967,9 +2197,10 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests name = "pytest-asyncio" version = "1.3.0" description = "Pytest support for asyncio" -optional = false +optional = true python-versions = ">=3.10" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"test\"" files = [ {file = "pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5"}, {file = "pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5"}, @@ -1984,6 +2215,22 @@ typing-extensions = {version = ">=4.12", markers = "python_version < \"3.13\""} docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1)"] testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] +[[package]] +name = "pytest-test-groups" +version = "1.2.1" +description = "A Pytest plugin for running a subset of your tests by splitting them in to equally sized groups." +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"integration\"" +files = [ + {file = "pytest_test_groups-1.2.1-py3-none-any.whl", hash = "sha256:8c7a016448f9ad347fb69a62f417f0a2358ecbf129fe44bc44ee991918a0bb73"}, + {file = "pytest_test_groups-1.2.1.tar.gz", hash = "sha256:67576b295522fc144b3a42fa1801f50ae962389e984b48bab4336686d09032f1"}, +] + +[package.dependencies] +pytest = ">=7.0.0" + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2020,9 +2267,10 @@ cli = ["click (>=5.0)"] name = "pytokens" version = "0.3.0" description = "A Fast, spec compliant Python 3.14+ tokenizer that runs on older Pythons." -optional = false +optional = true python-versions = ">=3.8" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"dev\"" files = [ {file = "pytokens-0.3.0-py3-none-any.whl", hash = "sha256:95b2b5eaf832e469d141a378872480ede3f251a5a5041b8ec6e581d3ac71bbf3"}, {file = "pytokens-0.3.0.tar.gz", hash = "sha256:2f932b14ed08de5fcf0b391ace2642f858f1394c0857202959000b68ed7a458a"}, @@ -2033,14 +2281,14 @@ dev = ["black", "build", "mypy", "pytest", "pytest-cov", "setuptools", "tox", "t [[package]] name = "rapyer" -version = "1.1.3" +version = "1.1.4" description = "Pydantic models with Redis as the backend" optional = false python-versions = "<3.14,>=3.10" groups = ["main"] files = [ - {file = "rapyer-1.1.3-py3-none-any.whl", hash = "sha256:3fe457c973f6de9bf8992141c9d268f7b1fb366e400b8cc2fa184085b77f542a"}, - {file = "rapyer-1.1.3.tar.gz", hash = "sha256:53a80ba672b2c96551f7143a8782fb693aa4f8e576f0cca806ebf233b1107441"}, + {file = "rapyer-1.1.4-py3-none-any.whl", hash = "sha256:0817c9c8a4fb0eaedfed2d1d11fc768df2ccf63177ad5f584df6201823a0fa96"}, + {file = "rapyer-1.1.4.tar.gz", hash = "sha256:6af975134be843572058a690cec47f2168d7cc91f5056377d34150d76eae71d9"}, ] [package.dependencies] @@ -2053,7 +2301,7 @@ version = "7.0.1" description = "Python client for Redis database and key-value store" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "redis-7.0.1-py3-none-any.whl", hash = "sha256:4977af3c7d67f8f0eb8b6fec0dafc9605db9343142f634041fb0235f67c0588a"}, {file = "redis-7.0.1.tar.gz", hash = "sha256:c949df947dca995dc68fdf5a7863950bf6df24f8d6022394585acc98e81624f1"}, @@ -2072,9 +2320,10 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (>=20.0.1)", "requests (>=2.31.0)" name = "requests" version = "2.32.5" description = "Python HTTP for Humans." -optional = false +optional = true python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main"] +markers = "extra == \"display\" or extra == \"integration\"" files = [ {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, @@ -2117,13 +2366,13 @@ files = [ ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] -core = ["importlib_metadata (>=6)", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "six" @@ -2142,9 +2391,10 @@ files = [ name = "sortedcontainers" version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" -optional = false +optional = true python-versions = "*" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"test\"" files = [ {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, @@ -2171,10 +2421,10 @@ test = ["pytest", "tornado (>=4.5)", "typeguard"] name = "tomli" version = "2.3.0" description = "A lil' TOML parser" -optional = false +optional = true python-versions = ">=3.8" -groups = ["dev"] -markers = "python_version < \"3.11\"" +groups = ["main"] +markers = "(extra == \"test\" or extra == \"integration\" or extra == \"coverage\" or extra == \"dev\") and python_version == \"3.10\" or extra == \"coverage\" and python_full_version <= \"3.11.0a6\"" files = [ {file = "tomli-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45"}, {file = "tomli-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba"}, @@ -2224,9 +2474,10 @@ files = [ name = "tox" version = "4.32.0" description = "tox is a generic virtualenv management and test command line tool" -optional = false +optional = true python-versions = ">=3.10" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"dev\"" files = [ {file = "tox-4.32.0-py3-none-any.whl", hash = "sha256:451e81dc02ba8d1ed20efd52ee409641ae4b5d5830e008af10fe8823ef1bd551"}, {file = "tox-4.32.0.tar.gz", hash = "sha256:1ad476b5f4d3679455b89a992849ffc3367560bbc7e9495ee8a3963542e7c8ff"}, @@ -2251,12 +2502,11 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, ] -markers = {dev = "python_version < \"3.13\""} [[package]] name = "typing-inspection" @@ -2277,27 +2527,29 @@ typing-extensions = ">=4.12.0" name = "urllib3" version = "2.6.2" description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false +optional = true python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main"] +markers = "extra == \"hatchet\" or extra == \"display\" or extra == \"integration\"" files = [ {file = "urllib3-2.6.2-py3-none-any.whl", hash = "sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd"}, {file = "urllib3-2.6.2.tar.gz", hash = "sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797"}, ] [package.extras] -brotli = ["brotli (>=1.2.0)", "brotlicffi (>=1.2.0.0)"] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "virtualenv" version = "20.35.4" description = "Virtual Python Environment builder" -optional = false +optional = true python-versions = ">=3.8" -groups = ["dev"] +groups = ["main"] +markers = "extra == \"dev\"" files = [ {file = "virtualenv-20.35.4-py3-none-any.whl", hash = "sha256:c21c9cede36c9753eeade68ba7d523529f228a403463376cf821eaae2b650f1b"}, {file = "virtualenv-20.35.4.tar.gz", hash = "sha256:643d3914d73d3eeb0c552cbb12d7e82adf0e504dbf86a3182f8771a153a1971c"}, @@ -2311,7 +2563,7 @@ typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\"" [package.extras] docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] -test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\" or platform_python_implementation == \"GraalVM\" or platform_python_implementation == \"CPython\" and sys_platform == \"win32\" and python_version >= \"3.13\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""] [[package]] name = "werkzeug" @@ -2492,7 +2744,7 @@ files = [ ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] @@ -2500,10 +2752,14 @@ test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_it type = ["pytest-mypy"] [extras] +coverage = ["coverage"] +dev = ["black", "tox"] display = ["dash", "dash-bootstrap-components", "dash-cytoscape"] hatchet = ["hatchet-sdk"] +integration = ["psutil", "pytest-test-groups", "requests"] +test = ["dynaconf", "fakeredis", "pytest", "pytest-asyncio"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "b27b900bf9b35d0331d8f742b2a8890f11d68ea3ca27b3011a16caa46070e551" +content-hash = "691819053e32a8228df1838fac913a4809ea12c6f84a25b41d48ae89cdcebdf5" diff --git a/pyproject.toml b/pyproject.toml index fb8740e5..fdcd5d69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,17 +42,35 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "rapyer>=1.1.3,<1.2.0" + "rapyer>=1.1.4,<1.2.0" ] [project.optional-dependencies] hatchet = [ - "hatchet-sdk (>=1.20.0,<1.22.0)" + "hatchet-sdk>=1.20.0,<1.22.0" ] display = [ - "dash[async] (>=3.3.0,<4.0.0)", - "dash-cytoscape (>=1.0.2,<2.0.0)", - "dash-bootstrap-components (>=2.0.4,<3.0.0)" + "dash[async]>=3.3.0,<4.0.0", + "dash-cytoscape>=1.0.2,<2.0.0", + "dash-bootstrap-components>=2.0.4,<3.0.0" +] +test = [ + "pytest>=9.0.2", + "pytest-asyncio>=1.2.0", + "fakeredis[json,lua]>=2.32.1", + "dynaconf>=3.2.12,<4.0.0", +] +integration = [ + "pytest-test-groups>=1.2.1", + "psutil>=7.1.3", + "requests>=2.32.5", +] +coverage = [ + "coverage[toml]>=7.0.0", +] +dev = [ + "black>=25.9.0,<26.0.0", + "tox>=4.32.0", ] [project.urls] @@ -66,16 +84,6 @@ Repository = "https://github.com/imaginary-cherry/mageflow" [tool.poetry] packages = [{include = "mageflow"}] -[tool.poetry.group.dev.dependencies] -black = ">=25.9.0,<26.0.0" -pytest-asyncio = "^1.2.0" -psutil = "^7.1.3" -requests = "^2.32.5" -fakeredis = {version = "^2.32.1", extras = ["json"]} -dynaconf = ">=3.2.12,<4.0.0" -pytest = "^9.0.2" -tox = "^4.32.0" - [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 38d56942..ce367faa 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -9,11 +9,13 @@ @pytest_asyncio.fixture(scope="session", loop_scope="session") -def redis_client(): +async def redis_client(): redis_client = redis.asyncio.from_url(settings.redis.url, decode_responses=True) yield redis_client + await redis_client.aclose() + @pytest_asyncio.fixture(scope="function", loop_scope="session") async def real_redis(redis_client): @@ -23,4 +25,3 @@ async def real_redis(redis_client): delete_keys = [key for key in all_keys if key not in current_keys] if delete_keys: await redis_client.delete(*delete_keys) - await redis_client.aclose() diff --git a/tests/integration/hatchet/assertions.py b/tests/integration/hatchet/assertions.py index db3e38ab..52dcbb42 100644 --- a/tests/integration/hatchet/assertions.py +++ b/tests/integration/hatchet/assertions.py @@ -3,7 +3,6 @@ from hatchet_sdk import Hatchet from hatchet_sdk.clients.rest import V1TaskStatus, V1TaskSummary - from mageflow.chain.model import ChainTaskSignature from mageflow.signature.consts import TASK_ID_PARAM_NAME, MAGEFLOW_TASK_INITIALS from mageflow.signature.model import TaskSignature @@ -223,6 +222,7 @@ def assert_swarm_task_done( tasks: list[TaskSignature], allow_fails: bool = True, check_callbacks: bool = True, + return_field_name: str = "task_result", ): task_map = {task.key: task for task in tasks} batch_map = {batch_item.key: batch_item for batch_item in batch_items} @@ -257,7 +257,7 @@ def assert_swarm_task_done( callback_wf = assert_signature_done( runs, callback_sign, check_called_once=True, **task.kwargs ) - for result in callback_wf.input["input"]["task_result"]: + for result in callback_wf.input["input"][return_field_name]: assert ( result in expected_output ), f"{result} not found in {expected_output} for callback {callback_wf.workflow_name}" diff --git a/tests/integration/hatchet/chain/test__chain.py b/tests/integration/hatchet/chain/test__chain.py index e9062edb..3117f3ac 100644 --- a/tests/integration/hatchet/chain/test__chain.py +++ b/tests/integration/hatchet/chain/test__chain.py @@ -1,8 +1,7 @@ import asyncio -import pytest - import mageflow +import pytest from mageflow.signature.model import TaskSignature from tests.integration.hatchet.assertions import ( assert_redis_is_clean, diff --git a/tests/integration/hatchet/chain/test_edge_cases.py b/tests/integration/hatchet/chain/test_edge_cases.py index d2941ef1..a9bdb2a8 100644 --- a/tests/integration/hatchet/chain/test_edge_cases.py +++ b/tests/integration/hatchet/chain/test_edge_cases.py @@ -1,8 +1,7 @@ import asyncio -import pytest - import mageflow +import pytest from tests.integration.hatchet.assertions import ( get_runs, assert_signature_done, diff --git a/tests/integration/hatchet/conftest.py b/tests/integration/hatchet/conftest.py index 5e3c3e89..7118df8d 100644 --- a/tests/integration/hatchet/conftest.py +++ b/tests/integration/hatchet/conftest.py @@ -12,19 +12,19 @@ from threading import Thread from typing import Generator, Callable, AsyncGenerator +import mageflow import psutil import pytest import pytest_asyncio import requests from hatchet_sdk import Hatchet from hatchet_sdk.clients.admin import TriggerWorkflowOptions -from redis.asyncio.client import Redis - -import mageflow from mageflow import Mageflow from mageflow.client import HatchetMageflow +from mageflow.signature.model import TaskSignature from mageflow.startup import mageflow_config, init_mageflow from mageflow.task.model import HatchetTaskModel +from redis.asyncio.client import Redis from tests.integration.hatchet.worker import ( config_obj, task1, @@ -280,3 +280,13 @@ async def sign_fail_task(): async def sign_chain_callback(): signature = await mageflow.sign(chain_callback) return signature + + +def convert_signature_mapping_to_list( + tasks: dict[str, TaskSignature], +) -> list[TaskSignature]: + final_tasks = [] + for sign_key, sign in tasks.items(): + sign.key = sign_key + final_tasks.append(sign) + return final_tasks diff --git a/tests/integration/hatchet/models.py b/tests/integration/hatchet/models.py index 33bce156..7835c781 100644 --- a/tests/integration/hatchet/models.py +++ b/tests/integration/hatchet/models.py @@ -1,8 +1,10 @@ from typing import Any, Annotated -from pydantic import BaseModel, Field - +from mageflow.chain.model import ChainTaskSignature from mageflow.models.message import ReturnValue +from mageflow.signature.model import TaskSignature +from mageflow.swarm.model import BatchItemTaskSignature, SwarmTaskSignature +from pydantic import BaseModel, Field class ContextMessage(BaseModel): @@ -31,3 +33,10 @@ class CommandMessageWithResult(ContextMessage): class SleepTaskMessage(ContextMessage): sleep_time: int = 2 result: Any = None + + +class SavedSignaturesResults(BaseModel): + signatures: dict[str, TaskSignature] = Field(default_factory=dict) + batch_items: dict[str, BatchItemTaskSignature] = Field(default_factory=dict) + swarms: dict[str, SwarmTaskSignature] = Field(default_factory=dict) + chains: dict[str, ChainTaskSignature] = Field(default_factory=dict) diff --git a/tests/integration/hatchet/root/__init__.py b/tests/integration/hatchet/root/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/hatchet/root/test__root.py b/tests/integration/hatchet/root/test__root.py new file mode 100644 index 00000000..39f1626c --- /dev/null +++ b/tests/integration/hatchet/root/test__root.py @@ -0,0 +1,183 @@ +import asyncio + +import mageflow +import pytest +from mageflow.callbacks import HatchetResult +from rapyer.types.base import REDIS_DUMP_FLAG_NAME +from tests.integration.hatchet.assertions import ( + assert_redis_is_clean, + get_runs, + assert_signature_done, + assert_signature_failed, + map_wf_by_id, + assert_signature_not_called, + assert_swarm_task_done, + assert_chain_done, +) +from tests.integration.hatchet.conftest import ( + HatchetInitData, + convert_signature_mapping_to_list, +) +from tests.integration.hatchet.models import ContextMessage, SavedSignaturesResults +from tests.integration.hatchet.worker import ( + root_with_chain_and_swarm, + root_with_failing_chains, + simple_root_task, + task1_callback, + error_callback, +) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_simple_root_task__sanity( + hatchet_client_init: HatchetInitData, + test_ctx, + ctx_metadata, + trigger_options, +): + # Arrange + redis_client, hatchet = ( + hatchet_client_init.redis_client, + hatchet_client_init.hatchet, + ) + message = ContextMessage(base_data=test_ctx) + + root_callback = await mageflow.sign(task1_callback) + error_sign = await mageflow.sign(error_callback) + root_signature = await mageflow.sign( + simple_root_task, + success_callbacks=[root_callback], + error_callbacks=[error_sign], + ) + + # Act + await root_signature.aio_run_no_wait(message, options=trigger_options) + + # Assert + await asyncio.sleep(11) + runs = await get_runs(hatchet, ctx_metadata) + + assert_signature_done(runs, root_callback) + assert_signature_not_called(runs, error_sign) + + await assert_redis_is_clean(redis_client) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_root_task_with_chain_and_swarm__callback_called_after_both_done__sanity( + hatchet_client_init: HatchetInitData, + test_ctx, + ctx_metadata, + trigger_options, +): + # Arrange + redis_client, hatchet = ( + hatchet_client_init.redis_client, + hatchet_client_init.hatchet, + ) + message = ContextMessage(base_data=test_ctx) + + root_callback = await mageflow.sign(task1_callback) + root_signature = await mageflow.sign( + root_with_chain_and_swarm, success_callbacks=[root_callback] + ) + + # Act + root_res: HatchetResult = await root_signature.aio_run( + message, options=trigger_options + ) + signatures = root_res["root_with_chain_and_swarm"]["hatchet_results"] + saved_signs = SavedSignaturesResults.model_validate( + signatures, context={REDIS_DUMP_FLAG_NAME: True} + ) + + # Take swarm and chain created by tasks + swarms, chains, signatures, batch_items = ( + saved_signs.swarms, + saved_signs.chains, + saved_signs.signatures, + saved_signs.batch_items, + ) + signatures = convert_signature_mapping_to_list(signatures) + chains = convert_signature_mapping_to_list(chains) + swarms = convert_signature_mapping_to_list(swarms) + batch_items = convert_signature_mapping_to_list(batch_items) + + # Assert + await asyncio.sleep(30) + runs = await get_runs(hatchet, ctx_metadata) + wf_map = map_wf_by_id(runs) + + # check callback was executed + assert_signature_done(runs, root_callback) + + # Check callback was executed last + latest_start_time = max([wf.started_at for wf in runs]) + assert wf_map[root_callback.key].started_at >= latest_start_time + + # Check inner tasks were called and done, also check input are good + assert len(swarms) == 2 + assert len(chains) == 1 + for swarm in swarms: + if swarm.task_name.startswith("root-swarm"): + continue + assert_swarm_task_done(runs, swarm, batch_items, signatures) + + assert_chain_done(runs, chains[0], signatures) + + await assert_redis_is_clean(redis_client) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_root_task_with_failing_chains__error_callback_called__failure( + hatchet_client_init: HatchetInitData, + test_ctx, + ctx_metadata, + trigger_options, +): + # Arrange + redis_client, hatchet = ( + hatchet_client_init.redis_client, + hatchet_client_init.hatchet, + ) + message = ContextMessage(base_data=test_ctx) + base_data2 = {"new_data": "23123"} + + success_callback = await mageflow.sign(task1_callback) + error_callback_sign = await mageflow.sign(error_callback) + error_callback_sign2 = await mageflow.sign(error_callback, base_data=base_data2) + root_signature = await mageflow.sign( + root_with_failing_chains, + success_callbacks=[success_callback], + error_callbacks=[error_callback_sign, error_callback_sign2], + ) + + # Act + root_res: HatchetResult = await root_signature.aio_run( + message, options=trigger_options + ) + signatures = root_res["root_with_failing_chains"]["hatchet_results"] + saved_signs = SavedSignaturesResults.model_validate( + signatures, context={REDIS_DUMP_FLAG_NAME: True} + ) + signatures_list = convert_signature_mapping_to_list(saved_signs.signatures) + + # Assert + await asyncio.sleep(25) + runs = await get_runs(hatchet, ctx_metadata) + wf_map = map_wf_by_id(runs, also_not_done=True) + + # Check error callback was called + assert_signature_done(runs, error_callback_sign) + assert_signature_done(runs, error_callback_sign2, base_data=base_data2) + + # Check success callback was NOT called + assert_signature_not_called(runs, success_callback) + + # Check that error callback started after the fail_task + error_callback_run = wf_map[error_callback_sign.key] + latest_start_time = max([wf.started_at for wf in runs]) + assert error_callback_run.started_at >= latest_start_time + + # Check redis is clean + await assert_redis_is_clean(redis_client) diff --git a/tests/integration/hatchet/signature/test__signature.py b/tests/integration/hatchet/signature/test__signature.py index 77610dfa..fb2dfa84 100644 --- a/tests/integration/hatchet/signature/test__signature.py +++ b/tests/integration/hatchet/signature/test__signature.py @@ -1,8 +1,7 @@ import asyncio -import pytest - import mageflow +import pytest from tests.integration.hatchet.assertions import ( assert_task_done, assert_redis_is_clean, diff --git a/tests/integration/hatchet/signature/test_edge_case.py b/tests/integration/hatchet/signature/test_edge_case.py index 6a015199..3512c6eb 100644 --- a/tests/integration/hatchet/signature/test_edge_case.py +++ b/tests/integration/hatchet/signature/test_edge_case.py @@ -1,9 +1,8 @@ import asyncio from datetime import datetime -import pytest - import mageflow +import pytest from tests.integration.hatchet.assertions import ( get_runs, assert_signature_done, diff --git a/tests/integration/hatchet/signature/test_stop_resume.py b/tests/integration/hatchet/signature/test_stop_resume.py index 4b39ee87..13750a0d 100644 --- a/tests/integration/hatchet/signature/test_stop_resume.py +++ b/tests/integration/hatchet/signature/test_stop_resume.py @@ -1,8 +1,7 @@ import asyncio -import pytest - import mageflow +import pytest from mageflow.signature.model import TaskSignature from tests.integration.hatchet.assertions import ( assert_redis_is_clean, diff --git a/tests/integration/hatchet/swarm/test__swarm.py b/tests/integration/hatchet/swarm/test__swarm.py index 63626516..43ca1662 100644 --- a/tests/integration/hatchet/swarm/test__swarm.py +++ b/tests/integration/hatchet/swarm/test__swarm.py @@ -1,10 +1,8 @@ import asyncio +import mageflow import pytest -from hatchet_sdk.clients.rest import V1TaskStatus from hatchet_sdk.runnables.types import EmptyModel - -import mageflow from mageflow.signature.model import TaskSignature from mageflow.swarm.model import SwarmConfig, BatchItemTaskSignature from tests.integration.hatchet.assertions import ( @@ -14,7 +12,6 @@ assert_signature_done, map_wf_by_id, assert_overlaps_leq_k_workflows, - is_wf_done, find_sub_calls_by_signature, ) from tests.integration.hatchet.conftest import HatchetInitData diff --git a/tests/integration/hatchet/swarm/test_edge_cases.py b/tests/integration/hatchet/swarm/test_edge_cases.py index e54829d5..048d3a0b 100644 --- a/tests/integration/hatchet/swarm/test_edge_cases.py +++ b/tests/integration/hatchet/swarm/test_edge_cases.py @@ -1,8 +1,7 @@ import asyncio -import pytest - import mageflow +import pytest from mageflow.signature.model import TaskSignature from mageflow.swarm.model import BatchItemTaskSignature, SwarmConfig from tests.integration.hatchet.assertions import get_runs, assert_swarm_task_done diff --git a/tests/integration/hatchet/swarm/test_stop_resume.py b/tests/integration/hatchet/swarm/test_stop_resume.py index 60602eef..4c87b3d2 100644 --- a/tests/integration/hatchet/swarm/test_stop_resume.py +++ b/tests/integration/hatchet/swarm/test_stop_resume.py @@ -1,9 +1,8 @@ import asyncio from datetime import datetime -import pytest - import mageflow +import pytest from mageflow.signature.model import TaskSignature from mageflow.swarm.model import SwarmConfig, BatchItemTaskSignature from tests.integration.hatchet.assertions import ( diff --git a/tests/integration/hatchet/test_complex_scerios.py b/tests/integration/hatchet/test_complex_scerios.py index 8e76134b..c364e203 100644 --- a/tests/integration/hatchet/test_complex_scerios.py +++ b/tests/integration/hatchet/test_complex_scerios.py @@ -1,7 +1,6 @@ import asyncio import pytest - from mageflow.chain.model import ChainTaskSignature from mageflow.signature.model import TaskSignature from mageflow.swarm.model import BatchItemTaskSignature, SwarmConfig diff --git a/tests/integration/hatchet/test_task_models.py b/tests/integration/hatchet/test_task_models.py index e6bcd96d..339f8021 100644 --- a/tests/integration/hatchet/test_task_models.py +++ b/tests/integration/hatchet/test_task_models.py @@ -1,7 +1,6 @@ import asyncio -import pytest -import pytest_asyncio +import pytest from mageflow.task.model import HatchetTaskModel from tests.integration.hatchet.conftest import HatchetInitData diff --git a/tests/integration/hatchet/worker.py b/tests/integration/hatchet/worker.py index 5365226f..b338872c 100644 --- a/tests/integration/hatchet/worker.py +++ b/tests/integration/hatchet/worker.py @@ -3,7 +3,10 @@ import os from datetime import datetime +from mageflow.chain.model import ChainTaskSignature from mageflow.signature.model import TaskSignature +from mageflow.swarm.model import SwarmTaskSignature, BatchItemTaskSignature +from rapyer.types.base import REDIS_DUMP_FLAG_NAME # Start coverage if COVERAGE_PROCESS_START is set if os.environ.get("COVERAGE_PROCESS_START"): @@ -29,6 +32,7 @@ MessageWithResult, CommandMessageWithResult, SleepTaskMessage, + SavedSignaturesResults, ) settings = Dynaconf( @@ -146,6 +150,58 @@ async def cancel_retry(msg): raise NonRetryableException("Test exception") +@hatchet.task(name="simple_root_task", input_validator=ContextMessage) +@hatchet.root_task(max_concurrency=4, stop_after_n_failures=2) +async def simple_root_task(msg: ContextMessage): + return msg + + +@hatchet.task(name="root_with_chain_and_swarm", input_validator=ContextMessage) +@hatchet.root_task(stop_after_n_failures=2) +async def root_with_chain_and_swarm(msg: ContextMessage): + chain_sig = await hatchet.chain([task1, task2]) + callback_sign = await hatchet.sign(task1_callback) + swarm_sig = await hatchet.swarm( + tasks=[task2, task3], success_callbacks=[callback_sign], is_swarm_closed=True + ) + swarms, chains, signatures, batch_items = await asyncio.gather( + SwarmTaskSignature.afind(), + ChainTaskSignature.afind(), + TaskSignature.afind(), + BatchItemTaskSignature.afind(), + ) + await chain_sig.aio_run_no_wait(msg) + await swarm_sig.aio_run_no_wait(msg) + results = SavedSignaturesResults( + signatures={sign.key: sign for sign in signatures}, + batch_items={item.key: item for item in batch_items}, + chains={chain.key: chain for chain in chains}, + swarms={swarm.key: swarm for swarm in swarms}, + ) + return results.model_dump(mode="json", context={REDIS_DUMP_FLAG_NAME: True}) + + +@hatchet.task(name="root_with_failing_chains", input_validator=ContextMessage) +@hatchet.root_task(stop_after_n_failures=1) +async def root_with_failing_chains(msg: ContextMessage): + success_chain_sig = await hatchet.chain([task1, task2]) + fail_chain_sig = await hatchet.chain([task1, fail_task, task3]) + + chains, signatures = await asyncio.gather( + ChainTaskSignature.afind(), + TaskSignature.afind(), + ) + + await success_chain_sig.aio_run_no_wait(msg) + await fail_chain_sig.aio_run_no_wait(msg) + + results = SavedSignaturesResults( + signatures={sign.key: sign for sign in signatures}, + chains={chain.key: chain for chain in chains}, + ) + return results.model_dump(mode="json", context={REDIS_DUMP_FLAG_NAME: True}) + + workflows = [ task1, task2, @@ -163,6 +219,9 @@ async def cancel_retry(msg): retry_once, retry_to_failure, cancel_retry, + simple_root_task, + root_with_chain_and_swarm, + root_with_failing_chains, ] diff --git a/tests/integration/test_redis_ttl.py b/tests/integration/test_redis_ttl.py index d8c91860..1a690a0c 100644 --- a/tests/integration/test_redis_ttl.py +++ b/tests/integration/test_redis_ttl.py @@ -1,10 +1,8 @@ import pytest -import pytest_asyncio - -from mageflow.signature.creator import sign from mageflow.chain.creator import chain -from mageflow.swarm.creator import swarm +from mageflow.signature.creator import sign from mageflow.startup import mageflow_config, init_mageflow +from mageflow.swarm.creator import swarm @pytest.fixture @@ -27,7 +25,7 @@ async def test_redis_ttl_verification_sanity(real_redis, dummy_task, entity_type redis_client = real_redis mageflow_config.redis_client = redis_client await init_mageflow() - + expected_ttl = 24 * 60 * 60 # 1 day in seconds ttl_tolerance = 100 # seconds tolerance for test execution time @@ -51,5 +49,9 @@ async def test_redis_ttl_verification_sanity(real_redis, dummy_task, entity_type ttl_result = await redis_client.ttl(key) # Assert - assert ttl_result > expected_ttl - ttl_tolerance, f"TTL for {entity_type} is too low: {ttl_result}" - assert ttl_result <= expected_ttl, f"TTL for {entity_type} is too high: {ttl_result}" \ No newline at end of file + assert ( + ttl_result > expected_ttl - ttl_tolerance + ), f"TTL for {entity_type} is too low: {ttl_result}" + assert ( + ttl_result <= expected_ttl + ), f"TTL for {entity_type} is too high: {ttl_result}" diff --git a/tests/unit/chain/test_chain.py b/tests/unit/chain/test_chain.py index 4da3468f..5e37107a 100644 --- a/tests/unit/chain/test_chain.py +++ b/tests/unit/chain/test_chain.py @@ -1,9 +1,8 @@ -import pytest - import mageflow +import pytest from mageflow.chain.consts import ON_CHAIN_ERROR, ON_CHAIN_END -from mageflow.signature.model import TaskSignature from mageflow.chain.model import ChainTaskSignature +from mageflow.signature.model import TaskSignature from tests.integration.hatchet.models import ContextMessage diff --git a/tests/unit/chain/test_status_changes.py b/tests/unit/chain/test_status_changes.py index 0777f1f9..c6764730 100644 --- a/tests/unit/chain/test_status_changes.py +++ b/tests/unit/chain/test_status_changes.py @@ -1,8 +1,7 @@ -import pytest - import mageflow -from mageflow.signature.model import TaskSignature +import pytest from mageflow.chain.model import ChainTaskSignature +from mageflow.signature.model import TaskSignature from mageflow.signature.status import SignatureStatus, TaskStatus from tests.integration.hatchet.models import ContextMessage from tests.unit.assertions import assert_redis_keys_do_not_contain_sub_task_ids diff --git a/tests/unit/complex/test_chain_and_swarm.py b/tests/unit/complex/test_chain_and_swarm.py index 8284fa3d..27a0770a 100644 --- a/tests/unit/complex/test_chain_and_swarm.py +++ b/tests/unit/complex/test_chain_and_swarm.py @@ -1,6 +1,5 @@ -import pytest - import mageflow +import pytest from mageflow.chain.consts import ON_CHAIN_END, ON_CHAIN_ERROR from mageflow.signature.model import TaskSignature from mageflow.swarm.model import BatchItemTaskSignature, SwarmTaskSignature diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 771b1089..0cb4a9b1 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -10,7 +10,7 @@ from mageflow.chain.model import ChainTaskSignature from mageflow.signature.model import TaskSignature from mageflow.startup import update_register_signature_models, mageflow_config -from tests.integration.hatchet.worker import ContextMessage +from tests.integration.hatchet.models import ContextMessage pytest.register_assert_rewrite("tests.assertions") diff --git a/tests/unit/swarm/conftest.py b/tests/unit/swarm/conftest.py index 5b509824..395315fe 100644 --- a/tests/unit/swarm/conftest.py +++ b/tests/unit/swarm/conftest.py @@ -3,12 +3,9 @@ import pytest import pytest_asyncio - -import mageflow -from mageflow.chain.model import ChainTaskSignature from mageflow.signature.model import TaskSignature from mageflow.swarm.model import SwarmTaskSignature -from tests.integration.hatchet.worker import ContextMessage +from tests.integration.hatchet.models import ContextMessage @dataclass diff --git a/tests/unit/swarm/test_call_item.py b/tests/unit/swarm/test_call_item.py index b4c03f81..4ad0ddf4 100644 --- a/tests/unit/swarm/test_call_item.py +++ b/tests/unit/swarm/test_call_item.py @@ -1,8 +1,8 @@ -import pytest -import pytest_asyncio from unittest.mock import AsyncMock, patch import mageflow +import pytest +import pytest_asyncio from mageflow.signature.model import TaskSignature from mageflow.swarm.model import SwarmTaskSignature from tests.integration.hatchet.models import ContextMessage diff --git a/tests/unit/swarm/test_config.py b/tests/unit/swarm/test_config.py index f538e850..3375165d 100644 --- a/tests/unit/swarm/test_config.py +++ b/tests/unit/swarm/test_config.py @@ -1,6 +1,5 @@ -import pytest - import mageflow +import pytest from mageflow.errors import TooManyTasksError from mageflow.signature.model import TaskSignature from mageflow.swarm.model import SwarmConfig diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 378e1ae3..6bdb5b88 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1,13 +1,12 @@ import asyncio -from hatchet_sdk import Context from datetime import timedelta from unittest.mock import MagicMock, AsyncMock import pytest +from hatchet_sdk import Context from hatchet_sdk import Hatchet -from redis import Redis - from mageflow.client import HatchetMageflow +from redis import Redis from tests.integration.hatchet.models import ContextMessage diff --git a/tests/unit/test_sign.py b/tests/unit/test_sign.py index 2f80ee53..f02fa22b 100644 --- a/tests/unit/test_sign.py +++ b/tests/unit/test_sign.py @@ -1,12 +1,11 @@ from datetime import datetime from typing import Optional, Any -import pytest -from pydantic import BaseModel - import mageflow +import pytest from mageflow.signature.model import TaskSignature from mageflow.signature.types import TaskIdentifierType +from pydantic import BaseModel class SignParamOptions(BaseModel): diff --git a/tox.ini b/tox.ini index 35c03f3c..19940cbd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,70 +1,53 @@ [tox] -envlist = - py{311,312,313}-hatchet{120,121} +envlist = + py{311,312,313}-unit + py{311,312,313}-hatchet{120,121}-integration coverage isolated_build = true -[testenv] -deps = - pytest - pytest-asyncio - psutil - requests - fakeredis[json] - dynaconf>=3.2.12,<4.0.0 - rapyer>=1.1.0,<1.2.0 +[testenv:py{311,312,313}-unit] +deps = + -e .[test,hatchet,display] +setenv = + PYTHONHASHSEED = 0 +commands = + pytest tests/unit {posargs} + +[testenv:py{311,312,313}-hatchet{120,121}-integration] +deps = + -e .[test,integration,display] hatchet120: hatchet-sdk>=1.20.0,<1.21.0 hatchet121: hatchet-sdk>=1.21.0,<1.22.0 - dash[async]>=3.3.0,<4.0.0 - dash-cytoscape>=1.0.2,<2.0.0 - dash-bootstrap-components>=2.0.4,<3.0.0 -allowlist_externals = - docker - sleep setenv = PYTHONHASHSEED = 0 SERVER_AUTH_COOKIE_INSECURE = t -passenv = +passenv = DYNACONF_* HATCHET_* -commands_pre = - docker compose -f tests/integration/hatchet/docker-compose.hatchet.yml up -d - sleep 10 -commands = - pytest tests -commands_post = - docker compose -f tests/integration/hatchet/docker-compose.hatchet.yml down + PYTEST_SPLITS + PYTEST_GROUP +commands = + pytest tests/integration --test-group-count {env:PYTEST_SPLITS:1} --test-group {env:PYTEST_GROUP:1} {posargs} [testenv:coverage] -deps = - coverage[toml] - pytest - pytest-asyncio - psutil - requests - fakeredis[json] - dynaconf>=3.2.12,<4.0.0 - rapyer>=1.1.0,<1.2.0 - hatchet-sdk>=1.20.0,<1.22.0 - dash[async]>=3.3.0,<4.0.0 - dash-cytoscape>=1.0.2,<2.0.0 - dash-bootstrap-components>=2.0.4,<3.0.0 -allowlist_externals = +deps = + -e .[test,integration,hatchet,display,coverage] +allowlist_externals = docker sleep setenv = PYTHONHASHSEED = 0 SERVER_AUTH_COOKIE_INSECURE = t -passenv = +passenv = DYNACONF_* HATCHET_* -commands_pre = +commands_pre = docker compose -f tests/integration/hatchet/docker-compose.hatchet.yml up -d sleep 10 -commands = +commands = coverage run --source=mageflow -m pytest --junitxml=junit.xml -o junit_family=legacy coverage combine coverage report --show-missing coverage xml -commands_post = - docker compose -f tests/integration/hatchet/docker-compose.hatchet.yml down \ No newline at end of file +commands_post = + docker compose -f tests/integration/hatchet/docker-compose.hatchet.yml down