diff --git a/.env.example b/.env.example index a65f7824a..23ae03d4d 100644 --- a/.env.example +++ b/.env.example @@ -58,6 +58,42 @@ POWERCONTEXT_SERVER_DATABASE_KIND=sqlite # POWERCONTEXT_SERVER_DATABASE_KIND=seekdb # POWERCONTEXT_SERVER_DATABASE_PATH=/absolute/path/to/seekdb +# Deployment and durable work ------------------------------------------------- +# Local installs run every role in one process. Distributed mode requires OceanBase and one role per process. +POWERCONTEXT_SERVER_DEPLOYMENT_MODE=single_node +POWERCONTEXT_SERVER_DEPLOYMENT_ROLE=all +POWERCONTEXT_SERVER_DEPLOYMENT_ID=local +POWERCONTEXT_SERVER_DEPLOYMENT_BEHAVIOR_REVISION=default + +# Scheduler and member leases use database time. Renewal must stay within the validated lease ratios. +POWERCONTEXT_SERVER_COORDINATION_SCHEDULER_LEASE_SECONDS=30 +POWERCONTEXT_SERVER_COORDINATION_SCHEDULER_RENEW_SECONDS=10 +POWERCONTEXT_SERVER_COORDINATION_SCAN_PAGE_SIZE=100 +POWERCONTEXT_SERVER_COORDINATION_MEMBER_TTL_SECONDS=30 +POWERCONTEXT_SERVER_COORDINATION_MEMBER_HEARTBEAT_SECONDS=10 +POWERCONTEXT_SERVER_COORDINATION_EMIT_PAYLOAD_VERSION=1 + +POWERCONTEXT_SERVER_WORKER_CONCURRENCY=4 +POWERCONTEXT_SERVER_WORKER_LEASE_SECONDS=120 +POWERCONTEXT_SERVER_WORKER_HEARTBEAT_SECONDS=30 +POWERCONTEXT_SERVER_WORKER_SHUTDOWN_GRACE_SECONDS=90 +POWERCONTEXT_SERVER_WORKER_MAX_ATTEMPTS=5 +POWERCONTEXT_SERVER_WORKER_RETRY_BASE_SECONDS=2 +POWERCONTEXT_SERVER_WORKER_RETRY_MAX_SECONDS=300 +POWERCONTEXT_SERVER_WORKER_POLL_SECONDS=1 + +POWERCONTEXT_SERVER_OPERATIONS_DEFAULT_WAIT_SECONDS=10 +POWERCONTEXT_SERVER_OPERATIONS_MAXIMUM_WAIT_SECONDS=30 +POWERCONTEXT_SERVER_OPERATIONS_POLL_SECONDS=0.2 +POWERCONTEXT_SERVER_OPERATIONS_RETENTION_DAYS=30 +POWERCONTEXT_SERVER_OPERATIONS_CLEANUP_BATCH_SIZE=500 +POWERCONTEXT_SERVER_OPERATIONS_CLEANUP_INTERVAL_SECONDS=3600 + +# Shared OceanBase fixed-window limiting is optional and disabled by default. +POWERCONTEXT_SERVER_RATE_LIMIT_ENABLED=false +POWERCONTEXT_SERVER_RATE_LIMIT_REQUESTS=120 +POWERCONTEXT_SERVER_RATE_LIMIT_WINDOW_SECONDS=60 + # Runtime --------------------------------------------------------------------- POWERCONTEXT_SERVER_RUNTIME_SOURCE_WINDOW_LIMIT=100 POWERCONTEXT_SERVER_RUNTIME_MEMORY_EXTRACTION_PROFILE=coding diff --git a/.licenserc.yaml b/.licenserc.yaml index 47dd00a1f..e0a103b50 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -59,6 +59,8 @@ header: - 'src/powercontext/http/_generated/**' - '.gitattributes' - '**/*.service.in' + # license-eye cannot determine the comment style of Alembic's Mako template. + - 'src/powercontext/builtin/persistence/migrations/script.py.mako' - '**/*.jsonl' # license-eye cannot determine the comment style of uv requirements files. - 'e2e/bub/source-overrides.txt' diff --git a/docker/README.md b/docker/README.md index bc8f4bbdd..ce838e996 100644 --- a/docker/README.md +++ b/docker/README.md @@ -11,7 +11,7 @@ docker build \ . ``` -Run the Server with persistent SQLite and scheduler data: +Run the Server with persistent SQLite application and Work Ledger data: ```bash docker run --rm \ @@ -34,6 +34,11 @@ requests share one refresh. Checks use the Runtime's credentials and never expos another database or inference provider with the same `POWERCONTEXT_SERVER_*` environment variables used by a regular Server installation. +For a role-separated OceanBase topology, use `docker/compose.distributed.yaml` as a deployment template. It runs a +one-shot migrator, two API replicas, two fenced Scheduler replicas, and two lease-based Worker replicas. Replace the +example database credentials, authentication token, and provider configuration before starting it; only Workers need +model credentials. + ## Network exposure PowerContext refuses to start an unauthenticated Server on a non-loopback address unless diff --git a/docker/compose.distributed.yaml b/docker/compose.distributed.yaml new file mode 100644 index 000000000..2ec3652b3 --- /dev/null +++ b/docker/compose.distributed.yaml @@ -0,0 +1,112 @@ +name: powercontext-distributed + +x-runtime: &runtime + image: ${POWERCONTEXT_IMAGE:-powercontext-server:local} + restart: unless-stopped + environment: &shared-environment + POWERCONTEXT_SERVER_DATABASE_KIND: oceanbase + POWERCONTEXT_SERVER_DATABASE_URL: ${POWERCONTEXT_SERVER_DATABASE_URL:?set an OceanBase URL} + POWERCONTEXT_SERVER_DEPLOYMENT_MODE: distributed + POWERCONTEXT_SERVER_DEPLOYMENT_BEHAVIOR_REVISION: ${POWERCONTEXT_BEHAVIOR_REVISION:-distributed-v1} + POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS: ${POWERCONTEXT_MEMORY_SCHEDULE_SECONDS:-60} + POWERCONTEXT_SERVER_RUNTIME_EXPERIENCE_SCHEDULE_SECONDS: ${POWERCONTEXT_EXPERIENCE_SCHEDULE_SECONDS:-60} + +services: + migrate: + <<: *runtime + restart: "no" + command: ["server", "migrate"] + environment: + <<: *shared-environment + # The migration command does not start a Runtime or register this role; + # a valid distributed role is still required while parsing ServerSettings. + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: api + POWERCONTEXT_SERVER_DEPLOYMENT_ID: migrator + + api-a: + <<: *runtime + command: ["server", "run"] + environment: + <<: *shared-environment + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: api + POWERCONTEXT_SERVER_DEPLOYMENT_ID: api-a + POWERCONTEXT_SERVER_AUTH_ENABLED: "true" + POWERCONTEXT_SERVER_AUTH_TOKEN: ${POWERCONTEXT_SERVER_AUTH_TOKEN:?set an API bearer token} + POWERCONTEXT_SERVER_RATE_LIMIT_ENABLED: "true" + ports: ["127.0.0.1:8001:8000"] + depends_on: + migrate: + condition: service_completed_successfully + + api-b: + <<: *runtime + command: ["server", "run"] + environment: + <<: *shared-environment + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: api + POWERCONTEXT_SERVER_DEPLOYMENT_ID: api-b + POWERCONTEXT_SERVER_AUTH_ENABLED: "true" + POWERCONTEXT_SERVER_AUTH_TOKEN: ${POWERCONTEXT_SERVER_AUTH_TOKEN:?set an API bearer token} + POWERCONTEXT_SERVER_RATE_LIMIT_ENABLED: "true" + ports: ["127.0.0.1:8002:8000"] + depends_on: + migrate: + condition: service_completed_successfully + + scheduler-a: + <<: *runtime + command: ["server", "run"] + environment: + <<: *shared-environment + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: scheduler + POWERCONTEXT_SERVER_DEPLOYMENT_ID: scheduler-a + POWERCONTEXT_SERVER_DASHBOARD_ENABLED: "false" + POWERCONTEXT_SERVER_MCP_ENABLED: "false" + depends_on: + migrate: + condition: service_completed_successfully + + scheduler-b: + <<: *runtime + command: ["server", "run"] + environment: + <<: *shared-environment + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: scheduler + POWERCONTEXT_SERVER_DEPLOYMENT_ID: scheduler-b + POWERCONTEXT_SERVER_DASHBOARD_ENABLED: "false" + POWERCONTEXT_SERVER_MCP_ENABLED: "false" + depends_on: + migrate: + condition: service_completed_successfully + + worker-a: + <<: *runtime + command: ["server", "run"] + environment: + <<: *shared-environment + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: worker + POWERCONTEXT_SERVER_DEPLOYMENT_ID: worker-a + POWERCONTEXT_SERVER_DASHBOARD_ENABLED: "false" + POWERCONTEXT_SERVER_MCP_ENABLED: "false" + POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL: ${POWERCONTEXT_GENERATION_MODEL:?set a generation model} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + depends_on: + migrate: + condition: service_completed_successfully + + worker-b: + <<: *runtime + command: ["server", "run"] + environment: + <<: *shared-environment + POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: worker + POWERCONTEXT_SERVER_DEPLOYMENT_ID: worker-b + POWERCONTEXT_SERVER_DASHBOARD_ENABLED: "false" + POWERCONTEXT_SERVER_MCP_ENABLED: "false" + POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL: ${POWERCONTEXT_GENERATION_MODEL:?set a generation model} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + depends_on: + migrate: + condition: service_completed_successfully diff --git a/docs/en/development/core-protocol.md b/docs/en/development/core-protocol.md index 1348e07c5..209fa5f63 100644 --- a/docs/en/development/core-protocol.md +++ b/docs/en/development/core-protocol.md @@ -97,8 +97,9 @@ conflicts and persistence; the family service is responsible for its domain beha A `Trigger` is a policy over a signal and prior state. It returns a `PolicyTransition` containing the next state and zero or more actions. A Trigger should not open storage, schedule itself, or perform the action it selects. -APScheduler belongs to the Builtin runtime lifecycle. It decides when to evaluate a policy. The Trigger decides what -the observed signal means. +The durable Scheduler belongs to the Builtin runtime lifecycle. It decides when to evaluate a policy and records work +in the database ledger. The Trigger decides what the observed signal means; a fenced Worker performs the selected +action. ## Ownership boundaries @@ -109,7 +110,7 @@ the observed signal means. | Environment-backed process configuration | `powercontext.client.settings`, `powercontext.server.settings` | | HTTP lifecycle and optional MCP transport | `powercontext.server` | | Provider-specific generation and embedding | Inference integration | -| Database and scheduler resource lifetime | Application entry point or Builtin runtime instance | +| Database, Scheduler, and Worker resource lifetime | Application entry point or Builtin runtime instance | Core models use Pydantic `BaseModel`. Add a validator when the value has a real domain constraint. Do not add wrapper properties for stored fields, custom JSON value hierarchies, or a second definition object when the model or protocol diff --git a/docs/en/development/remote-access-implementation.md b/docs/en/development/remote-access-implementation.md index e847fadc9..1a9c3b544 100644 --- a/docs/en/development/remote-access-implementation.md +++ b/docs/en/development/remote-access-implementation.md @@ -76,8 +76,9 @@ and OceanBase uses HNSW for `vector` and `hybrid` searches. Inference configuration is documented in [Configure Pydantic AI inference](pydantic-ai-inference.md). Set `POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS` to process pending Source windows on a persisted interval. Scheduled -jobs use the SQLite sidecar at `POWERCONTEXT_HOME/scheduler.db`. Scheduling works with either application database and -requires a configured generation pipeline. +discovery and execution use the primary database Work Ledger. In `single_node/all` mode the embedded Scheduler and +Worker share that ledger; distributed OceanBase deployments run them as separate roles. A generation pipeline is +required by the Worker. ## HTTP surface @@ -90,6 +91,7 @@ The source contract is `openapi/powercontext.yaml`. Generated Pydantic models an | Capabilities | source types, Artifact families, extraction, search modes | | Sources | capture durable content evidence | | Memory | flush pending Sources, remember explicit entries, search | +| Operations | inspect, list, cancel, and retry durable background work | | Memory entries | list, get, revise, retire | | History | list Memory changes | diff --git a/docs/en/docs/how-to/deploy-server.md b/docs/en/docs/how-to/deploy-server.md index a1e7ee16e..4fb7b3cb4 100644 --- a/docs/en/docs/how-to/deploy-server.md +++ b/docs/en/docs/how-to/deploy-server.md @@ -70,8 +70,9 @@ export POWERCONTEXT_HOME=/srv/powercontext powercontext server run ``` -The process must be able to create and update this directory. The default SQLite database and scheduler state are -stored below it. Supply the same environment variables whenever your service manager restarts the process. +The process must be able to create and update this directory. The default SQLite database also stores durable +Scheduler, Worker lease, and Operation state. Supply the same environment variables whenever your service manager +restarts the process. PowerContext does not search for a `.env` file automatically. Export the variables, configure them in the service manager or container platform, or pass one explicit file: @@ -110,7 +111,40 @@ docker run --rm \ ``` The image listens on `0.0.0.0:8000` inside the container, so the host-side address in `--publish` is important. The -named volume persists the SQLite database and scheduler state after the container stops. +named volume persists the SQLite database and durable work state after the container stops. + +## Run distributed roles + +Distributed mode requires OceanBase and a schema migration before any role starts. The repository includes +`docker/compose.distributed.yaml` as a topology example with two APIs, two Schedulers, and two Workers. One Scheduler is +leader and the other remains ready to take over; both APIs and both Workers are active. + +Export secrets and deployment choices without writing them into Compose: + +```bash +export POWERCONTEXT_SERVER_DATABASE_URL="$OCEANBASE_URL" +export POWERCONTEXT_SERVER_AUTH_TOKEN="$POWERCONTEXT_DEPLOYMENT_TOKEN" +export POWERCONTEXT_GENERATION_MODEL="openai:gpt-4.1-mini" +export OPENAI_API_KEY +docker compose --file docker/compose.distributed.yaml up migrate +docker compose --file docker/compose.distributed.yaml up -d api-a api-b scheduler-a scheduler-b worker-a worker-b +``` + +The two API examples listen on host ports 8001 and 8002. Put a TLS-terminating load balancer in front of them and use +round-robin routing without session affinity. Distributed MCP is stateless. The example intentionally supplies model +credentials only to Workers and no public port to Scheduler or Worker roles. A production deployment should also use +separate least-privilege database users instead of the shared demonstration URL. + +Every replica in one rollout must use the same behavior revision. Upgrade in this order: + +1. run `powercontext server migrate` with a dedicated DDL account; +2. replace Workers and wait for readiness; +3. replace Schedulers and confirm a leader can scan; +4. replace APIs. + +Keep `POWERCONTEXT_SERVER_COORDINATION_EMIT_PAYLOAD_VERSION` on the older supported value until old Workers have drained. +Rollback in the reverse order and never start a distributed role against a schema that has not reached the packaged +revision. ## Enable authentication diff --git a/docs/en/docs/how-to/full-capability-runtime.md b/docs/en/docs/how-to/full-capability-runtime.md index b4adb2ee9..07dab692f 100644 --- a/docs/en/docs/how-to/full-capability-runtime.md +++ b/docs/en/docs/how-to/full-capability-runtime.md @@ -126,13 +126,23 @@ evidence. Scheduled processing handles new Sources within the configured interva ## Data and restart behavior -With no database override, SQLite stores `powercontext.db` and `scheduler.db` under the user data directory: +The generated configuration leaves the database unset, so the Server stores data in the user data directory instead of +a project-local file. With `POWERCONTEXT_HOME` unset, SQLite keeps `powercontext.db`, including durable scheduling and +operation state, under: -- Linux: `$XDG_DATA_HOME/powercontext`, or `~/.local/share/powercontext`; -- macOS: `~/Library/Application Support/powercontext`. +- macOS: `~/Library/Application Support/powercontext/` +- Linux: `~/.local/share/powercontext/` -Press `Ctrl+C` to stop the Server. Restart it with the same `.env` and data directory. The default Scope and its opaque -ID remain stable because they are persisted in the database. +Set `POWERCONTEXT_HOME` before starting the Server to relocate all of this. Changing the database URL later points the +Server at a different (possibly empty) database; keep the previous value if you need the old data. + +## Stop and restart + +Press `Ctrl+C` in the Server terminal to stop it. Data persists in SQLite across restarts. To resume, load the same +`.env` and run `powercontext server run --env-file .env` again; pending Sources are processed on the next Scheduler run +or flush. The default Scope and its opaque ID also remain stable because they are persisted in the same database. + +## Quick troubleshooting | Symptom | Action | | --- | --- | diff --git a/docs/en/docs/how-to/troubleshoot.md b/docs/en/docs/how-to/troubleshoot.md index 1fb9bb7b4..d82780dd8 100644 --- a/docs/en/docs/how-to/troubleshoot.md +++ b/docs/en/docs/how-to/troubleshoot.md @@ -171,8 +171,8 @@ so the previous database remains available for recovery: 3. Export the PowerContext table data with OceanBase `obdumper` in CSV or SQL data mode **without `--ddl`**. Keep the export and the original database unchanged until the migration is verified. Supply credentials through your approved secret-handling process rather than placing them in logs or documentation. -4. Create a new empty OceanBase MySQL-mode database and point `POWERCONTEXT_SERVER_DATABASE_URL` at it. Start the - current PowerContext version once to create tables with `utf8mb4_bin`, then stop it before restoring data. +4. Create a new empty OceanBase MySQL-mode database, point `POWERCONTEXT_SERVER_DATABASE_URL` at it, and run + `powercontext server migrate`. Do not start an API, Scheduler, or Worker process before restoring the data. 5. Import only the exported row data into the existing new tables with OceanBase `obloader`, again **without `--ddl`**. Keep foreign-key enforcement enabled and run these three layers separately. The examples use CSV; if you exported SQL data, replace `--csv` with `--sql` in all three commands. Fill in `` through @@ -185,12 +185,15 @@ so the previous database remains available for recovery: exported `pc_scopes` data. If the source predates the three Skill lifecycle tables (`pc_skill_packages`, `pc_agent_skill_targets`, and `pc_skill_publications`), remove the absent tables from Layer 1. + Do not import `pc_scheduler_leases`, `pc_scheduler_scans`, `pc_runtime_members`, or `pc_rate_limit_windows`. + They contain transient coordination state. The migration command owns its migration lease row, and the runtime + recreates the remaining rows after startup. Layer 1 contains parents and tables without foreign keys: ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_work_lanes,pc_work_items,pc_work_keys,pc_work_attempts' \ -f ``` @@ -213,7 +216,8 @@ so the previous database remains available for recovery: Wait for each invocation to complete successfully before starting the next. Treat any OBLoader error, bad record, or conflict record as a failed restore. Order within a layer is irrelevant because no table in a layer references another table in the same layer. -6. If the installation has additional PowerContext-managed tables not listed above, these tested layers do not +6. Except for the four transient tables intentionally excluded above, if the installation has additional + PowerContext-managed tables not listed above, these tested layers do not classify them. Inspect their foreign-key constraints and place each table after all of its parents; do not add them to an all-table invocation. 7. Compare source and target row counts for every restored table, inspect the identity-column collations, and test diff --git a/docs/en/docs/reference/configuration.md b/docs/en/docs/reference/configuration.md index 77472d7d3..3a2b7f8ac 100644 --- a/docs/en/docs/reference/configuration.md +++ b/docs/en/docs/reference/configuration.md @@ -32,8 +32,8 @@ Without an override, the default is: - macOS: `~/Library/Application Support/powercontext`; - Windows: `%LOCALAPPDATA%\\powercontext`. -The default SQLite database is `powercontext.db` in this directory. Scheduled processing uses `scheduler.db` in the -same directory. +The default SQLite database is `powercontext.db` in this directory. Scheduled processing, leases, and operation state +use the same database; the former `scheduler.db` sidecar is no longer part of execution. ## Server @@ -61,6 +61,33 @@ Server settings use the `POWERCONTEXT_SERVER_` prefix. | `POWERCONTEXT_SERVER_DATABASE_KIND` | `sqlite` | Storage backend: `sqlite`, `seekdb`, or `oceanbase` | | `POWERCONTEXT_SERVER_DATABASE_URL` | user data SQLite file | SQLAlchemy async URL for SQLite or OceanBase; do not set for seekDB | | `POWERCONTEXT_SERVER_DATABASE_PATH` | user data `seekdb` directory | Embedded seekDB path; used only when `DATABASE_KIND=seekdb` | +| `POWERCONTEXT_SERVER_DEPLOYMENT_MODE` | `single_node` | `single_node` or `distributed` process topology | +| `POWERCONTEXT_SERVER_DEPLOYMENT_ROLE` | `all` | `all`, `api`, `scheduler`, or `worker`; distributed mode forbids `all` | +| `POWERCONTEXT_SERVER_DEPLOYMENT_ID` | `local` | Non-secret operator instance label; boot ownership remains unique | +| `POWERCONTEXT_SERVER_DEPLOYMENT_BEHAVIOR_REVISION` | `default` | Non-secret rollout compatibility revision shared by all replicas | +| `POWERCONTEXT_SERVER_COORDINATION_SCHEDULER_LEASE_SECONDS` | `30` | Scheduler leader lease duration using database time | +| `POWERCONTEXT_SERVER_COORDINATION_SCHEDULER_RENEW_SECONDS` | `10` | Scheduler renewal interval; at most one third of the lease | +| `POWERCONTEXT_SERVER_COORDINATION_SCAN_PAGE_SIZE` | `100` | Maximum scopes inspected in one discoverer page | +| `POWERCONTEXT_SERVER_COORDINATION_MEMBER_TTL_SECONDS` | `30` | Runtime member advertisement lifetime | +| `POWERCONTEXT_SERVER_COORDINATION_MEMBER_HEARTBEAT_SECONDS` | `10` | Runtime member heartbeat interval | +| `POWERCONTEXT_SERVER_COORDINATION_EMIT_PAYLOAD_VERSION` | `1` | Work payload version emitted during a rolling deployment | +| `POWERCONTEXT_SERVER_WORKER_CONCURRENCY` | `4` | Maximum attempts executed concurrently by one Worker | +| `POWERCONTEXT_SERVER_WORKER_LEASE_SECONDS` | `120` | Worker claim lease duration | +| `POWERCONTEXT_SERVER_WORKER_HEARTBEAT_SECONDS` | `30` | Claim heartbeat interval; less than one third of the lease | +| `POWERCONTEXT_SERVER_WORKER_SHUTDOWN_GRACE_SECONDS` | `90` | Maximum graceful drain time; less than the lease | +| `POWERCONTEXT_SERVER_WORKER_MAX_ATTEMPTS` | `5` | Automatic attempt budget before operator recovery is required | +| `POWERCONTEXT_SERVER_WORKER_RETRY_BASE_SECONDS` | `2` | Full-jitter exponential retry base | +| `POWERCONTEXT_SERVER_WORKER_RETRY_MAX_SECONDS` | `300` | Full-jitter retry ceiling | +| `POWERCONTEXT_SERVER_WORKER_POLL_SECONDS` | `1` | Idle claim polling interval | +| `POWERCONTEXT_SERVER_OPERATIONS_DEFAULT_WAIT_SECONDS` | `10` | Default HTTP Memory flush wait | +| `POWERCONTEXT_SERVER_OPERATIONS_MAXIMUM_WAIT_SECONDS` | `30` | Maximum accepted `Prefer: wait=N` value | +| `POWERCONTEXT_SERVER_OPERATIONS_POLL_SECONDS` | `0.2` | Local operation completion polling interval | +| `POWERCONTEXT_SERVER_OPERATIONS_RETENTION_DAYS` | `30` | Successful and cancelled operation history retention | +| `POWERCONTEXT_SERVER_OPERATIONS_CLEANUP_BATCH_SIZE` | `500` | Maximum records removed by one maintenance attempt | +| `POWERCONTEXT_SERVER_OPERATIONS_CLEANUP_INTERVAL_SECONDS` | `3600` | Durable maintenance discovery interval | +| `POWERCONTEXT_SERVER_RATE_LIMIT_ENABLED` | `false` | Enable shared database fixed-window limiting | +| `POWERCONTEXT_SERVER_RATE_LIMIT_REQUESTS` | `120` | Requests allowed for one principal and policy window | +| `POWERCONTEXT_SERVER_RATE_LIMIT_WINDOW_SECONDS` | `60` | Shared rate-limit window duration | | `POWERCONTEXT_SERVER_RUNTIME_SCOPE_CACHE_SIZE` | `128` | Inactive scope compositions retained by the Runtime; in-flight scopes are never evicted | | `POWERCONTEXT_SERVER_RUNTIME_SOURCE_WINDOW_LIMIT` | `100` | Maximum Sources processed in one activation | | `POWERCONTEXT_SERVER_RUNTIME_MEMORY_EXTRACTION_PROFILE` | `coding` | Memory selection policy: `coding` or `conversation` | @@ -197,14 +224,26 @@ and stores the canonical package bytes, then creates a pending Candidate with th generation model, semantic generation returns a capability error before persisting a Candidate; Review, package inspection and download, exact import, usage recording, and external Skill scan/list/resolve continue to work. -Experience incubation is a separate APScheduler job with its own persisted Source cursor. Each activation inspects a +Experience incubation is a separate durable Work handler with its own persisted Source cursor. Each activation inspects a fixed window of at most 32 Sources and exposes only Content Sources whose metadata contains `"kind": "task-outcome"` to the model. It creates pending Experience Candidates in the Review Inbox; it does not approve them, place them in PreparedContext, create a managed Skill, export it to an Agent target, or execute anything. -The Memory and Experience jobs share the APScheduler sidecar under `POWERCONTEXT_HOME`, but keep independent job -identities and business cursors. Unsetting one interval removes only that job. +Memory and Experience share the database Work Ledger but keep independent lanes, logical keys, and business cursors. +Unsetting one interval disables only its discoverer; already queued operations remain inspectable. See [Create and review an Experience](../how-to/create-and-review-experience.md) for setup and verification steps. +### Distributed roles and migrations + +Distributed mode requires OceanBase. Run `powercontext server migrate --env-file ...` with a DDL-capable account before +starting any role. Role processes never create or alter schema. Start or roll forward in this order: migrate, Workers, +Schedulers, then APIs. Use a new `POWERCONTEXT_SERVER_DEPLOYMENT_BEHAVIOR_REVISION` when a rollout changes non-secret +behavior that must not mix across replicas. + +An API replica can remain ready enough to accept durable work while Scheduler or Worker members are absent; readiness +is `degraded` and names the missing role. Scheduler and Worker roles expose health and metrics only. Distributed MCP is +stateless and needs no load-balancer affinity. Host-local External Skill targets are rejected because replicas could +otherwise return different results. + ### Agent Skill targets The zero-configuration flow uses the Codex and Claude Code project folders under the workspace. Provide a JSON override diff --git a/docs/en/docs/reference/http-api.md b/docs/en/docs/reference/http-api.md index 494b18a50..6734310ae 100644 --- a/docs/en/docs/reference/http-api.md +++ b/docs/en/docs/reference/http-api.md @@ -92,6 +92,27 @@ curl --fail \ "$POWERCONTEXT_URL/v1/memory/search" ``` +## Flush through a durable operation + +A Memory flush can finish during the request or continue on any Worker. Ask for an immediate handle when the caller +does not want to wait: + +```bash +curl --fail-with-body --include \ + --request POST \ + --header 'Content-Type: application/json' \ + --header 'Prefer: respond-async' \ + --header "$POWERCONTEXT_AUTH_HEADER" \ + --data '{"scope_id":"project:example"}' \ + "$POWERCONTEXT_URL/v1/memory/flush" +``` + +HTTP `200` contains the completed `FlushMemoryResponse`. HTTP `202` contains an operation ID and includes relative +`Location` and `Retry-After` headers. `Prefer: wait=N` waits up to 30 seconds. Poll +`GET /v1/operations/{operation_id}`; use the returned `state_version` as `expected_version` when cancelling a queued or +running operation, or retrying a blocked failed operation. A failed logical window returns `409 operation_blocked` on +another flush until an operator retries or cancels it. + ## Find an operation | Area | Main paths | Purpose | @@ -106,6 +127,7 @@ curl --fail \ | External Skills | `/v1/external-skills/*` | Scan configured targets and resolve or import packages | | Handoff Reports | `/v1/handoff-reports/*` | Generate a read-only report for a Scope selection | | Statistics | `/v1/stats` | Read scoped usage statistics | +| Durable operations | `/v1/operations/*` | Inspect, list, cancel, or recover Memory and Experience background work | The OpenAPI contract defines the complete path list, schemas, limits, and status codes. The higher-level workflow and Python examples are in [Interfaces](interfaces.md). @@ -129,6 +151,7 @@ Common statuses are: | Status | Meaning | | --- | --- | | `401` | The Server requires a valid bearer token | +| `429` | The shared request window is exhausted; wait for `Retry-After` | | `404` | The requested immutable value does not exist | | `409` | The request conflicts with current immutable state or an expected version | | `413` | A selected Handoff Report exceeds its output limit | diff --git a/docs/en/docs/reference/interfaces.md b/docs/en/docs/reference/interfaces.md index 0c76ae98f..d1cc5bc61 100644 --- a/docs/en/docs/reference/interfaces.md +++ b/docs/en/docs/reference/interfaces.md @@ -214,12 +214,12 @@ For the relationship between evidence, Candidate versions, approved Revisions, r ## Scheduled Experience incubation An integration can capture a completed task as a Content Source with metadata `"kind": "task-outcome"`. When the -Experience schedule is configured, APScheduler scans bounded Source windows and asks the configured schema-bound -pipeline for reusable situation, action, outcome, and lesson proposals. Each proposal cites exact Sources and enters -the Review Inbox as a pending Experience Candidate. +Experience schedule is configured, the durable Scheduler scans bounded Source windows and enqueues versioned work. +A fenced Worker asks the configured schema-bound pipeline for reusable situation, action, outcome, and lesson +proposals. Each proposal cites exact Sources and enters the Review Inbox as a pending Experience Candidate. -Experience incubation has its own persisted Source cursor, independent from Memory extraction. Candidate writes and -cursor advancement commit together; a generation or write failure leaves the window available for retry. Ordinary +Experience incubation has its own persisted Source cursor, independent from Memory extraction. Candidate writes, +cursor advancement, and Work success commit together; a generation or write failure leaves the window available for retry. Ordinary prompt Sources are not Task Outcomes and are ignored by this job. Scheduling stops at the review boundary. It never approves an Experience, includes pending content in diff --git a/docs/en/rfcs/0011_remote_access_architecture.md b/docs/en/rfcs/0011_remote_access_architecture.md index 54e239653..c158027d8 100644 --- a/docs/en/rfcs/0011_remote_access_architecture.md +++ b/docs/en/rfcs/0011_remote_access_architecture.md @@ -2,6 +2,10 @@ - Start Date: 2026-07-16 - RFC PR: [oceanbase/powercontext#11](https://github.com/oceanbase/powercontext/pull/11) +> **Deployment boundary:** [RFC 1430](1430_distributed_server_workers.md) defines the accepted durable Operation, +> stateless multi-replica API, role separation, and distributed coordination contract. It supersedes this RFC where +> this document leaves execution and deployment semantics open. + # Summary This RFC proposes a remote access boundary for PowerContext. A Server exposes application services through an diff --git a/docs/en/rfcs/0019_local_source_memory_runtime.md b/docs/en/rfcs/0019_local_source_memory_runtime.md index c67167772..f18c49b96 100644 --- a/docs/en/rfcs/0019_local_source_memory_runtime.md +++ b/docs/en/rfcs/0019_local_source_memory_runtime.md @@ -5,6 +5,10 @@ > **Note:** PowerContext now uses bundled sqlite-vec for SQLite vector search. Statements about Vec1 in this RFC no > longer apply and remain only as a record of the original design. +> **Execution update:** [RFC 1430](1430_distributed_server_workers.md) replaces the APScheduler sidecar and +> single-process execution assumptions with one database-backed Work Ledger for local and distributed modes. This RFC +> remains authoritative for Source window, cursor, Memory, and domain commit semantics. + # Summary This RFC proposes backend-neutral Runtime storage contracts and a built-in SQLite profile. The Runtime uses the diff --git a/docs/en/rfcs/0020_runtime_backed_memory_remote_access.md b/docs/en/rfcs/0020_runtime_backed_memory_remote_access.md index 251b9c34d..ef5d59803 100644 --- a/docs/en/rfcs/0020_runtime_backed_memory_remote_access.md +++ b/docs/en/rfcs/0020_runtime_backed_memory_remote_access.md @@ -6,6 +6,10 @@ > **Note:** PowerContext now uses bundled sqlite-vec for SQLite vector search. Statements about Vec1 in this RFC no > longer apply and remain only as a record of the original design. +> **Execution update:** [RFC 1430](1430_distributed_server_workers.md) supersedes this RFC's single-process scheduler, +> synchronous-only flush, and non-durable Operation assumptions. This RFC continues to define the HTTP mapping and +> generated-contract boundary. + # Summary This RFC defines the first concrete remote API built from the architecture in RFC 0011 and the local Runtime in diff --git a/docs/en/rfcs/1430_distributed_server_workers.md b/docs/en/rfcs/1430_distributed_server_workers.md new file mode 100644 index 000000000..b498dd5ee --- /dev/null +++ b/docs/en/rfcs/1430_distributed_server_workers.md @@ -0,0 +1,225 @@ +- Proposal Name: `distributed_server_workers` +- Start Date: 2026-09-03 +- Tracking Issue: [oceanbase/powercontext#1430](https://github.com/oceanbase/powercontext/issues/1430) +- Related RFCs: [RFC 0011](0011_remote_access_architecture.md), [RFC 0019](0019_local_source_memory_runtime.md), + [RFC 0020](0020_runtime_backed_memory_remote_access.md), and [RFC 0046](0046_observability_foundations.md) + +# Summary + +PowerContext uses one database-backed Work Ledger for both local and distributed background execution. An API replica +can enqueue and inspect work without process affinity. Schedulers elect one fenced leader while retaining durable +keyset scan positions. Workers claim only available capacity, renew database-time leases, and safely recover abandoned +attempts. Model calls remain at-least-once, while one logical Source window can commit at most one database result. + +OceanBase is the only coordination backend in distributed mode. The same ledger also replaces the APScheduler SQLite +sidecar in `single_node/all`, so there is one execution protocol to understand and test. Redis, Kafka, a public queue +SPI, arbitrary user code, DAG scheduling, priorities, multi-region consensus, and connector-specific checkpoints are +outside this RFC. + +# Decisions + +The design deliberately minimizes sources of truth: + +- OceanBase owns work state, leases, cursors, and domain commits in distributed mode. +- Execution is at-least-once. Provider work may repeat after a crash, but fencing and the domain cursor prevent an old + attempt from committing. +- Memory activation and Experience incubation use the ledger first. Future handlers keep their domain checkpoints in + their own modules. +- The default remains `single_node/all`; distributed processes use exactly one of `api`, `scheduler`, or `worker`. +- Work payloads are versioned references and numeric window boundaries, not Source bodies, prompts, model responses, + credentials, or exception text. + +This avoids entropy from parallel database, broker, and local-sidecar protocols. The compensating complexity is made +explicit as database constraints, a small state machine, fencing tokens, startup validation, and compatibility +advertisements rather than tacit assumptions about one process. + +# Architecture + +```text +API / Scheduler + | short transaction: identify window, deduplicate, enqueue + v +OceanBase Work Ledger + | short transaction: claim lane head and issue lease fence + v +Worker ---- provider/model call outside a transaction ----> + | short transaction: validate fence, commit domain state + cursor + work + v +one logical database result +``` + +The ledger consists of: + +| Table | Responsibility | +| --- | --- | +| `pc_work_items` | Versioned work, scope, lane sequence, safe payload, state, lease, attempt budget, and safe result/error | +| `pc_work_attempts` | Append-only owner/fence/timing/outcome audit and non-sensitive trace identity | +| `pc_work_lanes` | Strict sequence and one active head for each consistency domain | +| `pc_work_keys` | Unique reservation for the current logical window | +| `pc_scheduler_leases` | Leader and migrator ownership with monotonic fences and database-time expiry | +| `pc_scheduler_scans` | Next run time and keyset continuation for each discoverer | +| `pc_runtime_members` | Live role and schema/payload/behavior compatibility advertisements | +| `pc_rate_limit_windows` | Optional shared fixed-window counters keyed by an opaque principal digest and policy | + +The logical key for a Memory window is the SHA-256 digest of its handler kind, scope, cursor name, cursor generation, +and previous cursor. The enqueue-time high watermark and `through` boundary are stored in the work payload but are not +part of that key. Manual and scheduled discovery therefore join the same unadvanced cursor window; Sources arriving +later are handled by the next window. + +Memory and Experience use independent lanes (`memory:{scope}` and `experience:{scope}` before hashing). Different lanes +may execute concurrently, while a lane advances strictly in sequence. A terminal failure retains its logical-key and +lane reservation until an operator retries or cancels it, preventing a poison window from being recreated forever. + +# State machine + +```text +queued ----> running ----> succeeded + | |----> retry_wait ----> running + | |----> failed + | `----> cancelling ----> cancelled + `-----------------------------------> cancelled + +failed -- explicit operator retry --> queued +``` + +Cancellation linearizes at the work row. If cancellation wins before the final transaction, the Worker discards the +prepared provider result and completes the attempt as cancelled. If success has already committed, cancellation returns +`409`. Cancelling one operation does not pause future discovery for its scope. + +# Lock and transaction protocol + +OceanBase 4.3.5 with Read Committed isolation is the baseline. Correctness does not depend on an unlocked read, a +single predicate update, `SKIP LOCKED`, or `GET_LOCK()`. Every path follows this order: + +```text +scheduler lease (scheduler paths only) + -> lane + -> logical key + -> work item + -> domain cursor/head +``` + +Stable lease and lane rows are explicitly locked. A missing row is inserted under a unique key and the operation is +retried after a conflict. Claiming starts with a bounded indexed candidate scan, then locks and claims one lane at a +time in a short transaction. Workers never prefetch more work than their free execution slots. + +Scheduler enqueue and scan updates revalidate the exact leader owner, fence, and database expiry. Worker heartbeats, +failures, cancellation convergence, and completion validate the exact `(work_id, owner, fence)`. All expiry decisions +use `CURRENT_TIMESTAMP(6)` from the database; application time is used only for polling and deadlines. + +Provider, connector, network, and filesystem calls never run under a database transaction. The final Worker +transaction validates an unexpired, uncancelled claim and then invokes the handler's domain commit. Memory and +Experience reuse their existing cursor/head compare-and-swap. Domain writes, cursor advancement, attempt completion, +and Work success either commit together or roll back together. + +# Scheduler and Worker behavior + +Each discoverer stores a durable keyset continuation and scans at most 100 scopes per page. Repeating a page after a +crash is safe because logical keys deduplicate it. A former leader cannot enqueue or save a continuation after another +Scheduler acquires a higher fence. + +A Worker claims only its current free slots. Its attempt lease defaults to 120 seconds and is renewed every 30 seconds. +Expired attempts are closed in the audit trail, moved through retry state, and reclaimed with a higher fence. Retryable +failures use full-jitter exponential backoff, beginning at 2 seconds and capped at 5 minutes. Five automatic generation +attempts are allowed by default. Unsupported payload versions stay visible at the lane head and make Worker readiness +`misconfigured`; they are never guessed, silently dropped, or moved aside. + +# Process roles + +Configuration adds `deployment`, `coordination`, `worker`, `operations`, and `rate_limit` groups. + +- `single_node/all` runs API, Scheduler, and Worker together using the same ledger. A database ownership lease rejects a + mistakenly started second instance. +- `distributed/api` exposes HTTP, Dashboard, MCP, authentication, shared rate limiting, enqueue, and operation queries. + It does not run discovery or background Worker handlers. +- `distributed/scheduler` exposes only health and metrics, owns no provider credentials, and performs bounded discovery. +- `distributed/worker` exposes only health and metrics and owns only the provider credentials required by its handlers. + +Distributed mode requires OceanBase and rejects `role=all`, SQLite, seekDB, and explicit host-local External Skill +targets. All roles use one image but should use separate least-privilege database accounts. DDL belongs to a migrator +account and is never performed automatically by a distributed role. + +Every process heartbeats a boot-unique member identity with build version, current schema and payload range, and a +non-sensitive `behavior_revision`. Credentials, secret URLs, authorization data, and hashes that permit offline secret +guessing are not member metadata. + +# HTTP and Client contract + +`openapi/powercontext.yaml` remains authoritative. `POST /v1/memory/flush` behaves as follows: + +- no pending Source, or completion within the wait budget: `200 FlushMemoryResponse`; +- still queued, running, or waiting to retry: `202 OperationAccepted`, a relative `Location`, and `Retry-After: 2`; +- a failed operation already blocks that logical key: `409 operation_blocked` with its operation ID. + +`Prefer: respond-async` selects an immediate handle. `Prefer: wait=N` selects a bounded wait up to 30 seconds; the +default is 10 seconds. Operation endpoints provide authorized get/list, optimistic cancel, and operator retry. Mutation +bodies carry `expected_version`; illegal or stale transitions return `409`. Public status values are `queued`, +`running`, `retry_wait`, `cancelling`, `succeeded`, `failed`, and `cancelled`. Internal maintenance work is not exposed by +the Operation API. + +`PowerContextClient.flush_memory()` preserves its synchronous-looking result by submitting and polling until its total +deadline. If that deadline expires it raises `OperationPendingError` with the operation ID. Callers that want explicit +control use `submit_memory_flush()`, `get_operation()`, `list_operations()`, `cancel_operation()`, and +`retry_operation()`. + +Operation endpoints are not projected as MCP tools. In distributed mode FastMCP uses stateless HTTP, so consecutive +requests may reach different API replicas. Server-to-client elicitation and sampling are disabled; the Workstream picker +returns a structured `needs_selection` response. The internal ASGI bridge carries the already authenticated principal, +and tool visibility is never treated as authorization. + +# Health, shutdown, retention, and observability + +Liveness means the process can respond. API readiness requires database, schema, authentication policy, membership, +and behavior compatibility. Missing Scheduler or Worker members make API readiness degraded rather than preventing +durable enqueue and reads. A healthy Scheduler standby is ready. A Worker that lacks a required provider or handler +version stops claiming and reports the precise failing check. + +On SIGTERM, API readiness is removed first. Scheduler stops discovery and conditionally releases its lease. Worker +stops claiming, continues heartbeats for in-flight work, and drains for at most 90 seconds; an ungraceful exit recovers +through lease expiry. + +Succeeded and cancelled work and attempts are retained for 30 days. A durable maintenance handler deletes no more than +500 records per batch and also removes expired rate-limit counters. Failed blocking work is retained until operator +action. Future Source retention must treat every non-terminal work window as a retention root. + +Metrics use only bounded labels such as kind, status, outcome, role, and error category. They cover queue depth and age, +claim and attempt latency, lease expiry, retries, throughput, leadership changes, and member counts. Scope, principal, +and work IDs are not metric labels. Enqueue, claim, execute, commit, and retry are separate spans; retry attempts use span +links rather than pretending to be one uninterrupted trace. + +Logs, spans, work rows, and attempt rows must never contain Source content, prompts, model output, credentials, +authorization headers, or complete secret URLs. Persistent errors use bounded category and code values. + +# Schema and rollout + +Alembic owns a forward-only schema chain. `powercontext server migrate` acquires a database lease and upgrades through +the packaged head. A new database starts at the baseline; a known complete legacy schema is validated, receives only +recognized expansions, and is stamped before the ledger migration. Unknown or partially installed schemas are rejected. +Single-node startup runs the same migration path automatically; distributed roles only validate the current revision. + +Schema evolution follows expand, mixed-version deployment, then contract. Release N+1 must read N and N+1 layouts, and +destructive removal waits until N+2. Workers support the current and previous payload during an actual mixed-version +transition. `emit_payload_version` stays on the old format until old Workers drain. + +The deployment sequence is migrate, Workers, Schedulers, then APIs. Rollback reverses that order and drains any newer +payload before an older Worker is restored. The bridge release first puts `single_node/all` on the Work Ledger; only +after the APScheduler execution path is absent may the same database be served by multiple roles. The old sidecar is an +operator backup artifact and is not deleted automatically. + +# Validation + +Acceptance requires round-robin HTTP, Dashboard API, and stateless MCP tests across two API replicas; overlapping work +on different lanes and serialization on one lane; manual/scheduled deduplication; Worker crash points before claim, +during provider execution, during final commit, and after commit; Scheduler takeover fencing; retry, cancellation, +operator recovery, retention, and privacy tests; and real multi-process OceanBase 4.3.5 tests. Golden schema/payload +fixtures cover mixed versions and upgrade/rollback. SQLite and seekDB remain supported only for single-node regression +tests. + +# Alternatives rejected + +- Redis or Kafka would add another durable truth and a cross-system commit problem before throughput requires it. +- A public Queue/Coordinator SPI would freeze multiple coordination semantics before any second implementation exists. +- `GET_LOCK()` is session-scoped and unsafe as a pooled-connection fencing primitive. +- Keeping APScheduler for local mode would preserve two subtly different execution and recovery protocols. +- Exactly-once provider execution is not achievable across external calls; the enforceable contract is at-least-once + execution with at-most-one fenced database commit. diff --git a/docs/zh/development/core-protocol.md b/docs/zh/development/core-protocol.md index 914f4e83e..934f839f9 100644 --- a/docs/zh/development/core-protocol.md +++ b/docs/zh/development/core-protocol.md @@ -93,7 +93,8 @@ service 负责领域行为。 `Trigger` 是基于 signal 和先前 state 的策略。它返回 `PolicyTransition`,其中包含下一状态和零个或多个 action。 Trigger 不应打开存储、调度自身或执行它选择的 action。 -APScheduler 属于 Builtin runtime 的生命周期,负责决定何时评估策略。Trigger 只解释当前 signal 的含义。 +持久化 Scheduler 属于 Builtin runtime 生命周期,负责决定何时评估策略并把任务写入数据库 ledger。Trigger 只解释 +当前 signal 的含义;带 fence 的 Worker 执行被选择的 action。 ## 职责边界 @@ -104,7 +105,7 @@ APScheduler 属于 Builtin runtime 的生命周期,负责决定何时评估策 | 基于环境变量的进程配置 | `powercontext.client.settings`、`powercontext.server.settings` | | HTTP 生命周期与可选 MCP transport | `powercontext.server` | | Provider 相关的生成和 embedding | 推理集成 | -| 数据库和 scheduler 资源生命周期 | 应用入口或 Builtin runtime instance | +| 数据库、Scheduler 和 Worker 资源生命周期 | 应用入口或 Builtin runtime instance | Core model 使用 Pydantic `BaseModel`。只有存在真实领域约束时才增加 validator。对于普通存储字段,不需要包装 property;模型和协议已经能表达边界时,也不需要自定义 JSON value 层级或第二套 definition 对象。 diff --git a/docs/zh/development/remote-access-implementation.md b/docs/zh/development/remote-access-implementation.md index d3f1b867e..086e5b74f 100644 --- a/docs/zh/development/remote-access-implementation.md +++ b/docs/zh/development/remote-access-implementation.md @@ -72,8 +72,8 @@ export POWERCONTEXT_SERVER_DATABASE_URL="mysql+aoceanbase://user:password@host:2 inference 配置见[配置 Pydantic AI 推理](pydantic-ai-inference.md)。 设置 `POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS` 可以按持久化 interval 处理待消费的 Source window。 -定时 job 使用 `POWERCONTEXT_HOME/scheduler.db` 作为 SQLite sidecar。调度可以配合任一 application database -使用,但必须配置 generation pipeline。 +定时发现和执行使用主数据库中的 Work Ledger。`single_node/all` 模式内嵌 Scheduler 与 Worker;分布式 +OceanBase 部署将它们作为独立角色运行。Worker 必须配置 generation pipeline。 ## HTTP 接口 @@ -86,6 +86,7 @@ inference 配置见[配置 Pydantic AI 推理](pydantic-ai-inference.md)。 | Capabilities | source type、Artifact family、extraction、search mode | | Sources | capture 持久化 content evidence | | Memory | flush 待处理 Source、remember 显式 entry、search | +| Operations | 查询、列举、取消和重试持久后台任务 | | Memory entries | list、get、revise、retire | | History | list Memory change | diff --git a/docs/zh/docs/how-to/deploy-server.md b/docs/zh/docs/how-to/deploy-server.md index 9f71cdc22..e7fe81178 100644 --- a/docs/zh/docs/how-to/deploy-server.md +++ b/docs/zh/docs/how-to/deploy-server.md @@ -68,8 +68,8 @@ export POWERCONTEXT_HOME=/srv/powercontext powercontext server run ``` -运行进程必须能创建和更新该目录。默认 SQLite 数据库和 scheduler 状态都保存在这里。服务管理器每次重启进程时都应 -提供相同的环境变量。 +运行进程必须能创建和更新该目录。默认 SQLite 数据库也保存持久 Scheduler、Worker lease 和 Operation 状态。 +服务管理器每次重启进程时都应提供相同的环境变量。 PowerContext 不会自动搜索 `.env` 文件。可以导出变量、由服务管理器或容器平台提供,或者显式传入一个文件: @@ -106,7 +106,38 @@ docker run --rm \ ``` 镜像内部监听 `0.0.0.0:8000`,所以 `--publish` 中的宿主机地址非常重要。容器停止后,named volume 仍会保留 -SQLite 数据库和 scheduler 状态。 +SQLite 数据库和持久 work 状态。 + +## 运行分布式角色 + +分布式模式要求 OceanBase,并且任何角色启动前都必须先迁移 schema。仓库中的 +`docker/compose.distributed.yaml` 提供两 API、两 Scheduler、两 Worker 的拓扑示例。一个 Scheduler 成为 leader, +另一个保持 ready 并可接管;两个 API 和两个 Worker 都会同时工作。 + +通过环境传入 secret 和部署选择,不要把它们写进 Compose: + +```bash +export POWERCONTEXT_SERVER_DATABASE_URL="$OCEANBASE_URL" +export POWERCONTEXT_SERVER_AUTH_TOKEN="$POWERCONTEXT_DEPLOYMENT_TOKEN" +export POWERCONTEXT_GENERATION_MODEL="openai:gpt-4.1-mini" +export OPENAI_API_KEY +docker compose --file docker/compose.distributed.yaml up migrate +docker compose --file docker/compose.distributed.yaml up -d api-a api-b scheduler-a scheduler-b worker-a worker-b +``` + +两个 API 示例分别监听宿主机 8001 和 8002 端口。应在它们前面配置终止 TLS 的 load balancer,并使用不带 session +affinity 的 round-robin;分布式 MCP 是 stateless。示例有意只把模型 credential 交给 Worker,也不向宿主机发布 +Scheduler 或 Worker 的端口。生产环境还应为各角色使用独立最小权限数据库账号,而不是复用示例 URL。 + +同一轮发布的所有副本必须使用相同 behavior revision。升级顺序如下: + +1. 使用专门的 DDL 账号运行 `powercontext server migrate`; +2. 替换 Worker 并等待 readiness; +3. 替换 Scheduler 并确认 leader 可以扫描; +4. 替换 API。 + +旧 Worker 排空前,`POWERCONTEXT_SERVER_COORDINATION_EMIT_PAYLOAD_VERSION` 必须保持旧的受支持值。回滚顺序相反, +并且绝不能让分布式角色连接尚未升级到 packaged revision 的 schema。 ## 启用鉴权 diff --git a/docs/zh/docs/how-to/full-capability-runtime.md b/docs/zh/docs/how-to/full-capability-runtime.md index 206e5da2b..4cc6017f6 100644 --- a/docs/zh/docs/how-to/full-capability-runtime.md +++ b/docs/zh/docs/how-to/full-capability-runtime.md @@ -120,12 +120,22 @@ Codex 启动后发送普通 prompt。插件从绑定 Scope 召回内容,并把 ## 数据与重启 -没有覆盖数据库设置时,SQLite 在用户数据目录保存 `powercontext.db` 和 `scheduler.db`: +生成的配置不指定数据库位置,因此 Server 把数据保存在用户数据目录,而不是项目内文件。在未设置 +`POWERCONTEXT_HOME` 时,SQLite 的 `powercontext.db`(包含持久调度与 operation 状态)位于: -- Linux:`$XDG_DATA_HOME/powercontext`,或 `~/.local/share/powercontext`; -- macOS:`~/Library/Application Support/powercontext`。 +- macOS:`~/Library/Application Support/powercontext/` +- Linux:`~/.local/share/powercontext/` -按 `Ctrl+C` 停止 Server。使用同一 `.env` 和数据目录重启后,默认 Scope 及其不透明 ID 保持稳定,因为它们保存在数据库中。 +如需迁移,在启动 Server 前设置 `POWERCONTEXT_HOME` 即可。之后再修改数据库 URL 会把 Server 指向另一个(可能是 +空的)数据库;需要旧数据时请保留原来的值。 + +## 停止与恢复 + +在 Server 终端按 `Ctrl+C` 停止进程。数据持久保存在 SQLite 中,重启不会丢失。恢复时重新加载同一个 `.env`,再次 +执行 `powercontext server run --env-file .env`;pending 的 Source 会在下一次调度或 flush 时继续处理。默认 Scope 及其 +不透明 ID 也保存在同一数据库中,因此重启后保持稳定。 + +## 快速排障 | 现象 | 处理方式 | | --- | --- | diff --git a/docs/zh/docs/how-to/troubleshoot.md b/docs/zh/docs/how-to/troubleshoot.md index f6b0986c0..7d12310c0 100644 --- a/docs/zh/docs/how-to/troubleshoot.md +++ b/docs/zh/docs/how-to/troubleshoot.md @@ -165,8 +165,8 @@ collation,但不会包含数据库 URL 或凭据。 3. 使用 OceanBase `obdumper` 的 CSV 或 SQL 数据模式导出 PowerContext 表数据,且**不要使用 `--ddl`**。 在迁移验证完成之前,保持导出文件和原数据库不变。请通过获批的 secret 管理流程提供凭据,不要把凭据写入日志 或文档。 -4. 新建一个空的 OceanBase MySQL-mode 数据库,将 `POWERCONTEXT_SERVER_DATABASE_URL` 指向它。启动一次当前 - PowerContext,使其创建使用 `utf8mb4_bin` 的表;恢复数据前再次停止 Server。 +4. 新建一个空的 OceanBase MySQL-mode 数据库,将 `POWERCONTEXT_SERVER_DATABASE_URL` 指向它,并执行 + `powercontext server migrate`。恢复数据前不要启动 API、Scheduler 或 Worker 进程。 5. 使用 OceanBase `obloader` 只把导出的数据导入已经存在的新表,同样**不要使用 `--ddl`**。保持外键检查开启, 并分别运行下面三层命令。示例使用 CSV;如果导出的是 SQL 数据,请把三个命令中的 `--csv` 全部替换为 `--sql`。通过获批的 secret 管理流程填写 ``,并让 `` 指向第 4 步创建的 @@ -178,12 +178,14 @@ collation,但不会包含数据库 URL 或凭据。 后代记录之前。 源数据库早于三张 Skill 生命周期表(`pc_skill_packages`、`pc_agent_skill_targets` 和 `pc_skill_publications`)时,应从第 1 层删除缺失的表。 + 不要导入 `pc_scheduler_leases`、`pc_scheduler_scans`、`pc_runtime_members` 或 `pc_rate_limit_windows`。 + 这些表保存临时协调状态;迁移命令拥有自己的 migration lease 记录,其余记录会在 runtime 启动后重建。 第 1 层包含父表和无外键的表: ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_work_lanes,pc_work_items,pc_work_keys,pc_work_attempts' \ -f ``` @@ -205,7 +207,8 @@ collation,但不会包含数据库 URL 或凭据。 每个命令成功完成后才能开始下一层。OBLoader 出现任何错误、bad record 或 conflict record 时,都应判定恢复 失败。同一层内的表互不引用,因此层内顺序无关。 -6. 如果安装中还存在上面未列出的 PowerContext 管理表,则这些已测试的层并未对它们分类。检查其外键约束,将每张 +6. 除上面明确排除的四张临时表外,如果安装中还存在未列出的 PowerContext 管理表,则这些已测试的层并未对其分类。 + 检查其外键约束,将每张 表放在其所有父表之后;不要把它们加入全表导入命令。 7. 逐表比较源数据库和目标数据库的记录数,检查 identity column collation,并测试仅大小写或重音不同的 identity。所有检查通过后才能恢复正常流量。在整个回滚窗口内,保留源数据库、已验证的备份和导出文件。 diff --git a/docs/zh/docs/reference/configuration.md b/docs/zh/docs/reference/configuration.md index 6c3601422..fe929de56 100644 --- a/docs/zh/docs/reference/configuration.md +++ b/docs/zh/docs/reference/configuration.md @@ -28,8 +28,8 @@ export POWERCONTEXT_HOME=/srv/powercontext - macOS:`~/Library/Application Support/powercontext`; - Windows:`%LOCALAPPDATA%\\powercontext`。 -默认 SQLite 数据库是该目录下的 `powercontext.db`。启用定时处理时,调度状态保存在同一目录的 -`scheduler.db`。 +默认 SQLite 数据库是该目录下的 `powercontext.db`。定时处理、租约和 operation 状态使用同一个数据库;执行路径 +不再使用旧的 `scheduler.db` sidecar。 ## Server @@ -57,6 +57,33 @@ Server 配置使用 `POWERCONTEXT_SERVER_` 前缀。 | `POWERCONTEXT_SERVER_DATABASE_KIND` | `sqlite` | 存储后端:`sqlite`、`seekdb` 或 `oceanbase` | | `POWERCONTEXT_SERVER_DATABASE_URL` | 用户数据目录下的 SQLite 文件 | SQLite 或 OceanBase 的 SQLAlchemy 异步 URL;seekDB 不设置 | | `POWERCONTEXT_SERVER_DATABASE_PATH` | 用户数据目录下的 `seekdb` 目录 | 嵌入式 seekDB 路径;仅在 `DATABASE_KIND=seekdb` 时使用 | +| `POWERCONTEXT_SERVER_DEPLOYMENT_MODE` | `single_node` | `single_node` 或 `distributed` 进程拓扑 | +| `POWERCONTEXT_SERVER_DEPLOYMENT_ROLE` | `all` | `all`、`api`、`scheduler` 或 `worker`;分布式模式禁止 `all` | +| `POWERCONTEXT_SERVER_DEPLOYMENT_ID` | `local` | 非敏感运维实例标签;启动 owner identity 仍然唯一 | +| `POWERCONTEXT_SERVER_DEPLOYMENT_BEHAVIOR_REVISION` | `default` | 所有副本共享的非敏感发布兼容版本 | +| `POWERCONTEXT_SERVER_COORDINATION_SCHEDULER_LEASE_SECONDS` | `30` | 使用数据库时间的 Scheduler leader lease 时长 | +| `POWERCONTEXT_SERVER_COORDINATION_SCHEDULER_RENEW_SECONDS` | `10` | Scheduler 续租间隔;不超过 lease 的三分之一 | +| `POWERCONTEXT_SERVER_COORDINATION_SCAN_PAGE_SIZE` | `100` | discoverer 单页最多检查的 scope 数量 | +| `POWERCONTEXT_SERVER_COORDINATION_MEMBER_TTL_SECONDS` | `30` | Runtime member 声明有效期 | +| `POWERCONTEXT_SERVER_COORDINATION_MEMBER_HEARTBEAT_SECONDS` | `10` | Runtime member 心跳间隔 | +| `POWERCONTEXT_SERVER_COORDINATION_EMIT_PAYLOAD_VERSION` | `1` | 滚动发布期间发出的 Work payload version | +| `POWERCONTEXT_SERVER_WORKER_CONCURRENCY` | `4` | 单个 Worker 并发 attempt 上限 | +| `POWERCONTEXT_SERVER_WORKER_LEASE_SECONDS` | `120` | Worker claim lease 时长 | +| `POWERCONTEXT_SERVER_WORKER_HEARTBEAT_SECONDS` | `30` | Claim 心跳间隔;必须小于 lease 的三分之一 | +| `POWERCONTEXT_SERVER_WORKER_SHUTDOWN_GRACE_SECONDS` | `90` | 最大优雅 drain 时间;必须小于 lease | +| `POWERCONTEXT_SERVER_WORKER_MAX_ATTEMPTS` | `5` | 需要 operator 恢复前的自动 attempt 上限 | +| `POWERCONTEXT_SERVER_WORKER_RETRY_BASE_SECONDS` | `2` | full-jitter 指数退避基数 | +| `POWERCONTEXT_SERVER_WORKER_RETRY_MAX_SECONDS` | `300` | full-jitter 退避上限 | +| `POWERCONTEXT_SERVER_WORKER_POLL_SECONDS` | `1` | 空闲 claim 轮询间隔 | +| `POWERCONTEXT_SERVER_OPERATIONS_DEFAULT_WAIT_SECONDS` | `10` | HTTP Memory flush 默认等待时间 | +| `POWERCONTEXT_SERVER_OPERATIONS_MAXIMUM_WAIT_SECONDS` | `30` | `Prefer: wait=N` 最大允许值 | +| `POWERCONTEXT_SERVER_OPERATIONS_POLL_SECONDS` | `0.2` | 本地 operation 完成轮询间隔 | +| `POWERCONTEXT_SERVER_OPERATIONS_RETENTION_DAYS` | `30` | 成功和取消的 operation 历史保留天数 | +| `POWERCONTEXT_SERVER_OPERATIONS_CLEANUP_BATCH_SIZE` | `500` | 单次 maintenance attempt 最大清理数量 | +| `POWERCONTEXT_SERVER_OPERATIONS_CLEANUP_INTERVAL_SECONDS` | `3600` | 持久 maintenance discovery 间隔 | +| `POWERCONTEXT_SERVER_RATE_LIMIT_ENABLED` | `false` | 启用数据库共享固定窗口限流 | +| `POWERCONTEXT_SERVER_RATE_LIMIT_REQUESTS` | `120` | 每个 principal/policy 窗口允许的请求数 | +| `POWERCONTEXT_SERVER_RATE_LIMIT_WINDOW_SECONDS` | `60` | 共享限流窗口时长 | | `POWERCONTEXT_SERVER_RUNTIME_SCOPE_CACHE_SIZE` | `128` | Runtime 保留的非活动 scope composition 数量;进行中的 scope 不会被驱逐 | | `POWERCONTEXT_SERVER_RUNTIME_SOURCE_WINDOW_LIMIT` | `100` | 单次 activation 最多处理的 Source 数量 | | `POWERCONTEXT_SERVER_RUNTIME_MEMORY_EXTRACTION_PROFILE` | `coding` | Memory 选择策略:`coding` 或 `conversation` | @@ -190,13 +217,25 @@ fork/evolution。External Skill 精确导入和完整 package 上传不使用模 bytes,再创建 package digest 完全相同的 pending Candidate。未配置模型时,语义生成会在持久化 Candidate 前返回 capability error;Review、package 检查与下载、精确导入、usage recording 和 external Skill scan/list/resolve 仍可使用。 -Experience 孵化使用独立的 APScheduler job 和持久化 Source cursor。每次 activation 固定检查最多 32 条 Source,并且只把 metadata 包含 `"kind": "task-outcome"` 的 Content Source -暴露给模型。该 job 会在 Review Inbox 中创建 pending Experience Candidate;它不会自动批准、进入 +Experience 孵化使用独立的持久 Work handler 和 Source cursor。每次 activation 固定检查最多 32 条 Source,并且只把 +metadata 包含 `"kind": "task-outcome"` 的 Content Source 暴露给模型。该 handler 会在 Review Inbox 中创建 pending +Experience Candidate;它不会自动批准、进入 PreparedContext、创建 managed Skill、将它导出到 Agent target 或执行任何内容。Memory 和 Experience job 共用 -`POWERCONTEXT_HOME` 下的 APScheduler sidecar,但拥有独立的 job identity 和业务 cursor;取消其中一个 interval -只会移除对应 job。 +数据库 Work Ledger,但拥有独立 lane、logical key 和业务 cursor;取消其中一个 interval 只会关闭对应 discoverer, +已经入队的 operation 仍可查询。 设置与验证步骤见[创建并审核 Experience](../how-to/create-and-review-experience.md)。 +### 分布式角色与迁移 + +分布式模式要求 OceanBase。启动任何角色前,先使用有 DDL 权限的账号执行 +`powercontext server migrate --env-file ...`;角色进程不会创建或修改 schema。升级顺序固定为 migrate、Worker、 +Scheduler、API。当一次发布改变了不能混部的非敏感行为时,应为所有新副本设置新的 +`POWERCONTEXT_SERVER_DEPLOYMENT_BEHAVIOR_REVISION`。 + +Scheduler 或 Worker member 缺失时,API 仍可接受持久任务和读取请求,但 readiness 会是 `degraded` 并标出缺失角色。 +Scheduler 与 Worker 角色只暴露 health 和 metrics。分布式 MCP 为 stateless,不需要负载均衡粘性。由于不同副本可能 +返回不同结果,分布式模式会拒绝 host-local External Skill target。 + ### Agent Skill 目标 零配置流程使用上述 workspace 中的 Codex 和 Claude Code 项目级目录。只有需要自定义路径、用户级 target、环境兼容性 diff --git a/docs/zh/docs/reference/http-api.md b/docs/zh/docs/reference/http-api.md index 33e2edc17..43b6e9772 100644 --- a/docs/zh/docs/reference/http-api.md +++ b/docs/zh/docs/reference/http-api.md @@ -85,6 +85,25 @@ curl --fail \ "$POWERCONTEXT_URL/v1/memory/search" ``` +## 通过持久 Operation 执行 flush + +Memory flush 可以在当前请求内完成,也可以由任意 Worker 继续执行。不希望等待时,显式请求立即返回句柄: + +```bash +curl --fail-with-body --include \ + --request POST \ + --header 'Content-Type: application/json' \ + --header 'Prefer: respond-async' \ + --header "$POWERCONTEXT_AUTH_HEADER" \ + --data '{"scope_id":"project:example"}' \ + "$POWERCONTEXT_URL/v1/memory/flush" +``` + +HTTP `200` 返回已经完成的 `FlushMemoryResponse`;HTTP `202` 返回 operation ID,并携带相对 `Location` 和 +`Retry-After` header。`Prefer: wait=N` 最多等待 30 秒。通过 `GET /v1/operations/{operation_id}` 轮询;取消 queued +或 running operation,以及恢复被阻塞的 failed operation 时,必须把响应中的 `state_version` 作为 +`expected_version` 传回。同一逻辑窗口失败后,后续 flush 返回 `409 operation_blocked`,直到 operator retry 或 cancel。 + ## 查找操作 | 领域 | 主要路径 | 用途 | @@ -99,6 +118,7 @@ curl --fail \ | 外部 Skill | `/v1/external-skills/*` | 扫描已配置 target,解析或导入 package | | Handoff Report | `/v1/handoff-reports/*` | 按 Scope selection 生成只读报告 | | 统计 | `/v1/stats` | 读取指定 scope 的使用统计 | +| 持久 Operation | `/v1/operations/*` | 检查、列出、取消或恢复 Memory/Experience 后台任务 | 完整路径、schema、限制和状态码以 OpenAPI 契约为准。高层工作流和 Python 示例见[接口](interfaces.md)。 @@ -121,6 +141,7 @@ curl --fail \ | 状态码 | 含义 | | --- | --- | | `401` | Server 要求有效的 Bearer token | +| `429` | 共享请求窗口已耗尽;按 `Retry-After` 等待 | | `404` | 请求的不可变值不存在 | | `409` | 请求与当前不可变状态或 expected version 冲突 | | `413` | 选中的 Handoff Report 超过输出限制 | diff --git a/docs/zh/docs/reference/interfaces.md b/docs/zh/docs/reference/interfaces.md index af96e4a2c..b203ebbc0 100644 --- a/docs/zh/docs/reference/interfaces.md +++ b/docs/zh/docs/reference/interfaces.md @@ -197,12 +197,12 @@ Experience Revision 仍不会进入 PreparedContext。 ## 后台 Experience 孵化 Integration 可以把已完成任务采集为 metadata 含 `"kind": "task-outcome"` 的 Content Source。启用 -Experience schedule 后,APScheduler 会扫描有上限的 Source window,并让配置好的 schema-bound pipeline -生成可复用的 situation、action、outcome 和 lesson。每条 proposal 都引用精确 Source,并以 pending -Experience Candidate 进入 Review Inbox。 +Experience schedule 后,持久化 Scheduler 扫描有上限的 Source window 并写入带版本任务;带 fence 的 Worker 再让 +配置好的 schema-bound pipeline 生成可复用的 situation、action、outcome 和 lesson。每条 proposal 都引用精确 Source, +并以 pending Experience Candidate 进入 Review Inbox。 -Experience 孵化使用独立于 Memory extraction 的持久化 Source cursor。Candidate 写入和 cursor 推进会在同一 -事务提交;generation 或写入失败时,该 window 保留给下次重试。普通 Prompt Source 不是 Task Outcome, +Experience 孵化使用独立于 Memory extraction 的持久化 Source cursor。Candidate 写入、cursor 推进和 Work success +会在同一事务提交;generation 或写入失败时,该 window 保留给下次重试。普通 Prompt Source 不是 Task Outcome, 不会进入这个 job。 后台流程止于审核边界:它不会批准 Experience、把 pending 内容放入 PreparedContext、派生 managed Skill、 diff --git a/docs/zh/rfcs/0011_remote_access_architecture.md b/docs/zh/rfcs/0011_remote_access_architecture.md index 9224796f6..ebfe2a348 100644 --- a/docs/zh/rfcs/0011_remote_access_architecture.md +++ b/docs/zh/rfcs/0011_remote_access_architecture.md @@ -2,6 +2,9 @@ - Start Date: 2026-07-16 - RFC PR: [oceanbase/powercontext#11](https://github.com/oceanbase/powercontext/pull/11) +> **部署边界:** [RFC 1430](1430_distributed_server_workers.md) 定义已接受的持久 Operation、无状态多副本 API、 +> 角色拆分和分布式协调契约。本 RFC 未定义执行与部署语义之处,以 RFC 1430 为准。 + # Summary 本 RFC 提议为 PowerContext 建立远程访问边界。Server 通过 OpenAPI 定义的 HTTP contract 暴露 application diff --git a/docs/zh/rfcs/0019_local_source_memory_runtime.md b/docs/zh/rfcs/0019_local_source_memory_runtime.md index 692001d2d..636650210 100644 --- a/docs/zh/rfcs/0019_local_source_memory_runtime.md +++ b/docs/zh/rfcs/0019_local_source_memory_runtime.md @@ -2,6 +2,9 @@ - Start Date: 2026-07-24 - RFC PR: [oceanbase/powercontext#19](https://github.com/oceanbase/powercontext/pull/19) +> **执行更新:** [RFC 1430](1430_distributed_server_workers.md) 使用同一套数据库 Work Ledger 替代 APScheduler +> sidecar 和单进程执行假设,并同时服务本地与分布式模式。本 RFC 仍负责 Source window、cursor、Memory 和领域提交语义。 + > **注意:** PowerContext 当前使用随 builtin 依赖捆绑的 sqlite-vec 提供 SQLite 向量检索。本 RFC 中关于 Vec1 的 > 表述已不再适用,仅作为原始设计记录保留。 diff --git a/docs/zh/rfcs/0020_runtime_backed_memory_remote_access.md b/docs/zh/rfcs/0020_runtime_backed_memory_remote_access.md index 5a47c8ae3..be119de1a 100644 --- a/docs/zh/rfcs/0020_runtime_backed_memory_remote_access.md +++ b/docs/zh/rfcs/0020_runtime_backed_memory_remote_access.md @@ -6,6 +6,9 @@ > **注意:** PowerContext 当前使用随 builtin 依赖捆绑的 sqlite-vec 提供 SQLite 向量检索。本 RFC 中关于 Vec1 的 > 表述已不再适用,仅作为原始设计记录保留。 +> **执行更新:** [RFC 1430](1430_distributed_server_workers.md) 取代本 RFC 中的单进程调度、仅同步 flush 和 +> 非持久 Operation 假设。本 RFC 继续定义 HTTP 映射和生成契约边界。 + # Summary 本 RFC 定义首个基于 RFC 0011 架构和 RFC 0019 本地 Runtime 的具体远程 API。FastAPI Server 暴露 diff --git a/docs/zh/rfcs/1430_distributed_server_workers.md b/docs/zh/rfcs/1430_distributed_server_workers.md new file mode 100644 index 000000000..f6049127b --- /dev/null +++ b/docs/zh/rfcs/1430_distributed_server_workers.md @@ -0,0 +1,199 @@ +- Proposal Name: `distributed_server_workers` +- Start Date: 2026-09-03 +- Tracking Issue: [oceanbase/powercontext#1430](https://github.com/oceanbase/powercontext/issues/1430) +- Related RFCs: [RFC 0011](0011_remote_access_architecture.md)、[RFC 0019](0019_local_source_memory_runtime.md)、 + [RFC 0020](0020_runtime_backed_memory_remote_access.md) 和 [RFC 0046](0046_observability_foundations.md) + +# Summary + +PowerContext 在本地和分布式后台执行中统一使用数据库持久化的 Work Ledger。API 副本无需进程粘性即可入队和查询; +Scheduler 通过 fence 选出一个 leader,并持久保存 keyset 扫描位置;Worker 只领取当前空闲容量的任务,以数据库时间续租, +并安全恢复遗留 attempt。模型调用保持 at-least-once,但同一逻辑 Source window 最多提交一个数据库结果。 + +分布式模式只使用 OceanBase 作为协调后端。同一套 ledger 也替代 `single_node/all` 中的 APScheduler SQLite sidecar, +从而只保留一套需要理解和测试的执行协议。Redis、Kafka、公开 Queue SPI、任意用户代码、DAG、优先级、多区域共识以及 +Connector 自有 checkpoint 不属于本 RFC。 + +# 决策 + +本设计有意减少真相来源: + +- 分布式模式下,OceanBase 同时持有任务状态、租约、cursor 和领域提交。 +- 执行为 at-least-once。崩溃后 provider 调用可能重复,但 fence 和领域 cursor 会阻止旧 attempt 提交。 +- Memory activation 和 Experience incubation 首先接入 ledger;后续 handler 的领域 checkpoint 仍由各自模块持有。 +- 默认保持 `single_node/all`;分布式进程必须且只能选择 `api`、`scheduler` 或 `worker`。 +- Work payload 只保存带版本的引用和数字窗口边界,不保存 Source 正文、prompt、模型响应、credential 或原始异常。 + +这避免了数据库、broker 和本地 sidecar 多套协议自然积累的熵。新增复杂度被固化为数据库约束、小型状态机、fencing token、 +启动校验和兼容性声明,而不再依赖“系统只有一个进程”之类的隐性经验。 + +# 架构 + +```text +API / Scheduler + | 短事务:确定窗口、去重、入队 + v +OceanBase Work Ledger + | 短事务:领取 lane head 并签发 lease fence + v +Worker ---- 在事务外执行 provider/model 调用 ----> + | 短事务:验证 fence,提交领域状态 + cursor + work + v +一个逻辑数据库结果 +``` + +Ledger 包含: + +| 表 | 职责 | +| --- | --- | +| `pc_work_items` | 带版本任务、scope、lane sequence、安全 payload、状态、租约、attempt 预算及安全结果/错误 | +| `pc_work_attempts` | append-only owner/fence/时间/结果审计和非敏感 trace identity | +| `pc_work_lanes` | 每个一致性域的严格 sequence 和唯一 active head | +| `pc_work_keys` | 当前逻辑窗口的唯一占位 | +| `pc_scheduler_leases` | Scheduler leader 与 migrator 的 owner、单调 fence 和数据库时间 expiry | +| `pc_scheduler_scans` | 各 discoverer 的下次执行时间和 keyset continuation | +| `pc_runtime_members` | 存活角色以及 schema/payload/behavior 兼容性声明 | +| `pc_rate_limit_windows` | 可选共享固定窗口计数,以 principal 摘要和 policy 为键 | + +Memory 窗口的 logical key 是 handler kind、scope、cursor name、cursor generation 和 previous cursor 的 SHA-256。 +入队时的 high watermark 与 `through` 保存在 payload 中,但不进入 key。因此 manual 与 scheduled discovery 面对同一个 +未推进 cursor 时会加入同一任务;之后到达的 Source 由下一窗口处理。 + +Memory 和 Experience 使用独立 lane(散列前分别为 `memory:{scope}` 与 `experience:{scope}`)。不同 lane 可以并行, +同一 lane 严格按 sequence 推进。终态失败会保留 logical-key 和 lane 占位,直到 operator retry 或 cancel,避免 poison +window 被无限重建。 + +# 状态机 + +```text +queued ----> running ----> succeeded + | |----> retry_wait ----> running + | |----> failed + | `----> cancelling ----> cancelled + `-----------------------------------> cancelled + +failed -- 显式 operator retry --> queued +``` + +取消在 work row 上线性化。如果取消先于最终事务,Worker 丢弃已经准备好的 provider 结果,并把 attempt 完成为 +cancelled;如果成功已经提交,取消返回 `409`。取消某个 operation 不会暂停该 scope 后续的 discovery。 + +# 锁与事务协议 + +基线是 OceanBase 4.3.5 默认 Read Committed。正确性不依赖无锁读取、单条谓词 update、`SKIP LOCKED` 或 +`GET_LOCK()`。所有路径采用统一锁顺序: + +```text +scheduler lease(仅 Scheduler 路径) + -> lane + -> logical key + -> work item + -> domain cursor/head +``` + +稳定存在的 lease/lane row 会被显式锁定。缺失 row 通过唯一键插入,冲突后重试。claim 先做带索引且有上限的候选扫描, +再逐 lane 用短事务领取。Worker 永远不会预取超过空闲执行槽的任务。 + +Scheduler 的 enqueue 和 scan 更新都会重新验证精确 leader owner、fence 与数据库 expiry。Worker heartbeat、失败、 +取消收敛和最终提交都会验证 `(work_id, owner, fence)`。所有 expiry 判断使用数据库的 `CURRENT_TIMESTAMP(6)`; +应用时钟只用于 poll 和 deadline。 + +Provider、Connector、网络和文件系统调用绝不处于数据库事务中。Worker 最终事务先验证 claim 未过期且未取消,随后调用 +handler 的领域 commit。Memory 与 Experience 复用现有 cursor/head CAS。领域写入、cursor 推进、attempt 完成和 +Work success 要么一起提交,要么一起回滚。 + +# Scheduler 与 Worker + +每个 discoverer 持久保存 keyset continuation,每页最多扫描 100 个 scope。崩溃后重复扫描是安全的,因为 logical key +会去重。旧 leader 在另一个 Scheduler 获得更高 fence 后,不能继续 enqueue 或保存 continuation。 + +Worker 只 claim 当前空闲槽。attempt lease 默认 120 秒,每 30 秒续租。过期 attempt 会先写完审计并进入 retry,再由 +更高 fence 重新领取。可重试错误使用 full-jitter 指数退避,默认从 2 秒开始,上限 5 分钟,最多 5 次自动 generation +attempt。无法识别的 payload version 保持在 lane head 可见,并使 Worker readiness 成为 `misconfigured`;不得猜测、 +静默丢弃或跳过。 + +# 进程角色 + +配置新增 `deployment`、`coordination`、`worker`、`operations` 和 `rate_limit` 组。 + +- `single_node/all` 在同一进程运行 API、Scheduler 和 Worker,但仍走统一 ledger。数据库 owner lease 会拒绝误启动的 + 第二个实例。 +- `distributed/api` 暴露 HTTP、Dashboard、MCP、鉴权、共享限流、入队与 operation 查询,不运行 discovery 或后台 + Worker handler。 +- `distributed/scheduler` 只暴露 health 和 metrics,不持有 provider credential,只进行有界 discovery。 +- `distributed/worker` 只暴露 health 和 metrics,仅持有所注册 handler 必需的 provider credential。 + +分布式模式要求 OceanBase,并拒绝 `role=all`、SQLite、seekDB 和显式 host-local External Skill target。所有角色可以 +复用同一镜像,但应使用独立的最小权限数据库账号。DDL 只属于 migrator 账号,分布式角色绝不自动执行。 + +每个进程用 boot-unique member identity 心跳上报 build version、当前 schema/payload range 与非敏感 +`behavior_revision`。Member metadata 不保存 credential、secret URL、authorization 数据或允许离线猜测 secret 的 hash。 + +# HTTP 与 Client 契约 + +`openapi/powercontext.yaml` 仍是唯一来源。`POST /v1/memory/flush` 的行为是: + +- 没有待处理 Source,或在等待预算内完成:`200 FlushMemoryResponse`; +- 仍为 queued、running 或 retry_wait:`202 OperationAccepted`,带相对 `Location` 和 `Retry-After: 2`; +- 同一 logical key 已被失败任务阻塞:`409 operation_blocked`,返回 operation ID。 + +`Prefer: respond-async` 立即返回句柄;`Prefer: wait=N` 在最长 30 秒内等待,默认 10 秒。Operation endpoint 提供授权后 +的 get/list、乐观并发 cancel 和 operator retry。Mutation body 必须包含 `expected_version`,非法或过期转换返回 `409`。 +公开状态为 `queued`、`running`、`retry_wait`、`cancelling`、`succeeded`、`failed`、`cancelled`。内部 maintenance work +不会通过 Operation API 暴露。 + +`PowerContextClient.flush_memory()` 通过 submit + poll 保持原有同步式返回;总 deadline 到期时抛出携带 operation ID 的 +`OperationPendingError`。需要显式控制的调用方使用 `submit_memory_flush()`、`get_operation()`、`list_operations()`、 +`cancel_operation()` 和 `retry_operation()`。 + +Operation endpoint 不投影为 MCP tool。分布式模式强制 FastMCP stateless HTTP,连续请求可以命中不同 API 副本。 +Server-to-client elicitation 和 sampling 关闭;Workstream picker 返回结构化 `needs_selection`。内部 ASGI bridge 会传递 +已经认证的 principal,工具可见性绝不作为授权边界。 + +# 健康、关闭、保留与可观测性 + +Liveness 只表示进程能响应。API readiness 要求 database、schema、authentication policy、membership 和 behavior +兼容;缺失 Scheduler 或 Worker 只让 API degraded,仍允许持久入队和读取。健康的 Scheduler standby 也是 ready。 +缺少必需 provider 或 handler version 的 Worker 停止 claim,并报告精确失败项。 + +SIGTERM 时,API 先摘除 readiness;Scheduler 停止 discovery 并条件释放 lease;Worker 停止 claim,继续为在途任务 +heartbeat,并最多 drain 90 秒。非优雅退出由 lease expiry 恢复。 + +成功与取消的 work/attempt 默认保留 30 天。持久 maintenance handler 每批最多删除 500 条,并清理过期限流窗口。 +阻塞失败任务在 operator 处理前不得删除。未来 Source retention 必须把所有非终态 work window 当作 retention root。 + +Metric 只使用 kind、status、outcome、role 和 error category 等有界标签,覆盖 queue depth/age、claim/attempt latency、 +lease expiry、retry、throughput、leader change 和 member count。Scope、principal 和 work ID 不能作为 metric label。 +Enqueue、claim、execute、commit 和 retry 使用独立 span;retry attempt 通过 span link 关联。 + +日志、span、work row 与 attempt row 均不得包含 Source 正文、prompt、模型输出、credential、authorization header 或完整 +secret URL。持久错误只保存有界 category 与 code。 + +# Schema 与发布 + +Alembic 管理 forward-only schema chain。`powercontext server migrate` 获取数据库 lease 后升级到 packaged head。新库从 +baseline 创建;已知且完整的 legacy schema 先校验,只应用明确识别的扩展,再 stamp 并创建 ledger。未知或部分安装的 +schema 直接拒绝。单机启动自动运行同一迁移链;分布式角色只读校验当前 revision。 + +Schema 遵循 expand、mixed-version deploy、contract。N+1 必须能读取 N/N+1 layout,破坏性删除最早在 N+2。 +实际混部期间 Worker 支持当前和前一版 payload;`emit_payload_version` 在旧 Worker 排空前继续发送旧格式。 + +部署顺序是 migrate、Worker、Scheduler、API;回滚顺序相反,并在恢复旧 Worker 前排空新版 payload。Bridge release +先让 `single_node/all` 全部走 Work Ledger;只有 APScheduler 执行路径完全退出后,才允许同一数据库扩为多角色。 +旧 sidecar 只作为 operator 备份产物保留,系统不自动删除。 + +# 验收 + +验收覆盖:两个 API 副本间 round-robin 的 HTTP、Dashboard API 与 stateless MCP;不同 lane 并行与同 lane 串行; +manual/scheduled 去重;Worker 在 claim 前、provider 执行中、最终事务中和 commit 后的崩溃恢复;Scheduler takeover +fencing;retry、cancel、operator recovery、retention 与隐私;以及 OceanBase 4.3.5 真实多进程测试。Golden +schema/payload fixture 覆盖混合版本与升级回滚。SQLite 和 seekDB 只保留单机回归支持。 + +# 未采用方案 + +- Redis 或 Kafka 会在吞吐尚不需要时引入第二个持久真相和跨系统提交问题。 +- 公开 Queue/Coordinator SPI 会在不存在第二种实现时过早冻结多套协调语义。 +- `GET_LOCK()` 是 session 级锁,不适合作为连接池环境中的 fencing 原语。 +- 单机继续使用 APScheduler 会保留两套细微不同的执行和恢复协议。 +- 外部 provider 不可能实现真正 exactly-once;可执行的契约是 at-least-once execution 加 at-most-one fenced + database commit。 diff --git a/e2e/bub/uv.lock b/e2e/bub/uv.lock index 3cf7a2dc9..43360c466 100644 --- a/e2e/bub/uv.lock +++ b/e2e/bub/uv.lock @@ -1620,9 +1620,9 @@ requires-dist = [ { name = "aiosqlite", marker = "extra == 'builtin'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'seekdb'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'server'", specifier = ">=0.22,<1" }, - { name = "apscheduler", marker = "extra == 'builtin'", specifier = ">=3.11,<4" }, - { name = "apscheduler", marker = "extra == 'seekdb'", specifier = ">=3.11,<4" }, - { name = "apscheduler", marker = "extra == 'server'", specifier = ">=3.11,<4" }, + { name = "alembic", marker = "extra == 'builtin'", specifier = ">=1.14,<2" }, + { name = "alembic", marker = "extra == 'seekdb'", specifier = ">=1.14,<2" }, + { name = "alembic", marker = "extra == 'server'", specifier = ">=1.14,<2" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115,<1" }, { name = "fastmcp", marker = "extra == 'server'", specifier = ">=3.4,<4" }, { name = "httpx", extras = ["socks"], marker = "extra == 'cli'", specifier = ">=0.28,<1" }, diff --git a/integrations/dsh/plugins/powercontext/lib/index.js b/integrations/dsh/plugins/powercontext/lib/index.js index e527d310a..86cb0bb89 100644 --- a/integrations/dsh/plugins/powercontext/lib/index.js +++ b/integrations/dsh/plugins/powercontext/lib/index.js @@ -565,6 +565,34 @@ const OPERATIONS = { scopeMode: "selection", pathParameters: [] }, + list_operations: { + method: "GET", + path: "/v1/operations", + location: "query", + scopeMode: "none", + pathParameters: [] + }, + get_operation: { + method: "GET", + path: "/v1/operations/{operation_id}", + location: null, + scopeMode: "none", + pathParameters: ["operation_id"] + }, + cancel_operation: { + method: "POST", + path: "/v1/operations/{operation_id}/cancel", + location: "body", + scopeMode: "none", + pathParameters: ["operation_id"] + }, + retry_operation: { + method: "POST", + path: "/v1/operations/{operation_id}/retry", + location: "body", + scopeMode: "none", + pathParameters: ["operation_id"] + }, get_handoff_report: { method: "POST", path: "/v1/handoff-reports/get", diff --git a/integrations/dsh/plugins/powercontext/src/operations.generated.ts b/integrations/dsh/plugins/powercontext/src/operations.generated.ts index 962370efc..acddd24f0 100644 --- a/integrations/dsh/plugins/powercontext/src/operations.generated.ts +++ b/integrations/dsh/plugins/powercontext/src/operations.generated.ts @@ -86,6 +86,10 @@ export const OPERATIONS = { reject_artifact_candidate: { method: 'POST', path: '/v1/artifact-candidates/reject', location: "body", scopeMode: 'current', pathParameters: [] }, revise_artifact_candidate: { method: 'POST', path: '/v1/artifact-candidates/revise', location: "body", scopeMode: 'current', pathParameters: [] }, get_stats: { method: 'POST', path: '/v1/stats', location: "body", scopeMode: 'selection', pathParameters: [] }, + list_operations: { method: 'GET', path: '/v1/operations', location: "query", scopeMode: 'none', pathParameters: [] }, + get_operation: { method: 'GET', path: '/v1/operations/{operation_id}', location: null, scopeMode: 'none', pathParameters: ['operation_id'] }, + cancel_operation: { method: 'POST', path: '/v1/operations/{operation_id}/cancel', location: "body", scopeMode: 'none', pathParameters: ['operation_id'] }, + retry_operation: { method: 'POST', path: '/v1/operations/{operation_id}/retry', location: "body", scopeMode: 'none', pathParameters: ['operation_id'] }, get_handoff_report: { method: 'POST', path: '/v1/handoff-reports/get', location: "body", scopeMode: 'selection', pathParameters: [] }, } as const diff --git a/integrations/opencode/plugins/powercontext/src/operations.generated.ts b/integrations/opencode/plugins/powercontext/src/operations.generated.ts index 962370efc..acddd24f0 100644 --- a/integrations/opencode/plugins/powercontext/src/operations.generated.ts +++ b/integrations/opencode/plugins/powercontext/src/operations.generated.ts @@ -86,6 +86,10 @@ export const OPERATIONS = { reject_artifact_candidate: { method: 'POST', path: '/v1/artifact-candidates/reject', location: "body", scopeMode: 'current', pathParameters: [] }, revise_artifact_candidate: { method: 'POST', path: '/v1/artifact-candidates/revise', location: "body", scopeMode: 'current', pathParameters: [] }, get_stats: { method: 'POST', path: '/v1/stats', location: "body", scopeMode: 'selection', pathParameters: [] }, + list_operations: { method: 'GET', path: '/v1/operations', location: "query", scopeMode: 'none', pathParameters: [] }, + get_operation: { method: 'GET', path: '/v1/operations/{operation_id}', location: null, scopeMode: 'none', pathParameters: ['operation_id'] }, + cancel_operation: { method: 'POST', path: '/v1/operations/{operation_id}/cancel', location: "body", scopeMode: 'none', pathParameters: ['operation_id'] }, + retry_operation: { method: 'POST', path: '/v1/operations/{operation_id}/retry', location: "body", scopeMode: 'none', pathParameters: ['operation_id'] }, get_handoff_report: { method: 'POST', path: '/v1/handoff-reports/get', location: "body", scopeMode: 'selection', pathParameters: [] }, } as const diff --git a/integrations/pi/plugins/powercontext/src/operations.generated.ts b/integrations/pi/plugins/powercontext/src/operations.generated.ts index 962370efc..acddd24f0 100644 --- a/integrations/pi/plugins/powercontext/src/operations.generated.ts +++ b/integrations/pi/plugins/powercontext/src/operations.generated.ts @@ -86,6 +86,10 @@ export const OPERATIONS = { reject_artifact_candidate: { method: 'POST', path: '/v1/artifact-candidates/reject', location: "body", scopeMode: 'current', pathParameters: [] }, revise_artifact_candidate: { method: 'POST', path: '/v1/artifact-candidates/revise', location: "body", scopeMode: 'current', pathParameters: [] }, get_stats: { method: 'POST', path: '/v1/stats', location: "body", scopeMode: 'selection', pathParameters: [] }, + list_operations: { method: 'GET', path: '/v1/operations', location: "query", scopeMode: 'none', pathParameters: [] }, + get_operation: { method: 'GET', path: '/v1/operations/{operation_id}', location: null, scopeMode: 'none', pathParameters: ['operation_id'] }, + cancel_operation: { method: 'POST', path: '/v1/operations/{operation_id}/cancel', location: "body", scopeMode: 'none', pathParameters: ['operation_id'] }, + retry_operation: { method: 'POST', path: '/v1/operations/{operation_id}/retry', location: "body", scopeMode: 'none', pathParameters: ['operation_id'] }, get_handoff_report: { method: 'POST', path: '/v1/handoff-reports/get', location: "body", scopeMode: 'selection', pathParameters: [] }, } as const diff --git a/openapi/powercontext.yaml b/openapi/powercontext.yaml index b7c24f8c9..bc0a1b23a 100644 --- a/openapi/powercontext.yaml +++ b/openapi/powercontext.yaml @@ -79,6 +79,8 @@ paths: $ref: "#/components/schemas/Capabilities" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" /v1/scopes: get: tags: [scopes] @@ -365,6 +367,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -497,6 +501,8 @@ paths: $ref: "#/components/schemas/PreparedContext" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -532,6 +538,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -567,6 +575,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -602,6 +612,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -637,6 +649,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -670,6 +684,8 @@ paths: $ref: "#/components/responses/NotFound" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -702,6 +718,8 @@ paths: $ref: "#/components/responses/NotFound" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -734,6 +752,8 @@ paths: $ref: "#/components/responses/NotFound" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -768,6 +788,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -800,6 +822,8 @@ paths: $ref: "#/components/responses/NotFound" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -813,6 +837,13 @@ paths: description: Run one bounded Source-to-Memory activation for operational control and testing. operationId: flush_memory x-powercontext-scope-mode: current + parameters: + - name: Prefer + in: header + required: false + description: Use `respond-async` for an immediate handle or `wait=N` to wait at most 30 seconds. + schema: + type: string requestBody: required: true content: @@ -829,8 +860,29 @@ paths: application/json: schema: $ref: "#/components/schemas/FlushMemoryResponse" + "202": + description: The durable operation is still queued, running, or waiting to retry. + headers: + X-PowerContext-Request-ID: + $ref: "#/components/headers/RequestId" + Location: + description: Relative URL of the accepted operation. + schema: + type: string + Retry-After: + description: Suggested polling delay in seconds. + schema: + type: integer + content: + application/json: + schema: + $ref: "#/components/schemas/OperationAccepted" + "409": + $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -864,6 +916,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -897,6 +951,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -932,6 +988,8 @@ paths: $ref: "#/components/responses/NotFound" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -965,6 +1023,8 @@ paths: $ref: "#/components/responses/NotFound" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1000,6 +1060,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1035,6 +1097,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1068,6 +1132,8 @@ paths: $ref: "#/components/responses/NotFound" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1101,6 +1167,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1134,6 +1202,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1167,6 +1237,8 @@ paths: $ref: "#/components/responses/NotFound" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1200,6 +1272,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1233,6 +1307,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1266,6 +1342,8 @@ paths: $ref: "#/components/responses/NotFound" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1780,6 +1858,8 @@ paths: $ref: "#/components/schemas/ScanExternalSkillsResponse" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1811,6 +1891,8 @@ paths: $ref: "#/components/schemas/ListExternalSkillsResponse" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1844,6 +1926,8 @@ paths: $ref: "#/components/responses/NotFound" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1879,6 +1963,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1910,6 +1996,8 @@ paths: $ref: "#/components/schemas/ArtifactCandidatePage" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1943,6 +2031,8 @@ paths: $ref: "#/components/responses/NotFound" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -1978,6 +2068,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -2013,6 +2105,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -2048,6 +2142,8 @@ paths: $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": @@ -2083,12 +2179,170 @@ paths: $ref: "#/components/schemas/ScopedStats" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "503": $ref: "#/components/responses/Unavailable" "500": $ref: "#/components/responses/InternalError" + /v1/operations: + get: + tags: [operations] + summary: List durable operations visible to the caller + operationId: list_operations + parameters: + - name: scope_id + in: query + schema: + type: string + minLength: 1 + maxLength: 256 + - name: kind + in: query + schema: + $ref: "#/components/schemas/OperationKind" + - name: status + in: query + schema: + $ref: "#/components/schemas/OperationStatus" + - name: cursor + in: query + schema: + type: string + nullable: true + - name: limit + in: query + schema: + type: integer + minimum: 1 + maximum: 100 + default: 50 + responses: + "200": + description: A bounded cursor page of authorized operations. + headers: + X-PowerContext-Request-ID: + $ref: "#/components/headers/RequestId" + content: + application/json: + schema: + $ref: "#/components/schemas/OperationPage" + "401": + $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" + "422": + $ref: "#/components/responses/InvalidRequest" + "503": + $ref: "#/components/responses/Unavailable" + /v1/operations/{operation_id}: + get: + tags: [operations] + summary: Get one durable operation + operationId: get_operation + parameters: + - name: operation_id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: The authorized operation and its safe result or error metadata. + headers: + X-PowerContext-Request-ID: + $ref: "#/components/headers/RequestId" + content: + application/json: + schema: + $ref: "#/components/schemas/OperationRecord" + "401": + $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" + "404": + $ref: "#/components/responses/NotFound" + "503": + $ref: "#/components/responses/Unavailable" + /v1/operations/{operation_id}/cancel: + post: + tags: [operations] + summary: Cancel one durable operation using optimistic concurrency + operationId: cancel_operation + parameters: + - name: operation_id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/OperationMutationRequest" + responses: + "200": + description: The updated operation. + headers: + X-PowerContext-Request-ID: + $ref: "#/components/headers/RequestId" + content: + application/json: + schema: + $ref: "#/components/schemas/OperationRecord" + "401": + $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" + "404": + $ref: "#/components/responses/NotFound" + "409": + $ref: "#/components/responses/Conflict" + "422": + $ref: "#/components/responses/InvalidRequest" + /v1/operations/{operation_id}/retry: + post: + tags: [operations] + summary: Recover one blocked failed operation using optimistic concurrency + operationId: retry_operation + parameters: + - name: operation_id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/OperationMutationRequest" + responses: + "200": + description: The recovered queued operation. + headers: + X-PowerContext-Request-ID: + $ref: "#/components/headers/RequestId" + content: + application/json: + schema: + $ref: "#/components/schemas/OperationRecord" + "401": + $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" + "404": + $ref: "#/components/responses/NotFound" + "409": + $ref: "#/components/responses/Conflict" + "422": + $ref: "#/components/responses/InvalidRequest" /v1/handoff-reports/get: post: tags: [handoff-reports] @@ -2137,6 +2391,8 @@ paths: $ref: "#/components/responses/NotFound" "401": $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" "422": $ref: "#/components/responses/InvalidRequest" "413": @@ -2222,6 +2478,19 @@ components: application/json: schema: $ref: "#/components/schemas/ErrorResponse" + RateLimited: + description: The shared request policy rejected this fixed-window request. + headers: + Retry-After: + description: Seconds until the current shared window expires. + schema: + type: integer + X-PowerContext-Request-ID: + $ref: "#/components/headers/RequestId" + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" InternalError: description: The Server failed without exposing internal details. headers: @@ -5696,6 +5965,168 @@ components: type: array items: $ref: "#/components/schemas/SearchMemoryHit" + OperationStatus: + type: string + enum: [queued, running, retry_wait, cancelling, succeeded, failed, cancelled] + OperationKind: + type: string + enum: [memory_flush, experience_incubation] + MemoryOperationResult: + type: object + additionalProperties: false + required: [type, previous_cursor, high_watermark, current_cursor, processed_source_count] + properties: + type: + type: string + enum: [memory_flush] + previous_cursor: + type: integer + minimum: 0 + high_watermark: + type: integer + minimum: 0 + current_cursor: + type: integer + minimum: 0 + processed_source_count: + type: integer + minimum: 0 + memory: + $ref: "#/components/schemas/ArtifactReference" + nullable: true + ExperienceOperationResult: + type: object + additionalProperties: false + required: [type, previous_cursor, high_watermark, current_cursor, processed_source_count, candidate_count] + properties: + type: + type: string + enum: [experience_incubation] + previous_cursor: + type: integer + minimum: 0 + high_watermark: + type: integer + minimum: 0 + current_cursor: + type: integer + minimum: 0 + processed_source_count: + type: integer + minimum: 0 + candidate_count: + type: integer + minimum: 0 + OperationError: + type: object + additionalProperties: false + required: [category, code] + properties: + category: + type: string + minLength: 1 + maxLength: 64 + code: + type: string + minLength: 1 + maxLength: 128 + OperationRecord: + type: object + additionalProperties: false + required: + [operation_id, kind, scope_id, status, attempt_count, state_version, created_at, updated_at] + properties: + operation_id: + type: string + format: uuid + kind: + $ref: "#/components/schemas/OperationKind" + scope_id: + type: string + minLength: 1 + maxLength: 256 + status: + $ref: "#/components/schemas/OperationStatus" + attempt_count: + type: integer + minimum: 0 + state_version: + type: integer + minimum: 1 + created_at: + type: string + format: date-time + updated_at: + type: string + format: date-time + completed_at: + type: string + format: date-time + nullable: true + result: + oneOf: + - $ref: "#/components/schemas/MemoryOperationResult" + - $ref: "#/components/schemas/ExperienceOperationResult" + discriminator: + propertyName: type + nullable: true + error: + $ref: "#/components/schemas/OperationError" + nullable: true + OperationAccepted: + type: object + additionalProperties: false + required: [operation_id, status, status_url] + properties: + operation_id: + type: string + format: uuid + status: + $ref: "#/components/schemas/OperationStatus" + status_url: + type: string + pattern: '^/v1/operations/[0-9a-f-]{36}$' + OperationPage: + type: object + additionalProperties: false + required: [items] + properties: + items: + type: array + maxItems: 100 + items: + $ref: "#/components/schemas/OperationRecord" + next_cursor: + type: string + nullable: true + ListOperationsRequest: + type: object + additionalProperties: false + properties: + scope_id: + type: string + minLength: 1 + maxLength: 256 + kind: + $ref: "#/components/schemas/OperationKind" + status: + $ref: "#/components/schemas/OperationStatus" + cursor: + type: string + nullable: true + limit: + type: integer + minimum: 1 + maximum: 100 + default: 50 + OperationMutationRequest: + type: object + additionalProperties: false + required: [expected_version] + properties: + expected_version: + type: integer + minimum: 1 SourceReference: type: object additionalProperties: false diff --git a/pyproject.toml b/pyproject.toml index 25125163a..0da83322e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,8 +38,8 @@ classifiers = [ [project.optional-dependencies] builtin = [ + "alembic>=1.14,<2", "aiosqlite>=0.22,<1", - "apscheduler>=3.11,<4", "jsonschema>=4.23,<5", "packaging>=24,<27", "pydantic-ai-slim[anthropic,openai]>=2.27.1,<3", diff --git a/scripts/generate_api.py b/scripts/generate_api.py index 3bc865113..9f8edf7b4 100644 --- a/scripts/generate_api.py +++ b/scripts/generate_api.py @@ -160,9 +160,13 @@ def _generate_operations( if request_model is not None: imports.add(request_model[:2]) - success_status, success_response = _success_response(operation.responses, path) - response_model = _model_for_json_content(success_response.content, schemas, path) - imports.add(response_model[:2]) + success_responses = _success_responses(operation.responses, path) + response_models = tuple( + (status, _model_for_json_content(response.content, schemas, path)) + for status, response in success_responses + ) + for _, response_model in response_models: + imports.add(response_model[:2]) operations.append( _render_operation( constant_name=operation_id.upper(), @@ -174,8 +178,7 @@ def _generate_operations( path_parameters=tuple( parameter.name for parameter in parameters if parameter.in_ is ParameterInType.path ), - response_model=response_model[1], - success_status=success_status, + response_models=tuple((status, model[1]) for status, model in response_models), summary=operation.summary, tags=tuple(operation.tags or ()), scope_mode=_scope_mode(operation), @@ -192,7 +195,7 @@ def _generate_operations( from __future__ import annotations -from typing import Generic, Literal, TypeVar +from typing import Generic, Literal, TypeVar, cast from pydantic import BaseModel, JsonValue @@ -214,13 +217,24 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type: type[RequestT] | None request_location: Literal["body", "query"] | None path_parameters: tuple[str, ...] - response_type: type[ResponseT] - success_status: int + success_response_types: dict[int, type[BaseModel]] summary: str tags: tuple[str, ...] scope_mode: Literal["none", "current", "selection"] responses: dict[int | str, dict[str, JsonValue]] + @property + def success_statuses(self) -> tuple[int, ...]: + return tuple(sorted(self.success_response_types)) + + @property + def success_status(self) -> int: + return self.success_statuses[0] + + @property + def response_type(self) -> type[ResponseT]: + return cast(type[ResponseT], self.success_response_types[self.success_status]) + {rendered_operations} """ @@ -340,19 +354,20 @@ def _operation_parameters(path_item: PathItem, operation: OpenAPIOperation) -> t return tuple(parameters) -def _success_response( +def _success_responses( responses: dict[str, Response | object], path: str, -) -> tuple[int, Response]: - successes = [ - (int(code), response) for code, response in responses.items() if code.isdecimal() and 200 <= int(code) < 300 - ] - if len(successes) != 1: +) -> tuple[tuple[int, Response], ...]: + successes: list[tuple[int, Response]] = [] + for code, response in responses.items(): + if not code.isdecimal() or not 200 <= int(code) < 300: + continue + if not isinstance(response, Response): + raise ContractGenerationError("success response reference", path) # noqa: TRY003 + successes.append((int(code), response)) + if not successes: raise ContractGenerationError("success response", path) # noqa: TRY003 - success_status, response = successes[0] - if not isinstance(response, Response): - raise ContractGenerationError("success response reference", path) # noqa: TRY003 - return success_status, response + return tuple(sorted(successes, key=lambda item: item[0])) def _model_for_json_content( @@ -389,23 +404,23 @@ def _render_operation( request_model: str | None, request_location: Literal["body", "query"] | None, path_parameters: tuple[str, ...], - response_model: str, - success_status: int, + response_models: tuple[tuple[int, str], ...], summary: str, tags: tuple[str, ...], scope_mode: Literal["none", "current", "selection"], responses: dict[int | str, dict[str, JsonValue]], ) -> str: request_type = "None" if request_model is None else request_model - return f"""{constant_name} = Operation[{request_type}, {response_model}]( + response_type = " | ".join(dict.fromkeys(model for _, model in response_models)) + success_response_types = "{" + ", ".join(f"{status}: {model}" for status, model in response_models) + "}" + return f"""{constant_name} = Operation[{request_type}, {response_type}]( method={method!r}, path={path!r}, operation_id={operation_id!r}, request_type={request_type}, request_location={request_location!r}, path_parameters={path_parameters!r}, - response_type={response_model}, - success_status={success_status}, + success_response_types={success_response_types}, summary={summary!r}, tags={tags!r}, scope_mode={scope_mode!r}, diff --git a/src/powercontext/builtin/persistence/__init__.py b/src/powercontext/builtin/persistence/__init__.py index 7da8c4167..cb6772e22 100644 --- a/src/powercontext/builtin/persistence/__init__.py +++ b/src/powercontext/builtin/persistence/__init__.py @@ -21,6 +21,15 @@ ) from powercontext.builtin.persistence.candidates import CandidateRepository from powercontext.builtin.persistence.connectors import ConnectorCheckpointRepository +from powercontext.builtin.persistence.coordination import ( + CoordinationRepository, + CoordinatorLease, + RuntimeMember, + RuntimeMemberSpec, + SchedulerScan, + StaleCoordinatorLeaseError, + StaleScanStateError, +) from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.errors import ( DatabaseClosedError, @@ -35,6 +44,7 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.external_skills import ExternalSkillRepository +from powercontext.builtin.persistence.rate_limit import RateLimitDecision, RateLimitRepository from powercontext.builtin.persistence.skill_packages import SkillPackageRepository from powercontext.builtin.persistence.skill_publications import ( SkillPublication, @@ -48,12 +58,27 @@ StoredModelUsage, StoredRecallTokenUsage, ) +from powercontext.builtin.persistence.work import ( + EnqueueResult, + StaleWorkClaimError, + StoredWork, + WorkClaim, + WorkFailure, + WorkRepository, + WorkResult, + WorkSpec, + WorkStateConflictError, + WorkStatus, +) __all__ = ( "AsyncDatabase", "CandidateRepository", "ConnectorCheckpointRepository", + "CoordinationRepository", + "CoordinatorLease", "DatabaseClosedError", + "EnqueueResult", "ExternalSkillRepository", "GenerationConflictError", "IdentityMismatchError", @@ -61,19 +86,35 @@ "InvalidStoredColumnError", "InvalidStoredPayloadError", "PersistenceError", + "RateLimitDecision", + "RateLimitRepository", "RemoteAgentSkillTarget", "RemoteAgentSkillTargetRepository", "RemoteAgentSkillTargetState", "RepositoryError", "RepositoryNotFoundError", + "RuntimeMember", + "RuntimeMemberSpec", + "SchedulerScan", "SkillPackageRepository", "SkillPublication", "SkillPublicationDesiredState", "SkillPublicationRepository", "SourceDefinitionManifestRepository", + "StaleCoordinatorLeaseError", + "StaleScanStateError", + "StaleWorkClaimError", "StatisticsRepository", "StoredInventoryCounts", "StoredModelUsage", "StoredPayloadConflictError", "StoredRecallTokenUsage", + "StoredWork", + "WorkClaim", + "WorkFailure", + "WorkRepository", + "WorkResult", + "WorkSpec", + "WorkStateConflictError", + "WorkStatus", ) diff --git a/src/powercontext/builtin/persistence/coordination.py b/src/powercontext/builtin/persistence/coordination.py new file mode 100644 index 000000000..f163ec5c2 --- /dev/null +++ b/src/powercontext/builtin/persistence/coordination.py @@ -0,0 +1,416 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Database-time coordination primitives for schedulers and role members.""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import UTC, datetime, timedelta +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.database import database_now, insert_if_absent +from powercontext.builtin.persistence.errors import ( + InvalidRepositoryArgumentError, + InvalidStoredColumnError, + RepositoryError, +) +from powercontext.builtin.persistence.tables import ( + RUNTIME_MEMBERS_TABLE, + SCHEDULER_LEASES_TABLE, + SCHEDULER_SCANS_TABLE, +) +from powercontext.limits import MAX_SCOPE_ID_LENGTH + + +class CoordinatorLease(BaseModel): + """Monotonic fencing token held until database time reaches its expiry.""" + + model_config = ConfigDict(frozen=True) + + lease_name: str + owner_id: str + fence: int + lease_expires_at: datetime + + +class SchedulerScan(BaseModel): + """Durable keyset continuation for one bounded discoverer scan.""" + + model_config = ConfigDict(frozen=True) + + discoverer: str + next_run_at: datetime + continuation: str | None + state_version: int + updated_at: datetime + + +class RuntimeMember(BaseModel): + """Non-sensitive compatibility advertisement for one live process.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + member_id: str + role: Literal["all", "api", "scheduler", "worker"] + build_version: str + schema_min: int + schema_max: int + payload_min: int + payload_max: int + behavior_revision: str + heartbeat_at: datetime + expires_at: datetime + + +class RuntimeMemberSpec(BaseModel): + """Compatibility data supplied on member registration and heartbeat.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + member_id: str + role: Literal["all", "api", "scheduler", "worker"] + build_version: str + schema_min: int + schema_max: int + payload_min: int + payload_max: int + behavior_revision: str + + +class StaleCoordinatorLeaseError(RepositoryError): + """Raised when an old owner or fence attempts a coordinated write.""" + + def __init__(self, lease_name: str) -> None: + self.lease_name = lease_name + super().__init__(f"coordinator lease {lease_name!r} is no longer current") + + +class StaleScanStateError(RepositoryError): + """Raised when a scheduler scan continuation loses its version race.""" + + def __init__(self, discoverer: str) -> None: + self.discoverer = discoverer + super().__init__(f"scheduler scan {discoverer!r} changed concurrently") + + +class CoordinationRepository: + """Coordinate leases, scans, and role discovery in caller transactions.""" + + async def acquire_lease( + self, + connection: AsyncConnection, + *, + lease_name: str, + owner_id: str, + lease_seconds: int, + ) -> CoordinatorLease | None: + _require_text("lease_name", lease_name, 64) + _require_text("owner_id", owner_id, 128) + _require_positive("lease_seconds", lease_seconds) + now = await database_now(connection) + row = await _lock_or_create_lease(connection, lease_name, now) + current_owner = None if row["owner_id"] is None else str(row["owner_id"]) + current_expiry = _optional_datetime(row["lease_expires_at"], "lease_expires_at") + current_fence = _nonnegative_integer(row["fence"], "fence") + if current_owner == owner_id and current_expiry is not None and current_expiry > now: + fence = current_fence + elif current_expiry is None or current_expiry <= now: + fence = current_fence + 1 + else: + return None + expires_at = now + timedelta(seconds=lease_seconds) + await connection.execute( + update(SCHEDULER_LEASES_TABLE) + .where(SCHEDULER_LEASES_TABLE.c.lease_name == lease_name) + .values(owner_id=owner_id, fence=fence, lease_expires_at=expires_at, updated_at=now) + ) + return CoordinatorLease( + lease_name=lease_name, + owner_id=owner_id, + fence=fence, + lease_expires_at=expires_at, + ) + + async def assert_lease(self, connection: AsyncConnection, lease: CoordinatorLease, /) -> None: + row = await _lock_lease(connection, lease.lease_name) + now = await database_now(connection) + expires_at = None if row is None else _optional_datetime(row["lease_expires_at"], "lease_expires_at") + if ( + row is None + or row["owner_id"] != lease.owner_id + or row["fence"] != lease.fence + or expires_at is None + or expires_at <= now + ): + raise StaleCoordinatorLeaseError(lease.lease_name) + + async def release_lease(self, connection: AsyncConnection, lease: CoordinatorLease, /) -> bool: + now = await database_now(connection) + result = await connection.execute( + update(SCHEDULER_LEASES_TABLE) + .where( + SCHEDULER_LEASES_TABLE.c.lease_name == lease.lease_name, + SCHEDULER_LEASES_TABLE.c.owner_id == lease.owner_id, + SCHEDULER_LEASES_TABLE.c.fence == lease.fence, + ) + .values(owner_id=None, lease_expires_at=now, updated_at=now) + ) + return result.rowcount == 1 + + async def load_scan(self, connection: AsyncConnection, discoverer: str, /) -> SchedulerScan | None: + _require_text("discoverer", discoverer, 128) + row = ( + ( + await connection.execute( + select(SCHEDULER_SCANS_TABLE).where(SCHEDULER_SCANS_TABLE.c.discoverer == discoverer) + ) + ) + .mappings() + .one_or_none() + ) + return None if row is None else _decode_scan(row) + + async def save_scan( + self, + connection: AsyncConnection, + discoverer: str, + /, + *, + next_run_at: datetime, + continuation: str | None, + expected_version: int | None, + ) -> SchedulerScan: + _require_text("discoverer", discoverer, 128) + if continuation is not None: + _require_text("continuation", continuation, MAX_SCOPE_ID_LENGTH) + next_run_at = _normalized_datetime(next_run_at) + now = await database_now(connection) + if expected_version is None: + created = await insert_if_absent( + connection, + SCHEDULER_SCANS_TABLE, + { + "discoverer": discoverer, + "next_run_at": next_run_at, + "continuation": continuation, + "state_version": 1, + "updated_at": now, + }, + ) + if not created: + raise StaleScanStateError(discoverer) from None + else: + _require_positive("expected_version", expected_version) + result = await connection.execute( + update(SCHEDULER_SCANS_TABLE) + .where( + SCHEDULER_SCANS_TABLE.c.discoverer == discoverer, + SCHEDULER_SCANS_TABLE.c.state_version == expected_version, + ) + .values( + next_run_at=next_run_at, + continuation=continuation, + state_version=expected_version + 1, + updated_at=now, + ) + ) + if result.rowcount != 1: + raise StaleScanStateError(discoverer) + result = await self.load_scan(connection, discoverer) + if result is None: + raise InvalidStoredColumnError("discoverer", "a persisted scheduler scan") + return result + + async def heartbeat_member( + self, + connection: AsyncConnection, + spec: RuntimeMemberSpec, + /, + *, + ttl_seconds: int, + ) -> RuntimeMember: + _validate_member(spec) + _require_positive("ttl_seconds", ttl_seconds) + now = await database_now(connection) + expires_at = now + timedelta(seconds=ttl_seconds) + values = { + **spec.model_dump(mode="python"), + "heartbeat_at": now, + "expires_at": expires_at, + } + result = await connection.execute( + update(RUNTIME_MEMBERS_TABLE).where(RUNTIME_MEMBERS_TABLE.c.member_id == spec.member_id).values(**values) + ) + if result.rowcount != 1: + created = await insert_if_absent(connection, RUNTIME_MEMBERS_TABLE, values) + if not created: + result = await connection.execute( + update(RUNTIME_MEMBERS_TABLE) + .where(RUNTIME_MEMBERS_TABLE.c.member_id == spec.member_id) + .values(**values) + ) + if result.rowcount != 1: + raise InvalidStoredColumnError("member_id", "a concurrently persisted runtime member") + return RuntimeMember.model_validate(values) + + async def live_members( + self, + connection: AsyncConnection, + /, + *, + role: Literal["all", "api", "scheduler", "worker"] | None = None, + ) -> tuple[RuntimeMember, ...]: + now = await database_now(connection) + statement = select(RUNTIME_MEMBERS_TABLE).where(RUNTIME_MEMBERS_TABLE.c.expires_at > now) + if role is not None: + statement = statement.where(RUNTIME_MEMBERS_TABLE.c.role == role) + rows = (await connection.execute(statement.order_by(RUNTIME_MEMBERS_TABLE.c.member_id))).mappings().all() + return tuple(_decode_member(row) for row in rows) + + +async def _lock_or_create_lease( + connection: AsyncConnection, + lease_name: str, + now: datetime, +) -> Mapping[Any, Any]: + row = await _lock_lease(connection, lease_name) + if row is not None: + return row + await insert_if_absent( + connection, + SCHEDULER_LEASES_TABLE, + { + "lease_name": lease_name, + "owner_id": None, + "fence": 0, + "lease_expires_at": None, + "updated_at": now, + }, + ) + row = await _lock_lease(connection, lease_name) + if row is None: + raise InvalidStoredColumnError("lease_name", "an initialized coordinator lease") + return row + + +async def _lock_lease(connection: AsyncConnection, lease_name: str) -> Mapping[Any, Any] | None: + await connection.execute( + update(SCHEDULER_LEASES_TABLE) + .where(SCHEDULER_LEASES_TABLE.c.lease_name == lease_name) + .values(fence=SCHEDULER_LEASES_TABLE.c.fence) + ) + return ( + ( + await connection.execute( + select(SCHEDULER_LEASES_TABLE) + .where(SCHEDULER_LEASES_TABLE.c.lease_name == lease_name) + .with_for_update() + ) + ) + .mappings() + .one_or_none() + ) + + +def _decode_scan(row: Mapping[Any, Any]) -> SchedulerScan: + return SchedulerScan( + discoverer=str(row["discoverer"]), + next_run_at=_stored_datetime(row["next_run_at"], "next_run_at"), + continuation=None if row["continuation"] is None else str(row["continuation"]), + state_version=_positive_integer(row["state_version"], "state_version"), + updated_at=_stored_datetime(row["updated_at"], "updated_at"), + ) + + +def _decode_member(row: Mapping[Any, Any]) -> RuntimeMember: + return RuntimeMember.model_validate({ + "member_id": str(row["member_id"]), + "role": str(row["role"]), + "build_version": str(row["build_version"]), + "schema_min": _positive_integer(row["schema_min"], "schema_min"), + "schema_max": _positive_integer(row["schema_max"], "schema_max"), + "payload_min": _positive_integer(row["payload_min"], "payload_min"), + "payload_max": _positive_integer(row["payload_max"], "payload_max"), + "behavior_revision": str(row["behavior_revision"]), + "heartbeat_at": _stored_datetime(row["heartbeat_at"], "heartbeat_at"), + "expires_at": _stored_datetime(row["expires_at"], "expires_at"), + }) + + +def _validate_member(spec: RuntimeMemberSpec) -> None: + _require_text("member_id", spec.member_id, 128) + _require_text("build_version", spec.build_version, 64) + _require_text("behavior_revision", spec.behavior_revision, 128) + for name in ("schema_min", "schema_max", "payload_min", "payload_max"): + _require_positive(name, getattr(spec, name)) + if spec.schema_max < spec.schema_min: + raise InvalidRepositoryArgumentError("schema_max", "must not be less than schema_min") + if spec.payload_max < spec.payload_min: + raise InvalidRepositoryArgumentError("payload_max", "must not be less than payload_min") + + +def _stored_datetime(value: object, column: str) -> datetime: + if not isinstance(value, datetime): + raise InvalidStoredColumnError(column, "a datetime") + return _normalized_datetime(value) + + +def _optional_datetime(value: object, column: str) -> datetime | None: + return None if value is None else _stored_datetime(value, column) + + +def _normalized_datetime(value: datetime) -> datetime: + if value.tzinfo is None: + return value + return value.astimezone(UTC).replace(tzinfo=None) + + +def _positive_integer(value: object, column: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidStoredColumnError(column, "a positive integer") + return value + + +def _nonnegative_integer(value: object, column: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise InvalidStoredColumnError(column, "a non-negative integer") + return value + + +def _require_positive(field: str, value: int) -> None: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidRepositoryArgumentError(field, "must be a positive integer") + + +def _require_text(field: str, value: str, maximum: int) -> None: + if not isinstance(value, str) or not value.strip() or value != value.strip(): + raise InvalidRepositoryArgumentError(field, "must be a non-empty trimmed string") + if len(value) > maximum: + raise InvalidRepositoryArgumentError(field, f"must not exceed {maximum} characters") + + +__all__ = [ + "CoordinationRepository", + "CoordinatorLease", + "RuntimeMember", + "RuntimeMemberSpec", + "SchedulerScan", + "StaleCoordinatorLeaseError", + "StaleScanStateError", +] diff --git a/src/powercontext/builtin/persistence/cursors.py b/src/powercontext/builtin/persistence/cursors.py index 28c58d05b..aab8e7296 100644 --- a/src/powercontext/builtin/persistence/cursors.py +++ b/src/powercontext/builtin/persistence/cursors.py @@ -21,11 +21,10 @@ from pydantic import BaseModel from sqlalchemy import select, update -from sqlalchemy.dialects.mysql import insert as mysql_insert -from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.builtin.persistence.codec import dump_model, load_model, stored_bytes +from powercontext.builtin.persistence.database import insert_if_absent from powercontext.builtin.persistence.errors import ( GenerationConflictError, InvalidRepositoryArgumentError, @@ -87,12 +86,15 @@ async def save( if existing is not None: raise GenerationConflictError(binding_name, None, existing.generation) generation = 1 - created = await _insert_if_absent( + created = await insert_if_absent( connection, - scope_id=scope_id, - binding_name=binding_name, - cursor=payload, - generation=generation, + SOURCE_CURSORS_TABLE, + { + "scope_id": scope_id, + "binding_name": binding_name, + "cursor": payload, + "generation": generation, + }, ) if not created: # Another runtime may have inserted the same cursor after our @@ -133,34 +135,6 @@ async def save( ) -async def _insert_if_absent( - connection: AsyncConnection, - *, - scope_id: str, - binding_name: str, - cursor: bytes, - generation: int, -) -> bool: - values = { - "scope_id": scope_id, - "binding_name": binding_name, - "cursor": cursor, - "generation": generation, - } - if connection.dialect.name == "sqlite": - statement = sqlite_insert(SOURCE_CURSORS_TABLE).values(**values).on_conflict_do_nothing() - elif connection.dialect.name == "mysql": - # OceanBase MySQL mode accepts SAVEPOINT but does not retain it for - # RELEASE/ROLLBACK. INSERT IGNORE keeps first creation atomic without - # relying on a nested transaction; all values are validated first, so - # the only ignored error is the table's cursor identity conflict. - statement = mysql_insert(SOURCE_CURSORS_TABLE).values(**values).prefix_with("IGNORE") - else: - raise InvalidRepositoryArgumentError("dialect", f"{connection.dialect.name!r} does not support source cursors") - result = await connection.execute(statement) - return result.rowcount == 1 - - def _decode_row(row: Mapping[Any, Any]) -> StoredSourceCursor: return StoredSourceCursor( scope_id=str(row["scope_id"]), diff --git a/src/powercontext/builtin/persistence/database.py b/src/powercontext/builtin/persistence/database.py index 0fd1ff7a8..962850fee 100644 --- a/src/powercontext/builtin/persistence/database.py +++ b/src/powercontext/builtin/persistence/database.py @@ -17,12 +17,61 @@ from __future__ import annotations import asyncio -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Mapping from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext +from datetime import UTC, datetime +from typing import Any +from sqlalchemy import Table, insert +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine -from powercontext.builtin.persistence.errors import DatabaseClosedError +from powercontext.builtin.persistence.errors import DatabaseClosedError, InvalidStoredColumnError + + +async def database_now(connection: AsyncConnection, /) -> datetime: + """Return normalized UTC-naive database time for coordination decisions.""" + + statement = "SELECT UTC_TIMESTAMP(6)" if connection.dialect.name == "mysql" else "SELECT CURRENT_TIMESTAMP" + value = (await connection.exec_driver_sql(statement)).scalar_one() + if isinstance(value, str): + value = datetime.fromisoformat(value.replace("Z", "+00:00")) + if not isinstance(value, datetime): + raise InvalidStoredColumnError("CURRENT_TIMESTAMP", "a datetime") + if value.tzinfo is not None: + return value.astimezone(UTC).replace(tzinfo=None) + return value + + +async def insert_if_absent( + connection: AsyncConnection, + table: Table, + values: Mapping[str, Any], + /, +) -> bool: + """Insert validated values without aborting the transaction on a unique-key race. + + OceanBase's async dialect deliberately leaves ``do_begin`` empty. A + zero-row locking update therefore may not establish a server transaction, + which makes a SAVEPOINT-based insert race unsafe. The supported SQL + dialects provide a conflict-tolerant insert that starts the real + transaction and reports whether this caller created the row. + """ + + statement = insert(table).values(**values) + if connection.dialect.name == "mysql": + result = await connection.execute(statement.prefix_with("IGNORE")) + return result.rowcount == 1 + if connection.dialect.name == "sqlite": + result = await connection.execute(statement.prefix_with("OR IGNORE")) + return result.rowcount == 1 + + try: + async with connection.begin_nested(): + await connection.execute(statement) + except IntegrityError: + return False + return True class AsyncDatabase: @@ -32,9 +81,10 @@ class AsyncDatabase: object. Closing an attached database leaves the caller's engine available. """ - def __init__(self, engine: AsyncEngine, *, owns_engine: bool) -> None: + def __init__(self, engine: AsyncEngine, *, owns_engine: bool, serialize_transactions: bool = False) -> None: self._engine = engine self._owns_engine = owns_engine + self._transaction_lock = asyncio.Lock() if serialize_transactions else None self._closed = False self._closing = False self._active_transactions = 0 @@ -48,10 +98,10 @@ def attach(cls, engine: AsyncEngine, /) -> AsyncDatabase: return cls(engine, owns_engine=False) @classmethod - def own(cls, engine: AsyncEngine, /) -> AsyncDatabase: + def own(cls, engine: AsyncEngine, /, *, serialize_transactions: bool = False) -> AsyncDatabase: """Take disposal ownership of an already configured async engine.""" - return cls(engine, owns_engine=True) + return cls(engine, owns_engine=True, serialize_transactions=serialize_transactions) @property def engine(self) -> AsyncEngine: @@ -68,8 +118,12 @@ async def transaction(self) -> AsyncIterator[AsyncConnection]: raise DatabaseClosedError self._active_transactions += 1 try: - async with self._engine.begin() as connection: - yield connection + if self._transaction_lock is None: + async with self._engine.begin() as connection: + yield connection + else: + async with self._transaction_lock, self._engine.begin() as connection: + yield connection finally: async with self._state_changed: self._active_transactions -= 1 diff --git a/src/powercontext/builtin/persistence/memory_index.py b/src/powercontext/builtin/persistence/memory_index.py index f6fa9af9e..72d9cdedf 100644 --- a/src/powercontext/builtin/persistence/memory_index.py +++ b/src/powercontext/builtin/persistence/memory_index.py @@ -17,7 +17,7 @@ from __future__ import annotations from collections.abc import Iterable, Mapping -from typing import Any, Protocol +from typing import Any, Protocol, cast from sqlalchemy import Table from sqlalchemy.ext.asyncio import AsyncConnection @@ -100,6 +100,10 @@ async def hydrate( ) -> tuple[MemoryProjection, ...]: ... +class _SchemaVerifier(Protocol): + async def verify(self, connection: AsyncConnection, /) -> None: ... + + class NoMemoryIndex: """Truthfully expose an authoritative store without search projections.""" @@ -168,6 +172,12 @@ async def initialize(self, connection: AsyncConnection, /) -> None: for index in self.indexes: await index.initialize(connection) + async def verify(self, connection: AsyncConnection, /) -> None: + """Probe an already provisioned distributed schema without issuing DDL.""" + + for index in self.indexes: + await cast(_SchemaVerifier, index).verify(connection) + async def replace( self, connection: AsyncConnection, diff --git a/src/powercontext/builtin/persistence/migration.py b/src/powercontext/builtin/persistence/migration.py new file mode 100644 index 000000000..f373fdaba --- /dev/null +++ b/src/powercontext/builtin/persistence/migration.py @@ -0,0 +1,274 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Forward-only Alembic schema lifecycle with legacy baseline validation.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Iterable +from pathlib import Path +from uuid import uuid4 + +from alembic import command +from alembic.config import Config +from alembic.migration import MigrationContext +from alembic.script import ScriptDirectory +from sqlalchemy import Table, inspect +from sqlalchemy.engine import Connection +from sqlalchemy.engine.reflection import Inspector +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.coordination import CoordinationRepository, CoordinatorLease +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.errors import PersistenceError +from powercontext.builtin.persistence.schema import create_tables +from powercontext.builtin.persistence.tables import ( + ARTIFACT_HEADS_TABLE, + COORDINATION_TABLES, + MEMORY_TABLES, + SCHEDULER_LEASES_TABLE, + SCOPE_TABLES, + SHARED_TABLES, + STATISTICS_TABLES, + WORK_TABLES, +) + +BASELINE_REVISION = "0001_baseline" +CURRENT_SCHEMA_REVISION = "0003_scope_source_skill" +SCHEMA_VERSION_TABLE = "pc_schema_revisions" +_MIGRATION_LEASE = "schema-migration" +_MIGRATION_LEASE_SECONDS = 600 +_BASE_TABLES = SHARED_TABLES + MEMORY_TABLES + STATISTICS_TABLES +_NEW_TABLES = WORK_TABLES + COORDINATION_TABLES +_CURRENT_TABLES = SCOPE_TABLES + _BASE_TABLES + _NEW_TABLES +SchemaProvisioner = Callable[[AsyncConnection], Awaitable[None]] + + +class SchemaMigrationError(PersistenceError): + """Base class for stable migration startup failures.""" + + +class SchemaNotCurrentError(SchemaMigrationError): + """Raised when a role process starts before the migrator has completed.""" + + def __init__(self, actual: str | None) -> None: + self.actual = actual + super().__init__( + f"database schema revision is {actual or 'unversioned'}; expected {CURRENT_SCHEMA_REVISION}; " + "run `powercontext server migrate` before starting distributed roles" + ) + + +class SchemaCompatibilityError(SchemaMigrationError): + """Raised when an unversioned database cannot be safely baselined.""" + + def __init__(self, detail: str) -> None: + super().__init__(f"database schema cannot be baselined safely: {detail}") + + +class MigrationBusyError(SchemaMigrationError): + """Raised when another migrator owns the database lease.""" + + def __init__(self) -> None: + super().__init__("another schema migrator currently owns the database lease") + + +async def migrate_database( + database: AsyncDatabase, + *, + provision: SchemaProvisioner | None = None, +) -> str: + """Validate/stamp legacy state and upgrade through the current revision.""" + + lease = await _acquire_migration_lease(database) + try: + async with database.transaction() as connection: + await connection.run_sync(_prepare_legacy_revision) + await connection.run_sync(_upgrade_to_head) + if provision is not None: + await provision(connection) + actual = await connection.run_sync(_current_revision) + await connection.run_sync(_validate_current_schema) + if actual != CURRENT_SCHEMA_REVISION: + raise SchemaNotCurrentError(actual) + return actual + finally: + await _release_migration_lease(database, lease) + + +async def require_current_schema(database: AsyncDatabase) -> str: + """Read-only startup guard used by every distributed role.""" + + async with database.transaction() as connection: + actual = await connection.run_sync(_current_revision) + if actual == CURRENT_SCHEMA_REVISION: + await connection.run_sync(_validate_current_schema) + if actual != CURRENT_SCHEMA_REVISION: + raise SchemaNotCurrentError(actual) + return actual + + +async def _acquire_migration_lease(database: AsyncDatabase) -> CoordinatorLease: + # OceanBase/MySQL DDL commits the active transaction implicitly. Keep the + # one-time coordination-table bootstrap in its own transaction so a fresh + # database never enters the lease repository with SQLAlchemy's transaction + # state out of sync with the server (notably before its SAVEPOINT insert). + async with database.transaction() as connection: + await create_tables(connection, (SCHEDULER_LEASES_TABLE,)) + + async with database.transaction() as connection: + lease = await CoordinationRepository().acquire_lease( + connection, + lease_name=_MIGRATION_LEASE, + owner_id=uuid4().hex, + lease_seconds=_MIGRATION_LEASE_SECONDS, + ) + if lease is None: + raise MigrationBusyError + return lease + + +async def _release_migration_lease(database: AsyncDatabase, lease: CoordinatorLease) -> None: + try: + async with database.transaction() as connection: + await CoordinationRepository().release_lease(connection, lease) + except SQLAlchemyError: + # The lease expires by database time. Never mask the migration result + # with a best-effort release failure during connection loss. + return + + +def _prepare_legacy_revision(connection: Connection) -> None: + actual = _current_revision(connection) + if actual is not None: + return + inspector = inspect(connection) + existing = set(inspector.get_table_names()) + existing.discard(SCHEDULER_LEASES_TABLE.name) + existing.discard(SCHEMA_VERSION_TABLE) + powercontext_tables = {name for name in existing if name.startswith("pc_")} + if not powercontext_tables: + return + + required = {table.name for table in _BASE_TABLES} + missing = sorted(required - powercontext_tables) + if missing: + raise SchemaCompatibilityError( # noqa: TRY003 + f"missing baseline tables: {', '.join(missing)}" + ) + _upgrade_known_legacy_columns(connection, inspector) + inspector = inspect(connection) + _require_expected_columns(inspector, _BASE_TABLES) + + new_names = {table.name for table in _NEW_TABLES if table is not SCHEDULER_LEASES_TABLE} + present_new = new_names & powercontext_tables + if present_new and present_new != new_names: + missing_new = sorted(new_names - present_new) + raise SchemaCompatibilityError( # noqa: TRY003 + f"partially installed work-ledger tables: {', '.join(missing_new)}" + ) + if present_new: + _require_expected_columns(inspector, _NEW_TABLES) + command.stamp(_alembic_config(connection), CURRENT_SCHEMA_REVISION if present_new else BASELINE_REVISION) + + +def _upgrade_known_legacy_columns(connection: Connection, inspector: Inspector) -> None: + """Apply narrowly recognized pre-baseline expansions before stamping.""" + + table_name = ARTIFACT_HEADS_TABLE.name + if table_name not in set(inspector.get_table_names()): + return + columns = {str(column["name"]) for column in inspector.get_columns(table_name)} + mysql = connection.dialect.name in {"mysql", "oceanbase"} + additions = { + "searchable_text": "MEDIUMTEXT NULL" if mysql else "TEXT NULL", + "lifecycle_state": "VARCHAR(16) NOT NULL DEFAULT 'active'", + "replacement_artifact_id": "VARCHAR(128) NULL", + "governance_generation": "BIGINT NOT NULL DEFAULT 0", + } + for name, definition in additions.items(): + if name not in columns: + connection.exec_driver_sql(f"ALTER TABLE {table_name} ADD COLUMN {name} {definition}") + + +def _require_expected_columns(inspector: Inspector, tables: Iterable[Table]) -> None: + for table in tables: + actual = {str(column["name"]) for column in inspector.get_columns(table.name)} + expected = {column.name for column in table.columns} + missing = sorted(expected - actual) + if missing: + raise SchemaCompatibilityError( # noqa: TRY003 + f"table {table.name} is missing columns: {', '.join(missing)}" + ) + + +def _validate_current_schema(connection: Connection) -> None: + inspector = inspect(connection) + existing = set(inspector.get_table_names()) + expected = _CURRENT_TABLES + missing_tables = sorted(table.name for table in expected if table.name not in existing) + if missing_tables: + raise SchemaCompatibilityError(f"current revision is missing tables: {', '.join(missing_tables)}") # noqa: TRY003 + _require_expected_columns(inspector, expected) + + +def _upgrade_to_head(connection: Connection) -> None: + command.upgrade(_alembic_config(connection), "head") + + +def _current_revision(connection: Connection) -> str | None: + tables = set(inspect(connection).get_table_names()) + if SCHEMA_VERSION_TABLE not in tables: + return None + context = MigrationContext.configure(connection, opts={"version_table": SCHEMA_VERSION_TABLE}) + return context.get_current_revision() + + +def _alembic_config(connection: Connection | None = None) -> Config: + config = Config() + config.set_main_option("script_location", str(Path(__file__).with_name("migrations"))) + if connection is not None: + config.attributes["connection"] = connection + return config + + +def migration_head() -> str: + """Return the packaged Alembic head for contract tests and diagnostics.""" + + return str(ScriptDirectory.from_config(_alembic_config()).get_current_head()) + + +def baseline_tables() -> tuple[Table, ...]: + """Return the explicitly managed migration baseline.""" + + # Search projections and optional feature tables are provisioned by the + # configured migrator callback. Keeping the baseline limited to the + # invariant domain schema lets disabled features stay physically absent. + return _BASE_TABLES + + +__all__ = [ + "BASELINE_REVISION", + "CURRENT_SCHEMA_REVISION", + "SCHEMA_VERSION_TABLE", + "MigrationBusyError", + "SchemaCompatibilityError", + "SchemaMigrationError", + "SchemaNotCurrentError", + "baseline_tables", + "migrate_database", + "migration_head", + "require_current_schema", +] diff --git a/src/powercontext/builtin/persistence/migrations/__init__.py b/src/powercontext/builtin/persistence/migrations/__init__.py new file mode 100644 index 000000000..ca02bcd2d --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Packaged Alembic revisions for PowerContext persistence.""" diff --git a/src/powercontext/builtin/persistence/migrations/env.py b/src/powercontext/builtin/persistence/migrations/env.py new file mode 100644 index 000000000..cac14b1e6 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/env.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from alembic import context + +from powercontext.builtin.persistence.migration import SCHEMA_VERSION_TABLE + + +def run_migrations() -> None: + connection = context.config.attributes.get("connection") + if connection is None: + raise RuntimeError("PowerContext migrations require a caller-owned connection") # noqa: TRY003 + context.configure( + connection=connection, + target_metadata=None, + version_table=SCHEMA_VERSION_TABLE, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +run_migrations() diff --git a/src/powercontext/builtin/persistence/migrations/script.py.mako b/src/powercontext/builtin/persistence/migrations/script.py.mako new file mode 100644 index 000000000..6522ff221 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/script.py.mako @@ -0,0 +1,32 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""${message}""" + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + raise NotImplementedError("PowerContext schema migrations are forward-only") diff --git a/src/powercontext/builtin/persistence/migrations/versions/0001_baseline.py b/src/powercontext/builtin/persistence/migrations/versions/0001_baseline.py new file mode 100644 index 000000000..ae3db5952 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/versions/0001_baseline.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create the validated PowerContext baseline schema.""" + +from __future__ import annotations + +from collections import defaultdict + +from alembic import op +from sqlalchemy import MetaData, Table + +from powercontext.builtin.persistence.migration import baseline_tables + +revision = "0001_baseline" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + bind = op.get_bind() + grouped: dict[MetaData, list[Table]] = defaultdict(list) + for table in baseline_tables(): + grouped[table.metadata].append(table) + for metadata, tables in grouped.items(): + metadata.create_all(bind, tables=tables, checkfirst=True) + + +def downgrade() -> None: + raise NotImplementedError("PowerContext schema migrations are forward-only") diff --git a/src/powercontext/builtin/persistence/migrations/versions/0002_work_ledger.py b/src/powercontext/builtin/persistence/migrations/versions/0002_work_ledger.py new file mode 100644 index 000000000..825f19681 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/versions/0002_work_ledger.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add durable work, scheduler coordination, membership, and rate limiting.""" + +from __future__ import annotations + +from alembic import op + +from powercontext.builtin.persistence.tables import COORDINATION_TABLES, SHARED_METADATA, WORK_TABLES + +revision = "0002_work_ledger" +down_revision = "0001_baseline" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + SHARED_METADATA.create_all( + op.get_bind(), + tables=(*WORK_TABLES, *COORDINATION_TABLES), + checkfirst=True, + ) + + +def downgrade() -> None: + raise NotImplementedError("PowerContext schema migrations are forward-only") diff --git a/src/powercontext/builtin/persistence/migrations/versions/0003_scope_source_skill.py b/src/powercontext/builtin/persistence/migrations/versions/0003_scope_source_skill.py new file mode 100644 index 000000000..aa821a4da --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/versions/0003_scope_source_skill.py @@ -0,0 +1,74 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add Scope, Source Definition, publication, and Skill distribution state.""" + +from __future__ import annotations + +from alembic import op +from sqlalchemy import BigInteger, Column, inspect + +from powercontext.builtin.persistence.tables import ( + AGENT_SKILL_TARGETS_TABLE, + ARTIFACT_PUBLICATIONS_TABLE, + CONNECTOR_CHECKPOINTS_TABLE, + SCOPE_TABLES, + SHARED_METADATA, + SKILL_PACKAGES_TABLE, + SKILL_PUBLICATIONS_TABLE, + SOURCE_DEFINITION_MANIFESTS_TABLE, + identity_string, +) +from powercontext.limits import MAX_ARTIFACT_ID_LENGTH + +revision = "0003_scope_source_skill" +down_revision = "0002_work_ledger" +branch_labels = None +depends_on = None + +_NEW_TABLES = ( + *SCOPE_TABLES, + ARTIFACT_PUBLICATIONS_TABLE, + CONNECTOR_CHECKPOINTS_TABLE, + SOURCE_DEFINITION_MANIFESTS_TABLE, + SKILL_PACKAGES_TABLE, + AGENT_SKILL_TARGETS_TABLE, + SKILL_PUBLICATIONS_TABLE, +) + + +def upgrade() -> None: + bind = op.get_bind() + SHARED_METADATA.create_all(bind, tables=_NEW_TABLES, checkfirst=True) + + columns = {str(column["name"]) for column in inspect(bind).get_columns("pc_artifact_heads")} + if "lifecycle_state" not in columns: + op.add_column( + "pc_artifact_heads", + Column("lifecycle_state", identity_string(16), nullable=False, server_default="active"), + ) + if "replacement_artifact_id" not in columns: + op.add_column( + "pc_artifact_heads", + Column("replacement_artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH)), + ) + if "governance_generation" not in columns: + op.add_column( + "pc_artifact_heads", + Column("governance_generation", BigInteger, nullable=False, server_default="0"), + ) + + +def downgrade() -> None: + raise NotImplementedError("PowerContext schema migrations are forward-only") diff --git a/src/powercontext/builtin/persistence/oceanbase/experience_index.py b/src/powercontext/builtin/persistence/oceanbase/experience_index.py index 1736e6571..07b2ec9ca 100644 --- a/src/powercontext/builtin/persistence/oceanbase/experience_index.py +++ b/src/powercontext/builtin/persistence/oceanbase/experience_index.py @@ -49,6 +49,15 @@ AND index_name = :index_name """ ) +_SEARCHABLE_TEXT_EXISTS_SQL = text( + """ + SELECT COUNT(*) + FROM information_schema.columns + WHERE table_schema = DATABASE() + AND table_name = 'pc_artifact_heads' + AND column_name = 'searchable_text' + """ +) class OceanBaseExperienceFTSIndex: @@ -66,6 +75,27 @@ async def initialize(self, connection: AsyncConnection, /) -> None: ) if count == 0: await connection.exec_driver_sql(_OCEANBASE_CREATE_FTS_SQL) + await self.verify(connection) + + async def verify(self, connection: AsyncConnection, /) -> None: + """Require the migrator-provisioned Experience projection without DDL.""" + + if connection.dialect.name != "mysql": + raise CapabilityNotSupportedError("oceanbase-experience-fts") + if int(await connection.scalar(_SEARCHABLE_TEXT_EXISTS_SQL) or 0) == 0: + raise CapabilityNotSupportedError( + "oceanbase-experience-fts", + "search projection column is missing; run `powercontext server migrate`", + ) + count = await connection.scalar( + _OCEANBASE_FTS_INDEX_EXISTS_SQL, + {"index_name": _OCEANBASE_FTS_INDEX_NAME}, + ) + if count == 0: + raise CapabilityNotSupportedError( + "oceanbase-experience-fts", + "index is missing; run `powercontext server migrate`", + ) probe = match(ARTIFACT_HEADS_TABLE.c.searchable_text, against="powercontext") await connection.execute(select(ARTIFACT_HEADS_TABLE.c.artifact_id).where(probe).limit(1)) diff --git a/src/powercontext/builtin/persistence/oceanbase/memory_index.py b/src/powercontext/builtin/persistence/oceanbase/memory_index.py index d1dc5b81f..db9024f3c 100644 --- a/src/powercontext/builtin/persistence/oceanbase/memory_index.py +++ b/src/powercontext/builtin/persistence/oceanbase/memory_index.py @@ -77,6 +77,15 @@ AND column_name = 'embedding' """ ) +_OCEANBASE_VECTOR_INDEX_EXISTS_SQL = text( + """ + SELECT COUNT(*) + FROM information_schema.statistics + WHERE table_schema = DATABASE() + AND table_name = :table_name + AND index_name = :index_name + """ +) _OCEANBASE_VECTOR_SEARCH_SQL = """ SELECT m.memory_artifact_id, @@ -112,6 +121,19 @@ async def initialize(self, connection: AsyncConnection, /) -> None: ) if count == 0: await connection.exec_driver_sql(_OCEANBASE_CREATE_FTS_SQL) + await self.verify(connection) + + async def verify(self, connection: AsyncConnection, /) -> None: + """Require the migrator-provisioned FTS index without creating it.""" + + if connection.dialect.name != "mysql": + raise CapabilityNotSupportedError("oceanbase-fts") + count = await connection.scalar( + _OCEANBASE_FTS_INDEX_EXISTS_SQL, + {"index_name": _OCEANBASE_FTS_INDEX_NAME}, + ) + if count == 0: + raise CapabilityNotSupportedError("oceanbase-fts", "index is missing; run `powercontext server migrate`") probe = match(MEMORY_ENTRY_HEADS_TABLE.c.searchable_text, against="powercontext") await connection.execute(select(MEMORY_ENTRY_HEADS_TABLE.c.entry_version_id).where(probe).limit(1)) @@ -254,6 +276,28 @@ async def initialize(self, connection: AsyncConnection, /) -> None: ) await connection.run_sync(lambda sync_connection: self._vector_index.create(sync_connection, checkfirst=True)) + async def verify(self, connection: AsyncConnection, /) -> None: + """Require the configured vector column and index without issuing DDL.""" + + if connection.dialect.name != "mysql": + raise CapabilityNotSupportedError("oceanbase-vector") + column_type = await connection.scalar( + _OCEANBASE_VECTOR_TYPE_SQL, + {"table_name": _OCEANBASE_VECTOR_TABLE_NAME}, + ) + expected = f"VECTOR({self.profile.dimension})" + if str(column_type).upper() != expected: + raise CapabilityNotSupportedError( + "vector", + f"OceanBase projection uses {column_type!r}; expected {expected}", + ) + count = await connection.scalar( + _OCEANBASE_VECTOR_INDEX_EXISTS_SQL, + {"table_name": _OCEANBASE_VECTOR_TABLE_NAME, "index_name": _OCEANBASE_VECTOR_INDEX_NAME}, + ) + if count == 0: + raise CapabilityNotSupportedError("oceanbase-vector", "index is missing; run `powercontext server migrate`") + async def replace( self, connection: AsyncConnection, diff --git a/src/powercontext/builtin/persistence/oceanbase/profile.py b/src/powercontext/builtin/persistence/oceanbase/profile.py index 4a1133f98..146ed7272 100644 --- a/src/powercontext/builtin/persistence/oceanbase/profile.py +++ b/src/powercontext/builtin/persistence/oceanbase/profile.py @@ -120,6 +120,7 @@ async def open( config: OceanBaseConfig, *, tables: tuple[Table, ...], + create_schema: bool = True, ) -> AsyncIterator[OceanBaseProfile]: """Create, initialize and exclusively own an official OceanBase engine.""" @@ -131,7 +132,7 @@ async def open( pool_pre_ping=config.pool_pre_ping, ) database = AsyncDatabase.own(engine) - async with _initialized_profile(cls(database=database, tables=tables)) as profile: + async with _initialized_profile(cls(database=database, tables=tables), create_schema=create_schema) as profile: yield profile @classmethod @@ -141,22 +142,28 @@ async def attach( engine: AsyncEngine, *, tables: tuple[Table, ...], + create_schema: bool = True, ) -> AsyncIterator[OceanBaseProfile]: """Initialize a caller-owned official OceanBase engine without disposing it.""" _validate_oceanbase_url(engine.url) database = AsyncDatabase.attach(engine) - async with _initialized_profile(cls(database=database, tables=tables)) as profile: + async with _initialized_profile(cls(database=database, tables=tables), create_schema=create_schema) as profile: yield profile @asynccontextmanager -async def _initialized_profile(profile: OceanBaseProfile) -> AsyncIterator[OceanBaseProfile]: +async def _initialized_profile( + profile: OceanBaseProfile, + *, + create_schema: bool, +) -> AsyncIterator[OceanBaseProfile]: try: async with profile.database.transaction() as connection: await _require_mysql_tenant(connection) await _require_compatible_identity_collations(connection, profile.tables) - await create_tables(connection, profile.tables) + if create_schema: + await create_tables(connection, profile.tables) yield profile finally: await profile.database.close() diff --git a/src/powercontext/builtin/persistence/rate_limit.py b/src/powercontext/builtin/persistence/rate_limit.py new file mode 100644 index 000000000..a59c5131b --- /dev/null +++ b/src/powercontext/builtin/persistence/rate_limit.py @@ -0,0 +1,216 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Database-time fixed-window rate limiting for horizontally scaled APIs.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy import delete, select, update +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.database import database_now, insert_if_absent +from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError, InvalidStoredColumnError +from powercontext.builtin.persistence.tables import RATE_LIMIT_WINDOWS_TABLE + + +@dataclass(frozen=True, slots=True) +class RateLimitDecision: + """One atomic counter decision.""" + + allowed: bool + remaining: int + retry_after_seconds: int + + +class RateLimitRepository: + """Increment one shared fixed window under a row lock.""" + + async def consume( + self, + connection: AsyncConnection, + *, + principal_key: str, + policy_id: str, + limit: int, + window_seconds: int, + ) -> RateLimitDecision: + _require_digest(principal_key) + _require_text("policy_id", policy_id, 64) + _require_positive("limit", limit) + _require_positive("window_seconds", window_seconds) + now = await database_now(connection) + window_start = _window_start(now, window_seconds) + expires_at = window_start + timedelta(seconds=window_seconds) + row, created = await _lock_or_create_window( + connection, + principal_key=principal_key, + policy_id=policy_id, + window_start=window_start, + expires_at=expires_at, + ) + count = _positive_integer(row["request_count"], "request_count") + retry_after = max(1, math.ceil((expires_at - now).total_seconds())) + if created: + return RateLimitDecision( + allowed=True, + remaining=max(limit - count, 0), + retry_after_seconds=retry_after, + ) + if count >= limit: + return RateLimitDecision(allowed=False, remaining=0, retry_after_seconds=retry_after) + next_count = count + 1 + await connection.execute( + update(RATE_LIMIT_WINDOWS_TABLE) + .where( + RATE_LIMIT_WINDOWS_TABLE.c.principal_key == principal_key, + RATE_LIMIT_WINDOWS_TABLE.c.policy_id == policy_id, + RATE_LIMIT_WINDOWS_TABLE.c.window_started_at == window_start, + ) + .values(request_count=next_count) + ) + return RateLimitDecision( + allowed=True, + remaining=max(limit - next_count, 0), + retry_after_seconds=retry_after, + ) + + async def purge_expired(self, connection: AsyncConnection, /, *, limit: int = 500) -> int: + """Delete one bounded batch of expired fixed-window counters.""" + + if limit < 1 or limit > 500: + raise InvalidRepositoryArgumentError("limit", "must be between 1 and 500") + now = await database_now(connection) + rows = ( + await connection.execute( + select( + RATE_LIMIT_WINDOWS_TABLE.c.principal_key, + RATE_LIMIT_WINDOWS_TABLE.c.policy_id, + RATE_LIMIT_WINDOWS_TABLE.c.window_started_at, + ) + .where(RATE_LIMIT_WINDOWS_TABLE.c.expires_at <= now) + .order_by(RATE_LIMIT_WINDOWS_TABLE.c.expires_at) + .limit(limit) + ) + ).all() + deleted = 0 + for principal_key, policy_id, window_started_at in rows: + result = await connection.execute( + delete(RATE_LIMIT_WINDOWS_TABLE).where( + RATE_LIMIT_WINDOWS_TABLE.c.principal_key == principal_key, + RATE_LIMIT_WINDOWS_TABLE.c.policy_id == policy_id, + RATE_LIMIT_WINDOWS_TABLE.c.window_started_at == window_started_at, + RATE_LIMIT_WINDOWS_TABLE.c.expires_at <= now, + ) + ) + deleted += result.rowcount + return deleted + + +async def _lock_or_create_window( + connection: AsyncConnection, + *, + principal_key: str, + policy_id: str, + window_start: datetime, + expires_at: datetime, +) -> tuple[Mapping[Any, Any], bool]: + row = await _lock_window(connection, principal_key, policy_id, window_start) + if row is not None: + return row, False + created = await insert_if_absent( + connection, + RATE_LIMIT_WINDOWS_TABLE, + { + "principal_key": principal_key, + "policy_id": policy_id, + "window_started_at": window_start, + "request_count": 1, + "expires_at": expires_at, + }, + ) + row = await _lock_window(connection, principal_key, policy_id, window_start) + if row is None: + raise InvalidStoredColumnError( # noqa: TRY003 + "rate limit window", + "an initialized fixed window", + ) + return row, created + + +async def _lock_window( + connection: AsyncConnection, + principal_key: str, + policy_id: str, + window_start: datetime, +) -> Mapping[Any, Any] | None: + await connection.execute( + update(RATE_LIMIT_WINDOWS_TABLE) + .where( + RATE_LIMIT_WINDOWS_TABLE.c.principal_key == principal_key, + RATE_LIMIT_WINDOWS_TABLE.c.policy_id == policy_id, + RATE_LIMIT_WINDOWS_TABLE.c.window_started_at == window_start, + ) + .values(request_count=RATE_LIMIT_WINDOWS_TABLE.c.request_count) + ) + return ( + ( + await connection.execute( + select(RATE_LIMIT_WINDOWS_TABLE) + .where( + RATE_LIMIT_WINDOWS_TABLE.c.principal_key == principal_key, + RATE_LIMIT_WINDOWS_TABLE.c.policy_id == policy_id, + RATE_LIMIT_WINDOWS_TABLE.c.window_started_at == window_start, + ) + .with_for_update() + ) + ) + .mappings() + .one_or_none() + ) + + +def _window_start(now: datetime, window_seconds: int) -> datetime: + aware = now.replace(tzinfo=UTC) if now.tzinfo is None else now.astimezone(UTC) + epoch = int(aware.timestamp()) + return datetime.fromtimestamp(epoch - epoch % window_seconds, tz=UTC).replace(tzinfo=None) + + +def _require_digest(value: str) -> None: + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise InvalidRepositoryArgumentError("principal_key", "must be a lowercase SHA-256 digest") + + +def _require_text(field: str, value: str, maximum: int) -> None: + if not value.strip() or value != value.strip() or len(value) > maximum: + raise InvalidRepositoryArgumentError(field, f"must be trimmed and contain at most {maximum} characters") + + +def _require_positive(field: str, value: int) -> None: + if isinstance(value, bool) or value < 1: + raise InvalidRepositoryArgumentError(field, "must be a positive integer") + + +def _positive_integer(value: object, column: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidStoredColumnError(column, "a positive integer") + return value + + +__all__ = ["RateLimitDecision", "RateLimitRepository"] diff --git a/src/powercontext/builtin/persistence/seekdb/profile.py b/src/powercontext/builtin/persistence/seekdb/profile.py index 1a78b2298..2878bbf20 100644 --- a/src/powercontext/builtin/persistence/seekdb/profile.py +++ b/src/powercontext/builtin/persistence/seekdb/profile.py @@ -100,6 +100,7 @@ async def open( config: SeekDBConfig, *, tables: tuple[Table, ...], + create_schema: bool = True, ) -> AsyncIterator[SeekDBProfile]: """Start seekDB locally and connect through its async Unix socket.""" @@ -112,8 +113,9 @@ async def open( database = AsyncDatabase.own(engine) profile = cls(database=database, tables=tables) try: - async with database.transaction() as connection: - await create_tables(connection, tables) + if create_schema: + async with database.transaction() as connection: + await create_tables(connection, tables) yield profile finally: close_task = asyncio.create_task(database.close()) diff --git a/src/powercontext/builtin/persistence/sqlite/profile.py b/src/powercontext/builtin/persistence/sqlite/profile.py index 40836c739..a202e73f2 100644 --- a/src/powercontext/builtin/persistence/sqlite/profile.py +++ b/src/powercontext/builtin/persistence/sqlite/profile.py @@ -84,6 +84,7 @@ async def open( *, tables: tuple[Table, ...], load_vector_extension: bool = False, + create_schema: bool = True, ) -> AsyncIterator[SQLiteProfile]: """Create, initialize and exclusively own one SQLite engine.""" @@ -93,12 +94,13 @@ async def open( engine_options["poolclass"] = StaticPool engine = create_async_engine(config.url, **engine_options) _configure_sqlite(engine, config, load_vector_extension=load_vector_extension) - database = AsyncDatabase.own(engine) + database = AsyncDatabase.own(engine, serialize_transactions=config.is_in_memory) profile = cls(database=database, tables=tables) try: await _warm_sqlite(engine, config) - async with database.transaction() as connection: - await create_tables(connection, tables) + if create_schema: + async with database.transaction() as connection: + await create_tables(connection, tables) yield profile finally: await database.close() diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 2b01ce9d8..ccc16facf 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -22,6 +22,7 @@ Date, DateTime, ForeignKeyConstraint, + Index, Integer, LargeBinary, MetaData, @@ -30,7 +31,7 @@ Text, UniqueConstraint, ) -from sqlalchemy.dialects.mysql import MEDIUMBLOB, MEDIUMTEXT, VARCHAR +from sqlalchemy.dialects.mysql import DATETIME, MEDIUMBLOB, MEDIUMTEXT, VARCHAR from powercontext.limits import ( MAX_ARTIFACT_FAMILY_LENGTH, @@ -85,6 +86,12 @@ def _entry_text_type(): return Text().with_variant(MEDIUMTEXT(), "mysql") +def _coordination_timestamp_type(): + """Preserve the microsecond precision used by database-time leases.""" + + return DateTime(timezone=False).with_variant(DATETIME(fsp=6), "mysql") + + SCOPES_TABLE = Table( "pc_scopes", SHARED_METADATA, @@ -641,6 +648,163 @@ def _entry_text_type(): ) +WORK_LANES_TABLE = Table( + "pc_work_lanes", + SHARED_METADATA, + Column("lane_key", identity_string(64), primary_key=True), + Column("head_sequence", BigInteger), + Column("next_sequence", BigInteger, nullable=False), + CheckConstraint("head_sequence IS NULL OR head_sequence > 0", name="ck_pc_work_lanes_head_positive"), + CheckConstraint("next_sequence > 0", name="ck_pc_work_lanes_next_positive"), +) + +WORK_ITEMS_TABLE = Table( + "pc_work_items", + SHARED_METADATA, + Column("work_id", identity_string(36), primary_key=True), + Column("logical_key", identity_string(64), nullable=False), + Column("lane_key", identity_string(64), nullable=False), + Column("lane_sequence", BigInteger, nullable=False), + Column("kind", identity_string(128), nullable=False), + Column("payload_version", Integer, nullable=False), + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), nullable=False), + Column("payload", _canonical_payload_type(), nullable=False), + Column("status", identity_string(16), nullable=False), + Column("available_at", _coordination_timestamp_type(), nullable=False), + Column("lease_owner", identity_string(128)), + Column("lease_fence", BigInteger, nullable=False), + Column("lease_expires_at", _coordination_timestamp_type()), + Column("attempt_count", Integer, nullable=False), + Column("generation_attempt_count", Integer, nullable=False), + Column("recovery_generation", Integer, nullable=False), + Column("max_attempts", Integer, nullable=False), + Column("cancel_requested", Boolean, nullable=False), + Column("result_code", identity_string(128)), + Column("result_payload", _canonical_payload_type()), + Column("error_category", identity_string(64)), + Column("error_code", identity_string(128)), + Column("state_version", BigInteger, nullable=False), + Column("created_at", _coordination_timestamp_type(), nullable=False), + Column("updated_at", _coordination_timestamp_type(), nullable=False), + Column("completed_at", _coordination_timestamp_type()), + UniqueConstraint("lane_key", "lane_sequence", name="uq_pc_work_items_lane_sequence"), + CheckConstraint( + "status IN ('queued', 'running', 'retry_wait', 'cancelling', 'succeeded', 'failed', 'cancelled')", + name="ck_pc_work_items_status", + ), + CheckConstraint("lane_sequence > 0", name="ck_pc_work_items_lane_sequence_positive"), + CheckConstraint("payload_version > 0", name="ck_pc_work_items_payload_version_positive"), + CheckConstraint("lease_fence >= 0", name="ck_pc_work_items_fence_nonnegative"), + CheckConstraint("attempt_count >= 0", name="ck_pc_work_items_attempts_nonnegative"), + CheckConstraint("generation_attempt_count >= 0", name="ck_pc_work_items_generation_attempts_nonnegative"), + CheckConstraint("recovery_generation >= 0", name="ck_pc_work_items_recovery_nonnegative"), + CheckConstraint("max_attempts > 0", name="ck_pc_work_items_max_attempts_positive"), + CheckConstraint("state_version > 0", name="ck_pc_work_items_state_version_positive"), +) +Index("ix_pc_work_items_claim", WORK_ITEMS_TABLE.c.status, WORK_ITEMS_TABLE.c.available_at) +Index("ix_pc_work_items_lane_status", WORK_ITEMS_TABLE.c.lane_key, WORK_ITEMS_TABLE.c.status) +Index("ix_pc_work_items_completed", WORK_ITEMS_TABLE.c.completed_at) +Index("ix_pc_work_items_scope_created", WORK_ITEMS_TABLE.c.scope_id, WORK_ITEMS_TABLE.c.created_at) + +WORK_KEYS_TABLE = Table( + "pc_work_keys", + SHARED_METADATA, + Column("logical_key", identity_string(64), primary_key=True), + Column("work_id", identity_string(36), nullable=False, unique=True), + Column("lane_key", identity_string(64), nullable=False), + Column("created_at", _coordination_timestamp_type(), nullable=False), +) + +WORK_ATTEMPTS_TABLE = Table( + "pc_work_attempts", + SHARED_METADATA, + Column("work_id", identity_string(36), primary_key=True), + Column("attempt_no", Integer, primary_key=True), + Column("recovery_generation", Integer, nullable=False), + Column("owner_id", identity_string(128), nullable=False), + Column("fence", BigInteger, nullable=False), + Column("started_at", _coordination_timestamp_type(), nullable=False), + Column("heartbeat_at", _coordination_timestamp_type(), nullable=False), + Column("finished_at", _coordination_timestamp_type()), + Column("outcome", identity_string(32)), + Column("error_category", identity_string(64)), + Column("error_code", identity_string(128)), + Column("trace_id", identity_string(32)), + Column("span_id", identity_string(16)), + CheckConstraint("attempt_no > 0", name="ck_pc_work_attempts_attempt_positive"), + CheckConstraint("recovery_generation >= 0", name="ck_pc_work_attempts_recovery_nonnegative"), + CheckConstraint("fence > 0", name="ck_pc_work_attempts_fence_positive"), +) +Index("ix_pc_work_attempts_finished", WORK_ATTEMPTS_TABLE.c.finished_at) + +WORK_TABLES = ( + WORK_LANES_TABLE, + WORK_ITEMS_TABLE, + WORK_KEYS_TABLE, + WORK_ATTEMPTS_TABLE, +) + +SCHEDULER_LEASES_TABLE = Table( + "pc_scheduler_leases", + SHARED_METADATA, + Column("lease_name", identity_string(64), primary_key=True), + Column("owner_id", identity_string(128)), + Column("fence", BigInteger, nullable=False), + Column("lease_expires_at", _coordination_timestamp_type()), + Column("updated_at", _coordination_timestamp_type(), nullable=False), + CheckConstraint("fence >= 0", name="ck_pc_scheduler_leases_fence_nonnegative"), +) + +SCHEDULER_SCANS_TABLE = Table( + "pc_scheduler_scans", + SHARED_METADATA, + Column("discoverer", identity_string(128), primary_key=True), + Column("next_run_at", _coordination_timestamp_type(), nullable=False), + Column("continuation", identity_string(MAX_SCOPE_ID_LENGTH)), + Column("state_version", BigInteger, nullable=False), + Column("updated_at", _coordination_timestamp_type(), nullable=False), + CheckConstraint("state_version > 0", name="ck_pc_scheduler_scans_version_positive"), +) + +RUNTIME_MEMBERS_TABLE = Table( + "pc_runtime_members", + SHARED_METADATA, + Column("member_id", identity_string(128), primary_key=True), + Column("role", identity_string(16), nullable=False), + Column("build_version", identity_string(64), nullable=False), + Column("schema_min", Integer, nullable=False), + Column("schema_max", Integer, nullable=False), + Column("payload_min", Integer, nullable=False), + Column("payload_max", Integer, nullable=False), + Column("behavior_revision", identity_string(128), nullable=False), + Column("heartbeat_at", _coordination_timestamp_type(), nullable=False), + Column("expires_at", _coordination_timestamp_type(), nullable=False), + CheckConstraint("role IN ('all', 'api', 'scheduler', 'worker')", name="ck_pc_runtime_members_role"), + CheckConstraint("schema_min > 0 AND schema_max >= schema_min", name="ck_pc_runtime_members_schema_range"), + CheckConstraint("payload_min > 0 AND payload_max >= payload_min", name="ck_pc_runtime_members_payload_range"), +) +Index("ix_pc_runtime_members_role_expiry", RUNTIME_MEMBERS_TABLE.c.role, RUNTIME_MEMBERS_TABLE.c.expires_at) + +RATE_LIMIT_WINDOWS_TABLE = Table( + "pc_rate_limit_windows", + SHARED_METADATA, + Column("principal_key", identity_string(64), primary_key=True), + Column("policy_id", identity_string(64), primary_key=True), + Column("window_started_at", _coordination_timestamp_type(), primary_key=True), + Column("request_count", BigInteger, nullable=False), + Column("expires_at", _coordination_timestamp_type(), nullable=False), + CheckConstraint("request_count > 0", name="ck_pc_rate_limit_windows_count_positive"), +) +Index("ix_pc_rate_limit_windows_expiry", RATE_LIMIT_WINDOWS_TABLE.c.expires_at) + +COORDINATION_TABLES = ( + SCHEDULER_LEASES_TABLE, + SCHEDULER_SCANS_TABLE, + RUNTIME_MEMBERS_TABLE, + RATE_LIMIT_WINDOWS_TABLE, +) + + MAX_MEMORY_ENTRY_ID_LENGTH = 128 MAX_MEMORY_ENTRY_KIND_LENGTH = 128 MAX_MEMORY_HASH_LENGTH = 64 @@ -732,4 +896,4 @@ def _entry_text_type(): STATISTICS_TABLES = (MODEL_USAGE_DAILY_TABLE, RECALL_TOKEN_DAILY_TABLE) -BUILTIN_TABLES = SCOPE_TABLES + SHARED_TABLES + MEMORY_TABLES + STATISTICS_TABLES +BUILTIN_TABLES = SCOPE_TABLES + SHARED_TABLES + MEMORY_TABLES + STATISTICS_TABLES + WORK_TABLES + COORDINATION_TABLES diff --git a/src/powercontext/builtin/persistence/work.py b/src/powercontext/builtin/persistence/work.py new file mode 100644 index 000000000..36cd4d1fd --- /dev/null +++ b/src/powercontext/builtin/persistence/work.py @@ -0,0 +1,1375 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Durable work ledger with lane serialization, leases, and fencing.""" + +from __future__ import annotations + +import json +from collections.abc import Awaitable, Callable, Mapping +from contextlib import suppress +from datetime import UTC, datetime, timedelta +from enum import StrEnum +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator +from sqlalchemy import and_, delete, false, func, insert, not_, or_, select, update +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.codec import stored_bytes +from powercontext.builtin.persistence.database import database_now, insert_if_absent +from powercontext.builtin.persistence.errors import ( + InvalidRepositoryArgumentError, + InvalidStoredColumnError, + InvalidStoredPayloadError, + RepositoryError, + RepositoryNotFoundError, +) +from powercontext.builtin.persistence.tables import ( + WORK_ATTEMPTS_TABLE, + WORK_ITEMS_TABLE, + WORK_KEYS_TABLE, + WORK_LANES_TABLE, +) +from powercontext.limits import MAX_SCOPE_ID_LENGTH + +if TYPE_CHECKING: + from powercontext.builtin.runtime.work_observability import WorkObserver + +_MAX_PAYLOAD_BYTES = 16 * 1024 +_TERMINAL_STATUSES = frozenset({"succeeded", "cancelled"}) + + +class WorkStatus(StrEnum): + """Persisted lifecycle states for one logical work window.""" + + QUEUED = "queued" + RUNNING = "running" + RETRY_WAIT = "retry_wait" + CANCELLING = "cancelling" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + + +class WorkSpec(BaseModel): + """Safe, versioned references needed to execute one logical window.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: str + payload_version: int = Field(ge=1) + scope_id: str + lane_key: str + logical_key: str + payload: dict[str, JsonValue] + max_attempts: int = Field(default=5, ge=1, le=100) + available_at: datetime | None = None + + @field_validator("kind") + @classmethod + def validate_kind(cls, value: str) -> str: + return _validated_text("kind", value, 128) + + @field_validator("scope_id") + @classmethod + def validate_scope_id(cls, value: str) -> str: + return _validated_text("scope_id", value, MAX_SCOPE_ID_LENGTH) + + @field_validator("lane_key", "logical_key") + @classmethod + def validate_digest(cls, value: str, info: Any) -> str: + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise ValueError(f"{info.field_name} must be a lowercase SHA-256 digest") # noqa: TRY003 + return value + + +class WorkResult(BaseModel): + """Sanitized result metadata persisted after a successful commit.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + code: str + payload: dict[str, JsonValue] = Field(default_factory=dict) + + @field_validator("code") + @classmethod + def validate_code(cls, value: str) -> str: + return _validated_text("code", value, 128) + + +WorkCommit = Callable[[AsyncConnection], Awaitable[WorkResult | None]] + + +class WorkFailure(BaseModel): + """Bounded failure classification; raw exceptions are never persisted.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + category: str + code: str + retryable: bool + + @field_validator("category") + @classmethod + def validate_category(cls, value: str) -> str: + return _validated_text("category", value, 64) + + @field_validator("code") + @classmethod + def validate_code(cls, value: str) -> str: + return _validated_text("code", value, 128) + + +class StoredWork(BaseModel): + """Decoded work item returned by repository operations.""" + + model_config = ConfigDict(frozen=True) + + work_id: str + logical_key: str + lane_key: str + lane_sequence: int + kind: str + payload_version: int + scope_id: str + payload: dict[str, JsonValue] + status: WorkStatus + available_at: datetime + lease_owner: str | None + lease_fence: int + lease_expires_at: datetime | None + attempt_count: int + generation_attempt_count: int + recovery_generation: int + max_attempts: int + cancel_requested: bool + result_code: str | None + result_payload: dict[str, JsonValue] | None + error_category: str | None + error_code: str | None + state_version: int + created_at: datetime + updated_at: datetime + completed_at: datetime | None + + +class EnqueueResult(BaseModel): + """Whether enqueue created a row or joined an existing logical window.""" + + model_config = ConfigDict(frozen=True) + + created: bool + work: StoredWork + + +class WorkClaim(BaseModel): + """A fenced lease plus the safe payload needed by a Worker handler.""" + + model_config = ConfigDict(frozen=True) + + work_id: str + logical_key: str + lane_key: str + lane_sequence: int + kind: str + payload_version: int + scope_id: str + payload: dict[str, JsonValue] + owner_id: str + fence: int + attempt_no: int + generation_attempt_no: int + recovery_generation: int + created_at: datetime + claimed_at: datetime + previous_trace_id: str | None = None + previous_span_id: str | None = None + lease_expires_at: datetime + + +class WorkQueueStatistic(BaseModel): + """One bounded queue gauge sample grouped by kind and state.""" + + model_config = ConfigDict(frozen=True) + + kind: str + status: WorkStatus + depth: int + oldest_age_seconds: float + + +class WorkStateConflictError(RepositoryError): + """Raised when an optimistic operation observes a different state.""" + + def __init__(self, work_id: str, expected_version: int, status: WorkStatus, actual_version: int) -> None: + self.work_id = work_id + self.expected_version = expected_version + self.status = status + self.actual_version = actual_version + super().__init__( + f"work {work_id!r} changed: expected version {expected_version}, " + f"found {status.value} at version {actual_version}" + ) + + +class StaleWorkClaimError(RepositoryError): + """Raised when a lease owner or fence can no longer mutate a work item.""" + + def __init__(self, work_id: str) -> None: + self.work_id = work_id + super().__init__(f"work claim for {work_id!r} is no longer current") + + +class WorkRepository: + """Persist and transition work using short caller-owned transactions.""" + + def __init__(self, *, observer: WorkObserver | None = None) -> None: + self._observer = observer + + async def enqueue(self, connection: AsyncConnection, spec: WorkSpec, /) -> EnqueueResult: + payload = _dump_payload(spec.payload, kind="work", name=spec.logical_key) + now = await database_now(connection) + lane = await _lock_or_create_lane(connection, spec.lane_key) + existing = await _work_for_logical_key(connection, spec.logical_key, for_update=True) + if existing is not None: + _verify_logical_identity(existing, spec) + self._observe_enqueue(spec.kind, created=False) + return EnqueueResult(created=False, work=existing) + + work_id = str(uuid4()) + created = await insert_if_absent( + connection, + WORK_KEYS_TABLE, + { + "logical_key": spec.logical_key, + "work_id": work_id, + "lane_key": spec.lane_key, + "created_at": now, + }, + ) + if not created: + existing = await _work_for_logical_key(connection, spec.logical_key, for_update=True) + if existing is None: + raise InvalidStoredColumnError("logical_key", "a concurrently persisted work key") + _verify_logical_identity(existing, spec) + self._observe_enqueue(spec.kind, created=False) + return EnqueueResult(created=False, work=existing) + + sequence = _positive_integer(lane["next_sequence"], "next_sequence") + available_at = _normalized_datetime(spec.available_at) if spec.available_at is not None else now + await connection.execute( + update(WORK_LANES_TABLE) + .where(WORK_LANES_TABLE.c.lane_key == spec.lane_key) + .values( + next_sequence=sequence + 1, + head_sequence=sequence if lane["head_sequence"] is None else lane["head_sequence"], + ) + ) + await connection.execute( + insert(WORK_ITEMS_TABLE).values( + work_id=work_id, + logical_key=spec.logical_key, + lane_key=spec.lane_key, + lane_sequence=sequence, + kind=spec.kind, + payload_version=spec.payload_version, + scope_id=spec.scope_id, + payload=payload, + status=WorkStatus.QUEUED.value, + available_at=available_at, + lease_owner=None, + lease_fence=0, + lease_expires_at=None, + attempt_count=0, + generation_attempt_count=0, + recovery_generation=0, + max_attempts=spec.max_attempts, + cancel_requested=False, + result_code=None, + result_payload=None, + error_category=None, + error_code=None, + state_version=1, + created_at=now, + updated_at=now, + completed_at=None, + ) + ) + self._observe_enqueue(spec.kind, created=True) + return EnqueueResult(created=True, work=await self.get(connection, work_id)) + + async def claim( + self, + connection: AsyncConnection, + *, + worker_id: str, + supported: Mapping[str, frozenset[int]], + lease_seconds: int, + limit: int, + expired_retry_delay_seconds: int = 0, + ) -> tuple[WorkClaim, ...]: + _validated_text("worker_id", worker_id, 128) + _require_positive("lease_seconds", lease_seconds) + _require_positive("limit", limit) + _require_nonnegative("expired_retry_delay_seconds", expired_retry_delay_seconds) + if not supported: + return () + _validate_supported(supported) + + now = await database_now(connection) + supported_pairs = [ + and_(WORK_ITEMS_TABLE.c.kind == kind, WORK_ITEMS_TABLE.c.payload_version.in_(versions)) + for kind, versions in supported.items() + ] + supported_due = and_( + or_(*supported_pairs), + WORK_ITEMS_TABLE.c.status.in_((WorkStatus.QUEUED.value, WorkStatus.RETRY_WAIT.value)), + WORK_ITEMS_TABLE.c.available_at <= now, + ) + expired = and_( + WORK_ITEMS_TABLE.c.status.in_((WorkStatus.RUNNING.value, WorkStatus.CANCELLING.value)), + WORK_ITEMS_TABLE.c.lease_expires_at <= now, + ) + candidate_ids = ( + await connection.scalars( + select(WORK_ITEMS_TABLE.c.work_id) + .join( + WORK_LANES_TABLE, + and_( + WORK_LANES_TABLE.c.lane_key == WORK_ITEMS_TABLE.c.lane_key, + WORK_LANES_TABLE.c.head_sequence == WORK_ITEMS_TABLE.c.lane_sequence, + ), + ) + .where(or_(supported_due, expired)) + .order_by(WORK_ITEMS_TABLE.c.available_at, WORK_ITEMS_TABLE.c.created_at) + .limit(max(32, limit * 4)) + ) + ).all() + + claims: list[WorkClaim] = [] + for work_id_value in candidate_ids: + if len(claims) >= limit: + break + claim = await self._claim_candidate( + connection, + str(work_id_value), + worker_id=worker_id, + supported=supported, + lease_seconds=lease_seconds, + expired_retry_delay_seconds=expired_retry_delay_seconds, + ) + if claim is not None: + claims.append(claim) + return tuple(claims) + + async def heartbeat( + self, + connection: AsyncConnection, + claim: WorkClaim, + /, + *, + lease_seconds: int, + ) -> StoredWork: + _require_positive("lease_seconds", lease_seconds) + await _lock_lane(connection, claim.lane_key) + await _lock_logical_key(connection, claim.logical_key, claim.work_id) + work = await _lock_work(connection, claim.work_id) + now = await database_now(connection) + _verify_claim(work, claim, now) + if work.cancel_requested or work.status is WorkStatus.CANCELLING: + raise StaleWorkClaimError(claim.work_id) + expires_at = now + timedelta(seconds=lease_seconds) + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == claim.work_id) + .values(lease_expires_at=expires_at, updated_at=now, state_version=work.state_version + 1) + ) + await connection.execute( + update(WORK_ATTEMPTS_TABLE) + .where( + WORK_ATTEMPTS_TABLE.c.work_id == claim.work_id, + WORK_ATTEMPTS_TABLE.c.attempt_no == claim.attempt_no, + WORK_ATTEMPTS_TABLE.c.owner_id == claim.owner_id, + WORK_ATTEMPTS_TABLE.c.fence == claim.fence, + ) + .values(heartbeat_at=now) + ) + return await self.get(connection, claim.work_id) + + async def record_attempt_trace( + self, + connection: AsyncConnection, + claim: WorkClaim, + /, + *, + trace_id: str, + span_id: str, + ) -> None: + """Bind one non-sensitive span identity to the current fenced attempt.""" + + _require_hex("trace_id", trace_id, 32) + _require_hex("span_id", span_id, 16) + await _lock_lane(connection, claim.lane_key) + await _lock_logical_key(connection, claim.logical_key, claim.work_id) + work = await _lock_work(connection, claim.work_id) + _verify_claim(work, claim, await database_now(connection)) + result = await connection.execute( + update(WORK_ATTEMPTS_TABLE) + .where( + WORK_ATTEMPTS_TABLE.c.work_id == claim.work_id, + WORK_ATTEMPTS_TABLE.c.attempt_no == claim.attempt_no, + WORK_ATTEMPTS_TABLE.c.owner_id == claim.owner_id, + WORK_ATTEMPTS_TABLE.c.fence == claim.fence, + WORK_ATTEMPTS_TABLE.c.finished_at.is_(None), + ) + .values(trace_id=trace_id, span_id=span_id) + ) + if result.rowcount != 1: + raise StaleWorkClaimError(claim.work_id) + + async def complete( + self, + connection: AsyncConnection, + claim: WorkClaim, + result: WorkResult | None, + /, + *, + commit: WorkCommit | None = None, + ) -> StoredWork: + if result is None and commit is None: + raise InvalidRepositoryArgumentError("result", "result or commit is required") + await _lock_lane(connection, claim.lane_key) + await _lock_logical_key(connection, claim.logical_key, claim.work_id) + work = await _lock_work(connection, claim.work_id) + now = await database_now(connection) + _verify_claim(work, claim, now) + if work.cancel_requested or work.status is WorkStatus.CANCELLING: + raise StaleWorkClaimError(claim.work_id) + if commit is not None: + committed_result = await commit(connection) + if committed_result is not None: + result = committed_result + if result is None: + raise InvalidRepositoryArgumentError("result", "commit did not produce a result") + result_payload = _dump_payload(result.payload, kind="work-result", name=claim.work_id) + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == claim.work_id) + .values( + status=WorkStatus.SUCCEEDED.value, + lease_owner=None, + lease_expires_at=None, + result_code=result.code, + result_payload=result_payload, + error_category=None, + error_code=None, + state_version=work.state_version + 1, + updated_at=now, + completed_at=now, + ) + ) + await self._finish_attempt(connection, claim, now=now, outcome="succeeded") + self._observe_attempt(claim, now=now, outcome="succeeded") + await _release_logical_key(connection, work) + await _advance_lane(connection, work) + return await self.get(connection, claim.work_id) + + async def fail( + self, + connection: AsyncConnection, + claim: WorkClaim, + failure: WorkFailure, + /, + *, + retry_delay_seconds: int, + ) -> StoredWork: + _require_nonnegative("retry_delay_seconds", retry_delay_seconds) + await _lock_lane(connection, claim.lane_key) + await _lock_logical_key(connection, claim.logical_key, claim.work_id) + work = await _lock_work(connection, claim.work_id) + now = await database_now(connection) + _verify_claim(work, claim, now) + if work.cancel_requested or work.status is WorkStatus.CANCELLING: + return await self._cancel_running(connection, work, claim=claim, now=now) + retry = failure.retryable and work.generation_attempt_count < work.max_attempts + status = WorkStatus.RETRY_WAIT if retry else WorkStatus.FAILED + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == claim.work_id) + .values( + status=status.value, + available_at=now + timedelta(seconds=retry_delay_seconds) if retry else work.available_at, + lease_owner=None, + lease_expires_at=None, + error_category=failure.category, + error_code=failure.code, + state_version=work.state_version + 1, + updated_at=now, + ) + ) + await self._finish_attempt( + connection, + claim, + now=now, + outcome="retry_wait" if retry else "failed", + failure=failure, + ) + self._observe_attempt( + claim, + now=now, + outcome="retry_wait" if retry else "failed", + error_category=failure.category, + ) + return await self.get(connection, claim.work_id) + + async def retry(self, connection: AsyncConnection, work_id: str, /, *, expected_version: int) -> StoredWork: + _require_work_id(work_id) + _require_positive("expected_version", expected_version) + unlocked = await self.get(connection, work_id) + await _lock_lane(connection, unlocked.lane_key) + await _lock_logical_key(connection, unlocked.logical_key, work_id) + work = await _lock_work(connection, work_id) + _verify_version(work, expected_version) + if work.status is not WorkStatus.FAILED: + raise WorkStateConflictError(work_id, expected_version, work.status, work.state_version) + now = await database_now(connection) + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == work_id) + .values( + status=WorkStatus.QUEUED.value, + available_at=now, + generation_attempt_count=0, + recovery_generation=work.recovery_generation + 1, + cancel_requested=False, + error_category=None, + error_code=None, + state_version=work.state_version + 1, + updated_at=now, + ) + ) + return await self.get(connection, work_id) + + async def cancel(self, connection: AsyncConnection, work_id: str, /, *, expected_version: int) -> StoredWork: + _require_work_id(work_id) + _require_positive("expected_version", expected_version) + unlocked = await self.get(connection, work_id) + await _lock_lane(connection, unlocked.lane_key) + logical_key_locked = await _lock_logical_key_if_present(connection, unlocked.logical_key) + work = await _lock_work(connection, work_id) + _verify_version(work, expected_version) + if work.status in {WorkStatus.CANCELLING, WorkStatus.SUCCEEDED, WorkStatus.CANCELLED}: + raise WorkStateConflictError(work_id, expected_version, work.status, work.state_version) + if not logical_key_locked: + raise StaleWorkClaimError(work_id) + now = await database_now(connection) + if work.status is WorkStatus.RUNNING: + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == work_id) + .values( + status=WorkStatus.CANCELLING.value, + cancel_requested=True, + state_version=work.state_version + 1, + updated_at=now, + ) + ) + else: + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == work_id) + .values( + status=WorkStatus.CANCELLED.value, + cancel_requested=True, + lease_owner=None, + lease_expires_at=None, + state_version=work.state_version + 1, + updated_at=now, + completed_at=now, + ) + ) + await _release_logical_key(connection, work) + await _advance_lane(connection, work) + return await self.get(connection, work_id) + + async def get(self, connection: AsyncConnection, work_id: str, /) -> StoredWork: + _require_work_id(work_id) + row = ( + (await connection.execute(select(WORK_ITEMS_TABLE).where(WORK_ITEMS_TABLE.c.work_id == work_id))) + .mappings() + .one_or_none() + ) + if row is None: + raise RepositoryNotFoundError("work", work_id) + return _decode_work(row) + + async def list( + self, + connection: AsyncConnection, + /, + *, + scope_id: str | None = None, + kind: str | None = None, + allowed_kinds: frozenset[str] | None = None, + status: WorkStatus | None = None, + limit: int = 100, + before: tuple[datetime, str] | None = None, + ) -> tuple[StoredWork, ...]: + statement = _work_list_statement( + scope_id=scope_id, + kind=kind, + allowed_kinds=allowed_kinds, + status=status, + before=before, + limit=limit, + ) + if statement is None: + return () + rows = ( + ( + await connection.execute( + statement.order_by(WORK_ITEMS_TABLE.c.created_at.desc(), WORK_ITEMS_TABLE.c.work_id.desc()).limit( + limit + ) + ) + ) + .mappings() + .all() + ) + return tuple(_decode_work(row) for row in rows) + + async def purge_terminal( + self, + connection: AsyncConnection, + /, + *, + completed_before: datetime, + limit: int = 500, + ) -> int: + """Delete one bounded batch of successful or cancelled operation history.""" + + if limit < 1 or limit > 500: + raise InvalidRepositoryArgumentError("limit", "must be between 1 and 500") + cutoff = _normalized_datetime(completed_before) + work_ids = tuple( + str(value) + for value in ( + await connection.scalars( + select(WORK_ITEMS_TABLE.c.work_id) + .where( + WORK_ITEMS_TABLE.c.status.in_(_TERMINAL_STATUSES), + WORK_ITEMS_TABLE.c.completed_at.is_not(None), + WORK_ITEMS_TABLE.c.completed_at < cutoff, + ) + .order_by(WORK_ITEMS_TABLE.c.completed_at, WORK_ITEMS_TABLE.c.work_id) + .limit(limit) + ) + ).all() + ) + if not work_ids: + return 0 + await connection.execute(delete(WORK_ATTEMPTS_TABLE).where(WORK_ATTEMPTS_TABLE.c.work_id.in_(work_ids))) + result = await connection.execute( + delete(WORK_ITEMS_TABLE).where( + WORK_ITEMS_TABLE.c.work_id.in_(work_ids), + WORK_ITEMS_TABLE.c.status.in_(_TERMINAL_STATUSES), + WORK_ITEMS_TABLE.c.completed_at < cutoff, + ) + ) + return result.rowcount + + async def unsupported_head_count( + self, + connection: AsyncConnection, + supported: Mapping[str, frozenset[int]], + /, + ) -> int: + """Count visible lane heads that no registered handler can decode.""" + + _validate_supported(supported) + supported_pairs = [ + and_(WORK_ITEMS_TABLE.c.kind == kind, WORK_ITEMS_TABLE.c.payload_version.in_(versions)) + for kind, versions in supported.items() + ] + compatible = or_(*supported_pairs) if supported_pairs else false() + value = await connection.scalar( + select(func.count()) + .select_from(WORK_ITEMS_TABLE) + .join( + WORK_LANES_TABLE, + and_( + WORK_LANES_TABLE.c.lane_key == WORK_ITEMS_TABLE.c.lane_key, + WORK_LANES_TABLE.c.head_sequence == WORK_ITEMS_TABLE.c.lane_sequence, + ), + ) + .where( + WORK_ITEMS_TABLE.c.status.not_in(_TERMINAL_STATUSES), + not_(compatible), + ) + ) + return int(value or 0) + + async def queue_statistics(self, connection: AsyncConnection, /) -> tuple[WorkQueueStatistic, ...]: + """Aggregate non-terminal queue state without high-cardinality labels.""" + + now = await database_now(connection) + rows = ( + await connection.execute( + select( + WORK_ITEMS_TABLE.c.kind, + WORK_ITEMS_TABLE.c.status, + func.count().label("depth"), + func.min(WORK_ITEMS_TABLE.c.created_at).label("oldest_created_at"), + ) + .where(WORK_ITEMS_TABLE.c.status.not_in(_TERMINAL_STATUSES)) + .group_by(WORK_ITEMS_TABLE.c.kind, WORK_ITEMS_TABLE.c.status) + ) + ).mappings() + return tuple( + WorkQueueStatistic( + kind=str(row["kind"]), + status=WorkStatus(str(row["status"])), + depth=int(row["depth"]), + oldest_age_seconds=max( + 0.0, + (now - _stored_datetime(row["oldest_created_at"], "oldest_created_at")).total_seconds(), + ), + ) + for row in rows + ) + + async def _claim_locked( + self, + connection: AsyncConnection, + work: StoredWork, + *, + worker_id: str, + lease_seconds: int, + now: datetime, + ) -> WorkClaim: + previous_trace_id, previous_span_id = await _previous_attempt_trace(connection, work.work_id) + fence = work.lease_fence + 1 + attempt_no = work.attempt_count + 1 + expires_at = now + timedelta(seconds=lease_seconds) + result = await connection.execute( + update(WORK_ITEMS_TABLE) + .where( + WORK_ITEMS_TABLE.c.work_id == work.work_id, + WORK_ITEMS_TABLE.c.state_version == work.state_version, + WORK_ITEMS_TABLE.c.status == work.status.value, + ) + .values( + status=WorkStatus.RUNNING.value, + lease_owner=worker_id, + lease_fence=fence, + lease_expires_at=expires_at, + attempt_count=attempt_no, + generation_attempt_count=work.generation_attempt_count + 1, + state_version=work.state_version + 1, + updated_at=now, + ) + ) + if result.rowcount != 1: + raise StaleWorkClaimError(work.work_id) + await connection.execute( + insert(WORK_ATTEMPTS_TABLE).values( + work_id=work.work_id, + attempt_no=attempt_no, + recovery_generation=work.recovery_generation, + owner_id=worker_id, + fence=fence, + started_at=now, + heartbeat_at=now, + finished_at=None, + outcome=None, + error_category=None, + error_code=None, + trace_id=None, + span_id=None, + ) + ) + self._observe_claim(work, now=now) + return WorkClaim( + work_id=work.work_id, + logical_key=work.logical_key, + lane_key=work.lane_key, + lane_sequence=work.lane_sequence, + kind=work.kind, + payload_version=work.payload_version, + scope_id=work.scope_id, + payload=work.payload, + owner_id=worker_id, + fence=fence, + attempt_no=attempt_no, + generation_attempt_no=work.generation_attempt_count + 1, + recovery_generation=work.recovery_generation, + created_at=work.created_at, + claimed_at=now, + previous_trace_id=previous_trace_id, + previous_span_id=previous_span_id, + lease_expires_at=expires_at, + ) + + async def _claim_candidate( + self, + connection: AsyncConnection, + work_id: str, + *, + worker_id: str, + supported: Mapping[str, frozenset[int]], + lease_seconds: int, + expired_retry_delay_seconds: int, + ) -> WorkClaim | None: + unlocked = await self.get(connection, work_id) + await _lock_lane(connection, unlocked.lane_key) + await _lock_logical_key(connection, unlocked.logical_key, work_id) + work = await _lock_work(connection, work_id) + if not await _is_lane_head(connection, work): + return None + now = await database_now(connection) + if work.status in {WorkStatus.RUNNING, WorkStatus.CANCELLING}: + if work.lease_expires_at is None or work.lease_expires_at > now: + return None + await self._expire_claim( + connection, + work, + now=now, + retry_delay_seconds=expired_retry_delay_seconds, + ) + return None + if work.status not in {WorkStatus.QUEUED, WorkStatus.RETRY_WAIT} or work.available_at > now: + return None + if work.payload_version not in supported.get(work.kind, frozenset()): + return None + return await self._claim_locked( + connection, + work, + worker_id=worker_id, + lease_seconds=lease_seconds, + now=now, + ) + + async def _expire_claim( + self, + connection: AsyncConnection, + work: StoredWork, + *, + now: datetime, + retry_delay_seconds: int, + ) -> None: + claim = _claim_from_work(work) + if work.status is WorkStatus.CANCELLING or work.cancel_requested: + await self._cancel_running(connection, work, claim=claim, now=now) + return + retry = work.generation_attempt_count < work.max_attempts + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == work.work_id) + .values( + status=WorkStatus.RETRY_WAIT.value if retry else WorkStatus.FAILED.value, + available_at=now + timedelta(seconds=retry_delay_seconds) if retry else work.available_at, + lease_owner=None, + lease_expires_at=None, + error_category="lease", + error_code="lease_expired", + state_version=work.state_version + 1, + updated_at=now, + ) + ) + await self._finish_attempt( + connection, + claim, + now=now, + outcome="lease_expired_retry" if retry else "lease_expired_failed", + failure=WorkFailure(category="lease", code="lease_expired", retryable=retry), + ) + self._observe_attempt( + claim, + now=now, + outcome="lease_expired_retry" if retry else "lease_expired_failed", + error_category="lease", + ) + if self._observer is not None: + with suppress(Exception): + self._observer.observe_work_lease_expiry( + work.kind, + outcome="retry_wait" if retry else "failed", + ) + + async def _cancel_running( + self, + connection: AsyncConnection, + work: StoredWork, + *, + claim: WorkClaim, + now: datetime, + ) -> StoredWork: + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == work.work_id) + .values( + status=WorkStatus.CANCELLED.value, + lease_owner=None, + lease_expires_at=None, + cancel_requested=True, + state_version=work.state_version + 1, + updated_at=now, + completed_at=now, + ) + ) + await self._finish_attempt(connection, claim, now=now, outcome="cancelled") + self._observe_attempt(claim, now=now, outcome="cancelled") + await _release_logical_key(connection, work) + await _advance_lane(connection, work) + return await self.get(connection, work.work_id) + + async def _finish_attempt( + self, + connection: AsyncConnection, + claim: WorkClaim, + *, + now: datetime, + outcome: str, + failure: WorkFailure | None = None, + ) -> None: + result = await connection.execute( + update(WORK_ATTEMPTS_TABLE) + .where( + WORK_ATTEMPTS_TABLE.c.work_id == claim.work_id, + WORK_ATTEMPTS_TABLE.c.attempt_no == claim.attempt_no, + WORK_ATTEMPTS_TABLE.c.owner_id == claim.owner_id, + WORK_ATTEMPTS_TABLE.c.fence == claim.fence, + WORK_ATTEMPTS_TABLE.c.finished_at.is_(None), + ) + .values( + heartbeat_at=now, + finished_at=now, + outcome=outcome, + error_category=None if failure is None else failure.category, + error_code=None if failure is None else failure.code, + ) + ) + if result.rowcount != 1: + raise StaleWorkClaimError(claim.work_id) + + def _observe_enqueue(self, kind: str, *, created: bool) -> None: + if self._observer is not None: + with suppress(Exception): + self._observer.observe_work_enqueue(kind, created=created) + + def _observe_claim(self, work: StoredWork, *, now: datetime) -> None: + if self._observer is not None: + with suppress(Exception): + self._observer.observe_work_claim( + work.kind, + latency_seconds=max(0.0, (now - work.created_at).total_seconds()), + ) + + def _observe_attempt( + self, + claim: WorkClaim, + *, + now: datetime, + outcome: str, + error_category: str = "none", + ) -> None: + if self._observer is not None: + with suppress(Exception): + self._observer.observe_work_attempt( + claim.kind, + outcome=outcome, + error_category=error_category, + duration_seconds=max(0.0, (now - claim.claimed_at).total_seconds()), + ) + + +async def _lock_or_create_lane(connection: AsyncConnection, lane_key: str) -> Mapping[Any, Any]: + await connection.execute( + update(WORK_LANES_TABLE) + .where(WORK_LANES_TABLE.c.lane_key == lane_key) + .values(next_sequence=WORK_LANES_TABLE.c.next_sequence) + ) + row = await _lane_row(connection, lane_key) + if row is not None: + return row + await insert_if_absent( + connection, + WORK_LANES_TABLE, + {"lane_key": lane_key, "head_sequence": None, "next_sequence": 1}, + ) + row = await _lane_row(connection, lane_key) + if row is None: + raise InvalidStoredColumnError("lane_key", "an initialized work lane") + return row + + +async def _previous_attempt_trace(connection: AsyncConnection, work_id: str) -> tuple[str | None, str | None]: + row = ( + await connection.execute( + select(WORK_ATTEMPTS_TABLE.c.trace_id, WORK_ATTEMPTS_TABLE.c.span_id) + .where(WORK_ATTEMPTS_TABLE.c.work_id == work_id) + .order_by(WORK_ATTEMPTS_TABLE.c.attempt_no.desc()) + .limit(1) + ) + ).one_or_none() + if row is None or (row.trace_id is None and row.span_id is None): + return None, None + if row.trace_id is None or row.span_id is None: + raise InvalidStoredColumnError("attempt trace", "a complete trace/span pair") # noqa: TRY003 + trace_id = str(row.trace_id) + span_id = str(row.span_id) + _require_hex("trace_id", trace_id, 32) + _require_hex("span_id", span_id, 16) + return trace_id, span_id + + +async def _lock_lane(connection: AsyncConnection, lane_key: str) -> None: + await _lock_or_create_lane(connection, lane_key) + + +async def _lane_row(connection: AsyncConnection, lane_key: str) -> Mapping[Any, Any] | None: + return ( + ( + await connection.execute( + select(WORK_LANES_TABLE).where(WORK_LANES_TABLE.c.lane_key == lane_key).with_for_update() + ) + ) + .mappings() + .one_or_none() + ) + + +async def _lock_logical_key(connection: AsyncConnection, logical_key: str, work_id: str) -> None: + if not await _lock_logical_key_if_present(connection, logical_key): + raise StaleWorkClaimError(work_id) + + +async def _lock_logical_key_if_present(connection: AsyncConnection, logical_key: str) -> bool: + row = ( + await connection.execute( + select(WORK_KEYS_TABLE.c.work_id).where(WORK_KEYS_TABLE.c.logical_key == logical_key).with_for_update() + ) + ).one_or_none() + return row is not None + + +async def _lock_work(connection: AsyncConnection, work_id: str) -> StoredWork: + row = ( + ( + await connection.execute( + select(WORK_ITEMS_TABLE).where(WORK_ITEMS_TABLE.c.work_id == work_id).with_for_update() + ) + ) + .mappings() + .one_or_none() + ) + if row is None: + raise RepositoryNotFoundError("work", work_id) + return _decode_work(row) + + +async def _work_for_logical_key( + connection: AsyncConnection, + logical_key: str, + *, + for_update: bool, +) -> StoredWork | None: + statement = ( + select(WORK_ITEMS_TABLE) + .join(WORK_KEYS_TABLE, WORK_KEYS_TABLE.c.work_id == WORK_ITEMS_TABLE.c.work_id) + .where(WORK_KEYS_TABLE.c.logical_key == logical_key) + ) + if for_update: + statement = statement.with_for_update() + row = (await connection.execute(statement)).mappings().one_or_none() + return None if row is None else _decode_work(row) + + +async def _is_lane_head(connection: AsyncConnection, work: StoredWork) -> bool: + head = await connection.scalar( + select(WORK_LANES_TABLE.c.head_sequence).where(WORK_LANES_TABLE.c.lane_key == work.lane_key) + ) + return head == work.lane_sequence + + +async def _release_logical_key(connection: AsyncConnection, work: StoredWork) -> None: + result = await connection.execute( + delete(WORK_KEYS_TABLE).where( + WORK_KEYS_TABLE.c.logical_key == work.logical_key, + WORK_KEYS_TABLE.c.work_id == work.work_id, + ) + ) + if result.rowcount != 1: + raise StaleWorkClaimError(work.work_id) + + +async def _advance_lane(connection: AsyncConnection, work: StoredWork) -> None: + next_sequence = await connection.scalar( + select(WORK_ITEMS_TABLE.c.lane_sequence) + .where( + WORK_ITEMS_TABLE.c.lane_key == work.lane_key, + WORK_ITEMS_TABLE.c.lane_sequence > work.lane_sequence, + WORK_ITEMS_TABLE.c.status.not_in(_TERMINAL_STATUSES), + ) + .order_by(WORK_ITEMS_TABLE.c.lane_sequence) + .limit(1) + ) + await connection.execute( + update(WORK_LANES_TABLE) + .where( + WORK_LANES_TABLE.c.lane_key == work.lane_key, + WORK_LANES_TABLE.c.head_sequence == work.lane_sequence, + ) + .values(head_sequence=next_sequence) + ) + + +def _work_list_statement( + *, + scope_id: str | None, + kind: str | None, + allowed_kinds: frozenset[str] | None, + status: WorkStatus | None, + before: tuple[datetime, str] | None, + limit: int, +) -> Any | None: + if not _validate_work_list(scope_id=scope_id, kind=kind, allowed_kinds=allowed_kinds, limit=limit): + return None + + statement = select(WORK_ITEMS_TABLE) + if scope_id is not None: + statement = statement.where(WORK_ITEMS_TABLE.c.scope_id == scope_id) + if kind is not None: + statement = statement.where(WORK_ITEMS_TABLE.c.kind == kind) + if allowed_kinds is not None: + statement = statement.where(WORK_ITEMS_TABLE.c.kind.in_(allowed_kinds)) + if status is not None: + statement = statement.where(WORK_ITEMS_TABLE.c.status == status.value) + if before is not None: + created_at, work_id = before + created_at = _normalized_datetime(created_at) + _require_work_id(work_id) + statement = statement.where( + or_( + WORK_ITEMS_TABLE.c.created_at < created_at, + and_(WORK_ITEMS_TABLE.c.created_at == created_at, WORK_ITEMS_TABLE.c.work_id < work_id), + ) + ) + return statement + + +def _validate_work_list( + *, + scope_id: str | None, + kind: str | None, + allowed_kinds: frozenset[str] | None, + limit: int, +) -> bool: + if scope_id is not None: + _validated_text("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) + if kind is not None: + _validated_text("kind", kind, 128) + if allowed_kinds is not None: + if not allowed_kinds: + return False + for allowed_kind in allowed_kinds: + _validated_text("allowed_kind", allowed_kind, 128) + if limit < 1 or limit > 100: + raise InvalidRepositoryArgumentError("limit", "must be between 1 and 100") + return True + + +def _decode_work(row: Mapping[Any, Any]) -> StoredWork: + work_id = str(row["work_id"]) + return StoredWork( + work_id=work_id, + logical_key=str(row["logical_key"]), + lane_key=str(row["lane_key"]), + lane_sequence=_positive_integer(row["lane_sequence"], "lane_sequence"), + kind=str(row["kind"]), + payload_version=_positive_integer(row["payload_version"], "payload_version"), + scope_id=str(row["scope_id"]), + payload=_load_payload(row["payload"], kind="work", name=work_id), + status=WorkStatus(str(row["status"])), + available_at=_stored_datetime(row["available_at"], "available_at"), + lease_owner=None if row["lease_owner"] is None else str(row["lease_owner"]), + lease_fence=_nonnegative_integer(row["lease_fence"], "lease_fence"), + lease_expires_at=_optional_datetime(row["lease_expires_at"], "lease_expires_at"), + attempt_count=_nonnegative_integer(row["attempt_count"], "attempt_count"), + generation_attempt_count=_nonnegative_integer(row["generation_attempt_count"], "generation_attempt_count"), + recovery_generation=_nonnegative_integer(row["recovery_generation"], "recovery_generation"), + max_attempts=_positive_integer(row["max_attempts"], "max_attempts"), + cancel_requested=bool(row["cancel_requested"]), + result_code=None if row["result_code"] is None else str(row["result_code"]), + result_payload=( + None + if row["result_payload"] is None + else _load_payload(row["result_payload"], kind="work-result", name=work_id) + ), + error_category=None if row["error_category"] is None else str(row["error_category"]), + error_code=None if row["error_code"] is None else str(row["error_code"]), + state_version=_positive_integer(row["state_version"], "state_version"), + created_at=_stored_datetime(row["created_at"], "created_at"), + updated_at=_stored_datetime(row["updated_at"], "updated_at"), + completed_at=_optional_datetime(row["completed_at"], "completed_at"), + ) + + +def _verify_logical_identity(work: StoredWork, spec: WorkSpec) -> None: + if work.kind != spec.kind or work.scope_id != spec.scope_id or work.lane_key != spec.lane_key: + raise RepositoryError( # noqa: TRY003 + f"logical work key {spec.logical_key!r} is bound to a different work identity" + ) + + +def _verify_claim(work: StoredWork, claim: WorkClaim, now: datetime) -> None: + if ( + work.status not in {WorkStatus.RUNNING, WorkStatus.CANCELLING} + or work.lease_owner != claim.owner_id + or work.lease_fence != claim.fence + or work.attempt_count != claim.attempt_no + or work.lease_expires_at is None + or work.lease_expires_at <= now + ): + raise StaleWorkClaimError(claim.work_id) + + +def _verify_version(work: StoredWork, expected_version: int) -> None: + if work.state_version != expected_version: + raise WorkStateConflictError(work.work_id, expected_version, work.status, work.state_version) + + +def _claim_from_work(work: StoredWork) -> WorkClaim: + if work.lease_owner is None or work.lease_expires_at is None or work.attempt_count < 1: + raise InvalidStoredColumnError("work lease", "a complete active lease") # noqa: TRY003 + return WorkClaim( + work_id=work.work_id, + logical_key=work.logical_key, + lane_key=work.lane_key, + lane_sequence=work.lane_sequence, + kind=work.kind, + payload_version=work.payload_version, + scope_id=work.scope_id, + payload=work.payload, + owner_id=work.lease_owner, + fence=work.lease_fence, + attempt_no=work.attempt_count, + generation_attempt_no=work.generation_attempt_count, + recovery_generation=work.recovery_generation, + created_at=work.created_at, + claimed_at=work.updated_at, + previous_trace_id=None, + previous_span_id=None, + lease_expires_at=work.lease_expires_at, + ) + + +def _dump_payload(payload: dict[str, JsonValue], *, kind: str, name: str) -> bytes: + try: + encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + except (TypeError, ValueError) as error: + raise InvalidStoredPayloadError(kind, name, "value is not JSON serializable") from error + if len(encoded) > _MAX_PAYLOAD_BYTES: + raise InvalidRepositoryArgumentError("payload", f"must not exceed {_MAX_PAYLOAD_BYTES} encoded bytes") + return encoded + + +def _load_payload(payload: object, *, kind: str, name: str) -> dict[str, JsonValue]: + try: + decoded = json.loads(stored_bytes(payload, column="payload")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise InvalidStoredPayloadError(kind, name, "payload is not valid JSON") from error + if not isinstance(decoded, dict) or any(not isinstance(key, str) for key in decoded): + raise InvalidStoredPayloadError(kind, name, "payload is not an object") + return decoded + + +def _stored_datetime(value: object, column: str) -> datetime: + if not isinstance(value, datetime): + raise InvalidStoredColumnError(column, "a datetime") + return _normalized_datetime(value) + + +def _optional_datetime(value: object, column: str) -> datetime | None: + return None if value is None else _stored_datetime(value, column) + + +def _normalized_datetime(value: datetime) -> datetime: + if value.tzinfo is None: + return value + return value.astimezone(UTC).replace(tzinfo=None) + + +def _positive_integer(value: object, column: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidStoredColumnError(column, "a positive integer") + return value + + +def _nonnegative_integer(value: object, column: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise InvalidStoredColumnError(column, "a non-negative integer") + return value + + +def _require_positive(field: str, value: int) -> None: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidRepositoryArgumentError(field, "must be a positive integer") + + +def _require_nonnegative(field: str, value: int) -> None: + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise InvalidRepositoryArgumentError(field, "must be a non-negative integer") + + +def _require_work_id(value: str) -> None: + _validated_text("work_id", value, 36) + + +def _require_hex(field: str, value: str, length: int) -> None: + if len(value) != length or any(character not in "0123456789abcdef" for character in value): + raise InvalidRepositoryArgumentError(field, f"must be {length} lowercase hexadecimal characters") + + +def _validate_supported(supported: Mapping[str, frozenset[int]]) -> None: + for kind, versions in supported.items(): + _validated_text("supported kind", kind, 128) + if not versions or any(version < 1 for version in versions): + raise InvalidRepositoryArgumentError("supported", "each kind must have positive payload versions") + + +def _validated_text(field: str, value: str, maximum: int) -> str: + if not isinstance(value, str) or not value.strip() or value != value.strip(): + raise ValueError(f"{field} must be a non-empty trimmed string") # noqa: TRY003 + if len(value) > maximum: + raise ValueError(f"{field} must not exceed {maximum} characters") # noqa: TRY003 + return value + + +__all__ = [ + "EnqueueResult", + "StaleWorkClaimError", + "StoredWork", + "WorkClaim", + "WorkCommit", + "WorkFailure", + "WorkQueueStatistic", + "WorkRepository", + "WorkResult", + "WorkSpec", + "WorkStateConflictError", + "WorkStatus", +] diff --git a/src/powercontext/builtin/runtime/__init__.py b/src/powercontext/builtin/runtime/__init__.py index 1be806514..36be9d781 100644 --- a/src/powercontext/builtin/runtime/__init__.py +++ b/src/powercontext/builtin/runtime/__init__.py @@ -48,8 +48,6 @@ RemoteIngestionApplication, RemoteSkillApplication, ReviewApplication, - ScheduledExperienceProcessor, - ScheduledSourceProcessor, ScopedExperienceApplication, ScopedExternalSkillApplication, ScopedHandoffApplication, @@ -270,8 +268,6 @@ "RuntimeReadiness", "RuntimeReadinessChecks", "RuntimeReadinessStatus", - "ScheduledExperienceProcessor", - "ScheduledSourceProcessor", "ScopeStatistics", "ScopedExperienceApplication", "ScopedExternalSkillApplication", diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index 4438b31f8..af29b80d2 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -21,8 +21,6 @@ from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping from contextlib import AbstractContextManager, asynccontextmanager, nullcontext from datetime import UTC, datetime -from pathlib import Path -from time import perf_counter from typing import TYPE_CHECKING, Any from pydantic import BaseModel, ValidationError @@ -212,9 +210,8 @@ from powercontext.sources import ConnectorBinding, SourceDefinitionManifest, SourceRef if TYPE_CHECKING: - from apscheduler.schedulers.asyncio import AsyncIOScheduler - from powercontext.builtin.handoff_report.application import HandoffReportApplication + from powercontext.builtin.runtime.operations import OperationManager logger = logging.getLogger(__name__) @@ -225,7 +222,6 @@ _MEMORY_SEARCH_MODE = "powercontext.memory.search.mode" _MEMORY_SEARCH_RESULT_COUNT = "powercontext.memory.search.result_count" -ScopeIds = Callable[[], Awaitable[tuple[str, ...]]] ReviewServiceFactory = Callable[[str], ReviewService] GenerationServiceFactory = Callable[[str], ReviewedGenerationService] ExternalSkillRegistryFactory = Callable[[str], ExternalSkillRegistryService] @@ -247,6 +243,7 @@ SkillUsageRecorder = Callable[[str, SkillUsageCapture], Awaitable[SourceReceipt]] StatisticsServiceFactory = Callable[[str], RelationalScopedStatistics] RecallTokenEstimator = Callable[[str, PreparedContextBuild], Awaitable[RecallTokenMeasurement | None]] +MemoryFlusher = Callable[[str, int], Awaitable[MemoryFlushResult]] Clock = Callable[[], datetime] _MEMORY_SEARCH_ATTEMPTS = 3 @@ -266,7 +263,6 @@ def __init__(self, code: str) -> None: "remote-ingestion": "Remote Source ingestion is not configured", "review": "Candidate Review services are not configured", "remote-skill-distribution": "Remote Skill distribution services are not configured", - "scheduler": "Built-in Runtime scheduler is already started", "scope": "Scope services are not configured", "skill-publication": "Managed Skill publication services are not configured", "skill-provenance": "Managed Skill provenance services are not configured", @@ -1500,19 +1496,29 @@ async def changes(self, *, since_revision: int | None = None) -> MemoryChangesPa ) async def flush(self, /, *, limit: int | None = None) -> MemoryFlushResult: - async with self._runtime._context( - self.scope_id, - generation_purpose=ModelUsagePurpose.MEMORY_EXTRACTION, - embedding_purpose=ModelUsagePurpose.MEMORY_INDEXING, - ) as context: - window_limit = self._runtime.source_window_limit if limit is None else limit - async with self._runtime._locked(self.scope_id): + window_limit = self._runtime.source_window_limit if limit is None else limit + if self._runtime._memory_flusher is not None: + async with self._runtime._scope_operation(self.scope_id): with self._runtime._stage("memory.flush", attributes={}) as span: - result = await context.triggers.flush(limit=window_limit) + result = await self._runtime._memory_flusher(self.scope_id, window_limit) if span is not None: span.set_attributes({"powercontext.memory.flush.source_count": result.source_count}) span.set_outcome("success" if result.processed else "noop") return result + async with ( + self._runtime._context( + self.scope_id, + generation_purpose=ModelUsagePurpose.MEMORY_EXTRACTION, + embedding_purpose=ModelUsagePurpose.MEMORY_INDEXING, + ) as context, + self._runtime._locked(self.scope_id), + ): + with self._runtime._stage("memory.flush", attributes={}) as span: + result = await context.triggers.flush(limit=window_limit) + if span is not None: + span.set_attributes({"powercontext.memory.flush.source_count": result.source_count}) + span.set_outcome("success" if result.processed else "noop") + return result async def cursor(self) -> SourceCursor: async with self._runtime._context(self.scope_id) as context: @@ -1529,140 +1535,6 @@ def for_scope(self, scope_id: str, /) -> ScopedMemoryApplication: return ScopedMemoryApplication(self._runtime, scope_id) -class ScheduledSourceProcessor: - """Map APScheduler activations to scoped Source-window policies.""" - - def __init__(self, runtime: BuiltinRuntime, scope_ids: ScopeIds) -> None: - self._runtime = runtime - self._scope_ids = scope_ids - - async def run(self) -> None: - async with self._runtime._processor_lock: - if self._runtime._closing or self._runtime._closed: - return - for scope_id in await self._scope_ids(): - if self._runtime._closing or self._runtime._closed: - return - started_at = perf_counter() - with self._runtime._background( - "scheduled.process_source_window", - operation="process_source_window", - ) as span: - try: - result = await self._runtime.memory.for_scope(scope_id).flush() - except asyncio.CancelledError: - _log_scheduled_processing( - "cancelled", - operation="process_source_window", - started_at=started_at, - ) - raise - except Exception as error: - _log_scheduled_processing( - "failure", - operation="process_source_window", - started_at=started_at, - error=error, - ) - if span is not None: - span.set_outcome("failure") - else: - outcome = "success" if result.processed else "noop" - _log_scheduled_processing( - outcome, - operation="process_source_window", - started_at=started_at, - source_count=result.source_count, - ) - if span is not None: - span.set_outcome(outcome) - span.set_attributes({"powercontext.background.source_count": result.source_count}) - - -class ScheduledExperienceProcessor: - """Map APScheduler activations to scoped Experience incubation windows.""" - - def __init__(self, runtime: BuiltinRuntime, scope_ids: ScopeIds) -> None: - self._runtime = runtime - self._scope_ids = scope_ids - - async def run(self) -> None: - async with self._runtime._processor_lock: - if self._runtime._closing or self._runtime._closed: - return - for scope_id in await self._scope_ids(): - if self._runtime._closing or self._runtime._closed: - return - started_at = perf_counter() - with self._runtime._background( - "scheduled.incubate_experience_candidates", - operation="incubate_experience_candidates", - ) as span: - try: - result = await self._runtime.experience.for_scope(scope_id).incubate() - except asyncio.CancelledError: - _log_scheduled_processing( - "cancelled", - operation="incubate_experience_candidates", - started_at=started_at, - ) - raise - except Exception as error: - _log_scheduled_processing( - "failure", - operation="incubate_experience_candidates", - started_at=started_at, - error=error, - ) - if span is not None: - span.set_outcome("failure") - else: - outcome = "success" if result.processed else "noop" - _log_scheduled_processing( - outcome, - operation="incubate_experience_candidates", - started_at=started_at, - source_count=result.source_count, - candidate_count=result.candidate_count, - ) - if span is not None: - span.set_outcome(outcome) - span.set_attributes({ - "powercontext.background.source_count": result.source_count, - "powercontext.background.candidate_count": result.candidate_count, - }) - - -def _log_scheduled_processing( - outcome: str, - *, - operation: str, - started_at: float, - error: Exception | None = None, - source_count: int | None = None, - candidate_count: int | None = None, -) -> None: - extra = { - "event": "background.operation.completed", - "operation": operation, - "outcome": outcome, - "unit": "background", - "duration_ms": max(perf_counter() - started_at, 0) * 1_000, - } - if source_count is not None: - extra["source_count"] = source_count - if candidate_count is not None: - extra["candidate_count"] = candidate_count - level = logging.ERROR if error is not None else logging.INFO - log_safely( - logger, - level, - "Scheduled background processing completed" if error is None else "Scheduled background processing failed", - exc_info=error, - extra=extra, - ) - - class BuiltinRuntime: """Add business-specific operations over composed built-in contexts.""" @@ -1675,7 +1547,6 @@ def __init__( scope_cache_size: int = DEFAULT_SCOPE_CACHE_SIZE, scope_evictor: ScopeEvictor | None = None, scope_cache_observer: ScopeCacheObserver | None = None, - scope_ids: ScopeIds | None = None, review_service: ReviewServiceFactory | None = None, generation_service: GenerationServiceFactory | None = None, experience_recall: ExperienceRecall | None = None, @@ -1695,6 +1566,8 @@ def __init__( remote_skill_distribution: RemoteSkillDistributionService | None = None, statistics_service: StatisticsServiceFactory | None = None, recall_token_estimator: RecallTokenEstimator | None = None, + memory_flusher: MemoryFlusher | None = None, + operations: OperationManager | None = None, publication_application: ArtifactPublicationApplication | None = None, scope_application: ScopeApplication | None = None, readiness: RuntimeReadinessChecks | None = None, @@ -1727,6 +1600,7 @@ def __init__( self._remote_skill_distribution = remote_skill_distribution self._statistics_service = statistics_service self._recall_token_estimator = recall_token_estimator + self._memory_flusher = memory_flusher self.publications = publication_application self.scopes = scope_application self._readiness = RuntimeReadinessChecks() if readiness is None else readiness @@ -1738,15 +1612,12 @@ def __init__( evictor=scope_evictor, observer=scope_cache_observer, ) - self._processor_lock = asyncio.Lock() self._close_lock = asyncio.Lock() self._lifecycle = asyncio.Condition() self._active_operations = 0 self._operation_depths: dict[asyncio.Task[Any], int] = {} self._closing = False self._closed = False - self._scheduler: AsyncIOScheduler | None = None - self._scheduler_runtime_key: str | None = None self.sources = SourceApplication(self) self.ingestion = RemoteIngestionApplication(self, remote_ingestion) self.context = ContextApplication(self) @@ -1759,11 +1630,8 @@ def __init__( self.skill = SkillApplication(self) self.remote_skills = RemoteSkillApplication(self) self.statistics = StatisticsApplication(self) + self.operations = operations self.handoff_report: HandoffReportApplication | None = None - self.processor = None if scope_ids is None else ScheduledSourceProcessor(self, scope_ids) - self.experience_processor = ( - None if scope_ids is None or experience_incubator is None else ScheduledExperienceProcessor(self, scope_ids) - ) async def __aenter__(self) -> BuiltinRuntime: return self @@ -1785,95 +1653,15 @@ async def readiness(self) -> RuntimeReadiness: checks={"runtime": ReadinessCheckStatus.READY, **dependencies.checks}, ) - def start_scheduler( - self, - scheduler_path: str | Path, - schedule_seconds: float | None, - *, - experience_schedule_seconds: float | None = None, - ) -> None: - """Start the APScheduler time adapter for this Runtime.""" - - if schedule_seconds is None and experience_schedule_seconds is None: - raise _RuntimeConfigurationError("schedule_seconds") - if schedule_seconds is not None and schedule_seconds <= 0: - raise _RuntimeConfigurationError("schedule_seconds") - if experience_schedule_seconds is not None and experience_schedule_seconds <= 0: - raise _RuntimeConfigurationError("experience_schedule_seconds") - if schedule_seconds is not None and self.processor is None: - raise _RuntimeConfigurationError("scope_ids") - if experience_schedule_seconds is not None and self.experience_processor is None: - raise _RuntimeStateError("experience-incubation") - if self._scheduler is not None: - raise _RuntimeStateError("scheduler") - from powercontext.builtin.runtime.scheduler import ( - configure_experience_incubation_job, - configure_source_window_job, - create_scheduler, - register_processors, - scheduler_runtime_key, - unregister_processor, - ) - - runtime_key = scheduler_runtime_key(scheduler_path) - scheduler: AsyncIOScheduler | None = None - register_processors( - runtime_key, - source_window=None if schedule_seconds is None or self.processor is None else self.processor.run, - experience_incubation=( - None - if experience_schedule_seconds is None or self.experience_processor is None - else self.experience_processor.run - ), - ) - self._scheduler_runtime_key = runtime_key - try: - scheduler = create_scheduler(scheduler_path) - self._scheduler = scheduler - scheduler.start(paused=True) - configure_source_window_job( - scheduler, - runtime_key=runtime_key, - schedule_seconds=schedule_seconds, - ) - configure_experience_incubation_job( - scheduler, - runtime_key=runtime_key, - schedule_seconds=experience_schedule_seconds, - ) - scheduler.resume() - except BaseException: - if scheduler is not None and scheduler.running: - scheduler.shutdown(wait=False) - unregister_processor(runtime_key) - self._scheduler_runtime_key = None - self._scheduler = None - raise - async def close(self) -> None: """Stop accepting work and await in-flight operations without closing the provider.""" async with self._close_lock: if self._closed: return - if self._scheduler is not None and self._scheduler.running: - self._scheduler.pause() async with self._lifecycle: self._closing = True await self._lifecycle.wait_for(lambda: self._active_operations == 0) - async with self._processor_lock: - pass - try: - if self._scheduler is not None and self._scheduler.running: - self._scheduler.shutdown(wait=False) - await asyncio.sleep(0) - finally: - if self._scheduler_runtime_key is not None: - from powercontext.builtin.runtime.scheduler import unregister_processor - - unregister_processor(self._scheduler_runtime_key) - self._scheduler_runtime_key = None - self._scheduler = None self._scope_cache.clear() self._closed = True @@ -1976,16 +1764,6 @@ def _stage( return nullcontext(None) return self._tracing.stage(name, attributes=attributes) - def _background( - self, - name: str, - *, - operation: str, - ) -> AbstractContextManager[RuntimeSpan | None]: - if self._tracing is None: - return nullcontext(None) - return self._tracing.background(name, operation=operation, attributes={}) - def _review(self, scope_id: str) -> ReviewService: if self._review_service is None: raise _RuntimeStateError("review") diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index 4242be3a8..e8e9ed55a 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -16,16 +16,24 @@ from __future__ import annotations +import asyncio import os -from collections.abc import AsyncIterator, Callable, Mapping +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping from contextlib import AsyncExitStack, asynccontextmanager +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal, TypeVar, cast +from typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar, cast from pydantic import AnyHttpUrl, JsonValue, SecretStr +from sqlalchemy.ext.asyncio import AsyncConnection from typing_extensions import override -from powercontext.builtin.artifacts.experience import ExperienceCandidatePipeline, ExperienceGenerator +from powercontext.builtin.artifacts.experience import ( + EXPERIENCE_INCUBATION_WINDOW_LIMIT, + ExperienceCandidatePipeline, + ExperienceGenerator, +) from powercontext.builtin.artifacts.handoff import ( DefaultHandoffEvidenceProjector, HandoffGenerationPipeline, @@ -33,6 +41,7 @@ from powercontext.builtin.artifacts.memory import ( CandidatePipeline, DefaultMemoryEvidenceProjector, + EmbeddingProfile, MemoryCapabilities, MemoryHit, MemoryRerankDecision, @@ -46,13 +55,17 @@ UsageReportingEmbeddingModel, UsageReportingStructuredGenerator, ) +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.experience_index import ExperienceIndex from powercontext.builtin.persistence.memory_index import CompositeMemoryIndex, MemoryIndex +from powercontext.builtin.persistence.migration import migrate_database, require_current_schema from powercontext.builtin.persistence.oceanbase.experience_index import OceanBaseExperienceFTSIndex from powercontext.builtin.persistence.oceanbase.memory_index import ( OceanBaseMemoryFTSIndex, OceanBaseMemoryVectorIndex, ) from powercontext.builtin.persistence.oceanbase.profile import OceanBaseConfig, OceanBaseProfile +from powercontext.builtin.persistence.schema import create_tables from powercontext.builtin.persistence.seekdb.profile import SeekDBConfig, SeekDBProfile from powercontext.builtin.persistence.skill_distribution_schema import ensure_skill_distribution_schema from powercontext.builtin.persistence.sqlite.experience_index import SQLiteExperienceFTSIndex @@ -62,7 +75,10 @@ from powercontext.builtin.runtime._scope_cache import ScopeCacheObserver from powercontext.builtin.runtime.application import BuiltinRuntime from powercontext.builtin.runtime.config import BuiltinConfig, ExternalSkillsConfig, InferenceConfig, RuntimeConfig +from powercontext.builtin.runtime.durable_scheduler import DurableScheduler, WorkDiscoverer +from powercontext.builtin.runtime.membership import RuntimeMembership from powercontext.builtin.runtime.models import MemorySearchMode, RuntimeCapabilities +from powercontext.builtin.runtime.operations import OperationManager from powercontext.builtin.runtime.protocols import RuntimeTracing from powercontext.builtin.runtime.readiness import ( CachedReadinessProbe, @@ -72,6 +88,16 @@ dependency_readiness_probe, ) from powercontext.builtin.runtime.relational import RelationalContexts +from powercontext.builtin.runtime.work_handlers import ( + ExperienceWorkDiscoverer, + ExperienceWorkHandler, + MemoryWorkDiscoverer, + MemoryWorkHandler, + OperationMaintenanceDiscoverer, + OperationMaintenanceHandler, +) +from powercontext.builtin.runtime.work_observability import WorkObserver +from powercontext.builtin.runtime.worker import DurableWorker, WorkHandler from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY, TEXT_EVIDENCE_PROJECTION_KEY from powercontext.errors import InvalidSourceProjectionError, SourceProjectionNotFoundError from powercontext.sources import Source, SourceDefinitionRegistry, SourceProjectionKey @@ -84,6 +110,17 @@ ValueT = TypeVar("ValueT") +@dataclass(frozen=True, slots=True) +class _RuntimeStorage: + database: AsyncDatabase + index: CompositeMemoryIndex + experience_index: ExperienceIndex + + +class _SchemaVerifier(Protocol): + async def verify(self, connection: AsyncConnection, /) -> None: ... + + class BuiltinConfigurationError(RuntimeError): """Report a configuration that cannot assemble the built-in runtime.""" @@ -97,6 +134,7 @@ def __init__(self, issue: str) -> None: "memory-reranker": "Memory reranking requires a configured generation or rerank model, or injected reranker", "scheduled-experience-pipeline": "scheduled Experience incubation requires a candidate pipeline", "scheduled-pipeline": "scheduled Source processing requires a candidate pipeline", + "worker-pipeline": "the worker role requires a configured Memory candidate pipeline", "database": "unsupported built-in database", } super().__init__(messages[issue]) @@ -176,11 +214,19 @@ async def open_builtin_runtime( instrumentation: InstrumentationSettings | None = None, scope_cache_observer: ScopeCacheObserver | None = None, tracing: RuntimeTracing | None = None, + work_observer: WorkObserver | None = None, source_registry: SourceDefinitionRegistry | None = None, ) -> AsyncIterator[BuiltinRuntime]: - """Open the selected database, inference adapters, and built-in runtime.""" + """Open the selected database, inference adapters, and built-in runtime. + + ``scheduler_path`` is retained for bridge-release source compatibility and + is ignored because scheduling state now lives in the primary database. + """ + + del scheduler_path async with AsyncExitStack() as resources: + scheduler_only = config.deployment.role == "scheduler" configured_source_registry = source_registry or BUILTIN_SOURCE_REGISTRY ( generated_memory, @@ -199,7 +245,8 @@ async def open_builtin_runtime( instrumentation, configured_source_registry, ) - if ( + if not scheduler_only + and ( candidate_pipeline is None or experience_pipeline is None or experience_generator is None @@ -209,15 +256,28 @@ async def open_builtin_runtime( ) else (None, None, None, None, None, None, None, None) ) - configured_pipeline = generated_memory if candidate_pipeline is None else candidate_pipeline - configured_incubation = generated_incubation if experience_pipeline is None else experience_pipeline - configured_experience = generated_experience if experience_generator is None else experience_generator - configured_skill = generated_skill if skill_generator is None else skill_generator - configured_handoff = generated_handoff if handoff_pipeline is None else handoff_pipeline - configured_reranker = generated_reranker if memory_reranker is None else memory_reranker + configured_pipeline = ( + None if scheduler_only else generated_memory if candidate_pipeline is None else candidate_pipeline + ) + configured_incubation = ( + None if scheduler_only else generated_incubation if experience_pipeline is None else experience_pipeline + ) + configured_experience = ( + None if scheduler_only else generated_experience if experience_generator is None else experience_generator + ) + configured_skill = None if scheduler_only else generated_skill if skill_generator is None else skill_generator + configured_handoff = ( + None if scheduler_only else generated_handoff if handoff_pipeline is None else handoff_pipeline + ) + configured_reranker = ( + None if scheduler_only else generated_reranker if memory_reranker is None else memory_reranker + ) if configured_reranker is not None and tracing is not None: configured_reranker = _TracingMemoryReranker(configured_reranker, tracing) - if embedding_model is None: + if scheduler_only: + configured_embedding_source = None + readiness_embedding = None + elif embedding_model is None: configured_embedding_source, readiness_embedding = await _embedding_models( config.inference, resources, @@ -235,11 +295,13 @@ async def open_builtin_runtime( configured_embedding = ( None if configured_embedding_source is None else UsageReportingEmbeddingModel(configured_embedding_source) ) - configured_external_skills = ( - _external_skill_provider(config.external_skills) - if external_skill_provider is None - else external_skill_provider - ) + configured_external_skills = None + if not scheduler_only: + configured_external_skills = ( + _external_skill_provider(config.external_skills) + if external_skill_provider is None + else external_skill_provider + ) contexts = await resources.enter_async_context( open_builtin_contexts( config, @@ -255,23 +317,42 @@ async def open_builtin_runtime( source_registry=configured_source_registry, ) ) - readiness_probes: dict[str, ReadinessProbeDefinition] = { - "database": ReadinessProbeDefinition( - probe=dependency_readiness_probe(contexts.database.ping), - blocking=True, - ), - } - inference_readiness = ( - ("inference.generation", generation_readiness), - ("inference.rerank", rerank_readiness), - ( - "inference.embedding", - None if readiness_embedding is None else _embedding_readiness_probe(readiness_embedding), - ), + _validate_work_configuration(config, configured_pipeline, configured_incubation, configured_reranker) + membership = RuntimeMembership( + database=contexts.database, + deployment=config.deployment, + coordination=config.coordination, + build_version=_package_version(), + observer=work_observer, + ) + await membership.start() + membership_task = asyncio.create_task(membership.run(), name="powercontext-membership") + resources.push_async_callback(_stop_background_component, membership, membership_task) + local_worker, operation_manager = _work_services( + config, + contexts, + membership.instance_id, + claim_readiness=generation_readiness, + observer=work_observer, + tracing=tracing, + ) + local_scheduler = _scheduler_service( + config, + contexts, + membership.instance_id, + observer=work_observer, + tracing=tracing, + ) + readiness_probes = _runtime_readiness_probes( + config, + contexts, + membership, + worker=local_worker, + scheduler=local_scheduler, + generation=generation_readiness, + rerank=rerank_readiness, + embedding=None if readiness_embedding is None else _embedding_readiness_probe(readiness_embedding), ) - for name, readiness_probe in inference_readiness: - if readiness_probe is not None: - readiness_probes[name] = ReadinessProbeDefinition(probe=readiness_probe, blocking=False) runtime = await resources.enter_async_context( BuiltinRuntime( provider=contexts, @@ -287,7 +368,6 @@ async def open_builtin_runtime( scope_cache_size=config.runtime.scope_cache_size, scope_evictor=contexts.evict, scope_cache_observer=scope_cache_observer, - scope_ids=contexts.scope_ids, review_service=contexts.review, generation_service=contexts.generation, experience_recall=contexts.search_experience, @@ -307,6 +387,10 @@ async def open_builtin_runtime( remote_skill_distribution=contexts.remote_skill_distribution(), statistics_service=contexts.statistics, recall_token_estimator=contexts.estimate_recall_tokens, + memory_flusher=(lambda scope_id, limit: operation_manager.flush_memory(scope_id, limit=limit)) + if config.deployment.role == "all" + else None, + operations=operation_manager, publication_application=contexts.publications, scope_application=contexts.scopes, readiness=RuntimeReadinessChecks(readiness_probes), @@ -319,21 +403,195 @@ async def open_builtin_runtime( contexts.scopes, RuntimeHandoffReadAdapter(runtime.handoff), ) - if config.runtime.schedule_seconds is not None and configured_pipeline is None: - raise BuiltinConfigurationError("scheduled-pipeline") - if config.runtime.experience_schedule_seconds is not None and configured_incubation is None: - raise BuiltinConfigurationError("scheduled-experience-pipeline") - if config.runtime.memory_rerank_enabled and configured_reranker is None: - raise BuiltinConfigurationError("memory-reranker") - if config.runtime.schedule_seconds is not None or config.runtime.experience_schedule_seconds is not None: - runtime.start_scheduler( - scheduler_path, - config.runtime.schedule_seconds, - experience_schedule_seconds=config.runtime.experience_schedule_seconds, - ) + _start_work_background(local_worker, local_scheduler, resources) yield runtime +async def _stop_background_component(component: Any, task: asyncio.Task[None]) -> None: + await component.stop() + await task + + +def _validate_work_configuration( + config: BuiltinConfig, + memory: CandidatePipeline | None, + experience: ExperienceCandidatePipeline | None, + reranker: MemoryReranker | None, +) -> None: + if config.runtime.schedule_seconds is not None and memory is None and config.deployment.role == "all": + raise BuiltinConfigurationError("scheduled-pipeline") + if ( + config.runtime.experience_schedule_seconds is not None + and experience is None + and config.deployment.role == "all" + ): + raise BuiltinConfigurationError("scheduled-experience-pipeline") + if config.runtime.memory_rerank_enabled and reranker is None: + raise BuiltinConfigurationError("memory-reranker") + if config.deployment.role == "worker" and memory is None: + raise BuiltinConfigurationError("worker-pipeline") + + +def _work_services( + config: BuiltinConfig, + contexts: RelationalContexts, + owner_id: str, + *, + claim_readiness: ReadinessProbe | None, + observer: WorkObserver | None, + tracing: RuntimeTracing | None, +) -> tuple[DurableWorker | None, OperationManager]: + handlers: list[WorkHandler] = [ + MemoryWorkHandler(contexts), + OperationMaintenanceHandler( + retention_days=config.operations.retention_days, + batch_size=config.operations.cleanup_batch_size, + ), + ] + if contexts.experience_incubation: + handlers.append(ExperienceWorkHandler(contexts)) + local_worker = ( + DurableWorker( + database=contexts.database, + worker_id=owner_id, + handlers=handlers, + config=config.worker, + claim_readiness=claim_readiness, + observer=observer, + tracing=tracing, + ) + if config.deployment.role in {"all", "worker"} + else None + ) + return local_worker, OperationManager( + contexts=contexts, + operations=config.operations, + worker=config.worker, + local_worker=local_worker, + payload_version=config.coordination.emit_payload_version, + memory_window_limit=config.runtime.source_window_limit, + observer=observer, + tracing=tracing, + ) + + +def _scheduler_service( + config: BuiltinConfig, + contexts: RelationalContexts, + owner_id: str, + *, + observer: WorkObserver | None, + tracing: RuntimeTracing | None, +) -> DurableScheduler | None: + discoverers = _work_discoverers(config, contexts) + if not discoverers or config.deployment.role not in {"all", "scheduler"}: + return None + return DurableScheduler( + database=contexts.database, + scheduler_id=owner_id, + discoverers=discoverers, + config=config.coordination, + observer=observer, + tracing=tracing, + ) + + +def _start_work_background( + worker: DurableWorker | None, + scheduler: DurableScheduler | None, + resources: AsyncExitStack, +) -> None: + if worker is not None: + worker_task = asyncio.create_task(worker.run(), name="powercontext-worker") + resources.push_async_callback(_stop_background_component, worker, worker_task) + if scheduler is None: + return + scheduler_task = asyncio.create_task(scheduler.run(), name="powercontext-scheduler") + resources.push_async_callback(_stop_background_component, scheduler, scheduler_task) + + +def _package_version() -> str: + try: + return version("powercontext") + except PackageNotFoundError: + return "0+unknown" + + +def _runtime_readiness_probes( + config: BuiltinConfig, + contexts: RelationalContexts, + membership: RuntimeMembership, + *, + worker: DurableWorker | None, + scheduler: DurableScheduler | None, + generation: ReadinessProbe | None, + rerank: ReadinessProbe | None, + embedding: ReadinessProbe | None, +) -> dict[str, ReadinessProbeDefinition]: + probes = { + "database": ReadinessProbeDefinition( + probe=dependency_readiness_probe(contexts.database.ping), + blocking=True, + ), + } + if config.deployment.mode == "distributed": + probes["runtime.membership"] = ReadinessProbeDefinition(probe=membership.readiness, blocking=True) + if config.deployment.role == "api": + probes["runtime.scheduler"] = ReadinessProbeDefinition( + probe=lambda: membership.role_readiness("scheduler"), + blocking=False, + ) + probes["runtime.worker"] = ReadinessProbeDefinition( + probe=lambda: membership.role_readiness("worker"), + blocking=False, + ) + if config.deployment.mode == "distributed" and worker is not None: + probes["worker.handlers"] = ReadinessProbeDefinition(probe=worker.readiness, blocking=True) + if config.deployment.mode == "distributed" and scheduler is not None: + probes["scheduler.discovery"] = ReadinessProbeDefinition(probe=scheduler.readiness, blocking=True) + for name, probe in ( + ("inference.generation", generation), + ("inference.rerank", rerank), + ("inference.embedding", embedding), + ): + if probe is not None: + probes[name] = ReadinessProbeDefinition( + probe=probe, + blocking=config.deployment.role == "worker" and name == "inference.generation", + ) + return probes + + +def _work_discoverers(config: BuiltinConfig, contexts: RelationalContexts) -> list[WorkDiscoverer]: + discoverers: list[WorkDiscoverer] = [ + OperationMaintenanceDiscoverer( + interval_seconds=config.operations.cleanup_interval_seconds, + max_attempts=config.worker.max_attempts, + ) + ] + if config.runtime.schedule_seconds is not None: + discoverers.append( + MemoryWorkDiscoverer( + contexts, + interval_seconds=config.runtime.schedule_seconds, + window_limit=config.runtime.source_window_limit, + max_attempts=config.worker.max_attempts, + payload_version=config.coordination.emit_payload_version, + ) + ) + if config.runtime.experience_schedule_seconds is not None: + discoverers.append( + ExperienceWorkDiscoverer( + contexts, + interval_seconds=config.runtime.experience_schedule_seconds, + window_limit=EXPERIENCE_INCUBATION_WINDOW_LIMIT, + max_attempts=config.worker.max_attempts, + payload_version=config.coordination.emit_payload_version, + ) + ) + return discoverers + + @asynccontextmanager async def open_builtin_contexts( config: BuiltinConfig, @@ -351,77 +609,99 @@ async def open_builtin_contexts( ) -> AsyncIterator[RelationalContexts]: """Open the selected database and expose scope-bound PowerContext providers.""" - database = config.database configured_token_estimator = character_token_estimator() if token_estimator is None else token_estimator + embedding_profile = None if embedding_model is None else embedding_model.profile + async with _open_runtime_storage(config, embedding_profile=embedding_profile) as storage: + await _prepare_runtime_schema(config, storage) + contexts = RelationalContexts( + database=storage.database, + index=storage.index, + experience_index=storage.experience_index, + candidate_pipeline=candidate_pipeline, + experience_pipeline=experience_pipeline, + experience_generator=experience_generator, + skill_generator=skill_generator, + external_skill_provider=external_skill_provider, + handoff_pipeline=handoff_pipeline, + embedding_model=embedding_model, + token_estimator=configured_token_estimator, + memory_reranker=memory_reranker, + memory_rerank_candidate_limit=config.runtime.memory_rerank_candidate_limit, + source_registry=source_registry, + ) + await contexts.scopes.bootstrap_default() + yield contexts + + +async def migrate_builtin_database( + config: BuiltinConfig, + /, + *, + embedding_profile: EmbeddingProfile | None = None, +) -> str: + """Migrate the configured store using the same physical layout as the runtime.""" + + async with _open_runtime_storage(config, embedding_profile=embedding_profile) as storage: + return await migrate_database(storage.database, provision=_schema_provisioner(storage)) + + +@asynccontextmanager +async def _open_runtime_storage( + config: BuiltinConfig, + *, + embedding_profile: EmbeddingProfile | None, +) -> AsyncIterator[_RuntimeStorage]: + database = config.database if isinstance(database, SQLiteConfig): - experience_index = SQLiteExperienceFTSIndex() + experience_index: ExperienceIndex = SQLiteExperienceFTSIndex() indexes: list[MemoryIndex] = [SQLiteMemoryFTSIndex()] - if embedding_model is not None: - indexes.append(SQLiteMemoryVectorIndex(embedding_model.profile)) + if embedding_profile is not None: + indexes.append(SQLiteMemoryVectorIndex(embedding_profile)) index = CompositeMemoryIndex(*indexes) async with SQLiteProfile.open( database, tables=BUILTIN_TABLES + index.tables, - load_vector_extension=embedding_model is not None, + load_vector_extension=embedding_profile is not None, + create_schema=False, ) as profile: - async with profile.database.transaction() as connection: - await ensure_skill_distribution_schema(connection) - await index.initialize(connection) - await experience_index.initialize(connection) - contexts = RelationalContexts( - database=profile.database, - index=index, - experience_index=experience_index, - candidate_pipeline=candidate_pipeline, - experience_pipeline=experience_pipeline, - experience_generator=experience_generator, - skill_generator=skill_generator, - external_skill_provider=external_skill_provider, - handoff_pipeline=handoff_pipeline, - embedding_model=embedding_model, - token_estimator=configured_token_estimator, - memory_reranker=memory_reranker, - memory_rerank_candidate_limit=config.runtime.memory_rerank_candidate_limit, - source_registry=source_registry, - ) - await contexts.scopes.bootstrap_default() - yield contexts + yield _RuntimeStorage(profile.database, index, experience_index) return + experience_index = OceanBaseExperienceFTSIndex() indexes = [OceanBaseMemoryFTSIndex()] - if embedding_model is not None: - indexes.append(OceanBaseMemoryVectorIndex(embedding_model.profile)) + if embedding_profile is not None: + indexes.append(OceanBaseMemoryVectorIndex(embedding_profile)) index = CompositeMemoryIndex(*indexes) tables = BUILTIN_TABLES + index.tables if isinstance(database, OceanBaseConfig): - profile_context = OceanBaseProfile.open(database, tables=tables) + profile_context = OceanBaseProfile.open(database, tables=tables, create_schema=False) elif isinstance(database, SeekDBConfig): - profile_context = SeekDBProfile.open(database, tables=tables) + profile_context = SeekDBProfile.open(database, tables=tables, create_schema=False) else: raise BuiltinConfigurationError("database") async with profile_context as profile: - async with profile.database.transaction() as connection: - await ensure_skill_distribution_schema(connection) - await index.initialize(connection) - await experience_index.initialize(connection) - contexts = RelationalContexts( - database=profile.database, - index=index, - experience_index=experience_index, - candidate_pipeline=candidate_pipeline, - experience_pipeline=experience_pipeline, - experience_generator=experience_generator, - skill_generator=skill_generator, - external_skill_provider=external_skill_provider, - handoff_pipeline=handoff_pipeline, - embedding_model=embedding_model, - token_estimator=configured_token_estimator, - memory_reranker=memory_reranker, - memory_rerank_candidate_limit=config.runtime.memory_rerank_candidate_limit, - source_registry=source_registry, - ) - await contexts.scopes.bootstrap_default() - yield contexts + yield _RuntimeStorage(profile.database, index, experience_index) + + +async def _prepare_runtime_schema(config: BuiltinConfig, storage: _RuntimeStorage) -> None: + if config.deployment.mode == "distributed": + await require_current_schema(storage.database) + async with storage.database.transaction() as connection: + await storage.index.verify(connection) + await cast(_SchemaVerifier, storage.experience_index).verify(connection) + return + await migrate_database(storage.database, provision=_schema_provisioner(storage)) + + +def _schema_provisioner(storage: _RuntimeStorage) -> Callable[[AsyncConnection], Awaitable[None]]: + async def provision(connection: AsyncConnection) -> None: + await ensure_skill_distribution_schema(connection) + if storage.index.tables: + await create_tables(connection, storage.index.tables) + await storage.index.initialize(connection) + await storage.experience_index.initialize(connection) + + return provision async def _generation_pipelines( @@ -657,7 +937,7 @@ async def preflight_builtin_runtime(config: BuiltinConfig) -> None: """Validate Runtime composition without opening persistence or making requests.""" async with AsyncExitStack() as resources: - await _generation_pipelines( + memory, incubation, _, _, _, reranker, _, _ = await _generation_pipelines( config.inference, config.runtime, resources, @@ -666,14 +946,7 @@ async def preflight_builtin_runtime(config: BuiltinConfig) -> None: ) if config.inference.embedding_model is not None: await _embedding_models(config.inference, resources, None) - if config.runtime.schedule_seconds is not None and config.inference.generation_model is None: - raise BuiltinConfigurationError("scheduled-pipeline") - if config.runtime.experience_schedule_seconds is not None and config.inference.generation_model is None: - raise BuiltinConfigurationError("scheduled-experience-pipeline") - if config.runtime.memory_rerank_enabled and ( - config.inference.generation_model is None and config.inference.rerank_model is None - ): - raise BuiltinConfigurationError("memory-reranker") + _validate_work_configuration(config, memory, incubation, reranker) async def _open_pydantic_ai_model( @@ -872,4 +1145,10 @@ def _search_modes(capabilities: MemoryCapabilities) -> tuple[MemorySearchMode, . return tuple(modes) -__all__ = ["BuiltinConfigurationError", "open_builtin_contexts", "open_builtin_runtime", "preflight_builtin_runtime"] +__all__ = [ + "BuiltinConfigurationError", + "migrate_builtin_database", + "open_builtin_contexts", + "open_builtin_runtime", + "preflight_builtin_runtime", +] diff --git a/src/powercontext/builtin/runtime/config.py b/src/powercontext/builtin/runtime/config.py index d734c65fa..1d0095ea5 100644 --- a/src/powercontext/builtin/runtime/config.py +++ b/src/powercontext/builtin/runtime/config.py @@ -44,6 +44,95 @@ class RuntimeConfig(BaseModel): experience_schedule_seconds: float | None = Field(default=None, gt=0) +class DeploymentConfig(BaseModel): + """Process topology and non-sensitive compatibility identity.""" + + mode: Literal["single_node", "distributed"] = "single_node" + role: Literal["all", "api", "scheduler", "worker"] = "all" + id: str = Field(default="local", min_length=1, max_length=128) + behavior_revision: str = Field(default="default", min_length=1, max_length=128) + + @field_validator("id", "behavior_revision") + @classmethod + def validate_trimmed_identifier(cls, value: str) -> str: + if value != value.strip(): + raise ValueError("deployment identifiers must be trimmed") # noqa: TRY003 + return value + + +class CoordinationConfig(BaseModel): + """Database-backed scheduler and member lease policy.""" + + scheduler_lease_seconds: int = Field(default=30, ge=3) + scheduler_renew_seconds: int = Field(default=10, ge=1) + scan_page_size: int = Field(default=100, ge=1, le=100) + member_ttl_seconds: int = Field(default=30, ge=3) + member_heartbeat_seconds: int = Field(default=10, ge=1) + emit_payload_version: int = Field(default=1, ge=1, le=1) + + @model_validator(mode="after") + def validate_lease_intervals(self) -> CoordinationConfig: + if self.scheduler_renew_seconds * 3 > self.scheduler_lease_seconds: + raise ValueError( # noqa: TRY003 + "scheduler_renew_seconds must not exceed one third of scheduler_lease_seconds" + ) + if self.member_heartbeat_seconds * 3 > self.member_ttl_seconds: + raise ValueError( # noqa: TRY003 + "member_heartbeat_seconds must not exceed one third of member_ttl_seconds" + ) + return self + + +class WorkerConfig(BaseModel): + """Worker claim, retry, heartbeat, and drain policy.""" + + concurrency: int = Field(default=4, ge=1, le=256) + lease_seconds: int = Field(default=120, ge=3) + heartbeat_seconds: int = Field(default=30, ge=1) + shutdown_grace_seconds: int = Field(default=90, ge=0) + max_attempts: int = Field(default=5, ge=1, le=100) + retry_base_seconds: float = Field(default=2.0, gt=0) + retry_max_seconds: float = Field(default=300.0, gt=0) + poll_seconds: float = Field(default=1.0, gt=0) + + @model_validator(mode="after") + def validate_lease_intervals(self) -> WorkerConfig: + if self.heartbeat_seconds * 3 >= self.lease_seconds: + raise ValueError( # noqa: TRY003 + "heartbeat_seconds must be less than one third of lease_seconds" + ) + if self.shutdown_grace_seconds >= self.lease_seconds: + raise ValueError("shutdown_grace_seconds must be less than lease_seconds") # noqa: TRY003 + if self.retry_max_seconds < self.retry_base_seconds: + raise ValueError("retry_max_seconds must not be less than retry_base_seconds") # noqa: TRY003 + return self + + +class OperationsConfig(BaseModel): + """Synchronous wait facade and durable operation retention policy.""" + + default_wait_seconds: float = Field(default=10.0, ge=0, le=30) + maximum_wait_seconds: float = Field(default=30.0, ge=0, le=30) + poll_seconds: float = Field(default=0.2, gt=0) + retention_days: int = Field(default=30, ge=1) + cleanup_batch_size: int = Field(default=500, ge=1, le=500) + cleanup_interval_seconds: float = Field(default=3600.0, gt=0) + + @model_validator(mode="after") + def validate_wait_policy(self) -> OperationsConfig: + if self.default_wait_seconds > self.maximum_wait_seconds: + raise ValueError("default_wait_seconds must not exceed maximum_wait_seconds") # noqa: TRY003 + return self + + +class RateLimitConfig(BaseModel): + """Optional shared fixed-window request limit.""" + + enabled: bool = False + requests: int = Field(default=120, ge=1) + window_seconds: int = Field(default=60, ge=1) + + class HandoffReportConfig(BaseModel): """Optional Handoff Report feature registration.""" @@ -208,18 +297,42 @@ class BuiltinConfig(BaseModel): handoff_report: HandoffReportConfig = Field(default_factory=HandoffReportConfig) inference: InferenceConfig = Field(default_factory=InferenceConfig) external_skills: ExternalSkillsConfig = Field(default_factory=ExternalSkillsConfig) + deployment: DeploymentConfig = Field(default_factory=DeploymentConfig) + coordination: CoordinationConfig = Field(default_factory=CoordinationConfig) + worker: WorkerConfig = Field(default_factory=WorkerConfig) + operations: OperationsConfig = Field(default_factory=OperationsConfig) @model_validator(mode="before") @classmethod def default_database_to_sqlite(cls, value: Any) -> Any: return normalize_database_discriminator(value) + @model_validator(mode="after") + def validate_deployment(self) -> BuiltinConfig: + if self.deployment.mode == "single_node" and self.deployment.role != "all": + raise ValueError("single_node deployment role must be 'all'") # noqa: TRY003 + if self.deployment.mode == "distributed": + if not isinstance(self.database, OceanBaseConfig): + raise ValueError("distributed deployment requires OceanBase") # noqa: TRY003 + if self.deployment.role == "all": + raise ValueError("distributed deployment role must be api, scheduler, or worker") # noqa: TRY003 + if self.external_skills.agent_targets: + raise ValueError( # noqa: TRY003 + "distributed deployment does not support host-local external Skill targets" + ) + return self + __all__ = [ "BuiltinConfig", + "CoordinationConfig", "DatabaseConfig", + "DeploymentConfig", "ExternalSkillsConfig", "HandoffReportConfig", "InferenceConfig", + "OperationsConfig", + "RateLimitConfig", "RuntimeConfig", + "WorkerConfig", ] diff --git a/src/powercontext/builtin/runtime/durable_scheduler.py b/src/powercontext/builtin/runtime/durable_scheduler.py new file mode 100644 index 000000000..334ed2c43 --- /dev/null +++ b/src/powercontext/builtin/runtime/durable_scheduler.py @@ -0,0 +1,260 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fenced leader scheduler with durable bounded scan continuations.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Iterable +from contextlib import AbstractContextManager, nullcontext, suppress +from dataclasses import dataclass +from datetime import timedelta +from typing import Protocol + +from powercontext.builtin.persistence.coordination import ( + CoordinationRepository, + CoordinatorLease, + StaleCoordinatorLeaseError, +) +from powercontext.builtin.persistence.database import AsyncDatabase, database_now +from powercontext.builtin.persistence.work import WorkRepository, WorkSpec +from powercontext.builtin.runtime.config import CoordinationConfig +from powercontext.builtin.runtime.protocols import RuntimeSpan, RuntimeTracing +from powercontext.builtin.runtime.readiness import ReadinessCheckStatus +from powercontext.builtin.runtime.work_observability import WorkObserver, refresh_work_queue + +_SCHEDULER_LEASE_NAME = "work-discovery" +_EMPTY_DISCOVERERS = "at least one work discoverer is required" +_INVALID_DISCOVERER_INTERVAL = "discoverer interval must be positive" +_INVALID_DISCOVERER_NAME = "discoverer name must be a non-empty trimmed string" + + +@dataclass(frozen=True) +class DiscoveryPage: + """One bounded page and its next opaque keyset position.""" + + specs: tuple[WorkSpec, ...] + continuation: str | None + + +class WorkDiscoverer(Protocol): + """Find logical work without executing external side effects.""" + + name: str + interval_seconds: float + + async def page(self, continuation: str | None, limit: int, /) -> DiscoveryPage: ... + + +class DurableScheduler: + """Acquire one leader lease and enqueue pages under its exact fence.""" + + def __init__( + self, + *, + database: AsyncDatabase, + scheduler_id: str, + discoverers: Iterable[WorkDiscoverer], + config: CoordinationConfig, + coordination: CoordinationRepository | None = None, + work: WorkRepository | None = None, + observer: WorkObserver | None = None, + tracing: RuntimeTracing | None = None, + ) -> None: + self._database = database + self._scheduler_id = scheduler_id + self._discoverers = _discoverer_map(discoverers) + self._config = config + self._coordination = CoordinationRepository() if coordination is None else coordination + self._work = WorkRepository(observer=observer) if work is None else work + self._observer = observer + self._tracing = tracing + self._poll_seconds = min( + float(config.scheduler_renew_seconds), + *(discoverer.interval_seconds for discoverer in self._discoverers.values()), + ) + self._lease: CoordinatorLease | None = None + self._stop_requested = asyncio.Event() + self._failed = False + + async def tick(self) -> bool: + """Renew or acquire leadership and process at most one page per discoverer.""" + + previous = self._lease + with self._background( + "scheduler.tick", + operation="scheduler.tick", + attributes={"powercontext.scheduler.discoverer_count": len(self._discoverers)}, + ) as span: + async with self._database.transaction() as connection: + lease = await self._coordination.acquire_lease( + connection, + lease_name=_SCHEDULER_LEASE_NAME, + owner_id=self._scheduler_id, + lease_seconds=self._config.scheduler_lease_seconds, + ) + self._lease = lease + self._observe_leadership(previous, lease) + if lease is None: + if span is not None: + span.set_outcome("standby") + return False + for discoverer in self._discoverers.values(): + await self._scan_once(discoverer, lease) + if span is not None: + span.set_outcome("leader") + await refresh_work_queue(self._database, self._work, self._observer) + return True + + async def run(self) -> None: + """Maintain leadership until stopped; standby instances keep polling.""" + + while not self._stop_requested.is_set(): + try: + await self.tick() + self._failed = False + except asyncio.CancelledError: + raise + except StaleCoordinatorLeaseError: + self._observe_leadership(self._lease, None) + self._lease = None + except Exception: + self._observe_leadership(self._lease, None) + self._lease = None + self._failed = True + with suppress(TimeoutError): + await asyncio.wait_for( + self._stop_requested.wait(), + timeout=self._poll_seconds, + ) + + async def readiness(self) -> str: + """Treat both a healthy leader and a healthy standby as ready.""" + + return ReadinessCheckStatus.UNAVAILABLE if self._failed else ReadinessCheckStatus.READY + + async def stop(self) -> None: + """Stop discovery and conditionally release current leadership.""" + + self._stop_requested.set() + lease = self._lease + self._lease = None + self._observe_leadership(lease, None) + if lease is None: + return + async with self._database.transaction() as connection: + await self._coordination.release_lease(connection, lease) + + async def _scan_once(self, discoverer: WorkDiscoverer, lease: CoordinatorLease) -> None: + async with self._database.transaction() as connection: + await self._coordination.assert_lease(connection, lease) + scan = await self._coordination.load_scan(connection, discoverer.name) + now = await database_now(connection) + if scan is not None and scan.next_run_at > now: + return + + continuation = None if scan is None else scan.continuation + page = await discoverer.page(continuation, self._config.scan_page_size) + if len(page.specs) > self._config.scan_page_size: + message = f"discoverer {discoverer.name} exceeded the configured scan page" + raise ValueError(message) + for spec in page.specs: + with self._stage( + "work.enqueue", + attributes={ + "powercontext.work.kind": spec.kind, + "powercontext.work.payload_version": spec.payload_version, + }, + ) as span: + async with self._database.transaction() as connection: + await self._coordination.assert_lease(connection, lease) + result = await self._work.enqueue(connection, spec) + if span is not None: + span.set_outcome("created" if result.created else "joined") + + async with self._database.transaction() as connection: + await self._coordination.assert_lease(connection, lease) + now = await database_now(connection) + await self._coordination.save_scan( + connection, + discoverer.name, + next_run_at=now + if page.continuation is not None + else now + timedelta(seconds=discoverer.interval_seconds), + continuation=page.continuation, + expected_version=None if scan is None else scan.state_version, + ) + + def _observe_leadership( + self, + previous: CoordinatorLease | None, + current: CoordinatorLease | None, + ) -> None: + if self._observer is None: + return + outcome = None + if previous is None and current is not None: + outcome = "acquired" + elif previous is not None and current is None: + outcome = "lost" + elif previous is not None and current is not None and previous.fence != current.fence: + outcome = "reacquired" + if outcome is not None: + with suppress(Exception): + self._observer.observe_scheduler_leadership(outcome=outcome) + + def _stage( + self, + name: str, + *, + attributes: dict[str, str | bool | int | float], + ) -> AbstractContextManager[RuntimeSpan | None]: + if self._tracing is None: + return nullcontext(None) + return self._tracing.stage(name, attributes=attributes) + + def _background( + self, + name: str, + *, + operation: str, + attributes: dict[str, str | bool | int | float], + ) -> AbstractContextManager[RuntimeSpan | None]: + if self._tracing is None: + return nullcontext(None) + return self._tracing.background(name, operation=operation, attributes=attributes) + + +def _discoverer_map(discoverers: Iterable[WorkDiscoverer]) -> dict[str, WorkDiscoverer]: + registered: dict[str, WorkDiscoverer] = {} + for discoverer in discoverers: + if not discoverer.name.strip() or discoverer.name != discoverer.name.strip(): + raise ValueError(_INVALID_DISCOVERER_NAME) + if discoverer.interval_seconds <= 0: + raise ValueError(_INVALID_DISCOVERER_INTERVAL) + if discoverer.name in registered: + message = f"duplicate discoverer name: {discoverer.name}" + raise ValueError(message) + registered[discoverer.name] = discoverer + if not registered: + raise ValueError(_EMPTY_DISCOVERERS) + return registered + + +__all__ = [ + "DiscoveryPage", + "DurableScheduler", + "WorkDiscoverer", +] diff --git a/src/powercontext/builtin/runtime/membership.py b/src/powercontext/builtin/runtime/membership.py new file mode 100644 index 000000000..3d69880b3 --- /dev/null +++ b/src/powercontext/builtin/runtime/membership.py @@ -0,0 +1,197 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runtime role membership and single-node ownership leases.""" + +from __future__ import annotations + +import asyncio +import hashlib +from contextlib import suppress +from uuid import uuid4 + +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.coordination import ( + CoordinationRepository, + CoordinatorLease, + RuntimeMemberSpec, +) +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.runtime.config import CoordinationConfig, DeploymentConfig +from powercontext.builtin.runtime.readiness import ReadinessCheckStatus +from powercontext.builtin.runtime.work_observability import WorkObserver + +_SINGLE_NODE_LEASE = "single-node-runtime" +_SCHEMA_VERSION = 2 +_PAYLOAD_VERSION = 1 + + +class DuplicateSingleNodeError(RuntimeError): + """Raised before serving when another single-node runtime owns the database.""" + + def __init__(self) -> None: + super().__init__("another single-node runtime already owns this database") + + +class RuntimeMembership: + """Advertise compatibility and maintain exclusive local ownership.""" + + def __init__( + self, + *, + database: AsyncDatabase, + deployment: DeploymentConfig, + coordination: CoordinationConfig, + build_version: str, + repository: CoordinationRepository | None = None, + observer: WorkObserver | None = None, + ) -> None: + self._database = database + self._deployment = deployment + self._config = coordination + self._build_version = build_version + self._repository = CoordinationRepository() if repository is None else repository + self._observer = observer + boot_id = uuid4().hex + self._member_id = hashlib.sha256(f"{deployment.id}\0{boot_id}".encode()).hexdigest() + self._single_node_lease: CoordinatorLease | None = None + self._stop_requested = asyncio.Event() + self._failed = False + + @property + def instance_id(self) -> str: + """Return the boot-unique, non-sensitive owner used for fencing.""" + + return self._member_id + + async def start(self) -> None: + """Acquire required ownership and publish the first heartbeat.""" + + async with self._database.transaction() as connection: + if self._deployment.mode == "single_node": + self._single_node_lease = await self._repository.acquire_lease( + connection, + lease_name=_SINGLE_NODE_LEASE, + owner_id=self._member_id, + lease_seconds=self._config.member_ttl_seconds, + ) + if self._single_node_lease is None: + raise DuplicateSingleNodeError + await self._heartbeat(connection) + + async def run(self) -> None: + """Renew role and optional owner leases until shutdown.""" + + try: + while not self._stop_requested.is_set(): + with suppress(TimeoutError): + await asyncio.wait_for( + self._stop_requested.wait(), + timeout=self._config.member_heartbeat_seconds, + ) + if self._stop_requested.is_set(): + return + async with self._database.transaction() as connection: + if self._single_node_lease is not None: + renewed = await self._repository.acquire_lease( + connection, + lease_name=_SINGLE_NODE_LEASE, + owner_id=self._member_id, + lease_seconds=self._config.member_ttl_seconds, + ) + self._single_node_lease = _validated_renewal(self._single_node_lease, renewed) + await self._heartbeat(connection) + except asyncio.CancelledError: + raise + except Exception: + self._failed = True + raise + + async def stop(self) -> None: + """Stop heartbeats and conditionally release single-node ownership.""" + + self._stop_requested.set() + lease = self._single_node_lease + self._single_node_lease = None + if lease is not None: + async with self._database.transaction() as connection: + await self._repository.release_lease(connection, lease) + + async def readiness(self) -> str: + """Verify this member is still live and compatible with its peers.""" + + if self._failed: + return ReadinessCheckStatus.UNAVAILABLE + async with self._database.transaction() as connection: + members = await self._repository.live_members(connection) + current = next((member for member in members if member.member_id == self._member_id), None) + if current is None: + return ReadinessCheckStatus.UNAVAILABLE + if any( + member.behavior_revision != self._deployment.behavior_revision + or member.schema_min > _SCHEMA_VERSION + or member.schema_max < _SCHEMA_VERSION + or member.payload_min > _PAYLOAD_VERSION + or member.payload_max < _PAYLOAD_VERSION + for member in members + ): + return ReadinessCheckStatus.MISCONFIGURED + return ReadinessCheckStatus.READY + + async def role_readiness(self, role: str) -> str: + """Report whether at least one compatible live member serves ``role``.""" + + async with self._database.transaction() as connection: + members = await self._repository.live_members(connection) + return ( + ReadinessCheckStatus.READY + if any( + member.role in {role, "all"} and member.behavior_revision == self._deployment.behavior_revision + for member in members + ) + else ReadinessCheckStatus.UNAVAILABLE + ) + + async def _heartbeat(self, connection: AsyncConnection) -> None: + await self._repository.heartbeat_member( + connection, + RuntimeMemberSpec( + member_id=self._member_id, + role=self._deployment.role, + build_version=self._build_version, + schema_min=_SCHEMA_VERSION, + schema_max=_SCHEMA_VERSION, + payload_min=_PAYLOAD_VERSION, + payload_max=_PAYLOAD_VERSION, + behavior_revision=self._deployment.behavior_revision, + ), + ttl_seconds=self._config.member_ttl_seconds, + ) + if self._observer is not None: + members = await self._repository.live_members(connection) + counts = dict.fromkeys(("all", "api", "scheduler", "worker"), 0) + for member in members: + counts[member.role] += 1 + with suppress(Exception): + self._observer.set_runtime_members(counts) + + +def _validated_renewal(current: CoordinatorLease, renewed: CoordinatorLease | None) -> CoordinatorLease: + if renewed is None or renewed.fence != current.fence: + raise DuplicateSingleNodeError + return renewed + + +__all__ = ["DuplicateSingleNodeError", "RuntimeMembership"] diff --git a/src/powercontext/builtin/runtime/operations.py b/src/powercontext/builtin/runtime/operations.py new file mode 100644 index 000000000..dfd3d02a9 --- /dev/null +++ b/src/powercontext/builtin/runtime/operations.py @@ -0,0 +1,277 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Application service for durable operation submission and control.""" + +from __future__ import annotations + +import asyncio +import base64 +import binascii +import json +from contextlib import AbstractContextManager, nullcontext +from dataclasses import dataclass +from datetime import datetime +from time import monotonic + +from pydantic import ValidationError + +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError, RepositoryNotFoundError +from powercontext.builtin.persistence.work import EnqueueResult, StoredWork, WorkRepository, WorkStatus +from powercontext.builtin.runtime.config import OperationsConfig, WorkerConfig +from powercontext.builtin.runtime.models import MemoryFlushResult +from powercontext.builtin.runtime.protocols import RuntimeSpan, RuntimeTracing +from powercontext.builtin.runtime.relational import RelationalContexts +from powercontext.builtin.runtime.work_handlers import ( + EXPERIENCE_WORK_KIND, + MEMORY_WORK_KIND, + enqueue_memory_work, +) +from powercontext.builtin.runtime.work_observability import WorkObserver, refresh_work_queue +from powercontext.builtin.runtime.worker import DurableWorker + +_TERMINAL = frozenset({WorkStatus.SUCCEEDED, WorkStatus.FAILED, WorkStatus.CANCELLED}) +_PUBLIC_KINDS = frozenset({MEMORY_WORK_KIND, EXPERIENCE_WORK_KIND}) + + +@dataclass(frozen=True) +class OperationPage: + items: tuple[StoredWork, ...] + next_cursor: str | None + + +class RuntimeOperationError(RuntimeError): + """Base class for durable operation facade errors.""" + + +class RuntimeOperationPendingError(RuntimeOperationError): + def __init__(self, operation_id: str) -> None: + self.operation_id = operation_id + super().__init__(f"operation {operation_id} is still pending") + + +class RuntimeOperationFailedError(RuntimeOperationError): + def __init__(self, operation: StoredWork) -> None: + self.operation = operation + super().__init__(f"operation {operation.work_id} failed with {operation.error_code or 'unknown'}") + + +class RuntimeOperationCancelledError(RuntimeOperationError): + def __init__(self, operation_id: str) -> None: + self.operation_id = operation_id + super().__init__(f"operation {operation_id} was cancelled") + + +class OperationManager: + """Submit, wait for, query, and mutate durable work records.""" + + def __init__( + self, + *, + contexts: RelationalContexts, + operations: OperationsConfig, + worker: WorkerConfig, + local_worker: DurableWorker | None, + payload_version: int, + memory_window_limit: int, + repository: WorkRepository | None = None, + observer: WorkObserver | None = None, + tracing: RuntimeTracing | None = None, + ) -> None: + self._contexts = contexts + self._database: AsyncDatabase = contexts.database + self._operations = operations + self._worker_config = worker + self._local_worker = local_worker + self._payload_version = payload_version + self._memory_window_limit = memory_window_limit + self._repository = WorkRepository(observer=observer) if repository is None else repository + self._observer = observer + self._tracing = tracing + + @property + def default_wait_seconds(self) -> float: + return self._operations.default_wait_seconds + + @property + def maximum_wait_seconds(self) -> float: + return self._operations.maximum_wait_seconds + + @property + def memory_window_limit(self) -> int: + return self._memory_window_limit + + @property + def database(self) -> AsyncDatabase: + """Expose the shared database to process-level coordination adapters.""" + + return self._database + + async def submit_memory(self, scope_id: str, /, *, limit: int) -> MemoryFlushResult | EnqueueResult: + with self._stage( + "work.enqueue", + attributes={ + "powercontext.work.kind": "powercontext.memory.source-window", + "powercontext.work.payload_version": self._payload_version, + }, + ) as span: + submission = await enqueue_memory_work( + self._contexts, + scope_id, + limit=limit, + max_attempts=self._worker_config.max_attempts, + payload_version=self._payload_version, + repository=self._repository, + ) + if span is not None: + span.set_outcome( + "idle" + if isinstance(submission, MemoryFlushResult) + else "created" + if submission.created + else "joined" + ) + if isinstance(submission, EnqueueResult) and self._local_worker is not None: + self._local_worker.notify() + await refresh_work_queue(self._database, self._repository, self._observer) + return submission + + async def flush_memory(self, scope_id: str, /, *, limit: int) -> MemoryFlushResult: + submission = await self.submit_memory(scope_id, limit=limit) + if isinstance(submission, MemoryFlushResult): + return submission + operation = await self.wait( + submission.work.work_id, + timeout_seconds=self._operations.maximum_wait_seconds, + ) + if operation is None: + raise RuntimeOperationPendingError(submission.work.work_id) + return self.memory_result(operation) + + def memory_result(self, operation: StoredWork, /) -> MemoryFlushResult: + if operation.status is WorkStatus.FAILED: + raise RuntimeOperationFailedError(operation) + if operation.status is WorkStatus.CANCELLED: + raise RuntimeOperationCancelledError(operation.work_id) + if operation.status is not WorkStatus.SUCCEEDED or operation.result_payload is None: + raise RuntimeOperationPendingError(operation.work_id) + try: + return MemoryFlushResult.model_validate(operation.result_payload) + except ValidationError as error: + raise RuntimeOperationFailedError(operation) from error + + async def wait(self, operation_id: str, /, *, timeout_seconds: float) -> StoredWork | None: + if timeout_seconds < 0 or timeout_seconds > self._operations.maximum_wait_seconds: + raise InvalidRepositoryArgumentError( + "timeout_seconds", + f"must be between 0 and {self._operations.maximum_wait_seconds}", + ) + deadline = monotonic() + timeout_seconds + while True: + operation = await self.get(operation_id) + if operation.status in _TERMINAL: + return operation + remaining = deadline - monotonic() + if remaining <= 0: + return None + await asyncio.sleep(min(self._operations.poll_seconds, remaining)) + + async def get(self, operation_id: str, /) -> StoredWork: + async with self._database.transaction() as connection: + operation = await self._repository.get(connection, operation_id) + return _require_public_operation(operation) + + async def list( + self, + /, + *, + scope_id: str | None, + kind: str | None, + status: WorkStatus | None, + cursor: str | None, + limit: int, + ) -> OperationPage: + before = None if cursor is None else _decode_cursor(cursor) + async with self._database.transaction() as connection: + items = await self._repository.list( + connection, + scope_id=scope_id, + kind=kind, + allowed_kinds=_PUBLIC_KINDS, + status=status, + limit=limit, + before=before, + ) + next_cursor = None + if len(items) == limit: + last = items[-1] + next_cursor = _encode_cursor(last.created_at, last.work_id) + return OperationPage(items=items, next_cursor=next_cursor) + + async def cancel(self, operation_id: str, /, *, expected_version: int) -> StoredWork: + async with self._database.transaction() as connection: + _require_public_operation(await self._repository.get(connection, operation_id)) + operation = await self._repository.cancel(connection, operation_id, expected_version=expected_version) + await refresh_work_queue(self._database, self._repository, self._observer) + return operation + + async def retry(self, operation_id: str, /, *, expected_version: int) -> StoredWork: + async with self._database.transaction() as connection: + _require_public_operation(await self._repository.get(connection, operation_id)) + operation = await self._repository.retry(connection, operation_id, expected_version=expected_version) + if self._local_worker is not None: + self._local_worker.notify() + await refresh_work_queue(self._database, self._repository, self._observer) + return operation + + def _stage( + self, + name: str, + *, + attributes: dict[str, str | bool | int | float], + ) -> AbstractContextManager[RuntimeSpan | None]: + if self._tracing is None: + return nullcontext(None) + return self._tracing.stage(name, attributes=attributes) + + +def _require_public_operation(operation: StoredWork) -> StoredWork: + if operation.kind not in _PUBLIC_KINDS: + raise RepositoryNotFoundError("operation", operation.work_id) + return operation + + +def _encode_cursor(created_at: datetime, work_id: str) -> str: + payload = json.dumps([created_at.isoformat(), work_id], separators=(",", ":")).encode() + return base64.urlsafe_b64encode(payload).decode().rstrip("=") + + +def _decode_cursor(value: str) -> tuple[datetime, str]: + try: + padding = "=" * (-len(value) % 4) + created_at, work_id = json.loads(base64.urlsafe_b64decode(f"{value}{padding}")) + return datetime.fromisoformat(created_at), str(work_id) + except (ValueError, TypeError, json.JSONDecodeError, binascii.Error): + raise InvalidRepositoryArgumentError("cursor", "must be a valid operation cursor") from None + + +__all__ = [ + "OperationManager", + "OperationPage", + "RuntimeOperationCancelledError", + "RuntimeOperationError", + "RuntimeOperationFailedError", + "RuntimeOperationPendingError", +] diff --git a/src/powercontext/builtin/runtime/protocols.py b/src/powercontext/builtin/runtime/protocols.py index a2b71be08..2bc26df3f 100644 --- a/src/powercontext/builtin/runtime/protocols.py +++ b/src/powercontext/builtin/runtime/protocols.py @@ -16,9 +16,10 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from contextlib import AbstractContextManager -from typing import Protocol, TypeVar +from dataclasses import dataclass +from typing import Any, Protocol, TypeVar from powercontext.builtin.artifacts.handoff import ActivateHandoff, HandoffActivation from powercontext.builtin.runtime.models import ( @@ -38,6 +39,14 @@ TraceAttribute = str | bool | int | float +@dataclass(frozen=True, slots=True) +class RuntimeTraceContext: + """Portable non-sensitive trace identity used to link background attempts.""" + + trace_id: str + span_id: str + + class RuntimeSpan(Protocol): """Record bounded attributes and a deferred outcome for one internal Runtime stage.""" @@ -62,9 +71,17 @@ def background( *, operation: str, attributes: Mapping[str, TraceAttribute], + links: Sequence[RuntimeTraceContext] = (), ) -> AbstractContextManager[RuntimeSpan]: ... +def runtime_trace_context(span: RuntimeSpan | None) -> RuntimeTraceContext | None: + """Read an optional concrete tracing context without expanding the port.""" + + value: Any = getattr(span, "trace_context", None) + return value if isinstance(value, RuntimeTraceContext) else None + + class PowerContextProvider(Protocol[SourcesT, ArtifactsT, TriggersT]): """Resolve an already composed context without transferring lifecycle ownership.""" diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index dc997c73c..5c65d9930 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -876,6 +876,20 @@ async def scope_ids(self) -> tuple[str, ...]: ).scalars() return tuple(str(value) for value in values) + async def scope_ids_page(self, after: str | None, limit: int, /) -> tuple[str, ...]: + """Return one bounded keyset page of scopes that own a Source journal.""" + + if limit < 1 or limit > 100: + raise ValueError("scope page limit must be between 1 and 100") # noqa: TRY003 + async with self.database.transaction() as connection: + statement = select(SOURCE_JOURNAL_HEADS_TABLE.c.scope_id) + if after is not None: + statement = statement.where(SOURCE_JOURNAL_HEADS_TABLE.c.scope_id > validate_scope_id(after)) + values = ( + await connection.execute(statement.order_by(SOURCE_JOURNAL_HEADS_TABLE.c.scope_id).limit(limit)) + ).scalars() + return tuple(str(value) for value in values) + async def handoff_scope_ids(self) -> tuple[str, ...]: """Return scopes with a committed Handoff head, in deterministic order.""" diff --git a/src/powercontext/builtin/runtime/scheduler.py b/src/powercontext/builtin/runtime/scheduler.py deleted file mode 100644 index f12bdfe73..000000000 --- a/src/powercontext/builtin/runtime/scheduler.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Persistent APScheduler adapter for built-in Source-window activations.""" - -from __future__ import annotations - -from collections.abc import Awaitable, Callable -from pathlib import Path - -from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore -from apscheduler.schedulers.asyncio import AsyncIOScheduler -from apscheduler.triggers.interval import IntervalTrigger -from sqlalchemy import URL - -SOURCE_WINDOW_JOB_ID = "powercontext.memory.source-window.v1" -EXPERIENCE_INCUBATION_JOB_ID = "powercontext.experience.incubation.v1" -SCHEDULER_TABLE = "powercontext_scheduler_jobs" - -Processor = Callable[[], Awaitable[None]] -_SOURCE_WINDOW_PROCESSOR = "source-window" -_EXPERIENCE_INCUBATION_PROCESSOR = "experience-incubation" -_processors: dict[str, dict[str, Processor]] = {} - - -class SchedulerConfigurationError(ValueError): - """Report an unsupported scheduler sidecar configuration.""" - - def __init__(self) -> None: - super().__init__("scheduler_path must reference a file") - - -class SchedulerStateError(RuntimeError): - """Report conflicting runtime ownership for a persisted scheduler.""" - - def __init__(self, code: str, runtime_key: str) -> None: - messages = { - "duplicate": f"a scheduled runtime is already open for {runtime_key}", - "missing": f"no live scheduled runtime is registered for {runtime_key}", - } - super().__init__(messages[code]) - - -def scheduler_runtime_key(scheduler_path: str | Path) -> str: - """Return the stable in-process lookup key for one scheduler sidecar.""" - - if str(scheduler_path) == ":memory:": - raise SchedulerConfigurationError - return str(Path(scheduler_path).expanduser().resolve()) - - -def scheduler_database_path(scheduler_path: str | Path) -> str: - """Return the sidecar database used only by the persisted job store.""" - - return str(Path(scheduler_runtime_key(scheduler_path))) - - -def register_processors( - runtime_key: str, - *, - source_window: Processor | None, - experience_incubation: Processor | None, -) -> None: - """Register live processors referenced by persisted jobs.""" - - if runtime_key in _processors: - raise SchedulerStateError("duplicate", runtime_key) - registered = { - name: processor - for name, processor in ( - (_SOURCE_WINDOW_PROCESSOR, source_window), - (_EXPERIENCE_INCUBATION_PROCESSOR, experience_incubation), - ) - if processor is not None - } - if not registered: - raise SchedulerStateError("missing", runtime_key) - _processors[runtime_key] = registered - - -def unregister_processor(runtime_key: str) -> None: - """Remove a live processor after its scheduler has stopped.""" - - _processors.pop(runtime_key, None) - - -async def dispatch_source_windows(runtime_key: str) -> None: - """Dispatch a persisted job to the live runtime owning its database.""" - - await _processor(runtime_key, _SOURCE_WINDOW_PROCESSOR)() - - -async def dispatch_experience_incubation(runtime_key: str) -> None: - """Dispatch persisted Experience incubation to its live Runtime.""" - - await _processor(runtime_key, _EXPERIENCE_INCUBATION_PROCESSOR)() - - -def _processor(runtime_key: str, name: str) -> Processor: - try: - return _processors[runtime_key][name] - except KeyError: - raise SchedulerStateError("missing", runtime_key) from None - - -def create_scheduler(scheduler_path: str | Path) -> AsyncIOScheduler: - """Create an APScheduler instance with an isolated SQLite job-store.""" - - database = Path(scheduler_database_path(scheduler_path)) - database.parent.mkdir(parents=True, exist_ok=True) - url = URL.create("sqlite+pysqlite", database=str(database)) - return AsyncIOScheduler( - jobstores={ - "default": SQLAlchemyJobStore( - url=url, - tablename=SCHEDULER_TABLE, - engine_options={"connect_args": {"timeout": 30}}, - ) - }, - timezone="UTC", - ) - - -def configure_source_window_job( - scheduler: AsyncIOScheduler, - *, - runtime_key: str, - schedule_seconds: float | None, -) -> None: - """Create or reconcile the persisted Source-window interval job.""" - - _configure_interval_job( - scheduler, - job_id=SOURCE_WINDOW_JOB_ID, - processor=dispatch_source_windows, - runtime_key=runtime_key, - schedule_seconds=schedule_seconds, - ) - - -def configure_experience_incubation_job( - scheduler: AsyncIOScheduler, - *, - runtime_key: str, - schedule_seconds: float | None, -) -> None: - """Create or reconcile the persisted Experience incubation job.""" - - _configure_interval_job( - scheduler, - job_id=EXPERIENCE_INCUBATION_JOB_ID, - processor=dispatch_experience_incubation, - runtime_key=runtime_key, - schedule_seconds=schedule_seconds, - ) - - -def _configure_interval_job( - scheduler: AsyncIOScheduler, - *, - job_id: str, - processor: Callable[[str], Awaitable[None]], - runtime_key: str, - schedule_seconds: float | None, -) -> None: - job = scheduler.get_job(job_id) - if schedule_seconds is None: - if job is not None: - scheduler.remove_job(job_id) - return - if job is None: - scheduler.add_job( - processor, - "interval", - args=(runtime_key,), - seconds=schedule_seconds, - coalesce=True, - max_instances=1, - misfire_grace_time=None, - id=job_id, - ) - return - scheduler.modify_job( - job_id, - func=processor, - args=(runtime_key,), - coalesce=True, - max_instances=1, - misfire_grace_time=None, - ) - if not isinstance(job.trigger, IntervalTrigger) or job.trigger.interval.total_seconds() != schedule_seconds: - scheduler.reschedule_job( - job_id, - trigger="interval", - seconds=schedule_seconds, - ) diff --git a/src/powercontext/builtin/runtime/work_handlers.py b/src/powercontext/builtin/runtime/work_handlers.py new file mode 100644 index 000000000..72d8c5812 --- /dev/null +++ b/src/powercontext/builtin/runtime/work_handlers.py @@ -0,0 +1,636 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Built-in Memory and Experience work discovery and exact-window handlers.""" + +from __future__ import annotations + +import hashlib +import json +import logging +from collections.abc import Awaitable, Callable +from datetime import UTC, datetime, timedelta + +from pydantic import BaseModel, ConfigDict, Field, ValidationError +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.artifacts.experience import EXPERIENCE_INCUBATION_CURSOR_NAME +from powercontext.builtin.inference.models import InferenceUsage +from powercontext.builtin.inference.usage import bind_usage_reporter +from powercontext.builtin.persistence.cursors import StoredSourceCursor +from powercontext.builtin.persistence.database import database_now +from powercontext.builtin.persistence.rate_limit import RateLimitRepository +from powercontext.builtin.persistence.work import ( + EnqueueResult, + WorkClaim, + WorkRepository, + WorkResult, + WorkSpec, +) +from powercontext.builtin.runtime.durable_scheduler import DiscoveryPage +from powercontext.builtin.runtime.models import ExperienceIncubationResult, MemoryFlushResult +from powercontext.builtin.runtime.relational import RelationalContexts, _validate_experience_plans +from powercontext.builtin.runtime.worker import PreparedWork, WorkExecutionError +from powercontext.builtin.sources import SourceCursor, validate_scope_id +from powercontext.builtin.statistics import ModelUsageOperation, ModelUsagePurpose +from powercontext.builtin.triggers import SOURCE_WINDOW_TRIGGER_NAME, SourceHighWatermark, SourceWindowTrigger +from powercontext.errors import ArtifactNotFoundError + +MEMORY_WORK_KIND = "powercontext.memory.source-window" +EXPERIENCE_WORK_KIND = "powercontext.experience.incubation" +MAINTENANCE_WORK_KIND = "powercontext.maintenance.operations" +CURRENT_WORK_PAYLOAD_VERSION = 1 + +logger = logging.getLogger(__name__) + + +class SourceWindowPayload(BaseModel): + """Reference-only snapshot of one bounded Source cursor transition.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + cursor_name: str + cursor_generation: int = Field(ge=0) + after: int = Field(ge=0) + through: int = Field(ge=1) + high_watermark: int = Field(ge=1) + + +class MemoryWorkDiscoverer: + """Scan Source scopes and describe pending Memory windows.""" + + name = MEMORY_WORK_KIND + + def __init__( + self, + contexts: RelationalContexts, + *, + interval_seconds: float, + window_limit: int, + max_attempts: int, + payload_version: int = CURRENT_WORK_PAYLOAD_VERSION, + ) -> None: + self._contexts = contexts + self.interval_seconds = interval_seconds + self._window_limit = window_limit + self._max_attempts = max_attempts + self._payload_version = payload_version + + async def page(self, continuation: str | None, limit: int, /) -> DiscoveryPage: + scopes = await self._contexts.scope_ids_page(continuation, limit) + specs: list[WorkSpec] = [] + for scope_id in scopes: + spec = await memory_work_spec( + self._contexts, + scope_id, + limit=self._window_limit, + max_attempts=self._max_attempts, + payload_version=self._payload_version, + ) + if spec is not None: + specs.append(spec) + return DiscoveryPage( + specs=tuple(specs), + continuation=scopes[-1] if len(scopes) == limit else None, + ) + + +class ExperienceWorkDiscoverer: + """Scan Source scopes and describe pending Experience windows.""" + + name = EXPERIENCE_WORK_KIND + + def __init__( + self, + contexts: RelationalContexts, + *, + interval_seconds: float, + window_limit: int, + max_attempts: int, + payload_version: int = CURRENT_WORK_PAYLOAD_VERSION, + ) -> None: + self._contexts = contexts + self.interval_seconds = interval_seconds + self._window_limit = window_limit + self._max_attempts = max_attempts + self._payload_version = payload_version + + async def page(self, continuation: str | None, limit: int, /) -> DiscoveryPage: + scopes = await self._contexts.scope_ids_page(continuation, limit) + specs: list[WorkSpec] = [] + for scope_id in scopes: + spec = await experience_work_spec( + self._contexts, + scope_id, + limit=self._window_limit, + max_attempts=self._max_attempts, + payload_version=self._payload_version, + ) + if spec is not None: + specs.append(spec) + return DiscoveryPage( + specs=tuple(specs), + continuation=scopes[-1] if len(scopes) == limit else None, + ) + + +class OperationMaintenanceDiscoverer: + """Schedule one globally serialized, bounded history-retention batch.""" + + name = MAINTENANCE_WORK_KIND + + def __init__(self, *, interval_seconds: float, max_attempts: int) -> None: + self.interval_seconds = interval_seconds + self._max_attempts = max_attempts + + async def page(self, continuation: str | None, _limit: int, /) -> DiscoveryPage: + if continuation is not None: + return DiscoveryPage(specs=(), continuation=None) + return DiscoveryPage( + specs=( + WorkSpec( + kind=MAINTENANCE_WORK_KIND, + payload_version=CURRENT_WORK_PAYLOAD_VERSION, + scope_id="system:operations", + lane_key=_digest("maintenance:operations"), + logical_key=_digest(MAINTENANCE_WORK_KIND), + payload={}, + max_attempts=self._max_attempts, + ), + ), + continuation=None, + ) + + +class MemoryWorkHandler: + """Prepare Memory outside locks and atomically commit it with Work success.""" + + kind = MEMORY_WORK_KIND + supported_versions = frozenset({CURRENT_WORK_PAYLOAD_VERSION}) + + def __init__(self, contexts: RelationalContexts) -> None: + self._contexts = contexts + + async def prepare(self, claim: WorkClaim, /) -> PreparedWork: + payload = _payload(claim) + services = self._contexts._services_for(claim.scope_id) + async with self._contexts.database.transaction() as connection: + state_row = await services.repositories.cursors.load( + connection, + claim.scope_id, + SOURCE_WINDOW_TRIGGER_NAME, + ) + current_sequence, current_generation = _cursor_position(state_row) + if current_sequence >= payload.through: + return PreparedWork( + result=_memory_result( + previous=payload.after, + current=payload.through, + high_watermark=payload.high_watermark, + source_count=payload.through - payload.after, + memory_ref=None, + code="already_committed", + ) + ) + _require_exact_cursor(payload, current_sequence, current_generation) + rows = await services.repositories.sources.list( + connection, + claim.scope_id, + after=payload.after, + limit=payload.through - payload.after, + ) + _require_complete_window(rows, payload) + _, source_catalog = services.sources() + memory = services.memory(source_catalog) + try: + current = await memory.head(services.memory_artifact_id) + except ArtifactNotFoundError: + current = None + with bind_usage_reporter( + _usage_reporter(self._contexts, claim.scope_id), + generation_purpose=ModelUsagePurpose.MEMORY_EXTRACTION, + embedding_purpose=ModelUsagePurpose.MEMORY_INDEXING, + ): + plan = await memory.plan_remember( + memory=current, + sources=tuple(row.value for row in rows), + mode="extract", + ) + + async def commit(connection: AsyncConnection) -> WorkResult: + locked = await services.repositories.cursors.load( + connection, + claim.scope_id, + SOURCE_WINDOW_TRIGGER_NAME, + for_update=True, + ) + sequence, generation = _cursor_position(locked) + _require_exact_cursor(payload, sequence, generation) + _, bound_catalog = services.sources(connection) + updated = await services.memory(bound_catalog, connection).apply(plan) + await services.repositories.cursors.save( + connection, + claim.scope_id, + SOURCE_WINDOW_TRIGGER_NAME, + SourceCursor(sequence=payload.through), + expected_generation=None if payload.cursor_generation == 0 else payload.cursor_generation, + ) + return _memory_result( + previous=payload.after, + current=payload.through, + high_watermark=payload.high_watermark, + source_count=len(rows), + memory_ref=None if updated is None else updated.as_ref().model_dump(mode="json"), + code="processed", + ) + + return PreparedWork(result=None, commit=commit) + + +class ExperienceWorkHandler: + """Prepare Experience proposals and commit Candidates with cursor and Work.""" + + kind = EXPERIENCE_WORK_KIND + supported_versions = frozenset({CURRENT_WORK_PAYLOAD_VERSION}) + + def __init__(self, contexts: RelationalContexts) -> None: + self._contexts = contexts + + async def prepare(self, claim: WorkClaim, /) -> PreparedWork: + payload = _payload(claim) + services = self._contexts._services_for(claim.scope_id) + pipeline = services.experience_pipeline + if pipeline is None: + raise WorkExecutionError(category="configuration", code="experience_pipeline_unavailable", retryable=False) + async with self._contexts.database.transaction() as connection: + state_row = await services.repositories.cursors.load( + connection, + claim.scope_id, + EXPERIENCE_INCUBATION_CURSOR_NAME, + ) + current_sequence, current_generation = _cursor_position(state_row) + if current_sequence >= payload.through: + return PreparedWork(result=_experience_result(payload, candidate_count=0, code="already_committed")) + _require_exact_cursor(payload, current_sequence, current_generation) + rows = await services.repositories.sources.list( + connection, + claim.scope_id, + after=payload.after, + limit=payload.through - payload.after, + ) + _require_complete_window(rows, payload) + with bind_usage_reporter( + _usage_reporter(self._contexts, claim.scope_id), + generation_purpose=ModelUsagePurpose.EXPERIENCE_GENERATION, + ): + plans = await pipeline.incubate(tuple(row.value for row in rows)) + _validate_experience_plans(plans, rows) + + async def commit(connection: AsyncConnection) -> WorkResult: + locked = await services.repositories.cursors.load( + connection, + claim.scope_id, + EXPERIENCE_INCUBATION_CURSOR_NAME, + for_update=True, + ) + sequence, generation = _cursor_position(locked) + _require_exact_cursor(payload, sequence, generation) + review = services.review(connection) + for plan in plans: + await review.propose_experience( + plan.proposal, + sources=plan.sources, + artifacts=(), + target=None, + reason=plan.reason, + ) + await services.repositories.cursors.save( + connection, + claim.scope_id, + EXPERIENCE_INCUBATION_CURSOR_NAME, + SourceCursor(sequence=payload.through), + expected_generation=None if payload.cursor_generation == 0 else payload.cursor_generation, + ) + return _experience_result(payload, candidate_count=len(plans), code="processed") + + return PreparedWork(result=None, commit=commit) + + +def _usage_reporter( + contexts: RelationalContexts, + scope_id: str, +) -> Callable[[ModelUsagePurpose, ModelUsageOperation, InferenceUsage], Awaitable[None]]: + async def report( + purpose: ModelUsagePurpose, + operation: ModelUsageOperation, + usage: InferenceUsage, + ) -> None: + try: + await contexts.statistics(scope_id).record( + purpose, + operation, + usage, + datetime.now(UTC).date(), + ) + except Exception as error: + # Statistics are best effort and raw exception text may contain + # provider payloads. Record only a bounded event and type name. + logger.warning( + "Work model usage recording failed", + extra={ + "event": "statistics.model_usage.failed", + "operation": operation.value, + "outcome": "failure", + "error_type": type(error).__name__, + }, + ) + + return report + + +class OperationMaintenanceHandler: + """Delete one bounded batch of expired successful/cancelled operations.""" + + kind = MAINTENANCE_WORK_KIND + supported_versions = frozenset({CURRENT_WORK_PAYLOAD_VERSION}) + + def __init__(self, *, retention_days: int, batch_size: int) -> None: + self._retention_days = retention_days + self._batch_size = batch_size + self._repository = WorkRepository() + self._rate_limits = RateLimitRepository() + + async def prepare(self, claim: WorkClaim, /) -> PreparedWork: + if claim.payload: + raise WorkExecutionError(category="payload", code="invalid_payload", retryable=False) + + async def commit(connection: AsyncConnection) -> WorkResult: + now = await database_now(connection) + deleted = await self._repository.purge_terminal( + connection, + completed_before=now - timedelta(days=self._retention_days), + limit=self._batch_size, + ) + counters_deleted = await self._rate_limits.purge_expired(connection, limit=self._batch_size) + return WorkResult( + code="cleaned", + payload={ + "operations_deleted": deleted, + "rate_limit_windows_deleted": counters_deleted, + }, + ) + + return PreparedWork(result=None, commit=commit) + + +async def enqueue_memory_work( + contexts: RelationalContexts, + scope_id: str, + /, + *, + limit: int, + max_attempts: int, + payload_version: int = CURRENT_WORK_PAYLOAD_VERSION, + repository: WorkRepository | None = None, +) -> MemoryFlushResult | EnqueueResult: + """Determine and enqueue one manual Memory window in a short transaction.""" + + scope = validate_scope_id(scope_id) + async with contexts.database.transaction() as connection: + state_row, high_watermark = await _window_state( + contexts, + connection, + scope, + SOURCE_WINDOW_TRIGGER_NAME, + ) + payload = _window_payload(state_row, high_watermark, SOURCE_WINDOW_TRIGGER_NAME, limit) + if payload is None: + position, _ = _cursor_position(state_row) + return MemoryFlushResult( + previous_cursor=position, + high_watermark=high_watermark, + current_cursor=position, + source_count=0, + memory_ref=None, + ) + spec = _work_spec( + kind=MEMORY_WORK_KIND, + scope_id=scope, + payload=payload, + max_attempts=max_attempts, + payload_version=payload_version, + ) + return await (WorkRepository() if repository is None else repository).enqueue(connection, spec) + + +async def memory_work_spec( + contexts: RelationalContexts, + scope_id: str, + /, + *, + limit: int, + max_attempts: int, + payload_version: int, +) -> WorkSpec | None: + return await _discover_spec( + contexts, + scope_id, + cursor_name=SOURCE_WINDOW_TRIGGER_NAME, + kind=MEMORY_WORK_KIND, + limit=limit, + max_attempts=max_attempts, + payload_version=payload_version, + ) + + +async def experience_work_spec( + contexts: RelationalContexts, + scope_id: str, + /, + *, + limit: int, + max_attempts: int, + payload_version: int, +) -> WorkSpec | None: + return await _discover_spec( + contexts, + scope_id, + cursor_name=EXPERIENCE_INCUBATION_CURSOR_NAME, + kind=EXPERIENCE_WORK_KIND, + limit=limit, + max_attempts=max_attempts, + payload_version=payload_version, + ) + + +async def _discover_spec( + contexts: RelationalContexts, + scope_id: str, + *, + cursor_name: str, + kind: str, + limit: int, + max_attempts: int, + payload_version: int, +) -> WorkSpec | None: + scope = validate_scope_id(scope_id) + async with contexts.database.transaction() as connection: + state_row, high_watermark = await _window_state(contexts, connection, scope, cursor_name) + payload = _window_payload(state_row, high_watermark, cursor_name, limit) + return ( + None + if payload is None + else _work_spec( + kind=kind, + scope_id=scope, + payload=payload, + max_attempts=max_attempts, + payload_version=payload_version, + ) + ) + + +async def _window_state( + contexts: RelationalContexts, + connection: AsyncConnection, + scope_id: str, + cursor_name: str, +) -> tuple[StoredSourceCursor | None, int]: + services = contexts._services_for(scope_id) + state_row = await services.repositories.cursors.load(connection, scope_id, cursor_name) + high_watermark = await services.repositories.sources.journal_position(connection, scope_id) + return state_row, high_watermark + + +def _window_payload( + state_row: StoredSourceCursor | None, + high_watermark: int, + cursor_name: str, + limit: int, +) -> SourceWindowPayload | None: + state = SourceCursor() if state_row is None else state_row.cursor + transition = SourceWindowTrigger().activate( + SourceHighWatermark(sequence=high_watermark, limit=limit), + state, + ) + if not transition.actions: + return None + action = transition.actions[0] + return SourceWindowPayload( + cursor_name=cursor_name, + cursor_generation=0 if state_row is None else state_row.generation, + after=action.after, + through=action.through, + high_watermark=high_watermark, + ) + + +def _work_spec( + *, + kind: str, + scope_id: str, + payload: SourceWindowPayload, + max_attempts: int, + payload_version: int, +) -> WorkSpec: + lane_key = _digest(f"{kind.split('.')[1]}:{scope_id}") + logical_key = _digest( + json.dumps( + [kind, scope_id, payload.cursor_name, payload.cursor_generation, payload.after], + ensure_ascii=False, + separators=(",", ":"), + ) + ) + return WorkSpec( + kind=kind, + payload_version=payload_version, + scope_id=scope_id, + lane_key=lane_key, + logical_key=logical_key, + payload=payload.model_dump(mode="json"), + max_attempts=max_attempts, + ) + + +def _payload(claim: WorkClaim) -> SourceWindowPayload: + try: + return SourceWindowPayload.model_validate(claim.payload) + except ValidationError: + raise WorkExecutionError(category="payload", code="invalid_payload", retryable=False) from None + + +def _cursor_position(state_row: StoredSourceCursor | None) -> tuple[int, int]: + return (0, 0) if state_row is None else (state_row.cursor.sequence, state_row.generation) + + +def _require_exact_cursor(payload: SourceWindowPayload, sequence: int, generation: int) -> None: + if sequence != payload.after or generation != payload.cursor_generation: + raise WorkExecutionError(category="conflict", code="cursor_changed", retryable=False) + + +def _require_complete_window(rows, payload: SourceWindowPayload) -> None: + expected = payload.through - payload.after + if len(rows) != expected or any( + row.journal_position != payload.after + offset for offset, row in enumerate(rows, start=1) + ): + raise WorkExecutionError(category="retention", code="source_window_unavailable", retryable=False) + + +def _memory_result( + *, + previous: int, + current: int, + high_watermark: int, + source_count: int, + memory_ref, + code: str, +) -> WorkResult: + result = MemoryFlushResult( + previous_cursor=previous, + current_cursor=current, + high_watermark=high_watermark, + source_count=source_count, + memory_ref=memory_ref, + ) + return WorkResult(code=code, payload=result.model_dump(mode="json")) + + +def _experience_result(payload: SourceWindowPayload, *, candidate_count: int, code: str) -> WorkResult: + result = ExperienceIncubationResult( + previous_cursor=payload.after, + current_cursor=payload.through, + high_watermark=payload.high_watermark, + source_count=payload.through - payload.after, + candidate_count=candidate_count, + ) + return WorkResult(code=code, payload=result.model_dump(mode="json")) + + +def _digest(value: str) -> str: + return hashlib.sha256(value.encode()).hexdigest() + + +__all__ = [ + "CURRENT_WORK_PAYLOAD_VERSION", + "EXPERIENCE_WORK_KIND", + "MEMORY_WORK_KIND", + "ExperienceWorkDiscoverer", + "ExperienceWorkHandler", + "MemoryWorkDiscoverer", + "MemoryWorkHandler", + "SourceWindowPayload", + "enqueue_memory_work", + "experience_work_spec", + "memory_work_spec", +] diff --git a/src/powercontext/builtin/runtime/work_observability.py b/src/powercontext/builtin/runtime/work_observability.py new file mode 100644 index 000000000..143f835c7 --- /dev/null +++ b/src/powercontext/builtin/runtime/work_observability.py @@ -0,0 +1,69 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bounded-label observability port for durable background work.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from powercontext.builtin.persistence.database import AsyncDatabase + from powercontext.builtin.persistence.work import WorkQueueStatistic, WorkRepository + + +class WorkObserver(Protocol): + """Receive work events containing only bounded dimensions.""" + + def observe_work_enqueue(self, kind: str, *, created: bool) -> None: ... + + def observe_work_claim(self, kind: str, *, latency_seconds: float) -> None: ... + + def observe_work_attempt( + self, + kind: str, + *, + outcome: str, + error_category: str, + duration_seconds: float, + ) -> None: ... + + def observe_work_lease_expiry(self, kind: str, *, outcome: str) -> None: ... + + def observe_scheduler_leadership(self, *, outcome: str) -> None: ... + + def set_work_queue(self, samples: Sequence[WorkQueueStatistic]) -> None: ... + + def set_runtime_members(self, counts: Mapping[str, int]) -> None: ... + + +async def refresh_work_queue( + database: AsyncDatabase, + repository: WorkRepository, + observer: WorkObserver | None, +) -> None: + """Best-effort refresh aggregate queue gauges from durable state.""" + + if observer is None: + return + try: + async with database.transaction() as connection: + samples = await repository.queue_statistics(connection) + observer.set_work_queue(samples) + except Exception: + return + + +__all__ = ["WorkObserver", "refresh_work_queue"] diff --git a/src/powercontext/builtin/runtime/worker.py b/src/powercontext/builtin/runtime/worker.py new file mode 100644 index 000000000..7337d8881 --- /dev/null +++ b/src/powercontext/builtin/runtime/worker.py @@ -0,0 +1,385 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lease-aware Worker loop over the durable Work Ledger.""" + +from __future__ import annotations + +import asyncio +import random +from collections.abc import Awaitable, Callable, Iterable +from contextlib import AbstractContextManager, nullcontext, suppress +from dataclasses import dataclass +from typing import Protocol + +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.work import ( + StaleWorkClaimError, + WorkClaim, + WorkCommit, + WorkFailure, + WorkRepository, + WorkResult, +) +from powercontext.builtin.runtime.config import WorkerConfig +from powercontext.builtin.runtime.protocols import ( + RuntimeSpan, + RuntimeTraceContext, + RuntimeTracing, + runtime_trace_context, +) +from powercontext.builtin.runtime.readiness import ReadinessCheckStatus +from powercontext.builtin.runtime.work_observability import WorkObserver, refresh_work_queue + +ClaimReadiness = Callable[[], Awaitable[str]] +_EMPTY_HANDLERS = "at least one handler is required" +_INVALID_HANDLER_KIND = "handler kind must be a non-empty trimmed string" +_INVALID_HANDLER_VERSIONS = "handler versions must be positive" + + +class WorkHandler(Protocol): + """Prepare one versioned task outside a database transaction.""" + + kind: str + supported_versions: frozenset[int] + + async def prepare(self, claim: WorkClaim, /) -> PreparedWork: ... + + +@dataclass(frozen=True) +class PreparedWork: + """Sanitized result and optional domain write for the final transaction.""" + + result: WorkResult | None + commit: WorkCommit | None = None + + +class WorkExecutionError(RuntimeError): + """A handler failure safe to classify in persistent operation metadata.""" + + def __init__(self, *, category: str, code: str, retryable: bool) -> None: + self.failure = WorkFailure(category=category, code=code, retryable=retryable) + super().__init__(f"work handler failed with {category}/{code}") + + +class DurableWorker: + """Claim no more than available slots and recover through lease expiry.""" + + def __init__( + self, + *, + database: AsyncDatabase, + worker_id: str, + handlers: Iterable[WorkHandler], + config: WorkerConfig, + repository: WorkRepository | None = None, + random_source: Callable[[float, float], float] = random.uniform, + claim_readiness: ClaimReadiness | None = None, + observer: WorkObserver | None = None, + tracing: RuntimeTracing | None = None, + ) -> None: + self._database = database + self._worker_id = worker_id + self._config = config + self._repository = WorkRepository(observer=observer) if repository is None else repository + self._random_source = random_source + self._claim_readiness = claim_readiness + self._observer = observer + self._tracing = tracing + self._handlers = _handler_map(handlers) + self._stop_requested = asyncio.Event() + self._wake_requested = asyncio.Event() + self._claim_lock = asyncio.Lock() + self._running: set[asyncio.Task[None]] = set() + self._failed = False + + @property + def supported(self) -> dict[str, frozenset[int]]: + """Return bounded handler compatibility advertised by this Worker.""" + + return {kind: handler.supported_versions for kind, handler in self._handlers.items()} + + async def run_once(self) -> int: + """Claim current free capacity and await those attempts.""" + + async with self._claim_lock: + capacity = self._config.concurrency - len(self._running) + if capacity <= 0 or self._stop_requested.is_set(): + return 0 + if self._claim_readiness is not None and await self._claim_readiness() != ReadinessCheckStatus.READY: + return 0 + with self._background( + "work.claim", + operation="work.claim", + attributes={"powercontext.work.claim.capacity": capacity}, + ) as span: + claims = await self._claim_capacity(capacity) + if span is not None: + span.set_attributes({"powercontext.work.claim.count": len(claims)}) + tasks = {asyncio.create_task(self._execute(claim)) for claim in claims} + self._running.update(tasks) + for task in tasks: + task.add_done_callback(self._running.discard) + if tasks: + await asyncio.gather(*tasks) + await refresh_work_queue(self._database, self._repository, self._observer) + return len(claims) + + async def _claim_capacity(self, capacity: int) -> tuple[WorkClaim, ...]: + """Claim at most one lane per short transaction.""" + + claims: list[WorkClaim] = [] + supported = self.supported + for _ in range(capacity): + async with self._database.transaction() as connection: + claimed = await self._repository.claim( + connection, + worker_id=self._worker_id, + supported=supported, + lease_seconds=self._config.lease_seconds, + limit=1, + ) + if claimed: + claims.extend(claimed) + else: + break + return tuple(claims) + + async def run(self) -> None: + """Poll until shutdown, executing each claimed batch concurrently.""" + + while not self._stop_requested.is_set(): + self._wake_requested.clear() + try: + claimed = await self.run_once() + self._failed = False + except asyncio.CancelledError: + raise + except Exception: + self._failed = True + claimed = 0 + if claimed: + continue + with suppress(TimeoutError): + await asyncio.wait_for(self._wake_requested.wait(), timeout=self._config.poll_seconds) + + def notify(self) -> None: + """Wake the polling loop after a producer commits new work.""" + + self._wake_requested.set() + + async def readiness(self) -> str: + """Report database-loop and registered payload compatibility.""" + + if self._failed: + return ReadinessCheckStatus.UNAVAILABLE + if self._claim_readiness is not None: + dependency = await self._claim_readiness() + if dependency != ReadinessCheckStatus.READY: + return dependency + async with self._database.transaction() as connection: + unknown = await self._repository.unsupported_head_count(connection, self.supported) + return ReadinessCheckStatus.MISCONFIGURED if unknown else ReadinessCheckStatus.READY + + async def stop(self) -> None: + """Stop claiming, then give in-flight handlers their configured grace.""" + + self._stop_requested.set() + self._wake_requested.set() + if not self._running: + return + _, pending = await asyncio.wait(self._running, timeout=self._config.shutdown_grace_seconds) + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + + async def _execute(self, claim: WorkClaim) -> None: + links = ( + () + if claim.previous_trace_id is None or claim.previous_span_id is None + else (RuntimeTraceContext(trace_id=claim.previous_trace_id, span_id=claim.previous_span_id),) + ) + with self._background( + "work.execute", + operation="work.execute", + attributes={ + "powercontext.work.kind": claim.kind, + "powercontext.work.payload_version": claim.payload_version, + "powercontext.work.attempt": claim.attempt_no, + "powercontext.work.recovery_generation": claim.recovery_generation, + }, + links=links, + ) as span: + context = runtime_trace_context(span) + if context is not None and not await self._bind_attempt_trace(claim, context): + if span is not None: + span.set_outcome("stale") + return + outcome = await self._execute_claim(claim) + if span is not None: + span.set_outcome(outcome) + + async def _execute_claim(self, claim: WorkClaim) -> str: + handler = self._handlers[claim.kind] + heartbeat_stop = asyncio.Event() + heartbeat = asyncio.create_task(self._heartbeat(claim, heartbeat_stop)) + try: + prepared = await handler.prepare(claim) + if heartbeat.done() and heartbeat.exception() is not None: + await heartbeat + with self._stage( + "work.commit", + attributes={ + "powercontext.work.kind": claim.kind, + "powercontext.work.payload_version": claim.payload_version, + }, + ): + async with self._database.transaction() as connection: + await self._repository.complete( + connection, + claim, + prepared.result, + commit=prepared.commit, + ) + except asyncio.CancelledError: + raise + except StaleWorkClaimError: + return await self._finish_cancel_if_owned(claim) + except WorkExecutionError as error: + return await self._record_failure(claim, error.failure) + except Exception: + return await self._record_failure( + claim, + WorkFailure(category="internal", code="unhandled_handler_error", retryable=True), + ) + else: + return "succeeded" + finally: + heartbeat_stop.set() + await asyncio.gather(heartbeat, return_exceptions=True) + + async def _finish_cancel_if_owned(self, claim: WorkClaim) -> str: + """Converge a fenced cancelling claim without waiting for lease expiry.""" + + try: + async with self._database.transaction() as connection: + work = await self._repository.fail( + connection, + claim, + WorkFailure(category="cancellation", code="cancel_requested", retryable=False), + retry_delay_seconds=0, + ) + except StaleWorkClaimError: + return "stale" + return work.status.value + + async def _heartbeat(self, claim: WorkClaim, stop: asyncio.Event) -> None: + while not stop.is_set(): + try: + await asyncio.wait_for(stop.wait(), timeout=self._config.heartbeat_seconds) + except TimeoutError: + async with self._database.transaction() as connection: + await self._repository.heartbeat( + connection, + claim, + lease_seconds=self._config.lease_seconds, + ) + + async def _bind_attempt_trace(self, claim: WorkClaim, context: RuntimeTraceContext) -> bool: + try: + async with self._database.transaction() as connection: + await self._repository.record_attempt_trace( + connection, + claim, + trace_id=context.trace_id, + span_id=context.span_id, + ) + except StaleWorkClaimError: + return False + return True + + async def _record_failure(self, claim: WorkClaim, failure: WorkFailure) -> str: + ceiling = min( + self._config.retry_max_seconds, + self._config.retry_base_seconds * (2 ** max(claim.generation_attempt_no - 1, 0)), + ) + delay = self._random_source(0.0, ceiling) + try: + with self._stage( + "work.retry", + attributes={ + "powercontext.work.kind": claim.kind, + "powercontext.work.error_category": failure.category, + "powercontext.work.retryable": failure.retryable, + }, + ) as span: + async with self._database.transaction() as connection: + work = await self._repository.fail( + connection, + claim, + failure, + retry_delay_seconds=max(0, round(delay)), + ) + if span is not None: + span.set_outcome(work.status.value) + return work.status.value + except StaleWorkClaimError: + return "stale" + + def _stage( + self, + name: str, + *, + attributes: dict[str, str | bool | int | float], + ) -> AbstractContextManager[RuntimeSpan | None]: + if self._tracing is None: + return nullcontext(None) + return self._tracing.stage(name, attributes=attributes) + + def _background( + self, + name: str, + *, + operation: str, + attributes: dict[str, str | bool | int | float], + links: tuple[RuntimeTraceContext, ...] = (), + ) -> AbstractContextManager[RuntimeSpan | None]: + if self._tracing is None: + return nullcontext(None) + return self._tracing.background(name, operation=operation, attributes=attributes, links=links) + + +def _handler_map(handlers: Iterable[WorkHandler]) -> dict[str, WorkHandler]: + registered: dict[str, WorkHandler] = {} + for handler in handlers: + if not handler.kind.strip() or handler.kind != handler.kind.strip(): + raise ValueError(_INVALID_HANDLER_KIND) + if not handler.supported_versions or any(version < 1 for version in handler.supported_versions): + raise ValueError(_INVALID_HANDLER_VERSIONS) + if handler.kind in registered: + message = f"duplicate handler kind: {handler.kind}" + raise ValueError(message) + registered[handler.kind] = handler + if not registered: + raise ValueError(_EMPTY_HANDLERS) + return registered + + +__all__ = [ + "DurableWorker", + "PreparedWork", + "WorkExecutionError", + "WorkHandler", +] diff --git a/src/powercontext/cli/config.py b/src/powercontext/cli/config.py index 63c5fbd35..a3295462c 100644 --- a/src/powercontext/cli/config.py +++ b/src/powercontext/cli/config.py @@ -514,19 +514,8 @@ def _validate_builtin_runtime(values: Mapping[str, str]) -> None: try: settings = _server_settings_from_environment() from powercontext.builtin.runtime.composition import preflight_builtin_runtime - from powercontext.builtin.runtime.config import BuiltinConfig - - asyncio.run( - preflight_builtin_runtime( - BuiltinConfig( - runtime=settings.runtime, - database=settings.database, - handoff_report=settings.handoff_report, - inference=settings.inference, - external_skills=settings.external_skills, - ) - ) - ) + + asyncio.run(preflight_builtin_runtime(settings.to_builtin_config())) except ConfigError: raise except Exception as error: diff --git a/src/powercontext/client/__init__.py b/src/powercontext/client/__init__.py index b6af0d234..c61db5b85 100644 --- a/src/powercontext/client/__init__.py +++ b/src/powercontext/client/__init__.py @@ -15,7 +15,14 @@ """Python Client SDK package for the public PowerContext HTTP API.""" from powercontext.client.client import PowerContextClient -from powercontext.client.errors import ClientError, InvalidResponseError, ServerResponseError, TransportError +from powercontext.client.errors import ( + ClientError, + InvalidResponseError, + OperationFailedError, + OperationPendingError, + ServerResponseError, + TransportError, +) from powercontext.client.ingestion import RemoteConnectorWorker from powercontext.client.skill_receiver import ( RECEIVER_VERSION, @@ -32,6 +39,8 @@ "RECEIVER_VERSION", "ClientError", "InvalidResponseError", + "OperationFailedError", + "OperationPendingError", "PowerContextClient", "ReceiverSyncResult", "RemoteConnectorWorker", diff --git a/src/powercontext/client/client.py b/src/powercontext/client/client.py index f26f99873..7c4a4782d 100644 --- a/src/powercontext/client/client.py +++ b/src/powercontext/client/client.py @@ -18,14 +18,22 @@ import asyncio from collections.abc import Mapping +from time import monotonic from types import TracebackType -from typing import Self, TypeVar +from typing import Any, Self, TypeVar, cast from urllib.parse import quote +from uuid import UUID import httpx from pydantic import TypeAdapter, ValidationError -from powercontext.client.errors import InvalidResponseError, ServerResponseError, TransportError +from powercontext.client.errors import ( + InvalidResponseError, + OperationFailedError, + OperationPendingError, + ServerResponseError, + TransportError, +) from powercontext.client.tracing import ClientSpan from powercontext.http import ( AcknowledgeHandoffRequest, @@ -55,6 +63,7 @@ FinalizeHandoffRequest, FlushMemoryRequest, FlushMemoryResponse, + FlushStatus, GeneratedCandidateResponse, GenerateExperienceRequest, GenerateSkillRequest, @@ -83,10 +92,17 @@ ListMemoryChangesResponse, ListMemoryEntriesRequest, ListMemoryEntriesResponse, + ListOperationsRequest, ListRemoteSkillTargetsRequest, ListRemoteSkillTargetsResponse, MemoryEntry, MemoryMutationResponse, + MemoryOperationResult, + OperationAccepted, + OperationMutationRequest, + OperationPage, + OperationRecord, + OperationStatus, PrepareContextRequest, PreparedContext, PreparedHandoff, @@ -145,6 +161,7 @@ ACKNOWLEDGE_HANDOFF, ACTIVATE_HANDOFF, APPROVE_ARTIFACT_CANDIDATE, + CANCEL_OPERATION, CAPTURE_CONTENT_SOURCE, CLEAR_SCOPE_BINDING, COMMIT_CONNECTOR_CHECKPOINT, @@ -168,6 +185,7 @@ GET_HANDOFF_REPORT, GET_LIVENESS, GET_MEMORY_ENTRY, + GET_OPERATION, GET_READINESS, GET_SCOPE, GET_SKILL, @@ -180,6 +198,7 @@ LIST_MANAGED_SKILLS, LIST_MEMORY_CHANGES, LIST_MEMORY_ENTRIES, + LIST_OPERATIONS, LIST_REMOTE_SKILL_TARGETS, LIST_SCOPES, PREPARE_CONTEXT, @@ -201,6 +220,7 @@ RESOLVE_SCOPE_BINDING, RESOLVE_SCOPE_SELECTION, RETIRE_MEMORY_ENTRY, + RETRY_OPERATION, REVISE_ARTIFACT_CANDIDATE, REVISE_MEMORY_ENTRY, REVOKE_REMOTE_SKILL_TARGET, @@ -232,6 +252,8 @@ def __init__( timeout: float = 10.0, http_client: httpx.AsyncClient | None = None, trust_transport_security: bool = False, + operation_timeout: float = 30.0, + operation_poll_seconds: float = 0.2, allow_insecure_http: bool = False, ) -> None: self._base_url = base_url.rstrip("/") @@ -252,6 +274,12 @@ def __init__( if not transport_trusted and not allow_insecure_http and is_plaintext_non_loopback(self._base_url): raise ValueError("refusing to send requests over unencrypted non-loopback HTTP") # noqa: TRY003 self._headers = {"Authorization": f"Bearer {token}"} if token else None + if operation_timeout <= 0: + raise ValueError("operation_timeout must be positive") # noqa: TRY003 + if operation_poll_seconds <= 0: + raise ValueError("operation_poll_seconds must be positive") # noqa: TRY003 + self._operation_timeout = operation_timeout + self._operation_poll_seconds = operation_poll_seconds self._owned_http_client: httpx.AsyncClient | None = None if http_client is None: self._owned_http_client = httpx.AsyncClient(timeout=timeout) @@ -392,10 +420,10 @@ async def _request_handoff_report_content(self, request: GetHandoffReportRequest span.finish("failure", error=error) raise span.finish( - "success" if response.status_code == GET_HANDOFF_REPORT.success_status else "failure", + "success" if response.status_code in GET_HANDOFF_REPORT.success_statuses else "failure", status_code=response.status_code, ) - if response.status_code != GET_HANDOFF_REPORT.success_status: + if response.status_code not in GET_HANDOFF_REPORT.success_statuses: error = _decode_error(response.content) raise ServerResponseError( status_code=response.status_code, @@ -457,7 +485,75 @@ async def record_task_outcome(self, request: RecordTaskOutcomeRequest) -> WorkSo async def flush_memory(self, request: FlushMemoryRequest) -> FlushMemoryResponse: """Run one bounded Source-to-Memory activation.""" - return await self._request(FLUSH_MEMORY, request) + deadline = monotonic() + self._operation_timeout + submitted = await self.submit_memory_flush(request) + if isinstance(submitted, FlushMemoryResponse): + return submitted + operation_id = submitted.operation_id + while True: + remaining = deadline - monotonic() + if remaining <= 0: + raise OperationPendingError(str(operation_id)) + await asyncio.sleep(min(self._operation_poll_seconds, remaining)) + operation = await self.get_operation(operation_id) + if operation.status is OperationStatus.SUCCEEDED: + return _flush_response_from_operation(operation) + if operation.status in {OperationStatus.FAILED, OperationStatus.CANCELLED}: + raise OperationFailedError( + str(operation_id), + code=None if operation.error is None else operation.error.code, + ) + + async def submit_memory_flush( + self, + request: FlushMemoryRequest, + ) -> FlushMemoryResponse | OperationAccepted: + """Submit one Memory window and return immediately when it remains pending.""" + + return await self._request( + FLUSH_MEMORY, + request, + extra_headers={"Prefer": "respond-async"}, + ) + + async def get_operation(self, operation_id: str | UUID) -> OperationRecord: + """Read one durable operation.""" + + return await self._request( + GET_OPERATION, + path_parameters={"operation_id": str(UUID(str(operation_id)))}, + ) + + async def list_operations(self, request: ListOperationsRequest) -> OperationPage: + """List a bounded page of durable operations.""" + + return await self._request(LIST_OPERATIONS, request) + + async def cancel_operation( + self, + operation_id: str | UUID, + request: OperationMutationRequest, + ) -> OperationRecord: + """Cancel a queued or active operation using its state version.""" + + return await self._request( + CANCEL_OPERATION, + request, + path_parameters={"operation_id": str(UUID(str(operation_id)))}, + ) + + async def retry_operation( + self, + operation_id: str | UUID, + request: OperationMutationRequest, + ) -> OperationRecord: + """Recover one blocked failed operation using its state version.""" + + return await self._request( + RETRY_OPERATION, + request, + path_parameters={"operation_id": str(UUID(str(operation_id)))}, + ) async def remember_memory(self, request: RememberMemoryRequest) -> MemoryMutationResponse: """Save one explicit Memory entry without creating a Source.""" @@ -709,6 +805,7 @@ async def _request( request: _RequestT | None = None, *, path_parameters: Mapping[str, str] | None = None, + extra_headers: dict[str, str] | None = None, ) -> _ResponseT: operation_path = _bind_operation_path(operation, path_parameters) json_payload = None @@ -727,13 +824,45 @@ async def _request( else: json_payload = payload - span = ClientSpan.start(operation.operation_id) + response, request_id = await self._send( + method=operation.method, + path=operation_path, + operation_id=operation.operation_id, + json_payload=json_payload, + query_parameters=query_parameters, + extra_headers=extra_headers, + success_statuses=operation.success_statuses, + ) + + try: + response_type = operation.success_response_types[response.status_code] + return cast(_ResponseT, TypeAdapter(response_type).validate_json(response.content)) + except ValidationError as exc: + raise InvalidResponseError( + operation_path, + request_id=request_id, + ) from exc + + async def _send( + self, + *, + method: str, + path: str, + operation_id: str, + success_statuses: tuple[int, ...], + json_payload: Any = None, + query_parameters: Any = None, + extra_headers: dict[str, str] | None = None, + ) -> tuple[httpx.Response, str | None]: + headers = {} if self._headers is None else dict(self._headers) + if extra_headers is not None: + headers.update(extra_headers) + span = ClientSpan.start(operation_id) try: - headers = {} if self._headers is None else dict(self._headers) span.inject(headers) response = await self._http_client.request( - operation.method, - f"{self._base_url}{operation_path}", + method, + f"{self._base_url}{path}", json=json_payload, headers=headers, params=query_parameters, @@ -741,19 +870,16 @@ async def _request( except asyncio.CancelledError as error: span.finish("cancelled", error=error) raise - except httpx.HTTPError as exc: - span.finish("failure", error=exc) - raise TransportError(operation_path) from exc + except httpx.HTTPError as error: + span.finish("failure", error=error) + raise TransportError(path) from error except BaseException as error: span.finish("failure", error=error) raise - span.finish( - "success" if response.status_code == operation.success_status else "failure", - status_code=response.status_code, - ) - + success = response.status_code in success_statuses + span.finish("success" if success else "failure", status_code=response.status_code) request_id = response.headers.get(REQUEST_ID_HEADER) - if response.status_code != operation.success_status: + if not success: error = _decode_error(response.content) raise ServerResponseError( status_code=response.status_code, @@ -762,14 +888,7 @@ async def _request( message=None if error is None else error.error.message, details=None if error is None else error.error.details, ) - - try: - return TypeAdapter(operation.response_type).validate_json(response.content) - except ValidationError as exc: - raise InvalidResponseError( - operation_path, - request_id=request_id, - ) from exc + return response, request_id def _bind_operation_path( @@ -804,3 +923,18 @@ def _decode_error(content: bytes) -> ErrorResponse | None: return ErrorResponse.model_validate_json(content) except ValidationError: return None + + +def _flush_response_from_operation(operation: OperationRecord) -> FlushMemoryResponse: + result = operation.result + if not isinstance(result, MemoryOperationResult): + raise InvalidResponseError(GET_OPERATION.path, request_id=None) + flush_status = FlushStatus.PROCESSED if result.current_cursor > result.previous_cursor else FlushStatus.IDLE + return FlushMemoryResponse( + status=flush_status, + previous_cursor=result.previous_cursor, + current_cursor=result.current_cursor, + high_watermark=result.high_watermark, + processed_source_count=result.processed_source_count, + memory=result.memory, + ) diff --git a/src/powercontext/client/errors.py b/src/powercontext/client/errors.py index 24da5781f..787cbc79e 100644 --- a/src/powercontext/client/errors.py +++ b/src/powercontext/client/errors.py @@ -42,6 +42,24 @@ def __init__(self, path: str, *, request_id: str | None) -> None: super().__init__(f"response from {path} violated the API schema") +class OperationPendingError(ClientError): + """Raised when a durable operation outlives the Client deadline.""" + + def __init__(self, operation_id: str) -> None: + self.operation_id = operation_id + super().__init__(f"operation {operation_id} is still pending") + + +class OperationFailedError(ClientError): + """Raised when a durable operation reaches failed or cancelled.""" + + def __init__(self, operation_id: str, *, code: str | None) -> None: + self.operation_id = operation_id + self.code = code + suffix = "" if code is None else f" ({code})" + super().__init__(f"operation {operation_id} did not succeed{suffix}") + + class ServerResponseError(ClientError): """Raised when the Server returns a non-success status.""" diff --git a/src/powercontext/http/__init__.py b/src/powercontext/http/__init__.py index f69223518..c327f18da 100644 --- a/src/powercontext/http/__init__.py +++ b/src/powercontext/http/__init__.py @@ -58,6 +58,7 @@ ExactScopeSelection, Exclusion, ExperienceArtifact, + ExperienceOperationResult, ExperienceProposal, ExternalSkillImportMode, ExternalSkillInstallationScope, @@ -120,6 +121,7 @@ ListMemoryChangesResponse, ListMemoryEntriesRequest, ListMemoryEntriesResponse, + ListOperationsRequest, ListRemoteSkillTargetsRequest, ListRemoteSkillTargetsResponse, LiveStateCheckStatus, @@ -132,6 +134,7 @@ MemoryKindCount, MemoryMatchedBy, MemoryMutationResponse, + MemoryOperationResult, MemoryRevisionChanges, MemorySearchMode, MemoryUsedSearchMode, @@ -141,6 +144,13 @@ ModelUsageValue, Omission, OpenQuestion, + OperationAccepted, + OperationError, + OperationKind, + OperationMutationRequest, + OperationPage, + OperationRecord, + OperationStatus, PrepareContextRequest, PreparedContext, PreparedContextSchema, @@ -299,6 +309,7 @@ "ExactScopeSelection", "Exclusion", "ExperienceArtifact", + "ExperienceOperationResult", "ExperienceProposal", "ExternalSkillImportMode", "ExternalSkillInstallationScope", @@ -361,6 +372,7 @@ "ListMemoryChangesResponse", "ListMemoryEntriesRequest", "ListMemoryEntriesResponse", + "ListOperationsRequest", "ListRemoteSkillTargetsRequest", "ListRemoteSkillTargetsResponse", "LiveStateCheckStatus", @@ -373,6 +385,7 @@ "MemoryKindCount", "MemoryMatchedBy", "MemoryMutationResponse", + "MemoryOperationResult", "MemoryRevisionChanges", "MemorySearchMode", "MemoryUsedSearchMode", @@ -382,6 +395,13 @@ "ModelUsageValue", "Omission", "OpenQuestion", + "OperationAccepted", + "OperationError", + "OperationKind", + "OperationMutationRequest", + "OperationPage", + "OperationRecord", + "OperationStatus", "PrepareContextRequest", "PrepareHandoffRequest", "PreparedContext", diff --git a/src/powercontext/http/_generated/models.py b/src/powercontext/http/_generated/models.py index e144a6ba2..5f6b83f06 100644 --- a/src/powercontext/http/_generated/models.py +++ b/src/powercontext/http/_generated/models.py @@ -6,6 +6,7 @@ from datetime import date as date_aliased from enum import StrEnum from typing import Annotated, Any, Literal +from uuid import UUID from pydantic import ( AwareDatetime, @@ -1200,6 +1201,113 @@ class ImportExternalSkillRequest(BaseModel): reason: Annotated[StrictStr | None, Field(max_length=2000, min_length=1)] = None +class OperationStatus(StrEnum): + QUEUED = "queued" + RUNNING = "running" + RETRY_WAIT = "retry_wait" + CANCELLING = "cancelling" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + + +class OperationKind(StrEnum): + MEMORY_FLUSH = "memory_flush" + EXPERIENCE_INCUBATION = "experience_incubation" + + +class Type(StrEnum): + MEMORY_FLUSH = "memory_flush" + + +class MemoryOperationResult(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + type: Literal["memory_flush"] + previous_cursor: Annotated[StrictInt, Field(ge=0)] + high_watermark: Annotated[StrictInt, Field(ge=0)] + current_cursor: Annotated[StrictInt, Field(ge=0)] + processed_source_count: Annotated[StrictInt, Field(ge=0)] + memory: ArtifactReference | None = None + + +class Type1(StrEnum): + EXPERIENCE_INCUBATION = "experience_incubation" + + +class ExperienceOperationResult(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + type: Literal["experience_incubation"] + previous_cursor: Annotated[StrictInt, Field(ge=0)] + high_watermark: Annotated[StrictInt, Field(ge=0)] + current_cursor: Annotated[StrictInt, Field(ge=0)] + processed_source_count: Annotated[StrictInt, Field(ge=0)] + candidate_count: Annotated[StrictInt, Field(ge=0)] + + +class OperationError(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + category: Annotated[StrictStr, Field(max_length=64, min_length=1)] + code: Annotated[StrictStr, Field(max_length=128, min_length=1)] + + +class OperationRecord(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + operation_id: UUID + kind: OperationKind + scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1)] + status: OperationStatus + attempt_count: Annotated[StrictInt, Field(ge=0)] + state_version: Annotated[StrictInt, Field(ge=1)] + created_at: AwareDatetime + updated_at: AwareDatetime + completed_at: AwareDatetime | None = None + result: Annotated[MemoryOperationResult | ExperienceOperationResult | None, Field(discriminator="type")] = None + error: OperationError | None = None + + +class OperationAccepted(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + operation_id: UUID + status: OperationStatus + status_url: Annotated[StrictStr, Field(pattern="^/v1/operations/[0-9a-f-]{36}$")] + + +class OperationPage(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + items: Annotated[list[OperationRecord], Field(max_length=100)] + next_cursor: StrictStr | None = None + + +class ListOperationsRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + scope_id: Annotated[StrictStr | None, Field(max_length=256, min_length=1)] = None + kind: OperationKind | None = None + status: OperationStatus | None = None + cursor: StrictStr | None = None + limit: Annotated[StrictInt, Field(ge=1, le=100)] = 50 + + +class OperationMutationRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + expected_version: Annotated[StrictInt, Field(ge=1)] + + class SourceReference(BaseModel): model_config = ConfigDict( extra="forbid", diff --git a/src/powercontext/http/_generated/operations.py b/src/powercontext/http/_generated/operations.py index af53a697d..e8f50f82a 100644 --- a/src/powercontext/http/_generated/operations.py +++ b/src/powercontext/http/_generated/operations.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Generic, Literal, TypeVar +from typing import Generic, Literal, TypeVar, cast from pydantic import BaseModel, JsonValue @@ -61,10 +61,15 @@ ListMemoryChangesResponse, ListMemoryEntriesRequest, ListMemoryEntriesResponse, + ListOperationsRequest, ListRemoteSkillTargetsRequest, ListRemoteSkillTargetsResponse, MemoryEntry, MemoryMutationResponse, + OperationAccepted, + OperationMutationRequest, + OperationPage, + OperationRecord, PrepareContextRequest, PreparedContext, PreparedHandoff, @@ -136,13 +141,24 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type: type[RequestT] | None request_location: Literal["body", "query"] | None path_parameters: tuple[str, ...] - response_type: type[ResponseT] - success_status: int + success_response_types: dict[int, type[BaseModel]] summary: str tags: tuple[str, ...] scope_mode: Literal["none", "current", "selection"] responses: dict[int | str, dict[str, JsonValue]] + @property + def success_statuses(self) -> tuple[int, ...]: + return tuple(sorted(self.success_response_types)) + + @property + def success_status(self) -> int: + return self.success_statuses[0] + + @property + def response_type(self) -> type[ResponseT]: + return cast(type[ResponseT], self.success_response_types[self.success_status]) + GET_LIVENESS = Operation[None, HealthResponse]( method="GET", @@ -151,8 +167,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=None, request_location=None, path_parameters=(), - response_type=HealthResponse, - success_status=200, + success_response_types={200: HealthResponse}, summary="Get process liveness", tags=("health",), scope_mode="none", @@ -171,8 +186,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=None, request_location=None, path_parameters=(), - response_type=ReadinessResponse, - success_status=200, + success_response_types={200: ReadinessResponse}, summary="Get deployment readiness", tags=("health",), scope_mode="none", @@ -195,8 +209,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=None, request_location=None, path_parameters=(), - response_type=Capabilities, - success_status=200, + success_response_types={200: Capabilities}, summary="Get runtime capabilities", tags=("capabilities",), scope_mode="none", @@ -206,6 +219,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, }, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, }, ) @@ -216,8 +230,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=None, request_location=None, path_parameters=(), - response_type=ScopePage, - success_status=200, + success_response_types={200: ScopePage}, summary="List observable Scopes", tags=("scopes",), scope_mode="none", @@ -235,8 +248,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=CreateScopeRequest, request_location="body", path_parameters=(), - response_type=ScopeDescriptor, - success_status=201, + success_response_types={201: ScopeDescriptor}, summary="Create an independent Scope boundary", tags=("scopes",), scope_mode="none", @@ -256,8 +268,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=PublishArtifactRequest, request_location="body", path_parameters=(), - response_type=ArtifactPublication, - success_status=201, + success_response_types={201: ArtifactPublication}, summary="Publish one exact Artifact revision into another Scope", tags=("scopes",), scope_mode="none", @@ -277,8 +288,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=None, request_location=None, path_parameters=("scope_id",), - response_type=ScopeDescriptor, - success_status=200, + success_response_types={200: ScopeDescriptor}, summary="Get one Scope descriptor", tags=("scopes",), scope_mode="none", @@ -296,8 +306,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=UpdateScopeRequest, request_location="body", path_parameters=("scope_id",), - response_type=ScopeDescriptor, - success_status=200, + success_response_types={200: ScopeDescriptor}, summary="Replace mutable Scope metadata and relationships", tags=("scopes",), scope_mode="none", @@ -317,8 +326,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=None, request_location=None, path_parameters=(), - response_type=ScopeDescriptor, - success_status=200, + success_response_types={200: ScopeDescriptor}, summary="Get the default Scope binding target", tags=("scopes",), scope_mode="none", @@ -336,8 +344,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=SetDefaultScopeRequest, request_location="body", path_parameters=(), - response_type=ScopeDescriptor, - success_status=200, + success_response_types={200: ScopeDescriptor}, summary="Change the default Scope binding target", tags=("scopes",), scope_mode="none", @@ -355,8 +362,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ResolveScopeSelectionRequest, request_location="body", path_parameters=(), - response_type=ScopePage, - success_status=200, + success_response_types={200: ScopePage}, summary="Resolve an observation selection to a frozen Scope set", tags=("scopes",), scope_mode="none", @@ -375,8 +381,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ResolveScopeBindingRequest, request_location="body", path_parameters=(), - response_type=ScopeDescriptor, - success_status=200, + success_response_types={200: ScopeDescriptor}, summary="Resolve an explicit durable or default Scope binding", tags=("scope-bindings",), scope_mode="none", @@ -395,8 +400,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=SetScopeBindingRequest, request_location="body", path_parameters=(), - response_type=ScopeBinding, - success_status=200, + success_response_types={200: ScopeBinding}, summary="Persist an external identity to Scope binding", tags=("scope-bindings",), scope_mode="none", @@ -415,8 +419,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ClearScopeBindingRequest, request_location="body", path_parameters=(), - response_type=ClearScopeBindingResponse, - success_status=200, + success_response_types={200: ClearScopeBindingResponse}, summary="Remove one durable external Scope binding", tags=("scope-bindings",), scope_mode="none", @@ -434,8 +437,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=CaptureContentSourceRequest, request_location="body", path_parameters=(), - response_type=CaptureContentSourceResponse, - success_status=202, + success_response_types={202: CaptureContentSourceResponse}, summary="Capture durable ContentSource evidence", tags=("sources",), scope_mode="current", @@ -446,6 +448,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -459,8 +462,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=RegisterSourceDefinitionRequest, request_location="body", path_parameters=(), - response_type=SourceDefinitionManifest, - success_status=200, + success_response_types={200: SourceDefinitionManifest}, summary="Register a worker-owned Source Definition manifest", tags=("source-ingestion",), scope_mode="none", @@ -479,8 +481,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=GetConnectorCheckpointRequest, request_location="body", path_parameters=(), - response_type=ConnectorCheckpointState, - success_status=200, + success_response_types={200: ConnectorCheckpointState}, summary="Read a Connector binding checkpoint", tags=("source-ingestion",), scope_mode="none", @@ -499,8 +500,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=SubmitSourceObservationRequest, request_location="body", path_parameters=(), - response_type=SourceObservationReceipt, - success_status=202, + success_response_types={202: SourceObservationReceipt}, summary="Submit a worker-materialized Source observation", tags=("source-ingestion",), scope_mode="none", @@ -520,8 +520,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=CommitConnectorCheckpointRequest, request_location="body", path_parameters=(), - response_type=ConnectorCheckpointState, - success_status=200, + success_response_types={200: ConnectorCheckpointState}, summary="Commit a Connector binding checkpoint", tags=("source-ingestion",), scope_mode="none", @@ -540,8 +539,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=PrepareContextRequest, request_location="body", path_parameters=(), - response_type=PreparedContext, - success_status=200, + success_response_types={200: PreparedContext}, summary="Prepare bounded context for an Agent turn", tags=("context",), scope_mode="current", @@ -551,6 +549,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, }, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -564,8 +563,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=CreateWorkContractRequest, request_location="body", path_parameters=(), - response_type=WorkSourceReceipt, - success_status=202, + success_response_types={202: WorkSourceReceipt}, summary="Create a grounded Work Contract", tags=("work",), scope_mode="current", @@ -577,6 +575,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): 404: {"$ref": "#/components/responses/NotFound"}, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -590,8 +589,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=HandoffCurrentWorkRequest, request_location="body", path_parameters=(), - response_type=PreparedWorkHandoff, - success_status=200, + success_response_types={200: PreparedWorkHandoff}, summary="Hand off current work in one high-level operation", tags=("work",), scope_mode="current", @@ -603,6 +601,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): 404: {"$ref": "#/components/responses/NotFound"}, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -616,8 +615,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=AcknowledgeHandoffRequest, request_location="body", path_parameters=(), - response_type=HandoffAcknowledgement, - success_status=200, + success_response_types={200: HandoffAcknowledgement}, summary="Resolve and acknowledge a Handoff", tags=("work",), scope_mode="current", @@ -629,6 +627,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): 404: {"$ref": "#/components/responses/NotFound"}, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -642,8 +641,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=RecordTaskOutcomeRequest, request_location="body", path_parameters=(), - response_type=WorkSourceReceipt, - success_status=202, + success_response_types={202: WorkSourceReceipt}, summary="Record a completion-aware Task Outcome", tags=("work",), scope_mode="current", @@ -656,6 +654,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): 404: {"$ref": "#/components/responses/NotFound"}, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -669,8 +668,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ActivateHandoffRequest, request_location="body", path_parameters=(), - response_type=HandoffActivation, - success_status=200, + success_response_types={200: HandoffActivation}, summary="Activate Handoff generation at a Source boundary", tags=("handoff",), scope_mode="current", @@ -681,6 +679,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 404: {"$ref": "#/components/responses/NotFound"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -694,8 +693,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=PrepareHandoffRequest, request_location="body", path_parameters=(), - response_type=HandoffDraft, - success_status=200, + success_response_types={200: HandoffDraft}, summary="Generate an inspectable Handoff Draft", tags=("handoff",), scope_mode="current", @@ -706,6 +704,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 404: {"$ref": "#/components/responses/NotFound"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -719,8 +718,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=FinalizeHandoffRequest, request_location="body", path_parameters=(), - response_type=PreparedHandoff, - success_status=200, + success_response_types={200: PreparedHandoff}, summary="Finalize an inspected Handoff Draft", tags=("handoff",), scope_mode="current", @@ -731,6 +729,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 404: {"$ref": "#/components/responses/NotFound"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -744,8 +743,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=CommitHandoffRequest, request_location="body", path_parameters=(), - response_type=CommittedHandoff, - success_status=200, + success_response_types={200: CommittedHandoff}, summary="Commit an explicit Handoff milestone", tags=("handoff",), scope_mode="current", @@ -757,6 +755,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): 404: {"$ref": "#/components/responses/NotFound"}, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -770,8 +769,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ContinueHandoffRequest, request_location="body", path_parameters=(), - response_type=HandoffResolution, - success_status=200, + success_response_types={200: HandoffResolution}, summary="Resolve a Handoff as untrusted historical input", tags=("handoff",), scope_mode="current", @@ -782,21 +780,21 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 404: {"$ref": "#/components/responses/NotFound"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, }, ) -FLUSH_MEMORY = Operation[FlushMemoryRequest, FlushMemoryResponse]( +FLUSH_MEMORY = Operation[FlushMemoryRequest, FlushMemoryResponse | OperationAccepted]( method="POST", path="/v1/memory/flush", operation_id="flush_memory", request_type=FlushMemoryRequest, request_location="body", path_parameters=(), - response_type=FlushMemoryResponse, - success_status=200, + success_response_types={200: FlushMemoryResponse, 202: OperationAccepted}, summary="Process the pending Source window into Memory", tags=("memory",), scope_mode="current", @@ -805,7 +803,17 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): "description": "The activation completed or found no pending Sources.", "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, }, + 202: { + "description": "The durable operation is still queued, running, or waiting to retry.", + "headers": { + "X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}, + "Location": {"description": "Relative URL of the accepted operation.", "schema": {"type": "string"}}, + "Retry-After": {"description": "Suggested polling delay in seconds.", "schema": {"type": "integer"}}, + }, + }, + 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -819,8 +827,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=RememberMemoryRequest, request_location="body", path_parameters=(), - response_type=MemoryMutationResponse, - success_status=200, + success_response_types={200: MemoryMutationResponse}, summary="Remember explicit Memory content", tags=("memory",), scope_mode="current", @@ -831,6 +838,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -844,8 +852,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=SearchMemoryRequest, request_location="body", path_parameters=(), - response_type=SearchMemoryResponse, - success_status=200, + success_response_types={200: SearchMemoryResponse}, summary="Search active Memory entries", tags=("memory",), scope_mode="current", @@ -856,6 +863,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -869,8 +877,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ListMemoryEntriesRequest, request_location="body", path_parameters=(), - response_type=ListMemoryEntriesResponse, - success_status=200, + success_response_types={200: ListMemoryEntriesResponse}, summary="List Memory entries", tags=("memory",), scope_mode="current", @@ -881,6 +888,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 404: {"$ref": "#/components/responses/NotFound"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -894,8 +902,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=GetMemoryEntryRequest, request_location="body", path_parameters=(), - response_type=MemoryEntry, - success_status=200, + success_response_types={200: MemoryEntry}, summary="Get an exact Memory entry version", tags=("memory",), scope_mode="current", @@ -906,6 +913,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 404: {"$ref": "#/components/responses/NotFound"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -919,8 +927,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ReviseMemoryEntryRequest, request_location="body", path_parameters=(), - response_type=MemoryMutationResponse, - success_status=200, + success_response_types={200: MemoryMutationResponse}, summary="Revise an exact Memory entry", tags=("memory",), scope_mode="current", @@ -932,6 +939,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): 404: {"$ref": "#/components/responses/NotFound"}, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -945,8 +953,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=RetireMemoryEntryRequest, request_location="body", path_parameters=(), - response_type=MemoryMutationResponse, - success_status=200, + success_response_types={200: MemoryMutationResponse}, summary="Retire an exact Memory entry", tags=("memory",), scope_mode="current", @@ -958,6 +965,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): 404: {"$ref": "#/components/responses/NotFound"}, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -971,8 +979,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ListMemoryChangesRequest, request_location="body", path_parameters=(), - response_type=ListMemoryChangesResponse, - success_status=200, + success_response_types={200: ListMemoryChangesResponse}, summary="List Memory Revision changes", tags=("memory",), scope_mode="current", @@ -983,6 +990,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 404: {"$ref": "#/components/responses/NotFound"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -996,8 +1004,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ProposeExperienceRequest, request_location="body", path_parameters=(), - response_type=ArtifactCandidate, - success_status=201, + success_response_types={201: ArtifactCandidate}, summary="Propose Experience content", tags=("experience",), scope_mode="current", @@ -1008,6 +1015,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1021,8 +1029,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=GenerateExperienceRequest, request_location="body", path_parameters=(), - response_type=GeneratedCandidateResponse, - success_status=200, + success_response_types={200: GeneratedCandidateResponse}, summary="Generate an Experience Candidate", tags=("experience",), scope_mode="current", @@ -1033,6 +1040,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1046,8 +1054,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=GetExperienceRequest, request_location="body", path_parameters=(), - response_type=ExperienceArtifact, - success_status=200, + success_response_types={200: ExperienceArtifact}, summary="Get an exact Experience Revision", tags=("experience",), scope_mode="current", @@ -1058,6 +1065,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 404: {"$ref": "#/components/responses/NotFound"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1071,8 +1079,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ProposeSkillRequest, request_location="body", path_parameters=(), - response_type=ArtifactCandidate, - success_status=201, + success_response_types={201: ArtifactCandidate}, summary="Propose managed Skill content", tags=("skill",), scope_mode="current", @@ -1083,6 +1090,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1096,8 +1104,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=GenerateSkillRequest, request_location="body", path_parameters=(), - response_type=GeneratedCandidateResponse, - success_status=200, + success_response_types={200: GeneratedCandidateResponse}, summary="Generate a managed Skill Candidate", tags=("skill",), scope_mode="current", @@ -1108,6 +1115,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1121,8 +1129,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=GetSkillRequest, request_location="body", path_parameters=(), - response_type=SkillArtifact, - success_status=200, + success_response_types={200: SkillArtifact}, summary="Get an exact managed Skill Revision", tags=("skill",), scope_mode="current", @@ -1133,6 +1140,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 404: {"$ref": "#/components/responses/NotFound"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1146,8 +1154,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ListManagedSkillsRequest, request_location="body", path_parameters=(), - response_type=ListManagedSkillsResponse, - success_status=200, + success_response_types={200: ListManagedSkillsResponse}, summary="List or search current managed Skills", tags=("skill",), scope_mode="current", @@ -1170,8 +1177,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=UpdateSkillLifecycleRequest, request_location="body", path_parameters=(), - response_type=SkillGovernance, - success_status=200, + success_response_types={200: SkillGovernance}, summary="Update managed Skill lifecycle", tags=("skill",), scope_mode="current", @@ -1196,8 +1202,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=GetSkillPackageRequest, request_location="body", path_parameters=(), - response_type=SkillPackageManifest, - success_status=200, + success_response_types={200: SkillPackageManifest}, summary="Get an exact managed Skill package manifest", tags=("skill",), scope_mode="current", @@ -1218,8 +1223,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=GetSkillPackageRequest, request_location="body", path_parameters=(), - response_type=SkillPackageDownload, - success_status=200, + success_response_types={200: SkillPackageDownload}, summary="Download an exact managed Skill package", tags=("skill",), scope_mode="current", @@ -1240,8 +1244,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ProposeSkillPackageRequest, request_location="body", path_parameters=(), - response_type=ArtifactCandidate, - success_status=201, + success_response_types={201: ArtifactCandidate}, summary="Propose an uploaded standard Skill package", tags=("skill",), scope_mode="current", @@ -1262,8 +1265,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=RecordSkillUsageRequest, request_location="body", path_parameters=(), - response_type=CaptureContentSourceResponse, - success_status=201, + success_response_types={201: CaptureContentSourceResponse}, summary="Record a bounded Skill usage observation", tags=("skill",), scope_mode="current", @@ -1285,8 +1287,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ListRemoteSkillTargetsRequest, request_location="body", path_parameters=(), - response_type=ListRemoteSkillTargetsResponse, - success_status=200, + success_response_types={200: ListRemoteSkillTargetsResponse}, summary="List remote Agent Skill target status", tags=("skill",), scope_mode="current", @@ -1306,8 +1307,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=CreateRemoteSkillTargetRequest, request_location="body", path_parameters=(), - response_type=RemoteSkillTargetEnrollment, - success_status=201, + success_response_types={201: RemoteSkillTargetEnrollment}, summary="Create a remote Agent Skill target enrollment", tags=("skill",), scope_mode="current", @@ -1328,8 +1328,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=EnrollRemoteSkillTargetRequest, request_location="body", path_parameters=(), - response_type=RemoteSkillTargetCredential, - success_status=200, + success_response_types={200: RemoteSkillTargetCredential}, summary="Enroll a remote Agent Skill Receiver", tags=("skill",), scope_mode="none", @@ -1348,8 +1347,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=RenameRemoteSkillTargetRequest, request_location="body", path_parameters=(), - response_type=RemoteSkillTarget, - success_status=200, + success_response_types={200: RemoteSkillTarget}, summary="Rename a remote Agent Skill target", tags=("skill",), scope_mode="current", @@ -1370,8 +1368,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=RevokeRemoteSkillTargetRequest, request_location="body", path_parameters=(), - response_type=RemoteSkillTarget, - success_status=200, + success_response_types={200: RemoteSkillTarget}, summary="Revoke a remote Agent Skill target", tags=("skill",), scope_mode="current", @@ -1392,8 +1389,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=PublishRemoteSkillRequest, request_location="body", path_parameters=(), - response_type=RemoteSkillPublication, - success_status=200, + success_response_types={200: RemoteSkillPublication}, summary="Set a remote target Skill desired Revision", tags=("skill",), scope_mode="current", @@ -1414,8 +1410,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=UnpublishRemoteSkillRequest, request_location="body", path_parameters=(), - response_type=RemoteSkillPublication, - success_status=200, + success_response_types={200: RemoteSkillPublication}, summary="Set remote target Skill desired absence", tags=("skill",), scope_mode="current", @@ -1436,8 +1431,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ReconcileRemoteSkillsRequest, request_location="body", path_parameters=(), - response_type=ReconcileRemoteSkillsResponse, - success_status=200, + success_response_types={200: ReconcileRemoteSkillsResponse}, summary="Reconcile a remote Agent Skill target", tags=("skill",), scope_mode="none", @@ -1457,8 +1451,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=DownloadRemoteSkillPackageRequest, request_location="body", path_parameters=(), - response_type=SkillPackageDownload, - success_status=200, + success_response_types={200: SkillPackageDownload}, summary="Download the exact package desired by a remote target", tags=("skill",), scope_mode="none", @@ -1479,8 +1472,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=RecordRemoteSkillReceiptRequest, request_location="body", path_parameters=(), - response_type=RemoteSkillReceiptResponse, - success_status=200, + success_response_types={200: RemoteSkillReceiptResponse}, summary="Record an exact remote Skill delivery Receipt", tags=("skill",), scope_mode="none", @@ -1501,8 +1493,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ScanExternalSkillsRequest, request_location="body", path_parameters=(), - response_type=ScanExternalSkillsResponse, - success_status=200, + success_response_types={200: ScanExternalSkillsResponse}, summary="Scan configured external Skill roots", tags=("skill",), scope_mode="current", @@ -1512,6 +1503,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, }, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1525,8 +1517,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ListExternalSkillsRequest, request_location="body", path_parameters=(), - response_type=ListExternalSkillsResponse, - success_status=200, + success_response_types={200: ListExternalSkillsResponse}, summary="List external Skills visible on this host", tags=("skill",), scope_mode="current", @@ -1536,6 +1527,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, }, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1549,8 +1541,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ResolveExternalSkillRequest, request_location="body", path_parameters=(), - response_type=ExternalSkillResolution, - success_status=200, + success_response_types={200: ExternalSkillResolution}, summary="Resolve an exact external Skill fingerprint", tags=("skill",), scope_mode="current", @@ -1561,6 +1552,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 404: {"$ref": "#/components/responses/NotFound"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1574,8 +1566,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ImportExternalSkillRequest, request_location="body", path_parameters=(), - response_type=GeneratedCandidateResponse, - success_status=200, + success_response_types={200: GeneratedCandidateResponse}, summary="Import or fork an external Skill into Review", tags=("skill",), scope_mode="current", @@ -1587,6 +1578,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): 404: {"$ref": "#/components/responses/NotFound"}, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1600,8 +1592,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ListArtifactCandidatesRequest, request_location="body", path_parameters=(), - response_type=ArtifactCandidatePage, - success_status=200, + success_response_types={200: ArtifactCandidatePage}, summary="List Artifact Candidates", tags=("review",), scope_mode="current", @@ -1611,6 +1602,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, }, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1624,8 +1616,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=GetArtifactCandidateRequest, request_location="body", path_parameters=(), - response_type=ArtifactCandidate, - success_status=200, + success_response_types={200: ArtifactCandidate}, summary="Get an Artifact Candidate", tags=("review",), scope_mode="current", @@ -1636,6 +1627,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 404: {"$ref": "#/components/responses/NotFound"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1649,8 +1641,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ApproveArtifactCandidateRequest, request_location="body", path_parameters=(), - response_type=ArtifactCandidate, - success_status=200, + success_response_types={200: ArtifactCandidate}, summary="Approve an Artifact Candidate", tags=("review",), scope_mode="current", @@ -1662,6 +1653,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): 404: {"$ref": "#/components/responses/NotFound"}, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1675,8 +1667,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=RejectArtifactCandidateRequest, request_location="body", path_parameters=(), - response_type=ArtifactCandidate, - success_status=200, + success_response_types={200: ArtifactCandidate}, summary="Reject an Artifact Candidate", tags=("review",), scope_mode="current", @@ -1688,6 +1679,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): 404: {"$ref": "#/components/responses/NotFound"}, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1701,8 +1693,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=ReviseArtifactCandidateRequest, request_location="body", path_parameters=(), - response_type=ArtifactCandidate, - success_status=200, + success_response_types={200: ArtifactCandidate}, summary="Revise an Artifact Candidate", tags=("review",), scope_mode="current", @@ -1714,6 +1705,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): 404: {"$ref": "#/components/responses/NotFound"}, 409: {"$ref": "#/components/responses/Conflict"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, @@ -1727,8 +1719,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=GetStatsRequest, request_location="body", path_parameters=(), - response_type=ScopedStats, - success_status=200, + success_response_types={200: ScopedStats}, summary="Aggregate product statistics over a Scope selection", tags=("stats",), scope_mode="selection", @@ -1744,12 +1735,107 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, }, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 503: {"$ref": "#/components/responses/Unavailable"}, 500: {"$ref": "#/components/responses/InternalError"}, }, ) +LIST_OPERATIONS = Operation[ListOperationsRequest, OperationPage]( + method="GET", + path="/v1/operations", + operation_id="list_operations", + request_type=ListOperationsRequest, + request_location="query", + path_parameters=(), + success_response_types={200: OperationPage}, + summary="List durable operations visible to the caller", + tags=("operations",), + scope_mode="none", + responses={ + 200: { + "description": "A bounded cursor page of authorized operations.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + }, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, +) + +GET_OPERATION = Operation[None, OperationRecord]( + method="GET", + path="/v1/operations/{operation_id}", + operation_id="get_operation", + request_type=None, + request_location=None, + path_parameters=("operation_id",), + success_response_types={200: OperationRecord}, + summary="Get one durable operation", + tags=("operations",), + scope_mode="none", + responses={ + 200: { + "description": "The authorized operation and its safe result or error metadata.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + }, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, +) + +CANCEL_OPERATION = Operation[OperationMutationRequest, OperationRecord]( + method="POST", + path="/v1/operations/{operation_id}/cancel", + operation_id="cancel_operation", + request_type=OperationMutationRequest, + request_location="body", + path_parameters=("operation_id",), + success_response_types={200: OperationRecord}, + summary="Cancel one durable operation using optimistic concurrency", + tags=("operations",), + scope_mode="none", + responses={ + 200: { + "description": "The updated operation.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + }, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + }, +) + +RETRY_OPERATION = Operation[OperationMutationRequest, OperationRecord]( + method="POST", + path="/v1/operations/{operation_id}/retry", + operation_id="retry_operation", + request_type=OperationMutationRequest, + request_location="body", + path_parameters=("operation_id",), + success_response_types={200: OperationRecord}, + summary="Recover one blocked failed operation using optimistic concurrency", + tags=("operations",), + scope_mode="none", + responses={ + 200: { + "description": "The recovered queued operation.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + }, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 409: {"$ref": "#/components/responses/Conflict"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + }, +) + GET_HANDOFF_REPORT = Operation[GetHandoffReportRequest, HandoffReportResponse]( method="POST", path="/v1/handoff-reports/get", @@ -1757,8 +1843,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): request_type=GetHandoffReportRequest, request_location="body", path_parameters=(), - response_type=HandoffReportResponse, - success_status=200, + success_response_types={200: HandoffReportResponse}, summary="Generate a Handoff Report", tags=("handoff-reports",), scope_mode="selection", @@ -1787,6 +1872,7 @@ class Operation(BaseModel, Generic[RequestT, ResponseT]): }, 404: {"$ref": "#/components/responses/NotFound"}, 401: {"$ref": "#/components/responses/Unauthorized"}, + 429: {"$ref": "#/components/responses/RateLimited"}, 422: {"$ref": "#/components/responses/InvalidRequest"}, 413: {"$ref": "#/components/responses/ReportTooLarge"}, 503: {"$ref": "#/components/responses/Unavailable"}, diff --git a/src/powercontext/http/_generated/schema.py b/src/powercontext/http/_generated/schema.py index c6197870f..724028ae2 100644 --- a/src/powercontext/http/_generated/schema.py +++ b/src/powercontext/http/_generated/schema.py @@ -57,6 +57,7 @@ "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Capabilities"}}}, }, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, }, } }, @@ -320,6 +321,7 @@ }, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -450,6 +452,7 @@ "content": {"application/json": {"schema": {"$ref": "#/components/schemas/PreparedContext"}}}, }, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -478,6 +481,7 @@ "404": {"$ref": "#/components/responses/NotFound"}, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -512,6 +516,7 @@ "404": {"$ref": "#/components/responses/NotFound"}, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -547,6 +552,7 @@ "404": {"$ref": "#/components/responses/NotFound"}, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -587,6 +593,7 @@ "404": {"$ref": "#/components/responses/NotFound"}, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -622,6 +629,7 @@ }, "404": {"$ref": "#/components/responses/NotFound"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -646,6 +654,7 @@ }, "404": {"$ref": "#/components/responses/NotFound"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -672,6 +681,7 @@ }, "404": {"$ref": "#/components/responses/NotFound"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -697,6 +707,7 @@ "404": {"$ref": "#/components/responses/NotFound"}, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -723,6 +734,7 @@ }, "404": {"$ref": "#/components/responses/NotFound"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -736,9 +748,22 @@ "summary": "Process the pending Source window into Memory", "description": "Run one bounded Source-to-Memory activation for operational control and testing.", "operationId": "flush_memory", + "x-powercontext-scope-mode": "current", + "parameters": [ + { + "name": "Prefer", + "in": "header", + "required": False, + "description": "Use `respond-async` for " + "an immediate handle or " + "`wait=N` to wait at most " + "30 seconds.", + "schema": {"type": "string"}, + } + ], "requestBody": { - "content": {"application/json": {"schema": {"$ref": "#/components/schemas/FlushMemoryRequest"}}}, "required": True, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/FlushMemoryRequest"}}}, }, "responses": { "200": { @@ -748,12 +773,28 @@ "application/json": {"schema": {"$ref": "#/components/schemas/FlushMemoryResponse"}} }, }, + "202": { + "description": "The durable operation is still queued, running, or waiting to retry.", + "headers": { + "X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}, + "Location": { + "description": "Relative URL of the accepted operation.", + "schema": {"type": "string"}, + }, + "Retry-After": { + "description": "Suggested polling delay in seconds.", + "schema": {"type": "integer"}, + }, + }, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/OperationAccepted"}}}, + }, + "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, }, - "x-powercontext-scope-mode": "current", } }, "/v1/memory/remember": { @@ -778,6 +819,7 @@ }, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -805,6 +847,7 @@ }, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -837,6 +880,7 @@ }, "404": {"$ref": "#/components/responses/NotFound"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -862,6 +906,7 @@ }, "404": {"$ref": "#/components/responses/NotFound"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -892,6 +937,7 @@ "404": {"$ref": "#/components/responses/NotFound"}, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -924,6 +970,7 @@ "404": {"$ref": "#/components/responses/NotFound"}, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -953,6 +1000,7 @@ }, "404": {"$ref": "#/components/responses/NotFound"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -980,6 +1028,7 @@ }, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1012,6 +1061,7 @@ }, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1039,6 +1089,7 @@ }, "404": {"$ref": "#/components/responses/NotFound"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1064,6 +1115,7 @@ }, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1093,6 +1145,7 @@ }, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1118,6 +1171,7 @@ }, "404": {"$ref": "#/components/responses/NotFound"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1642,6 +1696,7 @@ }, }, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1681,6 +1736,7 @@ }, }, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1713,6 +1769,7 @@ }, "404": {"$ref": "#/components/responses/NotFound"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1746,6 +1803,7 @@ "404": {"$ref": "#/components/responses/NotFound"}, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1774,6 +1832,7 @@ }, }, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1801,6 +1860,7 @@ }, "404": {"$ref": "#/components/responses/NotFound"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1829,6 +1889,7 @@ "404": {"$ref": "#/components/responses/NotFound"}, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1860,6 +1921,7 @@ "404": {"$ref": "#/components/responses/NotFound"}, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1888,6 +1950,7 @@ "404": {"$ref": "#/components/responses/NotFound"}, "409": {"$ref": "#/components/responses/Conflict"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1920,6 +1983,7 @@ "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ScopedStats"}}}, }, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "503": {"$ref": "#/components/responses/Unavailable"}, "500": {"$ref": "#/components/responses/InternalError"}, @@ -1927,6 +1991,127 @@ "x-powercontext-scope-mode": "selection", } }, + "/v1/operations": { + "get": { + "tags": ["operations"], + "summary": "List durable operations visible to the caller", + "operationId": "list_operations", + "parameters": [ + {"name": "scope_id", "in": "query", "schema": {"type": "string", "minLength": 1, "maxLength": 256}}, + {"name": "kind", "in": "query", "schema": {"$ref": "#/components/schemas/OperationKind"}}, + {"name": "status", "in": "query", "schema": {"$ref": "#/components/schemas/OperationStatus"}}, + {"name": "cursor", "in": "query", "schema": {"type": "string", "nullable": True}}, + { + "name": "limit", + "in": "query", + "schema": {"type": "integer", "minimum": 1, "maximum": 100, "default": 50}, + }, + ], + "responses": { + "200": { + "description": "A bounded cursor page of authorized operations.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/OperationPage"}}}, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/operations/{operation_id}": { + "get": { + "tags": ["operations"], + "summary": "Get one durable operation", + "operationId": "get_operation", + "parameters": [ + { + "name": "operation_id", + "in": "path", + "required": True, + "schema": {"type": "string", "format": "uuid"}, + } + ], + "responses": { + "200": { + "description": "The authorized operation and its safe result or error metadata.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/OperationRecord"}}}, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + } + }, + "/v1/operations/{operation_id}/cancel": { + "post": { + "tags": ["operations"], + "summary": "Cancel one durable operation using optimistic concurrency", + "operationId": "cancel_operation", + "parameters": [ + { + "name": "operation_id", + "in": "path", + "required": True, + "schema": {"type": "string", "format": "uuid"}, + } + ], + "requestBody": { + "required": True, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/OperationMutationRequest"}} + }, + }, + "responses": { + "200": { + "description": "The updated operation.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/OperationRecord"}}}, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + }, + } + }, + "/v1/operations/{operation_id}/retry": { + "post": { + "tags": ["operations"], + "summary": "Recover one blocked failed operation using optimistic concurrency", + "operationId": "retry_operation", + "parameters": [ + { + "name": "operation_id", + "in": "path", + "required": True, + "schema": {"type": "string", "format": "uuid"}, + } + ], + "requestBody": { + "required": True, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/OperationMutationRequest"}} + }, + }, + "responses": { + "200": { + "description": "The recovered queued operation.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/OperationRecord"}}}, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "409": {"$ref": "#/components/responses/Conflict"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + }, + } + }, "/v1/handoff-reports/get": { "post": { "tags": ["handoff-reports"], @@ -1967,6 +2152,7 @@ }, "404": {"$ref": "#/components/responses/NotFound"}, "401": {"$ref": "#/components/responses/Unauthorized"}, + "429": {"$ref": "#/components/responses/RateLimited"}, "422": {"$ref": "#/components/responses/InvalidRequest"}, "413": {"$ref": "#/components/responses/ReportTooLarge"}, "503": {"$ref": "#/components/responses/Unavailable"}, @@ -4674,6 +4860,127 @@ "type": "object", "required": ["hits"], }, + "OperationStatus": { + "type": "string", + "enum": ["queued", "running", "retry_wait", "cancelling", "succeeded", "failed", "cancelled"], + }, + "OperationKind": {"type": "string", "enum": ["memory_flush", "experience_incubation"]}, + "MemoryOperationResult": { + "properties": { + "type": {"type": "string", "enum": ["memory_flush"]}, + "previous_cursor": {"type": "integer", "minimum": 0.0}, + "high_watermark": {"type": "integer", "minimum": 0.0}, + "current_cursor": {"type": "integer", "minimum": 0.0}, + "processed_source_count": {"type": "integer", "minimum": 0.0}, + "memory": {"$ref": "#/components/schemas/ArtifactReference", "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": ["type", "previous_cursor", "high_watermark", "current_cursor", "processed_source_count"], + }, + "ExperienceOperationResult": { + "properties": { + "type": {"type": "string", "enum": ["experience_incubation"]}, + "previous_cursor": {"type": "integer", "minimum": 0.0}, + "high_watermark": {"type": "integer", "minimum": 0.0}, + "current_cursor": {"type": "integer", "minimum": 0.0}, + "processed_source_count": {"type": "integer", "minimum": 0.0}, + "candidate_count": {"type": "integer", "minimum": 0.0}, + }, + "additionalProperties": False, + "type": "object", + "required": [ + "type", + "previous_cursor", + "high_watermark", + "current_cursor", + "processed_source_count", + "candidate_count", + ], + }, + "OperationError": { + "properties": { + "category": {"type": "string", "maxLength": 64, "minLength": 1}, + "code": {"type": "string", "maxLength": 128, "minLength": 1}, + }, + "additionalProperties": False, + "type": "object", + "required": ["category", "code"], + }, + "OperationRecord": { + "properties": { + "operation_id": {"type": "string", "format": "uuid"}, + "kind": {"$ref": "#/components/schemas/OperationKind"}, + "scope_id": {"type": "string", "maxLength": 256, "minLength": 1}, + "status": {"$ref": "#/components/schemas/OperationStatus"}, + "attempt_count": {"type": "integer", "minimum": 0.0}, + "state_version": {"type": "integer", "minimum": 1.0}, + "created_at": {"type": "string", "format": "date-time"}, + "updated_at": {"type": "string", "format": "date-time"}, + "completed_at": {"type": "string", "format": "date-time", "nullable": True}, + "result": { + "oneOf": [ + {"$ref": "#/components/schemas/MemoryOperationResult"}, + {"$ref": "#/components/schemas/ExperienceOperationResult"}, + ], + "discriminator": {"propertyName": "type"}, + "nullable": True, + }, + "error": {"$ref": "#/components/schemas/OperationError", "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": [ + "operation_id", + "kind", + "scope_id", + "status", + "attempt_count", + "state_version", + "created_at", + "updated_at", + ], + }, + "OperationAccepted": { + "properties": { + "operation_id": {"type": "string", "format": "uuid"}, + "status": {"$ref": "#/components/schemas/OperationStatus"}, + "status_url": {"type": "string", "pattern": "^/v1/operations/[0-9a-f-]{36}$"}, + }, + "additionalProperties": False, + "type": "object", + "required": ["operation_id", "status", "status_url"], + }, + "OperationPage": { + "properties": { + "items": { + "items": {"$ref": "#/components/schemas/OperationRecord"}, + "type": "array", + "maxItems": 100, + }, + "next_cursor": {"type": "string", "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": ["items"], + }, + "ListOperationsRequest": { + "properties": { + "scope_id": {"type": "string", "maxLength": 256, "minLength": 1}, + "kind": {"$ref": "#/components/schemas/OperationKind"}, + "status": {"$ref": "#/components/schemas/OperationStatus"}, + "cursor": {"type": "string", "nullable": True}, + "limit": {"type": "integer", "maximum": 100.0, "minimum": 1.0, "default": 50}, + }, + "additionalProperties": False, + "type": "object", + }, + "OperationMutationRequest": { + "properties": {"expected_version": {"type": "integer", "minimum": 1.0}}, + "additionalProperties": False, + "type": "object", + "required": ["expected_version"], + }, "SourceReference": { "properties": { "name": {"type": "string", "description": "Stable Source type."}, @@ -4740,6 +5047,17 @@ "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}, }, + "RateLimited": { + "description": "The shared request policy rejected this fixed-window request.", + "headers": { + "Retry-After": { + "description": "Seconds until the current shared window expires.", + "schema": {"type": "integer"}, + }, + "X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}, + }, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}, + }, "InternalError": { "description": "The Server failed without exposing internal details.", "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, diff --git a/src/powercontext/server/app.py b/src/powercontext/server/app.py index d5810e796..eaa650003 100644 --- a/src/powercontext/server/app.py +++ b/src/powercontext/server/app.py @@ -29,8 +29,9 @@ from functools import wraps from time import perf_counter from typing import TYPE_CHECKING, Annotated, Any, Protocol, TypeVar, cast +from uuid import UUID -from fastapi import Depends, FastAPI, Request, Response, status +from fastapi import Depends, FastAPI, Query, Request, Response, status from fastapi import Path as PathParameter from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse @@ -122,6 +123,7 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.skill_publications import SkillPublication +from powercontext.builtin.persistence.work import WorkStateConflictError, WorkStatus from powercontext.builtin.publication import ( ArtifactPublicationApplication, ArtifactPublicationConflictError, @@ -235,6 +237,7 @@ from powercontext.builtin.runtime import ( SubmitSourceObservation as RuntimeSubmitSourceObservation, ) +from powercontext.builtin.runtime.operations import OperationManager from powercontext.builtin.scope import ( ScopeApplication, ScopeBindingNotFoundError, @@ -342,10 +345,17 @@ ListMemoryChangesResponse, ListMemoryEntriesRequest, ListMemoryEntriesResponse, + ListOperationsRequest, ListRemoteSkillTargetsRequest, ListRemoteSkillTargetsResponse, MemoryEntry, MemoryMutationResponse, + OperationAccepted, + OperationKind, + OperationMutationRequest, + OperationPage, + OperationRecord, + OperationStatus, PrepareContextRequest, PreparedContext, PreparedWorkHandoff, @@ -427,6 +437,7 @@ API_TITLE, API_VERSION, APPROVE_ARTIFACT_CANDIDATE, + CANCEL_OPERATION, CAPTURE_CONTENT_SOURCE, CLEAR_SCOPE_BINDING, COMMIT_CONNECTOR_CHECKPOINT, @@ -450,6 +461,7 @@ GET_HANDOFF_REPORT, GET_LIVENESS, GET_MEMORY_ENTRY, + GET_OPERATION, GET_READINESS, GET_SCOPE, GET_SKILL, @@ -462,6 +474,7 @@ LIST_MANAGED_SKILLS, LIST_MEMORY_CHANGES, LIST_MEMORY_ENTRIES, + LIST_OPERATIONS, LIST_REMOTE_SKILL_TARGETS, LIST_SCOPES, OPENAPI_VERSION, @@ -484,6 +497,7 @@ RESOLVE_SCOPE_BINDING, RESOLVE_SCOPE_SELECTION, RETIRE_MEMORY_ENTRY, + RETRY_OPERATION, REVISE_ARTIFACT_CANDIDATE, REVISE_MEMORY_ENTRY, REVOKE_REMOTE_SKILL_TARGET, @@ -835,6 +849,7 @@ def create_app( metrics: ServerMetrics | None = None, tracing: ServerTracing | None = None, handoff_report_enabled: bool = False, + public_routes: bool = True, allow_insecure_remote_http: bool = False, ) -> FastAPI: """Build the HTTP adapter around an optional Runtime application binding.""" @@ -850,6 +865,7 @@ def create_app( ) app.openapi_version = OPENAPI_VERSION app.state.application = application + app.state.operation_manager = None app.state.capability_provider = capability_provider app.state.readiness_probe = readiness_probe app.state.metrics = metrics @@ -938,6 +954,10 @@ async def unexpected_error(request: Request, error: Exception) -> JSONResponse: _add_route(app, SUBMIT_SOURCE_OBSERVATION, submit_source_observation) _add_route(app, COMMIT_CONNECTOR_CHECKPOINT, commit_connector_checkpoint) _add_route(app, FLUSH_MEMORY, flush_memory) + _add_route(app, GET_OPERATION, get_operation) + _add_route(app, LIST_OPERATIONS, list_operations) + _add_route(app, CANCEL_OPERATION, cancel_operation) + _add_route(app, RETRY_OPERATION, retry_operation) _add_route(app, REMEMBER_MEMORY, remember_memory) _add_route(app, SEARCH_MEMORY, search_memory) _add_route(app, PREPARE_CONTEXT, prepare_context) @@ -992,21 +1012,45 @@ async def unexpected_error(request: Request, error: Exception) -> JSONResponse: include_in_schema=False, methods=["GET"], ) + _restrict_to_management_routes(app, public_routes=public_routes) def canonical_openapi() -> dict[str, Any]: - if app.openapi_schema is None: - app.openapi_schema = deepcopy(OPENAPI_SCHEMA) - if not handoff_report_enabled: - paths = cast(dict[str, Any], app.openapi_schema["paths"]) - app.openapi_schema["paths"] = { - path: value for path, value in paths.items() if not path.startswith("/v1/handoff-reports/") - } - return app.openapi_schema + return _canonical_openapi_schema( + app, + public_routes=public_routes, + handoff_report_enabled=handoff_report_enabled, + ) app.openapi = canonical_openapi # ty: ignore[invalid-assignment] return app +def _restrict_to_management_routes(app: FastAPI, *, public_routes: bool) -> None: + if public_routes: + return + management_paths = {GET_LIVENESS.path, GET_READINESS.path} + app.router.routes[:] = [route for route in app.router.routes if getattr(route, "path", None) in management_paths] + + +def _canonical_openapi_schema( + app: FastAPI, + *, + public_routes: bool, + handoff_report_enabled: bool, +) -> dict[str, Any]: + if app.openapi_schema is not None: + return app.openapi_schema + app.openapi_schema = deepcopy(OPENAPI_SCHEMA) + paths = cast(dict[str, Any], app.openapi_schema["paths"]) + if not public_routes: + management_paths = {GET_LIVENESS.path, GET_READINESS.path} + paths = {path: value for path, value in paths.items() if path in management_paths} + if not handoff_report_enabled: + paths = {path: value for path, value in paths.items() if not path.startswith("/v1/handoff-reports/")} + app.openapi_schema["paths"] = paths + return app.openapi_schema + + async def scalar_api_reference(request: Request) -> Response: """Render the runtime OpenAPI contract with Scalar.""" @@ -1281,11 +1325,119 @@ async def commit_connector_checkpoint( async def flush_memory( - request: FlushMemoryRequest, + payload: FlushMemoryRequest, + request: Request, application: Annotated[ServerApplication, Depends(_require_application)], -) -> FlushMemoryResponse: - result = await application.memory.for_scope(request.scope_id).flush() - return mapping.flush_response(result) +) -> FlushMemoryResponse | JSONResponse: + manager: OperationManager | None = request.app.state.operation_manager + if manager is None: + result = await application.memory.for_scope(payload.scope_id).flush() + return mapping.flush_response(result) + submission = await manager.submit_memory( + payload.scope_id, + limit=manager.memory_window_limit, + ) + if isinstance(submission, MemoryFlushResult): + return mapping.flush_response(submission) + operation = submission.work + if operation.status is WorkStatus.FAILED: + return _error_response( + status.HTTP_409_CONFLICT, + code="operation_blocked", + message="The logical window is blocked by a failed operation.", + details={"operation_id": operation.work_id}, + ) + completed = await manager.wait( + operation.work_id, + timeout_seconds=_preferred_wait(request, manager), + ) + if completed is not None and completed.status is WorkStatus.SUCCEEDED: + return mapping.flush_response(manager.memory_result(completed)) + if completed is not None and completed.status is WorkStatus.FAILED: + return _error_response( + status.HTTP_409_CONFLICT, + code="operation_failed", + message="The Memory operation failed and requires operator action.", + details={"operation_id": completed.work_id}, + ) + if completed is not None and completed.status is WorkStatus.CANCELLED: + return _error_response( + status.HTTP_409_CONFLICT, + code="operation_cancelled", + message="The Memory operation was cancelled.", + details={"operation_id": completed.work_id}, + ) + current = operation if completed is None else completed + accepted = OperationAccepted( + operation_id=UUID(current.work_id), + status=OperationStatus(current.status.value), + status_url=f"/v1/operations/{current.work_id}", + ) + return JSONResponse( + status_code=status.HTTP_202_ACCEPTED, + content=accepted.model_dump(mode="json"), + headers={"Location": accepted.status_url, "Retry-After": "2"}, + ) + + +async def get_operation( + operation_id: UUID, + manager: Annotated[OperationManager, Depends(_require_operation_manager)], +) -> OperationRecord: + return mapping.operation_response(await manager.get(str(operation_id))) + + +def _list_operations_query( + scope_id: Annotated[str | None, Query(min_length=1, max_length=256)] = None, + kind: Annotated[OperationKind | None, Query()] = None, + operation_status: Annotated[OperationStatus | None, Query(alias="status")] = None, + cursor: Annotated[str | None, Query()] = None, + limit: Annotated[int, Query(ge=1, le=100)] = 50, +) -> ListOperationsRequest: + """Coerce HTTP query strings before applying the strict generated contract model.""" + + return ListOperationsRequest( + scope_id=scope_id, + kind=kind, + status=operation_status, + cursor=cursor, + limit=limit, + ) + + +async def list_operations( + request: Annotated[ListOperationsRequest, Depends(_list_operations_query)], + manager: Annotated[OperationManager, Depends(_require_operation_manager)], +) -> OperationPage: + page = await manager.list( + scope_id=request.scope_id, + kind=None if request.kind is None else mapping.operation_kind_value(request.kind), + status=None if request.status is None else WorkStatus(request.status.value), + cursor=request.cursor, + limit=request.limit, + ) + return OperationPage( + items=[mapping.operation_response(item) for item in page.items], + next_cursor=page.next_cursor, + ) + + +async def cancel_operation( + operation_id: UUID, + request: OperationMutationRequest, + manager: Annotated[OperationManager, Depends(_require_operation_manager)], +) -> OperationRecord: + return mapping.operation_response( + await manager.cancel(str(operation_id), expected_version=request.expected_version) + ) + + +async def retry_operation( + operation_id: UUID, + request: OperationMutationRequest, + manager: Annotated[OperationManager, Depends(_require_operation_manager)], +) -> OperationRecord: + return mapping.operation_response(await manager.retry(str(operation_id), expected_version=request.expected_version)) async def remember_memory( @@ -1989,6 +2141,30 @@ def _require_application(request: Request) -> ServerApplication: return application +def _require_operation_manager(request: Request) -> OperationManager: + manager: OperationManager | None = request.app.state.operation_manager + if manager is None: + raise _RuntimeNotReadyError + return manager + + +def _preferred_wait(request: Request, manager: OperationManager) -> float: + preference = request.headers.get("Prefer", "") + values = {value.strip().lower() for value in preference.split(",") if value.strip()} + if "respond-async" in values: + return 0 + for value in values: + name, separator, raw_seconds = value.partition("=") + if name != "wait" or not separator: + continue + try: + seconds = max(0, int(raw_seconds)) + except ValueError: + continue + return min(seconds, manager.maximum_wait_seconds) + return manager.default_wait_seconds + + def _require_scope_application(request: Request) -> ScopeApplication: application = _require_application(request) if application.scopes is None: @@ -2208,12 +2384,10 @@ def _validation_error_details(error: RequestValidationError | PydanticValidation def _map_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | None]: if isinstance(error, _RuntimeNotReadyError): return status.HTTP_503_SERVICE_UNAVAILABLE, "runtime_not_ready", "The Runtime is not ready.", None - external_skill_error = _map_external_skill_error(error) - if external_skill_error is not None: - return external_skill_error - remote_skill_error = _map_remote_skill_error(error) - if remote_skill_error is not None: - return remote_skill_error + for mapper in (_map_operation_error, _map_external_skill_error, _map_remote_skill_error): + mapped = mapper(error) + if mapped is not None: + return mapped if isinstance(error, GenerationCapabilityUnavailableError): return ( status.HTTP_503_SERVICE_UNAVAILABLE, @@ -2221,21 +2395,16 @@ def _map_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | None]: "Artifact generation is not configured.", {"family": error.family}, ) - governance_error = _map_governance_error(error) - if governance_error is not None: - return governance_error - scope_error = _map_scope_error(error) - if scope_error is not None: - return scope_error - candidate_error = _map_candidate_error(error) - if candidate_error is not None: - return candidate_error - availability_error = _map_availability_error(error) - if availability_error is not None: - return availability_error - report_error = _map_report_error(error) - if report_error is not None: - return report_error + for mapper in ( + _map_governance_error, + _map_scope_error, + _map_candidate_error, + _map_availability_error, + _map_report_error, + ): + mapped = mapper(error) + if mapped is not None: + return mapped return _map_domain_error(error) @@ -2326,6 +2495,23 @@ def _map_scope_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | return None +def _map_operation_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | None] | None: + if isinstance(error, RepositoryNotFoundError) and error.kind in {"operation", "work"}: + return status.HTTP_404_NOT_FOUND, "operation_not_found", "The operation was not found.", None + if isinstance(error, WorkStateConflictError): + return ( + status.HTTP_409_CONFLICT, + "operation_conflict", + "The operation changed or does not allow this transition.", + { + "operation_id": error.work_id, + "status": error.status.value, + "actual_version": error.actual_version, + }, + ) + return None + + def _map_candidate_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | None] | None: if isinstance(error, CandidateNotFoundError): return status.HTTP_404_NOT_FOUND, "candidate_not_found", "The requested Candidate was not found.", None diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index 4c2be2481..a008ecb8f 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -16,12 +16,17 @@ from __future__ import annotations +import asyncio from pathlib import Path -from typing import Annotated, Any +from typing import Annotated, Any, cast import typer from pydantic import ValidationError +from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.persistence.migration import SchemaMigrationError +from powercontext.builtin.runtime.composition import migrate_builtin_database +from powercontext.builtin.runtime.config import InferenceConfig from powercontext.server.configuration import ServerConfigurationError, server_settings_context from powercontext.server.factory import create_server_app from powercontext.server.logging import configure_server_logging @@ -86,6 +91,48 @@ def run( raise typer.Exit(code=2) from error +@app.command() +def migrate( + env_file: Annotated[ + Path | None, + typer.Option(help="Load Server and database settings from this environment file."), + ] = None, +) -> None: + """Upgrade the configured database through the forward-only schema chain.""" + + try: + with server_settings_context(env_file=env_file) as settings: + revision = asyncio.run(_migrate_configured_database(settings)) + except ServerConfigurationError as error: + if isinstance(error.cause, ValidationError): + raise _friendly_bad_parameter(error.cause) from error + typer.echo(f"Invalid Server configuration: {error}", err=True) + raise typer.Exit(code=2) from error + except SchemaMigrationError as error: + typer.echo(f"Schema migration failed: {error}", err=True) + raise typer.Exit(code=1) from error + typer.echo(f"PowerContext database schema is at {revision}.") + + +async def _migrate_configured_database(settings: ServerSettings) -> str: + return await migrate_builtin_database( + settings.to_builtin_config(), + embedding_profile=_configured_embedding_profile(settings.inference), + ) + + +def _configured_embedding_profile(settings: InferenceConfig) -> EmbeddingProfile | None: + if settings.embedding_model is None: + return None + return EmbeddingProfile( + profile_id=cast(str, settings.embedding_profile_id), + model=settings.embedding_model, + dimension=cast(int, settings.embedding_dimension), + distance="l2", + normalization=settings.embedding_normalization, + ) + + def _run_configured_server(settings: ServerSettings) -> None: """Run one already-validated configuration in the current process.""" diff --git a/src/powercontext/server/factory.py b/src/powercontext/server/factory.py index 7470d6896..e0b645881 100644 --- a/src/powercontext/server/factory.py +++ b/src/powercontext/server/factory.py @@ -35,15 +35,14 @@ from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime import BuiltinRuntime from powercontext.builtin.runtime.composition import open_builtin_runtime -from powercontext.builtin.runtime.config import BuiltinConfig from powercontext.builtin.sources import CONTENT_SOURCE_NAME from powercontext.http import Capabilities, MemorySearchMode, PreparedContextSchema, ReadinessResponse, ReadinessStatus -from powercontext.paths import default_scheduler_path from powercontext.server.access import HttpAccessLogMiddleware from powercontext.server.app import create_app from powercontext.server.mcp import mount_mcp from powercontext.server.metrics import CONTENT_TYPE_LATEST, HttpMetricsMiddleware, ServerMetrics -from powercontext.server.middleware import StaticBearerMiddleware +from powercontext.server.middleware import LocalPrincipalMiddleware, StaticBearerMiddleware +from powercontext.server.rate_limit import SharedRateLimiter, SharedRateLimitMiddleware from powercontext.server.settings import ServerSettings from powercontext.server.tracing import HttpTracingMiddleware, ServerTracing from powercontext.server.web import mount_web_ui @@ -65,32 +64,30 @@ def create_server_app( middleware: Sequence[Middleware] = (), tracing: ServerTracing | None = None, ) -> FastAPI: - """Build the Server process and mount MCP when configured.""" + """Build the Server process and mount MCP when configured. + + ``scheduler_path`` remains an accepted bridge-release argument, but the + durable Scheduler stores all state in the primary database. + """ + + del scheduler_path resolved = ServerSettings() if settings is None else settings - config = BuiltinConfig( - runtime=resolved.runtime, - database=resolved.database, - handoff_report=resolved.handoff_report, - inference=resolved.inference, - external_skills=resolved.external_skills, - ) + config = resolved.to_builtin_config() metrics = ServerMetrics() if resolved.metrics.enabled else None + public_routes = config.deployment.role in {"all", "api"} resolved_tracing = ServerTracing.context_only() if tracing is None else tracing if metrics is not None: metrics.set_ready(False) readiness_probe = _ServerReadinessProbe(metrics, tracing=resolved_tracing) + rate_limiter = _shared_rate_limiter(resolved) @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: _log_lifecycle("server.starting", "PowerContext Server is starting") - if resolved.allow_insecure_http: - _log_insecure_remote_http_warning() - if isinstance(config.database, SQLiteConfig) and config.database.is_in_memory: - _log_in_memory_database_warning() + _log_startup_warnings(resolved) async with open_builtin_runtime( config, - scheduler_path=default_scheduler_path() if scheduler_path is None else scheduler_path, candidate_pipeline=candidate_pipeline, experience_pipeline=experience_pipeline, experience_generator=experience_generator, @@ -101,17 +98,31 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: instrumentation=resolved_tracing.instrumentation, scope_cache_observer=None if metrics is None else metrics.set_runtime_scopes, tracing=resolved_tracing, + work_observer=metrics, ) as runtime: readiness_probe.bind(runtime) app.state.application = runtime - app.state.capabilities = await _server_capabilities(runtime) + operations = runtime.operations + if operations is None: + raise RuntimeError("the built-in runtime did not expose operation coordination") # noqa: TRY003 + app.state.operation_manager = operations + if rate_limiter is not None: + rate_limiter.bind(operations.database) + app.state.capabilities = await _server_capabilities( + runtime, + accepts_distributed_memory_work=config.deployment.mode == "distributed" + and config.deployment.role == "api", + ) await readiness_probe() try: yield finally: _log_lifecycle("server.stopping", "PowerContext Server is stopping") readiness_probe.unbind() + if rate_limiter is not None: + rate_limiter.unbind() app.state.application = None + app.state.operation_manager = None app.state.capabilities = Capabilities( source_types=[], artifact_families=[], @@ -125,13 +136,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: ) _log_lifecycle("server.stopped", "PowerContext Server stopped") - configured_middleware = list(middleware) - auth_token = resolved.auth.token - if resolved.auth.enabled and auth_token is not None: - configured_middleware.insert( - 0, - Middleware(StaticBearerMiddleware, token=auth_token.get_secret_value()), - ) + configured_middleware = _process_middleware(middleware, resolved, rate_limiter) app = create_app( lifespan=lifespan, @@ -140,9 +145,10 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: metrics=metrics, tracing=resolved_tracing, handoff_report_enabled=resolved.handoff_report.enabled, + public_routes=public_routes, allow_insecure_remote_http=resolved.allow_insecure_http, ) - _mount_optional_web_ui(app, resolved) + _configure_web_ui(app, resolved, public_routes=public_routes) if metrics is not None: app.add_api_route( "/metrics", @@ -167,17 +173,58 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: operations=_http_operations(app), skip_paths=("/health/live", "/health/ready", "/metrics", resolved.mcp.path), ) - if resolved.mcp.enabled: + if public_routes and resolved.mcp.enabled: mount_mcp( app, path=resolved.mcp.path, access_log=resolved.logging.access, metrics=metrics, tracing=resolved_tracing, + stateless_http=config.deployment.mode == "distributed", ) return app +def _log_startup_warnings(settings: ServerSettings) -> None: + if settings.allow_insecure_http: + _log_insecure_remote_http_warning() + if isinstance(settings.database, SQLiteConfig) and settings.database.is_in_memory: + _log_in_memory_database_warning() + + +def _shared_rate_limiter(settings: ServerSettings) -> SharedRateLimiter | None: + if not settings.rate_limit.enabled: + return None + return SharedRateLimiter( + requests=settings.rate_limit.requests, + window_seconds=settings.rate_limit.window_seconds, + ) + + +def _process_middleware( + middleware: Sequence[Middleware], + settings: ServerSettings, + rate_limiter: SharedRateLimiter | None, +) -> list[Middleware]: + configured = list(middleware) + token = settings.auth.token + if settings.auth.enabled and token is not None: + configured.insert(0, Middleware(StaticBearerMiddleware, token=token.get_secret_value())) + else: + configured.insert(0, Middleware(LocalPrincipalMiddleware)) + if rate_limiter is not None: + configured.insert(1, Middleware(SharedRateLimitMiddleware, limiter=rate_limiter)) + return configured + + +def _configure_web_ui(app: FastAPI, settings: ServerSettings, *, public_routes: bool) -> None: + if public_routes: + _mount_optional_web_ui(app, settings) + return + app.state.dashboard_started = False + app.state.dashboard_startup_error = "the configured process role does not expose public routes" + + def _mount_optional_web_ui(app: FastAPI, settings: ServerSettings) -> None: app.state.dashboard_started = False app.state.dashboard_startup_error = None @@ -303,12 +350,16 @@ def _http_operations(app: FastAPI) -> dict[tuple[str, str], str]: } -async def _server_capabilities(runtime: BuiltinRuntime) -> Capabilities: +async def _server_capabilities( + runtime: BuiltinRuntime, + *, + accepts_distributed_memory_work: bool = False, +) -> Capabilities: capabilities = await runtime.capabilities() return Capabilities( source_types=[CONTENT_SOURCE_NAME], artifact_families=["memory", "experience", "skill", "handoff"], - memory_extraction=capabilities.memory_extraction, + memory_extraction=capabilities.memory_extraction or accepts_distributed_memory_work, experience_generation=capabilities.experience_generation, managed_skill_generation=capabilities.managed_skill_generation, external_skill_registry=capabilities.external_skill_registry, diff --git a/src/powercontext/server/mapping.py b/src/powercontext/server/mapping.py index 266d7b85f..61cb3bfb0 100644 --- a/src/powercontext/server/mapping.py +++ b/src/powercontext/server/mapping.py @@ -16,7 +16,9 @@ from __future__ import annotations +from datetime import UTC, datetime from typing import Any +from uuid import UUID from pydantic import ValidationError @@ -36,6 +38,7 @@ ExternalSkillResolution as RuntimeExternalSkillResolution, ) from powercontext.builtin.persistence.artifact_governance import ArtifactGovernance +from powercontext.builtin.persistence.work import StoredWork from powercontext.builtin.review import ArtifactCandidate as RuntimeArtifactCandidate from powercontext.builtin.review import ArtifactCandidatePage as RuntimeArtifactCandidatePage from powercontext.builtin.review import CandidateStatus as RuntimeCandidateStatus @@ -46,6 +49,7 @@ from powercontext.builtin.runtime import ( ActivateHandoff, CaptureSource, + ExperienceIncubationResult, Handoff, HandoffActivation, HandoffArtifactCitation, @@ -138,6 +142,7 @@ from powercontext.builtin.runtime import ( SubmitSourceObservation as RuntimeSubmitSourceObservation, ) +from powercontext.builtin.runtime.work_handlers import EXPERIENCE_WORK_KIND, MEMORY_WORK_KIND from powercontext.builtin.sources import ExternalSkillImportMode as RuntimeExternalSkillImportMode from powercontext.builtin.work import ( AcknowledgeHandoff as RuntimeAcknowledgeHandoff, @@ -181,6 +186,7 @@ EntryChange, EntryChangeOperation, ExperienceArtifact, + ExperienceOperationResult, ExperienceProposal, ExternalSkillRegistration, ExternalSkillResolution, @@ -216,8 +222,13 @@ MemoryEntryState, MemoryMatchedBy, MemoryMutationResponse, + MemoryOperationResult, MemoryRevisionChanges, MemoryUsedSearchMode, + OperationError, + OperationKind, + OperationRecord, + OperationStatus, PreparedContextSchema, PreparedContextStatus, PreparedHandoffSchema, @@ -523,6 +534,72 @@ def flush_response(value: MemoryFlushResult) -> FlushMemoryResponse: ) +def operation_response(value: StoredWork) -> OperationRecord: + """Project only bounded operation metadata and discriminated safe results.""" + + kind = _operation_kind(value.kind) + result = None + if value.result_payload is not None: + if kind is OperationKind.MEMORY_FLUSH: + memory = MemoryFlushResult.model_validate(value.result_payload) + result = MemoryOperationResult( + type="memory_flush", + previous_cursor=memory.previous_cursor, + high_watermark=memory.high_watermark, + current_cursor=memory.current_cursor, + processed_source_count=memory.source_count, + memory=None if memory.memory_ref is None else artifact_reference(memory.memory_ref), + ) + else: + experience = ExperienceIncubationResult.model_validate(value.result_payload) + result = ExperienceOperationResult( + type="experience_incubation", + previous_cursor=experience.previous_cursor, + high_watermark=experience.high_watermark, + current_cursor=experience.current_cursor, + processed_source_count=experience.source_count, + candidate_count=experience.candidate_count, + ) + error = ( + None + if value.error_category is None or value.error_code is None + else OperationError(category=value.error_category, code=value.error_code) + ) + return OperationRecord( + operation_id=UUID(value.work_id), + kind=kind, + scope_id=value.scope_id, + status=OperationStatus(value.status.value), + attempt_count=value.attempt_count, + state_version=value.state_version, + created_at=_aware_utc(value.created_at), + updated_at=_aware_utc(value.updated_at), + completed_at=None if value.completed_at is None else _aware_utc(value.completed_at), + result=result, + error=error, + ) + + +def _operation_kind(value: str) -> OperationKind: + if value == MEMORY_WORK_KIND: + return OperationKind.MEMORY_FLUSH + if value == EXPERIENCE_WORK_KIND: + return OperationKind.EXPERIENCE_INCUBATION + raise ValueError("unknown public operation kind") # noqa: TRY003 + + +def operation_kind_value(value: OperationKind) -> str: + """Map the public operation discriminator to its internal handler kind.""" + + if value is OperationKind.MEMORY_FLUSH: + return MEMORY_WORK_KIND + return EXPERIENCE_WORK_KIND + + +def _aware_utc(value: datetime) -> datetime: + return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) + + def remember_request(value: TransportRememberMemoryRequest) -> RememberMemoryRequest: return RememberMemoryRequest( entries=(MemoryEntryInput(kind=value.kind, text=value.text, reason=value.reason),), diff --git a/src/powercontext/server/mcp.py b/src/powercontext/server/mcp.py index f8cce4253..341ef7bda 100644 --- a/src/powercontext/server/mcp.py +++ b/src/powercontext/server/mcp.py @@ -223,6 +223,7 @@ def mount_mcp( access_log: bool = False, metrics: ServerMetrics | None = None, tracing: ServerTracing | None = None, + stateless_http: bool = False, ) -> FastAPI: """Mount the MCP transport while preserving the Server HTTP contract.""" @@ -232,7 +233,7 @@ def mount_mcp( metrics=metrics, tracing=tracing, ) - mcp_app = mcp_server.http_app(path="/") + mcp_app = mcp_server.http_app(path="/", stateless_http=stateless_http) server_app.router.lifespan_context = combine_lifespans( server_app.router.lifespan_context, diff --git a/src/powercontext/server/metrics.py b/src/powercontext/server/metrics.py index 19bd533e0..4eb93e305 100644 --- a/src/powercontext/server/metrics.py +++ b/src/powercontext/server/metrics.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +from collections.abc import Mapping, Sequence from contextlib import suppress from time import perf_counter from typing import Any @@ -36,6 +37,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send from typing_extensions import override +from powercontext.builtin.persistence.work import WorkQueueStatistic from powercontext.server.context import is_internal_bridge @@ -88,6 +90,61 @@ def __init__(self) -> None: ("state",), registry=self.registry, ) + self.work_enqueues = Counter( + "powercontext_work_enqueues_total", + "Durable logical work enqueue decisions.", + ("kind", "outcome"), + registry=self.registry, + ) + self.work_claim_latency = Histogram( + "powercontext_work_claim_latency_seconds", + "Time from durable work creation to a Worker claim.", + ("kind",), + registry=self.registry, + ) + self.work_attempts = Counter( + "powercontext_work_attempts_total", + "Completed durable work attempts.", + ("kind", "outcome", "error_category"), + registry=self.registry, + ) + self.work_attempt_duration = Histogram( + "powercontext_work_attempt_duration_seconds", + "Durable work attempt execution duration.", + ("kind", "outcome"), + registry=self.registry, + ) + self.work_lease_expirations = Counter( + "powercontext_work_lease_expirations_total", + "Worker leases recovered after database-time expiry.", + ("kind", "outcome"), + registry=self.registry, + ) + self.scheduler_leadership_changes = Counter( + "powercontext_scheduler_leadership_changes_total", + "Scheduler leadership transitions observed by this process.", + ("outcome",), + registry=self.registry, + ) + self.work_queue_depth = Gauge( + "powercontext_work_queue_depth", + "Durable non-terminal work grouped by bounded kind and state.", + ("kind", "status"), + registry=self.registry, + ) + self.work_queue_oldest_age = Gauge( + "powercontext_work_queue_oldest_age_seconds", + "Age of the oldest durable work item in each bounded queue group.", + ("kind", "status"), + registry=self.registry, + ) + self.runtime_role_members = Gauge( + "powercontext_runtime_role_members", + "Compatible live runtime members grouped by role.", + ("role",), + registry=self.registry, + ) + self._work_queue_labels: set[tuple[str, str]] = set() self.set_runtime_scopes(0, 0) def start_transport(self, transport: str, operation: str) -> float: @@ -128,6 +185,48 @@ def set_runtime_scopes(self, cached: int, active: int) -> None: with suppress(Exception): self.runtime_scopes.labels(state=state).set(value) + def observe_work_enqueue(self, kind: str, *, created: bool) -> None: + self.work_enqueues.labels(kind=kind, outcome="created" if created else "joined").inc() + + def observe_work_claim(self, kind: str, *, latency_seconds: float) -> None: + self.work_claim_latency.labels(kind=kind).observe(max(0.0, latency_seconds)) + + def observe_work_attempt( + self, + kind: str, + *, + outcome: str, + error_category: str, + duration_seconds: float, + ) -> None: + self.work_attempts.labels( + kind=kind, + outcome=outcome, + error_category=error_category, + ).inc() + self.work_attempt_duration.labels(kind=kind, outcome=outcome).observe(max(0.0, duration_seconds)) + + def observe_work_lease_expiry(self, kind: str, *, outcome: str) -> None: + self.work_lease_expirations.labels(kind=kind, outcome=outcome).inc() + + def observe_scheduler_leadership(self, *, outcome: str) -> None: + self.scheduler_leadership_changes.labels(outcome=outcome).inc() + + def set_work_queue(self, samples: Sequence[WorkQueueStatistic]) -> None: + current = {(sample.kind, str(sample.status)) for sample in samples} + for kind, status in self._work_queue_labels - current: + self.work_queue_depth.labels(kind=kind, status=status).set(0) + self.work_queue_oldest_age.labels(kind=kind, status=status).set(0) + for sample in samples: + status = str(sample.status) + self.work_queue_depth.labels(kind=sample.kind, status=status).set(sample.depth) + self.work_queue_oldest_age.labels(kind=sample.kind, status=status).set(sample.oldest_age_seconds) + self._work_queue_labels = current + + def set_runtime_members(self, counts: Mapping[str, int]) -> None: + for role in ("all", "api", "scheduler", "worker"): + self.runtime_role_members.labels(role=role).set(counts.get(role, 0)) + def render(self) -> bytes: return generate_latest(self.registry) diff --git a/src/powercontext/server/middleware.py b/src/powercontext/server/middleware.py index 17ba9d6bd..7d8473116 100644 --- a/src/powercontext/server/middleware.py +++ b/src/powercontext/server/middleware.py @@ -24,6 +24,7 @@ from powercontext.http import ErrorDetail, ErrorResponse from powercontext.server.context import is_internal_bridge +from powercontext.server.principal import PrincipalRef, bind_principal, reset_principal _PUBLIC_PATHS = frozenset({ "/", @@ -41,6 +42,12 @@ _PUBLIC_PATH_PREFIXES = ("/static/",) +def is_public_http_path(path: str) -> bool: + """Return whether a request bypasses Server bearer authentication.""" + + return path in _PUBLIC_PATHS or path.startswith(_PUBLIC_PATH_PREFIXES) + + class StaticBearerMiddleware: """Require one configured bearer token for external HTTP requests.""" @@ -51,10 +58,21 @@ def __init__(self, app: ASGIApp, *, token: str) -> None: self._token = token.encode() async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if self._allows(scope): + if scope["type"] != "http" or is_internal_bridge(): + await self.app(scope, receive, send) + return + if is_public_http_path(scope["path"]): await self.app(scope, receive, send) return + if self._has_valid_bearer(scope): + token = bind_principal(PrincipalRef(kind="static_bearer", subject="default")) + try: + await self.app(scope, receive, send) + finally: + reset_principal(token) + return + error = ErrorResponse( error=ErrorDetail( code="unauthorized", @@ -69,15 +87,7 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: ) await response(scope, receive, send) - def _allows(self, scope: Scope) -> bool: - if ( - scope["type"] != "http" - or scope["path"] in _PUBLIC_PATHS - or scope["path"].startswith(_PUBLIC_PATH_PREFIXES) - or is_internal_bridge() - ): - return True - + def _has_valid_bearer(self, scope: Scope) -> bool: authorization = Headers(scope=scope).get("authorization") if authorization is None: return False @@ -90,4 +100,21 @@ def _allows(self, scope: Scope) -> bool: ) -__all__ = ["StaticBearerMiddleware"] +class LocalPrincipalMiddleware: + """Bind the implicit principal used by an authenticated local transport.""" + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http" or is_internal_bridge(): + await self.app(scope, receive, send) + return + token = bind_principal(PrincipalRef(kind="local", subject="default")) + try: + await self.app(scope, receive, send) + finally: + reset_principal(token) + + +__all__ = ["LocalPrincipalMiddleware", "StaticBearerMiddleware", "is_public_http_path"] diff --git a/src/powercontext/server/principal.py b/src/powercontext/server/principal.py new file mode 100644 index 000000000..8c035e13c --- /dev/null +++ b/src/powercontext/server/principal.py @@ -0,0 +1,53 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Trusted request principal propagated through HTTP and the MCP ASGI bridge.""" + +from __future__ import annotations + +import hashlib +from contextvars import ContextVar, Token +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class PrincipalRef: + """A non-secret stable authorization identity.""" + + kind: str + subject: str + + @property + def storage_key(self) -> str: + """Return an opaque bounded key suitable for coordination tables.""" + + return hashlib.sha256(f"{self.kind}\0{self.subject}".encode()).hexdigest() + + +_CURRENT_PRINCIPAL: ContextVar[PrincipalRef | None] = ContextVar("powercontext_principal", default=None) + + +def bind_principal(principal: PrincipalRef) -> Token[PrincipalRef | None]: + return _CURRENT_PRINCIPAL.set(principal) + + +def current_principal() -> PrincipalRef | None: + return _CURRENT_PRINCIPAL.get() + + +def reset_principal(token: Token[PrincipalRef | None]) -> None: + _CURRENT_PRINCIPAL.reset(token) + + +__all__ = ["PrincipalRef", "bind_principal", "current_principal", "reset_principal"] diff --git a/src/powercontext/server/rate_limit.py b/src/powercontext/server/rate_limit.py new file mode 100644 index 000000000..f7c220041 --- /dev/null +++ b/src/powercontext/server/rate_limit.py @@ -0,0 +1,104 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared HTTP rate-limit adapter.""" + +from __future__ import annotations + +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Receive, Scope, Send + +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.rate_limit import RateLimitRepository +from powercontext.http import ErrorDetail, ErrorResponse +from powercontext.server.context import is_internal_bridge +from powercontext.server.middleware import is_public_http_path +from powercontext.server.principal import PrincipalRef, current_principal + +_POLICY_ID = "http.default.v1" +_RATE_LIMIT_EXEMPT_PATHS = frozenset({"/metrics"}) + + +class SharedRateLimiter: + """Lifecycle-bound database counter used by every API replica.""" + + def __init__(self, *, requests: int, window_seconds: int) -> None: + self._requests = requests + self._window_seconds = window_seconds + self._database: AsyncDatabase | None = None + self._repository = RateLimitRepository() + + def bind(self, database: AsyncDatabase) -> None: + self._database = database + + def unbind(self) -> None: + self._database = None + + async def consume(self, principal: PrincipalRef) -> tuple[bool, int]: + database = self._database + if database is None: + raise RuntimeError("rate limiter must be bound before handling requests") # noqa: TRY003 + async with database.transaction() as connection: + decision = await self._repository.consume( + connection, + principal_key=principal.storage_key, + policy_id=_POLICY_ID, + limit=self._requests, + window_seconds=self._window_seconds, + ) + return decision.allowed, decision.retry_after_seconds + + +class SharedRateLimitMiddleware: + """Reject protected requests after their shared fixed window is exhausted.""" + + def __init__(self, app: ASGIApp, *, limiter: SharedRateLimiter) -> None: + self.app = app + self._limiter = limiter + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if self._skip(scope): + await self.app(scope, receive, send) + return + principal = current_principal() + if principal is None: + raise RuntimeError("principal middleware must run before rate limiting") # noqa: TRY003 + allowed, retry_after = await self._limiter.consume(principal) + if allowed: + await self.app(scope, receive, send) + return + response = JSONResponse( + content=ErrorResponse( + error=ErrorDetail( + code="rate_limited", + message="The request rate limit was exceeded.", + details=None, + ) + ).model_dump(mode="json"), + status_code=429, + headers={"Retry-After": str(retry_after)}, + ) + await response(scope, receive, send) + + @staticmethod + def _skip(scope: Scope) -> bool: + return ( + scope["type"] != "http" + or is_internal_bridge() + or is_public_http_path(scope["path"]) + or scope["path"] in _RATE_LIMIT_EXEMPT_PATHS + ) + + +__all__ = ["SharedRateLimitMiddleware", "SharedRateLimiter"] diff --git a/src/powercontext/server/settings.py b/src/powercontext/server/settings.py index 56ea3a161..4318c71a1 100644 --- a/src/powercontext/server/settings.py +++ b/src/powercontext/server/settings.py @@ -29,11 +29,17 @@ from powercontext.builtin.artifacts.skill import AgentSkillTarget from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime.config import ( + BuiltinConfig, + CoordinationConfig, DatabaseConfig, + DeploymentConfig, ExternalSkillsConfig, HandoffReportConfig, InferenceConfig, + OperationsConfig, + RateLimitConfig, RuntimeConfig, + WorkerConfig, ) from powercontext.paths import default_database_path, default_seekdb_path, sqlite_url from powercontext.transport import is_loopback_host @@ -195,6 +201,26 @@ class ServerSettings(BaseSettings): handoff_report: HandoffReportConfig = Field(default_factory=HandoffReportConfig) inference: InferenceConfig = Field(default_factory=InferenceConfig) external_skills: ExternalSkillsConfig = Field(default_factory=ExternalSkillsConfig) + deployment: DeploymentConfig = Field(default_factory=DeploymentConfig) + coordination: CoordinationConfig = Field(default_factory=CoordinationConfig) + worker: WorkerConfig = Field(default_factory=WorkerConfig) + operations: OperationsConfig = Field(default_factory=OperationsConfig) + rate_limit: RateLimitConfig = Field(default_factory=RateLimitConfig) + + def to_builtin_config(self) -> BuiltinConfig: + """Return the runtime-owned subset of the Server configuration.""" + + return BuiltinConfig( + runtime=self.runtime, + database=self.database, + handoff_report=self.handoff_report, + inference=self.inference, + external_skills=self.external_skills, + deployment=self.deployment, + coordination=self.coordination, + worker=self.worker, + operations=self.operations, + ) @field_validator("workspace") @classmethod @@ -206,7 +232,7 @@ def resolve_workspace(cls, value: Path) -> Path: @model_validator(mode="after") def configure_default_local_skill_targets(self) -> ServerSettings: - if "external_skills" not in self.model_fields_set: + if "external_skills" not in self.model_fields_set and self.deployment.mode == "single_node": self.external_skills = _default_local_external_skills(self.workspace) return self @@ -276,6 +302,7 @@ def reject_unauthenticated_non_loopback_bind(self) -> ServerSettings: allow_unauthenticated_non_loopback=self.allow_unauthenticated_non_loopback, ): raise UnauthenticatedNonLoopbackBindError(_UNSAFE_BIND_MESSAGE) + self.to_builtin_config() return self diff --git a/src/powercontext/server/tracing.py b/src/powercontext/server/tracing.py index 2521a3058..75723c16b 100644 --- a/src/powercontext/server/tracing.py +++ b/src/powercontext/server/tracing.py @@ -17,7 +17,7 @@ from __future__ import annotations import asyncio -from collections.abc import Iterator, Mapping +from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager, suppress from contextvars import ContextVar from importlib import import_module @@ -33,10 +33,21 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry.sdk.trace.id_generator import RandomIdGenerator from opentelemetry.sdk.trace.sampling import ALWAYS_OFF, ParentBased -from opentelemetry.trace import Span, SpanKind, Status, StatusCode, Tracer, set_span_in_context +from opentelemetry.trace import ( + Link, + Span, + SpanContext, + SpanKind, + Status, + StatusCode, + TraceFlags, + Tracer, + set_span_in_context, +) from starlette.types import ASGIApp, Message, Receive, Scope, Send from typing_extensions import override +from powercontext.builtin.runtime.protocols import RuntimeTraceContext from powercontext.server.context import bind_request_id, is_internal_bridge, reset_request_id from powercontext.server.settings import TracingConfig @@ -103,6 +114,7 @@ def start_span( kind: SpanKind, attributes: dict[str, Any], context: Context | None = None, + links: Sequence[Link] = (), ) -> _ActiveSpan: return _ActiveSpan.start( self.tracer, @@ -110,6 +122,7 @@ def start_span( kind=kind, attributes=attributes, context=context, + links=links, ) @contextmanager @@ -147,6 +160,7 @@ def background( *, operation: str, attributes: Mapping[str, _TraceAttribute], + links: Sequence[RuntimeTraceContext] = (), ) -> Iterator[_ActiveSpan]: """Trace one scheduled activation as an independent trace root.""" @@ -165,6 +179,7 @@ def background( "powercontext.operation.name": operation, "powercontext.operation.unit": "background", }, + links=_otel_links(links), ) try: yield span @@ -224,6 +239,19 @@ def __init__(self, span: Span | None, token: Token[Context] | None) -> None: def outcome(self) -> str | None: return self._outcome + @property + def trace_context(self) -> RuntimeTraceContext | None: + if self.span is None: + return None + with suppress(Exception): + context = self.span.get_span_context() + if context.is_valid: + return RuntimeTraceContext( + trace_id=f"{context.trace_id:032x}", + span_id=f"{context.span_id:016x}", + ) + return None + def set_outcome(self, outcome: str) -> None: self._outcome = outcome @@ -236,10 +264,11 @@ def start( kind: SpanKind, attributes: dict[str, Any], context: Context | None, + links: Sequence[Link] = (), ) -> _ActiveSpan: span: Span | None = None try: - span = tracer.start_span(name, context=context, kind=kind, attributes=attributes) + span = tracer.start_span(name, context=context, kind=kind, attributes=attributes, links=links) token = otel_context.attach(set_span_in_context(span, context)) except Exception: if span is not None: @@ -385,6 +414,23 @@ async def on_request( return result +def _otel_links(contexts: Sequence[RuntimeTraceContext]) -> tuple[Link, ...]: + links: list[Link] = [] + for context in contexts: + try: + span_context = SpanContext( + trace_id=int(context.trace_id, 16), + span_id=int(context.span_id, 16), + is_remote=True, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + ) + except (TypeError, ValueError): + continue + if span_context.is_valid: + links.append(Link(span_context)) + return tuple(links) + + def configure_server_tracing(config: TracingConfig) -> ServerTracing: """Build request context and optional OTLP export from standard OTel configuration.""" diff --git a/tests/builtin/persistence/test_coordination.py b/tests/builtin/persistence/test_coordination.py new file mode 100644 index 000000000..24ac7454b --- /dev/null +++ b/tests/builtin/persistence/test_coordination.py @@ -0,0 +1,112 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime + +import pytest +from sqlalchemy import update + +from powercontext.builtin.persistence.coordination import ( + CoordinationRepository, + StaleCoordinatorLeaseError, + StaleScanStateError, +) +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import COORDINATION_TABLES, SCHEDULER_LEASES_TABLE + + +def test_scheduler_lease_takeover_increments_fence_and_rejects_the_old_owner() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=COORDINATION_TABLES) as profile: + repository = CoordinationRepository() + async with profile.database.transaction() as connection: + first = await repository.acquire_lease( + connection, + lease_name="scheduler", + owner_id="scheduler-a", + lease_seconds=30, + ) + assert first is not None + renewed = await repository.acquire_lease( + connection, + lease_name="scheduler", + owner_id="scheduler-a", + lease_seconds=30, + ) + assert renewed is not None + assert renewed.fence == first.fence + + async with profile.database.transaction() as connection: + await connection.execute( + update(SCHEDULER_LEASES_TABLE) + .where(SCHEDULER_LEASES_TABLE.c.lease_name == "scheduler") + .values(lease_expires_at=datetime(2000, 1, 1, tzinfo=UTC)) + ) + second = await repository.acquire_lease( + connection, + lease_name="scheduler", + owner_id="scheduler-b", + lease_seconds=30, + ) + + assert second is not None + assert second.fence > first.fence + with pytest.raises(StaleCoordinatorLeaseError): + async with profile.database.transaction() as connection: + await repository.assert_lease(connection, first) + + asyncio.run(scenario()) + + +def test_scheduler_scan_state_uses_versioned_keyset_continuation() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=COORDINATION_TABLES) as profile: + repository = CoordinationRepository() + async with profile.database.transaction() as connection: + initial = await repository.load_scan(connection, "memory") + assert initial is None + first = await repository.save_scan( + connection, + "memory", + next_run_at=datetime(2030, 1, 1, tzinfo=UTC), + continuation="project:a", + expected_version=None, + ) + assert first.state_version == 1 + assert first.continuation == "project:a" + + async with profile.database.transaction() as connection: + second = await repository.save_scan( + connection, + "memory", + next_run_at=datetime(2030, 1, 2, tzinfo=UTC), + continuation=None, + expected_version=first.state_version, + ) + assert second.state_version == 2 + + with pytest.raises(StaleScanStateError): + async with profile.database.transaction() as connection: + await repository.save_scan( + connection, + "memory", + next_run_at=datetime(2030, 1, 3, tzinfo=UTC), + continuation=None, + expected_version=first.state_version, + ) + + asyncio.run(scenario()) diff --git a/tests/builtin/persistence/test_database.py b/tests/builtin/persistence/test_database.py new file mode 100644 index 000000000..784d125f1 --- /dev/null +++ b/tests/builtin/persistence/test_database.py @@ -0,0 +1,47 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime +from types import SimpleNamespace +from typing import cast + +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.database import database_now + + +def test_mysql_coordination_time_is_queried_in_utc() -> None: + statements: list[str] = [] + expected = datetime(2026, 9, 4, 12, 0, tzinfo=UTC).replace(tzinfo=None) + + class Result: + @staticmethod + def scalar_one() -> datetime: + return expected + + class Connection: + dialect = SimpleNamespace(name="mysql") + + @staticmethod + async def exec_driver_sql(statement: str) -> Result: + statements.append(statement) + return Result() + + value = asyncio.run(database_now(cast(AsyncConnection, Connection()))) + + assert value == expected + assert statements == ["SELECT UTC_TIMESTAMP(6)"] diff --git a/tests/builtin/persistence/test_experience_index.py b/tests/builtin/persistence/test_experience_index.py index 85125fbfd..4b3d01261 100644 --- a/tests/builtin/persistence/test_experience_index.py +++ b/tests/builtin/persistence/test_experience_index.py @@ -29,8 +29,14 @@ from powercontext.builtin.artifacts.skill import Skill, SkillContent from powercontext.builtin.persistence.artifact_governance import ArtifactLifecycleState from powercontext.builtin.persistence.experience_index import ensure_artifact_head_searchable_text -from powercontext.builtin.persistence.sqlite import SQLiteConfig -from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, BUILTIN_TABLES +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import ( + ARTIFACT_HEADS_TABLE, + BUILTIN_TABLES, + MEMORY_TABLES, + SHARED_TABLES, + STATISTICS_TABLES, +) from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts from powercontext.builtin.sources import ContentCapture @@ -63,21 +69,29 @@ def test_artifact_head_search_projection_schema_is_mysql_compilable() -> None: def test_sqlite_startup_upgrades_legacy_artifact_heads_without_searchable_text(tmp_path) -> None: database = tmp_path / "legacy.db" - with sqlite3.connect(database) as connection: - connection.execute( - """ - CREATE TABLE pc_artifact_heads ( - scope_id VARCHAR(256) NOT NULL, - family VARCHAR(128) NOT NULL, - artifact_id VARCHAR(128) NOT NULL, - revision INTEGER NOT NULL, - PRIMARY KEY (scope_id, family, artifact_id) - ) - """ - ) async def scenario() -> None: - config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{database}")) + sqlite_config = SQLiteConfig(url=f"sqlite+aiosqlite:///{database}") + config = BuiltinConfig(database=sqlite_config) + async with SQLiteProfile.open( + sqlite_config, + tables=SHARED_TABLES + MEMORY_TABLES + STATISTICS_TABLES, + ): + pass + with sqlite3.connect(database) as connection: + connection.execute("PRAGMA foreign_keys = OFF") + connection.execute("DROP TABLE pc_artifact_heads") + connection.execute( + """ + CREATE TABLE pc_artifact_heads ( + scope_id VARCHAR(256) NOT NULL, + family VARCHAR(128) NOT NULL, + artifact_id VARCHAR(128) NOT NULL, + revision INTEGER NOT NULL, + PRIMARY KEY (scope_id, family, artifact_id) + ) + """ + ) for _ in range(2): async with ( open_builtin_contexts(config) as contexts, diff --git a/tests/builtin/persistence/test_migration.py b/tests/builtin/persistence/test_migration.py new file mode 100644 index 000000000..647f9e61b --- /dev/null +++ b/tests/builtin/persistence/test_migration.py @@ -0,0 +1,124 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from typing import Any, cast + +import pytest +from sqlalchemy import inspect + +from powercontext.builtin.persistence import migration as migration_module +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.migration import ( + CURRENT_SCHEMA_REVISION, + SchemaCompatibilityError, + SchemaNotCurrentError, + migrate_database, + require_current_schema, +) +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import ( + MEMORY_TABLES, + SHARED_TABLES, + STATISTICS_TABLES, + WORK_ITEMS_TABLE, +) + + +def test_migration_commits_coordination_ddl_before_lease_dml(monkeypatch: pytest.MonkeyPatch) -> None: + """Protect OceanBase/MySQL from DDL invalidating the lease transaction.""" + + async def scenario() -> None: + connections: list[object] = [] + lease = object() + + class _Database: + @asynccontextmanager + async def transaction(self): + connection = object() + connections.append(connection) + yield connection + + async def create_tables(connection: object, tables: object) -> None: + assert connection is connections[0] + assert tables == (migration_module.SCHEDULER_LEASES_TABLE,) + + class _Repository: + async def acquire_lease(self, connection: object, **_: Any) -> object: + assert connection is connections[1] + return lease + + monkeypatch.setattr(migration_module, "create_tables", create_tables) + monkeypatch.setattr(migration_module, "CoordinationRepository", _Repository) + + acquired = await migration_module._acquire_migration_lease(cast(AsyncDatabase, _Database())) + + assert acquired is lease + assert len(connections) == 2 + + asyncio.run(scenario()) + + +def test_clean_schema_migrates_to_head_idempotently(tmp_path) -> None: + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'clean.db'}") + async with SQLiteProfile.open(config, tables=(), create_schema=False) as profile: + assert await migrate_database(profile.database) == CURRENT_SCHEMA_REVISION + assert await migrate_database(profile.database) == CURRENT_SCHEMA_REVISION + assert await require_current_schema(profile.database) == CURRENT_SCHEMA_REVISION + async with profile.database.transaction() as connection: + table_names = await connection.run_sync(lambda value: set(inspect(value).get_table_names())) + assert WORK_ITEMS_TABLE.name in table_names + + asyncio.run(scenario()) + + +def test_complete_unversioned_baseline_is_validated_stamped_and_expanded(tmp_path) -> None: + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'legacy.db'}") + baseline = SHARED_TABLES + MEMORY_TABLES + STATISTICS_TABLES + async with SQLiteProfile.open(config, tables=baseline) as profile: + assert await migrate_database(profile.database) == CURRENT_SCHEMA_REVISION + await require_current_schema(profile.database) + + asyncio.run(scenario()) + + +def test_partial_unversioned_schema_is_rejected_without_blind_stamping(tmp_path) -> None: + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'partial.db'}") + async with SQLiteProfile.open(config, tables=(SHARED_TABLES[0],)) as profile: + with pytest.raises(SchemaCompatibilityError, match="missing baseline tables"): + await migrate_database(profile.database) + with pytest.raises(SchemaNotCurrentError): + await require_current_schema(profile.database) + + asyncio.run(scenario()) + + +def test_current_revision_with_missing_physical_table_is_rejected(tmp_path) -> None: + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'damaged.db'}") + async with SQLiteProfile.open(config, tables=(), create_schema=False) as profile: + await migrate_database(profile.database) + async with profile.database.transaction() as connection: + await connection.run_sync(WORK_ITEMS_TABLE.drop) + + with pytest.raises(SchemaCompatibilityError, match="current revision is missing tables"): + await require_current_schema(profile.database) + + asyncio.run(scenario()) diff --git a/tests/builtin/persistence/test_mysql_schema.py b/tests/builtin/persistence/test_mysql_schema.py index a9e846ec2..4a40a133c 100644 --- a/tests/builtin/persistence/test_mysql_schema.py +++ b/tests/builtin/persistence/test_mysql_schema.py @@ -15,7 +15,7 @@ import re from pathlib import Path -from sqlalchemy import BigInteger, Date, Integer, String, Table +from sqlalchemy import BigInteger, Date, DateTime, Integer, String, Table from sqlalchemy.dialects import mysql from sqlalchemy.schema import CreateTable, ForeignKeyConstraint, PrimaryKeyConstraint, UniqueConstraint @@ -48,6 +48,8 @@ def _column_budget(column) -> int: return 4 if isinstance(column.type, Date): return 3 + if isinstance(column.type, DateTime): + return 8 raise _UnbudgetedColumnTypeError(column.type) @@ -112,6 +114,12 @@ def _assert_restore_layers_are_parent_first( def test_documented_obloader_restore_layers_are_parent_first() -> None: + transient_tables = { + "pc_rate_limit_windows", + "pc_runtime_members", + "pc_scheduler_leases", + "pc_scheduler_scans", + } restore_guides = ( Path("docs/en/docs/how-to/troubleshoot.md"), Path("docs/zh/docs/how-to/troubleshoot.md"), @@ -127,7 +135,9 @@ def test_documented_obloader_restore_layers_are_parent_first() -> None: assert len(set(restore_plans)) == 1 restore_layers = restore_plans[0] - _assert_restore_layers_are_parent_first(restore_layers, BUILTIN_TABLES) + durable_tables = tuple(table for table in BUILTIN_TABLES if table.name not in transient_tables) + assert {table.name for table in BUILTIN_TABLES} - {table.name for table in durable_tables} == transient_tables + _assert_restore_layers_are_parent_first(restore_layers, durable_tables) def test_every_mysql_utf8mb4_key_stays_below_the_innodb_limit() -> None: diff --git a/tests/builtin/persistence/test_rate_limit.py b/tests/builtin/persistence/test_rate_limit.py new file mode 100644 index 000000000..8df8614ee --- /dev/null +++ b/tests/builtin/persistence/test_rate_limit.py @@ -0,0 +1,45 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio + +from powercontext.builtin.persistence.rate_limit import RateLimitRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import RATE_LIMIT_WINDOWS_TABLE + + +def test_fixed_window_counter_is_shared_by_repository_callers() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=(RATE_LIMIT_WINDOWS_TABLE,)) as profile: + repository = RateLimitRepository() + decisions = [] + for _ in range(3): + async with profile.database.transaction() as connection: + decisions.append( + await repository.consume( + connection, + principal_key="a" * 64, + policy_id="api.default", + limit=2, + window_seconds=60, + ) + ) + + assert [decision.allowed for decision in decisions] == [True, True, False] + assert [decision.remaining for decision in decisions] == [1, 0, 0] + assert decisions[-1].retry_after_seconds > 0 + + asyncio.run(scenario()) diff --git a/tests/builtin/persistence/test_work.py b/tests/builtin/persistence/test_work.py new file mode 100644 index 000000000..833d8e9a9 --- /dev/null +++ b/tests/builtin/persistence/test_work.py @@ -0,0 +1,418 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from sqlalchemy import func, select, update + +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import WORK_ATTEMPTS_TABLE, WORK_ITEMS_TABLE, WORK_TABLES +from powercontext.builtin.persistence.work import ( + StaleWorkClaimError, + WorkFailure, + WorkRepository, + WorkResult, + WorkSpec, + WorkStateConflictError, + WorkStatus, +) + + +def _spec(*, logical_key: str = "a" * 64, lane_key: str = "b" * 64) -> WorkSpec: + return WorkSpec( + kind="powercontext.memory.source-window.v1", + payload_version=1, + scope_id="project:test", + lane_key=lane_key, + logical_key=logical_key, + payload={"after": 0, "through": 2}, + ) + + +def test_enqueue_deduplicates_one_logical_window_and_orders_each_lane() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + first = await repository.enqueue(connection, _spec()) + duplicate = await repository.enqueue(connection, _spec()) + second = await repository.enqueue(connection, _spec(logical_key="c" * 64)) + + assert first.created is True + assert duplicate.created is False + assert duplicate.work.work_id == first.work.work_id + assert first.work.lane_sequence == 1 + assert second.work.lane_sequence == 2 + + asyncio.run(scenario()) + + +def test_only_the_lane_head_can_be_claimed_and_completion_advances_it() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + first = await repository.enqueue(connection, _spec()) + second = await repository.enqueue(connection, _spec(logical_key="c" * 64)) + + async with profile.database.transaction() as connection: + claims = await repository.claim( + connection, + worker_id="worker-a", + supported={first.work.kind: frozenset({1})}, + lease_seconds=120, + limit=2, + ) + assert len(claims) == 1 + assert claims[0].work_id == first.work.work_id + + async with profile.database.transaction() as connection: + await repository.complete( + connection, + claims[0], + WorkResult(code="processed", payload={"current_cursor": 2}), + ) + async with profile.database.transaction() as connection: + next_claims = await repository.claim( + connection, + worker_id="worker-b", + supported={second.work.kind: frozenset({1})}, + lease_seconds=120, + limit=2, + ) + + assert len(next_claims) == 1 + assert next_claims[0].work_id == second.work.work_id + + asyncio.run(scenario()) + + +def test_concurrent_workers_cannot_claim_the_same_work(tmp_path: Path) -> None: + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'work.db'}") + async with SQLiteProfile.open(config, tables=WORK_TABLES) as first_profile: + async with SQLiteProfile.open(config, tables=WORK_TABLES) as second_profile: + repository = WorkRepository() + async with first_profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec()) + + async def claim(profile: SQLiteProfile, worker_id: str): + async with profile.database.transaction() as connection: + return await repository.claim( + connection, + worker_id=worker_id, + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + + results = await asyncio.gather( + claim(first_profile, "worker-a"), + claim(second_profile, "worker-b"), + ) + + assert sum(len(result) for result in results) == 1 + + asyncio.run(scenario()) + + +def test_expired_claim_is_fenced_before_another_attempt_runs() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec()) + first = ( + await repository.claim( + connection, + worker_id="worker-a", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + + async with profile.database.transaction() as connection: + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == first.work_id) + .values(lease_expires_at=datetime(2000, 1, 1, tzinfo=UTC)) + ) + assert ( + await repository.claim( + connection, + worker_id="worker-b", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + expired_retry_delay_seconds=0, + ) + ) == () + + async with profile.database.transaction() as connection: + second = ( + await repository.claim( + connection, + worker_id="worker-b", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + + assert second.fence > first.fence + with pytest.raises(StaleWorkClaimError): + async with profile.database.transaction() as connection: + await repository.complete(connection, first, WorkResult(code="late", payload={})) + + asyncio.run(scenario()) + + +def test_retry_budget_blocks_the_lane_until_an_operator_recovers_or_cancels() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec().model_copy(update={"max_attempts": 1})) + claim = ( + await repository.claim( + connection, + worker_id="worker-a", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + failed = await repository.fail( + connection, + claim, + WorkFailure(category="provider", code="temporarily_unavailable", retryable=True), + retry_delay_seconds=0, + ) + + assert failed.status is WorkStatus.FAILED + async with profile.database.transaction() as connection: + duplicate = await repository.enqueue(connection, _spec()) + assert duplicate.created is False + assert duplicate.work.status is WorkStatus.FAILED + + async with profile.database.transaction() as connection: + recovered = await repository.retry( + connection, + failed.work_id, + expected_version=failed.state_version, + ) + assert recovered.status is WorkStatus.QUEUED + assert recovered.recovery_generation == 1 + + with pytest.raises(WorkStateConflictError): + async with profile.database.transaction() as connection: + await repository.cancel( + connection, + recovered.work_id, + expected_version=failed.state_version, + ) + + asyncio.run(scenario()) + + +def test_running_cancel_linearizes_before_commit_and_recovers_after_expiry() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec()) + claim = ( + await repository.claim( + connection, + worker_id="worker-a", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + running = await repository.get(connection, claim.work_id) + + async with profile.database.transaction() as connection: + cancelling = await repository.cancel( + connection, + claim.work_id, + expected_version=running.state_version, + ) + assert cancelling.status is WorkStatus.CANCELLING + with pytest.raises(WorkStateConflictError): + async with profile.database.transaction() as connection: + await repository.cancel( + connection, + claim.work_id, + expected_version=cancelling.state_version, + ) + + committed = False + + async def domain_commit(_connection): + nonlocal committed + committed = True + return WorkResult(code="late", payload={}) + + with pytest.raises(StaleWorkClaimError): + async with profile.database.transaction() as connection: + await repository.complete(connection, claim, None, commit=domain_commit) + assert committed is False + + async with profile.database.transaction() as connection: + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == claim.work_id) + .values(lease_expires_at=datetime(2000, 1, 1, tzinfo=UTC)) + ) + assert ( + await repository.claim( + connection, + worker_id="worker-b", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + ) == () + cancelled = await repository.get(connection, claim.work_id) + assert cancelled.status is WorkStatus.CANCELLED + + asyncio.run(scenario()) + + +def test_retry_claim_links_to_the_previous_attempt_trace_context() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec()) + first = ( + await repository.claim( + connection, + worker_id="worker-a", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + await repository.record_attempt_trace( + connection, + first, + trace_id="1" * 32, + span_id="2" * 16, + ) + await repository.fail( + connection, + first, + WorkFailure(category="provider", code="unavailable", retryable=True), + retry_delay_seconds=0, + ) + + async with profile.database.transaction() as connection: + second = ( + await repository.claim( + connection, + worker_id="worker-b", + supported={enqueued.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + + assert second.previous_trace_id == "1" * 32 + assert second.previous_span_id == "2" * 16 + + asyncio.run(scenario()) + + +def test_retention_purges_only_old_success_and_cancelled_history() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + succeeded = await repository.enqueue(connection, _spec()) + success_claim = ( + await repository.claim( + connection, + worker_id="worker-a", + supported={succeeded.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + await repository.complete(connection, success_claim, WorkResult(code="done", payload={})) + + cancelled = await repository.enqueue( + connection, + _spec(logical_key="c" * 64, lane_key="d" * 64), + ) + await repository.cancel( + connection, + cancelled.work.work_id, + expected_version=cancelled.work.state_version, + ) + + failed = await repository.enqueue( + connection, + _spec(logical_key="e" * 64, lane_key="f" * 64).model_copy(update={"max_attempts": 1}), + ) + failed_claim = ( + await repository.claim( + connection, + worker_id="worker-a", + supported={failed.work.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + await repository.fail( + connection, + failed_claim, + WorkFailure(category="provider", code="secret_free_code", retryable=False), + retry_delay_seconds=0, + ) + + old = datetime(2000, 1, 1, tzinfo=UTC) + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id.in_((succeeded.work.work_id, cancelled.work.work_id))) + .values(completed_at=old) + ) + assert ( + await repository.purge_terminal( + connection, + completed_before=datetime(2020, 1, 1, tzinfo=UTC), + limit=500, + ) + == 2 + ) + attempts = await connection.scalar(select(func.count()).select_from(WORK_ATTEMPTS_TABLE)) + + assert attempts == 1 + async with profile.database.transaction() as connection: + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, succeeded.work.work_id) + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, cancelled.work.work_id) + blocked = await repository.get(connection, failed.work.work_id) + assert blocked.status is WorkStatus.FAILED + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_deployment_config.py b/tests/builtin/runtime/test_deployment_config.py new file mode 100644 index 000000000..53fdd0a11 --- /dev/null +++ b/tests/builtin/runtime/test_deployment_config.py @@ -0,0 +1,65 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import pytest +from pydantic import SecretStr, ValidationError + +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime.config import ( + BuiltinConfig, + CoordinationConfig, + DeploymentConfig, + WorkerConfig, +) + + +def test_single_node_all_is_the_backwards_compatible_default() -> None: + config = BuiltinConfig(database=SQLiteConfig()) + + assert config.deployment.mode == "single_node" + assert config.deployment.role == "all" + assert config.worker.concurrency == 4 + assert config.worker.lease_seconds == 120 + + +def test_distributed_requires_oceanbase_and_one_process_role() -> None: + with pytest.raises(ValidationError, match="distributed deployment requires OceanBase"): + BuiltinConfig( + database=SQLiteConfig(), + deployment=DeploymentConfig(mode="distributed", role="api", id="api-a"), + ) + + with pytest.raises(ValidationError, match="distributed deployment role must be"): + BuiltinConfig( + database=OceanBaseConfig( + url=SecretStr("mysql+aoceanbase://root@localhost:2881/powercontext?charset=utf8mb4") + ), + deployment=DeploymentConfig(mode="distributed", role="all", id="all-a"), + ) + + +def test_worker_heartbeat_and_shutdown_must_fit_inside_the_lease() -> None: + with pytest.raises(ValidationError, match="heartbeat_seconds must be less than one third"): + WorkerConfig(lease_seconds=120, heartbeat_seconds=40) + + with pytest.raises(ValidationError, match="shutdown_grace_seconds must be less than lease_seconds"): + WorkerConfig(lease_seconds=120, shutdown_grace_seconds=120) + + +def test_scheduler_emits_only_the_supported_payload_version() -> None: + with pytest.raises(ValidationError): + CoordinationConfig.model_validate({"emit_payload_version": 2}) diff --git a/tests/builtin/runtime/test_durable_scheduler.py b/tests/builtin/runtime/test_durable_scheduler.py new file mode 100644 index 000000000..f217a1c58 --- /dev/null +++ b/tests/builtin/runtime/test_durable_scheduler.py @@ -0,0 +1,130 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime + +import pytest +from sqlalchemy import update + +from powercontext.builtin.persistence.coordination import CoordinationRepository, StaleCoordinatorLeaseError +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import COORDINATION_TABLES, SCHEDULER_LEASES_TABLE, WORK_TABLES +from powercontext.builtin.persistence.work import WorkRepository, WorkSpec +from powercontext.builtin.runtime.config import CoordinationConfig +from powercontext.builtin.runtime.durable_scheduler import DiscoveryPage, DurableScheduler + + +class _Discoverer: + name = "test-discoverer" + interval_seconds = 60.0 + + def __init__(self) -> None: + self.pages: list[str | None] = [] + + async def page(self, continuation: str | None, limit: int) -> DiscoveryPage: + self.pages.append(continuation) + if continuation is None: + return DiscoveryPage(specs=(_spec(1),), continuation="next") + return DiscoveryPage(specs=(_spec(2),), continuation=None) + + +class _PausingDiscoverer: + name = "pausing-discoverer" + interval_seconds = 60.0 + + def __init__(self) -> None: + self.entered = asyncio.Event() + self.resume = asyncio.Event() + + async def page(self, continuation: str | None, limit: int) -> DiscoveryPage: + del continuation, limit + self.entered.set() + await self.resume.wait() + return DiscoveryPage(specs=(_spec(1),), continuation=None) + + +def _spec(index: int) -> WorkSpec: + return WorkSpec( + kind="test.handler.v1", + payload_version=1, + scope_id=f"scope:{index}", + lane_key=f"{index + 1:064x}", + logical_key=f"{index + 101:064x}", + payload={}, + ) + + +def test_scheduler_persists_keyset_continuation_and_enqueues_under_its_fence() -> None: + async def scenario() -> None: + tables = WORK_TABLES + COORDINATION_TABLES + async with SQLiteProfile.open(SQLiteConfig(), tables=tables) as profile: + discoverer = _Discoverer() + scheduler = DurableScheduler( + database=profile.database, + scheduler_id="scheduler-a", + discoverers=(discoverer,), + config=CoordinationConfig(), + ) + assert await scheduler.tick() is True + assert await scheduler.tick() is True + + async with profile.database.transaction() as connection: + work = await WorkRepository().list(connection) + assert len(work) == 2 + assert discoverer.pages == [None, "next"] + + await scheduler.stop() + + asyncio.run(scenario()) + + +def test_old_scheduler_cannot_enqueue_after_a_higher_fence_takes_over() -> None: + async def scenario() -> None: + tables = WORK_TABLES + COORDINATION_TABLES + async with SQLiteProfile.open(SQLiteConfig(), tables=tables) as profile: + discoverer = _PausingDiscoverer() + scheduler = DurableScheduler( + database=profile.database, + scheduler_id="scheduler-a", + discoverers=(discoverer,), + config=CoordinationConfig(), + ) + stale_tick = asyncio.create_task(scheduler.tick()) + await asyncio.wait_for(discoverer.entered.wait(), timeout=1) + + coordination = CoordinationRepository() + async with profile.database.transaction() as connection: + await connection.execute( + update(SCHEDULER_LEASES_TABLE) + .where(SCHEDULER_LEASES_TABLE.c.lease_name == "work-discovery") + .values(lease_expires_at=datetime(2000, 1, 1, tzinfo=UTC)) + ) + replacement = await coordination.acquire_lease( + connection, + lease_name="work-discovery", + owner_id="scheduler-b", + lease_seconds=30, + ) + assert replacement is not None + + discoverer.resume.set() + with pytest.raises(StaleCoordinatorLeaseError): + await stale_tick + async with profile.database.transaction() as connection: + assert await WorkRepository().list(connection) == () + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_membership.py b/tests/builtin/runtime/test_membership.py new file mode 100644 index 000000000..81b256d45 --- /dev/null +++ b/tests/builtin/runtime/test_membership.py @@ -0,0 +1,73 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio + +import pytest + +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import COORDINATION_TABLES +from powercontext.builtin.runtime.config import CoordinationConfig, DeploymentConfig +from powercontext.builtin.runtime.membership import DuplicateSingleNodeError, RuntimeMembership + + +def test_single_node_membership_rejects_a_second_live_runtime() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=COORDINATION_TABLES) as profile: + config = CoordinationConfig(member_ttl_seconds=3, member_heartbeat_seconds=1) + deployment = DeploymentConfig() + first = RuntimeMembership( + database=profile.database, + deployment=deployment, + coordination=config, + build_version="test", + ) + second = RuntimeMembership( + database=profile.database, + deployment=deployment, + coordination=config, + build_version="test", + ) + + await first.start() + with pytest.raises(DuplicateSingleNodeError): + await second.start() + + await first.stop() + await second.start() + assert await second.readiness() == "ready" + await second.stop() + + asyncio.run(scenario()) + + +def test_api_membership_reports_missing_worker_and_scheduler_as_degraded_dependencies() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=COORDINATION_TABLES) as profile: + membership = RuntimeMembership( + database=profile.database, + deployment=DeploymentConfig(mode="distributed", role="api", id="api-a"), + coordination=CoordinationConfig(), + build_version="test", + ) + await membership.start() + + assert await membership.readiness() == "ready" + assert await membership.role_readiness("scheduler") == "unavailable" + assert await membership.role_readiness("worker") == "unavailable" + await membership.stop() + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_operation_maintenance.py b/tests/builtin/runtime/test_operation_maintenance.py new file mode 100644 index 000000000..df213c8a4 --- /dev/null +++ b/tests/builtin/runtime/test_operation_maintenance.py @@ -0,0 +1,85 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime + +import pytest +from sqlalchemy import update + +from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import COORDINATION_TABLES, WORK_ITEMS_TABLE, WORK_TABLES +from powercontext.builtin.persistence.work import WorkRepository, WorkResult, WorkSpec, WorkStatus +from powercontext.builtin.runtime.config import WorkerConfig +from powercontext.builtin.runtime.work_handlers import OperationMaintenanceDiscoverer, OperationMaintenanceHandler +from powercontext.builtin.runtime.worker import DurableWorker + + +def test_operation_retention_runs_as_bounded_durable_work() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES + COORDINATION_TABLES) as profile: + repository = WorkRepository() + old_spec = WorkSpec( + kind="test.completed", + payload_version=1, + scope_id="project:test", + lane_key="a" * 64, + logical_key="b" * 64, + payload={}, + ) + async with profile.database.transaction() as connection: + old = await repository.enqueue(connection, old_spec) + claim = ( + await repository.claim( + connection, + worker_id="setup", + supported={old_spec.kind: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + )[0] + await repository.complete(connection, claim, WorkResult(code="done", payload={})) + await connection.execute( + update(WORK_ITEMS_TABLE) + .where(WORK_ITEMS_TABLE.c.work_id == old.work.work_id) + .values(completed_at=datetime(2000, 1, 1, tzinfo=UTC)) + ) + + discoverer = OperationMaintenanceDiscoverer(interval_seconds=3600, max_attempts=5) + page = await discoverer.page(None, 100) + async with profile.database.transaction() as connection: + maintenance = await repository.enqueue(connection, page.specs[0]) + + worker = DurableWorker( + database=profile.database, + worker_id="maintenance-worker", + handlers=(OperationMaintenanceHandler(retention_days=1, batch_size=500),), + config=WorkerConfig(concurrency=1), + ) + assert await worker.run_once() == 1 + + async with profile.database.transaction() as connection: + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, old.work.work_id) + completed = await repository.get(connection, maintenance.work.work_id) + assert completed.status is WorkStatus.SUCCEEDED + assert completed.result_payload == { + "operations_deleted": 1, + "rate_limit_windows_deleted": 0, + } + + asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_scheduler.py b/tests/builtin/runtime/test_scheduler.py deleted file mode 100644 index 809bab1e9..000000000 --- a/tests/builtin/runtime/test_scheduler.py +++ /dev/null @@ -1,503 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import asyncio -import json -import logging -import sqlite3 -from typing import Any - -import pytest -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter - -from powercontext import PowerContext -from powercontext.builtin.runtime import ( - BuiltinRuntime, - ExperienceIncubationResult, - MemoryFlushResult, - RuntimeCapabilities, -) -from powercontext.builtin.runtime.scheduler import ( - EXPERIENCE_INCUBATION_JOB_ID, - SOURCE_WINDOW_JOB_ID, - SchedulerConfigurationError, - SchedulerStateError, - scheduler_database_path, -) -from powercontext.builtin.scope import ScopeDescriptor -from powercontext.builtin.sources import SourceCursor -from powercontext.server.tracing import ServerTracing - - -class _Provider: - def __init__(self, context: PowerContext[Any, Any, Any]) -> None: - self.context = context - - async def get(self, scope_id: str, /) -> PowerContext[Any, Any, Any]: - del scope_id - return self.context - - -class _Scopes: - async def get(self, scope_id: str, /) -> ScopeDescriptor: - return ScopeDescriptor(scope_id=scope_id, title="Scheduled", summary="Scheduled scope", version=1) - - -class _ScheduledTriggers: - def __init__(self, *, source_count: int = 0) -> None: - self.dispatched = asyncio.Event() - self.source_count = source_count - - async def flush(self, *, limit: int) -> MemoryFlushResult: - del limit - self.dispatched.set() - return MemoryFlushResult( - previous_cursor=0, - high_watermark=self.source_count, - current_cursor=self.source_count, - source_count=self.source_count, - memory_ref=None, - ) - - async def cursor(self) -> SourceCursor: - return SourceCursor() - - -class _FailingTriggers: - async def flush(self, *, limit: int) -> MemoryFlushResult: - del limit - raise RuntimeError("flush failed") # noqa: TRY003 - - async def cursor(self) -> SourceCursor: - return SourceCursor() - - -class _BlockingTriggers: - def __init__(self) -> None: - self.entered = asyncio.Event() - - async def flush(self, *, limit: int) -> MemoryFlushResult: - del limit - self.entered.set() - await asyncio.Event().wait() - raise AssertionError("unreachable") - - async def cursor(self) -> SourceCursor: - return SourceCursor() - - -class _ScheduledExperience: - def __init__(self) -> None: - self.dispatched = asyncio.Event() - - async def __call__(self, scope_id: str, limit: int) -> ExperienceIncubationResult: - del scope_id, limit - self.dispatched.set() - return ExperienceIncubationResult( - previous_cursor=0, - high_watermark=1, - current_cursor=1, - source_count=1, - candidate_count=1, - ) - - -class _NoopExperience: - async def __call__(self, scope_id: str, limit: int) -> ExperienceIncubationResult: - del scope_id, limit - return ExperienceIncubationResult( - previous_cursor=0, - high_watermark=0, - current_cursor=0, - source_count=0, - candidate_count=0, - ) - - -class _FailingExperience: - async def __call__(self, scope_id: str, limit: int) -> ExperienceIncubationResult: - del scope_id, limit - raise RuntimeError("incubation failed") # noqa: TRY003 - - -class _BlockingExperience: - def __init__(self) -> None: - self.entered = asyncio.Event() - - async def __call__(self, scope_id: str, limit: int) -> ExperienceIncubationResult: - del scope_id, limit - self.entered.set() - await asyncio.Event().wait() - raise AssertionError("unreachable") - - -async def _scope_ids() -> tuple[str, ...]: - return ("scheduled",) - - -def _runtime( - triggers: object, - *, - scope_ids=_scope_ids, - experience_incubator: ( - _ScheduledExperience | _NoopExperience | _FailingExperience | _BlockingExperience | None - ) = None, - tracing: ServerTracing | None = None, -) -> BuiltinRuntime: - return BuiltinRuntime( - provider=_Provider(PowerContext(sources=object(), artifacts=object(), triggers=triggers)), # type: ignore[arg-type] - capabilities=RuntimeCapabilities(memory_extraction=True, memory_search_modes=("fts",)), - scope_ids=scope_ids, - scope_application=_Scopes(), # ty: ignore[invalid-argument-type] - experience_incubator=experience_incubator, - tracing=tracing, - ) - - -def _tracing() -> tuple[ServerTracing, InMemorySpanExporter]: - exporter = InMemorySpanExporter() - provider = TracerProvider(shutdown_on_exit=False) - provider.add_span_processor(SimpleSpanProcessor(exporter)) - return ServerTracing(provider), exporter - - -def _scope_id_leak(spans) -> str: - return json.dumps( - [{"name": span.name, "attributes": dict(span.attributes or {})} for span in spans], - default=str, - ) - - -def _stored_jobs(database) -> list[tuple[str, float | None]]: - with sqlite3.connect(scheduler_database_path(database)) as connection: - return connection.execute("SELECT id, next_run_time FROM powercontext_scheduler_jobs ORDER BY id").fetchall() - - -def test_scheduler_persists_one_stable_source_window_job(tmp_path) -> None: - async def scenario() -> None: - database = tmp_path / "runtime.db" - first = _runtime(_ScheduledTriggers()) - first.start_scheduler(database, 3_600) - await first.close() - jobs = _stored_jobs(database) - - restored = _runtime(_ScheduledTriggers()) - restored.start_scheduler(database, 3_600) - await restored.close() - - assert jobs == _stored_jobs(database) - assert len(jobs) == 1 - assert jobs[0][0] == SOURCE_WINDOW_JOB_ID - - asyncio.run(scenario()) - - -def test_scheduler_creates_missing_database_parent_directory(tmp_path) -> None: - async def scenario() -> None: - database = tmp_path / "missing" / "nested" / "scheduler.db" - assert not database.parent.exists() - - runtime = _runtime(_ScheduledTriggers()) - try: - runtime.start_scheduler(database, 3_600) - assert database.is_file() - finally: - await runtime.close() - - asyncio.run(scenario()) - - -def test_scheduler_interval_activates_the_source_window_policy(tmp_path) -> None: - async def scenario() -> None: - triggers = _ScheduledTriggers() - runtime = _runtime(triggers) - runtime.start_scheduler(tmp_path / "runtime.db", 0.01) - try: - await asyncio.wait_for(triggers.dispatched.wait(), timeout=2) - finally: - await runtime.close() - - asyncio.run(scenario()) - - -def test_scheduler_interval_activates_experience_incubation(tmp_path) -> None: - async def scenario() -> None: - incubation = _ScheduledExperience() - runtime = _runtime(_ScheduledTriggers(), experience_incubator=incubation) - runtime.start_scheduler( - tmp_path / "runtime.db", - None, - experience_schedule_seconds=0.01, - ) - try: - await asyncio.wait_for(incubation.dispatched.wait(), timeout=2) - finally: - await runtime.close() - - asyncio.run(scenario()) - - -def test_scheduler_persists_and_reconciles_independent_jobs(tmp_path) -> None: - async def scenario() -> None: - database = tmp_path / "runtime.db" - first = _runtime(_ScheduledTriggers(), experience_incubator=_ScheduledExperience()) - first.start_scheduler(database, 3_600, experience_schedule_seconds=7_200) - await first.close() - - assert [job_id for job_id, _ in _stored_jobs(database)] == [ - EXPERIENCE_INCUBATION_JOB_ID, - SOURCE_WINDOW_JOB_ID, - ] - - restored = _runtime(_ScheduledTriggers(), experience_incubator=_ScheduledExperience()) - restored.start_scheduler(database, None, experience_schedule_seconds=7_200) - await restored.close() - - assert [job_id for job_id, _ in _stored_jobs(database)] == [EXPERIENCE_INCUBATION_JOB_ID] - - asyncio.run(scenario()) - - -def test_scheduled_noop_logs_a_bounded_outcome(caplog) -> None: - async def scenario() -> None: - runtime = _runtime(_ScheduledTriggers()) - assert runtime.processor is not None - await runtime.processor.run() - - with caplog.at_level(logging.INFO, logger="powercontext.builtin.runtime.application"): - asyncio.run(scenario()) - - record = next(record for record in caplog.records if record.event == "background.operation.completed") - assert record.operation == "process_source_window" - assert record.outcome == "noop" - assert record.source_count == 0 - assert "scope_id" not in vars(record) - - -def test_scheduled_experience_logs_candidate_count_without_scope(caplog) -> None: - async def scenario() -> None: - runtime = _runtime(_ScheduledTriggers(), experience_incubator=_ScheduledExperience()) - assert runtime.experience_processor is not None - await runtime.experience_processor.run() - - with caplog.at_level(logging.INFO, logger="powercontext.builtin.runtime.application"): - asyncio.run(scenario()) - - record = next(record for record in caplog.records if record.operation == "incubate_experience_candidates") - assert record.outcome == "success" - assert record.source_count == 1 - assert record.candidate_count == 1 - assert "scope_id" not in vars(record) - - -async def _private_scope_ids() -> tuple[str, ...]: - return ("project:private-scheduled-scope",) - - -def test_scheduled_processor_records_root_and_flush_spans() -> None: - tracing, exporter = _tracing() - - async def scenario() -> None: - runtime = _runtime(_ScheduledTriggers(), tracing=tracing, scope_ids=_private_scope_ids) - assert runtime.processor is not None - await runtime.processor.run() - - asyncio.run(scenario()) - - spans = {span.name: span for span in exporter.get_finished_spans()} - root = spans["scheduled.process_source_window"] - flush = spans["memory.flush"] - assert root.parent is None - assert flush.parent is not None and flush.parent.span_id == root.context.span_id - assert root.attributes is not None - assert root.attributes["powercontext.operation.name"] == "process_source_window" - assert root.attributes["powercontext.operation.unit"] == "background" - assert root.attributes["powercontext.operation.outcome"] == "noop" - assert root.attributes["powercontext.background.source_count"] == 0 - assert flush.attributes is not None - assert flush.attributes["powercontext.operation.unit"] == "stage" - assert flush.attributes["powercontext.operation.outcome"] == "noop" - assert flush.attributes["powercontext.memory.flush.source_count"] == 0 - assert "project:private-scheduled-scope" not in _scope_id_leak(exporter.get_finished_spans()) - - -def test_scheduled_processor_records_success_outcome() -> None: - tracing, exporter = _tracing() - - async def scenario() -> None: - runtime = _runtime(_ScheduledTriggers(source_count=3), tracing=tracing) - assert runtime.processor is not None - await runtime.processor.run() - - asyncio.run(scenario()) - - spans = {span.name: span for span in exporter.get_finished_spans()} - root = spans["scheduled.process_source_window"] - flush = spans["memory.flush"] - assert root.attributes is not None and root.attributes["powercontext.operation.outcome"] == "success" - assert root.attributes["powercontext.background.source_count"] == 3 - assert flush.attributes is not None and flush.attributes["powercontext.operation.outcome"] == "success" - assert flush.attributes["powercontext.memory.flush.source_count"] == 3 - - -def test_scheduled_processor_records_failure_and_swallows_error() -> None: - tracing, exporter = _tracing() - - async def scenario() -> None: - runtime = _runtime(_FailingTriggers(), tracing=tracing) - assert runtime.processor is not None - await runtime.processor.run() - - asyncio.run(scenario()) - - spans = {span.name: span for span in exporter.get_finished_spans()} - root = spans["scheduled.process_source_window"] - flush = spans["memory.flush"] - assert root.attributes is not None and root.attributes["powercontext.operation.outcome"] == "failure" - assert flush.attributes is not None and flush.attributes["powercontext.operation.outcome"] == "failure" - - -def test_scheduled_processor_records_cancellation() -> None: - tracing, exporter = _tracing() - - async def scenario() -> None: - triggers = _BlockingTriggers() - runtime = _runtime(triggers, tracing=tracing) - assert runtime.processor is not None - task = asyncio.create_task(runtime.processor.run()) - await triggers.entered.wait() - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - - asyncio.run(scenario()) - - spans = {span.name: span for span in exporter.get_finished_spans()} - root = spans["scheduled.process_source_window"] - assert root.attributes is not None and root.attributes["powercontext.operation.outcome"] == "cancelled" - - -def test_scheduled_experience_records_root_and_incubation_spans() -> None: - tracing, exporter = _tracing() - - async def scenario() -> None: - runtime = _runtime( - _ScheduledTriggers(), - experience_incubator=_ScheduledExperience(), - tracing=tracing, - scope_ids=_private_scope_ids, - ) - assert runtime.experience_processor is not None - await runtime.experience_processor.run() - - asyncio.run(scenario()) - - spans = {span.name: span for span in exporter.get_finished_spans()} - root = spans["scheduled.incubate_experience_candidates"] - incubation = spans["experience.incubation"] - assert root.parent is None - assert incubation.parent is not None and incubation.parent.span_id == root.context.span_id - assert root.attributes is not None - assert root.attributes["powercontext.operation.name"] == "incubate_experience_candidates" - assert root.attributes["powercontext.operation.unit"] == "background" - assert root.attributes["powercontext.operation.outcome"] == "success" - assert root.attributes["powercontext.background.source_count"] == 1 - assert root.attributes["powercontext.background.candidate_count"] == 1 - assert incubation.attributes is not None - assert incubation.attributes["powercontext.experience.incubation.source_count"] == 1 - assert incubation.attributes["powercontext.experience.incubation.candidate_count"] == 1 - assert "project:private-scheduled-scope" not in _scope_id_leak(exporter.get_finished_spans()) - - -def test_scheduled_experience_records_noop_outcome() -> None: - tracing, exporter = _tracing() - - async def scenario() -> None: - runtime = _runtime(_ScheduledTriggers(), experience_incubator=_NoopExperience(), tracing=tracing) - assert runtime.experience_processor is not None - await runtime.experience_processor.run() - - asyncio.run(scenario()) - - spans = {span.name: span for span in exporter.get_finished_spans()} - root = spans["scheduled.incubate_experience_candidates"] - incubation = spans["experience.incubation"] - assert root.attributes is not None and root.attributes["powercontext.operation.outcome"] == "noop" - assert root.attributes["powercontext.background.source_count"] == 0 - assert root.attributes["powercontext.background.candidate_count"] == 0 - assert incubation.attributes is not None and incubation.attributes["powercontext.operation.outcome"] == "noop" - assert incubation.attributes["powercontext.experience.incubation.source_count"] == 0 - assert incubation.attributes["powercontext.experience.incubation.candidate_count"] == 0 - - -def test_scheduled_experience_records_failure_and_swallows_error() -> None: - tracing, exporter = _tracing() - - async def scenario() -> None: - runtime = _runtime(_ScheduledTriggers(), experience_incubator=_FailingExperience(), tracing=tracing) - assert runtime.experience_processor is not None - await runtime.experience_processor.run() - - asyncio.run(scenario()) - - spans = {span.name: span for span in exporter.get_finished_spans()} - root = spans["scheduled.incubate_experience_candidates"] - incubation = spans["experience.incubation"] - assert root.attributes is not None and root.attributes["powercontext.operation.outcome"] == "failure" - assert incubation.attributes is not None and incubation.attributes["powercontext.operation.outcome"] == "failure" - - -def test_scheduled_experience_records_cancellation() -> None: - tracing, exporter = _tracing() - - async def scenario() -> None: - incubator = _BlockingExperience() - runtime = _runtime(_ScheduledTriggers(), experience_incubator=incubator, tracing=tracing) - assert runtime.experience_processor is not None - task = asyncio.create_task(runtime.experience_processor.run()) - await incubator.entered.wait() - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - - asyncio.run(scenario()) - - spans = {span.name: span for span in exporter.get_finished_spans()} - root = spans["scheduled.incubate_experience_candidates"] - assert root.attributes is not None and root.attributes["powercontext.operation.outcome"] == "cancelled" - - -def test_scheduler_requires_file_storage_and_one_live_owner(tmp_path) -> None: - async def scenario() -> None: - with pytest.raises(SchedulerConfigurationError): - _runtime(_ScheduledTriggers()).start_scheduler(":memory:", 60) - - database = tmp_path / "runtime.db" - first = _runtime(_ScheduledTriggers()) - second = _runtime(_ScheduledTriggers()) - first.start_scheduler(database, 60) - try: - with pytest.raises(SchedulerStateError): - second.start_scheduler(database, 60) - finally: - await first.close() - - second.start_scheduler(database, 60) - await second.close() - - asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_worker.py b/tests/builtin/runtime/test_worker.py new file mode 100644 index 000000000..ffc793071 --- /dev/null +++ b/tests/builtin/runtime/test_worker.py @@ -0,0 +1,339 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import WORK_ATTEMPTS_TABLE, WORK_ITEMS_TABLE, WORK_TABLES +from powercontext.builtin.persistence.work import WorkRepository, WorkResult, WorkSpec, WorkStatus +from powercontext.builtin.runtime.config import WorkerConfig +from powercontext.builtin.runtime.protocols import RuntimeTraceContext +from powercontext.builtin.runtime.worker import DurableWorker, PreparedWork, WorkExecutionError + + +class _Handler: + kind = "test.handler.v1" + supported_versions = frozenset({1}) + + def __init__(self, executed: list[str]) -> None: + self._executed = executed + + async def prepare(self, claim) -> PreparedWork: + async def commit(_connection: AsyncConnection) -> None: + self._executed.append(claim.work_id) + + return PreparedWork(result=WorkResult(code="done", payload={}), commit=commit) + + +class _OverlapHandler: + kind = "test.handler.v1" + supported_versions = frozenset({1}) + + def __init__(self) -> None: + self.active = 0 + self.maximum_active = 0 + self.both_entered = asyncio.Event() + + async def prepare(self, _claim) -> PreparedWork: + self.active += 1 + self.maximum_active = max(self.maximum_active, self.active) + if self.active == 2: + self.both_entered.set() + await asyncio.wait_for(self.both_entered.wait(), timeout=1) + self.active -= 1 + return PreparedWork(result=WorkResult(code="done", payload={})) + + +class _FlakyHandler: + kind = "test.handler.v1" + supported_versions = frozenset({1}) + + def __init__(self) -> None: + self.calls = 0 + + async def prepare(self, _claim) -> PreparedWork: + self.calls += 1 + if self.calls == 1: + raise WorkExecutionError(category="provider", code="unavailable", retryable=True) + return PreparedWork(result=WorkResult(code="done", payload={})) + + +class _BlockingHandler: + kind = "test.handler.v1" + supported_versions = frozenset({1}) + + def __init__(self) -> None: + self.started = asyncio.Event() + self.release = asyncio.Event() + + async def prepare(self, _claim) -> PreparedWork: + self.started.set() + await self.release.wait() + return PreparedWork(result=WorkResult(code="done", payload={})) + + +class _SensitiveFailureHandler: + kind = "test.handler.v1" + supported_versions = frozenset({1}) + + def __init__(self, sensitive_text: str) -> None: + self._sensitive_text = sensitive_text + + async def prepare(self, _claim) -> PreparedWork: + raise RuntimeError(self._sensitive_text) + + +class _TraceSpan: + def __init__(self, trace_context: RuntimeTraceContext | None = None) -> None: + self.trace_context = trace_context + + def set_attributes(self, _attributes) -> None: + return + + def set_outcome(self, _outcome: str) -> None: + return + + +class _Tracing: + def __init__(self) -> None: + self.execute_links: list[tuple[RuntimeTraceContext, ...]] = [] + self.attributes: list[dict[str, str | bool | int | float]] = [] + + @contextmanager + def stage( + self, + name: str, + *, + attributes: Mapping[str, str | bool | int | float], + ) -> Iterator[_TraceSpan]: + del name + self.attributes.append(dict(attributes)) + yield _TraceSpan() + + @contextmanager + def background( + self, + name: str, + *, + operation: str, + attributes: Mapping[str, str | bool | int | float], + links: Sequence[RuntimeTraceContext] = (), + ) -> Iterator[_TraceSpan]: + del operation + self.attributes.append(dict(attributes)) + links = tuple(links) + if name == "work.execute": + self.execute_links.append(links) + index = len(self.execute_links) + yield _TraceSpan(RuntimeTraceContext(trace_id=f"{index:032x}", span_id=f"{index:016x}")) + return + yield _TraceSpan() + + +def _spec(index: int) -> WorkSpec: + return WorkSpec( + kind="test.handler.v1", + payload_version=1, + scope_id=f"scope:{index}", + lane_key=f"{index + 1:064x}", + logical_key=f"{index + 101:064x}", + payload={}, + ) + + +def test_worker_claims_only_free_slots_and_executes_different_lanes_concurrently() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + executed: list[str] = [] + handler = _Handler(executed) + repository = WorkRepository() + async with profile.database.transaction() as connection: + first = await repository.enqueue(connection, _spec(1)) + second = await repository.enqueue(connection, _spec(2)) + + worker = DurableWorker( + database=profile.database, + worker_id="worker-a", + handlers=(handler,), + config=WorkerConfig(concurrency=2), + ) + assert await worker.run_once() == 2 + + async with profile.database.transaction() as connection: + first_stored = await repository.get(connection, first.work.work_id) + second_stored = await repository.get(connection, second.work.work_id) + assert first_stored.status is WorkStatus.SUCCEEDED + assert second_stored.status is WorkStatus.SUCCEEDED + assert set(executed) == {first.work.work_id, second.work.work_id} + + asyncio.run(scenario()) + + +def test_worker_overlaps_different_lanes_but_never_claims_an_unknown_payload() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + handler = _OverlapHandler() + repository = WorkRepository() + async with profile.database.transaction() as connection: + await repository.enqueue(connection, _spec(1)) + await repository.enqueue(connection, _spec(2)) + + worker = DurableWorker( + database=profile.database, + worker_id="worker-a", + handlers=(handler,), + config=WorkerConfig(concurrency=2), + ) + assert await worker.run_once() == 2 + assert handler.maximum_active == 2 + + unknown = _spec(3).model_copy(update={"payload_version": 2}) + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, unknown) + assert await worker.readiness() == "misconfigured" + assert await worker.run_once() == 0 + async with profile.database.transaction() as connection: + stored = await repository.get(connection, enqueued.work.work_id) + assert stored.status is WorkStatus.QUEUED + + asyncio.run(scenario()) + + +def test_retried_worker_attempt_links_to_the_persisted_previous_span() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec(1)) + + tracing = _Tracing() + worker = DurableWorker( + database=profile.database, + worker_id="worker-a", + handlers=(_FlakyHandler(),), + config=WorkerConfig(concurrency=1, retry_base_seconds=0.01, retry_max_seconds=0.01), + random_source=lambda _low, _high: 0, + tracing=tracing, + ) + assert await worker.run_once() == 1 + assert await worker.run_once() == 1 + async with profile.database.transaction() as connection: + completed = await repository.get(connection, enqueued.work.work_id) + + assert completed.status is WorkStatus.SUCCEEDED + assert tracing.execute_links == [ + (), + (RuntimeTraceContext(trace_id=f"{1:032x}", span_id=f"{1:016x}"),), + ] + + asyncio.run(scenario()) + + +def test_worker_converges_a_running_cancel_without_waiting_for_lease_expiry() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + handler = _BlockingHandler() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec(1)) + + worker = DurableWorker( + database=profile.database, + worker_id="worker-a", + handlers=(handler,), + config=WorkerConfig(concurrency=1), + ) + attempt = asyncio.create_task(worker.run_once()) + await asyncio.wait_for(handler.started.wait(), timeout=1) + async with profile.database.transaction() as connection: + running = await repository.get(connection, enqueued.work.work_id) + cancelling = await repository.cancel( + connection, + running.work_id, + expected_version=running.state_version, + ) + assert cancelling.status is WorkStatus.CANCELLING + + handler.release.set() + assert await asyncio.wait_for(attempt, timeout=1) == 1 + async with profile.database.transaction() as connection: + cancelled = await repository.get(connection, enqueued.work.work_id) + outcome = await connection.scalar( + select(WORK_ATTEMPTS_TABLE.c.outcome).where(WORK_ATTEMPTS_TABLE.c.work_id == enqueued.work.work_id) + ) + + assert cancelled.status is WorkStatus.CANCELLED + assert outcome == "cancelled" + + asyncio.run(scenario()) + + +def test_worker_does_not_persist_or_observe_sensitive_exception_text(caplog) -> None: + sensitive_text = "source-content=private credential=https://user:secret@example.test" + + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=WORK_TABLES) as profile: + repository = WorkRepository() + async with profile.database.transaction() as connection: + enqueued = await repository.enqueue(connection, _spec(1)) + + tracing = _Tracing() + worker = DurableWorker( + database=profile.database, + worker_id="worker-a", + handlers=(_SensitiveFailureHandler(sensitive_text),), + config=WorkerConfig(concurrency=1), + random_source=lambda _low, _high: 0, + tracing=tracing, + ) + with caplog.at_level(logging.DEBUG): + assert await worker.run_once() == 1 + + async with profile.database.transaction() as connection: + work = ( + ( + await connection.execute( + select(WORK_ITEMS_TABLE).where(WORK_ITEMS_TABLE.c.work_id == enqueued.work.work_id) + ) + ) + .mappings() + .one() + ) + attempt = ( + ( + await connection.execute( + select(WORK_ATTEMPTS_TABLE).where(WORK_ATTEMPTS_TABLE.c.work_id == enqueued.work.work_id) + ) + ) + .mappings() + .one() + ) + + assert work["status"] == WorkStatus.RETRY_WAIT.value + assert work["error_category"] == "internal" + assert work["error_code"] == "unhandled_handler_error" + assert sensitive_text not in repr(dict(work)) + assert sensitive_text not in repr(dict(attempt)) + assert sensitive_text not in caplog.text + assert sensitive_text not in repr(tracing.attributes) + + asyncio.run(scenario()) diff --git a/tests/client/test_receiver_service.py b/tests/client/test_receiver_service.py index 5886a97e9..6d928b954 100644 --- a/tests/client/test_receiver_service.py +++ b/tests/client/test_receiver_service.py @@ -26,6 +26,11 @@ from powercontext.client.skill_receiver import RemoteSkillReceiverConfig +@pytest.fixture(autouse=True) +def _linux_platform(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(service_module.sys, "platform", "linux") + + def _config(tmp_path: Path) -> RemoteSkillReceiverConfig: return RemoteSkillReceiverConfig( server_url="https://powercontext.example.com", diff --git a/tests/e2e/real_experience_skill/harness.py b/tests/e2e/real_experience_skill/harness.py index 3e5c7e17e..5d900b7b6 100644 --- a/tests/e2e/real_experience_skill/harness.py +++ b/tests/e2e/real_experience_skill/harness.py @@ -318,13 +318,13 @@ def main(argv: Sequence[str] | None = None) -> int: # noqa: C901 - one exceptio "branch": _run([git, "branch", "--show-current"], cwd=PROJECT_ROOT).stdout.strip(), "mode": "configured-real-services" if arguments.configured else "deterministic-runtime", "codex_home": "isolated temporary directory with a mode-0600 auth copy only", - "scheduler": "isolated APScheduler SQLite sidecar", + "scheduler": "database-backed fenced Scheduler and leased Worker", "preflight_output_directories_removed": removed_existing_outputs, }) if configured_settings is None: recorder.environment.update({ "runtime_generation_model": "not configured", - "experience_incubation": "APScheduler with a deterministic typed Task Outcome adapter", + "experience_incubation": "durable Work Ledger with a deterministic typed Task Outcome adapter", "database": "isolated SQLite", }) else: @@ -337,7 +337,7 @@ def main(argv: Sequence[str] | None = None) -> int: # noqa: C901 - one exceptio "embedding_dimension": configured_settings.inference.embedding_dimension, "database": configured_settings.database.kind, "database_isolated": isinstance(configured_settings.database, SQLiteConfig), - "experience_incubation": "APScheduler with the configured real generation model", + "experience_incubation": "durable Work Ledger with the configured real generation model", "experience_schedule_seconds": CONFIGURED_EXPERIENCE_SCHEDULE_SECONDS, }) recorder.write_report() @@ -354,10 +354,7 @@ def main(argv: Sequence[str] | None = None) -> int: # noqa: C901 - one exceptio validation_command="python test_config.py" if configured_settings is None else "python3 test_config.py", ) if configured_settings is None: - server = _start_server( - recorder.directory / "runtime.db", - recorder.directory / "scheduler.db", - ) + server = _start_server(recorder.directory / "runtime.db") else: external_skill = _prepare_external_skill(recorder.directory / "work" / "external-skills") configured_server_settings = _configured_server_settings( @@ -365,10 +362,7 @@ def main(argv: Sequence[str] | None = None) -> int: # noqa: C901 - one exceptio recorder.directory, external_skill.parent, ) - server = _start_configured_server( - configured_server_settings, - recorder.directory / "configured-scheduler.db", - ) + server = _start_configured_server(configured_server_settings) recorder.environment["server_url"] = server.base_url recorder.write_report() if configured_settings is None: @@ -401,10 +395,7 @@ def main(argv: Sequence[str] | None = None) -> int: # noqa: C901 - one exceptio ) server.stop() server = None - server = _start_configured_server( - _without_scheduled_processing(configured_server_settings), - recorder.directory / "configured-scheduler.db", - ) + server = _start_configured_server(_without_scheduled_processing(configured_server_settings)) asyncio.run( _verify_configured_restart( database=configured_settings.database, @@ -513,7 +504,7 @@ async def _run_journey( _require("passed" in proposal.outcome.lower(), "Codex Experience outcome did not report the verified pass") with recorder.scenario( - "APScheduler incubates an Experience Candidate that stays gated until Review approval", + "The durable Worker incubates an Experience Candidate that stays gated until Review approval", "api/experience-candidate.json", "api/experience-approved.json", ): @@ -863,7 +854,7 @@ async def _run_configured_journey( ) with recorder.scenario( - "configured LLM and APScheduler incubate one gated Experience Candidate", + "configured LLM and durable Worker incubate one gated Experience Candidate", "api/experience-source.json", "api/experience-candidate.json", "api/experience-approved.json", @@ -1975,14 +1966,13 @@ def _prepare_repositories( return repositories -def _start_server(database: Path, scheduler: Path) -> RunningServer: +def _start_server(database: Path) -> RunningServer: app = create_server_app( settings=ServerSettings( database=SQLiteConfig(url=f"sqlite+aiosqlite:///{database}"), runtime=RuntimeConfig(experience_schedule_seconds=0.05), mcp=McpConfig(enabled=False), ), - scheduler_path=scheduler, experience_pipeline=TaskOutcomeExperiencePipeline(), ) listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -2002,8 +1992,8 @@ def _start_server(database: Path, scheduler: Path) -> RunningServer: return RunningServer(server=server, thread=thread, listener=listener, base_url=f"http://127.0.0.1:{port}") -def _start_configured_server(settings: ServerSettings, scheduler_path: Path) -> RunningServer: - app = create_server_app(settings=settings, scheduler_path=scheduler_path) +def _start_configured_server(settings: ServerSettings) -> RunningServer: + app = create_server_app(settings=settings) listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) listener.bind(("127.0.0.1", 0)) diff --git a/tests/e2e/test_distributed_work_oceanbase.py b/tests/e2e/test_distributed_work_oceanbase.py new file mode 100644 index 000000000..094ab716a --- /dev/null +++ b/tests/e2e/test_distributed_work_oceanbase.py @@ -0,0 +1,231 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +import multiprocessing +import os +from typing import Any +from uuid import uuid4 + +import httpx +import pytest +from fastmcp import Client +from fastmcp.client.transports import StreamableHttpTransport +from pydantic import SecretStr +from sqlalchemy import delete + +from powercontext.builtin.persistence.migration import migrate_database +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile +from powercontext.builtin.persistence.tables import ( + BUILTIN_TABLES, + WORK_ATTEMPTS_TABLE, + WORK_ITEMS_TABLE, + WORK_KEYS_TABLE, + WORK_LANES_TABLE, +) +from powercontext.builtin.persistence.work import WorkRepository, WorkSpec +from powercontext.builtin.runtime.config import DeploymentConfig, HandoffReportConfig +from powercontext.server.cli import _migrate_configured_database +from powercontext.server.factory import create_server_app +from powercontext.server.settings import DashboardConfig, McpConfig, MetricsConfig, ServerSettings + +_OCEANBASE_URL = os.environ.get("POWERCONTEXT_TEST_OCEANBASE_URL") +_KIND = "test.distributed.claim" + + +class _RoundRobinASGITransport(httpx.AsyncBaseTransport): + def __init__(self, *apps) -> None: + self._transports = tuple(httpx.ASGITransport(app=app) for app in apps) + self._next = 0 + self.hits: list[int] = [] + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + index = self._next % len(self._transports) + self._next += 1 + self.hits.append(index) + return await self._transports[index].handle_async_request(request) + + async def aclose(self) -> None: + for transport in self._transports: + await transport.aclose() + + +def _api_settings(url: str, instance_id: str, *, mcp: bool) -> ServerSettings: + return ServerSettings( + database=OceanBaseConfig(url=SecretStr(url)), + deployment=DeploymentConfig(mode="distributed", role="api", id=instance_id), + dashboard=DashboardConfig(enabled=False), + handoff_report=HandoffReportConfig(enabled=False), + mcp=McpConfig(enabled=mcp), + metrics=MetricsConfig(enabled=False), + ) + + +def _claim_process(url: str, gate: Any, results: Any, worker_id: str) -> None: + async def claim() -> None: + config = OceanBaseConfig(url=SecretStr(url)) + async with OceanBaseProfile.open(config, tables=BUILTIN_TABLES, create_schema=False) as profile: + async with profile.database.transaction() as connection: + claims = await WorkRepository().claim( + connection, + worker_id=worker_id, + supported={_KIND: frozenset({1})}, + lease_seconds=120, + limit=1, + ) + results.put((worker_id, len(claims))) + + if not gate.wait(timeout=15): + results.put((worker_id, -1)) + return + asyncio.run(claim()) + + +@pytest.mark.skipif( + _OCEANBASE_URL is None, + reason="set POWERCONTEXT_TEST_OCEANBASE_URL to a dedicated OceanBase MySQL-mode test database", +) +def test_oceanbase_rc_allows_only_one_claim_across_worker_processes() -> None: + assert _OCEANBASE_URL is not None + marker = uuid4().hex + lane_key = marker.ljust(64, "0") + logical_key = marker.ljust(64, "1") + config = OceanBaseConfig(url=SecretStr(_OCEANBASE_URL)) + + async def arrange() -> str: + async with OceanBaseProfile.open(config, tables=BUILTIN_TABLES, create_schema=False) as profile: + await migrate_database(profile.database) + async with profile.database.transaction() as connection: + enqueued = await WorkRepository().enqueue( + connection, + WorkSpec( + kind=_KIND, + payload_version=1, + scope_id=f"oceanbase-multiprocess:{marker}", + lane_key=lane_key, + logical_key=logical_key, + payload={}, + ), + ) + return enqueued.work.work_id + + work_id = asyncio.run(arrange()) + context = multiprocessing.get_context("spawn") + gate = context.Event() + results = context.Queue() + processes = [ + context.Process(target=_claim_process, args=(_OCEANBASE_URL, gate, results, f"worker-{index}")) + for index in range(2) + ] + try: + for process in processes: + process.start() + gate.set() + outcomes = [results.get(timeout=30) for _ in processes] + for process in processes: + process.join(timeout=30) + assert process.exitcode == 0 + assert sorted(count for _, count in outcomes) == [0, 1] + finally: + for process in processes: + if process.is_alive(): + process.terminate() + process.join(timeout=5) + + async def cleanup() -> None: + async with ( + OceanBaseProfile.open(config, tables=BUILTIN_TABLES, create_schema=False) as profile, + profile.database.transaction() as connection, + ): + await connection.execute(delete(WORK_ATTEMPTS_TABLE).where(WORK_ATTEMPTS_TABLE.c.work_id == work_id)) + await connection.execute(delete(WORK_KEYS_TABLE).where(WORK_KEYS_TABLE.c.work_id == work_id)) + await connection.execute(delete(WORK_ITEMS_TABLE).where(WORK_ITEMS_TABLE.c.work_id == work_id)) + await connection.execute(delete(WORK_LANES_TABLE).where(WORK_LANES_TABLE.c.lane_key == lane_key)) + + asyncio.run(cleanup()) + + +@pytest.mark.skipif( + _OCEANBASE_URL is None, + reason="set POWERCONTEXT_TEST_OCEANBASE_URL to a dedicated OceanBase MySQL-mode test database", +) +def test_two_api_replicas_share_http_operations_and_stateless_mcp_without_affinity() -> None: + assert _OCEANBASE_URL is not None + marker = uuid4().hex + first_settings = _api_settings(_OCEANBASE_URL, f"api-a-{marker}", mcp=True) + second_settings = _api_settings(_OCEANBASE_URL, f"api-b-{marker}", mcp=True) + + async def scenario() -> None: + await _migrate_configured_database(first_settings) + first = create_server_app(settings=first_settings) + second = create_server_app(settings=second_settings) + round_robin = _RoundRobinASGITransport(first, second) + + def create_http_client( + headers: dict[str, str] | None = None, + timeout: httpx.Timeout | None = None, + auth: httpx.Auth | None = None, + **_: object, + ) -> httpx.AsyncClient: + return httpx.AsyncClient( + transport=round_robin, + base_url="http://testserver", + headers=headers, + timeout=timeout, + auth=auth, + follow_redirects=True, + ) + + async with ( + first.router.lifespan_context(first), + second.router.lifespan_context(second), + httpx.AsyncClient(transport=httpx.ASGITransport(app=first), base_url="http://first") as first_http, + httpx.AsyncClient(transport=httpx.ASGITransport(app=second), base_url="http://second") as second_http, + ): + capabilities = await first_http.get("/v1/capabilities") + assert capabilities.status_code == 200 + assert capabilities.json()["memory_extraction"] is True + + scope_id = f"distributed-api:{marker}" + captured = await first_http.post( + "/v1/sources/content", + json={"scope_id": scope_id, "source_id": "one", "content": "durable reference"}, + ) + assert captured.status_code == 202 + submitted = await second_http.post( + "/v1/memory/flush", + headers={"Prefer": "respond-async"}, + json={"scope_id": scope_id}, + ) + assert submitted.status_code == 202 + operation_id = submitted.json()["operation_id"] + visible = await first_http.get(f"/v1/operations/{operation_id}") + assert visible.status_code == 200 + assert visible.json()["status"] == "queued" + + transport = StreamableHttpTransport( + "http://testserver/mcp/", + httpx_client_factory=create_http_client, + ) + async with Client(transport) as client: + tools = {tool.name for tool in await client.list_tools()} + assert "list_memory_entries" in tools + result = await client.call_tool("list_memory_entries", {"scope_id": scope_id}) + assert result.structured_content == {"memory": None, "entries": []} + + assert set(round_robin.hits) == {0, 1} + + asyncio.run(scenario()) diff --git a/tests/e2e/test_durable_memory_work.py b/tests/e2e/test_durable_memory_work.py new file mode 100644 index 000000000..ae114c791 --- /dev/null +++ b/tests/e2e/test_durable_memory_work.py @@ -0,0 +1,81 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio + +from powercontext.builtin.artifacts.memory import MemoryCandidateRequest, MemoryEntryInput +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.work import EnqueueResult, WorkRepository, WorkStatus +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts +from powercontext.builtin.runtime.config import WorkerConfig +from powercontext.builtin.runtime.models import MemoryFlushResult +from powercontext.builtin.runtime.work_handlers import MemoryWorkHandler, enqueue_memory_work +from powercontext.builtin.runtime.worker import DurableWorker +from powercontext.builtin.sources import ContentCapture, ContentSource + + +class _ContentCandidatePipeline: + async def extract(self, request: MemoryCandidateRequest, /) -> tuple[MemoryEntryInput, ...]: + return tuple( + MemoryEntryInput(kind="fact", text=source.content, sources=(source,)) + for source in request.sources + if isinstance(source, ContentSource) + ) + + +def test_manual_and_scheduled_discovery_join_one_memory_window_and_commit_once(tmp_path) -> None: + async def scenario() -> None: + config = BuiltinConfig( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'durable.db'}"), + ) + async with open_builtin_contexts(config, candidate_pipeline=_ContentCandidatePipeline()) as contexts: + context = await contexts.get("project") + await context.sources.capture(ContentCapture(source_id="one", content="First durable fact.")) + await context.sources.capture(ContentCapture(source_id="two", content="Second durable fact.")) + + manual = await enqueue_memory_work(contexts, "project", limit=100, max_attempts=5) + scheduled = await enqueue_memory_work(contexts, "project", limit=100, max_attempts=5) + assert isinstance(manual, EnqueueResult) + assert isinstance(scheduled, EnqueueResult) + assert manual.created is True + assert scheduled.created is False + assert manual.work.work_id == scheduled.work.work_id + + worker = DurableWorker( + database=contexts.database, + worker_id="worker-a", + handlers=(MemoryWorkHandler(contexts),), + config=WorkerConfig(concurrency=1), + ) + assert await worker.run_once() == 1 + + async with contexts.database.transaction() as connection: + stored = await WorkRepository().get(connection, manual.work.work_id) + assert stored.status is WorkStatus.SUCCEEDED + assert stored.result_payload is not None + assert stored.result_payload["current_cursor"] == 2 + + memory = await context.artifacts.memory.head("memory") + assert memory.revision == 1 + assert {entry.text for entry in await context.artifacts.memory.entries(memory)} == { + "First durable fact.", + "Second durable fact.", + } + idle = await enqueue_memory_work(contexts, "project", limit=100, max_attempts=5) + assert isinstance(idle, MemoryFlushResult) + assert idle.current_cursor == 2 + + asyncio.run(scenario()) diff --git a/tests/e2e/test_observability.py b/tests/e2e/test_observability.py index d2ea729a1..8eed010cb 100644 --- a/tests/e2e/test_observability.py +++ b/tests/e2e/test_observability.py @@ -51,6 +51,7 @@ from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime import BuiltinConfig, RememberMemoryRequest, open_builtin_runtime from powercontext.builtin.runtime.config import InferenceConfig, RuntimeConfig +from powercontext.builtin.runtime.work_handlers import EXPERIENCE_WORK_KIND, MEMORY_WORK_KIND from powercontext.builtin.scope import ScopeDraft from powercontext.errors import RevisionConflictError from powercontext.server.factory import create_server_app @@ -135,6 +136,22 @@ "powercontext.background.source_count", "powercontext.background.candidate_count", }, + "work.commit": { + "powercontext.operation.name", + "powercontext.operation.unit", + "powercontext.operation.outcome", + "powercontext.work.kind", + "powercontext.work.payload_version", + }, + "work.execute": { + "powercontext.operation.name", + "powercontext.operation.unit", + "powercontext.operation.outcome", + "powercontext.work.kind", + "powercontext.work.payload_version", + "powercontext.work.attempt", + "powercontext.work.recovery_generation", + }, } _VECTOR_PROFILE = EmbeddingProfile( @@ -381,21 +398,24 @@ def test_inference_spans_join_the_operation_trace_only_when_instrumented(monkeyp transport = next(span for span in instrumented if span.name == "HTTP flush_memory") application = next(span for span in instrumented if span.name == "powercontext flush_memory") - flush_stage = next(span for span in instrumented if span.name == "memory.flush") + enqueue = _only_child(instrumented, application, "work.enqueue") + execute = _work_span(instrumented, "work.execute", MEMORY_WORK_KIND, outcome="succeeded") invoke_agent = next(span for span in instrumented if span.name == "invoke_agent memory_extraction") chat = next(span for span in instrumented if span.name.startswith("chat ")) + commit = _only_child(instrumented, execute, "work.commit") assert application.parent is not None assert application.parent.span_id == transport.context.span_id - assert flush_stage.parent is not None - assert flush_stage.parent.span_id == application.context.span_id + assert enqueue.context.trace_id == application.context.trace_id + assert execute.parent is None + assert execute.context.trace_id != application.context.trace_id assert invoke_agent.parent is not None - assert invoke_agent.parent.span_id == flush_stage.context.span_id + assert invoke_agent.parent.span_id == execute.context.span_id assert chat.parent is not None assert chat.parent.span_id == invoke_agent.context.span_id - assert {span.context.trace_id for span in (transport, application, flush_stage, invoke_agent, chat)} == { - transport.context.trace_id - } + assert commit.context.trace_id == execute.context.trace_id + assert {span.context.trace_id for span in (transport, application, enqueue)} == {transport.context.trace_id} + assert {span.context.trace_id for span in (execute, invoke_agent, chat, commit)} == {execute.context.trace_id} assert not any(_is_inference_span(span) for span in uninstrumented) @@ -666,7 +686,7 @@ async def scenario() -> None: (span.attributes or {})["powercontext.scope.lock.contended"] for span in spans if span.name == "scope.lock" ] == [False, True, False, False] # Every wait span succeeds, including the conflicting write's: the span closes before the critical section runs. - for span in spans: + for span in (span for span in spans if span.name == "scope.lock"): assert (span.attributes or {}).get("powercontext.operation.outcome") == "success" allowed_keys = _STAGE_ATTRIBUTE_KEYS.get(span.name) assert allowed_keys is None or (span.attributes or {}).keys() <= allowed_keys @@ -736,12 +756,10 @@ def test_scheduled_source_window_starts_an_independent_trace_root(tmp_path) -> N json={"scope_id": scope_id, "source_id": "task-1", "content": content}, ) assert captured.status_code == 202 - root = _wait_for_named_span(exporter, "scheduled.process_source_window", outcome="success") + root = _wait_for_work_span(exporter, MEMORY_WORK_KIND) spans = list(exporter.get_finished_spans()) - flush = _only_child(spans, root, "memory.flush") - _assert_scheduled_background_trace(root, flush, spans, scope_id=scope_id, content=content) - assert (root.attributes or {})["powercontext.background.source_count"] == 1 - assert (flush.attributes or {})["powercontext.memory.flush.source_count"] == 1 + commit = _only_child(spans, root, "work.commit") + _assert_scheduled_background_trace(root, commit, spans, scope_id=scope_id, content=content) def test_scheduled_experience_starts_an_independent_trace_root(tmp_path) -> None: @@ -767,14 +785,10 @@ def test_scheduled_experience_starts_an_independent_trace_root(tmp_path) -> None json={"scope_id": scope_id, "source_id": "task-1", "content": content}, ) assert captured.status_code == 202 - root = _wait_for_named_span(exporter, "scheduled.incubate_experience_candidates", outcome="success") + root = _wait_for_work_span(exporter, EXPERIENCE_WORK_KIND) spans = list(exporter.get_finished_spans()) - incubation = _only_child(spans, root, "experience.incubation") - _assert_scheduled_background_trace(root, incubation, spans, scope_id=scope_id, content=content) - assert (root.attributes or {})["powercontext.background.source_count"] == 1 - assert (root.attributes or {})["powercontext.background.candidate_count"] == 0 - assert (incubation.attributes or {})["powercontext.experience.incubation.source_count"] == 1 - assert (incubation.attributes or {})["powercontext.experience.incubation.candidate_count"] == 0 + commit = _only_child(spans, root, "work.commit") + _assert_scheduled_background_trace(root, commit, spans, scope_id=scope_id, content=content) def test_vector_search_exports_embedding_under_memory_search_without_recording_text(monkeypatch, tmp_path) -> None: @@ -873,9 +887,15 @@ def test_injected_always_on_embedding_skips_readiness_but_traces_vector_search(m tracing=tracing, ) with TestClient(app) as client: + exporter.clear() readiness = client.get("/health/ready") readiness_spans = list(exporter.get_finished_spans()) - assert not [span for span in readiness_spans if span.parent is None] + root_span_names = [ + span.name + for span in readiness_spans + if span.parent is None and (span.attributes or {}).get("powercontext.operation.unit") != "background" + ] + assert not root_span_names, root_span_names assert not any(_is_inference_span(span) for span in readiness_spans) scope_id = _get_default_scope_id(client) @@ -974,6 +994,32 @@ def _wait_for_named_span( raise AssertionError(f"{name} span was not exported") # noqa: TRY003 +def _wait_for_work_span( + exporter: InMemorySpanExporter, + kind: str, + *, + timeout: float = 3, +) -> ReadableSpan: + deadline = monotonic() + timeout + while monotonic() < deadline: + spans = list(exporter.get_finished_spans()) + try: + return _work_span(spans, "work.execute", kind, outcome="succeeded") + except StopIteration: + sleep(0.02) + raise AssertionError(f"work.execute span for {kind} was not exported") # noqa: TRY003 + + +def _work_span(spans: list[ReadableSpan], name: str, kind: str, *, outcome: str) -> ReadableSpan: + return next( + span + for span in spans + if span.name == name + and (span.attributes or {}).get("powercontext.work.kind") == kind + and (span.attributes or {}).get("powercontext.operation.outcome") == outcome + ) + + def _assert_stage_attribute_keys(span: ReadableSpan) -> None: allowed_keys = _STAGE_ATTRIBUTE_KEYS[span.name] attributes = dict(span.attributes or {}) @@ -992,7 +1038,7 @@ def _assert_scheduled_background_trace( assert root.parent is None assert stage.parent is not None and stage.parent.span_id == root.context.span_id assert (root.attributes or {})["powercontext.operation.unit"] == "background" - assert (root.attributes or {})["powercontext.operation.outcome"] == "success" + assert (root.attributes or {})["powercontext.operation.outcome"] == "succeeded" assert (stage.attributes or {})["powercontext.operation.unit"] == "stage" _assert_stage_attribute_keys(root) _assert_stage_attribute_keys(stage) diff --git a/tests/e2e/test_operation_api.py b/tests/e2e/test_operation_api.py new file mode 100644 index 000000000..3bb282c52 --- /dev/null +++ b/tests/e2e/test_operation_api.py @@ -0,0 +1,100 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from typing import Any, cast + +import httpx + +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.work import WorkRepository, WorkSpec +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts +from powercontext.builtin.runtime.config import OperationsConfig, WorkerConfig +from powercontext.builtin.runtime.operations import OperationManager +from powercontext.builtin.sources import ContentCapture +from powercontext.server.app import ServerApplication, create_app + + +def test_operation_http_api_is_stateless_and_uses_optimistic_mutations(tmp_path) -> None: + async def scenario() -> None: + config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'operations.db'}")) + async with open_builtin_contexts(config) as contexts: + context = await contexts.get("project") + await context.sources.capture(ContentCapture(source_id="one", content="queued work")) + manager = OperationManager( + contexts=contexts, + operations=OperationsConfig(), + worker=WorkerConfig(), + local_worker=None, + payload_version=1, + memory_window_limit=100, + ) + repository = WorkRepository() + async with contexts.database.transaction() as connection: + internal = await repository.enqueue( + connection, + WorkSpec( + kind="powercontext.maintenance.operations", + payload_version=1, + scope_id="system:operations", + lane_key="a" * 64, + logical_key="b" * 64, + payload={}, + ), + ) + app = create_app(application=cast(ServerApplication, cast(Any, object()))) + app.state.operation_manager = manager + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://testserver", + ) as client: + accepted = await client.post( + "/v1/memory/flush", + headers={"Prefer": "respond-async"}, + json={"scope_id": "project"}, + ) + assert accepted.status_code == 202 + operation_id = accepted.json()["operation_id"] + assert accepted.headers["location"] == f"/v1/operations/{operation_id}" + + stored = await client.get(f"/v1/operations/{operation_id}") + assert stored.status_code == 200 + assert stored.json()["status"] == "queued" + + listed = await client.get("/v1/operations", params={"scope_id": "project", "limit": 1}) + assert listed.status_code == 200 + assert [item["operation_id"] for item in listed.json()["items"]] == [operation_id] + + all_public = await client.get("/v1/operations") + assert [item["operation_id"] for item in all_public.json()["items"]] == [operation_id] + hidden = await client.get(f"/v1/operations/{internal.work.work_id}") + assert hidden.status_code == 404 + + cancelled = await client.post( + f"/v1/operations/{operation_id}/cancel", + json={"expected_version": stored.json()["state_version"]}, + ) + assert cancelled.status_code == 200 + assert cancelled.json()["status"] == "cancelled" + + conflict = await client.post( + f"/v1/operations/{operation_id}/cancel", + json={"expected_version": stored.json()["state_version"]}, + ) + assert conflict.status_code == 409 + assert conflict.json()["error"]["code"] == "operation_conflict" + + asyncio.run(scenario()) diff --git a/tests/e2e/test_scheduled_experience_incubation.py b/tests/e2e/test_scheduled_experience_incubation.py index 6014222d7..5838757c8 100644 --- a/tests/e2e/test_scheduled_experience_incubation.py +++ b/tests/e2e/test_scheduled_experience_incubation.py @@ -53,14 +53,13 @@ async def incubate(self, sources: tuple[Source, ...], /) -> tuple[ExperienceCand ) -def _app(database: Path, scheduler: Path): +def _app(database: Path): return create_server_app( settings=ServerSettings( database=SQLiteConfig(url=f"sqlite+aiosqlite:///{database}"), runtime=RuntimeConfig(experience_schedule_seconds=0.02), mcp=McpConfig(enabled=False), ), - scheduler_path=scheduler, experience_pipeline=_TaskOutcomePipeline(), ) @@ -83,8 +82,7 @@ async def _pending_experience(client: PowerContextClient, scope_id: str): def test_scheduler_incubates_task_outcome_once_and_preserves_review_gating(tmp_path: Path) -> None: async def scenario() -> None: database = tmp_path / "powercontext.db" - scheduler = tmp_path / "scheduler.db" - app = _app(database, scheduler) + app = _app(database) async with ( app.router.lifespan_context(app), httpx.AsyncClient( @@ -115,7 +113,7 @@ async def scenario() -> None: assert candidates[0].result_artifact is None assert prepared.status == "empty" - restored = _app(database, scheduler) + restored = _app(database) async with ( restored.router.lifespan_context(restored), httpx.AsyncClient( diff --git a/tests/e2e/test_shared_rate_limit.py b/tests/e2e/test_shared_rate_limit.py new file mode 100644 index 000000000..6a3d81dab --- /dev/null +++ b/tests/e2e/test_shared_rate_limit.py @@ -0,0 +1,66 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from fastapi.testclient import TestClient +from pydantic import SecretStr + +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime.config import RateLimitConfig +from powercontext.server.factory import create_server_app +from powercontext.server.settings import BearerAuthConfig, McpConfig, ServerSettings + + +def test_shared_rate_limit_rejects_only_protected_requests(tmp_path) -> None: + app = create_server_app( + settings=ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'runtime.db'}"), + mcp=McpConfig(enabled=False), + rate_limit=RateLimitConfig(enabled=True, requests=1, window_seconds=60), + ) + ) + + with TestClient(app) as client: + first = client.get("/v1/capabilities") + rejected = client.get("/v1/capabilities") + health = client.get("/health/ready") + + assert first.status_code == 200 + assert rejected.status_code == 429 + assert rejected.headers["Retry-After"] + assert rejected.json()["error"]["code"] == "rate_limited" + assert health.status_code == 200 + + +def test_shared_rate_limit_skips_unauthenticated_receiver_routes(tmp_path) -> None: + app = create_server_app( + settings=ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'runtime.db'}"), + mcp=McpConfig(enabled=False), + auth=BearerAuthConfig(enabled=True, token=SecretStr("server-token")), + rate_limit=RateLimitConfig(enabled=True, requests=1, window_seconds=60), + ) + ) + paths = ( + "/v1/skill/remote/target/enroll", + "/v1/skill/remote/reconcile", + "/v1/skill/remote/package/download", + "/v1/skill/remote/receipt", + ) + + with TestClient(app, raise_server_exceptions=False) as client: + responses = [client.post(path, json={}) for path in paths] + + assert {response.status_code for response in responses} == {422} diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index c5802d8d7..5833c88eb 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -48,6 +48,8 @@ SupportState, ) +_TEST_MEMBER_TTL_SECONDS = 3 + pytestmark = [ pytest.mark.native_service, pytest.mark.skipif( @@ -81,6 +83,9 @@ def test_native_personal_service_lifecycle(tmp_path: Path) -> None: assert adapter.loaded_registration().state is ManagerOwnershipState.OWNED assert adapter.manager_state() is ManagerState.INACTIVE + if isinstance(adapter, WindowsTaskSchedulerAdapter): + # schtasks /End cannot run the Server shutdown hook that releases the lease. + time.sleep(_TEST_MEMBER_TTL_SECONDS + 1) adapter.start(reload_definition=False) restarted = _wait_for_status(controller) assert restarted.ok @@ -273,6 +278,8 @@ def _environment_file(tmp_path: Path) -> Path: f"POWERCONTEXT_HOME={data_dir}", f"POWERCONTEXT_SERVER_HTTP_PORT={_unused_loopback_port()}", "POWERCONTEXT_SERVER_DASHBOARD_ENABLED=false", + f"POWERCONTEXT_SERVER_COORDINATION_MEMBER_TTL_SECONDS={_TEST_MEMBER_TTL_SECONDS}", + "POWERCONTEXT_SERVER_COORDINATION_MEMBER_HEARTBEAT_SECONDS=1", "", )), encoding="utf-8", diff --git a/tests/test_api_contract.py b/tests/test_api_contract.py index 7bf60ea23..fd80ab8a9 100644 --- a/tests/test_api_contract.py +++ b/tests/test_api_contract.py @@ -32,6 +32,7 @@ CreateWorkContractRequest, ExternalSkillResolution, FinalizeHandoffRequest, + FlushMemoryResponse, GeneratedCandidateResponse, GenerateExperienceRequest, GenerateSkillRequest, @@ -46,6 +47,7 @@ ListExternalSkillsRequest, ListExternalSkillsResponse, ListMemoryEntriesRequest, + OperationAccepted, PrepareContextRequest, PreparedContext, PreparedHandoff, @@ -193,6 +195,14 @@ def test_capture_operation_declares_its_typed_accepted_exchange() -> None: assert CAPTURE_CONTENT_SOURCE.success_status == 202 +def test_flush_operation_exposes_every_declared_success_response() -> None: + assert FLUSH_MEMORY.success_statuses == (200, 202) + assert FLUSH_MEMORY.success_response_types == { + 200: FlushMemoryResponse, + 202: OperationAccepted, + } + + def test_source_observation_contract_uses_explicit_connector_scope_and_captured_values() -> None: contract = yaml.safe_load(CONTRACT_PATH.read_text()) schemas = contract["components"]["schemas"] diff --git a/tests/test_client.py b/tests/test_client.py index 2947b7296..dde374735 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -19,11 +19,18 @@ import pytest from pydantic import ValidationError -from powercontext.client import InvalidResponseError, PowerContextClient, ServerResponseError, TransportError +from powercontext.client import ( + InvalidResponseError, + OperationPendingError, + PowerContextClient, + ServerResponseError, + TransportError, +) from powercontext.client.settings import ClientSettings from powercontext.http import ( CaptureContentSourceRequest, ExactScopeSelection, + FlushMemoryRequest, GetHandoffReportRequest, ReportFormat, ScopeId, @@ -32,6 +39,106 @@ ) +def test_flush_memory_follows_a_durable_operation_without_sticky_transport() -> None: + async def scenario() -> None: + operation_id = "4dfaf8c1-30e4-40c8-a37c-2b845dd8150e" + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.url.path == "/v1/memory/flush": + return httpx.Response( + 202, + json={ + "operation_id": operation_id, + "status": "queued", + "status_url": f"/v1/operations/{operation_id}", + }, + ) + return httpx.Response( + 200, + json={ + "operation_id": operation_id, + "kind": "memory_flush", + "scope_id": "project", + "status": "succeeded", + "attempt_count": 1, + "state_version": 3, + "created_at": "2026-09-03T00:00:00Z", + "updated_at": "2026-09-03T00:00:01Z", + "completed_at": "2026-09-03T00:00:01Z", + "result": { + "type": "memory_flush", + "previous_cursor": 0, + "high_watermark": 2, + "current_cursor": 2, + "processed_source_count": 2, + "memory": None, + }, + "error": None, + }, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + client = PowerContextClient( + "https://memory.example", + http_client=http_client, + operation_poll_seconds=0.001, + ) + result = await client.flush_memory(FlushMemoryRequest(scope_id="project")) + + assert result.current_cursor == 2 + assert requests[0].headers["Prefer"] == "respond-async" + assert requests[1].url.path == f"/v1/operations/{operation_id}" + + asyncio.run(scenario()) + + +def test_flush_memory_reports_the_pending_operation_at_the_client_deadline() -> None: + async def scenario() -> None: + operation_id = "4dfaf8c1-30e4-40c8-a37c-2b845dd8150e" + + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path == "/v1/memory/flush": + return httpx.Response( + 202, + json={ + "operation_id": operation_id, + "status": "running", + "status_url": f"/v1/operations/{operation_id}", + }, + ) + return httpx.Response( + 200, + json={ + "operation_id": operation_id, + "kind": "memory_flush", + "scope_id": "project", + "status": "running", + "attempt_count": 1, + "state_version": 2, + "created_at": "2026-09-03T00:00:00Z", + "updated_at": "2026-09-03T00:00:00Z", + "completed_at": None, + "result": None, + "error": None, + }, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + client = PowerContextClient( + "https://memory.example", + http_client=http_client, + operation_timeout=0.001, + operation_poll_seconds=0.001, + ) + with pytest.raises(OperationPendingError) as caught: + await client.flush_memory(FlushMemoryRequest(scope_id="project")) + assert caught.value.operation_id == operation_id + + asyncio.run(scenario()) + + def test_client_rejects_an_undeclared_success_status() -> None: async def scenario() -> None: response = httpx.Response( diff --git a/tests/test_config_cli.py b/tests/test_config_cli.py index 31921e899..81c3e3b04 100644 --- a/tests/test_config_cli.py +++ b/tests/test_config_cli.py @@ -143,6 +143,50 @@ def test_validate_accepts_minimal_server_environment_without_inference_models(tm assert settings.http.port == 8888 +@pytest.mark.parametrize("role", ("api", "scheduler")) +def test_validate_accepts_non_executing_distributed_roles_without_inference_models( + role: str, + tmp_path: Path, +) -> None: + environment = tmp_path / "server.env" + environment.write_text( + "\n".join(( + "POWERCONTEXT_SERVER_DATABASE_KIND=oceanbase", + "POWERCONTEXT_SERVER_DATABASE_URL=mysql+aoceanbase://root@127.0.0.1:2881/powercontext?charset=utf8mb4", + "POWERCONTEXT_SERVER_DEPLOYMENT_MODE=distributed", + f"POWERCONTEXT_SERVER_DEPLOYMENT_ROLE={role}", + "POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS=60", + "POWERCONTEXT_SERVER_RUNTIME_EXPERIENCE_SCHEDULE_SECONDS=60", + "", + )), + encoding="utf-8", + ) + + result = CliRunner().invoke(config_cli.app, ["validate", "--env-file", str(environment)]) + + assert result.exit_code == 0 + assert "Configuration is valid" in result.output + + +def test_validate_rejects_distributed_worker_without_inference_model(tmp_path: Path) -> None: + environment = tmp_path / "server.env" + environment.write_text( + "\n".join(( + "POWERCONTEXT_SERVER_DATABASE_KIND=oceanbase", + "POWERCONTEXT_SERVER_DATABASE_URL=mysql+aoceanbase://root@127.0.0.1:2881/powercontext?charset=utf8mb4", + "POWERCONTEXT_SERVER_DEPLOYMENT_MODE=distributed", + "POWERCONTEXT_SERVER_DEPLOYMENT_ROLE=worker", + "", + )), + encoding="utf-8", + ) + + result = CliRunner().invoke(config_cli.app, ["validate", "--env-file", str(environment)]) + + assert result.exit_code == 2 + assert "built-in runtime cannot be configured" in result.output + + @pytest.mark.parametrize( "runtime_setting", ( diff --git a/tests/test_docker_contract.py b/tests/test_docker_contract.py index 0df97467a..281276471 100644 --- a/tests/test_docker_contract.py +++ b/tests/test_docker_contract.py @@ -29,12 +29,14 @@ from unittest.mock import Mock import pytest +import yaml from typer.testing import CliRunner from powercontext.cli.app import create_cli from powercontext.server.cli import app as server_app _DOCKERFILE = Path(__file__).resolve().parent.parent / "docker" / "Dockerfile" +_DISTRIBUTED_COMPOSE = _DOCKERFILE.with_name("compose.distributed.yaml") _SERVER_ENV_PATTERN = re.compile(r"(POWERCONTEXT_SERVER_[A-Z0-9_]+)=([^\s\\]+)") @@ -69,3 +71,29 @@ def test_documented_container_invocation_starts_out_of_the_box(monkeypatch: pyte assert result.exit_code == 0, result.output run_server.assert_called_once() assert run_server.call_args.kwargs["host"] == "0.0.0.0" # noqa: S104 - the image's documented bind. + + +def test_distributed_compose_separates_roles_and_model_credentials() -> None: + document = yaml.safe_load(_DISTRIBUTED_COMPOSE.read_text(encoding="utf-8")) + services = document["services"] + assert set(services) == {"migrate", "api-a", "api-b", "scheduler-a", "scheduler-b", "worker-a", "worker-b"} + + assert services["migrate"]["command"] == ["server", "migrate"] + assert services["migrate"]["environment"]["POWERCONTEXT_SERVER_DEPLOYMENT_ROLE"] == "api" + assert services["migrate"]["environment"]["POWERCONTEXT_SERVER_DEPLOYMENT_ID"] == "migrator" + for name, service in services.items(): + environment = service["environment"] + assert environment["POWERCONTEXT_SERVER_DATABASE_KIND"] == "oceanbase" + assert environment["POWERCONTEXT_SERVER_DEPLOYMENT_MODE"] == "distributed" + if name == "migrate": + continue + role = name.split("-", 1)[0] + assert environment["POWERCONTEXT_SERVER_DEPLOYMENT_ROLE"] == role + assert environment["POWERCONTEXT_SERVER_DEPLOYMENT_ID"] == name + if role in {"scheduler", "worker"}: + assert "ports" not in service + assert environment["POWERCONTEXT_SERVER_DASHBOARD_ENABLED"] == "false" + assert environment["POWERCONTEXT_SERVER_MCP_ENABLED"] == "false" + assert ("POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL" in environment) is (role == "worker") + assert ("OPENAI_API_KEY" in environment) is (role == "worker") + assert ("ANTHROPIC_API_KEY" in environment) is (role == "worker") diff --git a/tests/test_server.py b/tests/test_server.py index ab19ba2f5..eaf4c3c27 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -17,6 +17,7 @@ import os import re import shlex +import sqlite3 from datetime import datetime, timedelta from pathlib import Path @@ -330,7 +331,7 @@ def test_server_settings_reject_custom_embedded_seekdb_database(tmp_path, monkey ServerSettings() -def test_server_scheduler_uses_the_powercontext_data_directory(tmp_path, monkeypatch) -> None: +def test_server_scheduler_uses_the_primary_work_ledger_without_a_sidecar(tmp_path, monkeypatch) -> None: data_dir = tmp_path / "powercontext-data" monkeypatch.setenv("POWERCONTEXT_HOME", str(data_dir)) app = create_server_app( @@ -342,7 +343,13 @@ def test_server_scheduler_uses_the_powercontext_data_directory(tmp_path, monkeyp ) with TestClient(app): - assert (data_dir / "scheduler.db").is_file() + database = data_dir / "powercontext.db" + assert database.is_file() + with sqlite3.connect(database) as connection: + tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'table'")} + assert "pc_scheduler_leases" in tables + assert "pc_work_items" in tables + assert not (data_dir / "scheduler.db").exists() def test_settings_load_bearer_authentication_without_exposing_token(monkeypatch) -> None: diff --git a/tests/test_server_metrics.py b/tests/test_server_metrics.py index 90a9fd0f4..6a310c643 100644 --- a/tests/test_server_metrics.py +++ b/tests/test_server_metrics.py @@ -26,9 +26,11 @@ from prometheus_client.parser import text_string_to_metric_families from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.work import WorkQueueStatistic, WorkStatus from powercontext.builtin.runtime import RuntimeConfig from powercontext.server.app import create_app from powercontext.server.factory import create_server_app +from powercontext.server.metrics import ServerMetrics from powercontext.server.settings import ( McpConfig, MetricsConfig, @@ -198,3 +200,34 @@ def observe_application(self, operation: str, outcome: str, started_at: float) - response = TestClient(create_app(metrics=metrics)).get("/v1/capabilities") assert response.status_code == 200 + + +def test_work_metrics_use_only_bounded_kind_status_and_outcome_labels() -> None: + metrics = ServerMetrics() + metrics.observe_work_enqueue("powercontext.memory.source-window", created=True) + metrics.observe_work_claim("powercontext.memory.source-window", latency_seconds=0.5) + metrics.observe_work_attempt( + "powercontext.memory.source-window", + outcome="retry_wait", + error_category="provider", + duration_seconds=1.25, + ) + metrics.observe_work_lease_expiry("powercontext.memory.source-window", outcome="retry_wait") + metrics.observe_scheduler_leadership(outcome="acquired") + metrics.set_work_queue(( + WorkQueueStatistic( + kind="powercontext.memory.source-window", + status=WorkStatus.QUEUED, + depth=2, + oldest_age_seconds=3.5, + ), + )) + metrics.set_runtime_members({"api": 2, "worker": 3}) + + rendered = metrics.render().decode() + assert 'powercontext_work_queue_depth{kind="powercontext.memory.source-window",status="queued"} 2.0' in rendered + assert 'powercontext_runtime_role_members{role="api"} 2.0' in rendered + assert 'powercontext_runtime_role_members{role="worker"} 3.0' in rendered + assert "scope_id" not in rendered + assert "work_id" not in rendered + assert "principal" not in rendered diff --git a/tests/test_server_migration_cli.py b/tests/test_server_migration_cli.py new file mode 100644 index 000000000..767ec9c7c --- /dev/null +++ b/tests/test_server_migration_cli.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import sqlite3 + +from typer.testing import CliRunner + +from powercontext.cli.app import create_cli +from powercontext.server.cli import app as server_app + + +def test_server_migrate_runs_the_packaged_forward_only_chain(tmp_path) -> None: + database = tmp_path / "runtime.db" + environment = tmp_path / "server.env" + environment.write_text( + "\n".join(( + "POWERCONTEXT_SERVER_DATABASE_KIND=sqlite", + f"POWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:///{database}", + )), + encoding="utf-8", + ) + + result = CliRunner().invoke( + create_cli((server_app,)), + ["server", "migrate", "--env-file", str(environment)], + ) + + assert result.exit_code == 0, result.output + assert "0003_scope_source_skill" in result.output + with sqlite3.connect(database) as connection: + revision = connection.execute("SELECT version_num FROM pc_schema_revisions").fetchone() + work_table = connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'pc_work_items'" + ).fetchone() + assert revision == ("0003_scope_source_skill",) + assert work_table == ("pc_work_items",) diff --git a/tests/test_server_roles.py b/tests/test_server_roles.py new file mode 100644 index 000000000..6f5d203f3 --- /dev/null +++ b/tests/test_server_roles.py @@ -0,0 +1,80 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from typing import Literal, cast + +from pydantic import SecretStr + +import powercontext.server.factory as factory +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from powercontext.builtin.runtime import BuiltinRuntime, RuntimeCapabilities +from powercontext.builtin.runtime.config import DeploymentConfig, HandoffReportConfig +from powercontext.server.settings import DashboardConfig, McpConfig, MetricsConfig, ServerSettings + + +def _distributed_settings( + role: Literal["api", "scheduler", "worker"], + *, + mcp: bool = True, + metrics: bool = True, +) -> ServerSettings: + return ServerSettings.model_validate({ + "database": OceanBaseConfig( + url=SecretStr("mysql+aoceanbase://root@127.0.0.1:2881/powercontext?charset=utf8mb4") + ), + "deployment": DeploymentConfig(mode="distributed", role=role, id=f"{role}-a"), + "dashboard": DashboardConfig(enabled=False), + "handoff_report": HandoffReportConfig(enabled=False), + "mcp": McpConfig(enabled=mcp), + "metrics": MetricsConfig(enabled=metrics), + }) + + +def test_scheduler_and_worker_roles_expose_only_the_management_plane() -> None: + for role in ("scheduler", "worker"): + app = factory.create_server_app(settings=_distributed_settings(role)) + paths = {getattr(route, "path", "") for route in app.routes} + assert paths == {"/health/live", "/health/ready", "/metrics"} + assert set(app.openapi()["paths"]) == {"/health/live", "/health/ready"} + + +def test_distributed_api_mounts_stateless_mcp(monkeypatch) -> None: + captured: dict[str, object] = {} + + def capture_mount(_app, **kwargs) -> None: + captured.update(kwargs) + + monkeypatch.setattr(factory, "mount_mcp", capture_mount) + app = factory.create_server_app(settings=_distributed_settings("api", metrics=False)) + + assert any(getattr(route, "path", None) == "/v1/operations" for route in app.routes) + assert captured["stateless_http"] is True + + +def test_distributed_api_advertises_worker_backed_memory_extraction() -> None: + class RuntimeWithoutLocalInference: + async def capabilities(self) -> RuntimeCapabilities: + return RuntimeCapabilities(memory_extraction=False, memory_search_modes=()) + + capabilities = asyncio.run( + factory._server_capabilities( + cast(BuiltinRuntime, RuntimeWithoutLocalInference()), + accepts_distributed_memory_work=True, + ) + ) + + assert capabilities.memory_extraction is True diff --git a/tests/test_server_tracing.py b/tests/test_server_tracing.py index 7d15e24a8..a976d8f88 100644 --- a/tests/test_server_tracing.py +++ b/tests/test_server_tracing.py @@ -27,6 +27,7 @@ from opentelemetry.trace import SpanKind, StatusCode from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime.protocols import RuntimeTraceContext from powercontext.client import PowerContextClient from powercontext.server.factory import create_server_app from powercontext.server.settings import ( @@ -278,6 +279,27 @@ def test_background_stage_starts_a_fresh_trace_outside_an_ambient_span() -> None assert attributes["powercontext.background.source_count"] == 0 +def test_background_stage_links_a_recovered_attempt_without_parenting_it() -> None: + tracing, exporter = _tracing() + + with tracing.background("work.execute", operation="work.execute", attributes={}) as first: + context = first.trace_context + assert context is not None + with tracing.background( + "work.execute", + operation="work.execute", + attributes={}, + links=(RuntimeTraceContext(trace_id=context.trace_id, span_id=context.span_id),), + ): + pass + + first_span, recovered_span = exporter.get_finished_spans() + assert recovered_span.parent is None + assert len(recovered_span.links) == 1 + assert recovered_span.links[0].context.trace_id == first_span.context.trace_id + assert recovered_span.links[0].context.span_id == first_span.context.span_id + + def test_background_isolates_child_spans_when_root_start_fails(monkeypatch) -> None: tracing, exporter = _tracing() ambient = tracing.start_span("HTTP flush_memory", kind=SpanKind.SERVER, attributes={}) diff --git a/uv.lock b/uv.lock index 7d9a02b87..9f167a7af 100644 --- a/uv.lock +++ b/uv.lock @@ -40,6 +40,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/b7/e3bf5133d697a08128598c8d0abc5e16377b51465a33756de24fa7dee953/aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb", size = 17405, upload-time = "2025-12-23T19:25:42.139Z" }, ] +[[package]] +name = "alembic" +version = "1.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/2b/e4153978368de59918115c9e01d3ebf58a558a7285efa7e960c383c4b59a/alembic-1.19.1.tar.gz", hash = "sha256:e0fca0518118c78acc493e31bcb5402f190057aaf6df8b5b95ce94c4789cf648", size = 2070816, upload-time = "2026-08-08T16:32:01.565Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/89/e62cc37b69ad357cc8ecd6e7367f5245f523d3cbb338a66197212bdf6749/alembic-1.19.1-py3-none-any.whl", hash = "sha256:b39018cb3d9413a19cbd54cf3c02ad33998641f0538eb77413a488a21c3e14be", size = 265946, upload-time = "2026-08-08T16:32:03.153Z" }, +] + [[package]] name = "annotated-doc" version = "0.0.4" @@ -90,18 +104,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] -[[package]] -name = "apscheduler" -version = "3.11.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tzlocal" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8c/6b/eeff360196bb20b312c9e762a820fd1b2c6d809466c755ef57863478e454/apscheduler-3.11.3.tar.gz", hash = "sha256:cd2fcc9330039a81a5893472ad49facf23a6d5604cbe1d918c835c6de7834d5a", size = 110312, upload-time = "2026-06-28T19:39:22.493Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/c9/8638db32514dbb9157b3d82680c6faea89283523edf9ed2415ea3884f2ae/apscheduler-3.11.3-py3-none-any.whl", hash = "sha256:bbeb2ec02d23d3c06a6c07ed7f0f3939ada6680eb121fae809a69bb42c537a30", size = 66024, upload-time = "2026-06-28T19:39:20.982Z" }, -] - [[package]] name = "argcomplete" version = "3.7.0" @@ -1423,6 +1425,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/26/22/598f988f293ce391e88baddeeaa1acaea625822653193935301e7149b5c6/logfire_api-4.38.0-py3-none-any.whl", hash = "sha256:6c0612d9200727b64b42eba6068d0d6afd45308f4b844ffd6290c2d33aa6a6f6", size = 140109, upload-time = "2026-07-20T12:15:31.973Z" }, ] +[[package]] +name = "mako" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2a/12/b5fa2353e2754cd67fb9f83793fa48ff42c213a5da7e719869d2301f6ab8/mako-1.4.1.tar.gz", hash = "sha256:d7904710b662996425a21627710c4777c45053146942cf8a7aebf757c92b8c27", size = 410165, upload-time = "2026-08-05T06:10:56.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/54/12ed58d458474aaab5c3d180173e745a4fe131bb330370596876d19ff60f/mako-1.4.1-py3-none-any.whl", hash = "sha256:a359d9a94a541213958742b2698d0a7757bb83551767bc468a74b9905aba9617", size = 80010, upload-time = "2026-08-05T06:10:58.248Z" }, +] + [[package]] name = "markdown" version = "3.10.2" @@ -2088,7 +2102,7 @@ dependencies = [ [package.optional-dependencies] builtin = [ { name = "aiosqlite" }, - { name = "apscheduler" }, + { name = "alembic" }, { name = "jsonschema" }, { name = "packaging" }, { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, @@ -2117,7 +2131,7 @@ client = [ ] seekdb = [ { name = "aiosqlite" }, - { name = "apscheduler" }, + { name = "alembic" }, { name = "jsonschema" }, { name = "packaging" }, { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, @@ -2130,7 +2144,7 @@ seekdb = [ ] server = [ { name = "aiosqlite" }, - { name = "apscheduler" }, + { name = "alembic" }, { name = "fastapi" }, { name = "fastmcp" }, { name = "jinja2" }, @@ -2179,9 +2193,9 @@ requires-dist = [ { name = "aiosqlite", marker = "extra == 'builtin'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'seekdb'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'server'", specifier = ">=0.22,<1" }, - { name = "apscheduler", marker = "extra == 'builtin'", specifier = ">=3.11,<4" }, - { name = "apscheduler", marker = "extra == 'seekdb'", specifier = ">=3.11,<4" }, - { name = "apscheduler", marker = "extra == 'server'", specifier = ">=3.11,<4" }, + { name = "alembic", marker = "extra == 'builtin'", specifier = ">=1.14,<2" }, + { name = "alembic", marker = "extra == 'seekdb'", specifier = ">=1.14,<2" }, + { name = "alembic", marker = "extra == 'server'", specifier = ">=1.14,<2" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115,<1" }, { name = "fastmcp", marker = "extra == 'server'", specifier = ">=3.4,<4" }, { name = "httpx", extras = ["socks"], marker = "extra == 'cli'", specifier = ">=0.28,<1" }, @@ -3625,27 +3639,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] -[[package]] -name = "tzdata" -version = "2026.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, -] - -[[package]] -name = "tzlocal" -version = "5.4.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tzdata", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/81/5b/879b2f932adfa7a053c360d50bc896c977fa6426109185f7c12ebdd0cb9d/tzlocal-5.4.4.tar.gz", hash = "sha256:8dbb8660838688a7b6ba4fed31d18dedf842afb4d47ca050d6d891c2c15f3be4", size = 31170, upload-time = "2026-06-29T08:03:40.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/a4/017a7a6cbe387d961a688ec31364ae60a5c4e22c96ae9921b79a947c855d/tzlocal-5.4.4-py3-none-any.whl", hash = "sha256:aae09f0126a8a86fa736be266eb4a471380d26a0de3bc14844e7821fee3e2a15", size = 18115, upload-time = "2026-06-29T08:03:38.666Z" }, -] - [[package]] name = "uncalled-for" version = "0.3.2" diff --git a/zensical.toml b/zensical.toml index d1ee08e22..cad6e783d 100644 --- a/zensical.toml +++ b/zensical.toml @@ -78,6 +78,7 @@ nav = [ { "RFCs & Meetings" = [ { "RFCs" = [ { "Overview" = "en/rfcs/README.md" }, + { "1430 Distributed Server Workers" = "en/rfcs/1430_distributed_server_workers.md" }, { "1400 Source Definition and Observation Model" = "en/rfcs/1400_source_definition_and_observation_model.md" }, { "1345 Scope Organization and Agent Integration" = "en/rfcs/1345_scope_organization_and_agent_integration.md" }, { "1229 Unified Workloads and Long-Horizon Memory Evaluation" = "en/rfcs/1229_unified_workloads_and_long_horizon_memory_evaluation.md" }, @@ -177,6 +178,7 @@ nav = [ { "RFC 与会议纪要" = [ { "RFC" = [ { "概览" = "zh/rfcs/README.md" }, + { "1430 分布式 Server Worker" = "zh/rfcs/1430_distributed_server_workers.md" }, { "1400 Source 定义与观察模型" = "zh/rfcs/1400_source_definition_and_observation_model.md" }, { "1345 Scope 组织与 Agent 集成" = "zh/rfcs/1345_scope_organization_and_agent_integration.md" }, { "1229 统一工作负载与长程 Memory 评估" = "zh/rfcs/1229_unified_workloads_and_long_horizon_memory_evaluation.md" },