feat: add database-backed distributed work runtime

.env.example

@@ -58,6 +58,42 @@ POWERCONTEXT_SERVER_DATABASE_KIND=sqlite
 # POWERCONTEXT_SERVER_DATABASE_KIND=seekdb
 # POWERCONTEXT_SERVER_DATABASE_PATH=/absolute/path/to/seekdb
 
+# Deployment and durable work -------------------------------------------------
+# Local installs run every role in one process. Distributed mode requires OceanBase and one role per process.
+POWERCONTEXT_SERVER_DEPLOYMENT_MODE=single_node
+POWERCONTEXT_SERVER_DEPLOYMENT_ROLE=all
+POWERCONTEXT_SERVER_DEPLOYMENT_ID=local
+POWERCONTEXT_SERVER_DEPLOYMENT_BEHAVIOR_REVISION=default
+
+# Scheduler and member leases use database time. Renewal must stay within the validated lease ratios.
+POWERCONTEXT_SERVER_COORDINATION_SCHEDULER_LEASE_SECONDS=30
+POWERCONTEXT_SERVER_COORDINATION_SCHEDULER_RENEW_SECONDS=10
+POWERCONTEXT_SERVER_COORDINATION_SCAN_PAGE_SIZE=100
+POWERCONTEXT_SERVER_COORDINATION_MEMBER_TTL_SECONDS=30
+POWERCONTEXT_SERVER_COORDINATION_MEMBER_HEARTBEAT_SECONDS=10
+POWERCONTEXT_SERVER_COORDINATION_EMIT_PAYLOAD_VERSION=1
+
+POWERCONTEXT_SERVER_WORKER_CONCURRENCY=4
+POWERCONTEXT_SERVER_WORKER_LEASE_SECONDS=120
+POWERCONTEXT_SERVER_WORKER_HEARTBEAT_SECONDS=30
+POWERCONTEXT_SERVER_WORKER_SHUTDOWN_GRACE_SECONDS=90
+POWERCONTEXT_SERVER_WORKER_MAX_ATTEMPTS=5
+POWERCONTEXT_SERVER_WORKER_RETRY_BASE_SECONDS=2
+POWERCONTEXT_SERVER_WORKER_RETRY_MAX_SECONDS=300
+POWERCONTEXT_SERVER_WORKER_POLL_SECONDS=1
+
+POWERCONTEXT_SERVER_OPERATIONS_DEFAULT_WAIT_SECONDS=10
+POWERCONTEXT_SERVER_OPERATIONS_MAXIMUM_WAIT_SECONDS=30
+POWERCONTEXT_SERVER_OPERATIONS_POLL_SECONDS=0.2
+POWERCONTEXT_SERVER_OPERATIONS_RETENTION_DAYS=30
+POWERCONTEXT_SERVER_OPERATIONS_CLEANUP_BATCH_SIZE=500
+POWERCONTEXT_SERVER_OPERATIONS_CLEANUP_INTERVAL_SECONDS=3600
+
+# Shared OceanBase fixed-window limiting is optional and disabled by default.
+POWERCONTEXT_SERVER_RATE_LIMIT_ENABLED=false
+POWERCONTEXT_SERVER_RATE_LIMIT_REQUESTS=120
+POWERCONTEXT_SERVER_RATE_LIMIT_WINDOW_SECONDS=60
+
 # Runtime ---------------------------------------------------------------------
 POWERCONTEXT_SERVER_RUNTIME_SOURCE_WINDOW_LIMIT=100
 POWERCONTEXT_SERVER_RUNTIME_MEMORY_EXTRACTION_PROFILE=coding

.licenserc.yaml

@@ -59,6 +59,8 @@ header:
     - 'src/powercontext/http/_generated/**'
     - '.gitattributes'
     - '**/*.service.in'
+    # license-eye cannot determine the comment style of Alembic's Mako template.
+    - 'src/powercontext/builtin/persistence/migrations/script.py.mako'
     - '**/*.jsonl'
     # license-eye cannot determine the comment style of uv requirements files.
     - 'e2e/bub/source-overrides.txt'

docker/README.md

@@ -11,7 +11,7 @@ docker build \
   .
 ```
 
-Run the Server with persistent SQLite and scheduler data:
+Run the Server with persistent SQLite application and Work Ledger data:
 
 ```bash
 docker run --rm \
@@ -34,6 +34,11 @@ requests share one refresh. Checks use the Runtime's credentials and never expos
 another database or inference provider with the same `POWERCONTEXT_SERVER_*` environment variables used by a regular
 Server installation.
 
+For a role-separated OceanBase topology, use `docker/compose.distributed.yaml` as a deployment template. It runs a
+one-shot migrator, two API replicas, two fenced Scheduler replicas, and two lease-based Worker replicas. Replace the
+example database credentials, authentication token, and provider configuration before starting it; only Workers need
+model credentials.
+
 ## Network exposure
 
 PowerContext refuses to start an unauthenticated Server on a non-loopback address unless

docker/compose.distributed.yaml

@@ -0,0 +1,112 @@
+name: powercontext-distributed
+
+x-runtime: &runtime
+  image: ${POWERCONTEXT_IMAGE:-powercontext-server:local}
+  restart: unless-stopped
+  environment: &shared-environment
+    POWERCONTEXT_SERVER_DATABASE_KIND: oceanbase
+    POWERCONTEXT_SERVER_DATABASE_URL: ${POWERCONTEXT_SERVER_DATABASE_URL:?set an OceanBase URL}
+    POWERCONTEXT_SERVER_DEPLOYMENT_MODE: distributed
+    POWERCONTEXT_SERVER_DEPLOYMENT_BEHAVIOR_REVISION: ${POWERCONTEXT_BEHAVIOR_REVISION:-distributed-v1}
+    POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS: ${POWERCONTEXT_MEMORY_SCHEDULE_SECONDS:-60}
+    POWERCONTEXT_SERVER_RUNTIME_EXPERIENCE_SCHEDULE_SECONDS: ${POWERCONTEXT_EXPERIENCE_SCHEDULE_SECONDS:-60}
+
+services:
+  migrate:
+    <<: *runtime
+    restart: "no"
+    command: ["server", "migrate"]
+    environment:
+      <<: *shared-environment
+      # The migration command does not start a Runtime or register this role;
+      # a valid distributed role is still required while parsing ServerSettings.
+      POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: api
+      POWERCONTEXT_SERVER_DEPLOYMENT_ID: migrator
+
+  api-a:
+    <<: *runtime
+    command: ["server", "run"]
+    environment:
+      <<: *shared-environment
+      POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: api
+      POWERCONTEXT_SERVER_DEPLOYMENT_ID: api-a
+      POWERCONTEXT_SERVER_AUTH_ENABLED: "true"
+      POWERCONTEXT_SERVER_AUTH_TOKEN: ${POWERCONTEXT_SERVER_AUTH_TOKEN:?set an API bearer token}
+      POWERCONTEXT_SERVER_RATE_LIMIT_ENABLED: "true"
+    ports: ["127.0.0.1:8001:8000"]
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+
+  api-b:
+    <<: *runtime
+    command: ["server", "run"]
+    environment:
+      <<: *shared-environment
+      POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: api
+      POWERCONTEXT_SERVER_DEPLOYMENT_ID: api-b
+      POWERCONTEXT_SERVER_AUTH_ENABLED: "true"
+      POWERCONTEXT_SERVER_AUTH_TOKEN: ${POWERCONTEXT_SERVER_AUTH_TOKEN:?set an API bearer token}
+      POWERCONTEXT_SERVER_RATE_LIMIT_ENABLED: "true"
+    ports: ["127.0.0.1:8002:8000"]
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+
+  scheduler-a:
+    <<: *runtime
+    command: ["server", "run"]
+    environment:
+      <<: *shared-environment
+      POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: scheduler
+      POWERCONTEXT_SERVER_DEPLOYMENT_ID: scheduler-a
+      POWERCONTEXT_SERVER_DASHBOARD_ENABLED: "false"
+      POWERCONTEXT_SERVER_MCP_ENABLED: "false"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+
+  scheduler-b:
+    <<: *runtime
+    command: ["server", "run"]
+    environment:
+      <<: *shared-environment
+      POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: scheduler
+      POWERCONTEXT_SERVER_DEPLOYMENT_ID: scheduler-b
+      POWERCONTEXT_SERVER_DASHBOARD_ENABLED: "false"
+      POWERCONTEXT_SERVER_MCP_ENABLED: "false"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+
+  worker-a:
+    <<: *runtime
+    command: ["server", "run"]
+    environment:
+      <<: *shared-environment
+      POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: worker
+      POWERCONTEXT_SERVER_DEPLOYMENT_ID: worker-a
+      POWERCONTEXT_SERVER_DASHBOARD_ENABLED: "false"
+      POWERCONTEXT_SERVER_MCP_ENABLED: "false"
+      POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL: ${POWERCONTEXT_GENERATION_MODEL:?set a generation model}
+      OPENAI_API_KEY: ${OPENAI_API_KEY:-}
+      ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+
+  worker-b:
+    <<: *runtime
+    command: ["server", "run"]
+    environment:
+      <<: *shared-environment
+      POWERCONTEXT_SERVER_DEPLOYMENT_ROLE: worker
+      POWERCONTEXT_SERVER_DEPLOYMENT_ID: worker-b
+      POWERCONTEXT_SERVER_DASHBOARD_ENABLED: "false"
+      POWERCONTEXT_SERVER_MCP_ENABLED: "false"
+      POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL: ${POWERCONTEXT_GENERATION_MODEL:?set a generation model}
+      OPENAI_API_KEY: ${OPENAI_API_KEY:-}
+      ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
+    depends_on:
+      migrate:
+        condition: service_completed_successfully

docs/en/development/core-protocol.md

@@ -97,8 +97,9 @@ conflicts and persistence; the family service is responsible for its domain beha
 A `Trigger` is a policy over a signal and prior state. It returns a `PolicyTransition` containing the next state and
 zero or more actions. A Trigger should not open storage, schedule itself, or perform the action it selects.
 
-APScheduler belongs to the Builtin runtime lifecycle. It decides when to evaluate a policy. The Trigger decides what
-the observed signal means.
+The durable Scheduler belongs to the Builtin runtime lifecycle. It decides when to evaluate a policy and records work
+in the database ledger. The Trigger decides what the observed signal means; a fenced Worker performs the selected
+action.
 
 ## Ownership boundaries
 
@@ -109,7 +110,7 @@ the observed signal means.
 | Environment-backed process configuration | `powercontext.client.settings`, `powercontext.server.settings` |
 | HTTP lifecycle and optional MCP transport | `powercontext.server` |
 | Provider-specific generation and embedding | Inference integration |
-| Database and scheduler resource lifetime | Application entry point or Builtin runtime instance |
+| Database, Scheduler, and Worker resource lifetime | Application entry point or Builtin runtime instance |
 
 Core models use Pydantic `BaseModel`. Add a validator when the value has a real domain constraint. Do not add wrapper
 properties for stored fields, custom JSON value hierarchies, or a second definition object when the model or protocol

docs/en/development/remote-access-implementation.md

@@ -76,8 +76,9 @@ and OceanBase uses HNSW for `vector` and `hybrid` searches.
 Inference configuration is documented in [Configure Pydantic AI inference](pydantic-ai-inference.md).
 
 Set `POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS` to process pending Source windows on a persisted interval. Scheduled
-jobs use the SQLite sidecar at `POWERCONTEXT_HOME/scheduler.db`. Scheduling works with either application database and
-requires a configured generation pipeline.
+discovery and execution use the primary database Work Ledger. In `single_node/all` mode the embedded Scheduler and
+Worker share that ledger; distributed OceanBase deployments run them as separate roles. A generation pipeline is
+required by the Worker.
 
 ## HTTP surface
 
@@ -90,6 +91,7 @@ The source contract is `openapi/powercontext.yaml`. Generated Pydantic models an
 | Capabilities | source types, Artifact families, extraction, search modes |
 | Sources | capture durable content evidence |
 | Memory | flush pending Sources, remember explicit entries, search |
+| Operations | inspect, list, cancel, and retry durable background work |
 | Memory entries | list, get, revise, retire |
 | History | list Memory changes |
 

docs/en/docs/how-to/deploy-server.md

@@ -70,8 +70,9 @@ export POWERCONTEXT_HOME=/srv/powercontext
 powercontext server run
 ```
 
-The process must be able to create and update this directory. The default SQLite database and scheduler state are
-stored below it. Supply the same environment variables whenever your service manager restarts the process.
+The process must be able to create and update this directory. The default SQLite database also stores durable
+Scheduler, Worker lease, and Operation state. Supply the same environment variables whenever your service manager
+restarts the process.
 
 PowerContext does not search for a `.env` file automatically. Export the variables, configure them in the service
 manager or container platform, or pass one explicit file:
@@ -110,7 +111,40 @@ docker run --rm \
 ```
 
 The image listens on `0.0.0.0:8000` inside the container, so the host-side address in `--publish` is important. The
-named volume persists the SQLite database and scheduler state after the container stops.
+named volume persists the SQLite database and durable work state after the container stops.
+
+## Run distributed roles
+
+Distributed mode requires OceanBase and a schema migration before any role starts. The repository includes
+`docker/compose.distributed.yaml` as a topology example with two APIs, two Schedulers, and two Workers. One Scheduler is
+leader and the other remains ready to take over; both APIs and both Workers are active.
+
+Export secrets and deployment choices without writing them into Compose:
+
+```bash
+export POWERCONTEXT_SERVER_DATABASE_URL="$OCEANBASE_URL"
+export POWERCONTEXT_SERVER_AUTH_TOKEN="$POWERCONTEXT_DEPLOYMENT_TOKEN"
+export POWERCONTEXT_GENERATION_MODEL="openai:gpt-4.1-mini"
+export OPENAI_API_KEY
+docker compose --file docker/compose.distributed.yaml up migrate
+docker compose --file docker/compose.distributed.yaml up -d api-a api-b scheduler-a scheduler-b worker-a worker-b
+```
+
+The two API examples listen on host ports 8001 and 8002. Put a TLS-terminating load balancer in front of them and use
+round-robin routing without session affinity. Distributed MCP is stateless. The example intentionally supplies model
+credentials only to Workers and no public port to Scheduler or Worker roles. A production deployment should also use
+separate least-privilege database users instead of the shared demonstration URL.
+
+Every replica in one rollout must use the same behavior revision. Upgrade in this order:
+
+1. run `powercontext server migrate` with a dedicated DDL account;
+2. replace Workers and wait for readiness;
+3. replace Schedulers and confirm a leader can scan;
+4. replace APIs.
+
+Keep `POWERCONTEXT_SERVER_COORDINATION_EMIT_PAYLOAD_VERSION` on the older supported value until old Workers have drained.
+Rollback in the reverse order and never start a distributed role against a schema that has not reached the packaged
+revision.
 
 ## Enable authentication
 

docs/en/docs/how-to/full-capability-runtime.md

@@ -126,13 +126,23 @@ evidence. Scheduled processing handles new Sources within the configured interva
 
 ## Data and restart behavior
 
-With no database override, SQLite stores `powercontext.db` and `scheduler.db` under the user data directory:
+The generated configuration leaves the database unset, so the Server stores data in the user data directory instead of
+a project-local file. With `POWERCONTEXT_HOME` unset, SQLite keeps `powercontext.db`, including durable scheduling and
+operation state, under:
 
-- Linux: `$XDG_DATA_HOME/powercontext`, or `~/.local/share/powercontext`;
-- macOS: `~/Library/Application Support/powercontext`.
+- macOS: `~/Library/Application Support/powercontext/`
+- Linux: `~/.local/share/powercontext/`
 
-Press `Ctrl+C` to stop the Server. Restart it with the same `.env` and data directory. The default Scope and its opaque
-ID remain stable because they are persisted in the database.
+Set `POWERCONTEXT_HOME` before starting the Server to relocate all of this. Changing the database URL later points the
+Server at a different (possibly empty) database; keep the previous value if you need the old data.
+
+## Stop and restart
+
+Press `Ctrl+C` in the Server terminal to stop it. Data persists in SQLite across restarts. To resume, load the same
+`.env` and run `powercontext server run --env-file .env` again; pending Sources are processed on the next Scheduler run
+or flush. The default Scope and its opaque ID also remain stable because they are persisted in the same database.
+
+## Quick troubleshooting
 
 | Symptom | Action |
 | --- | --- |

docs/en/docs/how-to/troubleshoot.md

@@ -171,8 +171,8 @@ so the previous database remains available for recovery:
 3. Export the PowerContext table data with OceanBase `obdumper` in CSV or SQL data mode **without `--ddl`**. Keep the
    export and the original database unchanged until the migration is verified. Supply credentials through your
    approved secret-handling process rather than placing them in logs or documentation.
-4. Create a new empty OceanBase MySQL-mode database and point `POWERCONTEXT_SERVER_DATABASE_URL` at it. Start the
-   current PowerContext version once to create tables with `utf8mb4_bin`, then stop it before restoring data.
+4. Create a new empty OceanBase MySQL-mode database, point `POWERCONTEXT_SERVER_DATABASE_URL` at it, and run
+   `powercontext server migrate`. Do not start an API, Scheduler, or Worker process before restoring the data.
 5. Import only the exported row data into the existing new tables with OceanBase `obloader`, again **without
    `--ddl`**. Keep foreign-key enforcement enabled and run these three layers separately. The examples use CSV; if
    you exported SQL data, replace `--csv` with `--sql` in all three commands. Fill in `<connection-options>` through
@@ -185,12 +185,15 @@ so the previous database remains available for recovery:
    exported `pc_scopes` data.
    If the source predates the three Skill lifecycle tables (`pc_skill_packages`, `pc_agent_skill_targets`, and
    `pc_skill_publications`), remove the absent tables from Layer 1.
+   Do not import `pc_scheduler_leases`, `pc_scheduler_scans`, `pc_runtime_members`, or `pc_rate_limit_windows`.
+   They contain transient coordination state. The migration command owns its migration lease row, and the runtime
+   recreates the remaining rows after startup.
 
    Layer 1 contains parents and tables without foreign keys:
 
    ```bash
    obloader <connection-options> -D <new-database> --csv \
-      --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily' \
+      --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_work_lanes,pc_work_items,pc_work_keys,pc_work_attempts' \
      -f <export-directory>
    ```
 
@@ -213,7 +216,8 @@ so the previous database remains available for recovery:
    Wait for each invocation to complete successfully before starting the next. Treat any OBLoader error, bad record,
    or conflict record as a failed restore. Order within a layer is irrelevant because no table in a layer references
    another table in the same layer.
-6. If the installation has additional PowerContext-managed tables not listed above, these tested layers do not
+6. Except for the four transient tables intentionally excluded above, if the installation has additional
+   PowerContext-managed tables not listed above, these tested layers do not
    classify them. Inspect their foreign-key constraints and place each table after all of its parents; do not add
    them to an all-table invocation.
 7. Compare source and target row counts for every restored table, inspect the identity-column collations, and test