diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f785a606..b50917ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -470,6 +470,10 @@ jobs: run: cargo run -p awa --example etl_pipeline env: DATABASE_URL: postgres://postgres:postgres@localhost:5432/awa_test + - name: Run Rust quickstart example + run: cargo run -p awa --example quickstart + env: + DATABASE_URL: postgres://postgres:postgres@localhost:5432/awa_test # ─── Chaos smoke test ───────────────────────────────────────── # Lightweight version of the nightly chaos suite — catches Python diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..1fac23d1 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,142 @@ +name: Docs + +on: + push: + branches: [main] + paths: + - .github/workflows/docs.yml + - awa/examples/quickstart.rs + - awa-python/examples/quickstart.py + - docs/** + - mkdocs.yml + - requirements-docs.txt + - scripts/build-agent-docs.py + - scripts/check-docs.sh + pull_request: + branches: [main] + types: [opened, reopened, synchronize, closed] + paths: + - .github/workflows/docs.yml + - awa/examples/quickstart.rs + - awa-python/examples/quickstart.py + - docs/** + - mkdocs.yml + - requirements-docs.txt + - scripts/build-agent-docs.py + - scripts/check-docs.sh + workflow_dispatch: + +concurrency: + # Serialize each preview's updates and cleanup without letting unrelated PRs + # evict one another from GitHub's single pending slot. Production deploys + # remain serialized separately; manual runs are intentionally independent. + group: >- + awa-docs-${{ + github.event_name == 'pull_request' + && format('pr-{0}', github.event.pull_request.number) + || github.event_name == 'push' + && 'production' + || format('manual-{0}', github.run_id) + }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + build: + if: github.event_name != 'pull_request' || github.event.action != 'closed' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: requirements-docs.txt + + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + - name: Install documentation dependencies + run: python -m pip install --requirement requirements-docs.txt + + - name: Build docs and check canonical examples + run: scripts/check-docs.sh + + - name: Prepare Pages artifact + run: touch site/.nojekyll + + - uses: actions/upload-artifact@v7 + with: + name: awa-docs-${{ github.run_id }} + path: site + include-hidden-files: true + if-no-files-found: error + + preview: + if: >- + github.event_name == 'pull_request' && + github.event.action != 'closed' && + github.event.pull_request.head.repo.full_name == github.repository + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v6 + + - uses: actions/download-artifact@v8 + with: + name: awa-docs-${{ github.run_id }} + path: site + + - name: Publish pull request preview + uses: rossjrw/pr-preview-action@ffa7509e91a3ec8dfc2e5536c4d5c1acdf7a6de9 # v1.8.1 + with: + source-dir: site + preview-branch: gh-pages + umbrella-dir: pr-preview + + remove-preview: + if: >- + github.event_name == 'pull_request' && + github.event.action == 'closed' && + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v6 + + - name: Remove pull request preview + uses: rossjrw/pr-preview-action@ffa7509e91a3ec8dfc2e5536c4d5c1acdf7a6de9 # v1.8.1 + with: + action: remove + preview-branch: gh-pages + umbrella-dir: pr-preview + + deploy: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v6 + + - uses: actions/download-artifact@v8 + with: + name: awa-docs-${{ github.run_id }} + path: site + + - name: Publish production site + uses: JamesIves/github-pages-deploy-action@fa24774553152dd7873cd16ebd8d959b010c5445 # v4.9.0 + with: + branch: gh-pages + folder: site + clean-exclude: pr-preview + force: false diff --git a/.gitignore b/.gitignore index d9c43fac..bb23a3d9 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,11 @@ awa-python/python/awa/lib_awa.dylib.dSYM/ /awa-ui/static/assets /awa-ui/static/index.html +# Documentation build and browser QA artifacts +/site/ +/.playwright-cli/ +/output/playwright/ + # Local worktrees and generated artifacts /.claude/ /artifacts/ diff --git a/AGENTS.md b/AGENTS.md index 1bddb00c..91d03643 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,13 +31,54 @@ exit code from a piped or backgrounded command. ## Schema Migrations -Migrations are forward-only and must stay rolling-upgrade compatible. Follow the -migration checklist and rolling-upgrade policy in -[`docs/development.md`](docs/development.md#authoring-schema-migrations) and -[ADR-041](docs/adr/041-rolling-upgrade-policy.md) before opening a migration PR. -Version floors, exclusive migrations, and the newer-schema fail-safe live in +Migrations are forward-only and must stay rolling-upgrade compatible. Version +floors, exclusive migrations, and the newer-schema fail-safe live in `awa-model/src/migrations.rs`. +The implementation checklist lives here rather than in `docs/`: it is +contributor-internal, and `docs/` is published as the public documentation site. +[`docs/development.md`](docs/development.md#authoring-schema-migrations) carries +the user-facing summary and points back here. + +Policy: [ADR-041 — rolling-upgrade policy](docs/adr/041-rolling-upgrade-policy.md). Use this checklist before opening a migration PR; version floors, exclusive migrations, and the newer-schema fail-safe live in `awa-model/src/migrations.rs`. + +Checklist for any new `awa-model/migrations/vNNN_*.sql`: + +**Every migration** + +- [ ] Keep every object used by N−1 binaries compatible: no drops, type changes, or tightened constraints; make new objects and columns additive. +- [ ] Make the migration safe to re-run: `IF NOT EXISTS` on `CREATE TABLE` / `SEQUENCE` / `INDEX`, `CREATE OR REPLACE` for functions and views, `DROP TRIGGER IF EXISTS` before each `CREATE TRIGGER`, guarded `DO` blocks for anything with no `IF NOT EXISTS` form (`CREATE TYPE`), and `ON CONFLICT (version) DO NOTHING` on the `awa.schema_version` row. `migrations::tests::every_migration_guards_its_ddl` enforces the top-level cases; `test_every_migration_is_individually_re_runnable` proves it against a real database. +- [ ] Keep every step transaction-safe — the runner applies the whole pending range in one transaction, so no `CREATE INDEX CONCURRENTLY`, `VACUUM`, or statement-level `BEGIN` / `COMMIT` / `ROLLBACK` / `SAVEPOINT`. `migrations::tests::every_migration_step_is_transaction_safe` enforces this. +- [ ] In the header, link the issue and state how N−1 binaries operate against the migrated schema. +- [ ] Safe under live load: no long `ACCESS EXCLUSIVE` holds on hot tables; note the expected wall time on realistic data volumes. +- [ ] The current binary remains operable before migration, or startup applies the migration before any changed path runs. Test binary-first as well as migrate-first ordering. +- [ ] Document requirements for external runners, which do not execute Rust preflights. + +**If compatibility first ships in an earlier-release patch** + +- [ ] Add the released, verified patch to `MIGRATION_RUNTIME_VERSION_FLOORS`; test old, unparseable, and stale runtimes plus `--allow-live-runtimes`. +- [ ] Keep the preflight race-free and record any observability-snapshot stall from its lock. Job and lease heartbeats must remain unaffected. +- [ ] Publish the patch prerequisite before the migration and document it in the CHANGELOG and upgrade guide. + +**If it changes an on-disk representation or hot-path structure (expand → flip → contract)** + +- [ ] Make the migration the **expand** phase only: seed the new representation, keep the old one authoritative, and store authority explicitly. Fresh installs may start on the new representation. +- [ ] Gate the runtime **flip** on fresh fleet capability. Install the schema-owned per-feature capability constant with the expand migration; treat missing or unparseable evidence as incapable and make any override explicit. +- [ ] Under the old-writer locks, the flip treats the old representation as source of truth, reconciles the complete new representation, verifies exact equivalence, and changes authority atomically. Shadow writes alone do not satisfy this requirement. +- [ ] The flip **fences** returning pre-flip binaries at the database boundary. Exercise the actual N−1 write path; a sentinel is insufficient if old code can advance through it. +- [ ] The **contract** migration (dropping the old representation) is deferred to a later minor, tracked as its own issue, and independently checked against that release's N−1 contract. +- [ ] Model mixed-version interleavings in TLA+ when a state machine or lock order changes. +- [ ] Rehearse migrate-first, binary-first, and overlapping rollouts with a released N−1 artifact. Include concurrent old/new workers, failures and retries, scheduled work, in-flight work, hard-kill and deadline rescue, flip/fence behavior, and exact job accounting; record the evidence. CI automation is [#427](https://github.com/hardbyte/awa/issues/427). + +**If no rolling-compatible design is practical** + +- [ ] Explain in an ADR why expand/flip/contract and a version floor are insufficient, then add the migration to `EXCLUSIVE_WINDOW_MIGRATIONS` with refusal, override, and stale-heartbeat tests plus explicit operator documentation. + +**Docs** + +- [ ] Update the CHANGELOG, the release upgrade guide when operator action is required, and `docs/stability.md` when the skew contract changes. Link compatibility claims to rehearsals of the claimed version topology; describe narrower evidence only by the behavior it covers. + + ## Agent Skills Canonical, portable [Agent Skills](https://agentskills.io/) live under diff --git a/README.md b/README.md index 6c630ecf..5c364f6e 100644 --- a/README.md +++ b/README.md @@ -92,10 +92,10 @@ See [docs/positioning.md](docs/positioning.md) for the category map and messagin ## Getting Started ```bash -# 1. Install -pip install 'awa-pg[ui]' # Python SDK + dashboard binary -# pip install awa-pg # SDK only (no dashboard, smaller wheel) -# or: cargo add awa # Rust +# 1. Add Awa to a Python project +uv add 'awa-pg[ui]==0.6.6' # Python SDK + dashboard binary +# uv add awa-pg==0.6.6 # SDK only (no dashboard, smaller wheel) +# or: cargo add awa@0.6.6 # Rust # 2. Start Postgres and run migrations awa --database-url $DATABASE_URL migrate @@ -310,13 +310,13 @@ Cancellation is cooperative for running handlers: ### Python ```bash -pip install awa-pg # SDK: insert, worker, admin, progress -pip install 'awa-pg[ui]' # SDK + bundled `awa` binary for the dashboard +uv add awa-pg==0.6.6 # SDK: insert, worker, admin, progress +uv add 'awa-pg[ui]==0.6.6' # SDK + bundled `awa` binary for the dashboard # or, just the CLI: -pip install awa-cli # CLI on its own: migrations, queue admin, web UI +uv tool install awa-cli==0.6.6 # CLI on its own: migrations, queue admin, web UI ``` -`pip install awa-pg` stays small for workers and producers. The `[ui]` extra pulls in [`awa-cli`](https://pypi.org/project/awa-cli/), which ships the `awa` binary plus the embedded React dashboard; afterwards `python -m awa serve` (or `awa serve` directly) launches it. +`uv add awa-pg==0.6.6` stays small for workers and producers. The `[ui]` extra pulls in [`awa-cli`](https://pypi.org/project/awa-cli/), which ships the `awa` binary plus the embedded React dashboard; afterwards `uv run python -m awa serve` (or `awa serve` directly) launches it. ### Rust @@ -327,10 +327,10 @@ awa = "0.6" ### CLI -Available via pip (no Rust toolchain needed) or cargo: +Available as a uv tool (no Rust toolchain needed) or through cargo: ```bash -pip install awa-cli +uv tool install awa-cli==0.6.6 # or: cargo install awa-cli awa --database-url $DATABASE_URL migrate @@ -376,15 +376,17 @@ All coordination through Postgres. The Rust runtime owns dispatch, leases, heart | `awa-worker` | Runtime: dispatch, heartbeat, maintenance | | `awa-ui` | Web UI (axum API + embedded React frontend) | | `awa-cli` | CLI binary (migrations, admin, serve) | -| `awa-python` | PyO3 extension module (`pip install awa-pg`) | +| `awa-python` | PyO3 extension module (`uv add awa-pg==0.6.6`) | | `awa-testing` | Test helpers (`TestClient`) | ## Documentation +**[Browse the documentation site →](https://hardbyte.github.io/awa/)** + | Doc | Description | | --- | --- | | [Rust getting started](docs/getting-started-rust.md) | From `cargo add` to a job reaching `completed` | -| [Python getting started](docs/getting-started-python.md) | From `pip install` to a job reaching `completed` | +| [Python getting started](docs/getting-started-python.md) | From `uv init` to a job reaching `completed` | | [Deployment guide](docs/deployment.md) | Docker, Kubernetes, pool sizing, graceful shutdown | | [Migration guide](docs/migrations.md) | Fresh installs, upgrades, extracted SQL, rollback strategy | | [0.5 → 0.6 upgrade](docs/upgrade-0.5-to-0.6.md) | Step-by-step operator checklist for the staged storage transition | diff --git a/awa-python/examples/quickstart.py b/awa-python/examples/quickstart.py index fb7691dd..2fb02d97 100644 --- a/awa-python/examples/quickstart.py +++ b/awa-python/examples/quickstart.py @@ -1,10 +1,10 @@ """Awa Python quickstart — a complete runnable example. -Requires: pip install awa-pg +Requires: uv add awa-pg==0.6.6 Requires: a running Postgres instance with DATABASE_URL set. -Usage: - DATABASE_URL=postgres://localhost/mydb python examples/quickstart.py +Usage from the repository's awa-python directory: + DATABASE_URL=postgres://localhost/mydb uv run python examples/quickstart.py """ import asyncio @@ -44,11 +44,43 @@ async def handle_email(job): ) print(f"Inserted job {job.id} (kind={job.kind}, state={job.state})") - await asyncio.sleep(1) - await client.shutdown() + # Verify it reaches a terminal state without relying on a fixed delay. + loop = asyncio.get_running_loop() + deadline = loop.time() + 10 + last_state = job.state + try: + while True: + remaining = deadline - loop.time() + if remaining <= 0: + raise TimeoutError( + f"timed out waiting for job {job.id} " + f"(last state: {last_state})" + ) + + # get_job is a single read-only query, so cancelling this await + # cannot leave an application transaction partially committed. + try: + result = await asyncio.wait_for( + client.get_job(job.id), timeout=remaining + ) + except asyncio.TimeoutError as error: + raise TimeoutError( + f"timed out waiting for job {job.id} " + f"(last state: {last_state})" + ) from error + + last_state = result.state + if result.state == awa.JobState.Completed: + break + if result.state in (awa.JobState.Failed, awa.JobState.Cancelled): + raise RuntimeError( + f"job {result.id} ended in terminal state {result.state}" + ) + await asyncio.sleep(min(0.1, max(0, deadline - loop.time()))) + finally: + await client.shutdown() + await client.close() - # Verify it completed - result = await client.get_job(job.id) print(f"Job {result.id} state: {result.state}") diff --git a/awa/examples/quickstart.rs b/awa/examples/quickstart.rs new file mode 100644 index 00000000..e1ad4cde --- /dev/null +++ b/awa/examples/quickstart.rs @@ -0,0 +1,105 @@ +//! A complete AWA Rust quickstart. +//! +//! Run with a PostgreSQL database available at `DATABASE_URL`: +//! `cargo run -p awa --example quickstart`. + +use awa::{ + admin, insert_with, migrations, Client, InsertOpts, JobArgs, JobResult, JobState, QueueConfig, +}; +use serde::{Deserialize, Serialize}; +use sqlx::postgres::PgPoolOptions; +use std::{env, time::Duration}; + +#[derive(Debug, Serialize, Deserialize)] +struct SendEmail { + to: String, + subject: String, +} + +impl JobArgs for SendEmail { + fn kind() -> &'static str { + "send_email" + } +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let database_url = env::var("DATABASE_URL")?; + let pool = PgPoolOptions::new() + .max_connections(10) + .connect(&database_url) + .await?; + + migrations::run(&pool).await?; + + let client = Client::builder(pool.clone()) + .queue( + "email", + QueueConfig { + max_workers: 2, + ..Default::default() + }, + ) + .register::(|args, _ctx| async move { + println!("sending email to {}: {}", args.to, args.subject); + Ok(JobResult::Completed) + }) + .build()?; + + client.start().await?; + + let job = insert_with( + &pool, + &SendEmail { + to: "alice@example.com".into(), + subject: "Welcome".into(), + }, + InsertOpts { + queue: "email".into(), + ..Default::default() + }, + ) + .await?; + + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut last_state = job.state; + let job = loop { + let current = tokio::time::timeout_at(deadline, admin::get_job(&pool, job.id)) + .await + .map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!( + "timed out waiting for job {} (last state: {})", + job.id, last_state + ), + ) + })??; + last_state = current.state; + match current.state { + JobState::Completed => break current, + JobState::Failed | JobState::Cancelled => { + return Err(std::io::Error::other(format!( + "job {} ended in terminal state {}", + current.id, current.state + )) + .into()); + } + _ if tokio::time::Instant::now() >= deadline => { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!( + "timed out waiting for job {} (last state: {})", + current.id, current.state + ), + ) + .into()); + } + _ => tokio::time::sleep(Duration::from_millis(100)).await, + } + }; + println!("job {} state = {:?}", job.id, job.state); + + client.shutdown(Duration::from_secs(5)).await; + Ok(()) +} diff --git a/docs/0.7-planning-brief.md b/docs/0.7-planning-brief.md index 900913c8..77c7828f 100644 --- a/docs/0.7-planning-brief.md +++ b/docs/0.7-planning-brief.md @@ -45,7 +45,7 @@ rescue, rotation, prune, and cron are all **leader-elected inside the worker fle | `awa-worker` | Runtime: dispatch, heartbeat, maintenance | | `awa-ui` | Web UI (axum API + embedded React) | | `awa-cli` | CLI binary | -| `awa-python` | PyO3 module (`pip install awa-pg`) | +| `awa-python` | PyO3 module (`uv add awa-pg`) | | `awa-testing` | Test helpers (`TestClient`) | | `awa-seaorm` | New in 0.6 — transactional enqueue alongside SeaORM writes | | `awa-metrics` | New in 0.6 — shared OTel counters for non-runtime callers | @@ -233,7 +233,7 @@ tuples** on a 5k-job runtime soak; enqueue ~30k/s single-producer, ~100k/s multi ## 4. Open issue inventory (20 open) > **Snapshot note.** This inventory reflects the tracker *before* the roadmap -> ([`0.7-roadmap.md`](0.7-roadmap.md)) was adopted. The roadmap has since filed #367–#383 and +> ([0.7 roadmap](0.7-roadmap.md)) was adopted. The roadmap has since filed #367–#383 and > moved the surviving candidates below into the `v0.7.0` milestone, so the milestone now holds > ~29 issues; the roadmap's §10 disposition table is the current authority. This section is > kept as the point-in-time context it was written to be. diff --git a/docs/0.7-roadmap.md b/docs/0.7-roadmap.md index 9cfe59ed..09f118c3 100644 --- a/docs/0.7-roadmap.md +++ b/docs/0.7-roadmap.md @@ -2,7 +2,7 @@ > **Status:** Proposed. This is the design document for the 0.7 release cycle: strategic > decisions, workstreams, fully-scoped issues (existing and new), experiments with decision -> rules, milestones, and release gates. Companion context: [`0.7-planning-brief.md`](0.7-planning-brief.md) +> rules, milestones, and release gates. Companion context: [0.7 planning brief](https://github.com/hardbyte/awa/blob/main/docs/0.7-planning-brief.md) > (0.6 recap, open-issue inventory, the #169 hot-row chronology, and the #197 process template). > No timeline, effort, or cost assumptions are made; sequencing is expressed as dependencies > and gates, not dates. diff --git a/docs/adr/019-queue-storage-redesign.md b/docs/adr/019-queue-storage-redesign.md index 4ee5caaf..56d6a05e 100644 --- a/docs/adr/019-queue-storage-redesign.md +++ b/docs/adr/019-queue-storage-redesign.md @@ -185,13 +185,13 @@ The decision above pins two hot-path requirements that surface repeatedly in imp ## Validation -Recorded local 5k-job runtime soak: **9,537 jobs/s**, **3.671 ms pickup p50**, **22.013 ms pickup p95**, **417 exact final dead tuples** (canonical runtime under the same workload: 9,686 jobs/s, 38.998 ms p95, dead tuples not sampled). The full command log, raw output, and per-table dead-tuple breakdown are in [`bench/019-queue-storage-validation-2026-04-19.md`](bench/019-queue-storage-validation-2026-04-19.md). +Recorded local 5k-job runtime soak: **9,537 jobs/s**, **3.671 ms pickup p50**, **22.013 ms pickup p95**, **417 exact final dead tuples** (canonical runtime under the same workload: 9,686 jobs/s, 38.998 ms p95, dead tuples not sampled). The [full validation record](bench/019-queue-storage-validation-2026-04-19.md) contains the command log, raw output, and per-table dead-tuple breakdown. -The phase-driven portable comparison harness lives in a separate repo: [postgresql-job-queue-benchmarking](https://github.com/hardbyte/postgresql-job-queue-benchmarking). That harness records producer, subscriber, and end-to-end latency on a shared timebase while also sampling throughput, queue depth, and dead tuples over time. Recent runs place Awa ahead of pgque on end-to-end latency and on sustained throughput in clean-phase scenarios, while pgque holds a comparable dead-tuple profile (both are append-only / partition-rotated). See [`SYSTEM_COMPARISONS.md`](https://github.com/hardbyte/postgresql-job-queue-benchmarking/blob/main/SYSTEM_COMPARISONS.md) for the per-system architectural notes and [docs/benchmarking.md](../benchmarking.md) for awa's own regression methodology. +The phase-driven portable comparison harness lives in a separate repo: [postgresql-job-queue-benchmarking](https://github.com/hardbyte/postgresql-job-queue-benchmarking). That harness records producer, subscriber, and end-to-end latency on a shared timebase while also sampling throughput, queue depth, and dead tuples over time. Recent runs place Awa ahead of pgque on end-to-end latency and on sustained throughput in clean-phase scenarios, while pgque holds a comparable dead-tuple profile (both are append-only / partition-rotated). See the [cross-system comparison](https://github.com/hardbyte/postgresql-job-queue-benchmarking/blob/main/SYSTEM_COMPARISONS.md) for the per-system architectural notes and [AWA benchmarking](../benchmarking.md) for awa's own regression methodology. -The current pressure frontier after the split-head change is the lease plane: `queue_lanes` is no longer the dominant MVCC hotspot, but the mutable `active_leases` family still absorbs steady insert/delete churn and heartbeat updates. The current implementation now includes a short-job receipt path (`lease_claims` plus lazy materialization) that substantially reduces dead tuples for zero-deadline short jobs. Long-horizon profiling also showed that the append-only history alone was not enough: open-claim reads and rescue scans needed bounded access paths so they would not degrade into history scans. The original ADR-019 design used a bounded `open_receipt_claims` table for this; ADR-023 replaces it with partitioned `lease_claims`, explicit `lease_claim_closures`, compact `lease_claim_closure_batches`, live-set anti-joins over active partitions, and tiny per-slot rescue cursors in `claim_ring_slots`, eliminating the last per-claim MVCC churn source on the receipt plane. Further lease-plane work is still tracked in [`lease-plane-redesign-spike.md`](../archive/0.6-storage-design/lease-plane-redesign-spike.md). The remaining queue-level coordination controls are implemented as bounded claimers, queue striping (`queue_stripe_count`), and per-queue enqueue-head sharding (`queue_meta.enqueue_shards`). The archived [`bounded-claimers-plan.md`](../archive/0.6-storage-design/bounded-claimers-plan.md) and [`queue-striping-plan.md`](../archive/0.6-storage-design/queue-striping-plan.md) capture design history; current operator guidance lives in [`configuration.md`](../configuration.md#queue-storage-tuning). +The current pressure frontier after the split-head change is the lease plane: `queue_lanes` is no longer the dominant MVCC hotspot, but the mutable `active_leases` family still absorbs steady insert/delete churn and heartbeat updates. The current implementation now includes a short-job receipt path (`lease_claims` plus lazy materialization) that substantially reduces dead tuples for zero-deadline short jobs. Long-horizon profiling also showed that the append-only history alone was not enough: open-claim reads and rescue scans needed bounded access paths so they would not degrade into history scans. The original ADR-019 design used a bounded `open_receipt_claims` table for this; ADR-023 replaces it with partitioned `lease_claims`, explicit `lease_claim_closures`, compact `lease_claim_closure_batches`, live-set anti-joins over active partitions, and tiny per-slot rescue cursors in `claim_ring_slots`, eliminating the last per-claim MVCC churn source on the receipt plane. Further lease-plane work is still tracked in the [lease-plane redesign spike](https://github.com/hardbyte/awa/blob/main/docs/archive/0.6-storage-design/lease-plane-redesign-spike.md). The remaining queue-level coordination controls are implemented as bounded claimers, queue striping (`queue_stripe_count`), and per-queue enqueue-head sharding (`queue_meta.enqueue_shards`). The archived [bounded-claimers plan](https://github.com/hardbyte/awa/blob/main/docs/archive/0.6-storage-design/bounded-claimers-plan.md) and [queue-striping plan](https://github.com/hardbyte/awa/blob/main/docs/archive/0.6-storage-design/queue-striping-plan.md) capture design history; current operator guidance lives in [Queue storage tuning](../configuration.md#queue-storage-tuning). -Spec-level safety is checked by the segmented-storage TLA+ family — `AwaSegmentedStorage`, `AwaSegmentedStorageRaces`, `AwaStorageLockOrder`, `AwaSegmentedStorageTrace` — under [`correctness/storage/`](../../correctness/storage/). The TLA+ action → Rust function correspondence is in [`correctness/storage/MAPPING.md`](../../correctness/storage/MAPPING.md). +Spec-level safety is checked by the segmented-storage TLA+ family — `AwaSegmentedStorage`, `AwaSegmentedStorageRaces`, `AwaStorageLockOrder`, `AwaSegmentedStorageTrace` — under [the storage correctness models](https://github.com/hardbyte/awa/tree/main/correctness/storage). The [TLA+ action → Rust function mapping](https://github.com/hardbyte/awa/blob/main/correctness/storage/MAPPING.md) records the correspondence. ## Consequences diff --git a/docs/adr/022-descriptor-catalog.md b/docs/adr/022-descriptor-catalog.md index df1fe412..1d9076e9 100644 --- a/docs/adr/022-descriptor-catalog.md +++ b/docs/adr/022-descriptor-catalog.md @@ -37,4 +37,4 @@ Descriptors are explicitly off the hot path — dispatcher, claim, completion ba Descriptors are deliberately off the queue-storage hot path. Dispatch, claim, and completion never touch `awa.queue_descriptors` or `awa.job_kind_descriptors`; only `awa.queue_meta` (pause/resume) remains on the dispatcher's queue-state read. This preserves the ADR-019 property that operator-facing metadata changes cannot affect dispatch throughput. -See [architecture.md → Descriptors And Runtime Liveness](../architecture.md#descriptors-and-runtime-liveness) for the implementation details, hashing algorithm, and measured performance profile. +See [Architecture → Descriptors and runtime liveness](../architecture.md#descriptors-and-runtime-liveness) for the implementation details, hashing algorithm, and measured performance profile. diff --git a/docs/adr/023-receipt-plane-ring-partitioning.md b/docs/adr/023-receipt-plane-ring-partitioning.md index 51de8ee0..5ff6c61e 100644 --- a/docs/adr/023-receipt-plane-ring-partitioning.md +++ b/docs/adr/023-receipt-plane-ring-partitioning.md @@ -36,7 +36,7 @@ Non-goals: - Do not change the heartbeat / deadline / callback-timeout rescue contract. Those continue to live on `attempt_state` and `active_leases`. - Do not change the external API or the `(job_id, run_lease)` stale-writer guard. -- Do not introduce any new reservation or pre-start state. The archived [`lease-plane-redesign-spike`](../archive/0.6-storage-design/lease-plane-redesign-spike.md) record shows that direction has been tried and rejected repeatedly on cost grounds. +- Do not introduce any new reservation or pre-start state. The archived [lease-plane redesign spike](https://github.com/hardbyte/awa/blob/main/docs/archive/0.6-storage-design/lease-plane-redesign-spike.md) record shows that direction has been tried and rejected repeatedly on cost grounds. ## Decision @@ -158,13 +158,13 @@ Rejected. Marking a `closed_at` column and sweeping closed rows periodically kee ### Ship 0.6 with receipts off -Rejected. Shipping with receipts off lets 0.6 hit the dead-tuple budget today, but it leaves the short-job path on the mutable `leases` ring and defers the work tracked in the archived [`lease-plane-redesign-spike`](../archive/0.6-storage-design/lease-plane-redesign-spike.md). ADR-019's vacuum-aware intent is only satisfied when receipts are on by default and do not regress the dead-tuple budget. This ADR is the path to that posture. +Rejected. Shipping with receipts off lets 0.6 hit the dead-tuple budget today, but it leaves the short-job path on the mutable `leases` ring and defers the work tracked in the archived [lease-plane redesign spike](https://github.com/hardbyte/awa/blob/main/docs/archive/0.6-storage-design/lease-plane-redesign-spike.md). ADR-019's vacuum-aware intent is only satisfied when receipts are on by default and do not regress the dead-tuple budget. This ADR is the path to that posture. ## Relationship to Earlier ADRs - ADR-019 established the vacuum-aware discipline. This ADR applies that discipline to the one remaining hot table that did not follow it. - ADR-013 (run-lease-guarded finalization) is unchanged. The authoritative record for `(job_id, run_lease)` staleness moves from a bounded mutable frontier to partitioned append-only tables; the guarantee does not. -- The archived [`lease-plane-redesign-spike`](../archive/0.6-storage-design/lease-plane-redesign-spike.md) identifies `open_receipt_claims` as the compromise that unblocked the receipt-backed path. This ADR is the follow-through that the spike anticipated. +- The archived [lease-plane redesign spike](https://github.com/hardbyte/awa/blob/main/docs/archive/0.6-storage-design/lease-plane-redesign-spike.md) identifies `open_receipt_claims` as the compromise that unblocked the receipt-backed path. This ADR is the follow-through that the spike anticipated. ## Implementation and Validation Status @@ -180,8 +180,8 @@ This ADR has been implemented for 0.6: Validation evidence is split by purpose: -- Runtime and long-horizon evidence lives in [`bench/023-receipt-ring-validation-2026-04-26.md`](bench/023-receipt-ring-validation-2026-04-26.md). The recorded runs include the 115-minute 4x8 receipts-on long-horizon run and the 12-hour overnight run; receipt closure partitions stayed at 0 dead tuples across every phase, and receipt claims remained bounded. -- Spec and implementation mapping lives in [`../../correctness/storage/MAPPING.md`](../../correctness/storage/MAPPING.md). The storage TLA+ family models claim-ring rotation, partition prune safety, receipt rescue, running cancel, and DLQ retry trace witnesses. -- Operator-facing tuning and defaults live in [`../configuration.md`](../configuration.md#queue-storage-tuning). +- Runtime and long-horizon evidence lives in the [receipt-ring validation record](bench/023-receipt-ring-validation-2026-04-26.md). The recorded runs include the 115-minute 4x8 receipts-on long-horizon run and the 12-hour overnight run; receipt closure partitions stayed at 0 dead tuples across every phase, and receipt claims remained bounded. +- Spec and implementation correspondence lives in the [storage model mapping](https://github.com/hardbyte/awa/blob/main/correctness/storage/MAPPING.md). The storage TLA+ family models claim-ring rotation, partition prune safety, receipt rescue, running cancel, and DLQ retry trace witnesses. +- Operator-facing tuning and defaults live in [Queue storage tuning](../configuration.md#queue-storage-tuning). The detailed phase-by-phase implementation notes were intentionally kept out of this ADR. ADRs record the decision and its consequences; dated build logs, benchmark output, and branch-era investigation notes belong in validation artifacts or the 0.6 storage-design archive. diff --git a/docs/adr/033-per-key-execution-control.md b/docs/adr/033-per-key-execution-control.md index e93edb61..bc924b45 100644 --- a/docs/adr/033-per-key-execution-control.md +++ b/docs/adr/033-per-key-execution-control.md @@ -4,7 +4,7 @@ Accepted. Tracked in [#340](https://github.com/hardbyte/awa/issues/340). The worker-local baseline and the fleet-exact protocol remain experiment-gated by E5 in -[`docs/0.7-roadmap.md`](../0.7-roadmap.md); acceptance fixes the contract and candidate constraints, +[the 0.7 roadmap](../0.7-roadmap.md); acceptance fixes the contract and candidate constraints, not evidence that either tier has shipped. ## Context diff --git a/docs/adr/034-job-dependencies.md b/docs/adr/034-job-dependencies.md index 3702f1c8..d965f8a1 100644 --- a/docs/adr/034-job-dependencies.md +++ b/docs/adr/034-job-dependencies.md @@ -2,7 +2,7 @@ ## Status -Proposed — number claimed; full design tracked in [#14](https://github.com/hardbyte/awa/issues/14) per roadmap decision D7 in [`docs/0.7-roadmap.md`](../0.7-roadmap.md). +Proposed — number claimed; full design tracked in [#14](https://github.com/hardbyte/awa/issues/14) per roadmap decision D7 in [the 0.7 roadmap](../0.7-roadmap.md). ## Context diff --git a/docs/adr/035-backpressure-flow-control.md b/docs/adr/035-backpressure-flow-control.md index 4df9f78e..aa679ee9 100644 --- a/docs/adr/035-backpressure-flow-control.md +++ b/docs/adr/035-backpressure-flow-control.md @@ -2,7 +2,7 @@ ## Status -Proposed — number claimed; full design tracked in [#341](https://github.com/hardbyte/awa/issues/341) and experiment E6 in [`docs/0.7-roadmap.md`](../0.7-roadmap.md). +Proposed — number claimed; full design tracked in [#341](https://github.com/hardbyte/awa/issues/341) and experiment E6 in [the 0.7 roadmap](../0.7-roadmap.md). ## Context diff --git a/docs/adr/036-public-surface-stability-policy.md b/docs/adr/036-public-surface-stability-policy.md index abf3e302..955643fb 100644 --- a/docs/adr/036-public-surface-stability-policy.md +++ b/docs/adr/036-public-surface-stability-policy.md @@ -2,7 +2,7 @@ ## Status -Accepted — the policy text is [`docs/stability.md`](../stability.md), which is normative +Accepted — the [stability policy](../stability.md) is normative ([#369](https://github.com/hardbyte/awa/issues/369), roadmap decision D6). ## Context @@ -20,7 +20,7 @@ required when a new surface replaces an older covered entry point. ## Decision -Publish and maintain [`docs/stability.md`](../stability.md) as the single normative statement +Publish and maintain the [stability policy](../stability.md) as the single normative statement of what is stable: - A **surface-by-surface map**: what each surface promises, what is explicitly internal. diff --git a/docs/adr/037-canonical-engine-deprecation.md b/docs/adr/037-canonical-engine-deprecation.md index 84bd2e6f..e61a47d9 100644 --- a/docs/adr/037-canonical-engine-deprecation.md +++ b/docs/adr/037-canonical-engine-deprecation.md @@ -33,7 +33,7 @@ claimable. the upgrade guides. 2. **Formal deprecation in 0.7.** Any runtime whose effective storage resolves to canonical logs a startup warning naming the transition steps and this ADR. Release notes and - [`docs/upgrade-0.6-to-0.7.md`](../upgrade-0.6-to-0.7.md) state the deprecation. + the [0.6 to 0.7 upgrade guide](../upgrade-0.6-to-0.7.md) state the deprecation. 3. **Removal in 0.8.** Canonical claim, execution, and trigger paths are deleted. Upgrades step 0.5 → 0.6 (finalize) → 0.7 → 0.8. A read-only drain/inspection check may remain so 0.8 can still refuse legibly rather than misbehave against a canonical remnant. diff --git a/docs/adr/041-rolling-upgrade-policy.md b/docs/adr/041-rolling-upgrade-policy.md index ad3d99c7..573b14e6 100644 --- a/docs/adr/041-rolling-upgrade-policy.md +++ b/docs/adr/041-rolling-upgrade-policy.md @@ -2,7 +2,7 @@ ## Status -Accepted. The migration-authoring checklist is in [`docs/development.md`](../development.md#authoring-schema-migrations), and automated mixed-version rehearsal is tracked by [#427](https://github.com/hardbyte/awa/issues/427). +Accepted. The [migration-authoring checklist](../development.md#authoring-schema-migrations) and automated mixed-version rehearsal tracked by [#427](https://github.com/hardbyte/awa/issues/427) carry this decision into practice. ## Context diff --git a/docs/adr/043-postgresql-capability-functions.md b/docs/adr/043-postgresql-capability-functions.md index 87b43f3b..187d8550 100644 --- a/docs/adr/043-postgresql-capability-functions.md +++ b/docs/adr/043-postgresql-capability-functions.md @@ -205,7 +205,7 @@ narrow direct privileges, but that profile is not the strict no-table-grant prof ### Public names and internal names -Only entry points listed in [`docs/stability.md`](../stability.md) are public SQL contracts. +Only entry points listed in the [stability policy](../stability.md) are public SQL contracts. Public functions use domain names, not implementation or rollout suffixes. The initial v1 surface is: diff --git a/docs/adr/README.md b/docs/adr/README.md index 61707d63..9bbae615 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -2,63 +2,59 @@ Each file in this directory captures a single architectural decision — its context, the decision itself, the alternatives considered, and the consequences. ADRs are written when a decision has a non-obvious rationale, trades off across concerns, or will be hard to change later. -Template: `Status / Context / Decision / Consequences (positive, negative) / Alternatives Considered / Relationship to other ADRs`. Superseded ADRs stay in-place as historical context. +Each record preserves its status, context, decision, consequences, and alternatives. Accepted decisions are the default in the index; only exceptional states are labelled. Superseded and rejected records remain available as historical context. ## Index -| # | Title | Status | Summary | Relationship | -| --: | --- | --- | --- | --- | -| 001 | [Postgres-only](001-postgres-only.md) | Accepted | Single storage backend, no pluggable adapter layer | Current layout per ADR-019 | -| 002 | [BLAKE3 uniqueness](002-blake3-uniqueness.md) | Accepted | Uniqueness keys hashed with BLAKE3, claims in `awa.job_unique_claims` | — | -| 003 | [Heartbeat + deadline hybrid](003-heartbeat-deadline-hybrid.md) | Accepted | Two independent rescue paths cover crash and runaway failure modes | Fields moved to `active_leases` per ADR-019 | -| 004 | [PyO3 async bridge](004-pyo3-async-bridge.md) | Accepted | Python workers are callbacks invoked by the Rust runtime via PyO3 | — | -| 005 | [Priority aging](005-priority-aging.md) | Accepted | Effective priority aging prevents starvation; canonical uses maintenance aging, queue storage uses claim-time aging | Aging target changed under ADR-019 | -| 006 | [AwaTransaction as narrow SQL surface](006-awa-transaction.md) | Accepted | Python transaction bridge exposes only insert + commit/rollback | — | -| 007 | [Periodic cron jobs](007-periodic-cron-jobs.md) | Accepted | Leader-elected scheduler with atomic CTE enqueue | — | -| 008 | [COPY batch ingestion](008-copy-batch-ingestion.md) | Accepted | Session-local staging table + COPY for 10k+-row inserts | Routes through `insert_job_compat` under ADR-019 | -| 009 | [Python sync support](009-python-sync-support.md) | Accepted | Every async method has a `_sync` counterpart for Django/Flask | — | -| 010 | [Per-queue rate limiting](010-rate-limiting.md) | Accepted | Per-worker token bucket composes with both concurrency modes | Storage-plane-agnostic | -| 011 | [Weighted concurrency](011-weighted-concurrency.md) | Accepted | Global worker pool with per-queue min guarantees and weighted overflow | Storage-plane-agnostic | -| 012 | [Hot / deferred job storage](012-hot-deferred-job-storage.md) | **Superseded by 019** | Manual hot/cold split of the `awa.jobs` heap | Superseded by ADR-019 | -| 013 | [Run lease and guarded finalization](013-run-lease-and-guarded-finalization.md) | Accepted | `run_lease` is the per-attempt identity; every finalize matches on it | Composite key on `active_leases` per ADR-019 | -| 014 | [Structured progress and metadata](014-structured-progress.md) | Accepted | JSONB progress buffer with heartbeat piggyback + atomic state-transition flush | Progress storage moved to `attempt_state` per ADR-019 | -| 015 | [Builder-side lifecycle hooks](015-post-commit-lifecycle-hooks.md) | Accepted | Builder-side hooks fire after claim start and guarded finalization commits | Guard lives on `active_leases` per ADR-019 | -| 016 | [Public Rust Postgres enqueue adapter API](016-rust-postgres-enqueue-adapter-api.md) | Accepted | Public Postgres insert-preparation contract plus built-in tokio-postgres adapter | Enables external Rust enqueue adapters | -| 017 | [Python insert-only transaction bridging](017-python-transaction-bridging.md) | Accepted | Python `awa.Transaction` is a thin wrapper over the Rust insert path | — | -| 018 | [HTTP Worker for serverless job dispatch](018-http-worker.md) | Accepted | `Worker` impl that dispatches to Lambda / Cloud Run via HTTP + BLAKE3-signed callbacks | Uses callback surface from ADR-021 | -| 019 | [Queue Storage Engine](019-queue-storage-redesign.md) | Accepted | Append-only ready / terminal entries, narrow `active_leases`, optional `attempt_state`, rotating segments | Supersedes ADR-012 | -| 020 | [Dead Letter Queue](020-dead-letter-queue.md) | Accepted | First-class DLQ storage family with per-queue opt-in, retention, and operator retry/purge | Lives inside ADR-019 | -| 021 | [Sequential callbacks and callback heartbeats](021-enhanced-external-wait.md) | Accepted | `wait_for_callback()` + `resume_external()` for multi-step orchestration; `heartbeat_callback` for long-running externals | Callback state moved to `active_leases` per ADR-019 | -| 022 | [Descriptor catalog](022-descriptor-catalog.md) | Accepted | `queue_descriptors` / `job_kind_descriptors` tables, BLAKE3-hashed, code-declared, off the hot path | Off the queue-storage hot path | -| 023 | [Receipt plane ring partitioning](023-receipt-plane-ring-partitioning.md) | Accepted | Partitioned `lease_claims`, explicit closures, and compact closure batches replace `open_receipt_claims`; receipts default on in 0.6 | Refines ADR-019 receipt plane | -| 024 | Deferred `done_entries` materialisation | Rejected | Investigated as a rotation guard; reverted in `053fec1` once a simpler integration test gave equivalent coverage | Historical | -| 025 | [Sharded enqueue heads](025-sharded-enqueue-heads.md) | Accepted | Per-queue `enqueue_shards` (default 1) spreads `queue_enqueue_heads` row-lock contention across N rows; FIFO becomes per-shard at S>1 | Refines ADR-019 enqueue path | -| 026 | [Narrow terminal history](026-narrow-terminal-history.md) | Accepted | Ready-backed terminal rows store only terminal facts, compact receipt completions use batch terminal history, and exact counts combine retained compact batches with append-only `done_entries` terminal-count deltas plus async sealed-slot rollup | Refines ADR-019 terminal path | -| 027 | [Callback ingress as a deployable surface](027-callback-ingress-surface.md) | Proposed | Separate signed callback ingress from the admin UI/API and expose callback-only embedding/CLI paths | Refines ADR-018 and ADR-021; uses ADR-029 for durable callback-driven side effects | -| 028 | [Maintenance-only runtime role](028-maintenance-only-runtime-role.md) | Proposed | Run promotion, rescue, pruning, and metadata maintenance without claiming or executing user jobs | Complements ADR-027 and ADR-018; uses ADR-029 for durable rescue-driven side effects | -| 029 | [Transactional follow-up jobs](029-transactional-followup-jobs.md) | Accepted | Durable lifecycle side effects are delivered by enqueuing follow-up Awa jobs — atomically with the triggering state UPDATE for worker-driven outcomes and for callback resolution via the worker `Client`, best-effort in a separate transaction for maintenance rescue; hooks remain for observation | Codifies ADR-015's "enqueue another job" guidance; addresses the durable-event punt in ADRs 027/028 | -| 030 | [Durable batch operations for operator bulk mutation](030-batch-operations.md) | Accepted | Filter-driven async bulk mutation with preview, progress, cancellation, retention, and maintenance-led execution; v0.6 starts with `set_priority` and `move_queue` | Refines ADR-019/025 operator mutation paths; complements ADR-028 | -| 031 | [Partitioned queues](031-partitioned-queues.md) | Accepted | First-class logical queue partitioning over ordinary physical queues, with domain-separated key routing and Python per-job COPY opts | Composes ADR-019/023/026 storage guarantees; refines the ADR-025 sharding interaction | -| 032 | [Failed terminal retention floor](032-failed-terminal-retention.md) | Accepted | Queue-storage prune carries in-floor `failed` terminal rows forward into the live segment as wide synthetic rows so they stay retryable for at least `failed_retention`; rows aged past the floor are folded into `queue_terminal_rollups.pruned_failed_count` and surfaced via `QueueCounts.pruned_failed` | Amends ADR-026's one-retention-unit consequence; refines ADR-019 terminal prune path | -| 033 | [Per-key execution control](033-per-key-execution-control.md) | Accepted | Fleet-exact keyed grants with shard locality, bounded lane probing, and transactional closure wakeups; fairness remains separate (#340) | Composes ADR-005/010/011/025/031; E5 compares row-local claim reuse, a separate ledger, and proved parking under ADR-023/026; caller completion per ADR-042 | -| 034 | [Job dependencies](034-job-dependencies.md) | Proposed | Single-parent A→B chaining: `waiting_on` parking state promoted transactionally by the parent's guarded finalization, with an `on_parent_failure` policy (#14) | Builds on ADR-029; workflow engine remains a non-goal | -| 035 | [Backpressure and flow control](035-backpressure-flow-control.md) | Proposed | Soft depth signals from lane-head cursors by default, opt-in hard rejection, paced-producer helpers (#341) | Makes the ADR-006 transactional-enqueue tension explicit; composes ADR-025/031 | -| 036 | [Public surface stability policy](036-public-surface-stability-policy.md) | Accepted | `docs/stability.md` is the normative surface-by-surface compatibility map, deprecation policy, and binary/schema skew statement (#369); enforced via #402 semver checks and the #367 compat matrix | Governs ADR-016 and the #342 SQL producer contract; constrains all future surface-touching ADRs | -| 037 | [Canonical engine deprecation](037-canonical-engine-deprecation.md) | Accepted | 0.7 `awa migrate` refuses unfinalized clusters (fresh installs exempt); canonical deprecated with a startup warning in 0.7, claim/execution/trigger paths removed in 0.8 (#370) | Completes ADR-019's supersession of the pre-0.6 model; bounds the #360 dual-engine matrix | -| 038 | [Queue runtime overrides](038-queue-runtime-overrides.md) | Accepted | Hot-reloadable per-queue dispatch knobs via nullable `queue_meta` override columns, refreshed by dispatchers on a slow cadence; rate-limit retune and non-zero deadline changes only (Tier 2: #397) | Extends the queue_meta pause/resume control-plane pattern; respects ADR-026; guards ADR-023 claim-mode selection | -| 039 | [End-to-end trace propagation](039-trace-propagation.md) | Accepted | W3C `traceparent` captured at enqueue into the reserved `awa:traceparent` metadata key; first attempts join the producer trace as remote children, retries start fresh root traces with span links; OTel messaging semantic conventions on both sides; default-on, `AWA_TRACE_CAPTURE=off` kill switch (#110) | Builds on ADR-004's single Rust execution path; composes ADR-032 SQL producers via the documented key; adds the reserved `awa:` metadata namespace to ADR-036's policy | -| 040 | [Append-only ring-rotation ledgers](040-append-only-ring-rotation-ledger.md) | Accepted | Ring cursors move from mutable `{ring}_ring_state` singletons to append-only `{ring}_ring_rotations` ledgers (cursor = max-generation row; CAS on the generation PK); staged `columns` -> `ledger` authority supports the 0.6.2/0.7 rollout; queue prune appends `queue_terminal_rollup_deltas` folded by horizon-gated maintenance (#371) | Extends ADR-023's ring plane; applies ADR-026's dead-tuple reclaim discipline to the ring control plane; modelled in AwaStorageLockOrder / AwaDeadTupleContract | -| 041 | [Rolling upgrade policy](041-rolling-upgrade-policy.md) | Accepted | Rolling upgrades use expand → capability-gated flip → later contract; version floors guard expand migrations, while database fences and real N-1 rehearsals guard irreversible flips. | Generalizes ADR-040; strengthens ADR-036; checklist in `docs/development.md` | -| 042 | [Caller-owned finalization transactions](042-caller-owned-finalization-transactions.md) | Accepted | A distinct handler type commits application rows and exact-lease completion in one transaction through a least-privilege finalization function (#401) | Extends ADR-013/021/029/034; hardened SQL boundary under ADR-036; conditionally composes with ADR-033 and #342 | -| 043 | [PostgreSQL capability functions and least-privilege runtime roles](043-postgresql-capability-functions.md) | Proposed | Replace blanket runtime table/function grants with allowlisted, role-specific capability entry points owned by a bounded execution role (#452); blanket definer conversion is rejected | Extends #91/ADR-036/041/042; separates producer, executor, maintenance, admin, callback, and application-finalizer authority | +| # | Decision | Summary | Status | +| --: | --- | --- | --- | +| 001 | [Postgres-only](001-postgres-only.md) | Single storage backend, no pluggable adapter layer. | | +| 002 | [BLAKE3 uniqueness](002-blake3-uniqueness.md) | Uniqueness keys hashed with BLAKE3, claims in `awa.job_unique_claims`. | | +| 003 | [Heartbeat + deadline hybrid](003-heartbeat-deadline-hybrid.md) | Two independent rescue paths cover crash and runaway failure modes. | | +| 004 | [PyO3 async bridge](004-pyo3-async-bridge.md) | Python workers are callbacks invoked by the Rust runtime via PyO3. | | +| 005 | [Priority aging](005-priority-aging.md) | Effective priority aging prevents starvation; canonical uses maintenance aging, queue storage uses claim-time aging. | | +| 006 | [AwaTransaction as narrow SQL surface](006-awa-transaction.md) | Python transaction bridge exposes only insert + commit/rollback. | | +| 007 | [Periodic cron jobs](007-periodic-cron-jobs.md) | Leader-elected scheduler with atomic CTE enqueue. | | +| 008 | [COPY batch ingestion](008-copy-batch-ingestion.md) | Session-local staging table + COPY for 10k+-row inserts. | | +| 009 | [Python sync support](009-python-sync-support.md) | Every async method has a `_sync` counterpart for Django/Flask. | | +| 010 | [Per-queue rate limiting](010-rate-limiting.md) | Per-worker token bucket composes with both concurrency modes. | | +| 011 | [Weighted concurrency](011-weighted-concurrency.md) | Global worker pool with per-queue min guarantees and weighted overflow. | | +| 012 | [Hot / deferred job storage](012-hot-deferred-job-storage.md) | Manual hot/cold split of the `awa.jobs` heap. | Superseded by 019 | +| 013 | [Run lease and guarded finalization](013-run-lease-and-guarded-finalization.md) | `run_lease` is the per-attempt identity; every finalize matches on it. | | +| 014 | [Structured progress and metadata](014-structured-progress.md) | JSONB progress buffer with heartbeat piggyback + atomic state-transition flush. | | +| 015 | [Builder-side lifecycle hooks](015-post-commit-lifecycle-hooks.md) | Builder-side hooks fire after claim start and guarded finalization commits. | | +| 016 | [Public Rust Postgres enqueue adapter API](016-rust-postgres-enqueue-adapter-api.md) | Public Postgres insert-preparation contract plus built-in tokio-postgres adapter. | | +| 017 | [Python insert-only transaction bridging](017-python-transaction-bridging.md) | Python `awa.Transaction` is a thin wrapper over the Rust insert path. | | +| 018 | [HTTP Worker for serverless job dispatch](018-http-worker.md) | `Worker` impl that dispatches to Lambda / Cloud Run via HTTP + BLAKE3-signed callbacks. | | +| 019 | [Queue Storage Engine](019-queue-storage-redesign.md) | Append-only ready / terminal entries, narrow `active_leases`, optional `attempt_state`, rotating segments. | | +| 020 | [Dead Letter Queue](020-dead-letter-queue.md) | First-class DLQ storage family with per-queue opt-in, retention, and operator retry/purge. | | +| 021 | [Sequential callbacks and callback heartbeats](021-enhanced-external-wait.md) | `wait_for_callback()` + `resume_external()` for multi-step orchestration; `heartbeat_callback` for long-running externals. | | +| 022 | [Descriptor catalog](022-descriptor-catalog.md) | `queue_descriptors` / `job_kind_descriptors` tables, BLAKE3-hashed, code-declared, off the hot path. | | +| 023 | [Receipt plane ring partitioning](023-receipt-plane-ring-partitioning.md) | Partitioned `lease_claims`, explicit closures, and compact closure batches replace `open_receipt_claims`; receipts default on in 0.6. | | +| 024 | Deferred `done_entries` materialisation | Investigated as a rotation guard; reverted in `053fec1` once a simpler integration test gave equivalent coverage. | Rejected | +| 025 | [Sharded enqueue heads](025-sharded-enqueue-heads.md) | Per-queue `enqueue_shards` (default 1) spreads `queue_enqueue_heads` row-lock contention across N rows; FIFO becomes per-shard at S>1. | | +| 026 | [Narrow terminal history](026-narrow-terminal-history.md) | Ready-backed terminal rows store only terminal facts, compact receipt completions use batch terminal history, and exact counts combine retained compact batches with append-only `done_entries` terminal-count deltas plus async sealed-slot rollup. | | +| 027 | [Callback ingress as a deployable surface](027-callback-ingress-surface.md) | Separate signed callback ingress from the admin UI/API and expose callback-only embedding/CLI paths. | Proposed | +| 028 | [Maintenance-only runtime role](028-maintenance-only-runtime-role.md) | Run promotion, rescue, pruning, and metadata maintenance without claiming or executing user jobs. | Proposed | +| 029 | [Transactional follow-up jobs](029-transactional-followup-jobs.md) | Durable lifecycle side effects are delivered by enqueuing follow-up Awa jobs — atomically with the triggering state UPDATE for worker-driven outcomes and for callback resolution via the worker `Client`, best-effort in a separate transaction for maintenance rescue; hooks remain for observation. | | +| 030 | [Durable batch operations for operator bulk mutation](030-batch-operations.md) | Filter-driven async bulk mutation with preview, progress, cancellation, retention, and maintenance-led execution; v0.6 starts with `set_priority` and `move_queue`. | | +| 031 | [Partitioned queues](031-partitioned-queues.md) | First-class logical queue partitioning over ordinary physical queues, with domain-separated key routing and Python per-job COPY opts. | | +| 032 | [Failed terminal retention floor](032-failed-terminal-retention.md) | Queue-storage prune carries in-floor `failed` terminal rows forward into the live segment as wide synthetic rows so they stay retryable for at least `failed_retention`; rows aged past the floor are folded into `queue_terminal_rollups.pruned_failed_count` and surfaced via `QueueCounts.pruned_failed`. | | +| 033 | [Per-key execution control](033-per-key-execution-control.md) | Fleet-exact keyed grants with shard locality, bounded lane probing, and transactional closure wakeups; fairness remains separate (#340). | | +| 034 | [Job dependencies](034-job-dependencies.md) | Single-parent A→B chaining: `waiting_on` parking state promoted transactionally by the parent's guarded finalization, with an `on_parent_failure` policy (#14). | Proposed | +| 035 | [Backpressure and flow control](035-backpressure-flow-control.md) | Soft depth signals from lane-head cursors by default, opt-in hard rejection, paced-producer helpers (#341). | Proposed | +| 036 | [Public surface stability policy](036-public-surface-stability-policy.md) | `docs/stability.md` is the normative surface-by-surface compatibility map, deprecation policy, and binary/schema skew statement (#369); enforced via #402 semver checks and the #367 compat matrix. | | +| 037 | [Canonical engine deprecation](037-canonical-engine-deprecation.md) | 0.7 `awa migrate` refuses unfinalized clusters (fresh installs exempt); canonical deprecated with a startup warning in 0.7, claim/execution/trigger paths removed in 0.8 (#370). | | +| 038 | [Queue runtime overrides](038-queue-runtime-overrides.md) | Hot-reloadable per-queue dispatch knobs via nullable `queue_meta` override columns, refreshed by dispatchers on a slow cadence; rate-limit retune and non-zero deadline changes only (Tier 2: #397). | | +| 039 | [End-to-end trace propagation](039-trace-propagation.md) | W3C `traceparent` captured at enqueue into the reserved `awa:traceparent` metadata key; first attempts join the producer trace as remote children, retries start fresh root traces with span links; OTel messaging semantic conventions on both sides; default-on, `AWA_TRACE_CAPTURE=off` kill switch (#110). | | +| 040 | [Append-only ring-rotation ledgers](040-append-only-ring-rotation-ledger.md) | Ring cursors move from mutable `{ring}_ring_state` singletons to append-only `{ring}_ring_rotations` ledgers (cursor = max-generation row; CAS on the generation PK); staged `columns` -> `ledger` authority supports the 0.6.2/0.7 rollout; queue prune appends `queue_terminal_rollup_deltas` folded by horizon-gated maintenance (#371). | | +| 041 | [Rolling upgrade policy](041-rolling-upgrade-policy.md) | Rolling upgrades use expand → capability-gated flip → later contract; version floors guard expand migrations, while database fences and real N-1 rehearsals guard irreversible flips. | | +| 042 | [Caller-owned finalization transactions](042-caller-owned-finalization-transactions.md) | A distinct handler type commits application rows and exact-lease completion in one transaction through a least-privilege finalization function (#401). | | +| 043 | [PostgreSQL capability functions and least-privilege runtime roles](043-postgresql-capability-functions.md) | Replace blanket runtime table/function grants with allowlisted, role-specific capability entry points owned by a bounded execution role (#452); blanket definer conversion is rejected. | Proposed | -## Validation artifacts +## Correctness evidence -Runtime validation for ADR-019 is recorded in [`bench/019-queue-storage-validation-2026-04-19.md`](bench/019-queue-storage-validation-2026-04-19.md) (exact commands, raw output, measured numbers). - -Runtime validation for ADR-023 is recorded in [`bench/023-receipt-ring-validation-2026-04-26.md`](bench/023-receipt-ring-validation-2026-04-26.md) (receipt-ring long-horizon and overnight evidence). - -TLA+ correctness models that pin spec-level invariants are under [`../../correctness/`](../../correctness/) — the segmented-storage family (`AwaSegmentedStorage`, `AwaSegmentedStorageRaces`, `AwaStorageLockOrder`, `AwaSegmentedStorageTrace`) maps to ADR-019 and ADR-020; the worker-runtime family (`AwaCore`, `AwaExtended`, `AwaBatcher`, `AwaCbk`, `AwaDispatchClaim`, `AwaViewTrigger`, `AwaCron`) covers rescue, batcher, callback race, dispatcher claim, view-trigger concurrency, and cron double-fire. +Executable TLA+ models live under [`correctness/`](https://github.com/hardbyte/awa/tree/main/correctness). The storage models cover segmented storage, storage races, lock ordering, and trace refinement; the runtime models cover claim, rescue, callbacks, batching, cron, and view-trigger concurrency. Benchmark evidence belongs with the benchmark artifacts, not in this decision index. ## Conventions diff --git a/docs/architecture.md b/docs/architecture.md index a159b612..2c870dab 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,17 +2,13 @@ Awa (Māori: river) is a Postgres-native background job queue for Rust and Python. Postgres is the sole infrastructure dependency: there is no Redis, RabbitMQ, sidecar scheduler, or separate lease store. Producers enqueue inside ordinary Postgres transactions, workers claim and complete jobs through the same database, and one elected worker runs cluster-wide maintenance. -This document is ordered by the questions operators and contributors usually need answered first: +Awa keeps application data and background work in one transactional system of record. Producers commit jobs with their business writes; workers claim attempts using lease-guarded PostgreSQL transitions; maintenance makes deferred work runnable, rescues abandoned attempts, and reclaims old ring partitions. There is no second broker whose acknowledgement state can diverge from the database transaction. -- What owns the runtime? -- What deployment assumptions shape that runtime? -- Where does state live? -- How does a job move through storage? -- How does Awa recover from crashes and stale attempts? -- How are partitions rotated and reclaimed? -- Which surfaces are operational rather than hot-path? +![Awa system architecture](assets/architecture-system.svg) -For migration details see [migrations.md](migrations.md). For user-facing knobs see [configuration.md](configuration.md). +The diagram is the useful boundary: application processes produce work, worker processes execute it, and PostgreSQL is authoritative for both job state and coordination. The admin surface observes and controls that system; it is not part of the dispatch hot path. + +For migration details see [Migrations](migrations.md). For user-facing knobs see [Configuration](configuration.md). ## Terms @@ -290,22 +286,22 @@ Core safety invariants are modeled in TLA+: | Model | Focus | | --- | --- | -| [`AwaCore`](../correctness/core/AwaCore.tla) | job lifecycle, retry/fail/cancel transitions, callback states | -| [`AwaBatcher`](../correctness/core/AwaBatcher.tla) | guarded completion batching and stale-result rejection | -| [`AwaExtended`](../correctness/protocol/AwaExtended.tla) | multi-instance shutdown, rescue, permit, leadership, and bounded fairness protocol | -| [`AwaSegmentedStorage`](../correctness/storage/AwaSegmentedStorage.tla) | queue-storage lifecycle, rotate/prune safety, DLQ round-trip, receipt rescue | -| [`AwaSegmentedStorageRaces`](../correctness/storage/AwaSegmentedStorageRaces.tla) | claim-vs-rotate/prune interleavings | -| [`AwaSegmentedStorageTrace`](../correctness/storage/AwaSegmentedStorageTrace.tla) | concrete runtime trace acceptance for representative queue-storage flows | -| [`AwaShardedPrune`](../correctness/storage/AwaShardedPrune.tla) | cross-shard ready/terminal prune matching by `enqueue_shard` | -| [`AwaStorageLockOrder`](../correctness/storage/AwaStorageLockOrder.tla) | Postgres lock ordering across claim, rotate, and prune | -| [`AwaStorageTransition`](../correctness/storage/AwaStorageTransition.tla) | queue-storage transition prepare, mixed-entry, finalize, and abort gates | -| [`AwaDeadTupleContract`](../correctness/storage/AwaDeadTupleContract.tla) | hot-table reclaim-kind contract for partition truncate and bounded warm tables | -| [`AwaCbk`](../correctness/races/AwaCbk.tla) | callback registration/resume/finalization races | -| [`AwaDispatchClaim`](../correctness/races/AwaDispatchClaim.tla) | availability re-check at dispatch claim commit | -| [`AwaViewTrigger`](../correctness/races/AwaViewTrigger.tla) | `awa.jobs` view trigger concurrency and version checks | -| [`AwaCron`](../correctness/races/AwaCron.tla) | cron double-fire prevention under leader failover | - -The storage model-to-code correspondence is maintained in [`correctness/storage/MAPPING.md`](../correctness/storage/MAPPING.md). Runtime tests replay representative storage traces against these models, and the benchmark notes document long-horizon partition and dead-tuple validation for ADR-019 and ADR-023. Public SQL projections such as `awa.jobs` and admin counts are treated as refinements over the modeled storage state; they need code-level regression tests as well as TLA+ lifecycle coverage. +| [`AwaCore`](https://github.com/hardbyte/awa/blob/main/correctness/core/AwaCore.tla) | job lifecycle, retry/fail/cancel transitions, callback states | +| [`AwaBatcher`](https://github.com/hardbyte/awa/blob/main/correctness/core/AwaBatcher.tla) | guarded completion batching and stale-result rejection | +| [`AwaExtended`](https://github.com/hardbyte/awa/blob/main/correctness/protocol/AwaExtended.tla) | multi-instance shutdown, rescue, permit, leadership, and bounded fairness protocol | +| [`AwaSegmentedStorage`](https://github.com/hardbyte/awa/blob/main/correctness/storage/AwaSegmentedStorage.tla) | queue-storage lifecycle, rotate/prune safety, DLQ round-trip, receipt rescue | +| [`AwaSegmentedStorageRaces`](https://github.com/hardbyte/awa/blob/main/correctness/storage/AwaSegmentedStorageRaces.tla) | claim-vs-rotate/prune interleavings | +| [`AwaSegmentedStorageTrace`](https://github.com/hardbyte/awa/blob/main/correctness/storage/AwaSegmentedStorageTrace.tla) | concrete runtime trace acceptance for representative queue-storage flows | +| [`AwaShardedPrune`](https://github.com/hardbyte/awa/blob/main/correctness/storage/AwaShardedPrune.tla) | cross-shard ready/terminal prune matching by `enqueue_shard` | +| [`AwaStorageLockOrder`](https://github.com/hardbyte/awa/blob/main/correctness/storage/AwaStorageLockOrder.tla) | Postgres lock ordering across claim, rotate, and prune | +| [`AwaStorageTransition`](https://github.com/hardbyte/awa/blob/main/correctness/storage/AwaStorageTransition.tla) | queue-storage transition prepare, mixed-entry, finalize, and abort gates | +| [`AwaDeadTupleContract`](https://github.com/hardbyte/awa/blob/main/correctness/storage/AwaDeadTupleContract.tla) | hot-table reclaim-kind contract for partition truncate and bounded warm tables | +| [`AwaCbk`](https://github.com/hardbyte/awa/blob/main/correctness/races/AwaCbk.tla) | callback registration/resume/finalization races | +| [`AwaDispatchClaim`](https://github.com/hardbyte/awa/blob/main/correctness/races/AwaDispatchClaim.tla) | availability re-check at dispatch claim commit | +| [`AwaViewTrigger`](https://github.com/hardbyte/awa/blob/main/correctness/races/AwaViewTrigger.tla) | `awa.jobs` view trigger concurrency and version checks | +| [`AwaCron`](https://github.com/hardbyte/awa/blob/main/correctness/races/AwaCron.tla) | cron double-fire prevention under leader failover | + +The storage model-to-code correspondence is maintained in [`correctness/storage/MAPPING.md`](https://github.com/hardbyte/awa/blob/main/correctness/storage/MAPPING.md). Runtime tests replay representative storage traces against these models, and the benchmark notes document long-horizon partition and dead-tuple validation for ADR-019 and ADR-023. Public SQL projections such as `awa.jobs` and admin counts are treated as refinements over the modeled storage state; they need code-level regression tests as well as TLA+ lifecycle coverage. ## Crate Structure diff --git a/docs/archive/0.6-storage-design/queue-striping-plan.md b/docs/archive/0.6-storage-design/queue-striping-plan.md index 5e80f301..7f9c24b2 100644 --- a/docs/archive/0.6-storage-design/queue-striping-plan.md +++ b/docs/archive/0.6-storage-design/queue-striping-plan.md @@ -14,7 +14,7 @@ Queue striping is now a queue-storage tuning knob, not a separate engine. - Stats aggregate back to the logical queue; stripe names are an internal diagnostic surface. - Maintenance rotates/prunes the existing queue-storage table families; striping does not add a new job lifecycle state. -The implementation lives in `awa-model/src/queue_storage.rs` (search for `queue_stripe_count`). Operator-facing configuration is documented in [`configuration.md`](../../configuration.md#queue-storage-tuning), and the runtime architecture is summarized in [`architecture.md`](../../architecture.md#queue-striping-and-claim-authority). +The implementation lives in `awa-model/src/queue_storage.rs` (search for `queue_stripe_count`). Operator-facing configuration is documented in [`configuration.md`](../../configuration.md#queue-storage-tuning), and the runtime architecture is summarized in [`architecture.md`](../../architecture.md#enqueue-and-claim). ## Why this exists diff --git a/docs/archive/prd.md b/docs/archive/prd.md index aac3d708..31473f45 100644 --- a/docs/archive/prd.md +++ b/docs/archive/prd.md @@ -1,6 +1,6 @@ # AWA — Product Requirements Document (archived) -> **Historical product brief — archived.** This document captures the 0.x design intent at the time it was written (early 2026) and is retained for context. It is **not** a description of current behavior; for that, see [`docs/architecture.md`](../architecture.md), [`docs/configuration.md`](../configuration.md), and the ADRs under [`docs/adr/`](../adr/). +> **Historical product brief — archived.** This document captures the 0.x design intent at the time it was written (early 2026) and is retained for context. It is **not** a description of current behavior; for that, see [`docs/architecture.md`](../architecture.md), [`docs/configuration.md`](../configuration.md), and the [architecture decision index](../adr/README.md). _Version: 1.0 — March 2026 (archived)_ diff --git a/docs/assets/architecture-flow.svg b/docs/assets/architecture-flow.svg new file mode 100644 index 00000000..776349a6 --- /dev/null +++ b/docs/assets/architecture-flow.svg @@ -0,0 +1,62 @@ + + AWA architecture + Applications atomically commit business data and jobs to PostgreSQL. Workers claim jobs from PostgreSQL and write durable outcomes back. Operators inspect the same system of record. + + + + + + + + + + + + + + + ONE POSTGRESQL SYSTEM OF RECORD + + + + + + Applications + Rust · Python · SQL bridges + Own the application transaction + + + + + + PostgreSQL + application data + AWA jobs + + + + + Workers + Claim · heartbeat · execute + Complete · retry · wait + + + + + ATOMIC COMMIT + + + CLAIM + + + DURABLE OUTCOME + + + + OPERATE · CLI · WEB UI · METRICS · TRACES · SQL + + + + diff --git a/docs/assets/architecture-system.svg b/docs/assets/architecture-system.svg new file mode 100644 index 00000000..62b86781 --- /dev/null +++ b/docs/assets/architecture-system.svg @@ -0,0 +1,30 @@ + + Awa system architecture + Applications enqueue jobs transactionally into PostgreSQL. Rust and Python worker processes claim jobs, execute handlers, and finalize guarded attempts. One elected maintenance leader promotes, rescues, and reclaims storage. Operator tools observe and control the database. + + + + + + PostgreSQL is the coordination boundary + + ApplicationsRust · Python · SQL adaptersBusiness write + enqueue + + + PostgreSQLjobs · attempts · control state + + Worker processesClaim · execute · heartbeatGuarded complete / retry / wait + + Maintenance leaderPromote · rescue · rotateOne elected worker at a time + + Operator surfacesCLI · admin UI · metrics · tracesInspect and control + COMMIT + CLAIM + FINALIZE + MAINTAIN + OBSERVE + Every state transition is durable; attempt tokens prevent stale workers from finalizing newer attempts. + diff --git a/docs/assets/job-lifecycle.svg b/docs/assets/job-lifecycle.svg new file mode 100644 index 00000000..32b1dd26 --- /dev/null +++ b/docs/assets/job-lifecycle.svg @@ -0,0 +1,78 @@ + + AWA job state machine + Scheduled and retryable jobs become available when due. Workers claim available jobs as running. A running attempt can complete, snooze, retry, wait for an external callback, fail, or cancel. Callback outcomes can resume, complete, fail, or retry the job. + + + + + + + + DURABLE JOB STATE + + + + + + + + + + + + + + + + + available + runnable now + + + running + claimed + heartbeating + + + completed + terminal success + + + scheduled + future run · same attempt + + + retryable + future run · next attempt + + + waiting_external + parked for callback + + + cancelled + terminal + + + failed + terminal · DLQ + + + + CLAIMSUCCESS + SNOOZERETRY + WAITCANCELTERMINAL / EXHAUSTED + + + + + + External callback outcomes + resume → running · complete → completed · retry → available · fail → failed · timeout → retryable or failed + + + diff --git a/docs/assets/logo.svg b/docs/assets/logo.svg new file mode 100644 index 00000000..f4e0d7cd --- /dev/null +++ b/docs/assets/logo.svg @@ -0,0 +1,6 @@ + + AWA + + + + diff --git a/docs/assets/transactional-enqueue.svg b/docs/assets/transactional-enqueue.svg new file mode 100644 index 00000000..0b377d2f --- /dev/null +++ b/docs/assets/transactional-enqueue.svg @@ -0,0 +1,54 @@ + + Transactional enqueue + Inside one PostgreSQL transaction, the application inserts business data and an AWA job, then commits both. Only after commit can a worker claim the job. + + + + + + + + ATOMIC HANDOFF + + + + Application + request · service · command + owns the transaction + + + ONE POSTGRESQL TRANSACTION + + + BEGIN + + + INSERT application row + for example, an order + + + INSERT AWA job + using the same connection + + + COMMIT + + + Worker + claims only after + the commit is visible + + + + + + + + + BOTH ROWS COMMIT — OR NEITHER DOES + + + diff --git a/docs/benchmarking.md b/docs/benchmarking.md index f77fa76f..fccdbee1 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -143,52 +143,6 @@ The companion repo currently benchmarks awa against pgque, procrastinate, pg-bos The two tracks are deliberately separate. If they ever diverge on workload shape or thresholds, the awa-only benches in this file are canonical for awa's own numbers and the cross-system runner defers. -### 2026-05-03 cross-system reference run - -An overnight run of the companion benchmark repo compared `awa` 0.6 alpha builds with the same phase-driven harness. Treat these as reference results for shape and regression tracking, not universal product guarantees. These numbers are from pre-0.6.0 alpha builds; for the current 0.6.0 pinned-MVCC shape see the [MVCC Horizon Benchmark](#mvcc-horizon-benchmark) section above and the #169 benchmark evidence in the [CHANGELOG](../CHANGELOG.md). - -Key observations: - -- `awa` peak throughput improved by `49%` from 0.6.0-alpha.2 to 0.6.0-alpha.3 at 128 workers and one replica: `4,576` to `6,834` jobs/s. The same run showed a roughly `1.5x` to `1.7x` improvement across the worker-count matrix. -- Phase A matrix shape for `awa`: `296` -> `1,115` -> `3,961` -> `6,834` jobs/s as worker count increased. -- Phase B `pgmq` peaked at `13,290` jobs/s at 16 workers and collapsed at 128 workers, matching the previously observed high-worker behaviour. -- Phase C multi-replica runs exposed the remaining `awa` topology sensitivity: at fixed total workers, throughput fell from `3,560` to `2,491` to `1,503` jobs/s across `1x64`, `2x32`, and `4x16`. That shape is consistent with fleet-wide completion flusher contention (`processes x AWA_COMPLETION_SHARDS`). `pgque` moved in the opposite direction in the same run (`18,660` -> `34,388` -> `38,810`), so this remains a useful comparison target rather than noise to smooth over. -- Phase D 60-minute `awa` soak sustained a median `5,369` jobs/s with median dead tuples around `396`, validating the ADR-019 / ADR-023 hot-path dead-tuple promise under sustained churn. - -Queue-storage e2e sweeps separate tuning from storage design. For the hot single-queue shape, `enqueue_shards = 4` plus larger completion batches sustained `7.9k` completed jobs/s with `200ms` p99 end-to-end latency and bounded depth. Increasing `claimers` did not materially improve that shape. - -When the application can accept partitioned ordering at the logical workload level, routing through several physical queues with `PartitionedQueue` is the preferred throughput lever: it creates independent claim and completion coordination streams rather than adding more claimers to one queue head. - -ADR-026 terminal-count deltas remove hot-path `queue_terminal_live_counts` updates. `done_entries` terminal paths append signed deltas, compact receipt completions are counted from retained `receipt_completion_batches_*` minus `receipt_completion_tombstones_*`, and exact reads include all retained evidence plus permanent rollups. Maintenance folds sealed `done_entries` delta slots into compact counters only when the MVCC horizon is not pinned by another backend snapshot or idle transaction id. Benchmark runs should therefore sample `queue_terminal_count_deltas_*`, `receipt_completion_batches_*`, and `queue_terminal_live_counts` so regressions distinguish pending append-only rows from mutable-counter dead tuples. - -The offered-rate benchmark exercises absorption directly and samples WAL. With one claimer, `claim_batch_size = 512`, `AWA_COMPLETION_BATCH_SIZE = 512`, the fused receipt completion path, and `max_workers = 1024` (which selects four queue-storage completion shards by default), a 10-second no-op run at `10k/s` offered load keeps durable completions at the offered rate, drains to zero backlog, and writes `1.80 KiB` WAL per completed job. `max_workers = 512` is close but does not consistently meet the 10k offered target, because no-op handlers hold permits while waiting for durable completion acknowledgement. - -First-principles WAL accounting compared the production path, a narrow `done_entries` row, and a deliberately non-production path that skipped durable terminal history entirely: - -| Shape | Completed jobs/s | p99 e2e | WAL/job | -| ------------------------------ | ---------------: | -------: | --------: | -| Tuned production queue storage | `7,885/s` | `203 ms` | `2,241 B` | -| Narrow terminal history | `8,337/s` | `205 ms` | `1,932 B` | -| Skip `done_entries` entirely | `8,402/s` | `230 ms` | `1,441 B` | - -The adopted design keeps the durable terminal fact and public `{schema}.terminal_jobs` surface while avoiding duplicated ready-body fields. Successful receipt completions can now write compact `receipt_completion_batches_*` rows instead of one `done_entries_*` row per job. The skip-durable-history experiment remains rejected because it weakens the terminal-history contract. - -### Queue-storage striping reference - -Queue striping is a contention-control knob for workloads dominated by one hot logical queue. The companion benchmark repo includes an awa-only sweep of `queue_storage_queue_stripe_count` over `1`, `2`, and `4` at `64`, `128`, `256`, and `512` workers. The source artifact lives in companion benchmark PR [#21](https://github.com/hardbyte/postgresql-job-queue-benchmarking/pull/21). - -Reference result: - -| Stripes | 64 workers | 128 workers | 256 workers | 512 workers | -| ------: | ---------: | ----------: | ----------: | ----------: | -| 1 | `3,408/s` | `4,741/s` | `9,530/s` | `11,188/s` | -| 2 | `4,654/s` | `7,667/s` | `11,975/s` | `11,173/s` | -| 4 | `4,378/s` | `7,596/s` | `11,418/s` | `11,443/s` | - -The clearest gain was the `1 -> 2` stripe step: at `128` workers throughput rose by `62%`, and at `256` workers throughput rose by `26%` while end-to-end p99 fell from `1,802 ms` to `1,027 ms`. `4` stripes mostly matched `2` stripes in this shape, with the only clear advantage at the `512` worker tail. - -Treat this as tuning guidance, not an out-of-the-box setting. The default stays `queue_storage_queue_stripe_count=1`; hot single-queue deployments should consider `2` after measuring their own worker/replica shape. - ## Python Runtime Benchmarks The Python benchmark script exercises the real `awa-python` worker path while reusing the same database-facing benchmark shapes as the Rust runtime: diff --git a/docs/callback-receivers.md b/docs/callback-receivers.md index 08ff036b..e00f9855 100644 --- a/docs/callback-receivers.md +++ b/docs/callback-receivers.md @@ -3,14 +3,14 @@ Awa supports three ways to host the HTTP callback ingress surface: 1. **Bundled with the admin UI** — the default. `awa serve` exposes the admin UI plus the three callback routes from the same router. -2. **Standalone receiver** — `awa callbacks serve` runs a router that mounts only the callback ingress endpoints. Use this when callbacks must be reachable from outside the operator network but the admin surface must remain private. See [`docs/http-callbacks.md`](./http-callbacks.md). +2. **Standalone receiver** — `awa callbacks serve` runs a router that mounts only the callback ingress endpoints. Use this when callbacks must be reachable from outside the operator network but the admin surface must remain private. See [HTTP callbacks](./http-callbacks.md). 3. **User-owned API layer** — mount the callback ingress routes inside your own application (FastAPI / Starlette / Flask / Django / axum / actix). This page covers option 3. The on-wire contract is identical across all three options: - Routes: `POST {prefix}/{callback_id}/{complete,fail,heartbeat}`. - Signature: BLAKE3 keyed-hash of the callback id, lowercase hex, in the `X-Awa-Signature` header. -- Payloads: see [`docs/http-callbacks.md`](./http-callbacks.md#callback-receiver-contract). +- Payloads: see the [callback receiver contract](./http-callbacks.md#callback-receiver-contract). When you host the routes yourself, **reuse the shared signing and URL helpers** from `awa::callback_contract` (Rust) or `awa.callback_contract` (Python) so your implementation cannot drift from the worker's. The helpers are exported specifically so this is a one-line dependency, not a copy-paste of the algorithm. diff --git a/docs/concepts/index.md b/docs/concepts/index.md new file mode 100644 index 00000000..54634207 --- /dev/null +++ b/docs/concepts/index.md @@ -0,0 +1,28 @@ +# How AWA works + +AWA is a library and schema, not a separate broker service. Producers, workers, and operational tools coordinate through PostgreSQL. + +
+![AWA producers, PostgreSQL storage, workers, and operator tools](../assets/architecture-flow.svg) +
+ +## The execution loop + +1. A producer serializes typed arguments and inserts a job, optionally inside an application transaction. +2. A worker asks PostgreSQL for runnable work in a configured queue. +3. A claim records the attempt and establishes an ownership token. Other workers skip the claimed row. +4. The worker heartbeats while the handler runs. +5. Completion is accepted only while the worker still owns the claim. A retry, snooze, callback wait, failure, or completion becomes durable state. +6. If heartbeats stop, another worker can rescue the expired claim and run the job again. + +This produces an **at-least-once** contract. The ownership guard prevents a stale worker from overwriting a newer attempt, but it cannot make an external side effect exactly once. + +## Storage and runtime are separate responsibilities + +- `awa-model` owns the schema, migrations, typed records, enqueue operations, and admin queries. +- `awa-worker` owns claiming, execution, heartbeats, retry scheduling, and graceful shutdown. +- `awa` is the Rust facade that joins those pieces. +- `awa-pg` exposes the same model and worker runtime to Python. +- `awa-cli` and the web UI provide migrations and operational inspection. + +See the detailed [architecture](../architecture.md) and [queue storage substrate](../queue-storage-substrate.md) when you need implementation-level invariants. diff --git a/docs/concepts/job-lifecycle.md b/docs/concepts/job-lifecycle.md new file mode 100644 index 00000000..2fe160c5 --- /dev/null +++ b/docs/concepts/job-lifecycle.md @@ -0,0 +1,49 @@ +--- +hide: + - toc +--- + +# Job lifecycle + +A job moves through durable states in PostgreSQL. The handler returns the next outcome; AWA validates that the current worker still owns the claim before applying it. + +
+![AWA job lifecycle from available through execution to completion, retry, callback wait, cancellation, or failure](../assets/job-lifecycle.svg) +
+ +## Common paths + +`available → running → completed` +: The normal path. Claiming increments the attempt before the handler runs. + +`running → retryable → available` +: A retry records the error and a future run time. Once due, the job is runnable again if attempts remain. + +`running → scheduled → available` +: A snooze defers the same attempt. It is useful for polling an external system without consuming the retry budget. + +`running → waiting_external → running` +: The handler parks while an external operation runs. A resuming callback stores the result and returns the job to `running`, where the still-live handler observes it. + +`waiting_external → completed | failed` +: A callback may finalize the job directly. Callback policy failure also moves the job to `failed`. + +`waiting_external → available` +: An explicit callback retry starts the job again from scratch and resets its attempt count. This is distinct from resuming the parked handler. + +`waiting_external → retryable | failed` +: When a callback deadline expires, maintenance retries the job if attempts remain and otherwise fails it. + +`running → failed | cancelled` +: Terminal outcomes. Failed jobs can be retained in the dead-letter queue according to policy. + +## What crash recovery means + +A running job is owned only for as long as its claim remains live. Heartbeats extend that claim. If the process stops, another worker can rescue the job after the claim expires. The original worker may still finish late, so completion writes are guarded by ownership and handlers must tolerate duplicate execution. + +## Where to go next + +- Configure attempts, delays, timeouts, and queue concurrency in [Configuration](../configuration.md). +- Design idempotent effects with [Transactional enqueue](transactional-enqueue.md). +- Inspect and redrive terminal failures with the [Dead-letter queue](../dead-letter-queue.md). +- Use [Callbacks](../http-callbacks.md) for externally completed work. diff --git a/docs/concepts/transactional-enqueue.md b/docs/concepts/transactional-enqueue.md new file mode 100644 index 00000000..b1a477bc --- /dev/null +++ b/docs/concepts/transactional-enqueue.md @@ -0,0 +1,45 @@ +# Transactional enqueue + +The central reason to keep AWA in PostgreSQL is atomic handoff. An application write and the job that follows it can commit together. + +
+![A single PostgreSQL transaction atomically commits an order and an AWA job before a worker can claim it](../assets/transactional-enqueue.svg) +
+ +Without atomic enqueue, an application can commit business data and crash before publishing work—or publish work and then roll back the data the handler expects. AWA avoids that gap by accepting the application's existing database transaction. + +## Rust + +Pass an open `sqlx` transaction to the same insertion functions used with a pool: + +```rust +let mut tx = pool.begin().await?; + +sqlx::query("INSERT INTO orders (id, email) VALUES ($1, $2)") + .bind(order_id) + .bind(email) + .execute(&mut *tx) + .await?; + +awa::insert_with(&mut *tx, &SendEmail { to: email.into() }, options).await?; +tx.commit().await?; +``` + +## Python + +Use `awa.bridge` with the connection or session that owns the application transaction: + +```python +async with session.begin(): + await session.execute(insert(Order).values(id=order_id, email=email)) + await awa.bridge.insert_job( + session, + SendEmail(to=email), + queue="email", + ) +``` + +The bridge supports asyncpg, psycopg 3, SQLAlchemy, and Django. It does not move transaction ownership into AWA: the application still decides whether to commit or roll back. See [Bridge adapters](../bridge-adapters.md) for exact driver behavior and tested examples. + +!!! warning "Atomic enqueue does not make arbitrary side effects exactly once" + AWA commits the job atomically with data in the same PostgreSQL transaction. A handler calling an external API still needs an idempotency key or another duplicate-safe design because delivery is at least once. diff --git a/docs/configuration.md b/docs/configuration.md index 5c16412a..0935182d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -226,7 +226,7 @@ Queue configuration is the normal way to express operational policy. Job kinds a | Per queue | `max_workers`, `rate_limit`, `deadline_duration`, `priority_aging_interval`, `poll_interval`, `min_workers`, `weight` | Runtime dispatch policy. `deadline_duration` is the hard per-attempt wall-clock timeout for every job claimed from that queue. | | Per job enqueue | `queue`, `priority`, `max_attempts`, `run_at`, `tags`, `metadata`, `unique` | Stored with the job. Use this for routing, priority, retry budget, scheduling, identity, and operator context. | | Per job kind | handler registration plus `JobKindDescriptor` | Descriptors cover display name, owner, docs link, tags, and extra metadata. They do not set dispatch limits or timeouts. | -| Per callback wait | callback `timeout` | Applies only to jobs that enter `wait_for_callback`; see [Callback timeout](#callback-timeout--bounding-wait_for_callback). | +| Per callback wait | callback `timeout` | Applies only to jobs that enter `wait_for_callback`; see [Callback timeout](#callback-timeout). | | Per queue / global retention and DLQ policy | `completed_retention`, `failed_retention`, `queue_retention(...)`, `dlq_enabled_by_default`, `queue_dlq_enabled`, `dlq_retention` | Retention has global defaults with per-queue overrides. DLQ enablement is queue-scoped. Split job kinds across queues if only some kinds should DLQ. | | Per runtime / fleet | heartbeat timings, rescue scan intervals, cleanup batch size, descriptor retention, queue-storage slots/stripes/rotation | Process or storage-engine policy, not job-kind policy. `queue_storage_queue_stripe_count` currently applies to the queue-storage engine rather than to one named queue. | @@ -459,7 +459,7 @@ Each queue has a `deadline_duration` (default `5m` on `QueueConfig`). At claim t Receipts mode (the 0.6 default storage) supports both shapes: the deadline lands on `lease_claims.deadline_at` and is rescued there for short claims, or carried onto `leases.deadline_at` if the claim materializes for a long-running attempt. See [Queue storage tuning](#queue-storage-tuning) and ADR-023. -### Callback timeout — bounding `wait_for_callback` +### Callback timeout — bounding `wait_for_callback` { #callback-timeout } If you suspend a handler with `wait_for_callback()` and the external system never resumes, a callback-timeout rescue brings the job back to ready (or DLQ if attempts are exhausted). @@ -594,7 +594,7 @@ A 16-producer same-queue reference sweep measured 1.0× / 1.60× / 2.75× / 3.69 Observability: the `awa.job.claimed` OTel counter carries an `awa.enqueue.shard` attribute on the queue-storage claim path. Dashboards can sum by that attribute to confirm the claim ordering is rotating fairly across shards. -Lowering the value is safe at any time — see [`docs/upgrade-0.5-to-0.6.md`](upgrade-0.5-to-0.6.md#lowering-enqueue_shards). See [ADR-025](adr/025-sharded-enqueue-heads.md) for the full design and contract. +Lowering the value is safe at any time. See [ADR-025](adr/025-sharded-enqueue-heads.md) for the full design and contract. ### Hot-Queue Claim Control @@ -633,7 +633,7 @@ AWA_HEALTH_ADDR=0.0.0.0:8321 Port `0` binds an ephemeral port (read it back with `Client::health_listener_addr()`). Unset means no listener. Endpoint semantics, response fields, and Kubernetes probe -examples live in [deployment.md](deployment.md#health-checks); `awa health` covers +examples live in [Deployment](deployment.md#health-checks); `awa health` covers probe-less environments from the CLI. ## Distributed tracing diff --git a/docs/contributing/index.md b/docs/contributing/index.md new file mode 100644 index 00000000..7f6c011c --- /dev/null +++ b/docs/contributing/index.md @@ -0,0 +1,10 @@ +# Contributing + +AWA's runtime behavior is backed by code tests, PostgreSQL integration tests, compatibility matrices, benchmarks, and TLA+ models. Changes should update the evidence at the same boundary they change. + +- [Development](../development.md) is the contributor workflow, including migrations and local checks. +- [Benchmarking](../benchmarking.md) explains which performance results are comparable and which are historical evidence only. +- [Architecture decisions](../adr/README.md) records durable design choices and their status. +- [Correctness models](https://github.com/hardbyte/awa/tree/main/correctness) hold the TLA+ specifications and model-to-code mapping. + +Start from the repository's [agent instructions](https://github.com/hardbyte/awa/blob/main/AGENTS.md) when working with a coding agent; it contains the always-on validation and Agent Skills rules. diff --git a/docs/dead-letter-queue.md b/docs/dead-letter-queue.md index 0a1d379b..3daae034 100644 --- a/docs/dead-letter-queue.md +++ b/docs/dead-letter-queue.md @@ -79,7 +79,7 @@ Purging is destructive: the rows are deleted from `dlq_entries` and not recovera ## Retention -The maintenance leader periodically prunes DLQ rows older than the configured retention window. See [`docs/configuration.md`](configuration.md) for the `dlq_retention_*` knobs. Retention runs alongside the rotation / prune work for the queue and lease rings, so a busy DLQ does not delay queue-plane reclamation. +The maintenance leader periodically prunes DLQ rows older than the configured retention window. See [Configuration](configuration.md) for the `dlq_retention_*` knobs. Retention runs alongside the rotation / prune work for the queue and lease rings, so a busy DLQ does not delay queue-plane reclamation. ## Programmatic access @@ -90,5 +90,5 @@ Python callers use direct client methods: `list_dlq`, `get_dlq_job`, `dlq_depth` ## See also - [ADR-020 — Dead Letter Queue](adr/020-dead-letter-queue.md) — design and trade-offs. -- [configuration.md](configuration.md) — `dlq_enabled` per queue, retention knobs. -- [troubleshooting.md](troubleshooting.md) — diagnosing why a particular job reached the DLQ. +- [Configuration](configuration.md) — `dlq_enabled` per queue, retention knobs. +- [Troubleshooting](troubleshooting.md) — diagnosing why a particular job reached the DLQ. diff --git a/docs/deploying-on-managed-postgres.md b/docs/deploying-on-managed-postgres.md index b1581439..86f11548 100644 --- a/docs/deploying-on-managed-postgres.md +++ b/docs/deploying-on-managed-postgres.md @@ -2,7 +2,7 @@ This page collects the operational gotchas and sizing data we learned running awa workers against Google Cloud SQL and AlloyDB in staging. Most of it applies to any managed Postgres (Amazon RDS / Aurora, Azure Database for PostgreSQL, etc.); GCP-specific advice is called out. -The reference data here was captured on a single 4-pod consumer × 4-pod producer fleet against a dedicated benchmarking database on each engine, with `enqueue_shards = 16` and the queue-storage direct COPY producer path. See [`benchmarking.md`](benchmarking.md) for methodology and the `awa-bench-driver` reports for raw numbers and EXPLAIN traces. +The reference data here was captured on a single 4-pod consumer × 4-pod producer fleet against a dedicated benchmarking database on each engine, with `enqueue_shards = 16` and the queue-storage direct COPY producer path. See [Benchmarking](benchmarking.md) for methodology and the `awa-bench-driver` reports for raw numbers and EXPLAIN traces. ## Pick a Postgres version @@ -31,7 +31,7 @@ Sizing rule of thumb: pick the vCPU count for your steady-state completion targe ## IAM and Cloud SQL connectivity -Migrations and custom queue-storage schema preparation need DDL-capable credentials. Ordinary workers can run with the runtime grants in [`security.md`](security.md) once `awa migrate` has materialized the default `awa` substrate, or once `awa storage prepare-queue-storage-schema` has materialized a custom queue-storage schema. If you rely on fresh-install auto-prepare from the first worker startup instead, that worker connection also needs the DDL privileges required by `prepare_schema()`. +Migrations and custom queue-storage schema preparation need DDL-capable credentials. Ordinary workers can run with the [runtime database grants](security.md) once `awa migrate` has materialized the default `awa` substrate, or once `awa storage prepare-queue-storage-schema` has materialized a custom queue-storage schema. If you rely on fresh-install auto-prepare from the first worker startup instead, that worker connection also needs the DDL privileges required by `prepare_schema()`. > **0.7 design note (not yet shipped):** [ADR-042](adr/042-caller-owned-finalization-transactions.md) uses one ordinary transaction, transaction-scoped advisory locks, and no session state, so its caller-owned finalization protocol is compatible with transaction-mode pgbouncer/pgcat and RDS Proxy. Queue `LISTEN` remains session-scoped: [ADR-033](adr/033-per-key-execution-control.md) grant-close notifications require a direct/session-pooled listener or the #374 gated polling fallback. The sending `pg_notify` call itself remains transactional and pooler-safe. @@ -94,7 +94,7 @@ DO UPDATE SET enqueue_shards = EXCLUDED.enqueue_shards; Use an upsert — first-enqueue may create lane rows before any operator inserts a `queue_meta` row, so a plain `UPDATE` can quietly affect zero rows. -Going higher than 4 only helps when producer-side head-row contention is your bottleneck; raise it after measuring. Lowering it later is safe (see [`upgrade-0.5-to-0.6.md`](upgrade-0.5-to-0.6.md#lowering-enqueue_shards)) but the FIFO contract changes — see [`configuration.md`](configuration.md#sharding-the-enqueue-head-per-queue) and [ADR-025](adr/025-sharded-enqueue-heads.md). +Going higher than 4 only helps when producer-side head-row contention is your bottleneck; raise it after measuring. Lowering it later is safe, but the FIFO contract changes — see [enqueue-head sharding](configuration.md#sharding-the-enqueue-head-per-queue) and [ADR-025](adr/025-sharded-enqueue-heads.md). ## Producer path: use the direct COPY entry point @@ -105,13 +105,13 @@ For any high-volume producer running against managed Postgres (rather than a Doc The compat-friendly `insert_many_copy_from_pool` / `client.insert_many_copy` path routes each row through the `awa.insert_job_compat()` SQL function once per row. On a real DB the per-row function-call cost (lane head update, NOTIFY, admin metadata) sums to ~100–150 ms per row on AlloyDB through the auth-proxy — fine when the goal is "one writer, strict compatibility", catastrophic when the goal is "burst 10⁶ jobs in seconds." The direct path runs at the inserts-per-second numbers in the sizing table above. -See [`configuration.md`](configuration.md#producer-path-choice) for the full surface comparison. +See [Producer path choice](configuration.md#producer-path-choice) for the full surface comparison. ## MVCC discipline: long-running readers pin the whole database Awa's queue storage keeps its hot path append-only and reclaims segments with `TRUNCATE`, but it is still a Postgres schema and is subject to the database-wide MVCC horizon. Any transaction holding an old snapshot — a `psql` session left `idle in transaction`, a BI tool with a long read transaction, `pg_dump`, a stuck migration — prevents vacuum and Awa's maintenance pruning from reclaiming row versions created after that snapshot, in every table in the database. -Under sustained load this degrades throughput, not correctness: jobs are not lost, but completion rate sags and backlog accumulates until the pinning transaction releases, after which the backlog drains. Every per-row Postgres queue (River, Oban, pg-boss, Graphile, pgmq) shares this failure mode. The long-horizon scenario in [`benchmarking.md`](benchmarking.md) reproduces it on demand; [`troubleshooting.md`](troubleshooting.md#dead-tuples-growing-in-queue-storage) covers diagnosis on a live system. +Under sustained load this degrades throughput, not correctness: jobs are not lost, but completion rate sags and backlog accumulates until the pinning transaction releases, after which the backlog drains. Every per-row Postgres queue (River, Oban, pg-boss, Graphile, pgmq) shares this failure mode. The long-horizon [benchmark scenario](benchmarking.md) reproduces it on demand; [Troubleshooting](troubleshooting.md#dead-tuples-growing-in-queue-storage) covers diagnosis on a live system. Operational rules, in priority order: @@ -142,11 +142,11 @@ WHERE datname = current_database() AND backend_type = 'client backend'; ``` -Alert when `max_xact_age_seconds` exceeds a few multiples of your normal longest job; `300` is a reasonable starting threshold for short-job workloads. The drill-down query for finding the offending session is in [`troubleshooting.md`](troubleshooting.md#inspect-long-transactions). +Alert when `max_xact_age_seconds` exceeds a few multiples of your normal longest job; `300` is a reasonable starting threshold for short-job workloads. The drill-down query for finding the offending session is in [Inspect long transactions](troubleshooting.md#inspect-long-transactions). ### 4. Give autovacuum enough capacity -The queue-storage substrate ships aggressive per-table autovacuum storage parameters on its hot mutable tables (ring state, heads, leases, claims), so per-table thresholds are normally not yours to tune. What managed-Postgres defaults often starve is instance-wide vacuum capacity. If the churn query in [`troubleshooting.md`](troubleshooting.md#inspect-table-churn) shows `autovacuum_count` flat while dead tuples climb on `leases%` or `attempt_state`, raise these flags: +The queue-storage substrate ships aggressive per-table autovacuum storage parameters on its hot mutable tables (ring state, heads, leases, claims), so per-table thresholds are normally not yours to tune. What managed-Postgres defaults often starve is instance-wide vacuum capacity. If the [table churn query](troubleshooting.md#inspect-table-churn) shows `autovacuum_count` flat while dead tuples climb on `leases%` or `attempt_state`, raise these flags: - `autovacuum_max_workers` (default 3 is low for a busy queue database sharing the instance with application tables) - `autovacuum_vacuum_cost_limit` / lower `autovacuum_vacuum_cost_delay` so workers actually keep up diff --git a/docs/deployment.md b/docs/deployment.md index a015075f..5d1123cf 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -25,7 +25,7 @@ In production, treat these as separate concerns: ## Queue Storage Cutover -Fresh 0.6 installs use queue storage as the worker engine. Existing 0.5.x clusters must follow the staged storage-transition procedure in [upgrade-0.5-to-0.6.md](upgrade-0.5-to-0.6.md); [migrations.md](migrations.md) covers the general migration contract and tooling. +Fresh 0.6 installs use queue storage as the worker engine. Existing 0.5.x clusters must follow the [staged storage-transition procedure](upgrade-0.5-to-0.6.md); [Migrations](migrations.md) covers the general migration contract and tooling. Operationally: @@ -93,8 +93,8 @@ The nightly MVCC benchmark exists to catch changes that make this failure mode w The repository already includes Dockerfiles for the `awa` CLI: -- [`docker/Dockerfile`](../docker/Dockerfile) -- [`docker/Dockerfile.runtime`](../docker/Dockerfile.runtime) +- [`docker/Dockerfile`](https://github.com/hardbyte/awa/blob/main/docker/Dockerfile) +- [`docker/Dockerfile.runtime`](https://github.com/hardbyte/awa/blob/main/docker/Dockerfile.runtime) Build the CLI image locally: @@ -276,7 +276,7 @@ For a UI pinned to a less-trusted network or shared with stakeholders who should awa --database-url "$DATABASE_URL" serve --read-only ``` -This forces read-only even when the Postgres connection is fully writable — the UI hides mutation buttons and every mutation endpoint returns 503. See [configuration.md#read-only-mode](configuration.md#read-only-mode) for the tradeoff versus pointing at a read replica. +This forces read-only even when the Postgres connection is fully writable — the UI hides mutation buttons and every mutation endpoint returns 503. See [Read-only mode](configuration.md#read-only-mode) for the tradeoff versus pointing at a read replica. If you expose the callback receiver endpoints for `HttpWorker`, also configure `AWA_CALLBACK_HMAC_SECRET` (or `--callback-hmac-secret`) so `awa-ui` verifies `X-Awa-Signature` on callback requests. See [HTTP workers and callback signatures](http-callbacks.md) for the worker, function, and receiver contract. diff --git a/docs/development.md b/docs/development.md index dc4d940b..58c92e38 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,149 +1,50 @@ -# Development Guide - -## Release Process - -Use pre-release tags before publishing a final version. Both crates.io and PyPI treat published versions as immutable — a botched release cannot be overwritten and the version number is burned. - -### Workflow - -1. **Alpha** — early integration testing: - ``` - v0.x.0-alpha.1 → v0.x.0-alpha.2 → ... - ``` -2. **Release candidate** — feature-complete, verifying in staging: - ``` - v0.x.0-rc.1 → v0.x.0-rc.2 → ... - ``` -3. **Final release** — CI green, all checks pass: - ``` - v0.x.0 - ``` - -### Steps - -1. Bump version in these release manifests: - - `Cargo.toml` (workspace `[workspace.package].version` and every workspace dependency whose version points at an Awa crate being released) - - `awa/Cargo.toml` (`awa-testing` dev-dependency version) - - `awa-cli/Cargo.toml` (`awa-ui` dependency version) - - `awa-cli/pyproject.toml` (`[project].version` — controls CLI wheel version on PyPI) - - `awa-python/Cargo.toml` (`version`, `awa-model`, `awa-worker` dep versions) - - `awa-python/pyproject.toml` (`[project].version` — controls SDK wheel version on PyPI) -2. Finalize the matching changelog section and commit the release preparation. -3. Push a branch and open a PR against the branch being released. Apply the - `full-ci` label and wait for every check, including the TLA+ storage models. -4. After merge, record the release branch's exact commit SHA and wait for the - automatically triggered **CI** workflow to pass completely. The Release - workflow refuses to publish unless this exact-SHA branch-push run succeeded; - a PR run against a pre-merge SHA is not sufficient. -5. Tag and push the validated commit, for example: - `git tag v0.x.0-alpha.1 && git push origin v0.x.0-alpha.1`. -6. The Release workflow reruns the rolling-upgrade rehearsal on the tagged - commit, builds wheels, publishes to crates.io and PyPI, and creates a GitHub - Release with binary assets. -7. When ready for final: bump the version to `0.x.0`, repeat steps 2–4 for the - final release preparation, then tag the validated commit as `v0.x.0`. - -### Why pre-releases matter - -v0.2.0 was published directly. The GitHub Release workflow tried to attach binary assets to an already-published release, which GitHub blocks. Pre-release tags avoid this because: - -- Draft releases are created by the workflow, not manually -- If a pre-release has problems, you bump to `-alpha.2` instead of fighting immutable registries - -## Crate Dependencies +# Development -``` -awa-macros (proc-macro, no runtime deps) - │ - ▼ -awa-model (core types + SQL, re-exports awa-macros::JobArgs) - │ - ├──────────────┬──────────────┐ - ▼ ▼ ▼ -awa-worker awa-ui awa-cli - │ (axum API + (depends on awa-ui) - │ embedded UI) - ├──────────────┐ - ▼ ▼ -awa (facade) awa-testing - │ - ▼ - awa-python (PyO3 bridge, separate workspace) -``` +This page is a short orientation for contributors building Awa locally. Release procedure, migration-author checklists, and CI policy are maintained in the repository's contributor files and workflow definitions because they change with the codebase and are not part of the user documentation contract. -Key dependencies per crate: +## Repository layout -| Crate | Key deps | -| ------------ | ------------------------------------------------ | -| `awa-model` | sqlx, blake3, serde, chrono, chrono-tz, croner | -| `awa-worker` | awa-model, tokio, opentelemetry | -| `awa-ui` | awa-model, axum, rust-embed | -| `awa-cli` | awa-model, awa-ui, axum, clap | -| `awa-python` | awa-model, awa-worker, pyo3, pyo3-async-runtimes | +| Path | Purpose | +| --- | --- | +| `awa-model` | Core types, SQL, and migrations | +| `awa-worker` | Dispatch, execution, maintenance, and telemetry | +| `awa` | Public Rust facade | +| `awa-cli` / `awa-ui` | Command-line tools and embedded admin interface | +| `awa-python` | PyO3 Python package and Python worker API | +| `awa-testing` | Test support | +| `correctness` | Executable TLA+ models and trace checks | +| `docs` | MkDocs site and source ADRs | -## Running Tests +## Local checks + +Start a supported PostgreSQL instance and provide its URL to the test suite: ```bash -# Start Postgres -docker run -d --name awa-pg -e POSTGRES_PASSWORD=test -e POSTGRES_DB=awa_test \ +docker run -d --name awa-pg \ + -e POSTGRES_PASSWORD=test \ + -e POSTGRES_DB=awa_test \ -p 15432:5432 postgres:17-alpine -# Rust -DATABASE_URL=postgres://postgres:test@localhost:15432/awa_test cargo test --workspace +export DATABASE_URL=postgres://postgres:test@localhost:15432/awa_test +cargo test --workspace +``` + +The Python package uses `uv`: -# Python +```bash cd awa-python uv run maturin develop -DATABASE_URL=postgres://postgres:test@localhost:15432/awa_test uv run pytest tests/ -v +uv run pytest tests/ -v +``` + +Run the core correctness models with the repository wrapper: -# TLA+ correctness models +```bash ./correctness/run-tlc.sh core/AwaCore.tla ./correctness/run-tlc.sh protocol/AwaExtended.tla ``` -## Authoring Schema Migrations - -Policy: [ADR-041 — rolling-upgrade policy](adr/041-rolling-upgrade-policy.md). Use this checklist before opening a migration PR; version floors, exclusive migrations, and the newer-schema fail-safe live in `awa-model/src/migrations.rs`. - -Checklist for any new `awa-model/migrations/vNNN_*.sql`: - -**Every migration** - -- [ ] Keep every object used by N−1 binaries compatible: no drops, type changes, or tightened constraints; make new objects and columns additive. -- [ ] Make the migration safe to re-run: `IF NOT EXISTS` on `CREATE TABLE` / `SEQUENCE` / `INDEX`, `CREATE OR REPLACE` for functions and views, `DROP TRIGGER IF EXISTS` before each `CREATE TRIGGER`, guarded `DO` blocks for anything with no `IF NOT EXISTS` form (`CREATE TYPE`), and `ON CONFLICT (version) DO NOTHING` on the `awa.schema_version` row. `migrations::tests::every_migration_guards_its_ddl` enforces the top-level cases; `test_every_migration_is_individually_re_runnable` proves it against a real database. -- [ ] Keep every step transaction-safe — the runner applies the whole pending range in one transaction, so no `CREATE INDEX CONCURRENTLY`, `VACUUM`, or statement-level `BEGIN` / `COMMIT` / `ROLLBACK` / `SAVEPOINT`. `migrations::tests::every_migration_step_is_transaction_safe` enforces this. -- [ ] In the header, link the issue and state how N−1 binaries operate against the migrated schema. -- [ ] Safe under live load: no long `ACCESS EXCLUSIVE` holds on hot tables; note the expected wall time on realistic data volumes. -- [ ] The current binary remains operable before migration, or startup applies the migration before any changed path runs. Test binary-first as well as migrate-first ordering. -- [ ] Document requirements for external runners, which do not execute Rust preflights. - -**If compatibility first ships in an earlier-release patch** - -- [ ] Add the released, verified patch to `MIGRATION_RUNTIME_VERSION_FLOORS`; test old, unparseable, and stale runtimes plus `--allow-live-runtimes`. -- [ ] Keep the preflight race-free and record any observability-snapshot stall from its lock. Job and lease heartbeats must remain unaffected. -- [ ] Publish the patch prerequisite before the migration and document it in the CHANGELOG and upgrade guide. - -**If it changes an on-disk representation or hot-path structure (expand → flip → contract)** - -- [ ] Make the migration the **expand** phase only: seed the new representation, keep the old one authoritative, and store authority explicitly. Fresh installs may start on the new representation. -- [ ] Gate the runtime **flip** on fresh fleet capability. Install the schema-owned per-feature capability constant with the expand migration; treat missing or unparseable evidence as incapable and make any override explicit. -- [ ] Under the old-writer locks, the flip treats the old representation as source of truth, reconciles the complete new representation, verifies exact equivalence, and changes authority atomically. Shadow writes alone do not satisfy this requirement. -- [ ] The flip **fences** returning pre-flip binaries at the database boundary. Exercise the actual N−1 write path; a sentinel is insufficient if old code can advance through it. -- [ ] The **contract** migration (dropping the old representation) is deferred to a later minor, tracked as its own issue, and independently checked against that release's N−1 contract. -- [ ] Model mixed-version interleavings in TLA+ when a state machine or lock order changes. -- [ ] Rehearse migrate-first, binary-first, and overlapping rollouts with a released N−1 artifact. Include concurrent old/new workers, failures and retries, scheduled work, in-flight work, hard-kill and deadline rescue, flip/fence behavior, and exact job accounting; record the evidence. CI automation is [#427](https://github.com/hardbyte/awa/issues/427). - -**If no rolling-compatible design is practical** - -- [ ] Explain in an ADR why expand/flip/contract and a version floor are insufficient, then add the migration to `EXCLUSIVE_WINDOW_MIGRATIONS` with refusal, override, and stale-heartbeat tests plus explicit operator documentation. - -**Docs** - -- [ ] Update the CHANGELOG, the release upgrade guide when operator action is required, and `docs/stability.md` when the skew contract changes. Link compatibility claims to rehearsals of the claimed version topology; describe narrower evidence only by the behavior it covers. - -## Pre-commit Checks (Rust) - -Always run before committing Rust changes: +Before submitting Rust changes, format and run the same offline checks used by CI: ```bash cargo fmt --all @@ -151,10 +52,17 @@ SQLX_OFFLINE=true cargo clippy --all-targets --all-features -- -D warnings SQLX_OFFLINE=true cargo build --workspace ``` -The Python crate lives in a separate workspace: +`awa-python` is a separate Rust workspace; run formatting and clippy from that directory too. -```bash -cd awa-python -cargo fmt --all -SQLX_OFFLINE=true cargo clippy --all-targets -- -D warnings -``` +## Where contributor policy lives + +- [Repository README](https://github.com/hardbyte/awa) — project overview and workspace entry point. +- [GitHub Actions workflows](https://github.com/hardbyte/awa/tree/main/.github/workflows) — the current CI and release gates. +- [ADR-041](adr/041-rolling-upgrade-policy.md) — architectural policy for rolling-compatible migrations. +- [Benchmarking](benchmarking.md) — reproducible performance suites and how to interpret their results. + +When a schema change affects users or operators, update the relevant upgrade guide and [stability policy](stability.md). Detailed migration implementation checklists belong beside the migration code and review process rather than in the public product guide — they live in [`AGENTS.md`](https://github.com/hardbyte/awa/blob/main/AGENTS.md#schema-migrations). + +## Authoring schema migrations + +Schema changes must follow [ADR-041's expand, capability-gated flip, and later contract policy](adr/041-rolling-upgrade-policy.md). The implementation checklist lives in [`AGENTS.md`](https://github.com/hardbyte/awa/blob/main/AGENTS.md#schema-migrations) alongside the migration tests that enforce it; upgrade guides document any action required from operators. diff --git a/docs/getting-started-python.md b/docs/getting-started-python.md index 8c7931d1..8096318d 100644 --- a/docs/getting-started-python.md +++ b/docs/getting-started-python.md @@ -1,22 +1,15 @@ # Python Getting Started -This guide takes you from `pip install` to a job reaching `completed`. +This guide takes you from `uv init` to a job reaching `completed`. -## Mental Model +!!! note "Version used in this guide" -Before the code, here is the operational model Awa is built around: - -- inserting a job writes durable job state to Postgres, so enqueuing can live inside the same transaction as your application write -- workers claim runnable jobs, heartbeat while they execute, and rescue them if the worker dies -- retries, callback waits, and progress checkpoints are persisted in Postgres and exposed as one hydrated job snapshot instead of being held only in memory -- when you debug or operate the system, inspect the job first; the CLI and UI are designed around that read-only inspection path - -That means “what happened?” is usually a database inspection question, not a worker-log archaeology exercise. + The install commands pin **v0.6.6**, the latest stable release. The canonical example is tested against both that release and the code on `main`; development-only 0.7 surfaces elsewhere on this site are identified by the site banner and stability labels. ## Prerequisites - PostgreSQL running locally or remotely -- Python 3.10+ +- [`uv`](https://docs.astral.sh/uv/getting-started/installation/) (it will use or install a compatible Python 3.10+ interpreter) - A database URL exported as `DATABASE_URL` Example local URL: @@ -25,19 +18,20 @@ Example local URL: export DATABASE_URL=postgres://postgres:test@localhost:15432/awa_test ``` -## 1. Install Packages +## 1. Create a Project ```bash -python -m venv .venv -source .venv/bin/activate - -pip install awa-pg +uv init awa-python-quickstart --bare +cd awa-python-quickstart +uv add awa-pg==0.6.6 ``` +`uv init` creates a minimal Python project. `uv add` creates and manages the project's virtual environment, records `awa-pg` in `pyproject.toml`, and writes a lockfile—there is no environment activation step. + ## 2. Run Migrations ```bash -python -m awa --database-url "$DATABASE_URL" migrate +uv run python -m awa --database-url "$DATABASE_URL" migrate ``` ## 3. Create a Worker @@ -45,66 +39,41 @@ python -m awa --database-url "$DATABASE_URL" migrate Create `quickstart.py`: ```python -import asyncio -import os -from dataclasses import dataclass - -import awa - -DATABASE_URL = os.environ["DATABASE_URL"] - - -@dataclass -class SendEmail: - to: str - subject: str - - -async def main() -> None: - client = awa.AsyncClient(DATABASE_URL) - - @client.task(SendEmail, queue="email") - async def handle_email(job): - print(f"sending email to {job.args.to}: {job.args.subject}") - - await client.start([("email", 2)]) - - job = await client.insert( - SendEmail(to="alice@example.com", subject="Welcome"), - queue="email", - ) - - await asyncio.sleep(1) - - result = await client.get_job(job.id) - print(f"job {result.id} state = {result.state}") - - await client.shutdown() - - -asyncio.run(main()) +--8<-- "awa-python/examples/quickstart.py" ``` +This page includes the repository's canonical example verbatim. CI runs it against PostgreSQL and the docs check compiles its Python syntax. + ## 4. Run It ```bash -python quickstart.py +uv run python quickstart.py ``` -Expected output is similar to: +You should see the inserted job, the handler output, and the terminal state. The first two lines can swap order because the worker starts before the insert: ```text -sending email to alice@example.com: Welcome -job 1 state = completed +Inserted job 1 (kind=send_email, state=available) +Sending email to alice@example.com: Welcome +Job 1 state: completed ``` +## What happened? + +1. `client.migrate()` made the example standalone; the explicit migration command in step 2 is the deployment-friendly path. +2. Inserting the job wrote durable state to PostgreSQL. +3. The worker claimed it, incremented the attempt, and kept the claim alive while the handler ran. +4. The handler result became durable `completed` state, which the final query read back. + +Retries, callback waits, and progress checkpoints follow the same rule: PostgreSQL is the system of record, not worker memory. When you debug a job, inspect its durable snapshot first instead of relying only on worker logs. + ## 5. Inspect the Queue ```bash -python -m awa --database-url "$DATABASE_URL" job list --queue email -python -m awa --database-url "$DATABASE_URL" job dump 1 -python -m awa --database-url "$DATABASE_URL" job dump-run 1 -python -m awa --database-url "$DATABASE_URL" queue stats +uv run python -m awa --database-url "$DATABASE_URL" job list --queue email +uv run python -m awa --database-url "$DATABASE_URL" job dump 1 +uv run python -m awa --database-url "$DATABASE_URL" job dump-run 1 +uv run python -m awa --database-url "$DATABASE_URL" queue stats ``` `job dump` gives you the whole job snapshot as JSON. `job dump-run` focuses on one attempt: the current attempt uses live row data, while historical attempts are reconstructed from the stored `errors[]` history. @@ -114,12 +83,12 @@ python -m awa --database-url "$DATABASE_URL" queue stats The dashboard ships in a separate wheel so the default `awa-pg` install stays small for workers and producers. Install the `[ui]` extra to bring in the `awa-cli` binary that hosts it: ```bash -pip install 'awa-pg[ui]' -python -m awa --database-url "$DATABASE_URL" serve +uv add 'awa-pg[ui]==0.6.6' +uv run python -m awa --database-url "$DATABASE_URL" serve # → http://127.0.0.1:3000 ``` -`python -m awa serve` delegates to the `awa serve` binary (you can also call `awa serve` directly once the extra is installed). The UI is read-only when the database reports `transaction_read_only = on` (e.g. on a replica) or when `--read-only` is passed. +`uv run python -m awa serve` delegates to the `awa serve` binary (you can also call `awa serve` directly once the extra is installed). The UI is read-only when the database reports `transaction_read_only = on` (e.g. on a replica) or when `--read-only` is passed. ## Useful Variants @@ -135,7 +104,7 @@ Most applications should keep using their normal database stack for business tab Install the app database libraries you already use, for example: ```bash -pip install 'sqlalchemy[asyncio]' asyncpg +uv add 'sqlalchemy[asyncio]' asyncpg ``` Then enqueue in the same SQLAlchemy transaction as your application write: @@ -182,7 +151,7 @@ await client.insert( ) ``` -At the default `enqueue_shards = 1` the key is ignored (everything is on shard 0 anyway). See [ADR-025](adr/025-sharded-enqueue-heads.md) for the partitioned-FIFO contract and [`docs/upgrade-0.5-to-0.6.md`](upgrade-0.5-to-0.6.md#raising-enqueue_shards) for the operator-side knob. +At the default `enqueue_shards = 1` the key is ignored (everything is on shard 0 anyway). See [ADR-025](adr/025-sharded-enqueue-heads.md) for the partitioned-FIFO contract and [queue configuration](configuration.md#sharding-the-enqueue-head-per-queue) for the operator-side knob. ### Exporting OpenTelemetry metrics @@ -199,7 +168,7 @@ awa.init_telemetry( # ... then build the client and start workers as normal. ``` -`init_telemetry` is idempotent; only the first call installs a provider. Call `awa.shutdown_telemetry()` at the end of short-lived scripts to flush pending metrics. See [`awa-python/examples/telemetry.py`](../awa-python/examples/telemetry.py) for a runnable example. +`init_telemetry` is idempotent; only the first call installs a provider. Call `awa.shutdown_telemetry()` at the end of short-lived scripts to flush pending metrics. See [`awa-python/examples/telemetry.py`](https://github.com/hardbyte/awa/blob/main/awa-python/examples/telemetry.py) for a runnable example. ### Distributed tracing @@ -228,8 +197,8 @@ the enqueue-site context so the trace still connects. ## More Examples -- [Bundled quickstart example](../awa-python/examples/quickstart.py) -- [ETL pipeline example](../examples/python/etl_pipeline.py) -- [Webhook callback example](../examples/python/webhook_payments.py) +- [Bundled quickstart example](https://github.com/hardbyte/awa/blob/main/awa-python/examples/quickstart.py) +- [ETL pipeline example](https://github.com/hardbyte/awa/blob/main/examples/python/etl_pipeline.py) +- [Webhook callback example](https://github.com/hardbyte/awa/blob/main/examples/python/webhook_payments.py) - [Deployment guide](deployment.md) - [Troubleshooting](troubleshooting.md) diff --git a/docs/getting-started-rust.md b/docs/getting-started-rust.md index 404abf96..433498a3 100644 --- a/docs/getting-started-rust.md +++ b/docs/getting-started-rust.md @@ -2,6 +2,10 @@ This guide takes you from `cargo add` to a job reaching `completed`. +!!! note "Version used in this guide" + + The install commands pin **v0.6.6**, the latest stable release. The canonical example is tested against both that release and the code on `main`; development-only 0.7 surfaces elsewhere on this site are identified by the site banner and stability labels. + ## Mental Model Before writing code, it helps to know what Awa is doing for you: @@ -31,7 +35,7 @@ export DATABASE_URL=postgres://postgres:test@localhost:15432/awa_test cargo new awa-rust-quickstart cd awa-rust-quickstart -cargo add awa +cargo add awa@0.6.6 cargo add sqlx --features runtime-tokio-rustls,postgres cargo add tokio --features macros,rt-multi-thread,time cargo add serde --features derive @@ -42,75 +46,11 @@ cargo add serde --features derive Put this in `src/main.rs`: ```rust -use awa::{admin, insert_with, migrations, Client, InsertOpts, JobArgs, JobResult, QueueConfig}; -use serde::{Deserialize, Serialize}; -use sqlx::postgres::PgPoolOptions; -use std::{env, time::Duration}; - -#[derive(Debug, Serialize, Deserialize)] -struct SendEmail { - to: String, - subject: String, -} - -impl JobArgs for SendEmail { - fn kind() -> &'static str { - "send_email" - } -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let database_url = env::var("DATABASE_URL")?; - - let pool = PgPoolOptions::new() - .max_connections(10) - .connect(&database_url) - .await?; - - // This is your application's sqlx pool. Awa uses it for queue storage, - // but it does not become your general database abstraction. - migrations::run(&pool).await?; - - let client = Client::builder(pool.clone()) - .queue( - "email", - QueueConfig { - max_workers: 2, - ..Default::default() - }, - ) - .register::(|args, _ctx| async move { - println!("sending email to {}: {}", args.to, args.subject); - Ok(JobResult::Completed) - }) - .build()?; - - client.start().await?; - - let job = insert_with( - &pool, - &SendEmail { - to: "alice@example.com".into(), - subject: "Welcome".into(), - }, - InsertOpts { - queue: "email".into(), - ..Default::default() - }, - ) - .await?; - - tokio::time::sleep(Duration::from_secs(1)).await; - - let job = admin::get_job(&pool, job.id).await?; - println!("job {} state = {:?}", job.id, job.state); - - client.shutdown(Duration::from_secs(5)).await; - Ok(()) -} +--8<-- "awa/examples/quickstart.rs" ``` +This page includes the repository's canonical example verbatim. The docs check compiles it on every change. + ## 3. Run It ```bash @@ -129,7 +69,14 @@ job 1 state = Completed Install the CLI if you want migration/admin/UI commands: ```bash -pip install awa-cli +uv tool install awa-cli==0.6.6 +``` + +If uv reports that its tool directory is not on `PATH`, update your shell and +open a new terminal before continuing: + +```bash +uv tool update-shell ``` Then inspect what happened: @@ -152,7 +99,7 @@ The UI starts on `http://127.0.0.1:3000` by default. - `Client::start()` spawns background tasks and returns immediately. Your service should usually stay alive until it receives a shutdown signal. - `Client::shutdown(Duration)` is the graceful drain path. Set your container or process shutdown timeout slightly above that duration. - If you only need to enqueue jobs from Rust, depend on `awa-model` instead of `awa`. -- If your service runs a `tracing-opentelemetry` layer, distributed tracing is automatic: enqueues capture the current span's context and the worker's `job.execute` span continues that trace (retries link back instead — see [`configuration.md`](configuration.md#distributed-tracing) and [ADR-039](adr/039-trace-propagation.md)). To propagate onward from a handler (outgoing HTTP headers), use the ambient context — `awa_model::trace::current_traceparent()` — so the downstream span is a child of the execution span; `ctx.traceparent()` returns the stored *enqueue-site* context for inspection. +- If your service runs a `tracing-opentelemetry` layer, distributed tracing is automatic: enqueues capture the current span's context and the worker's `job.execute` span continues that trace (retries link back instead — see [Distributed tracing](configuration.md#distributed-tracing) and [ADR-039](adr/039-trace-propagation.md)). To propagate onward from a handler (outgoing HTTP headers), use the ambient context — `awa_model::trace::current_traceparent()` — so the downstream span is a child of the execution span; `ctx.traceparent()` returns the stored *enqueue-site* context for inspection. When enqueueing from a request or service method that already writes app data, use your existing `sqlx` transaction and pass it to Awa: @@ -196,7 +143,7 @@ let opts = InsertOpts { awa::insert_with(&pool, &UpdateCustomer { customer_id, payload }, opts).await?; ``` -At the default `enqueue_shards = 1` the key is ignored. See [ADR-025](adr/025-sharded-enqueue-heads.md) for the partitioned-FIFO contract and [`docs/upgrade-0.5-to-0.6.md`](upgrade-0.5-to-0.6.md#raising-enqueue_shards) for the operator-side knob. +At the default `enqueue_shards = 1` the key is ignored. See [ADR-025](adr/025-sharded-enqueue-heads.md) for the partitioned-FIFO contract and [queue configuration](configuration.md#sharding-the-enqueue-head-per-queue) for the operator-side knob. ## Next @@ -204,8 +151,8 @@ At the default `enqueue_shards = 1` the key is ignored. See [ADR-025](adr/025-sh - [Deployment guide](deployment.md) - [Migration guide](migrations.md) - [Troubleshooting](troubleshooting.md) -- [Advanced Rust example](../awa/examples/etl_pipeline.rs) -- [Deadline-bounded polling pattern](../awa/examples/poll_until_deadline.rs) — poll an external system every X until it's ready or the deadline expires, using `JobResult::Snooze` so polls don't burn attempts. +- [Advanced Rust example](https://github.com/hardbyte/awa/blob/main/awa/examples/etl_pipeline.rs) +- [Deadline-bounded polling pattern](https://github.com/hardbyte/awa/blob/main/awa/examples/poll_until_deadline.rs) — poll an external system every X until it's ready or the deadline expires, using `JobResult::Snooze` so polls don't burn attempts. **Dashboard mid-run** — three polling jobs in flight (1 failed terminally, 1 scheduled between snoozes, 1 completed). diff --git a/docs/grafana/README.md b/docs/grafana/README.md index b8c47b73..da24fad3 100644 --- a/docs/grafana/README.md +++ b/docs/grafana/README.md @@ -11,7 +11,7 @@ Previews (live demo workload, both dashboards rendered against a local `grafana/ | --------------------------------- | ----------------------------- | | ![](screenshots/awa-postgres.png) | ![](screenshots/awa-otel.png) | -Alert rules (see [`alerts/`](alerts/)) import as Grafana unified-alerting file provisioning. Here's the rule browser after provisioning both variants, with the "no active runtime (Postgres)" rule firing after we stopped the demo worker: +Alert rules (see the [alert provisioning guide](alerts/README.md)) import as Grafana unified-alerting file provisioning. Here's the rule browser after provisioning both variants, with the "no active runtime (Postgres)" rule firing after we stopped the demo worker: ![](screenshots/awa-alerts.png) @@ -129,7 +129,7 @@ each heartbeat tick roots one at `heartbeat.tick`. Both are **`debug`**, so an `info` pipeline shows neither — they tick whether or not there is work, and at the default poll interval that would be ~5 traces/s per queue-claimer. Raise the filter when you want them; see -[configuration.md](../configuration.md#worker-side-traces). +[Worker-side traces](../configuration.md#worker-side-traces). Spans and log events for work on a *specific* queue carry both `queue` and the OTel `messaging.destination.name`, so diff --git a/docs/guides/index.md b/docs/guides/index.md new file mode 100644 index 00000000..c4220139 --- /dev/null +++ b/docs/guides/index.md @@ -0,0 +1,13 @@ +# Guides + +Use these guides once a quickstart is running. + +| Goal | Guide | +| --- | --- | +| Enqueue inside a Python framework transaction | [Bridge adapters](../bridge-adapters.md) | +| Run follow-up work after a durable outcome | [Lifecycle hooks](../lifecycle-hooks.md) | +| Send work to an HTTP function and wait for its callback | [HTTP callbacks](../http-callbacks.md) | +| Accept callbacks in an application-owned endpoint | [Callback receivers](../callback-receivers.md) | +| Retain, inspect, and redrive exhausted failures | [Dead-letter queue](../dead-letter-queue.md) | + +For runtime configuration, production rollout, role separation, monitoring, and incidents, continue to [Operations](../operations/index.md). diff --git a/docs/http-callbacks.md b/docs/http-callbacks.md index c4b4b729..ce00fc52 100644 --- a/docs/http-callbacks.md +++ b/docs/http-callbacks.md @@ -201,18 +201,17 @@ let router = callback_router( .await?; ``` -Use [`HttpWorkerConfig::callback_path_prefix`](#configuration) on the worker side to match a non-default `--path-prefix` so the URLs the worker hands to your function point at the receiver. +Use [`HttpWorkerConfig::callback_path_prefix`](#configure-the-worker) on the worker side to match a non-default `--path-prefix` so the URLs the worker hands to your function point at the receiver. -If you want callbacks to land inside your own application (FastAPI, axum, etc.) rather than running Awa's receiver at all, see [`docs/callback-receivers.md`](./callback-receivers.md) for the user-owned API integration pattern. +If you want callbacks to land inside your own application (FastAPI, axum, etc.) rather than running Awa's receiver at all, see [Callback receivers](./callback-receivers.md) for the user-owned API integration pattern. ## Function-side verification The signature primarily protects the Awa callback receiver from unauthorized completion requests. If the function endpoint is public, you may also verify the worker-to-function request before starting work. In that case the function must know the same 32-byte secret and recompute the BLAKE3 keyed hash over the received `callback_id`. -Python example: +Add the dependency with `uv add blake3`, then: ```python -# pip install blake3 import blake3 import hmac diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..f117cfe8 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,81 @@ +--- +hide: + - navigation + - toc +--- + +
+
+ +# Background jobs that live with your data + +AWA is a Postgres-native job queue for Rust and Python. Enqueue work in the same transaction as your application data, then run durable workers with retries, scheduling, progress, callbacks, and crash recovery. + +
+[Start with Rust](getting-started-rust.md){ .md-button .md-button--primary } +[Start with Python](getting-started-python.md){ .md-button } +
+ +
+
+![Application writes and AWA jobs flowing through PostgreSQL to Rust and Python workers](assets/architecture-flow.svg) +
+
+ +## One system of record + +Business data and job state share PostgreSQL. A transaction either commits both the application change and its follow-up work, or neither. Workers claim runnable rows without a separate broker, keep active claims alive, and leave enough state behind for another worker to recover work after a crash. + +
+ + Transactional by design + Commit an application write and its background job atomically. + + + Durable execution + Persist attempts, retries, schedules, callback waits, and progress. + + + Operable in production + Inspect queues with the CLI, web UI, metrics, traces, and SQL. + +
+ +## Pick the interface that fits your service + +=== "Rust" + + ```bash + cargo add awa@0.6.6 + ``` + + Define typed job arguments, register async handlers, and use your existing `sqlx` pool for migrations, enqueueing, and administration. + + [Build a Rust worker →](getting-started-rust.md) + +=== "Python" + + ```bash + uv add awa-pg==0.6.6 + ``` + + Register sync or async handlers around dataclasses and use direct clients or transaction bridges for asyncpg, psycopg, SQLAlchemy, and Django. + + [Build a Python worker →](getting-started-python.md) + +=== "Operations" + + ```bash + uv tool install awa-cli==0.6.6 + awa --database-url "$DATABASE_URL" health + ``` + + Run migrations, inspect jobs and queues, administer the dead-letter queue, and serve the optional web dashboard. + + [Explore the CLI →](reference/cli.md) + +## Understand the boundaries + +AWA provides **at-least-once delivery**: a handler may run more than once when a worker loses its claim or its completion write cannot be confirmed. Handlers should therefore be idempotent, or make their side effects transactional. + +Start with [how AWA works](concepts/index.md), then use the [deployment](deployment.md), [security](security.md), and [troubleshooting](troubleshooting.md) guides before production. diff --git a/docs/javascripts/agent-docs.js b/docs/javascripts/agent-docs.js new file mode 100644 index 00000000..24b38896 --- /dev/null +++ b/docs/javascripts/agent-docs.js @@ -0,0 +1,59 @@ +(() => { + const markdownUrl = () => new URL("index.md", window.location.href); + + const copyMarkdown = async (button) => { + const defaultLabel = "Copy Markdown"; + const response = await fetch(markdownUrl(), { + headers: { Accept: "text/markdown" }, + }); + if (!response.ok) { + throw new Error(`Markdown request failed with ${response.status}`); + } + await navigator.clipboard.writeText(await response.text()); + button.textContent = "Copied"; + window.setTimeout(() => { + button.textContent = defaultLabel; + }, 1600); + }; + + const mount = () => { + const content = document.querySelector("article.md-content__inner"); + if (!content || content.querySelector(".awa-agent-actions")) return; + + const actions = document.createElement("div"); + actions.className = "awa-agent-actions"; + + const copy = document.createElement("button"); + copy.type = "button"; + copy.className = "md-button md-button--primary awa-agent-copy"; + copy.textContent = "Copy Markdown"; + copy.addEventListener("click", async () => { + copy.disabled = true; + try { + await copyMarkdown(copy); + } catch (error) { + copy.textContent = "Copy failed"; + console.error(error); + window.setTimeout(() => { + copy.textContent = "Copy Markdown"; + }, 1600); + } finally { + copy.disabled = false; + } + }); + + const view = document.createElement("a"); + view.className = "md-button awa-agent-view"; + view.href = markdownUrl(); + view.textContent = "View Markdown"; + + actions.append(copy, view); + content.insertBefore(actions, content.firstChild); + }; + + if (typeof document$ !== "undefined") { + document$.subscribe(mount); + } else { + document.addEventListener("DOMContentLoaded", mount); + } +})(); diff --git a/docs/javascripts/language-switch.js b/docs/javascripts/language-switch.js new file mode 100644 index 00000000..e1daf2de --- /dev/null +++ b/docs/javascripts/language-switch.js @@ -0,0 +1,106 @@ +(function () { + "use strict"; + + const storageKey = "awa-docs-language"; + const languages = ["rust", "python"]; + const guidePaths = { + rust: "getting-started-rust/", + python: "getting-started-python/", + }; + + function savedLanguage() { + try { + const value = window.localStorage.getItem(storageKey); + return languages.includes(value) ? value : null; + } catch (_) { + return null; + } + } + + function saveLanguage(language) { + try { + window.localStorage.setItem(storageKey, language); + } catch (_) { + // The preference is optional; private browsing may disable storage. + } + } + + function languageTabs() { + return Array.from(document.querySelectorAll(".tabbed-labels > label")).filter( + (label) => languages.includes(label.textContent.trim().toLowerCase()), + ); + } + + function selectTabs(language) { + languageTabs() + .filter((label) => label.textContent.trim().toLowerCase() === language) + .forEach((label) => label.click()); + } + + function updateButtons(language) { + document.querySelectorAll("[data-awa-language]").forEach((button) => { + button.setAttribute( + "aria-pressed", + String(button.dataset.awaLanguage === language), + ); + }); + } + + function currentGuideLanguage() { + return languages.find((language) => + window.location.pathname.endsWith(guidePaths[language]), + ); + } + + function pairedGuideUrl(language) { + const current = currentGuideLanguage(); + if (!current) return null; + return new URL( + window.location.href.replace(guidePaths[current], guidePaths[language]), + ); + } + + function apply(language) { + updateButtons(language); + selectTabs(language); + } + + function choose(language) { + saveLanguage(language); + apply(language); + + const destination = pairedGuideUrl(language); + if (destination && destination.href !== window.location.href) { + window.location.assign(destination); + } + } + + function initialise() { + const switcher = document.querySelector("[data-awa-language-switch]"); + if (!switcher) return; + + const tabs = languageTabs(); + const hasPair = languages.every((language) => + tabs.some((label) => label.textContent.trim().toLowerCase() === language), + ); + if (!currentGuideLanguage() && !hasPair) return; + + switcher.hidden = false; + switcher.querySelectorAll("[data-awa-language]").forEach((button) => { + button.addEventListener("click", () => + choose(button.dataset.awaLanguage), + ); + }); + + // Reflect the page's own language when on a guide page; fall back to the + // saved preference for tabbed pages. Displaying is not choosing, so the + // stored preference is left untouched. + apply(currentGuideLanguage() || savedLanguage() || "rust"); + } + + if (typeof document$ !== "undefined") { + document$.subscribe(initialise); + } else { + document.addEventListener("DOMContentLoaded", initialise); + } +})(); diff --git a/docs/migrations.md b/docs/migrations.md index a7266e43..9e80d83c 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -30,23 +30,23 @@ The default queue-storage substrate in `awa.*` is migration-owned. Custom queue- awa --database-url "$DATABASE_URL" migrate ``` -### Rust +=== "Rust" -```rust -awa::migrations::run(&pool).await?; -``` + ```rust + awa::migrations::run(&pool).await?; + ``` -### Python +=== "Python" -```python -await client.migrate() -``` + ```python + await client.migrate() + ``` -or: + Or migrate without constructing a client: -```python -await awa.migrate(database_url) -``` + ```python + await awa.migrate(database_url) + ``` ## Upgrade an Existing Database diff --git a/docs/operations/index.md b/docs/operations/index.md new file mode 100644 index 00000000..072c7d63 --- /dev/null +++ b/docs/operations/index.md @@ -0,0 +1,21 @@ +# Operations + +AWA keeps its control plane in PostgreSQL, so safe operation begins with database privileges, migrations, compatibility, and observability. + +## Production path + +1. Review [Configuration](../configuration.md) for queues, workers, deadlines, priorities, and storage controls. +2. Read [Deployment](../deployment.md) for process topology, graceful shutdown, migration ordering, and container images. +3. Use [Managed Postgres](../deploying-on-managed-postgres.md) for hosted-service constraints and tuning. +4. Apply the role split in [Security](../security.md). +5. Import the [Grafana dashboards and alerts](../grafana/README.md), or query the same health surfaces with the CLI. +6. Keep [Troubleshooting](../troubleshooting.md) with your runbooks. + +## Upgrades + +- [Migrations](../migrations.md) explains forward-only schema changes and application/CLI entry points. +- [Upgrade 0.5 to 0.6](../upgrade-0.5-to-0.6.md) covers the queue-storage transition. +- [Upgrade 0.6 to 0.7](../upgrade-0.6-to-0.7.md) covers the current development-line rollout contract. + +!!! note "Match documentation to your installed version" + This site tracks `main` and 0.7 development. Use the release tag and package documentation for a stable 0.6 deployment, especially for migration and storage-transition procedures. diff --git a/docs/overrides/main.html b/docs/overrides/main.html new file mode 100644 index 00000000..7dc69e34 --- /dev/null +++ b/docs/overrides/main.html @@ -0,0 +1,36 @@ +{% extends "base.html" %} + +{% block announce %} + + These docs track main / 0.7 development. + Latest stable: + v0.6.6. + +{% endblock %} + +{% block content %} + + {{ super() }} +{% endblock %} + +{% block extrahead %} + {{ super() }} + {% set social_title = page.title ~ " — " ~ config.site_name if page else config.site_name %} + {% set social_description = page.meta.description if page and page.meta and page.meta.description else config.site_description %} + {% set social_url = page.canonical_url if page and page.canonical_url else config.site_url %} + + + + + + {% if page %} + + {% endif %} + +{% endblock %} diff --git a/docs/queue-storage-substrate.md b/docs/queue-storage-substrate.md index b5a20197..616a5f07 100644 --- a/docs/queue-storage-substrate.md +++ b/docs/queue-storage-substrate.md @@ -1,31 +1,35 @@ -# Queue-storage substrate: ownership and customisation +# Queue storage -Awa's queue-storage substrate is the set of per-schema tables, indexes, sequences, and helper functions the runtime relies on to claim, execute, and finalise jobs against the queue-storage engine. This page explains who owns those objects, how to customise them, and the guardrails that protect the default `awa` schema from accidental destructive operations. +Queue storage is Awa's PostgreSQL layout for runnable work, in-flight attempts, deferred work, terminal history, and the small control tables that coordinate workers. It is an implementation boundary, not an application API: producers and workers should use the Rust or Python clients, and operators should use the CLI, admin API, and documented read views. -## Ownership contract +This page explains the storage shape an operator needs to understand. For the end-to-end runtime design, see [Architecture](architecture.md). For migration ownership and external migration runners, see [Migrations](migrations.md). -| Owner | Scope | When it runs | +## Why the queue is split into planes + +A single frequently-updated jobs table accumulates dead tuples and makes claiming, history retention, and long-running attempts compete for the same indexes. Awa instead gives each workload a storage shape suited to its lifecycle: + +| Plane | Primary objects | Purpose | | --- | --- | --- | -| **`awa migrate`** | The canonical schema (`awa.schema_version`, `awa.runtime_instances`, `awa.storage_transition_state`, `awa.runtime_storage_backends`, etc.) **plus** the default queue-storage substrate at `awa.*` (`awa.ready_entries`, `awa.ready_tombstones`, `awa.ready_segments`, `awa.done_entries`, `awa.queue_terminal_count_deltas`, `awa.leases`, `awa.lease_claims`, `awa.lease_claim_batches`, `awa.lease_claim_closures`, `awa.lease_claim_closure_batches`, `awa.lease_claim_receipt_id_seq`, `awa.lease_claim_batch_id_seq`, `awa.queue_ring_state`, `awa.queue_claim_heads` (with its now-unused legacy ready-segment cache columns), `awa.queue_terminal_live_counts`, the partitions for each, the `awa.claim_ready_runtime` helper, and the `awa.install_queue_storage_substrate` function itself). | Install time and operator-driven upgrades. | -| **`QueueStorage::prepare_schema()` (Rust) / `awa storage prepare-queue-storage-schema` (CLI)** | Non-default queue-storage schemas (anything other than `awa`), repair flows, test setups. Calls `awa.install_queue_storage_substrate(, ...)` and performs a small set of legacy upgrade fixups that don't belong in a forward-only DDL function. | Custom-schema deployments; targeted repair; the test suite. | -| **The `awa.install_queue_storage_substrate(p_schema, ...)` SQL helper** | Per-schema DDL only. Idempotent. Activation-neutral — does NOT touch `awa.runtime_storage_backends` or `awa.storage_transition_state`. | Called by both `awa migrate` (for the default schema) and `prepare_schema()` (for custom schemas). Single source of truth for substrate DDL. | +| Ready queue | `ready_entries_*`, ready segments, tombstones | Append runnable work and claim it in ordered lanes. Whole ring slots can later be reclaimed. | +| Deferred queue | `deferred_jobs` | Hold scheduled and retryable work until maintenance promotes it. | +| Execution | claim receipts, claim batches, closures, `leases_*`, `attempt_state` | Prove which attempt owns a job. Short jobs use compact receipt evidence; jobs needing mutable state materialise a lease. | +| Terminal history | `done_entries_*`, compact completion batches, count deltas and rollups | Retain completion facts without copying the full job body into every terminal row. | +| Operator hold | `dlq_entries` | Keep explicitly dead-lettered work available for inspection, retry, or purge. | +| Control | queue metadata, lane heads, ring ledgers, runtimes, cron and uniqueness tables | Coordinate dispatch and maintenance without putting mutable metadata on the hot history path. | -The helper takes a per-schema advisory transaction lock (`awa.queue_storage.install:`) so concurrent installs from Rust workers, the CLI, Python, or externally-extracted migration SQL serialise on the same key. +The public `{schema}.terminal_jobs` view hydrates terminal facts with retained job bodies. Physical ring tables are internal and must not be mutated directly. -## The default `awa` schema is migration-owned and default-shaped +## How storage stays bounded -For `p_schema = 'awa'` the helper rejects non-default configuration with `ERRCODE = 22023`: +Ready, receipt, lease, and terminal families are partitioned into ring slots. Maintenance advances each ring only after its reclaimability checks succeed, then truncates the old slot as a unit. Long-lived database snapshots can delay reclamation; they do not allow maintenance to skip the safety checks. Deferred and DLQ rows use their own promotion and retention paths rather than the ring. -- `lease_claim_receipts` must be `TRUE` -- `queue_slot_count` must be `16` -- `lease_slot_count` must be `8` -- `claim_slot_count` must be `8` +One worker holds the maintenance advisory lock at a time. That leader promotes due work, rescues stale attempts, rotates rings, folds count deltas, and publishes queue health. If it exits, another worker can take over from durable PostgreSQL state. -The default `awa.*` substrate is the single stable shape every fresh installation gets. If you need different slot counts or `lease_claim_receipts = FALSE`, use a custom queue-storage schema (see below). Attempts to tune the default schema get a clear error pointing at the custom-schema path. +## Default and custom schemas -## Custom queue-storage schemas +`awa migrate` installs the canonical control objects and the default queue-storage substrate in the `awa` schema. That default has a stable shape and cannot be reset through `prepare-queue-storage-schema`; `DROP SCHEMA awa CASCADE` would also destroy migration and transition metadata and is not a supported recovery action. -For non-default deployments — say a high-throughput tenant that wants `queue_slot_count = 32`, or a side-by-side rebuild during an incident — install a substrate under a different schema name: +Custom storage schemas are an advanced operational tool for a separately sized substrate or a side-by-side transition: ```bash awa storage prepare-queue-storage-schema \ @@ -34,95 +38,22 @@ awa storage prepare-queue-storage-schema \ --lease-slot-count 16 ``` -The CLI calls `awa.install_queue_storage_substrate('my_jobs', 32, 16, 8, TRUE)` under the per-schema advisory lock. Activate the schema as the queue-storage backend via `awa storage prepare`, `awa storage enter-mixed-transition`, and `awa storage finalize` once you're ready. See [the upgrade guide](upgrade-0.5-to-0.6.md) for the staged flow. - -## `--reset` is rejected for `--schema awa` - -`awa storage prepare-queue-storage-schema --reset` runs `DROP SCHEMA IF EXISTS CASCADE` before re-preparing. For `--schema awa` that would also drop `schema_version`, `runtime_instances`, `storage_transition_state`, and every other canonical migration table, leaving the database in an unrecoverable state. - -The CLI rejects this combination: - -```text -Error: "Refusing to DROP SCHEMA awa CASCADE — schema 'awa' is the -default migration-owned queue-storage substrate and also contains the -canonical migration tables (schema_version, runtime_instances, -storage_transition_state, etc.). Use --schema for a throwaway -substrate, or 'awa storage abort' to rewind an in-flight transition." -``` - -To rebuild a queue-storage substrate from scratch: - -- **Testing or recovery:** target a custom schema name with `--schema ` and activate it via the storage transition flow. -- **Rewind an in-flight transition:** use `awa storage abort`. -- **Full cluster rebuild:** restore from backup, then run `awa migrate`. - -`DROP SCHEMA awa CASCADE` is not a supported operator action. - -## Driving installs and upgrades from external migration tooling - -Teams that already run their schema changes through a tool like Sqitch, Liquibase, or a hand-rolled migration runner do not need to invoke `awa migrate` or any other Rust binary. The migration set extracted with `awa migrate --sql` (or `--extract-to`) is complete: it includes the substrate DDL via the v023 helper, and the staged transition is driven by SQL functions. - -### Fresh install (no canonical data yet) - -After applying the migration files, the first runtime that boots calls `awa.storage_auto_finalize_if_fresh('awa')` which atomically advances `canonical → active` when `awa.jobs` is empty and no live runtimes have heartbeated. External tooling can call the same function as a post-migrate step to land in `active` before the first worker even starts: - -```sql -SELECT awa.storage_auto_finalize_if_fresh('awa'); -``` - -`storage_auto_finalize_if_fresh` has `GRANT EXECUTE ... TO PUBLIC`, so the EXECUTE bit is open to any role. The function is `SECURITY INVOKER` and reads/writes `awa.storage_transition_state`, `awa.jobs`, `awa.runtime_instances`, and `awa.runtime_storage_backends`, so callers still need the normal runtime/migrator table privileges on those. - -### Upgrade from an existing canonical-only deployment - -`storage_auto_finalize_if_fresh` refuses to short-circuit when canonical work or live runtimes exist. The operator drives the staged transition with three SQL function calls, each of which mirrors the equivalent `awa storage` CLI subcommand: - -```sql --- (1) Mark queue-storage as the prepared target. -SELECT awa.storage_prepare('queue_storage', '{"schema":"awa"}'::jsonb); - --- (2) Bring up at least one worker with --- transition_role=queue_storage_target. Stop any canonical-only --- workers. Then flip routing into mixed mode: -SELECT awa.storage_enter_mixed_transition(); - --- (3) Wait for workers to drain the canonical backlog onto --- queue-storage. The two SQL gates `storage_finalize` enforces --- are observable directly: --- --- SELECT awa.canonical_live_backlog(); --- -- must return 0 before finalize will advance. --- --- SELECT count(*) --- FROM awa.runtime_instances --- WHERE storage_capability = 'canonical' --- AND last_seen_at + make_interval( --- secs => GREATEST(((GREATEST(snapshot_interval_ms, 1000) / 1000) * 3)::int, 30) --- ) >= now(); --- -- must also be 0 (no live canonical-only runtimes). --- --- When both are 0, finalize: -SELECT awa.storage_finalize(); -``` - -`storage_enter_mixed_transition` rejects the call until at least one live `queue_storage_target` runtime is heartbeating. `storage_finalize` rejects while `awa.canonical_live_backlog() > 0` or a canonical-only runtime is live. - -Once routing has flipped, a canonical attempt that re-schedules itself — snooze, retry backoff, or `RetryAfter` — leaves the canonical plane instead of returning to canonical `scheduled_jobs` ([#456](https://github.com/hardbyte/awa/issues/456)). Ordinarily its fresh-id successor is written to the prepared schema's `deferred_jobs`. If the destination state claims uniqueness and a newer duplicate acquired that key while `running` was outside the mask, the attempted successor is recorded as a cancelled terminal row with a `rescheduled as duplicate` error; it never becomes executable and the newer claim holder wins. The canonical backlog therefore still converges for handlers that snooze on every run rather than completing. Pre-flip auto runtimes report `canonical_drain_only` after routing flips; once the backlog is empty they are idle and v040 allows finalization without waiting for those processes to restart or their heartbeat rows to expire. - -The orchestration (start new-mode workers, stop old-mode workers, wait for drain) is unchanged from the CLI flow — only the invocation surface is different. - -## Design rationale +The command invokes the idempotent `awa.install_queue_storage_substrate(...)` helper under a per-schema advisory transaction lock. The helper is activation-neutral: preparing a schema does not route work to it. Activation is a separate staged transition described in [Upgrading from 0.5 to 0.6](upgrade-0.5-to-0.6.md). -The split between migration-owned default substrate and helper-installed custom substrate exists for three reasons: +The installer is `SECURITY INVOKER`; its caller needs DDL privileges on the target schema. Workers need runtime DML privileges and `TRUNCATE` for guarded ring reclamation, but do not need DDL. See [Database roles](security/database-roles.md). -- **`awa migrate --sql` / `--extract-to` must reproduce the full default runtime schema** for external migration tools to be useful. All substrate DDL is reachable from the migration through the SQL helper, so the extracted SQL is complete. -- **Migrations that depend on queue-storage tables can write unconditional DDL.** Operations like the `done_entries` terminal-count delta append inside `awa.delete_job_compat()` need the delta table to exist. Migration ordering guarantees it. -- **The default schema cannot be accidentally destroyed.** The reset guard above protects operators from a `DROP SCHEMA awa CASCADE` that would take the canonical migration tables with it. +## Operator rules -The helper is `SECURITY INVOKER` so callers need their own DDL privileges on the target schema; the runtime role does not gain DDL through the helper, which keeps the principle-of-least-privilege role model intact. +- Treat physical queue-storage tables and helper functions as internal unless a page explicitly names a public surface. +- Use the CLI transition commands rather than editing `storage_transition_state` or `runtime_storage_backends`. +- Do not reset or drop the default `awa` schema. Restore from backup and rerun migrations for a full-cluster rebuild. +- Keep analytical transactions short on the primary; a pinned MVCC horizon delays best-effort ring reclamation. +- When using a custom schema, apply the same runtime grants to it and prepare it with the migrator role. -## See also +## Related reading -- [`docs/migrations.md`](migrations.md) — migration policy and the `awa migrate --sql` / `--extract-to` story. -- [`docs/architecture.md`](architecture.md) — overall storage layering. -- [`docs/security.md`](security.md) — role boundaries and the `SECURITY INVOKER` posture. -- [`docs/upgrade-0.5-to-0.6.md`](upgrade-0.5-to-0.6.md) — operator flow for an existing cluster. +- [Architecture](architecture.md) — runtime, storage, lifecycle, and recovery. +- [Migrations](migrations.md) — migration ownership and extracted SQL. +- [Database roles](security/database-roles.md) — production role separation and grants. +- [Storage upgrade guide](upgrade-0.5-to-0.6.md) — staged activation and drain. +- [ADR-019](adr/019-queue-storage-redesign.md) and [ADR-023](adr/023-receipt-plane-ring-partitioning.md) — decision rationale and consequences. diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 00000000..24c5d290 --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,34 @@ +# CLI command map + +The CLI is the migration and operations surface for AWA. The command itself is the exact reference for the installed version: + +```bash +awa --help +awa --help +``` + +| Command | Purpose | +| --- | --- | +| `migrate` | Apply AWA schema migrations | +| `health` | Check database and runtime health | +| `job` | List, inspect, dump, retry, cancel, or discard jobs | +| `queue` | Inspect queue statistics and runtime overrides | +| `dlq` | Inspect, redrive, or purge retained terminal failures | +| `batch-ops` | Operate on a selected set of jobs | +| `cron` | Inspect and manage periodic jobs | +| `storage` | Inspect or administer queue-storage transitions | +| `callbacks` | Run the callback receiver and administer callback state | +| `serve` | Host the web dashboard and admin API | +| `context` | Print shell or agent-oriented operational context | + +## Connection options + +Use `--database-url` or `DATABASE_URL`. Commands that mutate state can require stronger privileges than read-only inspection; use the [security guide](../security.md) to split migration, runtime, maintenance, and observer roles. + +```bash +awa --database-url "$DATABASE_URL" job list --queue payments +awa --database-url "$DATABASE_URL" job dump 42 +awa --database-url "$DATABASE_URL" queue stats +``` + +The web UI started by `awa serve` is read-only when PostgreSQL reports a read-only transaction or when the command receives `--read-only`. diff --git a/docs/reference/index.md b/docs/reference/index.md new file mode 100644 index 00000000..e4bfdd63 --- /dev/null +++ b/docs/reference/index.md @@ -0,0 +1,10 @@ +# Reference + +These pages map AWA's public surfaces without duplicating the versioned reference generated from source. + +- [CLI command map](cli.md) for migration and operations entry points. +- [Rust crates](rust.md) for choosing between the facade, storage model, worker, testing, and SeaORM crates. +- [Python API](python.md) for clients, job values, handler outcomes, and bridges. +- [Stability policy](../stability.md) for the guarantees attached to stable, experimental, and development surfaces. + +Exact signatures live with the installed artifact: use `awa --help`, Python's shipped type information, and the [AWA crate versions on docs.rs](https://docs.rs/crate/awa). Select the version that matches your installed dependency; the 0.7 development API is documented from source until that prerelease is published. diff --git a/docs/reference/python.md b/docs/reference/python.md new file mode 100644 index 00000000..ad6f5192 --- /dev/null +++ b/docs/reference/python.md @@ -0,0 +1,37 @@ +# Python API + +Install `awa-pg` for the Python clients and worker runtime: + +```bash +uv add awa-pg==0.6.6 +``` + +The package ships type information for its public surface. This page is a map; use Python help and your editor against the installed version for exact signatures. + +## Clients + +`awa.AsyncClient` +: Async migrations, workers, enqueue, admin queries, callbacks, and graceful shutdown. + +`awa.Client` +: Synchronous counterpart for scripts, workers, and producers. + +`awa.RawClient` +: Lower-level access when an application needs untyped job operations. + +`awa.PartitionedQueue` +: Deterministic routing across physical queues for a partitioned logical queue. + +## Common values + +- `Job` and `JobState` describe hydrated job state. +- `HealthCheck`, `QueueHealth`, and `QueueStat` expose operational status. +- `CallbackToken`, `WaitForCallback`, and `ResolveResult` model callback waits. +- `RetryAfter`, `Snooze`, and `Cancel` are handler outcomes. +- `DlqEntry` and `RetryFailedResult` support failure administration. + +## Transaction bridges + +`awa.bridge` inserts jobs through application-owned asyncpg, psycopg 3, SQLAlchemy, and Django transactions. See [Bridge adapters](../bridge-adapters.md) before using a framework session: the application, not AWA, remains responsible for commit and rollback. + +For a complete runnable program, follow the [Python getting started guide](../getting-started-python.md). diff --git a/docs/reference/rust.md b/docs/reference/rust.md new file mode 100644 index 00000000..2b4a30b2 --- /dev/null +++ b/docs/reference/rust.md @@ -0,0 +1,28 @@ +# Rust crates + +The workspace separates storage and runtime concerns so producers do not need to carry a worker runtime. + +| Crate | Use it for | Published versions | +| --- | --- | --- | +| `awa` | The usual Rust client: typed jobs, workers, migrations, enqueue, and admin APIs | [docs.rs versions](https://docs.rs/crate/awa) | +| `awa-model` | Schema, migrations, records, enqueue operations, and admin queries without workers | [docs.rs versions](https://docs.rs/crate/awa-model) | +| `awa-worker` | Worker runtime and execution internals | [docs.rs versions](https://docs.rs/crate/awa-worker) | +| `awa-testing` | PostgreSQL-backed fixtures and helpers for application tests | [docs.rs versions](https://docs.rs/crate/awa-testing) | +| `awa-seaorm` | SeaORM integration | [docs.rs versions](https://docs.rs/crate/awa-seaorm) | + +## Recommended entry point + +Most services should depend on `awa`. It re-exports the commonly used model and worker types, including `Client`, `JobArgs`, `JobResult`, `InsertOpts`, `QueueConfig`, migration helpers, and admin operations. + +```toml +[dependencies] +awa = "0.6" +``` + +These site docs track the 0.7 development branch. Select the version matching your dependency in the docs.rs version menu, and use the [stability policy](../stability.md) when evaluating alpha APIs. + +## Producers without workers + +Choose `awa-model` when a service only inserts or inspects jobs. It keeps the execution runtime out of that process while preserving the same PostgreSQL contract. + +For a complete compile-checked program, follow the [Rust getting started guide](../getting-started-rust.md). diff --git a/docs/security.md b/docs/security.md index 1ff8a50f..2b638da5 100644 --- a/docs/security.md +++ b/docs/security.md @@ -1,450 +1,25 @@ -# PostgreSQL Roles and Privileges +# Security -AWA can run with a single database user, but production deployments should separate **schema management** from **runtime execution**. This guide documents the minimum-privilege role model. +Awa's security boundary has two parts: PostgreSQL privileges determine who can read or mutate queue state, while network placement and callback authentication determine which HTTP surfaces are reachable. Production deployments should separate both. -## Role model +## Start here -``` -awa_owner NOLOGIN — owns all schema objects - ├── awa_migrator LOGIN — runs migrations (inherits awa_owner) - └── awa_runtime LOGIN — workers, producers, awa serve, CLI admin -``` +| Concern | Guidance | +| --- | --- | +| Database ownership and grants | [Database roles and privileges](security/database-roles.md) | +| Admin UI, callbacks, workers, and network exposure | [Deployable surfaces](security/deployable-surfaces.md) | +| Callback authentication and custom receivers | [Callback security](security/callback-security.md) | -**`awa_owner`** is a `NOLOGIN` group role that owns the `awa` schema and all objects in it. No process connects as `awa_owner` directly. +## Production baseline -**`awa_migrator`** is a `LOGIN` role that is a member of `awa_owner`. It runs `awa migrate` and can create/alter/drop schema objects. Use this role only for migrations — not for workers or the UI. +1. Use a non-login schema owner, a migration login, and a separate runtime login. +2. Keep `awa serve` on an authenticated operator network. It is a database administration surface, not a public application endpoint. +3. Put externally reachable callbacks on a callback-only listener or in your own application router; do not expose the admin router with them. +4. Configure callback signatures unless an authenticating proxy or trusted network already provides the boundary. +5. Use TLS, rotate secrets per environment, and avoid logging callback signatures. -**`awa_runtime`** is a `LOGIN` role with the minimum privileges needed to run workers, enqueue jobs, and serve the admin UI. It cannot modify the schema. - -## Setting up roles - -### 1. Create roles - -```sql --- Run as a superuser or database owner -CREATE ROLE awa_owner NOLOGIN; -CREATE ROLE awa_migrator LOGIN PASSWORD 'strong-password-here'; -CREATE ROLE awa_runtime LOGIN PASSWORD 'strong-password-here'; - --- awa_migrator inherits awa_owner (can create/alter schema objects) -GRANT awa_owner TO awa_migrator; - --- Both roles need to connect -GRANT CONNECT ON DATABASE mydb TO awa_migrator; -GRANT CONNECT ON DATABASE mydb TO awa_runtime; - --- awa_owner needs CREATE to make the schema -GRANT CREATE ON DATABASE mydb TO awa_owner; -``` - -### 2. Run migrations - -```bash -awa --database-url "postgres://awa_migrator:pass@host/mydb" migrate -``` - -The migrator creates the `awa` schema and all objects. Objects are owned by `awa_migrator` (who inherits `awa_owner`). - -### 3. Transfer ownership (recommended) - -After the initial migration, transfer object ownership to `awa_owner` so it's decoupled from the login role: - -```sql -ALTER SCHEMA awa OWNER TO awa_owner; - --- Transfer tables, partitioned tables, views, materialized views, and sequences. -DO $$ -DECLARE r RECORD; -BEGIN - FOR r IN - SELECT c.relkind, c.oid::regclass AS obj - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = 'awa' - AND c.relkind IN ('r', 'p', 'v', 'm', 'S') - LOOP - IF r.relkind = 'S' THEN - EXECUTE format('ALTER SEQUENCE %s OWNER TO awa_owner', r.obj); - ELSE - EXECUTE format('ALTER TABLE %s OWNER TO awa_owner', r.obj); - END IF; - END LOOP; -END$$; - --- Transfer functions. -DO $$ -DECLARE r RECORD; -BEGIN - FOR r IN SELECT p.oid::regprocedure AS func - FROM pg_proc p JOIN pg_namespace n ON p.pronamespace = n.oid - WHERE n.nspname = 'awa' LOOP - EXECUTE format('ALTER FUNCTION %s OWNER TO awa_owner', r.func); - END LOOP; -END$$; - --- Transfer standalone enum/domain types. Table row types and generated array --- types are owned through their base objects and should not be altered here. -DO $$ -DECLARE r RECORD; -BEGIN - FOR r IN - SELECT format('%I.%I', n.nspname, t.typname) AS typ - FROM pg_type t - JOIN pg_namespace n ON n.oid = t.typnamespace - WHERE n.nspname = 'awa' - AND t.typtype IN ('d', 'e') - LOOP - EXECUTE format('ALTER TYPE %s OWNER TO awa_owner', r.typ); - END LOOP; -END$$; -``` - -### 4. Grant runtime privileges - -```sql --- Schema access -GRANT USAGE ON SCHEMA awa TO awa_runtime; - --- Sequences: canonical `jobs_id_seq`, and queue-storage `job_id_seq` --- once prepare_schema has materialized it. -GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA awa TO awa_runtime; - --- All tables: the runtime needs full DML because triggers run as the --- invoking role (SECURITY INVOKER), so inserting a job also writes to --- the admin metadata cache tables via triggers. The maintenance leader --- also calls refresh_admin_metadata(), which truncates dirty-key tables --- after taking the metadata advisory lock. -GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA awa TO awa_runtime; - --- Functions (trigger functions execute with invoker privileges) -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA awa TO awa_runtime; --- Keep the queue-storage installer helper migrator/operator-only. -REVOKE EXECUTE ON FUNCTION awa.install_queue_storage_substrate(TEXT, INT, INT, INT, BOOLEAN) - FROM awa_runtime; - --- Default privileges for future migrations. -ALTER DEFAULT PRIVILEGES FOR ROLE awa_owner IN SCHEMA awa - GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE ON TABLES TO awa_runtime; -ALTER DEFAULT PRIVILEGES FOR ROLE awa_owner IN SCHEMA awa - GRANT USAGE, SELECT ON SEQUENCES TO awa_runtime; -ALTER DEFAULT PRIVILEGES FOR ROLE awa_owner IN SCHEMA awa - GRANT EXECUTE ON FUNCTIONS TO awa_runtime; - --- If migrations run as awa_migrator without `SET ROLE awa_owner`, future --- objects are owned by awa_migrator, so set defaults for that role too. -ALTER DEFAULT PRIVILEGES FOR ROLE awa_migrator IN SCHEMA awa - GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE ON TABLES TO awa_runtime; -ALTER DEFAULT PRIVILEGES FOR ROLE awa_migrator IN SCHEMA awa - GRANT USAGE, SELECT ON SEQUENCES TO awa_runtime; -ALTER DEFAULT PRIVILEGES FOR ROLE awa_migrator IN SCHEMA awa - GRANT EXECUTE ON FUNCTIONS TO awa_runtime; -``` - -If you use the compatibility `insert_many_copy` path (Rust `InsertOpts::copy()`), also grant: - -```sql -GRANT TEMP ON DATABASE mydb TO awa_runtime; -``` - -This allows creating temporary staging tables for the `COPY` bulk insert path. Queue-storage direct COPY (`QueueStorage::enqueue_params_copy` in Rust, `enqueue_many_copy` in Python) writes to the queue-storage tables directly and does not need this temporary-table grant. - -### Custom queue-storage schema - -The queue-storage backend defaults to keeping its tables in the same `awa` schema, so the grants above cover both control-plane and queue-storage tables. See [Queue-storage substrate](queue-storage-substrate.md) for the full ownership contract — what `awa migrate` installs by default, how custom schemas are materialised via `awa.install_queue_storage_substrate()`, and why `awa storage prepare-queue-storage-schema --schema awa --reset` is rejected. - -If you override the schema name (Rust: `QueueStorageConfig.schema`; Python: `queue_storage_schema=...`; CLI: `awa storage prepare-queue-storage-schema --schema `), repeat the grant block against that schema: - -```sql -GRANT USAGE ON SCHEMA my_qs_schema TO awa_runtime; -GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA my_qs_schema TO awa_runtime; -GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA my_qs_schema TO awa_runtime; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA my_qs_schema TO awa_runtime; -ALTER DEFAULT PRIVILEGES FOR ROLE awa_owner IN SCHEMA my_qs_schema - GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE ON TABLES TO awa_runtime; -ALTER DEFAULT PRIVILEGES FOR ROLE awa_owner IN SCHEMA my_qs_schema - GRANT USAGE, SELECT ON SEQUENCES TO awa_runtime; -ALTER DEFAULT PRIVILEGES FOR ROLE awa_owner IN SCHEMA my_qs_schema - GRANT EXECUTE ON FUNCTIONS TO awa_runtime; -ALTER DEFAULT PRIVILEGES FOR ROLE awa_migrator IN SCHEMA my_qs_schema - GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE ON TABLES TO awa_runtime; -ALTER DEFAULT PRIVILEGES FOR ROLE awa_migrator IN SCHEMA my_qs_schema - GRANT USAGE, SELECT ON SEQUENCES TO awa_runtime; -ALTER DEFAULT PRIVILEGES FOR ROLE awa_migrator IN SCHEMA my_qs_schema - GRANT EXECUTE ON FUNCTIONS TO awa_runtime; -``` - -The default `awa` queue-storage substrate is migrated by `awa migrate`. Custom queue-storage schemas are migrated by `awa storage prepare-queue-storage-schema`, which should also run as the migrator role and creates objects owned by `awa_migrator` / `awa_owner`. The runtime role needs read/write/execute privileges plus `TRUNCATE` for ring-partition rotation; it never needs DDL. - -### 5. Configure your processes - -| Process | Role | Connection string | -| --- | --- | --- | -| `awa migrate` | `awa_migrator` | `postgres://awa_migrator:pass@host/db` | -| Workers (Rust/Python) | `awa_runtime` | `postgres://awa_runtime:pass@host/db` | -| `awa serve` (UI + API) | `awa_runtime` | `postgres://awa_runtime:pass@host/db` | -| `awa job`, `awa queue`, etc | `awa_runtime` | `postgres://awa_runtime:pass@host/db` | - -## What the runtime role needs and why - -The runtime grants look broad (`SELECT, INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES`) because AWA's internal tables are maintained by `SECURITY INVOKER` trigger functions. When a worker inserts a job, triggers automatically update: - -- `awa.queue_state_counts` — per-queue job state tallies -- `awa.job_kind_catalog` — distinct job kinds -- `awa.job_queue_catalog` — distinct queues -- `awa.job_unique_claims` — unique key deduplication - -Since these triggers run with the caller's privileges, the runtime role needs write access to these internal tables even though application code never touches them directly. - -The maintenance leader also calls `awa.refresh_admin_metadata()` as a full reconciliation safety net. That function runs with invoker privileges and uses `TRUNCATE` on `awa.admin_dirty_queues` and `awa.admin_dirty_kinds` after taking the metadata advisory lock, so `awa_runtime` needs the `TRUNCATE` table privilege too. - -The runtime also directly upserts into the descriptor catalogs on startup and on each runtime snapshot tick: - -- `awa.queue_descriptors` — declared queue display names, ownership, tags -- `awa.job_kind_descriptors` — declared job-kind display names, ownership, tags -- `awa.runtime_instances` — the runtime's own liveness row, including per-queue and per-kind descriptor hashes used for drift detection - -These writes are not trigger-driven; they come from `ClientBuilder::build()` / `AsyncClient.start()` and from the snapshot reporter. The broad grant already covers them. - -Other PostgreSQL features used at runtime: - -| Feature | Purpose | Privilege needed | -| --- | --- | --- | -| `LISTEN` / `NOTIFY` | Queue wakeup without polling | `CONNECT` (no extra grant) | -| `pg_try_advisory_lock` | Leader election for maintenance | Built-in function (no grant) | -| `COPY ... FROM STDIN` | Bulk insert path | `TEMP` on database | -| `FOR UPDATE SKIP LOCKED` | Non-blocking job claiming | `SELECT`, `UPDATE` on table | - -## Planned caller-owned finalization role (ADR-042) - -> **Accepted design for 0.7; not shipped in the current release.** This section records the -> privilege boundary the implementation and migration must satisfy. - -The #401 implementation owns the narrow ADR-043 substrate required to make that boundary strict in -0.7: a pre-provisioned bounded execution owner, the exact `complete_job` manifest entry, transactional -ownership transfer and `PUBLIC` revocation, the application-role grant, and `awa doctor` validation. -The broader producer/executor/maintenance/admin capability split remains deferred under #452. - -The generic ownership-transfer examples above describe today's broad-grant profile. Once the -strict capability profile ships, its generated ownership manifest must exclude `complete_job` and -other bounded-owner definers from blanket transfer to `awa_owner`; `awa doctor` treats such a -transfer as strict-profile drift. - -Strict validation also audits the complete PostgreSQL role graph. It follows both transitive -inherited privileges and nested `SET ROLE` paths from every runtime, application, callback, and -admin login, and rejects any non-allowlisted path to the bounded execution owner, migrator, schema -owner, or a role that can reach them. Direct-membership checks alone are not sufficient. The -conformance matrix runs this audit on the oldest and newest supported PostgreSQL majors — a pair -that straddles PostgreSQL 16's separate membership `SET`/`INHERIT`/`ADMIN` semantics — so the newer -role model cannot be mistaken for the older one. - -[ADR-042](adr/042-caller-owned-finalization-transactions.md) lets a handler commit application rows -and guarded Awa completion in one transaction. That transaction normally comes from a separate -application pool whose login can write the application's billing/inbox tables. Do **not** make that -login a member of `awa_runtime`: the current runtime role can directly mutate and `TRUNCATE` Awa -tables, which is far broader than caller-owned completion needs. - -The completion migration installs `complete_job` as the canonical public finalization entry point. -Its exact argument signature, -`FinalizationReceipt` result, stale-token SQLSTATE, and schema-version compatibility semantics are -part of the ADR-036 SQL worker contract. Compatible implementation changes keep this name. Breaking -improvements require ADR-036 deprecation and ADR-041 expand/migrate/contract; a versioned -coexistence name may be used during migration without making version suffixes permanent policy. - -`complete_job` is a hardened `SECURITY DEFINER` function owned by the bounded execution owner -defined by ADR-043. A queue-storage schema owner may be used only as a transitional, non-strict -fallback; it does not satisfy the hardened owner model, and `awa doctor` rejects that ownership -when the deployment declares the strict profile. Its migration: - -- fixes `search_path` to trusted schemas with `pg_temp` last and fully qualifies every referenced - object; -- accepts no caller-controlled schema, table, function, or SQL identifier; -- revokes `EXECUTE` from `PUBLIC` in the same migration transaction that creates the function; -- transfers the function to the pre-provisioned bounded execution owner before commit in a strict - deployment, or explicitly records the schema-owner fallback as non-strict; -- performs only guarded completion of the supplied finalization token and never commits; and -- has the same hardened per-schema installation for a custom queue-storage schema. - -The operator grants the application-worker login (or a dedicated `NOLOGIN` group role used by -those logins) only `USAGE` on the Awa/queue-storage schema and `EXECUTE` on the exact versioned -completion function. It receives no direct Awa table, sequence, maintenance-function, or -`TRUNCATE` privilege. The Rust `complete_in_tx` helper calls this same SQL function through the -application transaction, so Rust and non-Rust workers share one privilege and conformance boundary. -Provisioning grants by the exact `complete_job(...)` signature only after ownership transfer; -replacement reasserts the `PUBLIC` revocation and preserves or reapplies only that manifest-listed -ACL. Removal follows ADR-036 rather than silently redirecting the grant to an internal helper. - -Function execution authorizes completion of any valid token presented by that trusted worker -application role. Do not grant it to producer-only roles, browser/admin clients, or public callback -ingress roles. `awa doctor` resolves the exact `regprocedure` signature and checks its owner, -`SECURITY DEFINER` flag, fixed `search_path`, `PUBLIC` revocation, and application-role ACL. Negative -tests prove the finalizer role cannot read or mutate Awa tables directly or redirect the function -through `search_path`, inherited membership, or a transitive `SET ROLE` chain. - -## Managing roles with pgroles - -For teams that manage PostgreSQL access declaratively, [pgroles](https://github.com/hardbyte/pgroles) can maintain the role model as a YAML manifest: - -```yaml -profiles: - runtime: - grants: - - on: { type: schema } - privileges: [USAGE] - - on: { type: table, name: "*" } - privileges: [SELECT, INSERT, UPDATE, DELETE, TRUNCATE] - - on: { type: sequence, name: "*" } - privileges: [USAGE, SELECT] - - on: { type: function, name: "*" } - privileges: [EXECUTE] - default_privileges: - - on_type: table - privileges: [SELECT, INSERT, UPDATE, DELETE, TRUNCATE] - - on_type: sequence - privileges: [USAGE, SELECT] - - on_type: function - privileges: [EXECUTE] - -schemas: - - name: awa - profiles: [runtime] - -roles: - - name: awa_owner - login: false - - name: awa_migrator - login: true - password: - from_env: AWA_MIGRATOR_PASSWORD - - name: awa_runtime - login: true - password: - from_env: AWA_RUNTIME_PASSWORD - -memberships: - - role: awa_owner - members: - - name: awa_migrator -``` - -`pgroles diff` shows planned changes, `pgroles apply` converges. You can also run `pgroles generate` against an existing AWA database to produce an initial manifest. - -## Migrating from a single-user setup - -If you're currently running everything as one superuser or app role: - -1. Create `awa_owner`, `awa_migrator`, `awa_runtime` as above -2. Transfer ownership to `awa_owner` -3. Grant runtime privileges -4. Update your migration tooling to use `awa_migrator` -5. Update worker/serve connection strings to use `awa_runtime` -6. Verify with `awa --database-url postgres://awa_runtime:... job list` - -This is additive — no schema changes, no downtime. - -## Future: tighter runtime grants - -The current model requires broad table grants because compatibility triggers and helper functions -still use `SECURITY INVOKER`. ADR-042's completion function is a deliberately narrow -`SECURITY DEFINER` exception for an application role that must join business writes without gaining -runtime privileges; it does not reduce what the ordinary runtime needs. - -[ADR-043](adr/043-postgresql-capability-functions.md) defines the long-term tightening path. Awa -will not convert the entire function surface to `SECURITY DEFINER`: generic internal, -caller-controlled dynamic-SQL, arbitrary-schema, and DDL helpers would become excessive -privilege-escalation entry points. Instead, an allowlisted set of producer, executor, maintenance, -admin, callback, and finalizer capability functions runs under a bounded `NOLOGIN` execution owner. -Maintenance partition reclamation is the sole relation-dispatch exception: it accepts only a -bounded ring slot, verifies each manifest-listed Awa child by catalog OID/namespace/owner/attachment, -uses a short lock timeout, revalidates every target's catalog identity again after the locks are -held, and may issue only `LOCK TABLE ONLY ... ACCESS EXCLUSIVE` plus -`TRUNCATE TABLE ONLY ... CONTINUE IDENTITY RESTRICT` after the reclaimability recheck. It never accepts a caller relation or SQL fragment. Internal helpers remain -inaccessible, `PUBLIC EXECUTE` is revoked, and broad table grants are removed only after an ADR-041 -capability rollout. This work is tracked in [#452](https://github.com/hardbyte/awa/issues/452). +The current 0.6 runtime and the 0.7 development runtime still need broad DML and `TRUNCATE` privileges on Awa's internal tables. [ADR-043](adr/043-postgresql-capability-functions.md) defines a proposed capability-function design for narrower roles; do not treat that future design as a shipped security control. ## Deployable roles -Awa is one process binary, but for production it splits into several _deployable roles_. Each role has a different exposure profile, and mixing them onto the same listener is the most common source of operational risk. See [ADR-027](adr/027-callback-ingress-surface.md) for the design rationale and [`docs/http-callbacks.md`](http-callbacks.md) for the per-role deployment shape. - -| Role | Purpose | Exposure | Mutates job state? | -| --- | --- | --- | --- | -| **Admin UI / API** (`awa serve`) | operator inspection and mutation — jobs, queues, runtime, DLQ, stats | private operator network | yes | -| **Callback receiver** (`awa callbacks serve` or user-owned router) | `complete` / `fail` / `heartbeat` for `HttpWorker` async mode and external systems | public or partner-facing, signed | yes | -| **Workers / dispatchers** (`awa::Client` with registered workers) | claim jobs, execute handlers, dispatch via `HttpWorker` | internal | yes | -| **Maintenance runtime** (background tasks on any worker process; future: dedicated role per ADR-028) | promotion, rescue, pruning, metadata refresh | internal | yes | -| **Database** | storage and coordination | private | yes | - -A single development setup can collapse these onto one process: `awa serve` runs admin + callback receiver, an embedded `awa::Client` runs workers, and Postgres is reachable on localhost. Production deployments **should** split at least the admin UI from the callback receiver — the admin surface stays on the operator network, the receiver lives wherever it needs to be reachable from the function (often public). - -### Common deployment shapes - -- **All-in-one dev:** `awa serve` + an embedded `awa::Client`. Admin UI and callback receiver share the same listener with permissive CORS. Fine for local development; never run this on a public listener. -- **Private admin + public callback receiver:** `awa serve` on a private VPC subnet, `awa callbacks serve --callback-hmac-secret …` on an external load balancer. The receiver router omits static UI assets, the admin REST routes, and permissive CORS. -- **User-owned callback API:** mount the three callback ingress routes inside your existing FastAPI / axum / Flask app, using `awa_model::callback_contract::verify` (Rust) or `awa.callback_contract.verify` (Python) so the signature contract cannot drift. See [`docs/callback-receivers.md`](callback-receivers.md). -- **Receiver + maintenance-only runtime:** the receiver handles ingress while a dedicated runtime instance runs promotion / rescue / pruning. Workers stay on their own deployment. Maintenance-only runtime is tracked in [ADR-028](adr/028-maintenance-only-runtime-role.md). -- **HTTP-worker deployments:** the worker process still runs an `awa::Client` — it claims jobs, calls the function, and registers the callback. The receiver is a separate listener. A function endpoint without a corresponding dispatcher process will _never_ see jobs. - -## Admin Surface - -`awa serve` is an **operator surface**: it bundles the admin REST API, the React dashboard, the static fallback, permissive CORS, and (today) the callback receiver routes behind a single router. Treat it like a database admin console: - -- Put it behind your normal authentication and authorization layer. -- Restrict network access with ingress policy, firewall rules, or private networking. -- Prefer binding to localhost or an internal service address unless you explicitly need external access. - -When callbacks must be externally reachable but the admin surface must stay private, run them on separate listeners using `awa callbacks serve` or a user-owned receiver — the admin endpoints simply do not exist on those routers. - -## Callback Endpoints - -The callback receiver exposes three **mutating ingress** endpoints, regardless of which deployable role hosts them: - -- `POST {prefix}/:callback_id/complete` -- `POST {prefix}/:callback_id/fail` -- `POST {prefix}/:callback_id/heartbeat` - -`{prefix}` defaults to `/api/callbacks`, matching the historical `awa serve` shape so existing deployments keep working unchanged. It is configurable on both sides: - -- Worker side: `HttpWorkerConfig::callback_path_prefix` -- `awa callbacks serve` side: `--path-prefix` / `AWA_CALLBACK_PATH_PREFIX` -- Custom-receiver side: mount your routes at whatever prefix you want and pass that prefix back to the worker config - -These endpoints mutate job state and must not be exposed without protection. The full HTTP worker flow, callback payloads, and signature contract are documented in [HTTP workers and callback signatures](http-callbacks.md). - -## Callback Signature Verification - -Awa supports per-callback request authentication with a 32-byte BLAKE3 keyed hash. - -- Configure the callback receiver with `--callback-hmac-secret <64-hex-chars>` or `AWA_CALLBACK_HMAC_SECRET`. -- Configure `HttpWorkerConfig.hmac_secret` with the same 32-byte key. -- The worker signs the callback ID and sends the signature as `X-Awa-Signature`. -- The function normally forwards that same header when it calls Awa back. -- The callback receiver verifies that header before accepting `complete`, `fail`, or `heartbeat`. - -Example: - -```bash -export AWA_CALLBACK_HMAC_SECRET=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef -awa --database-url "$DATABASE_URL" serve --host 0.0.0.0 --port 3000 -``` - -If no callback secret is configured, signature verification is disabled. That is acceptable only for trusted internal deployments where the callback receiver is already protected by network boundaries or an authenticating proxy. - -The option name says `hmac` for operational familiarity, but the implementation uses BLAKE3 keyed hashing over the callback ID string, not RFC HMAC. - -## Custom Callback Receivers - -When you host the callback ingress routes inside your own application (FastAPI, axum, Flask, etc.), reuse the shared helpers in `awa::callback_contract` (Rust) or `awa.callback_contract` (Python) rather than re-implementing the signature algorithm. The Python wrappers are thin PyO3 bindings around the same Rust functions, with a pinned BLAKE3 test vector asserted from both sides so the bindings cannot drift. - -See [callback receivers](callback-receivers.md) for end-to-end custom-axum and FastAPI examples. - -## Operational Guidance - -- Rotate callback secrets like any other shared secret. -- Use different secrets per environment. -- Prefer HTTPS/TLS termination in front of any externally reachable callback receiver. -- Avoid logging callback signatures or other shared-secret material. - -## Next - -- [Deployment guide](deployment.md) -- [Configuration](configuration.md) -- [Troubleshooting](troubleshooting.md) +The admin UI, callback ingress, workers, and maintenance tasks have different exposure profiles. See [Deployable surfaces](security/deployable-surfaces.md) for supported deployment shapes and network boundaries. diff --git a/docs/security/callback-security.md b/docs/security/callback-security.md new file mode 100644 index 00000000..36d99f67 --- /dev/null +++ b/docs/security/callback-security.md @@ -0,0 +1,34 @@ +# Callback security + +The callback receiver exposes state-changing endpoints for externally executed attempts: + +```text +POST {prefix}/:callback_id/complete +POST {prefix}/:callback_id/fail +POST {prefix}/:callback_id/heartbeat +``` + +The default prefix is `/api/callbacks`. These routes can complete, fail, or extend an attempt, so they require an authentication boundary. + +## Signed callbacks + +Awa supports a 32-byte BLAKE3 keyed hash over the callback ID. Despite the historical `hmac` option name, this is BLAKE3 keyed hashing, not RFC HMAC. + +- Configure the receiver with `--callback-hmac-secret` or `AWA_CALLBACK_HMAC_SECRET` using 64 hexadecimal characters. +- Configure `HttpWorkerConfig.hmac_secret` with the same 32-byte key. +- The dispatcher sends `X-Awa-Signature`; the external worker forwards it when calling the receiver. +- The receiver verifies the signature before accepting a callback mutation. + +If no secret is configured, signature verification is disabled. Use that only when a trusted network or authenticating proxy already protects the receiver. + +## Custom receivers + +Use `awa::callback_contract` in Rust or `awa.callback_contract` in Python rather than reimplementing signature verification. Both language surfaces call the same Rust implementation and share a pinned test vector. The [callback receiver guide](../callback-receivers.md) includes axum and FastAPI examples. + +## Operational checklist + +- Terminate TLS before any externally reachable receiver. +- Use a different secret in each environment and rotate it like any shared credential. +- Do not log callback signatures or secret material. +- Expose only the callback routes, not the admin router. +- Grant the receiver only the database/runtime authority its deployment model requires; never give it migrator credentials. diff --git a/docs/security/database-roles.md b/docs/security/database-roles.md new file mode 100644 index 00000000..35568ef2 --- /dev/null +++ b/docs/security/database-roles.md @@ -0,0 +1,81 @@ +# Database roles and privileges + +AWA can run with one database user, but production deployments should separate schema management from runtime execution. + +## Role model + +```text +awa_owner NOLOGIN owns the schema and its objects +└── awa_migrator LOGIN runs migrations; member of awa_owner + +awa_runtime LOGIN workers, producers, admin UI and CLI operations +``` + +Create the roles as a superuser or a role with the cluster-wide `CREATEROLE` attribute. Database ownership alone cannot create roles: + +```sql +CREATE ROLE awa_owner NOLOGIN; +CREATE ROLE awa_migrator LOGIN PASSWORD 'replace-me'; +CREATE ROLE awa_runtime LOGIN PASSWORD 'replace-me'; + +GRANT awa_owner TO awa_migrator; +GRANT CONNECT ON DATABASE mydb TO awa_migrator, awa_runtime; +GRANT CREATE ON DATABASE mydb TO awa_owner; +``` + +Run migrations through the migrator login while making `awa_owner` the effective role for every migration connection: + +```bash +PGOPTIONS='-c role=awa_owner' \ + awa --database-url "$AWA_MIGRATOR_DATABASE_URL" migrate +``` + +This makes the `awa` schema and the tables, sequences, functions, and standalone enum/domain types created by migrations belong to the non-login owner from the outset. It also avoids relying on membership or default privileges to repair ownership later. The `awa_migrator` login must be allowed to `SET ROLE awa_owner`; on PostgreSQL 16 and newer, preserve that option when granting the membership. + +## Runtime grants + +The 0.6 runtime and current 0.7 development runtime use `SECURITY INVOKER` triggers and maintenance helpers, so their grants are intentionally broader than an application's enqueue-only privileges: + +```sql +GRANT USAGE ON SCHEMA awa TO awa_runtime; +GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE + ON ALL TABLES IN SCHEMA awa TO awa_runtime; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA awa TO awa_runtime; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA awa TO awa_runtime; + +REVOKE EXECUTE ON FUNCTION + awa.install_queue_storage_substrate(TEXT, INT, INT, INT, BOOLEAN) + FROM awa_runtime; +``` + +The runtime needs `TRUNCATE` for guarded ring-partition reclamation. Compatibility COPY through `InsertOpts::copy()` also needs `TEMP` on the database; direct queue-storage COPY does not use that temporary staging table. + +Set matching default privileges for every role that creates objects during migrations: + +```sql +ALTER DEFAULT PRIVILEGES FOR ROLE awa_owner IN SCHEMA awa + GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE ON TABLES TO awa_runtime; +ALTER DEFAULT PRIVILEGES FOR ROLE awa_owner IN SCHEMA awa + GRANT USAGE, SELECT ON SEQUENCES TO awa_runtime; +ALTER DEFAULT PRIVILEGES FOR ROLE awa_owner IN SCHEMA awa + GRANT EXECUTE ON FUNCTIONS TO awa_runtime; +``` + +If an existing installation created objects as `awa_migrator` without first setting the owner role, transfer every existing schema object to `awa_owner` before relying on owner-scoped defaults. New migrations should use the effective-role command above. + +## Custom queue-storage schemas + +Repeat the schema, table, sequence, function, and default-privilege grants for a custom queue-storage schema. Prepare custom schemas as the migrator; the runtime needs DML, sequence access, function execution, and `TRUNCATE`, but never DDL. See [Queue storage](../queue-storage-substrate.md). + +## Why the grants are broad + +Runtime triggers maintain queue counts, descriptors, uniqueness claims, and other metadata as the invoking role. The elected maintenance task promotes and rescues jobs, refreshes metadata, and reclaims eligible ring slots. Consequently a login holding `awa_runtime` is trusted Awa infrastructure: do not grant it to a producer-only application or a public callback service. + +[ADR-042](../adr/042-caller-owned-finalization-transactions.md) and [ADR-043](../adr/043-postgresql-capability-functions.md) describe accepted/proposed boundaries for a narrower application finalizer and capability-specific runtime roles. They are not implemented by the current 0.6 or 0.7 development runtime. + +## Verify the split + +- Connect as the migrator to run `awa migrate`. +- Connect as the runtime to start workers and run ordinary `awa job` / `awa queue` commands. +- Confirm the runtime cannot create or alter schema objects. +- Keep the migrator credential out of worker and admin-service configuration. diff --git a/docs/security/deployable-surfaces.md b/docs/security/deployable-surfaces.md new file mode 100644 index 00000000..51704fb5 --- /dev/null +++ b/docs/security/deployable-surfaces.md @@ -0,0 +1,26 @@ +# Deployable surfaces + +Awa ships one binary, but its runtime surfaces have different trust boundaries. Production deployments should not expose them all on one listener. + +| Surface | Purpose | Recommended exposure | +| --- | --- | --- | +| Admin UI and API (`awa serve`) | Inspect and mutate jobs, queues, runtime and DLQ state | Authenticated operator network | +| Callback receiver | Complete, fail, or heartbeat externally executed jobs | Public or partner-facing only when authenticated | +| Workers and dispatchers | Claim jobs and execute handlers | Internal network | +| Maintenance | Promote, rescue, prune, and refresh metadata | Internal; elected from the worker fleet | +| PostgreSQL | Authoritative storage and coordination | Private network | + +## Admin UI + +`awa serve` is an operator surface. It includes the dashboard and mutating administration routes and currently can also include callback routes. Put it behind normal authentication and authorization, restrict it with ingress or firewall policy, and prefer a private address. + +Do not publish the all-in-one development router to the internet. When callbacks must be reachable externally, run `awa callbacks serve` on a separate listener or mount the callback contract in an existing application. The callback-only router omits the admin API, UI assets, and permissive admin CORS behavior. + +## Common deployment shapes + +- **Local development:** admin UI, callback routes, workers, and PostgreSQL can share one machine. +- **Private admin, public callbacks:** place `awa serve` inside the operator network and expose only `awa callbacks serve` through the external load balancer. +- **Application-owned callback API:** mount the verified callback routes in an existing FastAPI, axum, or Flask service. +- **HTTP worker:** an Awa client still needs to claim jobs and dispatch the function. A function endpoint alone does not consume queued work. + +See [Callback security](callback-security.md) and [HTTP callbacks](../http-callbacks.md) before exposing callback ingress. diff --git a/docs/stability.md b/docs/stability.md index 5819afca..1bf891d0 100644 --- a/docs/stability.md +++ b/docs/stability.md @@ -1,7 +1,7 @@ # Public Surface Stability Policy > **Status: Accepted** ([ADR-036](adr/036-public-surface-stability-policy.md), from -> [`0.7-roadmap.md`](0.7-roadmap.md) decision D6). This document is normative: release notes +> [0.7 roadmap](0.7-roadmap.md) decision D6). This document is normative: release notes > list breaking changes against its surface list, and changes to the promises below are made > by amending this document in an ordinary reviewed PR. diff --git a/docs/start/cli.md b/docs/start/cli.md new file mode 100644 index 00000000..4f0bde50 --- /dev/null +++ b/docs/start/cli.md @@ -0,0 +1,34 @@ +# Install the CLI + +The `awa` command runs migrations, inspects and administers queues, and can host the optional web dashboard. + +=== "uv tool (recommended)" + + ```bash + uv tool install awa-cli==0.6.6 + awa --help + ``` + +=== "Project dependency" + + ```bash + uv add 'awa-pg[ui]==0.6.6' + uv run python -m awa --help + ``` + + `python -m awa` delegates to the bundled `awa` binary inside the project environment. + +=== "Release binary" + + Download the archive for your platform from the [GitHub Releases](https://github.com/hardbyte/awa/releases) page, then put `awa` on your `PATH`. + +Point commands at PostgreSQL with `--database-url` or `DATABASE_URL`: + +```bash +export DATABASE_URL=postgres://awa_runtime:secret@db.example.com/app +awa health +awa queue stats +awa job list --queue email +``` + +Continue to the [CLI command map](../reference/cli.md) or launch the dashboard with `awa serve`. diff --git a/docs/start/index.md b/docs/start/index.md new file mode 100644 index 00000000..4e52bf42 --- /dev/null +++ b/docs/start/index.md @@ -0,0 +1,23 @@ +# Choose a client + +All AWA clients use the same PostgreSQL schema and job model. Choose by where the code runs, not by a separate server protocol. + +| You are building… | Start with | Why | +| --- | --- | --- | +| A Rust producer and worker | [`awa`](../getting-started-rust.md) | Typed arguments, worker runtime, admin API, and migrations in one facade crate | +| A Rust producer only | [`awa-model`](../reference/rust.md#producers-without-workers) | Enqueue and inspect jobs without the worker runtime | +| A Python producer and worker | [`awa-pg`](../getting-started-python.md) | Async and sync clients plus the compiled worker runtime | +| A Python web request with an open transaction | [`awa.bridge`](../bridge-adapters.md) | Enqueue on the application's asyncpg, psycopg, SQLAlchemy, or Django transaction | +| An operator or migration job | [`awa-cli`](cli.md) | Migrations, health checks, queue/job inspection, DLQ administration, and the web UI | + +## What every deployment needs + +1. A supported PostgreSQL database and credentials. +2. A migration owner that can create or upgrade the `awa` schema. +3. One or more producers that insert jobs. +4. One or more workers registered for the queues and kinds they process. + +The migration owner and runtime role can be separate. See [Security](../security.md) for the privilege model and [Deployment](../deployment.md) for rollout and shutdown guidance. + +!!! tip "Start locally, keep the production boundary explicit" + The quickstarts use one database role for clarity. Production deployments should use the least-privilege role split described in the security guide. diff --git a/docs/stylesheets/agent-docs.css b/docs/stylesheets/agent-docs.css new file mode 100644 index 00000000..364863df --- /dev/null +++ b/docs/stylesheets/agent-docs.css @@ -0,0 +1,24 @@ +.awa-agent-actions { + display: flex; + float: right; + gap: 0.4rem; + margin: 0 0 0.8rem 1rem; +} + +.awa-agent-actions .md-button { + cursor: pointer; + font-size: 0.64rem; + padding: 0.35rem 0.6rem; +} + +.awa-agent-actions button:disabled { + cursor: wait; + opacity: 0.72; +} + +@media screen and (max-width: 44.9844em) { + .awa-agent-actions { + float: none; + margin: 0 0 1rem; + } +} diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css new file mode 100644 index 00000000..378c79e8 --- /dev/null +++ b/docs/stylesheets/extra.css @@ -0,0 +1,184 @@ +:root, +[data-md-color-scheme="awa-light"] { + --md-primary-fg-color: #17633e; + --md-primary-fg-color--light: #287a51; + --md-primary-fg-color--dark: #0d4d2e; + --md-accent-fg-color: #14865a; + --md-default-bg-color: #fafcf9; + --md-default-fg-color: #17231c; + --md-code-bg-color: #f0f5f1; + --awa-surface: #f1f7f3; + --awa-border: #c9ddd0; + --awa-muted: #52675a; +} + +[data-md-color-scheme="slate"] { + color-scheme: dark; + --md-primary-fg-color: #287a51; + --md-primary-fg-color--light: #3f9468; + --md-primary-fg-color--dark: #17633e; + --md-accent-fg-color: #55c58a; + --md-default-bg-color: #101713; + --md-default-fg-color: #e6efe9; + --md-default-fg-color--light: #b9c9bf; + --md-code-bg-color: #17221b; + --md-typeset-color: var(--md-default-fg-color); + --md-typeset-a-color: #55c58a; + --awa-surface: #15231b; + --awa-border: #2b4937; + --awa-muted: #a8b9ae; +} + +html { + scroll-behavior: smooth; +} + +.md-header, +.md-tabs { + background: linear-gradient(115deg, #0d4d2e, #17633e 58%, #1e6c48); +} + +.md-banner { + background: #0d3120; + color: #e9f7ee; +} + +.md-banner a { + color: #8de2b1; + font-weight: 700; +} + +.awa-version-note { + display: block; + text-align: center; +} + +.md-main__inner { + margin-top: 1.4rem; +} + +.md-content h1, +.md-content h2, +.md-content h3 { + letter-spacing: -0.025em; +} + +.md-content h1 { + font-weight: 760; +} + +.awa-hero { + display: grid; + grid-template-columns: minmax(0, 1.05fr) minmax(18rem, 0.95fr); + gap: 2.5rem; + align-items: center; + margin: 0.8rem 0 3rem; + padding: 2.2rem; + border: 1px solid var(--awa-border); + border-radius: 1rem; + background: + radial-gradient(circle at 85% 10%, color-mix(in srgb, var(--md-accent-fg-color) 15%, transparent), transparent 45%), + var(--awa-surface); +} + +.awa-hero h1 { + margin: 0 0 0.7rem; + font-size: clamp(2.25rem, 5vw, 4.1rem); + line-height: 0.98; +} + +.awa-hero__lead { + max-width: 38rem; + color: var(--awa-muted); + font-size: 1.05rem; +} + +.awa-hero__actions { + display: flex; + flex-wrap: wrap; + gap: 0.65rem; + margin-top: 1.3rem; +} + +.awa-hero__visual img { + width: 100%; +} + +.md-typeset .awa-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 1rem; + margin: 1.5rem 0 2.5rem; +} + +.md-typeset .awa-card { + display: flex; + min-width: 0; + min-height: 8.25rem; + flex-direction: column; + justify-content: space-between; + gap: 0.8rem; + padding: 1.15rem 1.2rem; + border: 1px solid var(--awa-border); + border-radius: 0.75rem; + background: var(--awa-surface); + color: var(--md-default-fg-color) !important; + transition: border-color 120ms ease, transform 120ms ease; +} + +.md-typeset .awa-card:hover { + border-color: var(--md-accent-fg-color); + transform: translateY(-2px); +} + +.md-typeset .awa-card strong { + display: block; + margin-bottom: 0.35rem; + color: var(--md-accent-fg-color); + font-size: 1.05rem; +} + +.md-typeset .awa-card span { + color: var(--awa-muted); +} + +.awa-diagram { + margin: 1.8rem auto 2.5rem; + padding: 1rem; + border: 1px solid var(--awa-border); + border-radius: 0.8rem; + background: var(--awa-surface); +} + +.awa-diagram img { + display: block; + width: 100%; + height: auto; +} + +.md-typeset .md-button--primary { + background-color: var(--md-primary-fg-color); + border-color: var(--md-primary-fg-color); + color: white; +} + +.md-typeset table:not([class]) { + border-radius: 0.5rem; + overflow: hidden; +} + +@media screen and (max-width: 760px) { + .awa-hero { + grid-template-columns: 1fr; + gap: 1.2rem; + padding: 1.4rem; + } + + .md-typeset .awa-grid { + grid-template-columns: 1fr; + } + + .md-typeset .awa-card { + min-height: auto; + } +} diff --git a/docs/stylesheets/language-switch.css b/docs/stylesheets/language-switch.css new file mode 100644 index 00000000..e143b1dd --- /dev/null +++ b/docs/stylesheets/language-switch.css @@ -0,0 +1,54 @@ +.awa-language-switch { + align-items: center; + display: flex; + gap: 0.6rem; + justify-content: flex-end; + margin: 0 0 1rem; +} + +.awa-language-switch[hidden] { + display: none; +} + +.awa-language-switch__label { + color: var(--md-default-fg-color--light); + font-size: 0.7rem; + font-weight: 600; +} + +.awa-language-switch__options { + background: var(--md-code-bg-color); + border: 1px solid var(--md-default-fg-color--lightest); + border-radius: 999px; + display: inline-flex; + padding: 0.15rem; +} + +.awa-language-switch button { + background: transparent; + border: 0; + border-radius: 999px; + color: var(--md-default-fg-color--light); + cursor: pointer; + font: inherit; + font-size: 0.7rem; + font-weight: 650; + line-height: 1.5; + padding: 0.2rem 0.65rem; +} + +.awa-language-switch button[aria-pressed="true"] { + background: var(--md-primary-fg-color); + color: var(--md-primary-bg-color); +} + +.awa-language-switch button:focus-visible { + outline: 2px solid var(--md-accent-fg-color); + outline-offset: 2px; +} + +@media (max-width: 44.984375em) { + .awa-language-switch { + justify-content: flex-start; + } +} diff --git a/docs/stylesheets/reference.css b/docs/stylesheets/reference.css new file mode 100644 index 00000000..04d6d2ba --- /dev/null +++ b/docs/stylesheets/reference.css @@ -0,0 +1,30 @@ +.awa-status { + display: inline-block; + white-space: nowrap; + padding: 0.12rem 0.45rem; + border: 1px solid currentcolor; + border-radius: 999px; + font-size: 0.64rem; + font-weight: 700; + line-height: 1.35; +} + +.awa-status--proposed { + color: #8a6116; + background: color-mix(in srgb, #e3b75b 18%, transparent); +} + +.awa-status--superseded, +.awa-status--rejected { + color: #a3473e; + background: color-mix(in srgb, #d87569 15%, transparent); +} + +[data-md-color-scheme="slate"] .awa-status--proposed { + color: #f0c96f; +} + +[data-md-color-scheme="slate"] .awa-status--superseded, +[data-md-color-scheme="slate"] .awa-status--rejected { + color: #f09b91; +} diff --git a/docs/test-plan.md b/docs/test-plan.md index 08a5276d..60dae45b 100644 --- a/docs/test-plan.md +++ b/docs/test-plan.md @@ -249,7 +249,7 @@ The formal suite includes passing configs and expected-counterexample configs. T ## 0.7 Planned Validation -Planned test matrix for the 0.7 cycle, mapped to the roadmap ([`0.7-roadmap.md`](0.7-roadmap.md)) and the release gates on the [#383 tracker](https://github.com/hardbyte/awa/issues/383). Rows move into the matrix above as they are implemented. +Planned test matrix for the 0.7 cycle, mapped to the [0.7 roadmap](0.7-roadmap.md) and the release gates on the [#383 tracker](https://github.com/hardbyte/awa/issues/383). Rows move into the matrix above as they are implemented. V25 adds a focused `AwaKeyedExecution` model for grant safety, partial-legacy coverage scope, and the drain/epoch/verify/resume/activate-or-cancel lowering protocol; it also extends `AwaStorageLockOrder` for the short policy transition windows. V26 adds `AwaCallerOwnedCompletion` for completion-versus-rescue atomicity. These names are planned deliverables, not claims that model files already exist. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 223e07ac..b04913a0 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -216,7 +216,7 @@ Use an upsert: first-enqueue may create lane rows before any operator inserts a **3. WAL or commit pressure on the database.** -If batch size and shard count both look healthy, sample `pg_stat_activity` for `LWLock:WALWrite` / `LWLock:WALSync` waits and confirm the database is sized for the offered rate. The per-vCPU sustained-completion and burst-enqueue numbers in [`docs/deploying-on-managed-postgres.md`](deploying-on-managed-postgres.md#pick-a-vcpu-size) are useful reference points. +If batch size and shard count both look healthy, sample `pg_stat_activity` for `LWLock:WALWrite` / `LWLock:WALSync` waits and confirm the database is sized for the offered rate. The per-vCPU sustained-completion and burst-enqueue numbers in [Managed Postgres sizing](deploying-on-managed-postgres.md#pick-a-vcpu-size) are useful reference points. ## Leader Election Delays @@ -379,7 +379,7 @@ Pay particular attention to: - reduce terminal-row retention if the terminal history is much larger than needed; note that `failed_retention` is a floor — non-DLQ `failed` rows stay retryable for at least that long in both engines (queue storage carries in-floor failed rows forward at prune time), and the cumulative count of failed rows aged past the floor is visible as `QueueCounts.pruned_failed` - review autovacuum settings if lease churn is expected continuously -If you want to reproduce the behavior locally before changing settings, run the MVCC benchmark documented in `docs/benchmarking.md`. Preventative guidance — reader placement, session timeouts, alerting, and autovacuum capacity flags — lives in [`deploying-on-managed-postgres.md`](deploying-on-managed-postgres.md#mvcc-discipline-long-running-readers-pin-the-whole-database). +If you want to reproduce the behavior locally before changing settings, run the MVCC benchmark documented in [Benchmarking](benchmarking.md). Preventative guidance — reader placement, session timeouts, alerting, and autovacuum capacity flags — lives in [Managed Postgres: MVCC discipline](deploying-on-managed-postgres.md#mvcc-discipline-long-running-readers-pin-the-whole-database). ## Something's In The DLQ diff --git a/docs/upgrade-0.5-to-0.6.md b/docs/upgrade-0.5-to-0.6.md index 5e5cd8ce..3a5948fb 100644 --- a/docs/upgrade-0.5-to-0.6.md +++ b/docs/upgrade-0.5-to-0.6.md @@ -1,8 +1,8 @@ # Upgrade Checklist: 0.5.x → 0.6 -> **Planning to run 0.7?** Finalize before upgrading: the 0.7 `awa migrate` refuses unfinalized clusters ([ADR-037](adr/037-canonical-engine-deprecation.md), [upgrade-0.6-to-0.7.md](upgrade-0.6-to-0.7.md)). +> **Planning to run 0.7?** Finalize before upgrading: the 0.7 `awa migrate` refuses unfinalized clusters ([ADR-037](adr/037-canonical-engine-deprecation.md), [Upgrade 0.6 to 0.7](upgrade-0.6-to-0.7.md)). -This is the operator-facing source of truth for moving an existing 0.5.x cluster to 0.6 (queue-storage-by-default). It defines the pre-flight, rollout phases, rollback boundary, and health checks; [migrations.md](migrations.md) covers the general migration contract and external tooling. +This is the operator-facing source of truth for moving an existing 0.5.x cluster to 0.6 (queue-storage-by-default). It defines the pre-flight, rollout phases, rollback boundary, and health checks; [Migrations](migrations.md) covers the general migration contract and external tooling. > **Fresh installs do not need this file.** A new cluster runs `awa migrate` and starts workers; the first worker auto-finalizes via `awa.storage_auto_finalize_if_fresh()`. See migrations.md ["Fresh install"](migrations.md#fresh-install-no-prior-canonical-data). This checklist is for **upgrading existing 0.5.x clusters**, where canonical drain is unavoidable and auto-finalize correctly defers to the staged path. @@ -176,8 +176,8 @@ If any of these go wrong **before** any queue-storage work is accepted, `awa sto ## Cross-references -- [migrations.md](migrations.md) — general migration contract and external tooling -- [configuration.md](configuration.md) — claim-ring / lease-ring sizing knobs -- [`docs/adr/023-receipt-plane-ring-partitioning.md`](adr/023-receipt-plane-ring-partitioning.md) — receipt-plane partition design and reverse-migration recipe -- [`docs/adr/025-sharded-enqueue-heads.md`](adr/025-sharded-enqueue-heads.md) — enqueue-head sharding design and partitioned-FIFO contract +- [Migrations](migrations.md) — general migration contract and external tooling +- [Configuration](configuration.md) — claim-ring / lease-ring sizing knobs +- [ADR-023: Receipt-plane ring partitioning](adr/023-receipt-plane-ring-partitioning.md) — receipt-plane partition design and reverse-migration recipe +- [ADR-025: Sharded enqueue heads](adr/025-sharded-enqueue-heads.md) — enqueue-head sharding design and partitioned-FIFO contract - [`docs/grafana/awa-dashboard.json`](grafana/awa-dashboard.json) — Prometheus dashboard with the rotation/prune panels diff --git a/docs/upgrade-0.6-to-0.7.md b/docs/upgrade-0.6-to-0.7.md index f754d432..c2629f88 100644 --- a/docs/upgrade-0.6-to-0.7.md +++ b/docs/upgrade-0.6-to-0.7.md @@ -29,7 +29,7 @@ You are done with the storage step. Deploy 0.7 binaries and run `awa migrate` as ## If you are on 0.6, not yet finalized Complete the staged transition **on your 0.6 binaries** first -(full procedure: [upgrade-0.5-to-0.6.md](upgrade-0.5-to-0.6.md)): +(full procedure: [Upgrade 0.5 to 0.6](upgrade-0.5-to-0.6.md)): ```bash awa storage prepare --engine queue_storage @@ -39,7 +39,7 @@ awa storage finalize --wait Then upgrade binaries to 0.7 and run `awa migrate`. -If `finalize --wait` sits at a non-zero backlog that never falls while jobs are visibly executing, your workload probably contains **perpetually snoozing jobs** — handlers that end every run in `JobResult::Snooze`. On builds before [#456](https://github.com/hardbyte/awa/issues/456) those re-entered canonical `scheduled_jobs` after each post-flip run, replenishing the backlog forever. Roll to a build carrying that fix (or apply the manual backlog migration in [upgrade-0.5-to-0.6.md](upgrade-0.5-to-0.6.md#known-issues)) before waiting on finalize. +If `finalize --wait` sits at a non-zero backlog that never falls while jobs are visibly executing, your workload probably contains **perpetually snoozing jobs** — handlers that end every run in `JobResult::Snooze`. On builds before [#456](https://github.com/hardbyte/awa/issues/456) those re-entered canonical `scheduled_jobs` after each post-flip run, replenishing the backlog forever. Roll to a build carrying that fix (or apply the [manual backlog migration](upgrade-0.5-to-0.6.md#known-issues)) before waiting on finalize. Remember the transition is a one-way door once queue-storage work is accepted; the 0.5→0.6 guide covers the abort boundaries. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..a190e89d --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,162 @@ +site_name: AWA +site_description: Postgres-native background jobs for Rust and Python +site_url: https://hardbyte.github.io/awa/ +repo_url: https://github.com/hardbyte/awa +repo_name: hardbyte/awa +edit_uri: edit/main/docs/ + +docs_dir: docs +site_dir: site + +theme: + name: material + custom_dir: docs/overrides + language: en + logo: assets/logo.svg + favicon: assets/logo.svg + icon: + repo: fontawesome/brands/github + font: + text: Inter + code: Geist Mono + features: + - content.code.annotate + - content.code.copy + - content.tabs.link + - content.tooltips + - navigation.footer + - navigation.indexes + - navigation.instant + - navigation.instant.progress + - navigation.sections + - navigation.tabs + - navigation.top + - search.highlight + - search.suggest + - toc.follow + palette: + - media: "(prefers-color-scheme: light)" + scheme: awa-light + primary: custom + accent: custom + toggle: + icon: material/weather-night + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: custom + accent: custom + toggle: + icon: material/weather-sunny + name: Switch to light mode + +extra_css: + - stylesheets/extra.css + - stylesheets/reference.css + - stylesheets/language-switch.css + - stylesheets/agent-docs.css + +extra_javascript: + - javascripts/language-switch.js + - javascripts/agent-docs.js + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/hardbyte/awa + generator: false + +copyright: Copyright © AWA contributors · MIT OR Apache-2.0 + +markdown_extensions: + - abbr + - admonition + - attr_list + - def_list + - footnotes + - md_in_html + - tables + - toc: + permalink: true + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets: + base_path: + - . + check_paths: true + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.tasklist: + custom_checkbox: true + +nav: + - Home: index.md + - Start: + - Choose a client: start/index.md + - Rust: getting-started-rust.md + - Python: getting-started-python.md + - CLI: start/cli.md + - Concepts: + - How AWA works: concepts/index.md + - Job lifecycle: concepts/job-lifecycle.md + - Transactional enqueue: concepts/transactional-enqueue.md + - Architecture: architecture.md + - Queue storage: queue-storage-substrate.md + - Guides: + - Guide index: guides/index.md + - Bridge adapters: bridge-adapters.md + - Lifecycle hooks: lifecycle-hooks.md + - HTTP callbacks: http-callbacks.md + - Callback receivers: callback-receivers.md + - Dead-letter queue: dead-letter-queue.md + - Operations: + - Operations index: operations/index.md + - Configuration: configuration.md + - Deployment: deployment.md + - Managed Postgres: deploying-on-managed-postgres.md + - Migrations: + - Migration overview: migrations.md + - Upgrade 0.5 to 0.6: upgrade-0.5-to-0.6.md + - Upgrade 0.6 to 0.7: upgrade-0.6-to-0.7.md + - Observability: grafana/README.md + - Troubleshooting: troubleshooting.md + - Security: + - Security overview: security.md + - Database roles: security/database-roles.md + - Deployable surfaces: security/deployable-surfaces.md + - Callback security: security/callback-security.md + - Reference: + - Reference index: reference/index.md + - CLI commands: reference/cli.md + - Rust crates: reference/rust.md + - Python API: reference/python.md + - Stability policy: stability.md + - Architecture decisions: adr/README.md + - Contributing: + - Contributing index: contributing/index.md + - Development: development.md + - Benchmarking: benchmarking.md + +exclude_docs: | + /0.7-planning-brief.md + /positioning.md + /test-plan.md + /ui-design.md + /archive/** + +not_in_nav: | + /0.7-roadmap.md + /adr/[0-9]*.md + /adr/bench/** + /grafana/alerts/** + +validation: + omitted_files: warn + absolute_links: relative_to_docs + unrecognized_links: warn + anchors: warn diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 00000000..291c8729 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,3 @@ +mkdocs==1.6.1 +mkdocs-material==9.7.7 +pymdown-extensions==11.0.1 diff --git a/scripts/build-agent-docs.py b/scripts/build-agent-docs.py new file mode 100644 index 00000000..e9c6be64 --- /dev/null +++ b/scripts/build-agent-docs.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""Build and validate agent-readable companions for the MkDocs site.""" + +from __future__ import annotations + +import argparse +import os +import re +from pathlib import Path, PurePosixPath +from urllib.parse import unquote, urlsplit, urlunsplit + +import yaml + + +MARKDOWN_LINK = re.compile(r"(!?\[[^\]]*\])\(([^)]+)\)") +SNIPPET = re.compile(r'^([ \t]*)--8<--\s+"([^"]+)"[ \t]*$', re.MULTILINE) + + +def page_output(source: PurePosixPath) -> PurePosixPath: + if source.name.lower() in {"index.md", "readme.md"}: + return source.parent / "index.md" + return source.with_suffix("") / "index.md" + + +def html_output(source: PurePosixPath) -> PurePosixPath: + return page_output(source).with_suffix(".html") + + +def strip_front_matter(text: str) -> str: + if not text.startswith("---\n"): + return text + marker = text.find("\n---\n", 4) + return text[marker + 5 :] if marker >= 0 else text + + +def rewrite_link( + destination: str, + source: PurePosixPath, + published_sources: set[PurePosixPath], +) -> str: + wrapped = destination.startswith("<") and destination.endswith(">") + value = destination[1:-1] if wrapped else destination + parts = urlsplit(value) + if parts.scheme or parts.netloc or value.startswith(("#", "/", "mailto:")): + return destination + + resolved = PurePosixPath(os.path.normpath(str(source.parent / parts.path))) + if parts.path.endswith(".md") and resolved not in published_sources: + return urlunsplit(("https", "github.com", f"/hardbyte/awa/blob/main/docs/{resolved.as_posix()}", parts.query, parts.fragment)) + target = page_output(resolved) if parts.path.endswith(".md") else resolved + current_dir = page_output(source).parent + relative = os.path.relpath(target, current_dir).replace(os.sep, "/") + rewritten = urlunsplit(("", "", relative, parts.query, parts.fragment)) + return f"<{rewritten}>" if wrapped else rewritten + + +def expand_snippets(text: str, repository_root: Path) -> str: + root = repository_root.resolve() + + def replace(match: re.Match[str]) -> str: + include = (root / match.group(2)).resolve() + try: + include.relative_to(root) + except ValueError: + raise SystemExit(f"snippet escapes repository: {match.group(2)}") + if not include.is_file(): + raise SystemExit(f"snippet does not exist: {match.group(2)}") + indent = match.group(1) + return "\n".join(indent + line if line else "" for line in include.read_text().splitlines()) + + return SNIPPET.sub(replace, text) + + +def markdown_variant( + text: str, + source: PurePosixPath, + repository_root: Path, + published_sources: set[PurePosixPath], +) -> str: + body = strip_front_matter(text).lstrip() + body = expand_snippets(body, repository_root) + body = MARKDOWN_LINK.sub( + lambda match: f"{match.group(1)}({rewrite_link(match.group(2), source, published_sources)})", + body, + ) + index_path = os.path.relpath("llms.txt", page_output(source).parent).replace(os.sep, "/") + directive = ( + f"> For the complete AWA documentation index, see " + f"[`llms.txt`]({index_path}).\n\n" + ) + return directive + body.rstrip() + "\n" + + +def first_description(text: str) -> str: + body = strip_front_matter(text) + in_fence = False + for block in re.split(r"\n\s*\n", body): + candidate = block.strip() + if candidate.startswith("```"): + in_fence = not in_fence + continue + if ( + not candidate + or in_fence + or candidate.startswith(("#", ">", "- ", "* ", "<", "!!!", "???")) + ): + continue + candidate = re.sub(r"\[([^]]+)\]\([^)]+\)", r"\1", candidate) + candidate = re.sub(r"[`*_]", "", candidate) + candidate = " ".join(candidate.split()) + if len(candidate) > 240: + candidate = candidate[:240].rsplit(" ", 1)[0] + return candidate.rstrip(" .,:;—-") + "." + return "AWA documentation." + + +def nav_links(value: object) -> list[tuple[str, str]]: + links: list[tuple[str, str]] = [] + if not isinstance(value, list): + return links + for child in value: + if not isinstance(child, dict): + continue + label, target = next(iter(child.items())) + if isinstance(target, str): + links.append((str(label), target)) + else: + links.extend(nav_links(target)) + return links + + +def nav_sections(nav: list[object]) -> list[tuple[str, list[tuple[str, str]]]]: + sections: list[tuple[str, list[tuple[str, str]]]] = [] + for item in nav: + if not isinstance(item, dict): + continue + heading, value = next(iter(item.items())) + links: list[tuple[str, str]] = [] + if isinstance(value, str): + links.append((str(heading), value)) + elif isinstance(value, list): + links.extend(nav_links(value)) + sections.append((str(heading), links)) + return sections + + +def build(docs_dir: Path, site_dir: Path, config_path: Path) -> None: + config = yaml.safe_load(config_path.read_text()) + + repository_root = config_path.resolve().parent + source_paths = sorted(docs_dir.rglob("*.md")) + published_sources = { + PurePosixPath(path.relative_to(docs_dir).as_posix()) + for path in source_paths + if (site_dir / html_output(PurePosixPath(path.relative_to(docs_dir).as_posix()))).exists() + } + generated: dict[PurePosixPath, Path] = {} + for path in source_paths: + source = PurePosixPath(path.relative_to(docs_dir).as_posix()) + if source not in published_sources: + continue + destination = site_dir / page_output(source) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + markdown_variant(path.read_text(), source, repository_root, published_sources) + ) + generated[source] = destination + + lines = [ + "# AWA", + "", + "> Postgres-native background jobs for Rust and Python, with durable job state and transactional enqueue.", + "", + "These links are curated for agents. Each target is the Markdown representation of the corresponding documentation page.", + ] + indexed: set[PurePosixPath] = set() + for section, links in nav_sections(config["nav"]): + available = [] + for label, path_value in links: + source = PurePosixPath(path_value) + if source in generated: + available.append((label, source)) + indexed.add(source) + if not available: + continue + lines.extend(("", f"## {section}", "")) + for label, source in available: + url = page_output(source).as_posix() + description = first_description((docs_dir / source).read_text()) + lines.append(f"- [{label}]({url}): {description}") + + lines.extend( + ( + "", + "## Optional", + "", + "- [GitHub repository](https://github.com/hardbyte/awa): Source code, issues, releases, and repository-only contributor material.", + ) + ) + llms = "\n".join(lines) + "\n" + if len(llms) >= 50_000: + raise SystemExit(f"llms.txt is too large: {len(llms)} characters") + (site_dir / "llms.txt").write_text(llms) + + expected_nav = { + PurePosixPath(path) + for _, links in nav_sections(config["nav"]) + for _, path in links + } + all_sources = { + PurePosixPath(path.relative_to(docs_dir).as_posix()) for path in source_paths + } + validate( + site_dir, + generated, + indexed, + expected_nav, + all_sources - published_sources, + ) + + +def validate( + site_dir: Path, + generated: dict[PurePosixPath, Path], + indexed: set[PurePosixPath], + expected_nav: set[PurePosixPath], + excluded_sources: set[PurePosixPath], +) -> None: + llms_path = site_dir / "llms.txt" + llms = llms_path.read_text() + errors: list[str] = [] + if not llms.startswith("# AWA\n\n> "): + errors.append("llms.txt must start with an H1 and blockquote summary") + if len(llms) >= 50_000: + errors.append("llms.txt must remain below 50,000 characters") + if not indexed: + errors.append("llms.txt contains no documentation pages") + missing_nav = expected_nav - indexed + if missing_nav: + errors.append( + "llms.txt omits nav pages: " + ", ".join(sorted(map(str, missing_nav))) + ) + + for _, destination in MARKDOWN_LINK.findall(llms): + parts = urlsplit(destination) + if parts.scheme or parts.netloc: + continue + target = site_dir / unquote(parts.path) + if not target.is_file(): + errors.append(f"llms.txt: target does not exist: {destination}") + + for source, output in generated.items(): + text = output.read_text() + if "[`llms.txt`]" not in text: + errors.append(f"{output}: missing llms.txt discovery directive") + if len(text) >= 100_000: + errors.append(f"{output}: Markdown representation exceeds 100,000 characters") + if output.stat().st_size == 0: + errors.append(f"{output}: empty Markdown representation") + expected_html = site_dir / html_output(source) + if not expected_html.exists(): + errors.append(f"{output}: corresponding HTML page is missing") + else: + html = expected_html.read_text() + if 'rel="alternate"' not in html or 'type="text/markdown"' not in html: + errors.append(f"{expected_html}: missing Markdown alternate link") + if "llms.txt" not in html: + errors.append(f"{expected_html}: missing llms.txt discovery link") + for _, destination in MARKDOWN_LINK.findall(text): + parts = urlsplit(destination.strip("<>")) + if parts.scheme or parts.netloc or destination.startswith(("#", "/", "mailto:")): + continue + target = (output.parent / unquote(parts.path)).resolve() + if parts.path and not target.exists(): + errors.append(f"{output}: local target does not exist: {destination}") + + for source in excluded_sources: + output = site_dir / page_output(source) + if output.exists(): + errors.append(f"excluded source unexpectedly emitted: {source}") + + if errors: + raise SystemExit("agent documentation checks failed:\n- " + "\n- ".join(errors)) + print( + f"agent documentation checks passed: {len(generated)} Markdown pages, " + f"{len(indexed)} pages indexed, {len(llms)}-character llms.txt" + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--docs-dir", type=Path, default=Path("docs")) + parser.add_argument("--site-dir", type=Path, default=Path("site")) + parser.add_argument("--config", type=Path, default=Path("mkdocs.yml")) + args = parser.parse_args() + build(args.docs_dir, args.site_dir, args.config) + + +if __name__ == "__main__": + main() diff --git a/scripts/check-docs.sh b/scripts/check-docs.sh new file mode 100755 index 00000000..aaf02d50 --- /dev/null +++ b/scripts/check-docs.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +mkdocs build --strict +python3 scripts/build-agent-docs.py +env -u RUSTC_WRAPPER cargo check -p awa --example quickstart +python3 -m py_compile awa-python/examples/quickstart.py + +# The repository CI workflow owns workspace lint/build/database tests and runs +# both canonical quickstarts. This script keeps the docs-only check fast while +# verifying that included source still parses and compiles. + +if rg --glob '*.md' '\]\((?:\.\./)+(?:awa|awa-python|correctness|docker|examples|CHANGELOG)' docs; then + echo "docs contain repository-relative links that will break on the published site" >&2 + exit 1 +fi