From 21ba28095c369d444b7011444370fcd385dd2eb2 Mon Sep 17 00:00:00 2001 From: "fineas-bot[bot]" <258147136+fineas-bot[bot]@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:04:15 -0400 Subject: [PATCH 1/9] feat: add schema-forward rollback bridge (1/3) Reviewed emergency bridge for preserving the frozen forward migration ledger and workflow state during a staged rollback. This commit does not deploy or publish the bridge. --- .../rollback-bridge-reviewed-forward-commit | 1 + .../check-rollback-bridge-workflows.sh | 108 ++++ .github/workflows/ci.yml | 165 ++++- .github/workflows/publish-images.yml | 574 +++++++----------- .github/workflows/validate-images.yml | 88 +++ .../operate/upstream-rollback-bridge.mdx | 169 ++++++ docs/pages/reference/configuration.mdx | 2 + .../md/operate/upstream-rollback-bridge.md | 169 ++++++ docs/public/md/reference/configuration.md | 2 + packages/api-client/src/client.ts | 6 +- packages/api-client/test/client.test.ts | 2 + .../centaur-api-integration-test/src/main.rs | 89 ++- .../crates/centaur-api-server/src/error.rs | 4 + .../crates/centaur-api-server/src/lib.rs | 98 +-- .../crates/centaur-api-server/src/main.rs | 111 +++- .../crates/centaur-api-server/src/routes.rs | 310 ++++++++-- .../crates/centaur-api-server/src/types.rs | 7 +- .../forward_migrations/0033_session_title.sql | 2 + .../0034_session_sandbox_activity.sql | 21 + .../0035_session_execution_stdout_owner.sql | 7 + 20 files changed, 1496 insertions(+), 439 deletions(-) create mode 100644 .github/rollback-bridge-reviewed-forward-commit create mode 100644 .github/scripts/check-rollback-bridge-workflows.sh create mode 100644 .github/workflows/validate-images.yml create mode 100644 docs/pages/operate/upstream-rollback-bridge.mdx create mode 100644 docs/public/md/operate/upstream-rollback-bridge.md create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0033_session_title.sql create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0034_session_sandbox_activity.sql create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0035_session_execution_stdout_owner.sql diff --git a/.github/rollback-bridge-reviewed-forward-commit b/.github/rollback-bridge-reviewed-forward-commit new file mode 100644 index 000000000..dd177d51c --- /dev/null +++ b/.github/rollback-bridge-reviewed-forward-commit @@ -0,0 +1 @@ +6847616b6fcbad3c0ba51fef25f8b97845fd5aec diff --git a/.github/scripts/check-rollback-bridge-workflows.sh b/.github/scripts/check-rollback-bridge-workflows.sh new file mode 100644 index 000000000..6413ed135 --- /dev/null +++ b/.github/scripts/check-rollback-bridge-workflows.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +publisher=.github/workflows/publish-images.yml +validator=.github/workflows/validate-images.yml +ci=.github/workflows/ci.yml +pin=.github/rollback-bridge-reviewed-forward-commit + +fail() { + echo "rollback bridge workflow safety check failed: $*" >&2 + exit 1 +} + +[[ "$(head -n 1 "$validator")" == "name: Publish Images" ]] || + fail "PR validator must preserve the historical Publish Images check context" +grep -q '^ pull_request:$' "$validator" || fail "PR validator must run on pull requests" +if grep -Eq '^ (branches|paths):' "$validator"; then + fail "PR validator must be unfiltered so the historical required check is always reported" +fi +if grep -Eq '^ (push|workflow_dispatch):' "$validator"; then + fail "PR validator must not have a publication trigger" +fi +grep -q '^ contents: read$' "$validator" || fail "PR validator must be read-only" +grep -q '^ push: false$' "$validator" || fail "PR validator must build with push disabled" +if grep -q 'docker/login-action' "$validator" || grep -q 'packages: write' "$validator"; then + fail "PR validator must not receive registry credentials" +fi +grep -Fq 'service: [api-rs, slackbotv2, linearbot, discordbot, teamsbot, agent, iron-proxy, console]' "$validator" || + fail "PR validator must preserve the historical eight-image required-check matrix" + +grep -q '^ workflow_dispatch:$' "$publisher" || fail "publisher must be manual-only" +if grep -Eq '^ (push|pull_request):' "$publisher"; then + fail "publisher must not run on push or pull request events" +fi +grep -q '^ live_updater_scope_verified:$' "$publisher" || + fail "publisher must require an explicit live updater-scope acknowledgement" +grep -Fq 'LIVE_UPDATER_SCOPE_VERIFIED: ${{ inputs.live_updater_scope_verified }}' "$publisher" || + fail "publisher must bind the live updater-scope acknowledgement into its release gate" +grep -Fq 'none of the four exact bridge repositories is in image-list' "$publisher" || + fail "publisher must state the fail-closed live updater-scope proof" +if grep -qE 'image_updater_disabled|IMAGE_UPDATER_DISABLED' "$publisher"; then + fail "publisher must not claim that global Image Updater disablement is a publication precondition" +fi +if grep -Eq 'kubectl|KUBECONFIG|kubeconfig' "$publisher"; then + fail "publisher must not receive Kubernetes access" +fi +grep -Fq 'service: [api-rs, slackbotv2, agent, iron-proxy]' "$publisher" || + fail "publisher build matrix must contain exactly the four rollback runtime images" +grep -q '^ cancel-in-progress: false$' "$publisher" || + fail "publisher dispatches must serialize rather than cancel an in-flight publication" +grep -q '^ group: publish-reviewed-rollback-bridge-images$' "$publisher" || + fail "publisher concurrency must serialize globally across refs" +grep -q '^ needs: tag-absence-gate$' "$publisher" || + fail "digest publication must wait for the reviewed-tag absence gate" +grep -q '^ tag-absence-gate:$' "$publisher" || + fail "publisher is missing the reviewed-tag absence gate" +grep -Fq 'case "$status" in' "$publisher" || + fail "reviewed-tag absence gate must distinguish explicit registry HTTP status" +if [[ "$(grep -Fc '.code == "MANIFEST_UNKNOWN"' "$publisher")" -ne 2 ]]; then + fail "publisher must require explicit MANIFEST_UNKNOWN at both absence checks" +fi +if [[ "$(grep -Fc 'application/vnd.oci.image.manifest.v1+json' "$publisher")" -ne 2 || + "$(grep -Fc 'application/vnd.docker.distribution.manifest.v2+json' "$publisher")" -ne 2 ]]; then + fail "both absence checks must negotiate single-platform and multi-platform manifests" +fi +grep -Fq 'refusing to overwrite immutable reviewed tag during final recheck' "$publisher" || + fail "each manifest publication must recheck tag absence immediately before creation" +if grep -Eq 'centaur-(linearbot|discordbot|teamsbot|console)' "$publisher"; then + fail "publisher must not build or describe non-bridge images" +fi +grep -Fq 'type=raw,value=reviewed-${{ github.sha }}' "$publisher" || + fail "publisher must use the reviewed-full-commit tag namespace" +grep -Fq 'tag="reviewed-${RELEASE_REVISION}"' "$publisher" || + fail "release descriptor must use the reviewed-full-commit tag namespace" +grep -Fq 'pattern: digests-*-linux-arm64' "$publisher" || + fail "release descriptor must download this run's arm64 digest artifacts" +grep -Fq 'if [[ "$digest" != "$built_digest" ]]' "$publisher" || + fail "release descriptor must bind each tagged arm64 digest to this run's build artifact" +grep -Fq 'refusing to overwrite immutable reviewed tag' "$publisher" || + fail "publisher must refuse to overwrite an existing reviewed tag" +if grep -Eq 'type=sha|value=(latest|main|edge)|sha-\$\{|::7' "$publisher"; then + fail "publisher contains a legacy, mutable, or shortened deploy-shaped tag" +fi + +expected_descriptor_rows=$'api-rs\tcentaur-api-rs\nslackbotv2\tcentaur-slackbotv2\nsandbox\tcentaur-agent\niron-proxy\tcentaur-iron-proxy' +actual_descriptor_rows="$(awk ' + /done <<'"'"'COMPONENTS'"'"'/ { capture = 1; next } + capture && /^[[:space:]]*COMPONENTS$/ { exit } + capture { sub(/^[[:space:]]+/, ""); print } +' "$publisher")" +[[ "$actual_descriptor_rows" == "$expected_descriptor_rows" ]] || + fail "release descriptor rows do not exactly match the four infra rollback runtime rows" + +grep -Fq '.github/rollback-bridge-reviewed-forward-commit' "$ci" || + fail "CI does not read the central reviewed forward commit pin" +grep -Fq '.github/rollback-bridge-reviewed-forward-commit' "$publisher" || + fail "publisher does not read the central reviewed forward commit pin" +placeholder="__REVIEWED_FORWARD_""COMMIT_REQUIRED__" +unexpected_placeholder_files="$( + git grep -lF "$placeholder" -- .github services docs 2>/dev/null | + grep -vxF "$pin" || true +)" +if [[ -n "$unexpected_placeholder_files" ]]; then + fail "the unresolved forward commit placeholder may exist only in $pin" +fi + +echo "rollback bridge workflow safety checks passed" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f8c25b8c..233600c21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,6 +77,10 @@ jobs: '^\.github/workflows/ci\.yml$' set_output rust_api \ '^services/api-rs/' \ + '^\.github/rollback-bridge-reviewed-forward-commit$' \ + '^\.github/scripts/check-rollback-bridge-workflows\.sh$' \ + '^\.github/workflows/publish-images\.yml$' \ + '^\.github/workflows/validate-images\.yml$' \ '^\.github/workflows/ci\.yml$' set_output sandbox_config_tests \ '^services/sandbox/configure_codex_config\.py$' \ @@ -146,24 +150,23 @@ jobs: runs-on: ubuntu-latest needs: ci_changes if: needs.ci_changes.outputs.rust_api == 'true' - services: - postgres: - image: postgres:17 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - options: >- - --health-cmd="pg_isready -U postgres" - --health-interval=10s - --health-timeout=5s - --health-retries=5 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: persist-credentials: false + - name: Resolve the single reviewed forward commit pin + id: forward_pin + run: | + set -euo pipefail + IFS=$'\n\t' + commit="$(tr -d '\r\n' < .github/rollback-bridge-reviewed-forward-commit)" + if [[ ! "$commit" =~ ^[0-9a-f]{40}$ ]]; then + echo ".github/rollback-bridge-reviewed-forward-commit must contain the frozen lowercase 40-character commit SHA" >&2 + exit 1 + fi + echo "commit=$commit" >> "$GITHUB_OUTPUT" + - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1 with: toolchain: stable @@ -177,24 +180,82 @@ jobs: working-directory: services/api-rs run: cargo fmt --all --check + - name: Verify rollback bridge workflow safety + run: bash .github/scripts/check-rollback-bridge-workflows.sh + - name: Clippy working-directory: services/api-rs run: cargo clippy --workspace --all-targets -- -D warnings + - name: Verify rollback bridge forward migration provenance + env: + REVIEWED_FORWARD_COMMIT: ${{ steps.forward_pin.outputs.commit }} + run: | + set -euo pipefail + IFS=$'\n\t' + + fixture_dir=services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations + embedded_dir=services/api-rs/crates/centaur-session-sqlx/migrations + ( + cd "$fixture_dir" + sha256sum --check SHA256SUMS + ) + sha384sum --check "$embedded_dir/.checksums.sha384" + git fetch --no-tags --depth=1 origin "$REVIEWED_FORWARD_COMMIT" + while IFS=' ' read -r _ file; do + source_copy="$RUNNER_TEMP/$file" + git show "$REVIEWED_FORWARD_COMMIT:$embedded_dir/$file" > "$source_copy" + cmp "$source_copy" "$fixture_dir/$file" + cmp "$source_copy" "$embedded_dir/$file" + done < "$fixture_dir/SHA256SUMS" + + - name: Start Rust test database + run: | + set -euo pipefail + IFS=$'\n\t' + + docker run --detach --name centaur-rust-test-postgres \ + --publish 127.0.0.1:5432:5432 \ + --env POSTGRES_USER=postgres \ + --env POSTGRES_PASSWORD=postgres \ + --env POSTGRES_DB=postgres \ + paradedb/paradedb@sha256:e41e0c742ef91ece4fc7c08dda7f24e5a8f818563b164bbfea3a4364941d75f7 \ + -c shared_preload_libraries=pg_search,pg_cron \ + -c max_connections=500 + for _ in {1..60}; do + if docker logs centaur-rust-test-postgres 2>&1 \ + | grep -q 'PostgreSQL init process complete' \ + && docker exec centaur-rust-test-postgres \ + psql -U postgres -d postgres -tAc 'select 1' >/dev/null; then + exit 0 + fi + sleep 1 + done + docker logs centaur-rust-test-postgres + exit 1 + - name: Test working-directory: services/api-rs env: + SESSION_RUNTIME_TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres SESSION_SQLX_TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres run: cargo test --workspace + - name: Cleanup Rust test database + if: ${{ always() }} + run: docker rm -f centaur-rust-test-postgres || true + - name: Start API integration test dependencies run: | + set -euo pipefail + IFS=$'\n\t' + docker run --detach --name centaur-api-integration-test-postgres \ --publish 127.0.0.1:15432:5432 \ --env POSTGRES_USER=postgres \ --env POSTGRES_PASSWORD=postgres \ --env POSTGRES_DB=centaur \ - paradedb/paradedb:0.23.0-pg16 \ + paradedb/paradedb@sha256:e41e0c742ef91ece4fc7c08dda7f24e5a8f818563b164bbfea3a4364941d75f7 \ -c shared_preload_libraries=pg_search,pg_cron \ -c max_connections=500 for _ in {1..60}; do @@ -209,22 +270,96 @@ jobs: docker logs centaur-api-integration-test-postgres exit 1 + - name: Checkout reviewed forward source for cross-version rehearsal + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: TipLink/centaur + ref: ${{ steps.forward_pin.outputs.commit }} + path: reviewed-forward + persist-credentials: false + + - name: Build and verify reviewed forward runtime + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/reviewed-forward-target + SESSION_RUNTIME_TEST_DATABASE_URL: postgres://postgres:postgres@127.0.0.1:15432/centaur?sslmode=disable + run: | + set -euo pipefail + IFS=$'\n\t' + + cargo build --locked \ + --manifest-path reviewed-forward/services/api-rs/Cargo.toml \ + -p centaur-api-server + cargo test --locked \ + --manifest-path reviewed-forward/services/api-rs/Cargo.toml \ + -p centaur-session-runtime \ + rollback_era_clear_and_reassignment_cannot_spoof_forward_content_stamp \ + -- --nocapture + + - name: Apply API integration schema with reviewed forward runtime + working-directory: reviewed-forward/services/api-rs + env: + BIND_ADDR: 127.0.0.1:18081 + CENTAUR_CONTROL_API_KEY: integration-control-key + DATABASE_URL: postgres://postgres:postgres@127.0.0.1:15432/centaur?sslmode=disable + PYTHON_WORKFLOW_HOST_PATH: ${{ github.workspace }}/reviewed-forward/services/workflow-python/workflow_host.py + PYTHON_WORKFLOW_HOST_PYTHON: python3 + RUN_MIGRATIONS: "true" + RUST_LOG: warn + SESSION_EXECUTION_ADOPTION_INTERVAL_SECS: "0" + WORKFLOW_DIRS: ${{ runner.temp }}/reviewed-forward-schema-workflows + WORKFLOW_HOST_SANDBOX: "false" + WORKFLOW_REAP_REMOVED_AFTER_TICKS: "0" + WORKFLOW_RECONCILE_INTERVAL_SECS: "0" + run: | + set -euo pipefail + IFS=$'\n\t' + + mkdir -p "$WORKFLOW_DIRS" + "${{ runner.temp }}/reviewed-forward-target/debug/centaur-api-server" \ + > "$RUNNER_TEMP/reviewed-forward-schema.log" 2>&1 & + forward_pid="$!" + trap 'kill "$forward_pid" 2>/dev/null || true' EXIT + for _ in {1..60}; do + if curl --fail --silent http://127.0.0.1:18081/readyz >/dev/null; then + kill "$forward_pid" + wait "$forward_pid" || true + trap - EXIT + exit 0 + fi + if ! kill -0 "$forward_pid" 2>/dev/null; then + break + fi + sleep 1 + done + cat "$RUNNER_TEMP/reviewed-forward-schema.log" + exit 1 + - name: Run API integration test working-directory: services/api-rs env: API_INTEGRATION_WORKFLOW_DIR: ${{ runner.temp }}/api-integration-workflows BIND_ADDR: 127.0.0.1:18080 CENTAUR_API_URL: http://127.0.0.1:18080 + CENTAUR_CONTROL_API_KEY: integration-control-key + CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS: "true" DATABASE_URL: postgres://postgres:postgres@127.0.0.1:15432/centaur?sslmode=disable - RUN_MIGRATIONS: "true" + ROLLBACK_BRIDGE_FORWARD_TEST_DATABASE_URL: postgres://postgres:postgres@127.0.0.1:15432/centaur?sslmode=disable + ROLLBACK_BRIDGE_REHEARSAL_FORWARD_BIN: ${{ runner.temp }}/reviewed-forward-target/debug/centaur-api-server + ROLLBACK_BRIDGE_REHEARSAL_FORWARD_WORKDIR: ${{ github.workspace }}/reviewed-forward/services/api-rs + ROLLBACK_BRIDGE_REHEARSAL_FORWARD_WORKFLOW_HOST: ${{ github.workspace }}/reviewed-forward/services/workflow-python/workflow_host.py + RUN_MIGRATIONS: "false" RUST_LOG: info WORKFLOW_DIRS: ${{ runner.temp }}/api-integration-workflows WORKFLOW_HOST_SANDBOX: "false" WORKFLOW_REAP_REMOVED_AFTER_TICKS: "1" WORKFLOW_RECONCILE_INTERVAL_SECS: "1" run: | + set -euo pipefail + IFS=$'\n\t' + mkdir -p "$API_INTEGRATION_WORKFLOW_DIR" cargo build --locked -p centaur-api-server -p centaur-api-integration-test + cargo test --locked -p centaur-api-server --test rollback_bridge_forward_schema -- --nocapture ./target/debug/centaur-api-server > "$RUNNER_TEMP/centaur-api-server.log" 2>&1 & api_pid="$!" diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index 89f75fc74..0d9580c8d 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -1,55 +1,136 @@ -name: Publish Images +name: Publish Reviewed Rollback Bridge Images on: - push: - branches: [main] - tags: [v*] - paths: - - .github/workflows/publish-images.yml - - services/** - - crates/harness-server/** - - centaur_sdk/** - - packages/** - - tools/** - - scripts/bootstrap-k8s-secrets.sh - - package.json - - pnpm-lock.yaml - - pnpm-workspace.yaml - - .agents/skills/** - pull_request: workflow_dispatch: + inputs: + live_updater_scope_verified: + description: Operator attests live child annotations exclude all four bridge repositories and every relevant rule excludes reviewed-40 tags + required: true + type: boolean + default: false + reviewed_forward_commit: + description: Exact frozen reviewed-forward commit recorded by this bridge + required: true + type: string concurrency: - group: publish-images-${{ github.ref }} - cancel-in-progress: true + # Tags are global registry state, so every dispatch must serialize across + # branches and refs before any digest or manifest is published. + group: publish-reviewed-rollback-bridge-images + # A second dispatch must wait and observe the first run's immutable tags. + cancel-in-progress: false permissions: contents: read packages: write - pull-requests: read env: REGISTRY: ghcr.io IMAGE_NAMESPACE: tiplink/centaur IMAGE_SOURCE: https://github.com/TipLink/centaur - # Main/tags keep optimized release images. PRs and manual non-release - # branch publishes use debug builds so staging/dev iteration does not spend - # minutes optimizing Rust binaries that are immediately replaced by the next - # test build. - RUST_BUILD_PROFILE: ${{ (github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/v'))) && 'debug' || 'release' }} + RUST_BUILD_PROFILE: release jobs: + release-gate: + name: Confirm rollback publication barriers + runs-on: ubuntu-latest + outputs: + forward_commit: ${{ steps.forward_pin.outputs.commit }} + env: + DISPATCH_FORWARD_COMMIT: ${{ inputs.reviewed_forward_commit }} + LIVE_UPDATER_SCOPE_VERIFIED: ${{ inputs.live_updater_scope_verified }} + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - name: Require live updater scope proof and the frozen forward commit + id: forward_pin + run: | + set -euo pipefail + IFS=$'\n\t' + if [[ "$LIVE_UPDATER_SCOPE_VERIFIED" != "true" ]]; then + echo "read the live child annotations and verify that none of the four exact bridge repositories is in image-list and every relevant allow-tags rule excludes reviewed-40 before publication" >&2 + exit 1 + fi + commit="$(tr -d '\r\n' < .github/rollback-bridge-reviewed-forward-commit)" + if [[ ! "$commit" =~ ^[0-9a-f]{40}$ ]]; then + echo ".github/rollback-bridge-reviewed-forward-commit must contain the frozen lowercase 40-character commit SHA" >&2 + exit 1 + fi + if [[ "$DISPATCH_FORWARD_COMMIT" != "$commit" ]]; then + echo "dispatch reviewed_forward_commit does not match the bridge's frozen forward commit" >&2 + exit 1 + fi + echo "commit=$commit" >> "$GITHUB_OUTPUT" + + tag-absence-gate: + name: Prove reviewed tags do not already exist before publishing digests + runs-on: ubuntu-latest + needs: release-gate + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REVIEWED_TAG: reviewed-${{ github.sha }} + steps: + - name: Reject overwrite and fail closed on registry errors + run: | + set -euo pipefail + IFS=$'\n\t' + for image in centaur-api-rs centaur-slackbotv2 centaur-agent centaur-iron-proxy; do + repository="${IMAGE_NAMESPACE}/${image}" + token_json="$(curl --fail --silent --show-error \ + --user "${GITHUB_ACTOR}:${GHCR_TOKEN}" \ + --get \ + --data-urlencode "scope=repository:${repository}:pull" \ + --data-urlencode "service=ghcr.io" \ + https://ghcr.io/token)" + registry_token="$(jq -er '.token' <<<"$token_json")" + response_body="$(mktemp)" + if ! status="$(curl --silent --show-error \ + --output "$response_body" \ + --write-out '%{http_code}' \ + --header "Authorization: Bearer ${registry_token}" \ + --header 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json' \ + "https://ghcr.io/v2/${repository}/manifests/${REVIEWED_TAG}")"; then + echo "registry request failed while checking ${repository}:${REVIEWED_TAG}" >&2 + exit 1 + fi + case "$status" in + 404) + if ! jq -e 'any(.errors[]?; .code == "MANIFEST_UNKNOWN")' "$response_body" >/dev/null; then + echo "registry returned HTTP 404 without MANIFEST_UNKNOWN for ${repository}:${REVIEWED_TAG}" >&2 + cat "$response_body" >&2 + exit 1 + fi + ;; + 200) + echo "refusing to overwrite immutable reviewed tag: ${repository}:${REVIEWED_TAG}" >&2 + exit 1 + ;; + *) + echo "registry returned HTTP $status while checking ${repository}:${REVIEWED_TAG}" >&2 + cat "$response_body" >&2 + exit 1 + ;; + esac + rm -f "$response_body" + done + # Build each image natively per platform (amd64 on x64 runners, arm64 on # arm runners), push by digest, and hand the digests to the merge job below - # which assembles the multi-arch manifest. PR and manual non-release branch - # builds stay amd64-only to keep iteration fast. + # which assembles the multi-arch manifest. This package-write workflow is + # manual-only and requires confirmation from a read of the live child + # annotations that the four repositories and reviewed-full namespace are + # outside Image Updater's scope. build: + needs: tag-absence-gate runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} strategy: fail-fast: false matrix: - service: [api-rs, slackbotv2, linearbot, discordbot, teamsbot, agent, iron-proxy, console] - platform: ${{ (github.event_name == 'push' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && fromJSON('["linux/amd64", "linux/arm64"]') || fromJSON('["linux/amd64"]') }} + service: [api-rs, slackbotv2, agent, iron-proxy] + platform: [linux/amd64, linux/arm64] include: - service: api-rs image: centaur-api-rs @@ -61,21 +142,6 @@ jobs: context: . dockerfile: services/slackbotv2/Dockerfile target: "" - - service: linearbot - image: centaur-linearbot - context: . - dockerfile: services/linearbot/Dockerfile - target: "" - - service: discordbot - image: centaur-discordbot - context: . - dockerfile: services/discordbot/Dockerfile - target: "" - - service: teamsbot - image: centaur-teamsbot - context: . - dockerfile: services/teamsbot/Dockerfile - target: "" - service: agent image: centaur-agent context: . @@ -86,11 +152,6 @@ jobs: context: . dockerfile: services/iron-proxy/Dockerfile target: "" - - service: console - image: centaur-console - context: services/console - dockerfile: services/console/Dockerfile - target: "" steps: - name: Checkout @@ -100,6 +161,7 @@ jobs: - name: Derive platform slug run: | + set -euo pipefail platform="${{ matrix.platform }}" echo "PLATFORM_SLUG=${platform//\//-}" >> "$GITHUB_ENV" @@ -121,8 +183,6 @@ jobs: labels: | org.opencontainers.image.title=${{ matrix.image }} org.opencontainers.image.source=${{ env.IMAGE_SOURCE }} - env: - DOCKER_METADATA_PR_HEAD_SHA: true - name: Build and push ${{ matrix.image }} (${{ matrix.platform }}) id: build @@ -134,29 +194,25 @@ jobs: platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} # Tags are applied by the merge job on the multi-arch manifest; - # per-arch builds are pushed by digest only. Fork PRs run with a - # read-only GITHUB_TOKEN, so they build without pushing. - outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.image }},push-by-digest=true,name-canonical=true,push=${{ !github.event.pull_request.head.repo.fork }} + # per-arch release builds are pushed by digest only. + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.image }},push-by-digest=true,name-canonical=true,push=true build-args: | RUST_BUILD_PROFILE=${{ env.RUST_BUILD_PROFILE }} cache-from: | type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.image }}:buildcache-${{ env.PLATFORM_SLUG }} type=gha,scope=${{ matrix.image }}-${{ env.PLATFORM_SLUG }} - # Fork PRs run with a read-only GITHUB_TOKEN: exporting the registry - # cache would fail the build, so only export it when we can push. cache-to: | - ${{ !github.event.pull_request.head.repo.fork && format('type=registry,ref={0}/{1}/{2}:buildcache-{3},mode=max', env.REGISTRY, env.IMAGE_NAMESPACE, matrix.image, env.PLATFORM_SLUG) || '' }} + type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.image }}:buildcache-${{ env.PLATFORM_SLUG }},mode=max type=gha,mode=max,scope=${{ matrix.image }}-${{ env.PLATFORM_SLUG }} - name: Export digest - if: ${{ !github.event.pull_request.head.repo.fork }} run: | + set -euo pipefail mkdir -p ${{ runner.temp }}/digests digest="${{ steps.build.outputs.digest }}" touch "${{ runner.temp }}/digests/${digest#sha256:}" - name: Upload digest - if: ${{ !github.event.pull_request.head.repo.fork }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: digests-${{ matrix.image }}-${{ env.PLATFORM_SLUG }} @@ -167,19 +223,14 @@ jobs: merge: runs-on: ubuntu-latest needs: build - if: ${{ !github.event.pull_request.head.repo.fork }} strategy: fail-fast: false matrix: include: - image: centaur-api-rs - image: centaur-slackbotv2 - - image: centaur-linearbot - image: centaur-agent - image: centaur-iron-proxy - - image: centaur-console - - image: centaur-discordbot - - image: centaur-teamsbot steps: - name: Download digests @@ -207,22 +258,57 @@ jobs: flavor: | latest=false tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=raw,value=main,enable={{is_default_branch}} - type=raw,value=edge,enable={{is_default_branch}} - type=sha,prefix=main-sha-,enable={{is_default_branch}} - type=semver,pattern={{raw}} - type=ref,event=pr - type=sha + type=raw,value=reviewed-${{ github.sha }} labels: | org.opencontainers.image.title=${{ matrix.image }} org.opencontainers.image.source=${{ env.IMAGE_SOURCE }} - env: - DOCKER_METADATA_PR_HEAD_SHA: true - name: Create multi-arch manifest for ${{ matrix.image }} working-directory: ${{ runner.temp }}/digests + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REVIEWED_TAG: reviewed-${{ github.sha }} run: | + set -euo pipefail + IFS=$'\n\t' + repository="${IMAGE_NAMESPACE}/${{ matrix.image }}" + token_json="$(curl --fail --silent --show-error \ + --user "${GITHUB_ACTOR}:${GHCR_TOKEN}" \ + --get \ + --data-urlencode "scope=repository:${repository}:pull" \ + --data-urlencode "service=ghcr.io" \ + https://ghcr.io/token)" + registry_token="$(jq -er '.token' <<<"$token_json")" + response_body="$(mktemp)" + if ! status="$(curl --silent --show-error \ + --output "$response_body" \ + --write-out '%{http_code}' \ + --header "Authorization: Bearer ${registry_token}" \ + --header 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json' \ + "https://ghcr.io/v2/${repository}/manifests/${REVIEWED_TAG}")"; then + echo "registry request failed while rechecking ${repository}:${REVIEWED_TAG}" >&2 + exit 1 + fi + case "$status" in + 404) + if ! jq -e 'any(.errors[]?; .code == "MANIFEST_UNKNOWN")' "$response_body" >/dev/null; then + echo "registry returned HTTP 404 without MANIFEST_UNKNOWN for ${repository}:${REVIEWED_TAG}" >&2 + cat "$response_body" >&2 + exit 1 + fi + ;; + 200) + echo "refusing to overwrite immutable reviewed tag during final recheck: ${repository}:${REVIEWED_TAG}" >&2 + exit 1 + ;; + *) + echo "registry returned HTTP $status while rechecking ${repository}:${REVIEWED_TAG}" >&2 + cat "$response_body" >&2 + exit 1 + ;; + esac + rm -f "$response_body" + mapfile -t tags < <(jq -r '.tags[]' <<< "$DOCKER_METADATA_OUTPUT_JSON") tag_args=() for tag in "${tags[@]}"; do @@ -238,279 +324,91 @@ jobs: - name: Inspect manifest run: | + set -euo pipefail docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.image }}:${{ steps.meta.outputs.version }} - promote-fineas-infra: - name: Open Fineas infra promotion PR + release-descriptor: + name: Publish reviewed linux/arm64 release descriptor runs-on: ubuntu-latest needs: merge - if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main' steps: - - name: Create Fineas infra GitHub App token - id: fineas_token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 + - name: Download this run's linux/arm64 digests + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - app-id: ${{ vars.FINEAS_GITHUB_APP_ID }} - private-key: ${{ secrets.FINEAS_GITHUB_APP_PRIVATE_KEY }} - owner: TipLink - repositories: fineas-centaur-infra - permission-contents: write - permission-pull-requests: write + pattern: digests-*-linux-arm64 + path: ${{ runner.temp }}/arm64-digests - - name: Checkout Fineas infra - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - with: - repository: TipLink/fineas-centaur-infra - token: ${{ steps.fineas_token.outputs.token }} - path: fineas-centaur-infra - persist-credentials: false + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - - name: Update Fineas Centaur pins - working-directory: fineas-centaur-infra - run: | - set -euo pipefail - bash scripts/bump-centaur-pins.sh "${GITHUB_SHA}" - scripts/audit-supply-chain.sh + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Open or update Fineas infra PR - id: promote + - name: Write release descriptor env: - FINEAS_INFRA_TOKEN: ${{ steps.fineas_token.outputs.token }} - CENTAUR_TOKEN: ${{ github.token }} - CENTAUR_SHA: ${{ github.sha }} - CENTAUR_REPO: ${{ github.repository }} - CENTAUR_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + RELEASE_REVISION: ${{ github.sha }} + ARM64_DIGEST_ROOT: ${{ runner.temp }}/arm64-digests run: | - python3 <<'PY' - from __future__ import annotations - - import base64 - import json - import os - import re - import subprocess - import urllib.error - import urllib.parse - import urllib.request - from pathlib import Path - - owner = "TipLink" - repo = "fineas-centaur-infra" - base_branch = "main" - root = Path("fineas-centaur-infra") - centaur_sha = os.environ["CENTAUR_SHA"] - centaur_short = centaur_sha[:7] - centaur_tag = f"sha-{centaur_short}" - branch = f"automation/promote-centaur-{centaur_short}" - message = f"chore: promote centaur {centaur_short}" - title = f"Promote Centaur {centaur_short} to Fineas" - - headers = { - "Authorization": f"Bearer {os.environ['FINEAS_INFRA_TOKEN']}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - "User-Agent": "centaur-promote-fineas", - } - api = f"https://api.github.com/repos/{owner}/{repo}" - centaur_headers = dict(headers) - centaur_headers["Authorization"] = f"Bearer {os.environ['CENTAUR_TOKEN']}" - - def request( - method: str, - url: str, - payload: dict | None = None, - *, - ok: tuple[int, ...] = (200, 201, 204), - request_headers: dict[str, str] | None = None, - ): - data = None - req_headers = dict(request_headers or headers) - if payload is not None: - data = json.dumps(payload).encode() - req_headers["Content-Type"] = "application/json" - req = urllib.request.Request(url, data=data, headers=req_headers, method=method) - try: - with urllib.request.urlopen(req) as resp: - body_text = resp.read().decode() - if resp.status not in ok: - raise RuntimeError(f"{method} {url} returned {resp.status}: {body_text}") - return json.loads(body_text) if body_text else None - except urllib.error.HTTPError as exc: - body_text = exc.read().decode() - if exc.code in ok: - if exc.code == 404: - return None - return json.loads(body_text) if body_text else None - raise RuntimeError(f"{method} {url} returned {exc.code}: {body_text}") from None - - def centaur_pr_url_for_commit() -> str: - repo_full = os.environ["CENTAUR_REPO"] - url = f"https://api.github.com/repos/{repo_full}/commits/{centaur_sha}/pulls" - try: - pulls = request("GET", url, request_headers=centaur_headers) - except RuntimeError as exc: - print(f"warning: could not look up Centaur PR for {centaur_sha}: {exc}") - return "" - - merged_pulls = [pull for pull in pulls if pull.get("merged_at")] - pull = (merged_pulls or pulls or [None])[0] - return pull["html_url"] if pull else "" - - def git(*args: str) -> str: - return subprocess.check_output(["git", "-C", str(root), *args], text=True) - - def set_application_annotation(key: str, value: str) -> bool: - application_path = root / "clusters/centaur-sandbox/argocd/applications/centaur-sandbox.yaml" - text = application_path.read_text(encoding="utf-8") - pattern = re.compile(rf"^ {re.escape(key)}: .*$", flags=re.MULTILINE) - - if value: - line = f" {key}: {json.dumps(value)}" - if pattern.search(text): - new_text = pattern.sub(line, text, count=1) - else: - marker = " argocd.argoproj.io/compare-options: ServerSideDiff=true\n" - if marker not in text: - raise RuntimeError(f"could not place Application annotation {key}") - new_text = text.replace(marker, f"{marker}{line}\n", 1) - else: - new_text = pattern.sub("", text) - new_text = re.sub(r"\n{3,}", "\n\n", new_text) - - if new_text == text: - return False - application_path.write_text(new_text, encoding="utf-8") - return True - - def put_file_to_branch(rel: str, encoded_branch: str) -> bool: - encoded_path = urllib.parse.quote(rel, safe="") - existing = request("GET", f"{api}/contents/{encoded_path}?ref={encoded_branch}", ok=(200, 404)) - content_bytes = (root / rel).read_bytes() - if existing: - current = base64.b64decode(existing["content"]).replace(b"\r\n", b"\n") - if current == content_bytes: - return False - - payload = { - "message": message, - "content": base64.b64encode(content_bytes).decode(), - "branch": branch, - } - if existing: - payload["sha"] = existing["sha"] - request("PUT", f"{api}/contents/{encoded_path}", payload) - return True - - centaur_pr_url = centaur_pr_url_for_commit() - source_lines = [ - f"- Centaur commit: https://github.com/{os.environ['CENTAUR_REPO']}/commit/{centaur_sha}", - f"- Image publish run: {os.environ['CENTAUR_RUN_URL']}", - ] - if centaur_pr_url: - source_lines.append(f"- Centaur PR: {centaur_pr_url}") - - body = f"""## Summary - - update Fineas Centaur base image pins to `{centaur_tag}` - - pin the Centaur chart source to `{centaur_sha}` - - ## Source - {chr(10).join(source_lines)} - - ## Tests - - `bash scripts/bump-centaur-pins.sh {centaur_sha}` - - `scripts/audit-supply-chain.sh` - """ - - status_lines = [line for line in git("status", "--porcelain").splitlines() if line] - changed_paths: list[tuple[str, str]] = [] - for line in status_lines: - status = line[:2] - path = line[3:] - if " -> " in path: - path = path.split(" -> ", 1)[1] - if status == "??": - kind = "A" - elif "D" in status: - kind = "D" - else: - kind = "M" - changed_paths.append((kind, path)) - - output_path = os.environ["GITHUB_OUTPUT"] - if not changed_paths: - with open(output_path, "a", encoding="utf-8") as output: - output.write("changed=false\n") - output.write(f"branch={branch}\n") - output.write("pr_url=\n") - raise SystemExit(0) - - base_ref = request("GET", f"{api}/git/ref/heads/{base_branch}") - base_sha = base_ref["object"]["sha"] - encoded_branch = urllib.parse.quote(branch, safe="") - try: - request("POST", f"{api}/git/refs", {"ref": f"refs/heads/{branch}", "sha": base_sha}) - except RuntimeError as exc: - if "Reference already exists" not in str(exc): - raise - - changed = False - for kind, rel in changed_paths: - encoded_path = urllib.parse.quote(rel, safe="") - existing = request("GET", f"{api}/contents/{encoded_path}?ref={encoded_branch}", ok=(200, 404)) - if kind == "D": - if existing: - request( - "DELETE", - f"{api}/contents/{encoded_path}", - {"message": message, "branch": branch, "sha": existing["sha"]}, - ) - changed = True - continue - - content_bytes = (root / rel).read_bytes() - if existing: - current = base64.b64decode(existing["content"]).replace(b"\r\n", b"\n") - if current == content_bytes: - continue - - payload = { - "message": message, - "content": base64.b64encode(content_bytes).decode(), - "branch": branch, - } - if existing: - payload["sha"] = existing["sha"] - request("PUT", f"{api}/contents/{encoded_path}", payload) - changed = True - - pulls = request( - "GET", - f"{api}/pulls?state=open&head={urllib.parse.quote(f'{owner}:{branch}', safe='')}", - ) - if pulls: - pr = request("PATCH", f"{api}/pulls/{pulls[0]['number']}", {"title": title, "body": body}) - elif changed: - pr = request("POST", f"{api}/pulls", {"title": title, "head": branch, "base": base_branch, "body": body}) - else: - pr = None - - if pr: - annotations_changed = False - annotations_changed |= set_application_annotation("fineas.dev/deployment-pr-url", pr["html_url"]) - annotations_changed |= set_application_annotation("fineas.dev/centaur-pr-url", centaur_pr_url) - if annotations_changed: - put_file_to_branch( - "clusters/centaur-sandbox/argocd/applications/centaur-sandbox.yaml", - encoded_branch, - ) - - with open(output_path, "a", encoding="utf-8") as output: - output.write(f"changed={'true' if changed else 'false'}\n") - output.write(f"branch={branch}\n") - output.write(f"centaur_short={centaur_short}\n") - if pr: - output.write(f"pr_url={pr['html_url']}\n") - output.write(f"pr_number={pr['number']}\n") - else: - output.write("pr_url=\n") - PY + set -euo pipefail + if [[ ! "$RELEASE_REVISION" =~ ^[0-9a-f]{40}$ ]]; then + echo "invalid release revision: $RELEASE_REVISION" >&2 + exit 1 + fi + tag="reviewed-${RELEASE_REVISION}" + printf 'component\trepository\ttag\tdigest\trevision\n' > centaur-release.tsv + while IFS=$'\t' read -r component image; do + repository="${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${image}" + index_json="$(docker buildx imagetools inspect "${repository}:${tag}" --raw)" + mapfile -t arm64_digests < <(jq -r ' + [.manifests[] + | select(.platform.os == "linux" and .platform.architecture == "arm64") + | .digest] | .[]' <<<"$index_json") + if [[ "${#arm64_digests[@]}" -ne 1 ]]; then + echo "expected one runnable linux/arm64 manifest for $image; found ${#arm64_digests[@]}" >&2 + exit 1 + fi + digest="${arm64_digests[0]}" + if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "invalid arm64 digest for $image: $digest" >&2 + exit 1 + fi + artifact_dir="${ARM64_DIGEST_ROOT}/digests-${image}-linux-arm64" + mapfile -t built_digest_files < <(find "$artifact_dir" -maxdepth 1 -type f -print) + if [[ "${#built_digest_files[@]}" -ne 1 ]]; then + echo "expected one linux/arm64 digest artifact for $image; found ${#built_digest_files[@]}" >&2 + exit 1 + fi + built_digest="sha256:$(basename "${built_digest_files[0]}")" + if [[ ! "$built_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "invalid built linux/arm64 digest for $image: $built_digest" >&2 + exit 1 + fi + if [[ "$digest" != "$built_digest" ]]; then + echo "reviewed tag arm64 digest for $image does not match this run: tag=$digest built=$built_digest" >&2 + exit 1 + fi + # Prove the platform child itself is pullable. The multi-arch tag + # points at an OCI index; Kubernetes imageID reports this child + # manifest, so the reviewed lock records the child, not the index. + docker buildx imagetools inspect "${repository}@${digest}" >/dev/null + printf '%s\tghcr.io/tiplink/centaur/%s\t%s\t%s\t%s\n' \ + "$component" "$image" "$tag" "$digest" "$RELEASE_REVISION" \ + >> centaur-release.tsv + done <<'COMPONENTS' + api-rs centaur-api-rs + slackbotv2 centaur-slackbotv2 + sandbox centaur-agent + iron-proxy centaur-iron-proxy + COMPONENTS + + - name: Upload release descriptor + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: centaur-linux-arm64-release-${{ github.sha }} + path: centaur-release.tsv + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/validate-images.yml b/.github/workflows/validate-images.yml new file mode 100644 index 000000000..bba42accf --- /dev/null +++ b/.github/workflows/validate-images.yml @@ -0,0 +1,88 @@ +name: Publish Images + +on: + pull_request: + +concurrency: + group: validate-images-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + RUST_BUILD_PROFILE: debug + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Preserve the historical per-service required-check matrix while this + # PR-only workflow builds without registry credentials or publication. + service: [api-rs, slackbotv2, linearbot, discordbot, teamsbot, agent, iron-proxy, console] + platform: [linux/amd64] + include: + - service: api-rs + image: centaur-api-rs + context: . + dockerfile: services/api-rs/Dockerfile + target: "" + - service: slackbotv2 + image: centaur-slackbotv2 + context: . + dockerfile: services/slackbotv2/Dockerfile + target: "" + - service: linearbot + image: centaur-linearbot + context: . + dockerfile: services/linearbot/Dockerfile + target: "" + - service: discordbot + image: centaur-discordbot + context: . + dockerfile: services/discordbot/Dockerfile + target: "" + - service: teamsbot + image: centaur-teamsbot + context: . + dockerfile: services/teamsbot/Dockerfile + target: "" + - service: agent + image: centaur-agent + context: . + dockerfile: services/sandbox/Dockerfile + target: sandbox + - service: iron-proxy + image: centaur-iron-proxy + context: . + dockerfile: services/iron-proxy/Dockerfile + target: "" + - service: console + image: centaur-console + context: services/console + dockerfile: services/console/Dockerfile + target: "" + + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Build ${{ matrix.image }} without registry credentials + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + target: ${{ matrix.target }} + platforms: ${{ matrix.platform }} + push: false + build-args: | + RUST_BUILD_PROFILE=${{ env.RUST_BUILD_PROFILE }} + cache-from: type=gha,scope=validate-${{ matrix.image }}-linux-amd64 + cache-to: type=gha,mode=max,scope=validate-${{ matrix.image }}-linux-amd64 diff --git a/docs/pages/operate/upstream-rollback-bridge.mdx b/docs/pages/operate/upstream-rollback-bridge.mdx new file mode 100644 index 000000000..c822d1bf7 --- /dev/null +++ b/docs/pages/operate/upstream-rollback-bridge.mdx @@ -0,0 +1,169 @@ +# Upstream-sync rollback bridge + +This branch is an emergency, schema-forward bridge based on TipLink Centaur +`ba2c01f5`. It is not a general downgrade. Use it only through the audited +Fineas upstream-sync runbook in +`fineas-centaur-infra/docs/runbooks/centaur-upstream-sync-20260711.md`. + +## Hard safety contract + +- Keep `RUN_MIGRATIONS=false`. SQLx has no down migrations. This binary embeds + byte-identical migrations 1–43 so its ledger matches the forward database, + but the emergency rollback must not take ownership of schema migration. The + bridge rejects `RUN_MIGRATIONS=true` before binding or database access; only + the exact reviewed forward binary may migrate a database. +- Use the forward chart and provision a distinct `CENTAUR_CONTROL_API_KEY` + before the bridge starts. Control, Slack, GitHub, Linear, Discord, Teams, + workflow, and Slack-feedback service credentials must all be pairwise + distinct. The bridge refuses startup on a missing control key or any reused + service key; the old ba2c chart does not inject the control key. +- Make the API cutover with zero replica overlap. This bridge has release and + assignment compare-and-swap fences, but it does not implement the forward + runtime's cross-replica stdout-owner lease protocol. +- Do not run the branch image publisher until an operator has read the live + child annotations and proved that Argo Image Updater cannot manage any of + the four exact bridge repositories or the `reviewed-` tag + namespace, and core review has frozen the exact forward commit. The current + legacy updater manages only the separate Fineas overlay repository and its + allow-list accepts only deploy-shaped `sha-<7>` tags. The manual workflow + requires an explicit acknowledgement of that live scope proof and the exact + frozen commit as a matching dispatch input before it emits reviewed + `reviewed-` tags for exactly four linux/arm64 bridge runtime + rows: API, Slackbot v2, agent, and IronProxy. This namespace cannot match the + legacy updater's deploy-shaped `sha-<7>` tags; Console web/worker remains on + the forward image. Publication creates inert artifacts and a descriptor; it + is not rollout authorization. The infra runbook still requires global Image + Updater removal and the Argo automation freeze before consuming the + descriptor in Git or changing any Argo pin. The separate pull-request + workflow preserves the + historical `Publish Images` check context but has read-only repository + permissions, receives no registry credentials, and builds with `push: false`. + Push/tag events on this emergency branch do not publish; only the confirmed + manual dispatch can mutate GHCR. +- Set `CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS=true` explicitly. The bridge + refuses to start when the value is absent, false, or malformed. With the + fence acknowledged, it starts no absurd workers, schedule ticks, metadata + reconciler, or removed-workflow reaper. Workflow create/cancel/event + mutations return 403. This preserves pending, running, and sleeping rows for + re-forwarding. Startup is schema-read-only: all five forward absurd queues, + including `centaur_workflow_schedules`, must already exist. A missing queue + fails startup without creating it or changing the migration ledger. +- Do not treat cancellation as a workflow-drain strategy. Forward-only task + params, retry policy, cancellation policy, checkpoints, and wait rows must + survive byte-for-byte. + +Migrations 33–43 do not change absurd's task, run, checkpoint, event, or wait +schema. Migrations 7–9, which define that contract, are byte-identical between +the ba2c baseline and the forward integration. Handler source is not +backward-compatible, however: a task created for a workflow absent from the +rollback overlay would be failed or reaped by active ba2c workers. The +workflow pause is therefore mandatory. + +The forward chart value for the rollback infra state is exactly: + +```yaml +apiRs: + extraEnv: + CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS: "true" +``` + +The server validates pause and migration ownership before binding its listener +or touching the database. The workflow runtime then reads the existing absurd +queue registry and refuses incomplete forward state without running queue DDL. + +## Protected routes + +The bridge accepts `Authorization: Bearer $CENTAUR_CONTROL_API_KEY` for session +release, sandbox drain, global workflow routes, and admin routes. Anonymous and +Slackbot-key requests are rejected. The Slack archive workflow has one narrow +exception: its download-url request carries the exact workflow run and task +IDs, which must match the import row. + +Call release with the caller's observed sandbox ID: + +```bash +curl --fail-with-body \ + -H "Authorization: Bearer $CENTAUR_CONTROL_API_KEY" \ + -H 'Content-Type: application/json' \ + --data "{\"release_id\":\"rollback-window\",\"expected_sandbox_id\":\"$SANDBOX_ID\",\"cancel_inflight\":true}" \ + "$CENTAUR_API_URL/api/session/$THREAD_KEY/release" +``` + +A mismatch is a hard retry signal. The bridge must never stop the newly bound +sandbox from a stale observation. A backend stop failure returns HTTP 503 with +`ok: false`, `sandbox_released: false`, and `sandbox_release_error`; operators +must validate those fields and the returned null sandbox assignment rather than +trusting transport success alone. + +Before replacing the forward API, quiesce ingress and call drain once. Drain +permanently closes this runtime's allocation gate, pauses the warm replenisher, +waits for in-flight replenish and session allocation work, and only then takes +the backend inventory and stops it. New warm claims, resumes, and cold creates +return 503 after the gate closes. Any stop or warm-row failure makes the drain +request itself return 503 with the partial report, so `curl --fail-with-body` +is a real hard gate. + +```bash +curl --fail-with-body \ + -H "Authorization: Bearer $CENTAUR_CONTROL_API_KEY" \ + -X POST "$CENTAUR_API_URL/api/sandboxes/drain" +``` + +## Acceptance and re-forward + +After the bridge is ready, verify all of the following before reopening Slack +ingress: + +1. Exactly one bridge API replica exists and no forward API replica remains. +2. `RUN_MIGRATIONS=false` and workflow pause are effective. +3. Anonymous and Slackbot-key drain/release requests return 401; the distinct + control key reaches the handler. +4. Workflow mutation requests return 403 and the counts plus JSON contents of + all non-terminal absurd rows remain unchanged. +5. Session create, execute, expected-sandbox release, and one Slack turn pass. + +Workflow processing remains unavailable throughout the rollback. Re-forward +to the reviewed integration image before resuming those workers. On the +forward image, verify that pending rows are claimed, expired running claims are +adopted after their lease, and sleeping rows retain their checkpoint/wait +state. Never set `CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS=false` in the rollback +deployment. + +The focused source checks are: + +```bash +cd services/api-rs +cargo test -p centaur-api-server --lib +cargo test -p centaur-api-server --test rollback_bridge_startup +cargo test -p centaur-session-runtime adoption_tests::release_and_sandbox_assignment_race_has_exactly_one_winner +cargo test -p centaur-session-runtime adoption_tests::release_winning_allocation_race_stays_cancelled_not_failed +cargo test -p centaur-session-runtime adoption_tests::drain_waits_for_inflight_allocation_and_rejects_new_allocations +cargo test -p centaur-workflows --lib rollback_bridge_requires_workflow_pause_to_be_explicitly_enabled +``` + +CI fetches the pinned forward source commit, proves the test fixtures and +embedded migrations 33–43 are byte-identical to it, verifies both SHA-256 and +SQLx SHA-384 manifests, then applies the full embedded ledger 1–43 to a +disposable ParadeDB and seeds pending/running/sleeping tasks plus their runs, +retry/cancellation policy, checkpoints, events, waits, and migration ledger, +then runs the bridge with `RUN_MIGRATIONS=false` past worker/reaper intervals. +Another negative case omits the forward schedule queue and requires startup to +fail without changing the absurd schema or SQLx ledger. +The ordinary API integration database is likewise migrated by the exact +reviewed forward binary before the bridge integration server starts with +`RUN_MIGRATIONS=false`; the deployed-shape bridge integration server never +takes schema ownership. +It exercises read, create, cancel, event, webhook, and admin lanes and requires +an exact before/after JSON snapshot: + +```bash +ROLLBACK_BRIDGE_FORWARD_TEST_DATABASE_URL="$PARADEDB_URL" \ + cargo test -p centaur-api-server --test rollback_bridge_forward_schema +``` + +The CI-only cross-version case also builds the exact pinned forward commit. It +uses that binary to create real pending, running, and sleeping tasks, stops it, +starts the bridge in pause mode, checks an exact durable snapshot, then starts +the same forward binary again. Acceptance requires expired-claim adoption, +event and timed-wait resumption, preserved checkpoints and task contracts, and +actual replacement/restamping of a rollback-era sandbox assignment. diff --git a/docs/pages/reference/configuration.mdx b/docs/pages/reference/configuration.mdx index a1d3cfe8e..380bc9e26 100644 --- a/docs/pages/reference/configuration.mdx +++ b/docs/pages/reference/configuration.mdx @@ -81,6 +81,8 @@ Optional required-by-mode variables: | `CENTAUR_ENVIRONMENT`, `DEPLOY_ENV`, `ENVIRONMENT` | `apiRs.extraEnv` or deployment env. | Deployment environment resource attribute for telemetry. | | `OTEL_TRACES_EXPORTER` | `apiRs.extraEnv`. | Set to `otlp` to force OTLP trace export, or `none`/`off` to disable it. | | `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | `apiRs.extraEnv`. | Enables OTLP trace export to Tempo, Jaeger, or another OTLP collector. | +| `CENTAUR_CONTROL_API_KEY` | Forward chart Secret mapping. | Dedicated bearer key for destructive, admin, and global workflow routes in the rollback bridge. Startup fails if it is missing or any configured control, bot, workflow, or feedback service credentials are reused across trust lanes. | +| `CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS` | Required as `apiRs.extraEnv.CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS: "true"` in rollback infra. | Preserves non-terminal forward workflow rows by disabling workers, schedules, reconciliation, and workflow mutations. The rollback bridge refuses startup when missing, false, or malformed. | | `apiRs.metrics.scrapeAnnotations` | Helm value, default `true`. | Adds Prometheus scrape annotations to the API-RS Pod template and Service. | | `apiRs.metrics.path` | Helm value, default `/metrics`. | Metrics scrape path for annotation-based discovery. | | `apiRs.metrics.annotations` | Helm value. | Additional scrape annotations for Prometheus-compatible collectors. | diff --git a/docs/public/md/operate/upstream-rollback-bridge.md b/docs/public/md/operate/upstream-rollback-bridge.md new file mode 100644 index 000000000..c822d1bf7 --- /dev/null +++ b/docs/public/md/operate/upstream-rollback-bridge.md @@ -0,0 +1,169 @@ +# Upstream-sync rollback bridge + +This branch is an emergency, schema-forward bridge based on TipLink Centaur +`ba2c01f5`. It is not a general downgrade. Use it only through the audited +Fineas upstream-sync runbook in +`fineas-centaur-infra/docs/runbooks/centaur-upstream-sync-20260711.md`. + +## Hard safety contract + +- Keep `RUN_MIGRATIONS=false`. SQLx has no down migrations. This binary embeds + byte-identical migrations 1–43 so its ledger matches the forward database, + but the emergency rollback must not take ownership of schema migration. The + bridge rejects `RUN_MIGRATIONS=true` before binding or database access; only + the exact reviewed forward binary may migrate a database. +- Use the forward chart and provision a distinct `CENTAUR_CONTROL_API_KEY` + before the bridge starts. Control, Slack, GitHub, Linear, Discord, Teams, + workflow, and Slack-feedback service credentials must all be pairwise + distinct. The bridge refuses startup on a missing control key or any reused + service key; the old ba2c chart does not inject the control key. +- Make the API cutover with zero replica overlap. This bridge has release and + assignment compare-and-swap fences, but it does not implement the forward + runtime's cross-replica stdout-owner lease protocol. +- Do not run the branch image publisher until an operator has read the live + child annotations and proved that Argo Image Updater cannot manage any of + the four exact bridge repositories or the `reviewed-` tag + namespace, and core review has frozen the exact forward commit. The current + legacy updater manages only the separate Fineas overlay repository and its + allow-list accepts only deploy-shaped `sha-<7>` tags. The manual workflow + requires an explicit acknowledgement of that live scope proof and the exact + frozen commit as a matching dispatch input before it emits reviewed + `reviewed-` tags for exactly four linux/arm64 bridge runtime + rows: API, Slackbot v2, agent, and IronProxy. This namespace cannot match the + legacy updater's deploy-shaped `sha-<7>` tags; Console web/worker remains on + the forward image. Publication creates inert artifacts and a descriptor; it + is not rollout authorization. The infra runbook still requires global Image + Updater removal and the Argo automation freeze before consuming the + descriptor in Git or changing any Argo pin. The separate pull-request + workflow preserves the + historical `Publish Images` check context but has read-only repository + permissions, receives no registry credentials, and builds with `push: false`. + Push/tag events on this emergency branch do not publish; only the confirmed + manual dispatch can mutate GHCR. +- Set `CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS=true` explicitly. The bridge + refuses to start when the value is absent, false, or malformed. With the + fence acknowledged, it starts no absurd workers, schedule ticks, metadata + reconciler, or removed-workflow reaper. Workflow create/cancel/event + mutations return 403. This preserves pending, running, and sleeping rows for + re-forwarding. Startup is schema-read-only: all five forward absurd queues, + including `centaur_workflow_schedules`, must already exist. A missing queue + fails startup without creating it or changing the migration ledger. +- Do not treat cancellation as a workflow-drain strategy. Forward-only task + params, retry policy, cancellation policy, checkpoints, and wait rows must + survive byte-for-byte. + +Migrations 33–43 do not change absurd's task, run, checkpoint, event, or wait +schema. Migrations 7–9, which define that contract, are byte-identical between +the ba2c baseline and the forward integration. Handler source is not +backward-compatible, however: a task created for a workflow absent from the +rollback overlay would be failed or reaped by active ba2c workers. The +workflow pause is therefore mandatory. + +The forward chart value for the rollback infra state is exactly: + +```yaml +apiRs: + extraEnv: + CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS: "true" +``` + +The server validates pause and migration ownership before binding its listener +or touching the database. The workflow runtime then reads the existing absurd +queue registry and refuses incomplete forward state without running queue DDL. + +## Protected routes + +The bridge accepts `Authorization: Bearer $CENTAUR_CONTROL_API_KEY` for session +release, sandbox drain, global workflow routes, and admin routes. Anonymous and +Slackbot-key requests are rejected. The Slack archive workflow has one narrow +exception: its download-url request carries the exact workflow run and task +IDs, which must match the import row. + +Call release with the caller's observed sandbox ID: + +```bash +curl --fail-with-body \ + -H "Authorization: Bearer $CENTAUR_CONTROL_API_KEY" \ + -H 'Content-Type: application/json' \ + --data "{\"release_id\":\"rollback-window\",\"expected_sandbox_id\":\"$SANDBOX_ID\",\"cancel_inflight\":true}" \ + "$CENTAUR_API_URL/api/session/$THREAD_KEY/release" +``` + +A mismatch is a hard retry signal. The bridge must never stop the newly bound +sandbox from a stale observation. A backend stop failure returns HTTP 503 with +`ok: false`, `sandbox_released: false`, and `sandbox_release_error`; operators +must validate those fields and the returned null sandbox assignment rather than +trusting transport success alone. + +Before replacing the forward API, quiesce ingress and call drain once. Drain +permanently closes this runtime's allocation gate, pauses the warm replenisher, +waits for in-flight replenish and session allocation work, and only then takes +the backend inventory and stops it. New warm claims, resumes, and cold creates +return 503 after the gate closes. Any stop or warm-row failure makes the drain +request itself return 503 with the partial report, so `curl --fail-with-body` +is a real hard gate. + +```bash +curl --fail-with-body \ + -H "Authorization: Bearer $CENTAUR_CONTROL_API_KEY" \ + -X POST "$CENTAUR_API_URL/api/sandboxes/drain" +``` + +## Acceptance and re-forward + +After the bridge is ready, verify all of the following before reopening Slack +ingress: + +1. Exactly one bridge API replica exists and no forward API replica remains. +2. `RUN_MIGRATIONS=false` and workflow pause are effective. +3. Anonymous and Slackbot-key drain/release requests return 401; the distinct + control key reaches the handler. +4. Workflow mutation requests return 403 and the counts plus JSON contents of + all non-terminal absurd rows remain unchanged. +5. Session create, execute, expected-sandbox release, and one Slack turn pass. + +Workflow processing remains unavailable throughout the rollback. Re-forward +to the reviewed integration image before resuming those workers. On the +forward image, verify that pending rows are claimed, expired running claims are +adopted after their lease, and sleeping rows retain their checkpoint/wait +state. Never set `CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS=false` in the rollback +deployment. + +The focused source checks are: + +```bash +cd services/api-rs +cargo test -p centaur-api-server --lib +cargo test -p centaur-api-server --test rollback_bridge_startup +cargo test -p centaur-session-runtime adoption_tests::release_and_sandbox_assignment_race_has_exactly_one_winner +cargo test -p centaur-session-runtime adoption_tests::release_winning_allocation_race_stays_cancelled_not_failed +cargo test -p centaur-session-runtime adoption_tests::drain_waits_for_inflight_allocation_and_rejects_new_allocations +cargo test -p centaur-workflows --lib rollback_bridge_requires_workflow_pause_to_be_explicitly_enabled +``` + +CI fetches the pinned forward source commit, proves the test fixtures and +embedded migrations 33–43 are byte-identical to it, verifies both SHA-256 and +SQLx SHA-384 manifests, then applies the full embedded ledger 1–43 to a +disposable ParadeDB and seeds pending/running/sleeping tasks plus their runs, +retry/cancellation policy, checkpoints, events, waits, and migration ledger, +then runs the bridge with `RUN_MIGRATIONS=false` past worker/reaper intervals. +Another negative case omits the forward schedule queue and requires startup to +fail without changing the absurd schema or SQLx ledger. +The ordinary API integration database is likewise migrated by the exact +reviewed forward binary before the bridge integration server starts with +`RUN_MIGRATIONS=false`; the deployed-shape bridge integration server never +takes schema ownership. +It exercises read, create, cancel, event, webhook, and admin lanes and requires +an exact before/after JSON snapshot: + +```bash +ROLLBACK_BRIDGE_FORWARD_TEST_DATABASE_URL="$PARADEDB_URL" \ + cargo test -p centaur-api-server --test rollback_bridge_forward_schema +``` + +The CI-only cross-version case also builds the exact pinned forward commit. It +uses that binary to create real pending, running, and sleeping tasks, stops it, +starts the bridge in pause mode, checks an exact durable snapshot, then starts +the same forward binary again. Acceptance requires expired-claim adoption, +event and timed-wait resumption, preserved checkpoints and task contracts, and +actual replacement/restamping of a rollback-era sandbox assignment. diff --git a/docs/public/md/reference/configuration.md b/docs/public/md/reference/configuration.md index a1d3cfe8e..380bc9e26 100644 --- a/docs/public/md/reference/configuration.md +++ b/docs/public/md/reference/configuration.md @@ -81,6 +81,8 @@ Optional required-by-mode variables: | `CENTAUR_ENVIRONMENT`, `DEPLOY_ENV`, `ENVIRONMENT` | `apiRs.extraEnv` or deployment env. | Deployment environment resource attribute for telemetry. | | `OTEL_TRACES_EXPORTER` | `apiRs.extraEnv`. | Set to `otlp` to force OTLP trace export, or `none`/`off` to disable it. | | `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | `apiRs.extraEnv`. | Enables OTLP trace export to Tempo, Jaeger, or another OTLP collector. | +| `CENTAUR_CONTROL_API_KEY` | Forward chart Secret mapping. | Dedicated bearer key for destructive, admin, and global workflow routes in the rollback bridge. Startup fails if it is missing or any configured control, bot, workflow, or feedback service credentials are reused across trust lanes. | +| `CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS` | Required as `apiRs.extraEnv.CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS: "true"` in rollback infra. | Preserves non-terminal forward workflow rows by disabling workers, schedules, reconciliation, and workflow mutations. The rollback bridge refuses startup when missing, false, or malformed. | | `apiRs.metrics.scrapeAnnotations` | Helm value, default `true`. | Adds Prometheus scrape annotations to the API-RS Pod template and Service. | | `apiRs.metrics.path` | Helm value, default `/metrics`. | Metrics scrape path for annotation-based discovery. | | `apiRs.metrics.annotations` | Helm value. | Additional scrape annotations for Prometheus-compatible collectors. | diff --git a/packages/api-client/src/client.ts b/packages/api-client/src/client.ts index 527d5d781..639d9d8e4 100644 --- a/packages/api-client/src/client.ts +++ b/packages/api-client/src/client.ts @@ -305,11 +305,15 @@ export class CentaurClient { } } - async releaseThread(threadKey: string, opts?: { releaseId?: string; cancelInflight?: boolean }) { + async releaseThread( + threadKey: string, + opts?: { releaseId?: string; expectedSandboxId?: string; cancelInflight?: boolean }, + ) { const { data } = await this.http.post( `/api/session/${encodeURIComponent(threadKey)}/release`, { release_id: opts?.releaseId, + expected_sandbox_id: opts?.expectedSandboxId, cancel_inflight: opts?.cancelInflight ?? false, }, ); diff --git a/packages/api-client/test/client.test.ts b/packages/api-client/test/client.test.ts index 39ed55d35..05af74689 100644 --- a/packages/api-client/test/client.test.ts +++ b/packages/api-client/test/client.test.ts @@ -129,6 +129,7 @@ describe("CentaurClient", () => { }); await client.releaseThread(threadKey, { releaseId: "release:1", + expectedSandboxId: "sbx:1", cancelInflight: true, }); @@ -182,6 +183,7 @@ describe("CentaurClient", () => { "/api/session/slack%3AT123%3AC123%3A1700000000.000100/release", { release_id: "release:1", + expected_sandbox_id: "sbx:1", cancel_inflight: true, }, ); diff --git a/services/api-rs/crates/centaur-api-integration-test/src/main.rs b/services/api-rs/crates/centaur-api-integration-test/src/main.rs index 61a6e638c..477126836 100644 --- a/services/api-rs/crates/centaur-api-integration-test/src/main.rs +++ b/services/api-rs/crates/centaur-api-integration-test/src/main.rs @@ -8,7 +8,10 @@ use anyhow::{Context, Result, bail}; use centaur_session_core::HarnessType; use eventsource_stream::Eventsource; use futures_util::StreamExt; -use reqwest::{Client as HttpClient, StatusCode}; +use reqwest::{ + Client as HttpClient, StatusCode, + header::{AUTHORIZATION, HeaderMap, HeaderValue}, +}; use serde_json::{Value, json}; use tokio::time::{Instant, sleep, timeout}; use uuid::Uuid; @@ -23,7 +26,17 @@ async fn main() -> Result<()> { .unwrap_or_else(|_| DEFAULT_API_URL.to_owned()) .trim_end_matches('/') .to_owned(); - let http = HttpClient::new(); + let control_key = env::var("CENTAUR_CONTROL_API_KEY") + .context("CENTAUR_CONTROL_API_KEY is required by protected API integration routes")?; + let mut headers = HeaderMap::new(); + let mut authorization = HeaderValue::from_str(&format!("Bearer {}", control_key.trim())) + .context("CENTAUR_CONTROL_API_KEY is not valid HTTP header material")?; + authorization.set_sensitive(true); + headers.insert(AUTHORIZATION, authorization); + let http = HttpClient::builder() + .default_headers(headers) + .build() + .context("build integration HTTP client")?; let mut results = Vec::new(); @@ -60,9 +73,14 @@ async fn main() -> Result<()> { ); let line = line!() + 1; + let workflow_test_summary = if rollback_bridge_workflow_pause_enabled() { + "Rollback bridge rejects workflow mutations while preserving reads" + } else { + "Workflows API runs added workflows and cancels removed workflows" + }; record_result( &mut results, - "Workflows API runs added workflows and cancels removed workflows", + workflow_test_summary, line, test_workflows_api(&http, &base_url).await, ); @@ -492,6 +510,10 @@ async fn test_metrics(http: &HttpClient, base_url: &str) -> Result<()> { } async fn test_workflows_api(http: &HttpClient, base_url: &str) -> Result<()> { + if rollback_bridge_workflow_pause_enabled() { + return test_rollback_bridge_workflow_pause(http, base_url).await; + } + let workflow_dir = integration_workflow_dir()?; fs::create_dir_all(&workflow_dir) .with_context(|| format!("create workflow dir {}", workflow_dir.display()))?; @@ -561,6 +583,67 @@ async fn test_workflows_api(http: &HttpClient, base_url: &str) -> Result<()> { Ok(()) } +fn rollback_bridge_workflow_pause_enabled() -> bool { + matches!( + env::var("CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS") + .ok() + .map(|raw| raw.trim().to_ascii_lowercase()) + .as_deref(), + Some("true") + ) +} + +async fn test_rollback_bridge_workflow_pause(http: &HttpClient, base_url: &str) -> Result<()> { + // Read-only workflow inspection stays available for rollback validation. + get_json_ok(http, format!("{base_url}/api/workflows/schedules")) + .await + .context("list workflow schedules while rollback pause is active")?; + + for (path, body) in [ + ( + "/api/workflows/runs".to_owned(), + json!({ + "workflow_name": "must_not_start", + "input": {"source": "rollback-bridge-integration-test"}, + "idempotency_key": format!("rollback-pause-{}", Uuid::new_v4().simple()), + }), + ), + ( + format!("/api/workflows/runs/{}/cancel", Uuid::new_v4()), + json!({}), + ), + ( + "/api/workflows/events".to_owned(), + json!({ + "event_name": "rollback_bridge_must_not_emit", + "payload": {"source": "rollback-bridge-integration-test"}, + }), + ), + ] { + let url = format!("{base_url}{path}"); + let response = http + .post(&url) + .json(&body) + .send() + .await + .with_context(|| format!("POST {url}"))?; + let status = response.status(); + let response_body = response.text().await.unwrap_or_default(); + if status != StatusCode::FORBIDDEN { + bail!( + "rollback workflow mutation POST {url} returned {status}, expected 403: {response_body}" + ); + } + if !response_body.contains("rollback bridge") { + bail!( + "rollback workflow mutation POST {url} did not report the rollback fence: {response_body}" + ); + } + } + + Ok(()) +} + fn integration_workflow_dir() -> Result { let path = env::var("API_INTEGRATION_WORKFLOW_DIR") .context("API_INTEGRATION_WORKFLOW_DIR must point at the mounted workflow test dir")?; diff --git a/services/api-rs/crates/centaur-api-server/src/error.rs b/services/api-rs/crates/centaur-api-server/src/error.rs index a144442a3..aa79043db 100644 --- a/services/api-rs/crates/centaur-api-server/src/error.rs +++ b/services/api-rs/crates/centaur-api-server/src/error.rs @@ -17,6 +17,8 @@ pub enum ApiError { #[error("{0}")] Unauthorized(String), #[error("{0}")] + Forbidden(String), + #[error("{0}")] NotFound(String), #[error("{0}")] MethodNotAllowed(String), @@ -49,11 +51,13 @@ impl IntoResponse for ApiError { let status = match &self { Self::BadRequest(_) => StatusCode::BAD_REQUEST, Self::Unauthorized(_) => StatusCode::UNAUTHORIZED, + Self::Forbidden(_) => StatusCode::FORBIDDEN, Self::NotFound(_) => StatusCode::NOT_FOUND, Self::MethodNotAllowed(_) => StatusCode::METHOD_NOT_ALLOWED, Self::PayloadTooLarge(_) => StatusCode::PAYLOAD_TOO_LARGE, Self::ServiceUnavailable(_) => StatusCode::SERVICE_UNAVAILABLE, Self::Runtime(SessionRuntimeError::BadRequest(_)) => StatusCode::BAD_REQUEST, + Self::Runtime(SessionRuntimeError::Draining) => StatusCode::SERVICE_UNAVAILABLE, Self::Runtime(SessionRuntimeError::Store(SessionStoreError::NotFound { .. })) => { StatusCode::NOT_FOUND } diff --git a/services/api-rs/crates/centaur-api-server/src/lib.rs b/services/api-rs/crates/centaur-api-server/src/lib.rs index 0b0b3819b..7e9b76aee 100644 --- a/services/api-rs/crates/centaur-api-server/src/lib.rs +++ b/services/api-rs/crates/centaur-api-server/src/lib.rs @@ -176,43 +176,6 @@ mod tests { .uri("/api/session/slack%3AC123%3A123.456/events") .body(Body::empty()) .unwrap(), - Request::builder() - .method(Method::POST) - .uri("/api/sandboxes/drain") - .body(Body::empty()) - .unwrap(), - Request::builder() - .method(Method::GET) - .uri("/api/workflows/schedules") - .body(Body::empty()) - .unwrap(), - Request::builder() - .method(Method::GET) - .uri("/api/workflows/runs") - .body(Body::empty()) - .unwrap(), - Request::builder() - .method(Method::POST) - .uri("/api/workflows/runs") - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(r#"{"workflow_name":"agent_turn","input":{}}"#)) - .unwrap(), - Request::builder() - .method(Method::GET) - .uri("/api/workflows/runs/run-1") - .body(Body::empty()) - .unwrap(), - Request::builder() - .method(Method::POST) - .uri("/api/workflows/runs/run-1/cancel") - .body(Body::empty()) - .unwrap(), - Request::builder() - .method(Method::POST) - .uri("/api/workflows/events") - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(r#"{"event_name":"test.event","payload":{}}"#)) - .unwrap(), Request::builder() .method(Method::POST) .uri("/api/webhooks/test") @@ -225,6 +188,67 @@ mod tests { } } + #[tokio::test] + async fn control_routes_reject_anonymous_and_bot_keys_before_runtime_access() { + unsafe { + std::env::set_var("CENTAUR_CONTROL_API_KEY", "control-test-key"); + std::env::set_var("SLACKBOT_API_KEY", "bot-test-key"); + } + + for (uri, method) in [ + ("/api/sandboxes/drain", Method::POST), + ("/api/workflows/runs", Method::GET), + ("/api/admin/slack/archive-imports", Method::GET), + ( + "/api/admin/slack/archive-imports/import-1/download-url", + Method::POST, + ), + ] { + let anonymous = build_router_with_app_state(AppState::unready()) + .oneshot( + Request::builder() + .method(method.clone()) + .uri(uri) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(anonymous.status(), StatusCode::UNAUTHORIZED, "{uri}"); + + let bot_key = build_router_with_app_state(AppState::unready()) + .oneshot( + Request::builder() + .method(method.clone()) + .uri(uri) + .header(header::AUTHORIZATION, "Bearer bot-test-key") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(bot_key.status(), StatusCode::UNAUTHORIZED, "{uri}"); + + let control = build_router_with_app_state(AppState::unready()) + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header(header::AUTHORIZATION, "Bearer control-test-key") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(control.status(), StatusCode::SERVICE_UNAVAILABLE, "{uri}"); + } + + unsafe { + std::env::remove_var("CENTAUR_CONTROL_API_KEY"); + std::env::remove_var("SLACKBOT_API_KEY"); + } + } + #[tokio::test] async fn append_messages_does_not_apply_a_session_body_limit() { let pool = diff --git a/services/api-rs/crates/centaur-api-server/src/main.rs b/services/api-rs/crates/centaur-api-server/src/main.rs index be8c1e61c..8e091d61a 100644 --- a/services/api-rs/crates/centaur-api-server/src/main.rs +++ b/services/api-rs/crates/centaur-api-server/src/main.rs @@ -5,7 +5,7 @@ use centaur_api_server::{AppState, build_router_with_app_state}; use centaur_session_runtime::SessionRuntime; use centaur_session_sqlx::PgSessionStore; use centaur_telemetry::{TelemetryConfig, init_telemetry}; -use centaur_workflows::WorkflowRuntime; +use centaur_workflows::{WorkflowRuntime, require_rollback_bridge_workflow_pause}; use clap::Parser; use thiserror::Error; use tokio::net::TcpListener; @@ -13,12 +13,28 @@ use tracing::info; use args::Args; +const CONTROL_API_KEY_ENV: &str = "CENTAUR_CONTROL_API_KEY"; +const SERVICE_API_KEY_ENVS: [&str; 8] = [ + CONTROL_API_KEY_ENV, + "SLACKBOT_API_KEY", + "GITHUBBOT_API_KEY", + "LINEARBOT_API_KEY", + "DISCORDBOT_API_KEY", + "TEAMSBOT_API_KEY", + "WORKFLOW_API_KEY", + "SLACK_FEEDBACK_API_KEY", +]; + #[tokio::main] async fn main() -> Result<(), ServerError> { init_crypto_provider(); - let telemetry = init_telemetry(TelemetryConfig::from_env())?; - + // Fail before binding a listener or touching the database if the rollback + // deployment omitted its mandatory durable-workflow preservation fence. + require_rollback_bridge_workflow_pause()?; + require_distinct_service_api_keys()?; let args = Args::parse(); + require_rollback_bridge_migrations_disabled(args.server.run_migrations)?; + let telemetry = init_telemetry(TelemetryConfig::from_env())?; let listener = TcpListener::bind(args.server.bind_addr).await?; info!( bind_addr = %args.server.bind_addr, @@ -53,11 +69,61 @@ async fn main() -> Result<(), ServerError> { Ok(()) } +fn require_distinct_service_api_keys() -> Result<(), ServerError> { + let configured = SERVICE_API_KEY_ENVS + .iter() + .filter_map(|name| std::env::var(name).ok().map(|value| (*name, value))) + .collect::>(); + validate_service_api_key_separation( + configured + .iter() + .map(|(name, value)| (*name, value.as_str())), + ) + .map_err(ServerError::UnsupportedConfig) +} + +fn require_rollback_bridge_migrations_disabled(run_migrations: bool) -> Result<(), ServerError> { + if run_migrations { + return Err(ServerError::UnsupportedConfig( + "rollback bridge refuses to start with RUN_MIGRATIONS=true; the reviewed forward runtime owns schema migration" + .to_owned(), + )); + } + Ok(()) +} + +fn validate_service_api_key_separation<'a>( + configured: impl IntoIterator, +) -> Result<(), String> { + use std::collections::BTreeMap; + + let configured = configured + .into_iter() + .map(|(name, value)| (name, value.trim())) + .filter(|(_, value)| !value.is_empty()) + .collect::>(); + if !configured + .iter() + .any(|(name, _)| *name == CONTROL_API_KEY_ENV) + { + return Err(format!( + "rollback bridge refuses to start unless {CONTROL_API_KEY_ENV} is configured" + )); + } + + let mut owners = BTreeMap::new(); + for (name, value) in configured { + if let Some(existing) = owners.insert(value, name) { + return Err(format!( + "rollback bridge refuses to start because {existing} and {name} must contain distinct service credentials" + )); + } + } + Ok(()) +} + async fn initialize_runtime(args: Args, app_state: AppState) -> Result<(), ServerError> { let store = PgSessionStore::connect(&args.server.database_url).await?; - if args.server.run_migrations { - store.run_migrations().await?; - } let pool = store.pool().clone(); let sandbox_runtime = args.sandbox_runtime().await?; let mut runtime = SessionRuntime::new(store.clone(), sandbox_runtime); @@ -146,3 +212,36 @@ pub(crate) enum ServerError { #[error("{0}")] UnsupportedConfig(String), } + +#[cfg(test)] +mod rollback_bridge_config_tests { + use super::*; + + #[test] + fn control_key_is_required_and_every_service_key_is_pairwise_distinct() { + assert!(validate_service_api_key_separation([]).is_err()); + assert!(validate_service_api_key_separation([(CONTROL_API_KEY_ENV, "control")]).is_ok()); + for (index, left) in SERVICE_API_KEY_ENVS.iter().enumerate() { + for right in &SERVICE_API_KEY_ENVS[index + 1..] { + let error = validate_service_api_key_separation([ + (CONTROL_API_KEY_ENV, "control"), + (left, "shared"), + (right, "shared"), + ]) + .expect_err("reused service key must fail startup"); + assert!(error.contains(left), "unexpected error: {error}"); + assert!(error.contains(right), "unexpected error: {error}"); + assert!(!error.contains("shared"), "secret leaked in error: {error}"); + } + } + assert!( + validate_service_api_key_separation([ + (CONTROL_API_KEY_ENV, "control"), + ("SLACKBOT_API_KEY", "bot"), + ("WORKFLOW_API_KEY", "workflow"), + ("GITHUBBOT_API_KEY", ""), + ]) + .is_ok() + ); + } +} diff --git a/services/api-rs/crates/centaur-api-server/src/routes.rs b/services/api-rs/crates/centaur-api-server/src/routes.rs index ece7253c4..28fbf2212 100644 --- a/services/api-rs/crates/centaur-api-server/src/routes.rs +++ b/services/api-rs/crates/centaur-api-server/src/routes.rs @@ -29,8 +29,8 @@ use axum::{ use base64::{Engine as _, engine::general_purpose}; use centaur_session_core::ThreadKey; use centaur_session_runtime::{ - ExecuteSessionInput, HarnessConflictPolicy, PersonaSummary, SandboxRuntime, SessionRuntime, - thread_trace_id, thread_trace_parent_span_id, + DrainReport, ExecuteSessionInput, HarnessConflictPolicy, PersonaSummary, ReleaseThreadOutcome, + SandboxRuntime, SessionRuntime, thread_trace_id, thread_trace_parent_span_id, }; use centaur_session_sqlx::PgSessionStore; use centaur_telemetry::{ @@ -184,24 +184,12 @@ pub fn build_router_with_session_and_workflow_runtime( } pub fn build_router_with_app_state(state: AppState) -> Router { - Router::new() - .route("/healthz", get(healthz)) - .route("/readyz", get(readyz)) - .route("/metrics", get(metrics)) - .route("/api/personas", get(list_personas)) - .route( - "/api/session/{thread_key}", - post(create_or_get_session).get(get_session_context), - ) - .route( - "/api/session/{thread_key}/messages", - post(append_messages).layer(DefaultBodyLimit::disable()), - ) - .route( - "/api/session/{thread_key}/execute", - post(execute_session).layer(DefaultBodyLimit::disable()), - ) - .route("/api/session/{thread_key}/events", get(stream_events)) + // Keep destructive, administrative, and global workflow surfaces behind + // the dedicated operator key. Webhooks remain on their per-webhook auth; + // ordinary session routes retain their existing caller contract so this + // narrow rollback bridge does not pretend to implement the forward + // principal-JWT architecture. + let control_routes = Router::new() .route("/api/session/{thread_key}/release", post(release_thread)) .route("/agent/threads/{thread_key}/release", post(release_thread)) .route("/api/sandboxes/drain", post(drain_sandboxes)) @@ -248,10 +236,6 @@ pub fn build_router_with_app_state(state: AppState) -> Router { "/api/admin/slack/archive-imports/{import_id}/upload-url", post(refresh_slack_archive_import_upload_url), ) - .route( - "/api/admin/slack/archive-imports/{import_id}/download-url", - post(create_slack_archive_import_download_url), - ) .route( "/api/admin/slack/archive-imports/{import_id}/start", post(start_slack_archive_import), @@ -276,7 +260,32 @@ pub fn build_router_with_app_state(state: AppState) -> Router { "/api/admin/google/docs-sync/batch", post(ingest_google_docs_sync_batch).layer(DefaultBodyLimit::disable()), ) + .route_layer(middleware::from_fn(require_control_authorization)); + + Router::new() + .route("/healthz", get(healthz)) + .route("/readyz", get(readyz)) + .route("/metrics", get(metrics)) + .route("/api/personas", get(list_personas)) + .route( + "/api/session/{thread_key}", + post(create_or_get_session).get(get_session_context), + ) + .route( + "/api/session/{thread_key}/messages", + post(append_messages).layer(DefaultBodyLimit::disable()), + ) + .route( + "/api/session/{thread_key}/execute", + post(execute_session).layer(DefaultBodyLimit::disable()), + ) + .route("/api/session/{thread_key}/events", get(stream_events)) .route("/api/webhooks/{slug}", any(invoke_workflow_webhook)) + .route( + "/api/admin/slack/archive-imports/{import_id}/download-url", + post(create_slack_archive_import_download_url), + ) + .merge(control_routes) .layer( TraceLayer::new_for_http() .make_span_with(|request: &Request| { @@ -352,6 +361,78 @@ async fn metrics(State(state): State) -> Response { .into_response() } +async fn require_control_authorization(request: Request, next: Next) -> Response { + let expected = env::var("CENTAUR_CONTROL_API_KEY").unwrap_or_default(); + if !control_token_authorized(request.headers(), &expected) { + return ApiError::Unauthorized("invalid control service token".to_owned()).into_response(); + } + next.run(request).await +} + +fn control_token_authorized(headers: &HeaderMap, expected: &str) -> bool { + let expected = expected.trim(); + !expected.is_empty() + && bearer_token(headers) + .is_some_and(|presented| constant_time_eq(presented.as_bytes(), expected.as_bytes())) +} + +fn bearer_token(headers: &HeaderMap) -> Option<&str> { + let value = headers.get("Authorization")?.to_str().ok()?.trim(); + value + .strip_prefix("Bearer ") + .or_else(|| value.strip_prefix("bearer ")) + .map(str::trim) + .filter(|value| !value.is_empty()) +} + +#[derive(Debug)] +enum ArchiveDownloadAuthorization { + Control, + WorkflowTask { run_id: String, task_id: String }, +} + +fn authorize_archive_download( + headers: &HeaderMap, +) -> Result { + let control_key = env::var("CENTAUR_CONTROL_API_KEY").unwrap_or_default(); + if control_token_authorized(headers, &control_key) { + return Ok(ArchiveDownloadAuthorization::Control); + } + + let run_id = header_value(headers, "X-Centaur-Workflow-Run-Id") + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()); + let task_id = header_value(headers, "X-Centaur-Workflow-Task-Id") + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()); + match (run_id, task_id) { + (Some(run_id), Some(task_id)) => { + Ok(ArchiveDownloadAuthorization::WorkflowTask { run_id, task_id }) + } + _ => Err(ApiError::Unauthorized( + "archive download requires control or workflow-task authorization".to_owned(), + )), + } +} + +fn ensure_archive_download_authorized( + authorization: &ArchiveDownloadAuthorization, + import: &SlackArchiveImportRow, +) -> Result<(), ApiError> { + match authorization { + ArchiveDownloadAuthorization::Control => Ok(()), + ArchiveDownloadAuthorization::WorkflowTask { run_id, task_id } + if import.workflow_run_id.as_deref() == Some(run_id.as_str()) + && import.workflow_task_id.as_deref() == Some(task_id.as_str()) => + { + Ok(()) + } + ArchiveDownloadAuthorization::WorkflowTask { .. } => Err(ApiError::Forbidden( + "workflow task is not authorized for this archive import".to_owned(), + )), + } +} + async fn http_metrics(req: Request, next: Next) -> Response { let method = req.method().clone(); let route = matched_route(&req); @@ -506,41 +587,83 @@ async fn release_thread( State(state): State, Path(raw_thread_key): Path, Json(request): Json, -) -> Result, ApiError> { +) -> Result<(StatusCode, Json), ApiError> { let thread_key = ThreadKey::try_from(raw_thread_key)?; let outcome = state .runtime()? .release_thread( &thread_key, request.release_id.as_deref(), + request.expected_sandbox_id.as_deref(), request.cancel_inflight, ) .await?; - Ok(Json(crate::types::ReleaseThreadResponse { - ok: true, - session: outcome.session, - release_id: outcome.release_id, - cancel_inflight: outcome.cancel_inflight, - sandbox_released: outcome.sandbox_released, - sandbox_release_error: outcome.sandbox_release_error, - execution_id: outcome.execution_id, - execution_cancelled: outcome.execution_cancelled, - })) + Ok(release_thread_response( + outcome, + request.expected_sandbox_id, + )) } -async fn drain_sandboxes(State(state): State) -> Result, ApiError> { +fn release_thread_response( + outcome: ReleaseThreadOutcome, + expected_sandbox_id: Option, +) -> (StatusCode, Json) { + let ReleaseThreadOutcome { + session, + release_id, + cancel_inflight, + sandbox_released, + sandbox_release_error, + execution_id, + execution_cancelled, + } = outcome; + let ok = sandbox_release_error.is_none(); + let status = if ok { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + ( + status, + Json(crate::types::ReleaseThreadResponse { + ok, + session, + release_id, + expected_sandbox_id, + cancel_inflight, + sandbox_released, + sandbox_release_error, + execution_id, + execution_cancelled, + }), + ) +} + +async fn drain_sandboxes( + State(state): State, +) -> Result<(StatusCode, Json), ApiError> { let report = state.runtime()?.drain().await?; + Ok(drain_report_response(report)) +} + +fn drain_report_response(report: DrainReport) -> (StatusCode, Json) { + let status = if report.failed.is_empty() { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; let failed = report .failed .iter() .map(|failure| json!({ "sandbox_id": failure.sandbox_id, "error": failure.error })) .collect::>(); - Ok(Json(json!({ + let body = json!({ "ok": report.failed.is_empty(), "stopped_count": report.stopped.len(), "stopped": report.stopped, "failed": failed, - }))) + }); + (status, Json(body)) } async fn stream_events( @@ -1223,10 +1346,13 @@ async fn refresh_slack_archive_import_upload_url( async fn create_slack_archive_import_download_url( State(state): State, + headers: HeaderMap, Path(import_id): Path, ) -> Result, ApiError> { + let authorization = authorize_archive_download(&headers)?; let pool = db_pool(&state)?; let import = load_slack_archive_import(&pool, &import_id).await?; + ensure_archive_download_authorized(&authorization, &import)?; ensure_archive_import_status( &import.status, &["uploaded", "importing", "failed"], @@ -3034,6 +3160,37 @@ mod slack_archive_import_tests { } } + #[test] + fn workflow_task_archive_capability_requires_exact_run_and_task_match() { + let mut import = archive_row("uploaded"); + import.workflow_run_id = Some("run-123".to_owned()); + import.workflow_task_id = Some("task-456".to_owned()); + + ensure_archive_download_authorized( + &ArchiveDownloadAuthorization::WorkflowTask { + run_id: "run-123".to_owned(), + task_id: "task-456".to_owned(), + }, + &import, + ) + .expect("matching task capability"); + for (run_id, task_id) in [ + ("run-other", "task-456"), + ("run-123", "task-other"), + ("run-other", "task-other"), + ] { + let error = ensure_archive_download_authorized( + &ArchiveDownloadAuthorization::WorkflowTask { + run_id: run_id.to_owned(), + task_id: task_id.to_owned(), + }, + &import, + ) + .expect_err("mismatched task capability must fail"); + assert!(matches!(error, ApiError::Forbidden(_))); + } + } + #[test] fn archive_import_status_gate_allows_only_requested_statuses() { assert!( @@ -3164,6 +3321,19 @@ mod slack_archive_import_tests { mod webhook_tests { use super::*; + #[test] + fn control_auth_accepts_only_the_dedicated_bearer_key() { + let mut headers = HeaderMap::new(); + assert!(!control_token_authorized(&headers, "control-key")); + + headers.insert("authorization", "Bearer bot-key".parse().unwrap()); + assert!(!control_token_authorized(&headers, "control-key")); + + headers.insert("authorization", "Bearer control-key".parse().unwrap()); + assert!(control_token_authorized(&headers, "control-key")); + assert!(!control_token_authorized(&headers, "")); + } + #[test] fn session_thread_key_from_path_decodes_session_routes() { assert_eq!( @@ -3421,3 +3591,67 @@ mod webhook_tests { assert!(matches!(error, ApiError::Internal(_))); } } + +#[cfg(test)] +mod drain_tests { + use centaur_session_core::{HarnessType, Session, SessionStatus}; + use centaur_session_runtime::{DrainFailure, DrainReport, ReleaseThreadOutcome}; + + use super::*; + + #[test] + fn partial_drain_failure_returns_non_success_status_with_report() { + let (status, Json(body)) = drain_report_response(DrainReport { + stopped: vec!["sbx-stopped".to_owned()], + failed: vec![DrainFailure { + sandbox_id: "sbx-live".to_owned(), + error: "stop failed".to_owned(), + }], + }); + + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(body["ok"], false); + assert_eq!(body["stopped"], json!(["sbx-stopped"])); + assert_eq!(body["failed"][0]["sandbox_id"], "sbx-live"); + } + + #[test] + fn sandbox_stop_failure_returns_service_unavailable_release_response() { + let (status, Json(body)) = release_thread_response( + ReleaseThreadOutcome { + session: Session { + thread_key: ThreadKey::parse("test:release-stop-failure").unwrap(), + sandbox_id: None, + sandbox_capabilities: None, + harness_type: HarnessType::Codex, + harness_thread_id: None, + persona_id: None, + status: SessionStatus::Idle, + iron_control_principal: None, + created_at: OffsetDateTime::UNIX_EPOCH, + updated_at: OffsetDateTime::UNIX_EPOCH, + }, + release_id: Some("release-stop-failure".to_owned()), + cancel_inflight: false, + sandbox_released: false, + sandbox_release_error: Some("backend refused stop".to_owned()), + execution_id: None, + execution_cancelled: false, + }, + Some("sbx-stop-failure".to_owned()), + ); + + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert!(!body.ok); + assert!(!body.sandbox_released); + assert_eq!( + body.sandbox_release_error.as_deref(), + Some("backend refused stop") + ); + assert_eq!( + body.expected_sandbox_id.as_deref(), + Some("sbx-stop-failure") + ); + assert!(body.session.sandbox_id.is_none()); + } +} diff --git a/services/api-rs/crates/centaur-api-server/src/types.rs b/services/api-rs/crates/centaur-api-server/src/types.rs index 09924cbea..f07c0bb5a 100644 --- a/services/api-rs/crates/centaur-api-server/src/types.rs +++ b/services/api-rs/crates/centaur-api-server/src/types.rs @@ -75,9 +75,13 @@ pub struct ExecuteSessionResponse { pub status: String, } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct ReleaseThreadRequest { pub release_id: Option, + /// Compare-and-swap fence supplied by the caller. It is required whenever + /// the thread currently has a sandbox, and release is rejected unless the + /// thread is still assigned to this exact sandbox. + pub expected_sandbox_id: Option, #[serde(default)] pub cancel_inflight: bool, } @@ -88,6 +92,7 @@ pub struct ReleaseThreadResponse { #[serde(flatten)] pub session: Session, pub release_id: Option, + pub expected_sandbox_id: Option, pub cancel_inflight: bool, pub sandbox_released: bool, pub sandbox_release_error: Option, diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0033_session_title.sql b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0033_session_title.sql new file mode 100644 index 000000000..7e20b465e --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0033_session_title.sql @@ -0,0 +1,2 @@ +alter table sessions + add column if not exists title text; diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0034_session_sandbox_activity.sql b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0034_session_sandbox_activity.sql new file mode 100644 index 000000000..eac1434ca --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0034_session_sandbox_activity.sql @@ -0,0 +1,21 @@ +alter table sessions + add column if not exists sandbox_last_active_at timestamptz; + +update sessions +set sandbox_last_active_at = coalesce(sandbox_last_active_at, updated_at, created_at) +where sandbox_id is not null; + +create index if not exists sessions_sandbox_activity_idx + on sessions (sandbox_last_active_at, thread_key) + where sandbox_id is not null; + +alter table session_warm_sandboxes + drop constraint if exists session_warm_sandboxes_status_supported; + +alter table session_warm_sandboxes + add constraint session_warm_sandboxes_status_supported + check (status in ('ready', 'claimed', 'evicting', 'failed')); + +create index if not exists session_warm_sandboxes_evicting_idx + on session_warm_sandboxes (updated_at, sandbox_id) + where status = 'evicting'; diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0035_session_execution_stdout_owner.sql b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0035_session_execution_stdout_owner.sql new file mode 100644 index 000000000..9dd0c44b1 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0035_session_execution_stdout_owner.sql @@ -0,0 +1,7 @@ +alter table session_executions + add column if not exists stdout_owner_id text, + add column if not exists stdout_owner_lease_expires_at timestamptz; + +create index if not exists session_executions_stdout_owner_lease_idx + on session_executions (stdout_owner_lease_expires_at) + where status in ('queued', 'running') and stdout_owner_id is not null; From 47c660bff66b750e72e181ec35ea08251c283890 Mon Sep 17 00:00:00 2001 From: "fineas-bot[bot]" <258147136+fineas-bot[bot]@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:04:17 -0400 Subject: [PATCH 2/9] feat: add schema-forward rollback bridge (2/3) Reviewed emergency bridge for preserving the frozen forward migration ledger and workflow state during a staged rollback. This commit does not deploy or publish the bridge. --- ..._session_sandbox_api_server_capability.sql | 7 + .../0037_readonly_all_workflow_queues.sql | 91 + ...0038_session_sandbox_repo_cache_access.sql | 10 + .../0039_slack_private_channels.sql | 130 ++ .../0040_granola_sync_tables.sql | 277 +++ .../0041_attio_sync_tables.sql | 131 ++ .../0042_centaur_readonly_slack_dm_rls.sql | 139 ++ .../0043_session_sandbox_content_revision.sql | 21 + .../fixtures/forward_migrations/README.md | 16 + .../fixtures/forward_migrations/SHA256SUMS | 11 + .../tests/rollback_bridge_forward_schema.rs | 1488 +++++++++++++++++ .../tests/rollback_bridge_startup.rs | 105 ++ .../centaur-sandbox-manager/src/warm_pool.rs | 30 +- .../crates/centaur-session-runtime/src/lib.rs | 934 +++++++++-- .../migrations/.checksums.sha384 | 43 + .../migrations/0033_session_title.sql | 2 + .../0034_session_sandbox_activity.sql | 21 + .../0035_session_execution_stdout_owner.sql | 7 + ..._session_sandbox_api_server_capability.sql | 7 + .../0037_readonly_all_workflow_queues.sql | 91 + 20 files changed, 3458 insertions(+), 103 deletions(-) create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0036_session_sandbox_api_server_capability.sql create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0037_readonly_all_workflow_queues.sql create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0038_session_sandbox_repo_cache_access.sql create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0039_slack_private_channels.sql create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0040_granola_sync_tables.sql create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0041_attio_sync_tables.sql create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0042_centaur_readonly_slack_dm_rls.sql create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0043_session_sandbox_content_revision.sql create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/README.md create mode 100644 services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/SHA256SUMS create mode 100644 services/api-rs/crates/centaur-api-server/tests/rollback_bridge_forward_schema.rs create mode 100644 services/api-rs/crates/centaur-api-server/tests/rollback_bridge_startup.rs create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/.checksums.sha384 create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/0033_session_title.sql create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/0034_session_sandbox_activity.sql create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/0035_session_execution_stdout_owner.sql create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/0036_session_sandbox_api_server_capability.sql create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/0037_readonly_all_workflow_queues.sql diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0036_session_sandbox_api_server_capability.sql b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0036_session_sandbox_api_server_capability.sql new file mode 100644 index 000000000..f7faa61c5 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0036_session_sandbox_api_server_capability.sql @@ -0,0 +1,7 @@ +alter table sessions + add column if not exists sandbox_api_server_enabled boolean; + +update sessions +set sandbox_api_server_enabled = true +where sandbox_observability_enabled is not null + and sandbox_api_server_enabled is null; diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0037_readonly_all_workflow_queues.sql b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0037_readonly_all_workflow_queues.sql new file mode 100644 index 000000000..5a0095898 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0037_readonly_all_workflow_queues.sql @@ -0,0 +1,91 @@ +select absurd.create_queue('centaur_workflows'); +select absurd.create_queue('centaur_workflows_slack_live'); +select absurd.create_queue('centaur_workflows_etl'); +select absurd.create_queue('centaur_workflows_etl_backfill'); + +create or replace view centaur_readonly_workflow_runs as +select + 'centaur_workflows'::text as queue_name, + r.run_id::text as run_id, + t.task_id::text as task_id, + t.task_name, + t.params ->> 'workflow_name' as workflow_name, + t.params ->> 'harness_type' as harness_type, + t.state, + t.attempts, + t.max_attempts, + t.enqueue_at as created_at, + t.first_started_at, + r.started_at, + r.completed_at, + r.failed_at, + r.available_at, + r.claimed_by is not null as claimed, + t.cancelled_at +from absurd.t_centaur_workflows t +left join absurd.r_centaur_workflows r on r.run_id = t.last_attempt_run +union all +select + 'centaur_workflows_slack_live'::text as queue_name, + r.run_id::text as run_id, + t.task_id::text as task_id, + t.task_name, + t.params ->> 'workflow_name' as workflow_name, + t.params ->> 'harness_type' as harness_type, + t.state, + t.attempts, + t.max_attempts, + t.enqueue_at as created_at, + t.first_started_at, + r.started_at, + r.completed_at, + r.failed_at, + r.available_at, + r.claimed_by is not null as claimed, + t.cancelled_at +from absurd.t_centaur_workflows_slack_live t +left join absurd.r_centaur_workflows_slack_live r on r.run_id = t.last_attempt_run +union all +select + 'centaur_workflows_etl'::text as queue_name, + r.run_id::text as run_id, + t.task_id::text as task_id, + t.task_name, + t.params ->> 'workflow_name' as workflow_name, + t.params ->> 'harness_type' as harness_type, + t.state, + t.attempts, + t.max_attempts, + t.enqueue_at as created_at, + t.first_started_at, + r.started_at, + r.completed_at, + r.failed_at, + r.available_at, + r.claimed_by is not null as claimed, + t.cancelled_at +from absurd.t_centaur_workflows_etl t +left join absurd.r_centaur_workflows_etl r on r.run_id = t.last_attempt_run +union all +select + 'centaur_workflows_etl_backfill'::text as queue_name, + r.run_id::text as run_id, + t.task_id::text as task_id, + t.task_name, + t.params ->> 'workflow_name' as workflow_name, + t.params ->> 'harness_type' as harness_type, + t.state, + t.attempts, + t.max_attempts, + t.enqueue_at as created_at, + t.first_started_at, + r.started_at, + r.completed_at, + r.failed_at, + r.available_at, + r.claimed_by is not null as claimed, + t.cancelled_at +from absurd.t_centaur_workflows_etl_backfill t +left join absurd.r_centaur_workflows_etl_backfill r on r.run_id = t.last_attempt_run; + +grant select on table centaur_readonly_workflow_runs to centaur_readonly; diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0038_session_sandbox_repo_cache_access.sql b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0038_session_sandbox_repo_cache_access.sql new file mode 100644 index 000000000..a8e4dab30 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0038_session_sandbox_repo_cache_access.sql @@ -0,0 +1,10 @@ +alter table sessions + add column if not exists sandbox_repo_cache_access text; + +update sessions +set sandbox_repo_cache_access = case + when sandbox_repo_cache_enabled then 'all' + else 'none' +end +where sandbox_repo_cache_access is null + and sandbox_repo_cache_enabled is not null; diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0039_slack_private_channels.sql b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0039_slack_private_channels.sql new file mode 100644 index 000000000..eda032787 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0039_slack_private_channels.sql @@ -0,0 +1,130 @@ +alter table slack_sync_channels + add column if not exists is_private boolean; + +-- Existing rows predate the dedicated privacy column. Trust an explicit +-- boolean from the stored Slack payload; when privacy is absent or malformed, +-- fail closed until a live Slack sync can classify the channel. +update slack_sync_channels +set is_private = case + when jsonb_typeof(raw_payload -> 'is_private') = 'boolean' + then (raw_payload ->> 'is_private')::boolean + else true +end; + +alter table slack_sync_channels + alter column is_private set default true, + alter column is_private set not null; + +create index if not exists idx_slack_sync_channels_private + on slack_sync_channels (is_private, channel_id); + +drop policy if exists centaur_readonly_slack_sync_channels_select + on slack_sync_channels; +create policy centaur_readonly_slack_sync_channels_select + on slack_sync_channels + for select + to centaur_readonly + using ( + not is_private + or channel_id = centaur_current_slack_channel_id() + ); + +drop policy if exists centaur_readonly_slack_sync_message_attachments_select + on slack_sync_message_attachments; +create policy centaur_readonly_slack_sync_message_attachments_select + on slack_sync_message_attachments + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_sync_channels channels + where channels.channel_id = slack_sync_message_attachments.channel_id + ) + ); + +drop policy if exists centaur_readonly_slack_sync_messages_select + on slack_sync_messages; +create policy centaur_readonly_slack_sync_messages_select + on slack_sync_messages + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_sync_channels channels + where channels.channel_id = slack_sync_messages.channel_id + ) + ); + +drop policy if exists centaur_readonly_company_context_documents_select + on company_context_documents; +create policy centaur_readonly_company_context_documents_select + on company_context_documents + for select + to centaur_readonly + using ( + source <> 'slack' + or exists ( + select 1 + from slack_sync_channels channels + where channels.channel_id = metadata ->> 'channel_id' + ) + ); + +-- Fineas company context intentionally exposes documents from public, +-- syncable Slack channels across channel-scoped principals. Keep direct access +-- to the principal's current channel (including a private channel), but never +-- use the Slack channel-id prefix as a privacy signal. +create or replace function centaur_slack_channel_is_public_syncable( + _schema name, + _channel_id text +) +returns boolean +language plpgsql +stable +security definer +set search_path = pg_catalog +as $$ +declare + public_syncable boolean; +begin + execute format( + 'select exists ( + select 1 + from %I.slack_sync_channels channels + where channels.channel_id = $1 + and channels.is_syncable + and not channels.is_private + )', + _schema + ) + into public_syncable + using _channel_id; + return coalesce(public_syncable, false); +end +$$; + +revoke all on function centaur_slack_channel_is_public_syncable(name, text) + from public; +grant execute on function centaur_slack_channel_is_public_syncable(name, text) + to centaur_slack_reader; + +drop policy if exists centaur_context_docs_reader_select + on company_context_documents; +create policy centaur_context_docs_reader_select + on company_context_documents + for select + to centaur_slack_reader + using ( + source <> 'slack' + or metadata ->> 'channel_id' = centaur_current_slack_channel_id() + or centaur_slack_channel_is_public_syncable( + current_schema(), + metadata ->> 'channel_id' + ) + ); + +-- The old helper treated a C-prefixed id as public and was executable by +-- PUBLIC. Its only policy dependency was replaced immediately above. +drop function if exists centaur_slack_channel_is_syncable(name, text); diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0040_granola_sync_tables.sql b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0040_granola_sync_tables.sql new file mode 100644 index 000000000..20284fba3 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0040_granola_sync_tables.sql @@ -0,0 +1,277 @@ +create extension if not exists pg_search; + +create table if not exists granola_sync_runs ( + run_id text primary key, + workflow_run_id text, + mode text not null default 'incremental', + status text not null, + scopes_requested jsonb not null default '[]'::jsonb, + scopes_synced jsonb not null default '[]'::jsonb, + scopes_failed jsonb not null default '[]'::jsonb, + notes_seen integer not null default 0, + notes_upserted integer not null default 0, + transcripts_seen integer not null default 0, + transcripts_upserted integer not null default 0, + started_at timestamptz not null default now(), + finished_at timestamptz, + error_text text not null default '', + metadata jsonb not null default '{}'::jsonb +); + +create index if not exists idx_granola_sync_runs_started + on granola_sync_runs (started_at desc); + +create table if not exists granola_sync_notes ( + note_id text primary key, + title text not null default '', + owner_id text not null default '', + owner_email text not null default '', + owner_name text not null default '', + attendees jsonb not null default '[]'::jsonb, + access_emails text[] not null default array[]::text[], + calendar_event jsonb not null default '{}'::jsonb, + summary_markdown text not null default '', + summary_text text not null default '', + transcript_text text not null default '', + transcript_payload jsonb not null default '[]'::jsonb, + url text not null default '', + content_text text not null default '', + content_hash text not null default '', + source_created_at timestamptz, + source_updated_at timestamptz, + raw_payload jsonb not null default '{}'::jsonb, + source_run_id text references granola_sync_runs(run_id) on delete set null, + first_seen_at timestamptz not null default now(), + last_seen_at timestamptz not null default now(), + last_error text not null default '', + updated_at timestamptz not null default now() +); + +create index if not exists idx_granola_sync_notes_source_updated + on granola_sync_notes (source_updated_at desc); + +create index if not exists idx_granola_sync_notes_owner + on granola_sync_notes (owner_email, source_created_at desc); + +create index if not exists idx_granola_sync_notes_access_emails + on granola_sync_notes using gin (access_emails); + +create index if not exists idx_granola_sync_notes_text + on granola_sync_notes + using gin (to_tsvector('english', coalesce(content_text, ''))); + +create table if not exists granola_context_documents ( + document_id text primary key, + note_id text not null references granola_sync_notes(note_id) on delete cascade, + title text not null default '', + body text not null default '', + url text not null default '', + owner_id text not null default '', + owner_email text not null default '', + owner_name text not null default '', + access_emails text[] not null default array[]::text[], + attendee_labels text[] not null default array[]::text[], + occurred_at timestamptz, + source_updated_at timestamptz, + content_hash text not null default '', + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + unique (note_id), + check (document_id <> ''), + check (note_id <> '') +); + +create index if not exists idx_granola_context_documents_note_time + on granola_context_documents (note_id, occurred_at desc); + +create index if not exists idx_granola_context_documents_owner_time + on granola_context_documents (owner_email, occurred_at desc); + +create index if not exists idx_granola_context_documents_access_emails + on granola_context_documents using gin (access_emails); + +create index if not exists idx_granola_context_documents_metadata + on granola_context_documents using gin (metadata); + +drop index if exists idx_granola_context_documents_bm25; + +create index idx_granola_context_documents_bm25 + on granola_context_documents + using bm25 ( + document_id, + note_id, + title, + body, + url, + owner_id, + owner_email, + owner_name, + occurred_at, + source_updated_at, + metadata + ) + with ( + key_field = 'document_id', + text_fields = '{ + "document_id": { + "tokenizer": {"type": "keyword"} + }, + "note_id": { + "tokenizer": {"type": "keyword"} + }, + "owner_id": { + "tokenizer": {"type": "keyword"} + }, + "owner_email": { + "tokenizer": {"type": "keyword"} + } + }' + ); + +create table if not exists granola_sync_checkpoints ( + scope_id text primary key, + watermark_time timestamptz, + last_run_id text references granola_sync_runs(run_id) on delete set null, + last_success_at timestamptz, + last_error text not null default '', + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +do $$ +declare + role_name text; +begin + foreach role_name in array array[ + 'centaur_slack_reader', + 'centaur_slack_admin', + 'centaur_readonly' + ] loop + if exists (select 1 from pg_roles where rolname = role_name) then + execute format( + 'grant select on %s to %I', + 'granola_sync_runs, granola_sync_notes, granola_context_documents, granola_sync_checkpoints', + role_name + ); + end if; + end loop; +end $$; + +alter table granola_sync_runs enable row level security; +alter table granola_sync_notes enable row level security; +alter table granola_context_documents enable row level security; +alter table granola_sync_checkpoints enable row level security; + +create or replace function centaur_current_slack_user_email() +returns text +language sql +stable +security definer +set search_path = public +as $$ + select coalesce( + lower(nullif(current_setting('centaur.user_email', true), '')), + ( + select lower(nullif(coalesce( + users.raw_payload #>> '{profile,email}', + users.raw_payload ->> 'email' + ), '')) + from slack_sync_users users + where users.team_id = centaur_current_slack_team_id() + and users.user_id = centaur_current_slack_user_id() + limit 1 + ) + ) +$$; + +create or replace function centaur_granola_current_user_can_read( + p_access_emails text[] +) +returns boolean +language sql +stable +as $$ + select coalesce( + centaur_current_slack_user_email() = any(coalesce(p_access_emails, array[]::text[])), + false + ) +$$; + +do $$ +declare + role_name text; +begin + foreach role_name in array array[ + 'centaur_slack_reader', + 'centaur_slack_admin', + 'centaur_readonly' + ] loop + if exists (select 1 from pg_roles where rolname = role_name) then + execute format( + 'grant execute on function centaur_current_slack_user_email() to %I', + role_name + ); + execute format( + 'grant execute on function centaur_granola_current_user_can_read(text[]) to %I', + role_name + ); + end if; + end loop; +end $$; + +drop policy if exists centaur_granola_runs_admin_select on granola_sync_runs; +drop policy if exists centaur_granola_runs_reader_select on granola_sync_runs; +create policy centaur_granola_runs_reader_select + on granola_sync_runs for select to centaur_slack_reader + using (false); +drop policy if exists centaur_readonly_granola_sync_runs_select on granola_sync_runs; +create policy centaur_readonly_granola_sync_runs_select + on granola_sync_runs for select to centaur_readonly using (false); + +drop policy if exists centaur_granola_notes_admin_select on granola_sync_notes; +drop policy if exists centaur_granola_notes_reader_select on granola_sync_notes; +create policy centaur_granola_notes_reader_select + on granola_sync_notes for select to centaur_slack_reader + using (centaur_granola_current_user_can_read(access_emails)); +drop policy if exists centaur_readonly_granola_sync_notes_select on granola_sync_notes; +create policy centaur_readonly_granola_sync_notes_select + on granola_sync_notes for select to centaur_readonly using (false); + +drop policy if exists centaur_granola_context_documents_admin_select + on granola_context_documents; +drop policy if exists centaur_granola_context_documents_reader_select + on granola_context_documents; +create policy centaur_granola_context_documents_reader_select + on granola_context_documents for select to centaur_slack_reader + using (centaur_granola_current_user_can_read(access_emails)); +drop policy if exists centaur_readonly_granola_context_documents_select + on granola_context_documents; +create policy centaur_readonly_granola_context_documents_select + on granola_context_documents for select to centaur_readonly using (false); + +drop policy if exists centaur_granola_checkpoints_admin_select + on granola_sync_checkpoints; +drop policy if exists centaur_granola_checkpoints_reader_select + on granola_sync_checkpoints; +create policy centaur_granola_checkpoints_reader_select + on granola_sync_checkpoints for select to centaur_slack_reader + using (false); +drop policy if exists centaur_readonly_granola_sync_checkpoints_select + on granola_sync_checkpoints; +create policy centaur_readonly_granola_sync_checkpoints_select + on granola_sync_checkpoints for select to centaur_readonly using (false); + +do $$ +begin + if exists (select 1 from pg_roles where rolname = 'centaur_slack_admin') then + create policy centaur_granola_runs_admin_select + on granola_sync_runs for select to centaur_slack_admin using (true); + create policy centaur_granola_notes_admin_select + on granola_sync_notes for select to centaur_slack_admin using (true); + create policy centaur_granola_context_documents_admin_select + on granola_context_documents for select to centaur_slack_admin using (true); + create policy centaur_granola_checkpoints_admin_select + on granola_sync_checkpoints for select to centaur_slack_admin using (true); + end if; +end $$; diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0041_attio_sync_tables.sql b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0041_attio_sync_tables.sql new file mode 100644 index 000000000..442a374fa --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0041_attio_sync_tables.sql @@ -0,0 +1,131 @@ +create table if not exists attio_sync_runs ( + run_id text primary key, + workflow_run_id text, + mode text not null default 'incremental', + status text not null, + scopes_requested jsonb not null default '[]'::jsonb, + scopes_synced jsonb not null default '[]'::jsonb, + scopes_failed jsonb not null default '[]'::jsonb, + meetings_seen integer not null default 0, + meetings_upserted integer not null default 0, + call_recordings_seen integer not null default 0, + transcripts_upserted integer not null default 0, + started_at timestamptz not null default now(), + finished_at timestamptz, + error_text text not null default '', + metadata jsonb not null default '{}'::jsonb +); + +create index if not exists idx_attio_sync_runs_started + on attio_sync_runs (started_at desc); + +create table if not exists attio_sync_meetings ( + meeting_id text primary key, + title text not null default '', + description text not null default '', + url text not null default '', + linked_records jsonb not null default '[]'::jsonb, + participants jsonb not null default '[]'::jsonb, + organizer_id text not null default '', + organizer_name text not null default '', + organizer_email text not null default '', + call_recording_ids jsonb not null default '[]'::jsonb, + transcript_text text not null default '', + transcript_payload jsonb not null default '[]'::jsonb, + content_text text not null default '', + content_hash text not null default '', + started_at timestamptz, + ended_at timestamptz, + source_created_at timestamptz, + source_updated_at timestamptz, + raw_payload jsonb not null default '{}'::jsonb, + source_run_id text references attio_sync_runs(run_id) on delete set null, + first_seen_at timestamptz not null default now(), + last_seen_at timestamptz not null default now(), + last_error text not null default '', + updated_at timestamptz not null default now() +); + +create index if not exists idx_attio_sync_meetings_source_updated + on attio_sync_meetings (source_updated_at desc); + +create index if not exists idx_attio_sync_meetings_time + on attio_sync_meetings (started_at desc); + +create index if not exists idx_attio_sync_meetings_text + on attio_sync_meetings + using gin (to_tsvector('english', coalesce(content_text, ''))); + +create table if not exists attio_sync_checkpoints ( + scope_id text primary key, + watermark_time timestamptz, + last_run_id text references attio_sync_runs(run_id) on delete set null, + last_success_at timestamptz, + last_error text not null default '', + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +do $$ +declare + role_name text; +begin + foreach role_name in array array[ + 'centaur_slack_reader', + 'centaur_slack_admin', + 'centaur_readonly' + ] loop + if exists (select 1 from pg_roles where rolname = role_name) then + execute format( + 'grant select on %s to %I', + 'attio_sync_runs, attio_sync_meetings, attio_sync_checkpoints', + role_name + ); + end if; + end loop; +end $$; + +alter table attio_sync_runs enable row level security; +alter table attio_sync_meetings enable row level security; +alter table attio_sync_checkpoints enable row level security; + +drop policy if exists centaur_attio_runs_admin_select on attio_sync_runs; +drop policy if exists centaur_attio_runs_reader_select on attio_sync_runs; +create policy centaur_attio_runs_reader_select + on attio_sync_runs for select to centaur_slack_reader + using (false); +drop policy if exists centaur_readonly_attio_sync_runs_select on attio_sync_runs; +create policy centaur_readonly_attio_sync_runs_select + on attio_sync_runs for select to centaur_readonly using (true); + +drop policy if exists centaur_attio_meetings_admin_select on attio_sync_meetings; +drop policy if exists centaur_attio_meetings_reader_select on attio_sync_meetings; +create policy centaur_attio_meetings_reader_select + on attio_sync_meetings for select to centaur_slack_reader + using (false); +drop policy if exists centaur_readonly_attio_sync_meetings_select + on attio_sync_meetings; +create policy centaur_readonly_attio_sync_meetings_select + on attio_sync_meetings for select to centaur_readonly using (true); + +drop policy if exists centaur_attio_checkpoints_admin_select on attio_sync_checkpoints; +drop policy if exists centaur_attio_checkpoints_reader_select on attio_sync_checkpoints; +create policy centaur_attio_checkpoints_reader_select + on attio_sync_checkpoints for select to centaur_slack_reader + using (false); +drop policy if exists centaur_readonly_attio_sync_checkpoints_select + on attio_sync_checkpoints; +create policy centaur_readonly_attio_sync_checkpoints_select + on attio_sync_checkpoints for select to centaur_readonly using (true); + +do $$ +begin + if exists (select 1 from pg_roles where rolname = 'centaur_slack_admin') then + create policy centaur_attio_runs_admin_select + on attio_sync_runs for select to centaur_slack_admin using (true); + create policy centaur_attio_meetings_admin_select + on attio_sync_meetings for select to centaur_slack_admin using (true); + create policy centaur_attio_checkpoints_admin_select + on attio_sync_checkpoints for select to centaur_slack_admin using (true); + end if; +end $$; diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0042_centaur_readonly_slack_dm_rls.sql b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0042_centaur_readonly_slack_dm_rls.sql new file mode 100644 index 000000000..4d0cf6dde --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0042_centaur_readonly_slack_dm_rls.sql @@ -0,0 +1,139 @@ +-- Keep centaur_readonly useful for public channel context while allowing a +-- principal that carries Slack identity settings to see only its own DMs. + +drop policy if exists centaur_readonly_slack_dm_sync_conversations_select + on slack_dm_sync_conversations; +create policy centaur_readonly_slack_dm_sync_conversations_select + on slack_dm_sync_conversations + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_dm_sync_conversation_members members + where members.home_team_id = slack_dm_sync_conversations.home_team_id + and members.home_team_id = centaur_current_slack_team_id() + and members.conversation_id = slack_dm_sync_conversations.conversation_id + and members.user_id = centaur_current_slack_user_id() + and members.is_current_member + ) + ); + +drop policy if exists centaur_readonly_slack_dm_sync_conversation_members_select + on slack_dm_sync_conversation_members; +create policy centaur_readonly_slack_dm_sync_conversation_members_select + on slack_dm_sync_conversation_members + for select + to centaur_readonly + using ( + home_team_id = centaur_current_slack_team_id() + and user_id = centaur_current_slack_user_id() + and is_current_member + ); + +drop policy if exists centaur_readonly_slack_dm_sync_messages_select + on slack_dm_sync_messages; +create policy centaur_readonly_slack_dm_sync_messages_select + on slack_dm_sync_messages + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_dm_sync_conversation_members members + where members.home_team_id = slack_dm_sync_messages.home_team_id + and members.home_team_id = centaur_current_slack_team_id() + and members.conversation_id = slack_dm_sync_messages.conversation_id + and members.user_id = centaur_current_slack_user_id() + and members.is_current_member + ) + ); + +drop policy if exists centaur_readonly_slack_dm_sync_message_attachments_select + on slack_dm_sync_message_attachments; +create policy centaur_readonly_slack_dm_sync_message_attachments_select + on slack_dm_sync_message_attachments + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_dm_sync_conversation_members members + where members.home_team_id = slack_dm_sync_message_attachments.home_team_id + and members.home_team_id = centaur_current_slack_team_id() + and members.conversation_id = slack_dm_sync_message_attachments.conversation_id + and members.user_id = centaur_current_slack_user_id() + and members.is_current_member + ) + ); + +drop policy if exists centaur_readonly_slack_dm_sync_checkpoints_select + on slack_dm_sync_checkpoints; +create policy centaur_readonly_slack_dm_sync_checkpoints_select + on slack_dm_sync_checkpoints + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_dm_sync_conversation_members members + where members.home_team_id = slack_dm_sync_checkpoints.home_team_id + and members.home_team_id = centaur_current_slack_team_id() + and members.conversation_id = slack_dm_sync_checkpoints.conversation_id + and members.user_id = centaur_current_slack_user_id() + and members.is_current_member + ) + ); + +-- Operational rows never belong in user-visible company context. +drop policy if exists centaur_readonly_slack_dm_sync_runs_select + on slack_dm_sync_runs; +create policy centaur_readonly_slack_dm_sync_runs_select + on slack_dm_sync_runs + for select + to centaur_readonly + using (false); + +drop policy if exists centaur_readonly_slack_dm_sync_backfill_jobs_select + on slack_dm_sync_backfill_jobs; +create policy centaur_readonly_slack_dm_sync_backfill_jobs_select + on slack_dm_sync_backfill_jobs + for select + to centaur_readonly + using (false); + +drop policy if exists centaur_readonly_slack_dm_context_documents_select + on slack_dm_context_documents; +create policy centaur_readonly_slack_dm_context_documents_select + on slack_dm_context_documents + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_dm_sync_conversation_members members + where members.home_team_id = slack_dm_context_documents.home_team_id + and members.home_team_id = centaur_current_slack_team_id() + and members.conversation_id = slack_dm_context_documents.conversation_id + and members.user_id = centaur_current_slack_user_id() + and members.is_current_member + ) + ); + +drop policy if exists centaur_readonly_slack_dm_conversation_context_documents_select + on slack_dm_conversation_context_documents; +create policy centaur_readonly_slack_dm_conversation_context_documents_select + on slack_dm_conversation_context_documents + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_dm_sync_conversation_members members + where members.home_team_id = slack_dm_conversation_context_documents.home_team_id + and members.home_team_id = centaur_current_slack_team_id() + and members.conversation_id = slack_dm_conversation_context_documents.conversation_id + and members.user_id = centaur_current_slack_user_id() + and members.is_current_member + ) + ); diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0043_session_sandbox_content_revision.sql b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0043_session_sandbox_content_revision.sql new file mode 100644 index 000000000..74bf6f4d0 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/0043_session_sandbox_content_revision.sql @@ -0,0 +1,21 @@ +alter table sessions + add column if not exists sandbox_content_revision text; + +comment on column sessions.sandbox_content_revision is + 'Assignment-bound digest of the immutable deployment boot-content generation and sandbox ID; NULL on legacy assignments.'; + +create or replace view centaur_readonly_sessions as +select + thread_key, + sandbox_id, + harness_type, + harness_thread_id, + persona_id, + status, + metadata ->> 'source' as source, + metadata ->> 'platform' as platform, + metadata ->> 'thread_id' as external_thread_id, + created_at, + updated_at, + sandbox_content_revision +from sessions; diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/README.md b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/README.md new file mode 100644 index 000000000..fe00327d0 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/README.md @@ -0,0 +1,16 @@ +# Forward migration fixtures + +These provisional test fixtures are migrations 0033–0043 from the reviewed +forward candidate. The final forward commit has not yet been frozen: replace +the sole value in `.github/rollback-bridge-reviewed-forward-commit` only after +re-copying and verifying these bytes against that exact commit. CI, the manual +publisher, and the Rust rehearsal all read that same file. The same files are vendored into +`centaur-session-sqlx/migrations`, so SQLx embeds the forward ledger through +0043. The emergency bridge still runs with +`RUN_MIGRATIONS=false`: schema ownership stays with the already-completed +forward rollout. + +`SHA256SUMS` pins the copied source, and CI compares every byte to the frozen +central commit. Update the fixtures, central pin, and checksums together if the +reviewed forward migration set changes. CI fails closed while the pin is not a +lowercase 40-character commit SHA. diff --git a/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/SHA256SUMS b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/SHA256SUMS new file mode 100644 index 000000000..b97b6a278 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/fixtures/forward_migrations/SHA256SUMS @@ -0,0 +1,11 @@ +2a62ca1e986f18823a71bf43821704fdeff2e4b9944960944e264cb50f528a49 0033_session_title.sql +d2362fcf77283ca6e4f4f78101aeaca66d89c0b513700ea440126c271add431f 0034_session_sandbox_activity.sql +44caf1809f16c631e39fda9cfbd846ff678072954afaef68055d44cd5713fbc6 0035_session_execution_stdout_owner.sql +dc439073ba2b463bd65aa040254876d6aa212634944a72b2cba4a81672b344ef 0036_session_sandbox_api_server_capability.sql +03a4b1b6b0273d884d0fad3c209e76419220ac7f9921ead17816e4f1a3f0a552 0037_readonly_all_workflow_queues.sql +b620f8b1d7fbf29c4c4eedd14e6dae27d0134f6783d39b0d7772ac5400a80df8 0038_session_sandbox_repo_cache_access.sql +0291d1b6f7670f77c606fb0f6924050f903623914aed8993b2dc58d31210bc20 0039_slack_private_channels.sql +2a63ba12978c77889786ae098ecf840aa735354e8f30ec653a03c397cf84dd09 0040_granola_sync_tables.sql +21bb6ae387ffa14185b1c9c0dbf1344c5f4caf6b0c66916eb8d443b0ad4e9ae8 0041_attio_sync_tables.sql +915d21f7a78b4201c67a2e0c72491ad4245dcc052a968209f68783c6399fdacf 0042_centaur_readonly_slack_dm_rls.sql +1d8cc24dda26a7a511e964fb51f1366ee4a6cad0dc0a5ad132dccfea92c63d39 0043_session_sandbox_content_revision.sql diff --git a/services/api-rs/crates/centaur-api-server/tests/rollback_bridge_forward_schema.rs b/services/api-rs/crates/centaur-api-server/tests/rollback_bridge_forward_schema.rs new file mode 100644 index 000000000..20f77a9a8 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/rollback_bridge_forward_schema.rs @@ -0,0 +1,1488 @@ +use std::{ + collections::BTreeMap, + env, + net::TcpListener, + path::Path, + process::{Child, Command, Stdio}, + time::Duration, +}; + +use centaur_session_core::{HarnessType, SandboxCapabilities, ThreadKey}; +use centaur_session_sqlx::PgSessionStore; +use reqwest::{Client, StatusCode}; +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; +use sqlx::PgPool; +use tokio::sync::Mutex; +use tokio::time::{Instant, sleep}; +use uuid::Uuid; + +const DATABASE_ENV: &str = "ROLLBACK_BRIDGE_FORWARD_TEST_DATABASE_URL"; +const FORWARD_BINARY_ENV: &str = "ROLLBACK_BRIDGE_REHEARSAL_FORWARD_BIN"; +const FORWARD_WORKFLOW_HOST_ENV: &str = "ROLLBACK_BRIDGE_REHEARSAL_FORWARD_WORKFLOW_HOST"; +const FORWARD_WORKDIR_ENV: &str = "ROLLBACK_BRIDGE_REHEARSAL_FORWARD_WORKDIR"; +const CONTROL_KEY: &str = "rollback-preservation-control-key"; +const WORKFLOW_KEY: &str = "rollback-preservation-workflow-key"; +const WEBHOOK_KEY: &str = "rollback-preservation-webhook-key"; +const FORWARD_COMMIT_FILE: &str = + include_str!("../../../../../.github/rollback-bridge-reviewed-forward-commit"); +static DATABASE_REHEARSAL_LOCK: Mutex<()> = Mutex::const_new(()); +const QUEUES: [&str; 5] = [ + "centaur_workflows", + "centaur_workflows_slack_live", + "centaur_workflows_etl", + "centaur_workflows_etl_backfill", + "centaur_workflow_schedules", +]; +const FORWARD_MIGRATIONS: [(&str, &str); 11] = [ + ( + "0033_session_title.sql", + include_str!("fixtures/forward_migrations/0033_session_title.sql"), + ), + ( + "0034_session_sandbox_activity.sql", + include_str!("fixtures/forward_migrations/0034_session_sandbox_activity.sql"), + ), + ( + "0035_session_execution_stdout_owner.sql", + include_str!("fixtures/forward_migrations/0035_session_execution_stdout_owner.sql"), + ), + ( + "0036_session_sandbox_api_server_capability.sql", + include_str!("fixtures/forward_migrations/0036_session_sandbox_api_server_capability.sql"), + ), + ( + "0037_readonly_all_workflow_queues.sql", + include_str!("fixtures/forward_migrations/0037_readonly_all_workflow_queues.sql"), + ), + ( + "0038_session_sandbox_repo_cache_access.sql", + include_str!("fixtures/forward_migrations/0038_session_sandbox_repo_cache_access.sql"), + ), + ( + "0039_slack_private_channels.sql", + include_str!("fixtures/forward_migrations/0039_slack_private_channels.sql"), + ), + ( + "0040_granola_sync_tables.sql", + include_str!("fixtures/forward_migrations/0040_granola_sync_tables.sql"), + ), + ( + "0041_attio_sync_tables.sql", + include_str!("fixtures/forward_migrations/0041_attio_sync_tables.sql"), + ), + ( + "0042_centaur_readonly_slack_dm_rls.sql", + include_str!("fixtures/forward_migrations/0042_centaur_readonly_slack_dm_rls.sql"), + ), + ( + "0043_session_sandbox_content_revision.sql", + include_str!("fixtures/forward_migrations/0043_session_sandbox_content_revision.sql"), + ), +]; +const EMBEDDED_FORWARD_MIGRATIONS: [(&str, &str); 11] = [ + ( + "0033_session_title.sql", + include_str!("../../centaur-session-sqlx/migrations/0033_session_title.sql"), + ), + ( + "0034_session_sandbox_activity.sql", + include_str!("../../centaur-session-sqlx/migrations/0034_session_sandbox_activity.sql"), + ), + ( + "0035_session_execution_stdout_owner.sql", + include_str!( + "../../centaur-session-sqlx/migrations/0035_session_execution_stdout_owner.sql" + ), + ), + ( + "0036_session_sandbox_api_server_capability.sql", + include_str!( + "../../centaur-session-sqlx/migrations/0036_session_sandbox_api_server_capability.sql" + ), + ), + ( + "0037_readonly_all_workflow_queues.sql", + include_str!("../../centaur-session-sqlx/migrations/0037_readonly_all_workflow_queues.sql"), + ), + ( + "0038_session_sandbox_repo_cache_access.sql", + include_str!( + "../../centaur-session-sqlx/migrations/0038_session_sandbox_repo_cache_access.sql" + ), + ), + ( + "0039_slack_private_channels.sql", + include_str!("../../centaur-session-sqlx/migrations/0039_slack_private_channels.sql"), + ), + ( + "0040_granola_sync_tables.sql", + include_str!("../../centaur-session-sqlx/migrations/0040_granola_sync_tables.sql"), + ), + ( + "0041_attio_sync_tables.sql", + include_str!("../../centaur-session-sqlx/migrations/0041_attio_sync_tables.sql"), + ), + ( + "0042_centaur_readonly_slack_dm_rls.sql", + include_str!( + "../../centaur-session-sqlx/migrations/0042_centaur_readonly_slack_dm_rls.sql" + ), + ), + ( + "0043_session_sandbox_content_revision.sql", + include_str!( + "../../centaur-session-sqlx/migrations/0043_session_sandbox_content_revision.sql" + ), + ), +]; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn paused_bridge_preserves_forward_schema_workflows_and_reassignment_fence() { + let _database_guard = DATABASE_REHEARSAL_LOCK.lock().await; + let Ok(admin_url) = env::var(DATABASE_ENV) else { + eprintln!("skipping: {DATABASE_ENV} not set (required by rollback preservation CI)"); + return; + }; + assert_fixture_provenance(); + + let database_name = format!("rollback_bridge_{}", Uuid::new_v4().simple()); + let admin_pool = PgPool::connect(&admin_url) + .await + .expect("connect forward-test admin database"); + sqlx::query(&format!("create database {database_name}")) + .execute(&admin_pool) + .await + .expect("create disposable forward database"); + let database_url = database_url_with_name(&admin_url, &database_name); + + let store = PgSessionStore::connect(&database_url) + .await + .expect("connect disposable forward database"); + store + .run_migrations() + .await + .expect("apply reviewed forward migration ledger 0001-0043"); + let pool = store.pool().clone(); + let (migration_count, migration_max) = + sqlx::query_as::<_, (i64, i64)>("select count(*), max(version) from _sqlx_migrations") + .fetch_one(&pool) + .await + .expect("read embedded migration ledger"); + assert_eq!((migration_count, migration_max), (43, 43)); + let migrated_queues = sqlx::query_scalar::<_, String>( + "select queue_name from absurd.list_queues() order by queue_name", + ) + .fetch_all(&pool) + .await + .expect("list forward migration-created queues"); + assert_eq!(migrated_queues.len(), QUEUES.len()); + assert!( + QUEUES + .iter() + .all(|queue| migrated_queues.iter().any(|value| value == queue)) + ); + let seeded = seed_representative_workflow_rows(&pool).await; + let reassigned = seed_forward_assignment_and_simulate_bridge_reassignment(&store, &pool).await; + let before = canonical_forward_snapshot(&pool).await; + assert_seeded_states(&pool).await; + assert_reassignment_requires_forward_replacement(&pool, &reassigned).await; + + let port = unused_local_port(); + let workflow_dir = env::temp_dir().join(format!( + "centaur-rollback-preservation-{}", + Uuid::new_v4().simple() + )); + std::fs::create_dir_all(&workflow_dir).expect("create workflow fixture directory"); + std::fs::write( + workflow_dir.join("rollback_preservation_sentinel.py"), + r#"WORKFLOW_NAME = "rollback_preservation_sentinel" + +async def handler(params, ctx): + return {"ok": True} +"#, + ) + .expect("write workflow discovery sentinel"); + let mut server = BridgeProcess::spawn(&database_url, port, &workflow_dir); + let client = Client::new(); + let base_url = format!("http://127.0.0.1:{port}"); + wait_for_ready(&client, &base_url, &mut server).await; + + exercise_read_and_mutation_lanes(&client, &base_url, &seeded.running_run_id).await; + // This exceeds both test-configured one-second reconcile/reaper intervals. + // Any accidentally started worker or metadata reaper has time to mutate + // the pending, expired-running, sleeping, or forward-only handler rows. + sleep(Duration::from_secs(3)).await; + server.stop(); + + let after = canonical_forward_snapshot(&pool).await; + assert_eq!( + after, before, + "paused rollback bridge changed forward sessions, workflows, or migration ledger" + ); + assert_seeded_states(&pool).await; + assert_reassignment_requires_forward_replacement(&pool, &reassigned).await; + + pool.close().await; + drop(store); + sqlx::query(&format!("drop database {database_name} with (force)")) + .execute(&admin_pool) + .await + .expect("drop disposable forward database"); + admin_pool.close().await; + let _ = std::fs::remove_dir_all(workflow_dir); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn paused_bridge_missing_forward_queue_fails_without_schema_mutation() { + let _database_guard = DATABASE_REHEARSAL_LOCK.lock().await; + let Ok(admin_url) = env::var(DATABASE_ENV) else { + eprintln!("skipping: {DATABASE_ENV} not set (required by rollback preservation CI)"); + return; + }; + + let database_name = format!("rollback_missing_queue_{}", Uuid::new_v4().simple()); + let admin_pool = PgPool::connect(&admin_url) + .await + .expect("connect missing-queue admin database"); + sqlx::query(&format!("create database {database_name}")) + .execute(&admin_pool) + .await + .expect("create missing-queue database"); + let database_url = database_url_with_name(&admin_url, &database_name); + let store = PgSessionStore::connect(&database_url) + .await + .expect("connect missing-queue database"); + store + .run_migrations() + .await + .expect("apply embedded forward migration ledger"); + let pool = store.pool().clone(); + + let migrated_queues = sqlx::query_scalar::<_, String>( + "select queue_name from absurd.list_queues() order by queue_name", + ) + .fetch_all(&pool) + .await + .expect("list migration-created workflow queues"); + assert_eq!( + migrated_queues.len(), + QUEUES.len(), + "forward migrations must create all required queues: {migrated_queues:?}" + ); + sqlx::query("select absurd.drop_queue($1)") + .bind(QUEUES[4]) + .execute(&pool) + .await + .expect("remove schedule queue for negative startup fixture"); + let queues = sqlx::query_scalar::<_, String>( + "select queue_name from absurd.list_queues() order by queue_name", + ) + .fetch_all(&pool) + .await + .expect("list workflow queues after negative-fixture removal"); + assert_eq!( + queues.len(), + 4, + "unexpected migration-created queues: {queues:?}" + ); + assert!( + !queues.iter().any(|queue| queue == QUEUES[4]), + "schedule queue must be absent in the negative fixture" + ); + let before = rollback_schema_identity_snapshot(&pool).await; + + let workflow_dir = env::temp_dir().join(format!( + "centaur-rollback-missing-queue-{}", + Uuid::new_v4().simple() + )); + std::fs::create_dir_all(&workflow_dir).expect("create missing-queue workflow directory"); + let port = unused_local_port(); + let mut command = bridge_command(&database_url, port, &workflow_dir); + command.stdout(Stdio::null()).stderr(Stdio::piped()); + let mut child = command + .spawn() + .expect("run bridge against missing forward queue"); + let client = Client::new(); + let ready_url = format!("http://127.0.0.1:{port}/readyz"); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if child + .try_wait() + .expect("inspect missing-queue bridge process") + .is_some() + { + break; + } + if let Ok(response) = client.get(&ready_url).send().await + && response.status() == StatusCode::OK + { + let _ = child.kill(); + let output = child + .wait_with_output() + .expect("collect unexpectedly ready bridge output"); + panic!( + "bridge became ready after creating the missing queue: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + if Instant::now() >= deadline { + let _ = child.kill(); + let output = child + .wait_with_output() + .expect("collect timed-out missing-queue bridge output"); + panic!( + "bridge did not fail missing-queue startup within 10 seconds: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + sleep(Duration::from_millis(100)).await; + } + let output = child + .wait_with_output() + .expect("collect missing-queue bridge output"); + assert!(!output.status.success(), "bridge unexpectedly became ready"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("rollback bridge requires all forward absurd queues to exist") + && stderr.contains(QUEUES[4]), + "unexpected missing-queue startup failure: {stderr}" + ); + + let after = rollback_schema_identity_snapshot(&pool).await; + assert_eq!( + after, before, + "bridge startup created or changed schema objects while rejecting a missing queue" + ); + + pool.close().await; + drop(store); + sqlx::query(&format!("drop database {database_name} with (force)")) + .execute(&admin_pool) + .await + .expect("drop missing-queue database"); + admin_pool.close().await; + let _ = std::fs::remove_dir_all(workflow_dir); +} + +#[derive(Clone)] +struct RehearsalTask { + queue: &'static str, + task_id: Uuid, + run_id: Uuid, +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn reviewed_forward_bridge_reviewed_forward_rehearsal() { + let _database_guard = DATABASE_REHEARSAL_LOCK.lock().await; + let Ok(admin_url) = env::var(DATABASE_ENV) else { + eprintln!("skipping: {DATABASE_ENV} not set"); + return; + }; + let Ok(forward_binary) = env::var(FORWARD_BINARY_ENV) else { + eprintln!("skipping: {FORWARD_BINARY_ENV} not set (explicit cross-version CI sets it)"); + return; + }; + let forward_workflow_host = env::var(FORWARD_WORKFLOW_HOST_ENV).unwrap_or_else(|_| { + panic!("{FORWARD_WORKFLOW_HOST_ENV} is required with {FORWARD_BINARY_ENV}") + }); + let forward_workdir = env::var(FORWARD_WORKDIR_ENV) + .unwrap_or_else(|_| panic!("{FORWARD_WORKDIR_ENV} is required with {FORWARD_BINARY_ENV}")); + assert_fixture_provenance(); + + let database_name = format!("rollback_rehearsal_{}", Uuid::new_v4().simple()); + let admin_pool = PgPool::connect(&admin_url) + .await + .expect("connect cross-version admin database"); + sqlx::query(&format!("create database {database_name}")) + .execute(&admin_pool) + .await + .expect("create cross-version database"); + let database_url = database_url_with_name(&admin_url, &database_name); + let store = PgSessionStore::connect(&database_url) + .await + .expect("connect cross-version database"); + store + .run_migrations() + .await + .expect("apply embedded reviewed forward migrations"); + let pool = store.pool().clone(); + + let workflow_dir = env::temp_dir().join(format!( + "centaur-cross-version-workflows-{}", + Uuid::new_v4().simple() + )); + write_cross_version_workflows(&workflow_dir); + let client = Client::new(); + + // Phase 1: the pinned reviewed forward binary creates real durable tasks. + let prepare_port = unused_local_port(); + let mut prepare = ForwardProcess::spawn(ForwardProcessConfig { + binary: &forward_binary, + workdir: &forward_workdir, + workflow_host: &forward_workflow_host, + database_url: &database_url, + workflow_dir: &workflow_dir, + port: prepare_port, + phase: "prepare", + }); + let prepare_url = format!("http://127.0.0.1:{prepare_port}"); + wait_for_forward_ready(&client, &prepare_url, &mut prepare).await; + let running = create_forward_workflow_run( + &client, + &prepare_url, + "rollback_running", + "rehearsal-running", + json!({"kind": "expired-claim", "payload": {"must": "survive"}}), + ) + .await; + wait_for_task_state(&pool, running.queue, running.task_id, "running").await; + let pending = create_forward_workflow_run( + &client, + &prepare_url, + "rollback_pending", + "rehearsal-pending", + json!({"kind": "pending", "payload": [1, 2, 3]}), + ) + .await; + let waiting = create_forward_workflow_run( + &client, + &prepare_url, + "rollback_waiting", + "rehearsal-waiting", + json!({"kind": "await-event", "payload": {"opaque": true}}), + ) + .await; + wait_for_task_state(&pool, pending.queue, pending.task_id, "pending").await; + wait_for_task_state(&pool, waiting.queue, waiting.task_id, "pending").await; + let sleeping = create_forward_workflow_run( + &client, + &prepare_url, + "slack_sync", + "rehearsal-sleeping", + json!({"kind": "sleeping", "cursor": 17}), + ) + .await; + wait_for_task_state(&pool, sleeping.queue, sleeping.task_id, "sleeping").await; + prepare.stop(); + + expire_running_claim(&pool, &running).await; + convert_pending_to_event_wait(&pool, &waiting).await; + customize_retry_and_cancellation_contract(&pool, [&running, &pending, &waiting, &sleeping]) + .await; + let reassigned = seed_forward_assignment_and_simulate_bridge_reassignment(&store, &pool).await; + let before_bridge = canonical_cross_version_snapshot(&pool).await; + let immutable_task_contract = + task_contract_snapshot(&pool, [&running, &pending, &waiting, &sleeping]).await; + + // Phase 2: the rollback bridge starts with workflow mutation hard-paused. + let bridge_port = unused_local_port(); + let mut bridge = BridgeProcess::spawn(&database_url, bridge_port, &workflow_dir); + let bridge_url = format!("http://127.0.0.1:{bridge_port}"); + wait_for_ready(&client, &bridge_url, &mut bridge).await; + exercise_read_and_mutation_lanes(&client, &bridge_url, &running.run_id.to_string()).await; + sleep(Duration::from_secs(3)).await; + bridge.stop(); + let after_bridge = canonical_cross_version_snapshot(&pool).await; + assert_eq!( + after_bridge, before_bridge, + "rollback bridge changed durable forward workflow/session state" + ); + assert_reassignment_requires_forward_replacement(&pool, &reassigned).await; + + // Phase 3: the exact reviewed forward binary resumes every durable shape. + let resume_port = unused_local_port(); + let mut resume = ForwardProcess::spawn(ForwardProcessConfig { + binary: &forward_binary, + workdir: &forward_workdir, + workflow_host: &forward_workflow_host, + database_url: &database_url, + workflow_dir: &workflow_dir, + port: resume_port, + phase: "resume", + }); + let resume_url = format!("http://127.0.0.1:{resume_port}"); + wait_for_forward_ready(&client, &resume_url, &mut resume).await; + emit_forward_workflow_event(&client, &resume_url, "rollback.rehearsal.resume").await; + execute_reassigned_session(&client, &resume_url, &reassigned).await; + for task in [&running, &pending, &waiting, &sleeping] { + wait_for_task_state(&pool, task.queue, task.task_id, "completed").await; + } + wait_for_forward_sandbox_replacement(&pool, &reassigned).await; + let after_resume_contract = + task_contract_snapshot(&pool, [&running, &pending, &waiting, &sleeping]).await; + assert_eq!( + after_resume_contract, immutable_task_contract, + "re-forward changed task payload, retry, cancellation, or idempotency contract" + ); + assert_checkpoint_survived_resume(&pool, &sleeping).await; + resume.stop(); + + pool.close().await; + drop(store); + sqlx::query(&format!("drop database {database_name} with (force)")) + .execute(&admin_pool) + .await + .expect("drop cross-version database"); + admin_pool.close().await; + let _ = std::fs::remove_dir_all(workflow_dir); +} + +struct BridgeProcess { + child: Option, +} + +impl BridgeProcess { + fn spawn(database_url: &str, port: u16, workflow_dir: &Path) -> Self { + let mut command = bridge_command(database_url, port, workflow_dir); + command.stdout(Stdio::null()).stderr(Stdio::null()); + Self { + child: Some(command.spawn().expect("start rollback bridge binary")), + } + } + + fn exited(&mut self) -> Option { + self.child + .as_mut() + .expect("bridge child") + .try_wait() + .expect("inspect rollback bridge process") + } + + fn stop(&mut self) { + if let Some(mut child) = self.child.take() { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +fn bridge_command(database_url: &str, port: u16, workflow_dir: &Path) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_centaur-api-server")); + command + .env("BIND_ADDR", format!("127.0.0.1:{port}")) + .env("DATABASE_URL", database_url) + .env("RUN_MIGRATIONS", "false") + .env("CENTAUR_CONTROL_API_KEY", CONTROL_KEY) + .env("CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS", "true") + .env("TRIVY_INTAKE_WEBHOOK_TOKEN", WEBHOOK_KEY) + .env("WORKFLOW_DIRS", workflow_dir) + .env("WORKFLOW_HOST_SANDBOX", "false") + .env("WORKFLOW_REAP_REMOVED_AFTER_TICKS", "1") + .env("WORKFLOW_RECONCILE_INTERVAL_SECS", "1") + .env("RUST_LOG", "warn") + .env_remove("SLACKBOT_API_KEY") + .env_remove("GITHUBBOT_API_KEY") + .env_remove("LINEARBOT_API_KEY") + .env_remove("DISCORDBOT_API_KEY") + .env_remove("TEAMSBOT_API_KEY") + .env_remove("WORKFLOW_API_KEY") + .env_remove("SLACK_FEEDBACK_API_KEY"); + command +} + +impl Drop for BridgeProcess { + fn drop(&mut self) { + self.stop(); + } +} + +struct ForwardProcess { + child: Option, +} + +struct ForwardProcessConfig<'a> { + binary: &'a str, + workdir: &'a str, + workflow_host: &'a str, + database_url: &'a str, + workflow_dir: &'a Path, + port: u16, + phase: &'a str, +} + +impl ForwardProcess { + fn spawn(config: ForwardProcessConfig<'_>) -> Self { + let mut command = Command::new(config.binary); + command + .current_dir(config.workdir) + .env("BIND_ADDR", format!("127.0.0.1:{}", config.port)) + .env("DATABASE_URL", config.database_url) + .env("RUN_MIGRATIONS", "false") + .env("CENTAUR_CONTROL_API_KEY", CONTROL_KEY) + .env("WORKFLOW_API_KEY", WORKFLOW_KEY) + .env("WORKFLOW_DIRS", config.workflow_dir) + .env("WORKFLOW_HOST_SANDBOX", "false") + .env("PYTHON_WORKFLOW_HOST_PATH", config.workflow_host) + .env("PYTHON_WORKFLOW_HOST_PYTHON", "python3") + .env("WORKFLOW_WORKER_CONCURRENCY", "1") + .env("WORKFLOW_ETL_WORKER_CONCURRENCY", "1") + .env("WORKFLOW_REAP_REMOVED_AFTER_TICKS", "0") + .env("WORKFLOW_RECONCILE_INTERVAL_SECS", "0") + .env("SESSION_EXECUTION_ADOPTION_INTERVAL_SECS", "0") + .env("ROLLBACK_REHEARSAL_PHASE", config.phase) + .env("RUST_LOG", "warn") + .env_remove("CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS") + .env_remove("SLACKBOT_API_KEY") + .env_remove("GITHUBBOT_API_KEY") + .env_remove("LINEARBOT_API_KEY") + .env_remove("DISCORDBOT_API_KEY") + .env_remove("TEAMSBOT_API_KEY") + .env_remove("SLACK_FEEDBACK_API_KEY") + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + Self { + child: Some(command.spawn().expect("start reviewed forward binary")), + } + } + + fn exited(&mut self) -> Option { + self.child + .as_mut() + .expect("forward child") + .try_wait() + .expect("inspect forward process") + } + + fn stop(&mut self) { + if let Some(mut child) = self.child.take() { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +impl Drop for ForwardProcess { + fn drop(&mut self) { + self.stop(); + } +} + +fn write_cross_version_workflows(workflow_dir: &Path) { + std::fs::create_dir_all(workflow_dir).expect("create cross-version workflow directory"); + let files = [ + ( + "rollback_running.py", + r#"import asyncio +import os + +WORKFLOW_NAME = "rollback_running" + +async def handler(params, ctx): + if os.environ.get("ROLLBACK_REHEARSAL_PHASE") == "prepare": + await asyncio.sleep(300) + return {"resumed": True, "input": params} +"#, + ), + ( + "rollback_pending.py", + r#"WORKFLOW_NAME = "rollback_pending" + +async def handler(params, ctx): + return {"resumed": True, "input": params} +"#, + ), + ( + "rollback_waiting.py", + r#"WORKFLOW_NAME = "rollback_waiting" + +async def handler(params, ctx): + return {"resumed": True, "input": params} +"#, + ), + ( + "slack_sync.py", + r#"WORKFLOW_NAME = "slack_sync" + +async def handler(params, ctx): + checkpoint = await ctx.step( + "rehearsal_checkpoint", + lambda: {"cursor": 17, "opaque": "preserve-through-rollback"}, + ) + await ctx.sleep("rehearsal_sleep", 2.0) + return {"resumed": True, "checkpoint": checkpoint, "input": params} +"#, + ), + ]; + for (name, source) in files { + std::fs::write(workflow_dir.join(name), source) + .unwrap_or_else(|error| panic!("write {name}: {error}")); + } +} + +async fn wait_for_forward_ready(client: &Client, base_url: &str, server: &mut ForwardProcess) { + let deadline = Instant::now() + Duration::from_secs(45); + let mut last = String::new(); + while Instant::now() < deadline { + if let Some(status) = server.exited() { + panic!("reviewed forward binary exited before readiness: {status}"); + } + match client.get(format!("{base_url}/readyz")).send().await { + Ok(response) if response.status() == StatusCode::OK => return, + Ok(response) => last = format!("readyz returned {}", response.status()), + Err(error) => last = error.to_string(), + } + sleep(Duration::from_millis(100)).await; + } + panic!("reviewed forward binary did not become ready: {last}"); +} + +async fn create_forward_workflow_run( + client: &Client, + base_url: &str, + workflow_name: &str, + idempotency_key: &str, + input: Value, +) -> RehearsalTask { + let response = client + .post(format!("{base_url}/api/workflows/runs")) + .header("Authorization", format!("Bearer {CONTROL_KEY}")) + .json(&json!({ + "workflow_name": workflow_name, + "input": input, + "idempotency_key": idempotency_key, + "max_attempts": 7, + })) + .send() + .await + .unwrap_or_else(|error| panic!("create forward workflow {workflow_name}: {error}")); + let status = response.status(); + let body = response + .json::() + .await + .expect("decode forward workflow create response"); + assert_eq!(status, StatusCode::OK, "create {workflow_name}: {body}"); + let task_id = Uuid::parse_str(body["task_id"].as_str().expect("workflow create task_id")) + .expect("workflow task UUID"); + let run_id = Uuid::parse_str(body["run_id"].as_str().expect("workflow create run_id")) + .expect("workflow run UUID"); + RehearsalTask { + queue: if workflow_name == "slack_sync" { + "centaur_workflows_slack_live" + } else { + "centaur_workflows" + }, + task_id, + run_id, + } +} + +fn task_table(queue: &str) -> &'static str { + match queue { + "centaur_workflows" => "absurd.t_centaur_workflows", + "centaur_workflows_slack_live" => "absurd.t_centaur_workflows_slack_live", + "centaur_workflows_etl" => "absurd.t_centaur_workflows_etl", + other => panic!("unsupported rehearsal queue {other}"), + } +} + +fn run_table(queue: &str) -> &'static str { + match queue { + "centaur_workflows" => "absurd.r_centaur_workflows", + "centaur_workflows_slack_live" => "absurd.r_centaur_workflows_slack_live", + "centaur_workflows_etl" => "absurd.r_centaur_workflows_etl", + other => panic!("unsupported rehearsal queue {other}"), + } +} + +async fn wait_for_task_state(pool: &PgPool, queue: &str, task_id: Uuid, expected: &str) { + let deadline = Instant::now() + Duration::from_secs(45); + let table = task_table(queue); + let mut last = String::new(); + while Instant::now() < deadline { + match sqlx::query_scalar::<_, String>(&format!( + "select state from {table} where task_id = $1::uuid" + )) + .bind(task_id.to_string()) + .fetch_optional(pool) + .await + { + Ok(Some(state)) if state == expected => return, + Ok(Some(state)) => last = state, + Ok(None) => last = "missing".to_owned(), + Err(error) => last = error.to_string(), + } + sleep(Duration::from_millis(100)).await; + } + panic!("task {task_id} did not reach {expected}; last state {last}"); +} + +async fn expire_running_claim(pool: &PgPool, task: &RehearsalTask) { + sqlx::query(&format!( + "update {} set claim_expires_at = now() - interval '1 second' where run_id = $1::uuid", + run_table(task.queue) + )) + .bind(task.run_id.to_string()) + .execute(pool) + .await + .expect("expire interrupted forward claim"); +} + +async fn convert_pending_to_event_wait(pool: &PgPool, task: &RehearsalTask) { + let task_table = task_table(task.queue); + let run_table = run_table(task.queue); + sqlx::query(&format!( + "update {task_table} set state = 'sleeping' where task_id = $1::uuid" + )) + .bind(task.task_id.to_string()) + .execute(pool) + .await + .expect("set waiting task state"); + sqlx::query(&format!( + "update {run_table} set state = 'sleeping', claimed_by = null, \ + claim_expires_at = null, available_at = '2099-01-01T00:00:00Z', \ + wake_event = 'rollback.rehearsal.resume' where run_id = $1::uuid" + )) + .bind(task.run_id.to_string()) + .execute(pool) + .await + .expect("set waiting run state"); + sqlx::query(&format!( + "insert into absurd.c_{} \ + (task_id, checkpoint_name, state, status, owner_run_id, updated_at) \ + values ($1::uuid, 'event-checkpoint', $2, 'committed', $3::uuid, now())", + task.queue + )) + .bind(task.task_id.to_string()) + .bind(json!({"cursor": "event-wait-preserved"})) + .bind(task.run_id.to_string()) + .execute(pool) + .await + .expect("seed waiting checkpoint"); + sqlx::query(&format!( + "insert into absurd.w_{} \ + (task_id, run_id, step_name, event_name, timeout_at, created_at) \ + values ($1::uuid, $2::uuid, 'await-rehearsal', 'rollback.rehearsal.resume', \ + '2099-01-01T00:00:00Z', now())", + task.queue + )) + .bind(task.task_id.to_string()) + .bind(task.run_id.to_string()) + .execute(pool) + .await + .expect("seed waiting row"); +} + +async fn customize_retry_and_cancellation_contract<'a>( + pool: &PgPool, + tasks: impl IntoIterator, +) { + for task in tasks { + sqlx::query(&format!( + "update {} set retry_strategy = $2, cancellation = $3 where task_id = $1::uuid", + task_table(task.queue) + )) + .bind(task.task_id.to_string()) + .bind(json!({ + "kind": "exponential", + "base_seconds": 0.125, + "factor": 2.0, + "max_seconds": 9.0, + })) + .bind(json!({"max_duration": 3600, "max_delay": 60})) + .execute(pool) + .await + .expect("set rehearsal task policy"); + } +} + +async fn task_contract_snapshot<'a>( + pool: &PgPool, + tasks: impl IntoIterator, +) -> Value { + let mut snapshot = Map::new(); + for task in tasks { + let table = task_table(task.queue); + let value = sqlx::query_scalar::<_, Value>(&format!( + "select jsonb_build_object( \ + 'params', params, 'headers', headers, 'retry_strategy', retry_strategy, \ + 'max_attempts', max_attempts, 'cancellation', cancellation, \ + 'idempotency_key', idempotency_key) \ + from {table} where task_id = $1::uuid" + )) + .bind(task.task_id.to_string()) + .fetch_one(pool) + .await + .expect("snapshot task contract"); + snapshot.insert(format!("{}:{}", task.queue, task.task_id), value); + } + Value::Object(snapshot) +} + +async fn canonical_cross_version_snapshot(pool: &PgPool) -> Value { + let mut snapshot = Map::new(); + for table in [ + "public._sqlx_migrations", + "public.sessions", + "absurd.t_centaur_workflows", + "absurd.r_centaur_workflows", + "absurd.c_centaur_workflows", + "absurd.e_centaur_workflows", + "absurd.w_centaur_workflows", + "absurd.t_centaur_workflows_slack_live", + "absurd.r_centaur_workflows_slack_live", + "absurd.c_centaur_workflows_slack_live", + "absurd.e_centaur_workflows_slack_live", + "absurd.w_centaur_workflows_slack_live", + "absurd.t_centaur_workflows_etl", + "absurd.r_centaur_workflows_etl", + "absurd.c_centaur_workflows_etl", + "absurd.e_centaur_workflows_etl", + "absurd.w_centaur_workflows_etl", + ] { + snapshot.insert(table.to_owned(), canonical_rows(pool, table).await); + } + Value::Object(snapshot) +} + +async fn rollback_schema_identity_snapshot(pool: &PgPool) -> Value { + let queues = sqlx::query_scalar::<_, String>( + "select queue_name from absurd.list_queues() order by queue_name", + ) + .fetch_all(pool) + .await + .expect("snapshot absurd queue registry"); + let absurd_objects = sqlx::query_scalar::<_, Value>( + r#" + select coalesce( + jsonb_agg( + jsonb_build_object('name', c.relname, 'kind', c.relkind::text) + order by c.relname, c.relkind::text + ), + '[]'::jsonb + ) + from pg_class c + join pg_namespace n on n.oid = c.relnamespace + where n.nspname = 'absurd' + "#, + ) + .fetch_one(pool) + .await + .expect("snapshot absurd schema objects"); + json!({ + "migration_ledger": canonical_rows(pool, "public._sqlx_migrations").await, + "queues": queues, + "absurd_objects": absurd_objects, + }) +} + +async fn emit_forward_workflow_event(client: &Client, base_url: &str, event_name: &str) { + let response = client + .post(format!("{base_url}/api/workflows/events")) + .header("Authorization", format!("Bearer {CONTROL_KEY}")) + .json(&json!({"event_name": event_name, "payload": {"resumed": true}})) + .send() + .await + .expect("emit re-forward workflow event"); + assert_eq!(response.status(), StatusCode::OK); +} + +async fn execute_reassigned_session( + client: &Client, + base_url: &str, + reassigned: &ReassignedSession, +) { + let input_line = serde_json::to_string(&json!({ + "type": "user", + "model": "rollback-rehearsal-model", + "message": {"role": "user", "content": [{"type": "text", "text": "PONG"}]}, + })) + .expect("serialize rehearsal input"); + let url = format!( + "{base_url}/api/session/{}/execute", + urlencoding::encode(reassigned.thread_key.as_str()) + ); + let response = client + .post(url) + .header("Authorization", format!("Bearer {CONTROL_KEY}")) + .json(&json!({ + "idempotency_key": format!("re-forward-{}", Uuid::new_v4()), + "metadata": {"source": "rollback-cross-version-rehearsal"}, + "input_lines": [input_line], + "idle_timeout_ms": 5_000, + "max_duration_ms": 15_000, + })) + .send() + .await + .expect("execute rollback-era session after re-forward"); + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + assert_eq!(status, StatusCode::OK, "re-forward execute failed: {body}"); +} + +async fn wait_for_forward_sandbox_replacement(pool: &PgPool, reassigned: &ReassignedSession) { + let deadline = Instant::now() + Duration::from_secs(30); + let mut last = None; + while Instant::now() < deadline { + let row = sqlx::query_as::<_, (Option, Option)>( + "select sandbox_id, sandbox_content_revision from sessions where thread_key = $1", + ) + .bind(reassigned.thread_key.as_str()) + .fetch_one(pool) + .await + .expect("read re-forward session assignment"); + if row.0.as_deref() != Some(reassigned.rollback_sandbox_id.as_str()) + && row.0.is_some() + && row.1.as_deref() != Some(reassigned.stale_forward_stamp.as_str()) + && row.1.is_some() + { + return; + } + last = Some(row); + sleep(Duration::from_millis(100)).await; + } + panic!("re-forward did not replace and restamp rollback-era sandbox: {last:?}"); +} + +async fn assert_checkpoint_survived_resume(pool: &PgPool, task: &RehearsalTask) { + let rows = sqlx::query_scalar::<_, Value>(&format!( + "select coalesce(jsonb_agg(state order by checkpoint_name), '[]'::jsonb) \ + from absurd.c_{} where task_id = $1::uuid", + task.queue + )) + .bind(task.task_id.to_string()) + .fetch_one(pool) + .await + .expect("read resumed checkpoints"); + assert!( + rows.as_array().is_some_and(|rows| { + rows.iter() + .any(|state| state["opaque"] == "preserve-through-rollback") + }), + "sleeping workflow checkpoint did not survive re-forward: {rows}" + ); +} + +fn assert_fixture_provenance() { + let forward_commit = FORWARD_COMMIT_FILE.trim(); + assert!( + forward_commit.len() == 40 + && forward_commit + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')), + ".github/rollback-bridge-reviewed-forward-commit must contain the frozen lowercase 40-character commit SHA" + ); + let expected = include_str!("fixtures/forward_migrations/SHA256SUMS") + .lines() + .filter_map(|line| line.split_once(" ")) + .map(|(checksum, file)| (file.to_owned(), checksum.to_owned())) + .collect::>(); + assert_eq!(expected.len(), FORWARD_MIGRATIONS.len()); + for ((file, sql), (embedded_file, embedded_sql)) in FORWARD_MIGRATIONS + .into_iter() + .zip(EMBEDDED_FORWARD_MIGRATIONS) + { + assert_eq!(file, embedded_file); + assert_eq!( + sql, embedded_sql, + "embedded production migration {file} differs from its reviewed fixture" + ); + let checksum = hex::encode(Sha256::digest(sql.as_bytes())); + assert_eq!( + expected.get(file).map(String::as_str), + Some(checksum.as_str()), + "test-only forward migration {file} drifted from pinned commit {forward_commit}" + ); + } +} + +struct SeededRows { + running_run_id: String, +} + +struct ReassignedSession { + thread_key: ThreadKey, + forward_content_generation: String, + rollback_sandbox_id: String, + stale_forward_stamp: String, +} + +async fn seed_forward_assignment_and_simulate_bridge_reassignment( + store: &PgSessionStore, + pool: &PgPool, +) -> ReassignedSession { + let thread_key = ThreadKey::parse(format!( + "rollback:content-revision:{}", + Uuid::new_v4().simple() + )) + .expect("forward assignment thread key"); + store + .create_or_get_session(&thread_key, &HarnessType::Codex, None, json!({})) + .await + .expect("create forward assignment session"); + + let forward_content_generation = "sandbox-spec-sha256:forward-reviewed-generation".to_owned(); + let forward_sandbox_id = "asbx-forward-content"; + let rollback_sandbox_id = "asbx-rollback-content".to_owned(); + let stale_forward_stamp = + forward_assignment_content_revision(&forward_content_generation, forward_sandbox_id); + sqlx::query( + "update sessions \ + set sandbox_id = $2, sandbox_content_revision = $3, \ + sandbox_repo_cache_enabled = true, sandbox_observability_enabled = true \ + where thread_key = $1", + ) + .bind(thread_key.as_str()) + .bind(forward_sandbox_id) + .bind(&stale_forward_stamp) + .execute(pool) + .await + .expect("seed forward content-stamped assignment"); + + let execution_id = store + .create_execution(&thread_key, None, json!({"rollback_bridge": true})) + .await + .expect("create rollback bridge assignment execution") + .execution + .execution_id; + let assigned = store + .assign_sandbox_to_active_execution( + &thread_key, + &execution_id, + Some(forward_sandbox_id), + &rollback_sandbox_id, + &SandboxCapabilities::default_enabled(), + ) + .await + .expect("bridge assignment against forward schema") + .expect("active bridge assignment wins"); + assert_eq!( + assigned.sandbox_id.as_deref(), + Some(rollback_sandbox_id.as_str()) + ); + store + .complete_execution(&execution_id) + .await + .expect("complete bridge assignment fixture execution"); + + ReassignedSession { + thread_key, + forward_content_generation, + rollback_sandbox_id, + stale_forward_stamp, + } +} + +async fn assert_reassignment_requires_forward_replacement( + pool: &PgPool, + reassigned: &ReassignedSession, +) { + let (sandbox_id, persisted_stamp) = sqlx::query_as::<_, (Option, Option)>( + "select sandbox_id, sandbox_content_revision from sessions where thread_key = $1", + ) + .bind(reassigned.thread_key.as_str()) + .fetch_one(pool) + .await + .expect("read rollback-era assignment after bridge startup"); + assert_eq!( + sandbox_id.as_deref(), + Some(reassigned.rollback_sandbox_id.as_str()) + ); + assert_eq!( + persisted_stamp.as_deref(), + Some(reassigned.stale_forward_stamp.as_str()) + ); + + let desired_re_forward_stamp = forward_assignment_content_revision( + &reassigned.forward_content_generation, + &reassigned.rollback_sandbox_id, + ); + assert_ne!( + reassigned.stale_forward_stamp, desired_re_forward_stamp, + "a rollback-era sandbox must not authenticate as current forward boot content" + ); +} + +fn forward_assignment_content_revision(generation: &str, sandbox_id: &str) -> String { + let mut digest = Sha256::new(); + digest.update(b"centaur-sandbox-assignment-v1\0"); + digest.update(generation.as_bytes()); + digest.update(b"\0"); + digest.update(sandbox_id.as_bytes()); + format!("sandbox-assignment-sha256:{:x}", digest.finalize()) +} + +async fn seed_representative_workflow_rows(pool: &PgPool) -> SeededRows { + let states = ["pending", "running", "sleeping", "pending", "pending"]; + let mut running_run_id = None; + for (index, (queue, state)) in QUEUES.iter().zip(states).enumerate() { + let task_id = Uuid::new_v4(); + let run_id = Uuid::new_v4(); + if state == "running" { + running_run_id = Some(run_id); + } + let task_name = if *queue == "centaur_workflow_schedules" { + "centaur.workflow.schedule_tick" + } else { + "centaur.workflow" + }; + let workflow_name = if index == 0 { + // Present only in the forward overlay: an active rollback worker + // would fail or reap this row. + "fineas_google_drive_folder_sync" + } else { + [ + "slack_live_forward", + "sleeping_forward", + "etl_backfill_forward", + "schedule_forward", + ][index - 1] + }; + let task_table = format!("absurd.t_{queue}"); + let run_table = format!("absurd.r_{queue}"); + sqlx::query(&format!( + "insert into {task_table} \ + (task_id, task_name, params, headers, retry_strategy, max_attempts, cancellation, \ + enqueue_at, first_started_at, state, attempts, last_attempt_run, idempotency_key) \ + values ($1::uuid, $2, $3, $4, $5, 7, $6, \ + '2026-07-01T01:02:03Z', '2026-07-01T01:03:03Z', $7, 2, $8::uuid, $9)" + )) + .bind(task_id.to_string()) + .bind(task_name) + .bind(json!({ + "workflow_name": workflow_name, + "input": {"fixture": index, "forward_only": index == 0}, + "harness_type": "codex" + })) + .bind(json!({"traceparent": format!("fixture-{index}"), "x-forward": true})) + .bind(json!({"type": "exponential", "initial_ms": 125, "max_ms": 9000})) + .bind(json!({"mode": "cooperative", "grace_ms": 4321})) + .bind(state) + .bind(run_id.to_string()) + .bind(format!("rollback-preservation-{index}")) + .execute(pool) + .await + .expect("seed forward task row"); + + let (claimed_by, claim_expires_at, wake_event, available_at) = match state { + "running" => ( + Some("forward-worker"), + Some("2026-07-01T01:04:03Z"), + None, + "2026-07-01T01:02:03Z", + ), + "sleeping" => (None, None, Some("forward.resume"), "2099-07-01T01:02:03Z"), + _ => (None, None, None, "2026-07-01T01:02:03Z"), + }; + sqlx::query(&format!( + "insert into {run_table} \ + (run_id, task_id, attempt, state, claimed_by, claim_expires_at, available_at, \ + wake_event, event_payload, started_at, result, failure_reason, created_at) \ + values ($1::uuid, $2::uuid, 2, $3, $4, $5::timestamptz, $6::timestamptz, \ + $7, $8, '2026-07-01T01:03:03Z', $9, $10, '2026-07-01T01:02:03Z')" + )) + .bind(run_id.to_string()) + .bind(task_id.to_string()) + .bind(state) + .bind(claimed_by) + .bind(claim_expires_at) + .bind(available_at) + .bind(wake_event) + .bind(json!({"delivery": "preserve"})) + .bind(json!({"partial": true})) + .bind(json!({"previous_attempt": "preserve"})) + .execute(pool) + .await + .expect("seed forward run row"); + + if state == "sleeping" { + sqlx::query(&format!( + "insert into absurd.c_{queue} \ + (task_id, checkpoint_name, state, status, owner_run_id, updated_at) \ + values ($1::uuid, 'forward-checkpoint', $2, 'committed', $3::uuid, \ + '2026-07-01T01:05:03Z')" + )) + .bind(task_id.to_string()) + .bind(json!({"cursor": "opaque-forward-cursor", "page": 17})) + .bind(run_id.to_string()) + .execute(pool) + .await + .expect("seed sleeping checkpoint"); + sqlx::query(&format!( + "insert into absurd.e_{queue} (event_name, payload, emitted_at) \ + values ('forward.resume', $1, '2026-07-01T01:05:03Z')" + )) + .bind(json!({"event": "must-remain"})) + .execute(pool) + .await + .expect("seed sleeping event"); + sqlx::query(&format!( + "insert into absurd.w_{queue} \ + (task_id, run_id, step_name, event_name, timeout_at, created_at) \ + values ($1::uuid, $2::uuid, 'await-forward', 'forward.resume', \ + '2099-07-01T01:02:03Z', '2026-07-01T01:05:03Z')" + )) + .bind(task_id.to_string()) + .bind(run_id.to_string()) + .execute(pool) + .await + .expect("seed sleeping wait"); + } + } + SeededRows { + running_run_id: running_run_id.expect("running fixture run").to_string(), + } +} + +async fn assert_seeded_states(pool: &PgPool) { + let mut counts = BTreeMap::new(); + for queue in QUEUES { + let rows = sqlx::query_as::<_, (String, i64)>(&format!( + "select state, count(*) from absurd.t_{queue} group by state order by state" + )) + .fetch_all(pool) + .await + .expect("count seeded states"); + counts.insert(queue, rows); + } + assert_eq!(counts["centaur_workflows"], vec![("pending".to_owned(), 1)]); + assert_eq!( + counts["centaur_workflows_slack_live"], + vec![("running".to_owned(), 1)] + ); + assert_eq!( + counts["centaur_workflows_etl"], + vec![("sleeping".to_owned(), 1)] + ); + assert_eq!( + counts["centaur_workflows_etl_backfill"], + vec![("pending".to_owned(), 1)] + ); + assert_eq!( + counts["centaur_workflow_schedules"], + vec![("pending".to_owned(), 1)] + ); +} + +async fn canonical_forward_snapshot(pool: &PgPool) -> Value { + let mut snapshot = Map::new(); + snapshot.insert( + "_sqlx_migrations".to_owned(), + canonical_rows(pool, "public._sqlx_migrations").await, + ); + snapshot.insert( + "absurd.queues".to_owned(), + canonical_rows(pool, "absurd.queues").await, + ); + snapshot.insert( + "public.sessions".to_owned(), + canonical_rows(pool, "public.sessions").await, + ); + for queue in QUEUES { + for prefix in ["t", "r", "c", "e", "w"] { + let table = format!("absurd.{prefix}_{queue}"); + snapshot.insert(table.clone(), canonical_rows(pool, &table).await); + } + } + Value::Object(snapshot) +} + +async fn canonical_rows(pool: &PgPool, table: &str) -> Value { + sqlx::query_scalar::<_, Value>(&format!( + "select coalesce( \ + jsonb_agg(to_jsonb(snapshot_row) order by to_jsonb(snapshot_row)::text), \ + '[]'::jsonb) \ + from (select * from {table}) snapshot_row" + )) + .fetch_one(pool) + .await + .unwrap_or_else(|error| panic!("snapshot {table}: {error}")) +} + +async fn exercise_read_and_mutation_lanes(client: &Client, base_url: &str, running_run_id: &str) { + let control = format!("Bearer {CONTROL_KEY}"); + for path in ["/api/workflows/runs?limit=50", "/api/workflows/schedules"] { + let response = client + .get(format!("{base_url}{path}")) + .header("Authorization", &control) + .send() + .await + .expect("read paused workflow state"); + assert_eq!(response.status(), StatusCode::OK, "GET {path}"); + } + + let mutations = [ + ( + "/api/workflows/runs".to_owned(), + json!({"workflow_name": "must_not_start", "input": {"fixture": true}}), + ), + ( + format!("/api/workflows/runs/{running_run_id}/cancel"), + json!({}), + ), + ( + "/api/workflows/events".to_owned(), + json!({"event_name": "forward.resume", "payload": {"must_not_write": true}}), + ), + ]; + for (path, body) in mutations { + let response = client + .post(format!("{base_url}{path}")) + .header("Authorization", &control) + .json(&body) + .send() + .await + .expect("exercise paused workflow mutation"); + assert_eq!(response.status(), StatusCode::FORBIDDEN, "POST {path}"); + } + + let webhook = client + .post(format!( + "{base_url}/api/webhooks/trivy-vulnerability-intake" + )) + .header("Authorization", format!("Bearer {WEBHOOK_KEY}")) + .json(&json!({"alerts": [{"fixture": true}]})) + .send() + .await + .expect("exercise authenticated webhook mutation lane"); + assert_eq!(webhook.status(), StatusCode::FORBIDDEN); + + let admin = client + .post(format!("{base_url}/api/admin/slack/dm-sync/batch")) + .header("Authorization", &control) + .json(&json!({})) + .send() + .await + .expect("exercise control-authenticated admin lane"); + assert_eq!(admin.status(), StatusCode::OK); +} + +async fn wait_for_ready(client: &Client, base_url: &str, server: &mut BridgeProcess) { + let deadline = Instant::now() + Duration::from_secs(30); + let mut last = String::new(); + while Instant::now() < deadline { + if let Some(status) = server.exited() { + panic!("rollback bridge exited before readiness: {status}"); + } + match client.get(format!("{base_url}/readyz")).send().await { + Ok(response) if response.status() == StatusCode::OK => return, + Ok(response) => last = format!("readyz returned {}", response.status()), + Err(error) => last = error.to_string(), + } + sleep(Duration::from_millis(100)).await; + } + panic!("rollback bridge did not become ready: {last}"); +} + +fn unused_local_port() -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral test port"); + listener + .local_addr() + .expect("ephemeral local address") + .port() +} + +fn database_url_with_name(admin_url: &str, database_name: &str) -> String { + let (without_query, query) = admin_url + .split_once('?') + .map_or((admin_url, ""), |(base, query)| (base, query)); + let slash = without_query + .rfind('/') + .expect("database URL must include a path"); + let query = if query.is_empty() { + String::new() + } else { + format!("?{query}") + }; + format!("{}{database_name}{query}", &without_query[..=slash]) +} diff --git a/services/api-rs/crates/centaur-api-server/tests/rollback_bridge_startup.rs b/services/api-rs/crates/centaur-api-server/tests/rollback_bridge_startup.rs new file mode 100644 index 000000000..950087cb0 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/tests/rollback_bridge_startup.rs @@ -0,0 +1,105 @@ +use std::process::Command; + +const PAUSE_ENV: &str = "CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS"; +const CONTROL_KEY_ENV: &str = "CENTAUR_CONTROL_API_KEY"; +const OTHER_SERVICE_KEY_ENVS: [&str; 7] = [ + "SLACKBOT_API_KEY", + "GITHUBBOT_API_KEY", + "LINEARBOT_API_KEY", + "DISCORDBOT_API_KEY", + "TEAMSBOT_API_KEY", + "WORKFLOW_API_KEY", + "SLACK_FEEDBACK_API_KEY", +]; + +#[test] +fn rollback_bridge_binary_refuses_startup_without_explicit_workflow_pause() { + for value in [None, Some("false"), Some("1")] { + let mut command = Command::new(env!("CARGO_BIN_EXE_centaur-api-server")); + command.env_remove(PAUSE_ENV); + if let Some(value) = value { + command.env(PAUSE_ENV, value); + } + + let output = command.output().expect("run rollback bridge binary"); + assert!( + !output.status.success(), + "rollback bridge unexpectedly started with {PAUSE_ENV}={value:?}" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains(&format!( + "rollback bridge refuses to start unless {PAUSE_ENV}=true" + )), + "unexpected startup error for {PAUSE_ENV}={value:?}: {stderr}" + ); + } +} + +#[test] +fn rollback_bridge_binary_refuses_missing_or_reused_control_key() { + let mut missing = Command::new(env!("CARGO_BIN_EXE_centaur-api-server")); + missing.env(PAUSE_ENV, "true").env_remove(CONTROL_KEY_ENV); + for name in OTHER_SERVICE_KEY_ENVS { + missing.env_remove(name); + } + let output = missing.output().expect("run rollback bridge binary"); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("CENTAUR_CONTROL_API_KEY is configured"), + "unexpected missing-control startup error: {stderr}" + ); + + for reused_name in OTHER_SERVICE_KEY_ENVS { + let mut command = Command::new(env!("CARGO_BIN_EXE_centaur-api-server")); + command + .env(PAUSE_ENV, "true") + .env(CONTROL_KEY_ENV, "shared-key"); + for name in OTHER_SERVICE_KEY_ENVS { + command.env_remove(name); + } + command.env(reused_name, "shared-key"); + + let output = command.output().expect("run rollback bridge binary"); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("must contain distinct service credentials") + && stderr.contains(CONTROL_KEY_ENV) + && stderr.contains(reused_name), + "unexpected startup error for duplicate {reused_name}: {stderr}" + ); + assert!(!stderr.contains("shared-key"), "secret leaked: {stderr}"); + } +} + +#[test] +fn rollback_bridge_binary_refuses_migration_ownership_before_database_access() { + let mut command = Command::new(env!("CARGO_BIN_EXE_centaur-api-server")); + command + .env(PAUSE_ENV, "true") + .env(CONTROL_KEY_ENV, "control-key") + .env("RUN_MIGRATIONS", "true") + .env( + "DATABASE_URL", + "postgresql://must-not-connect.invalid/rollback_bridge", + ); + for name in OTHER_SERVICE_KEY_ENVS { + command.env_remove(name); + } + let output = command.output().expect("run rollback bridge binary"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains( + "rollback bridge refuses to start with RUN_MIGRATIONS=true; the reviewed forward runtime owns schema migration" + ), + "unexpected migration-ownership startup error: {stderr}" + ); + assert!( + !stderr.contains("must-not-connect.invalid"), + "bridge attempted to report database access before rejecting migrations: {stderr}" + ); +} diff --git a/services/api-rs/crates/centaur-sandbox-manager/src/warm_pool.rs b/services/api-rs/crates/centaur-sandbox-manager/src/warm_pool.rs index 8d1cf611a..919660678 100644 --- a/services/api-rs/crates/centaur-sandbox-manager/src/warm_pool.rs +++ b/services/api-rs/crates/centaur-sandbox-manager/src/warm_pool.rs @@ -1,9 +1,18 @@ -use std::{sync::Arc, time::Duration}; +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; use centaur_sandbox_core::{SandboxError, SandboxId, SandboxSpec, SandboxStatus}; use centaur_session_sqlx::{PgSessionStore, SessionStoreError}; use thiserror::Error; -use tokio::time::{MissedTickBehavior, interval}; +use tokio::{ + sync::Mutex, + time::{MissedTickBehavior, interval}, +}; use tracing::warn; use crate::SandboxManager; @@ -22,6 +31,8 @@ pub struct WarmPoolManager { spec_factory: WarmSandboxSpecFactory, workload_key: String, config: WarmPoolConfig, + paused: AtomicBool, + reconcile_lock: Mutex<()>, } impl WarmPoolManager { @@ -38,6 +49,8 @@ impl WarmPoolManager { spec_factory, workload_key: workload_key.into(), config, + paused: AtomicBool::new(false), + reconcile_lock: Mutex::new(()), } } @@ -45,6 +58,15 @@ impl WarmPoolManager { &self.workload_key } + /// Permanently pause this process's replenisher and wait for any in-flight + /// reconciliation to finish. Deployment drains call this before listing + /// sandboxes so the background loop cannot recreate a warm sandbox between + /// the drain and the zero-overlap rollback cutover. + pub async fn pause_and_wait(&self) { + self.paused.store(true, Ordering::SeqCst); + let _guard = self.reconcile_lock.lock().await; + } + pub fn spawn_replenisher(self: Arc) { tokio::spawn(async move { let mut tick = interval(self.config.replenish_interval); @@ -115,6 +137,10 @@ impl WarmPoolManager { } async fn replenish_once(&self) -> Result<(), WarmPoolError> { + let _guard = self.reconcile_lock.lock().await; + if self.paused.load(Ordering::SeqCst) { + return Ok(()); + } let needed = self.config.target_size.saturating_sub( self.store .count_ready_warm_sandboxes(self.workload_key.as_str()) diff --git a/services/api-rs/crates/centaur-session-runtime/src/lib.rs b/services/api-rs/crates/centaur-session-runtime/src/lib.rs index 131d231a6..02aa3cf1a 100644 --- a/services/api-rs/crates/centaur-session-runtime/src/lib.rs +++ b/services/api-rs/crates/centaur-session-runtime/src/lib.rs @@ -2,7 +2,10 @@ mod cleanup; use std::{ collections::{BTreeMap, HashMap, HashSet, VecDeque}, - sync::Arc, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, time::{Duration, SystemTime}, }; @@ -20,7 +23,7 @@ use centaur_session_core::{ Session, SessionEvent, SessionExecution, SessionMessageInput, ThreadKey, }; use centaur_session_sqlx::{ - PgSessionStore, SessionEventListener, SessionStoreError, default_metadata, + PgSessionStore, ReleaseSessionResult, SessionEventListener, SessionStoreError, default_metadata, }; use centaur_telemetry::{ export_thread_trace_root_span, record_sandbox_warm_pool_claim, @@ -35,7 +38,7 @@ use sha2::{Digest, Sha256}; use thiserror::Error; use tokio::{ io, - sync::Mutex, + sync::{Mutex, RwLock}, time::{Instant, Interval, MissedTickBehavior, interval_at, sleep}, }; use tokio_util::codec::{FramedRead, FramedWrite, LinesCodec, LinesCodecError}; @@ -63,6 +66,7 @@ type SessionInputSink = FramedWrite; type ExecutionSpanRegistry = Arc>>; type SessionPipeMap = Arc>; type SessionPipeOpenLocks = Arc>>>; +type SessionOperationLocks = Arc>>>; #[derive(Clone)] pub struct SessionRuntime { @@ -70,6 +74,14 @@ pub struct SessionRuntime { sandbox_runtime: SandboxRuntime, sandbox_pipes: SessionPipeMap, sandbox_pipe_open_locks: SessionPipeOpenLocks, + /// Serializes assignment/input with explicit release in this process. + /// Cross-replica safety comes from the database CAS below; the rollback + /// bridge is still deployed with a zero-overlap API cutover. + session_operation_locks: SessionOperationLocks, + /// Once drain starts, reject new sandbox allocations and wait for any + /// allocation already in progress before taking the backend inventory. + sandbox_allocation_gate: Arc>, + draining: Arc, execution_spans: ExecutionSpanRegistry, iron_control: Option, warm_pool: Option>, @@ -327,6 +339,9 @@ impl SessionRuntime { sandbox_runtime, sandbox_pipes: Arc::new(DashMap::new()), sandbox_pipe_open_locks: Arc::new(DashMap::new()), + session_operation_locks: Arc::new(DashMap::new()), + sandbox_allocation_gate: Arc::new(RwLock::new(())), + draining: Arc::new(AtomicBool::new(false)), execution_spans: Arc::new(Mutex::new(HashMap::new())), iron_control: None, warm_pool: None, @@ -403,6 +418,19 @@ impl SessionRuntime { } } + fn session_operation_lock(&self, thread_key: &ThreadKey) -> Arc> { + self.session_operation_locks + .entry(thread_key.as_str().to_owned()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + } + + fn release_session_operation_lock(&self, thread_key: &ThreadKey, lock: Arc>) { + drop(lock); + self.session_operation_locks + .remove_if(thread_key.as_str(), |_, lock| Arc::strong_count(lock) == 1); + } + /// Attach an iron-control registrar so each new session upserts its /// principal and assigns the configured roles. pub fn with_iron_control(mut self, registrar: SessionRegistrar) -> Self { @@ -734,6 +762,15 @@ impl SessionRuntime { /// rest, and the [`DrainReport`] records which were stopped and which /// failed so the caller can surface partial failure. pub async fn drain(&self) -> Result { + // The flag closes the gate to newcomers. Taking the write guard then + // waits for every allocator that crossed the gate before this drain, + // so the inventory below cannot miss a late warm claim, resume, or + // cold create from this runtime. + self.draining.store(true, Ordering::SeqCst); + if let Some(warm_pool) = &self.warm_pool { + warm_pool.pause_and_wait().await; + } + let _allocation_guard = self.sandbox_allocation_gate.write().await; let observed = self.sandbox_runtime.manager.list_observed().await?; let mut report = DrainReport::default(); for sandbox in observed { @@ -766,6 +803,23 @@ impl SessionRuntime { } } } + let failed_ids = report + .failed + .iter() + .map(|failure| failure.sandbox_id.clone()) + .collect::>(); + for sandbox in self.sandbox_runtime.manager.list_observed().await? { + if sandbox.status.is_terminal() || failed_ids.contains(sandbox.id.as_str()) { + continue; + } + report.failed.push(DrainFailure { + sandbox_id: sandbox.id.as_str().to_owned(), + error: format!( + "sandbox remained non-terminal after drain: {:?}", + sandbox.status + ), + }); + } Ok(report) } @@ -886,83 +940,127 @@ impl SessionRuntime { &self, thread_key: &ThreadKey, release_id: Option<&str>, + expected_sandbox_id: Option<&str>, cancel_inflight: bool, ) -> Result { - let session = self.store.get_session(thread_key).await?; - let sandbox_id = session.sandbox_id.clone(); - let mut execution_id = None; - let mut execution_cancelled = false; + let operation_lock = self.session_operation_lock(thread_key); + let result = { + let _guard = operation_lock.lock().await; + self.release_thread_locked(thread_key, release_id, expected_sandbox_id, cancel_inflight) + .await + }; + self.release_session_operation_lock(thread_key, operation_lock); + result + } - if let Some(active_execution) = self.store.active_execution_for_thread(thread_key).await? { - execution_id = Some(active_execution.execution_id.clone()); - if !cancel_inflight { + async fn release_thread_locked( + &self, + thread_key: &ThreadKey, + release_id: Option<&str>, + expected_sandbox_id: Option<&str>, + cancel_inflight: bool, + ) -> Result { + let snapshot = self.store.get_session(thread_key).await?; + if expected_sandbox_id.is_some_and(|value| value.trim().is_empty()) { + return Err(SessionRuntimeError::BadRequest( + "expected_sandbox_id must not be empty".to_owned(), + )); + } + let expected_sandbox_id = expected_sandbox_id.map(str::trim); + if snapshot.sandbox_id.is_some() && expected_sandbox_id.is_none() { + return Err(SessionRuntimeError::BadRequest( + "expected_sandbox_id is required when the session has an assigned sandbox" + .to_owned(), + )); + } + if let Some(expected_sandbox_id) = expected_sandbox_id + && snapshot.sandbox_id.as_deref() != Some(expected_sandbox_id) + { + return Err(SessionRuntimeError::BadRequest(format!( + "session sandbox does not match caller fence (expected {expected_sandbox_id:?}, current {:?})", + snapshot.sandbox_id + ))); + } + let sandbox_id = expected_sandbox_id + .map(ToOwned::to_owned) + .or_else(|| snapshot.sandbox_id.clone()); + let cancellation_reason = + Self::release_error_message(release_id, thread_key, sandbox_id.as_deref()); + let (session, cancelled_execution) = match self + .store + .release_session_if_sandbox_matches( + thread_key, + sandbox_id.as_deref(), + cancel_inflight, + &cancellation_reason, + ) + .await? + { + ReleaseSessionResult::Released { + session, + cancelled_execution, + } => (session, cancelled_execution), + ReleaseSessionResult::ActiveExecution(execution) => { return Err(SessionRuntimeError::BadRequest(format!( - "thread {} has active execution {}; pass cancel_inflight=true to release it", - thread_key.as_str(), - active_execution.execution_id + "thread has active execution {}; retry with cancel_inflight=true", + execution.execution_id ))); } - if matches!( - active_execution.status, - ExecutionStatus::Queued | ExecutionStatus::Running - ) { - let cancel_error = Self::release_error_message( - release_id, - thread_key, - sandbox_id.as_deref(), - active_execution.execution_id.as_str(), - ); - if let Some(cancelled) = self - .store - .cancel_execution_if_active(&active_execution.execution_id, &cancel_error) - .await? - { - execution_cancelled = true; - self.execution_spans - .lock() - .await - .remove(&cancelled.execution_id); - self.store - .append_event( - thread_key, - Some(&cancelled.execution_id), - "session.execution_cancelled", - json!({ - "execution_id": cancelled.execution_id.as_str(), - "thread_key": thread_key.as_str(), - "sandbox_id": sandbox_id.as_deref(), - "release_id": release_id, - "reason": "thread_released", - }), - ) - .await?; - } + ReleaseSessionResult::SandboxMismatch { current_sandbox_id } => { + return Err(SessionRuntimeError::BadRequest(format!( + "session sandbox changed during release (expected {sandbox_id:?}, current {current_sandbox_id:?}); retry" + ))); } + }; + + let execution_id = cancelled_execution + .as_ref() + .map(|execution| execution.execution_id.clone()); + if let Some(execution_id) = execution_id.as_deref() { + self.execution_spans.lock().await.remove(execution_id); + } + if let Some(sandbox_id) = sandbox_id.as_deref() { + self.sandbox_pipes.remove(sandbox_id); } let mut sandbox_released = false; let mut sandbox_release_error = None; - if let Some(ref sandbox_id) = sandbox_id { - self.sandbox_pipes.remove(sandbox_id); - let id = SandboxId::new(sandbox_id.clone()); - match self.sandbox_runtime.manager.stop(&id).await { - Ok(()) | Err(SandboxError::NotFound(_)) => { - sandbox_released = true; - } + if let Some(sandbox_id) = sandbox_id.as_deref() { + match self + .sandbox_runtime + .manager + .stop(&SandboxId::new(sandbox_id)) + .await + { + Ok(()) | Err(SandboxError::NotFound(_)) => sandbox_released = true, Err(error) => { - warn!( - thread_key = %thread_key, - sandbox_id = %sandbox_id, - %error, - "failed to release sandbox" - ); + warn!(%thread_key, %sandbox_id, %error, "failed to stop released sandbox"); sandbox_release_error = Some(error.to_string()); } } } - let session = self.store.release_session(thread_key).await?; - self.store + if let Some(execution) = cancelled_execution.as_ref() + && let Err(error) = self + .store + .append_event( + thread_key, + Some(&execution.execution_id), + "session.execution_cancelled", + json!({ + "execution_id": execution.execution_id, + "thread_key": thread_key.as_str(), + "sandbox_id": sandbox_id.as_deref(), + "release_id": release_id, + "reason": "thread_released", + }), + ) + .await + { + warn!(%thread_key, %error, "failed to record release cancellation event"); + } + if let Err(error) = self + .store .append_event( thread_key, execution_id.as_deref(), @@ -975,10 +1073,13 @@ impl SessionRuntime { "sandbox_released": sandbox_released, "sandbox_release_error": sandbox_release_error.as_deref(), "execution_id": execution_id.as_deref(), - "execution_cancelled": execution_cancelled, + "execution_cancelled": cancelled_execution.is_some(), }), ) - .await?; + .await + { + warn!(%thread_key, %error, "failed to record session release event"); + } Ok(ReleaseThreadOutcome { session, @@ -987,7 +1088,7 @@ impl SessionRuntime { sandbox_released, sandbox_release_error, execution_id, - execution_cancelled, + execution_cancelled: cancelled_execution.is_some(), }) } @@ -995,6 +1096,20 @@ impl SessionRuntime { &self, thread_key: &ThreadKey, input: ExecuteSessionInput, + ) -> Result { + let operation_lock = self.session_operation_lock(thread_key); + let result = { + let _guard = operation_lock.lock().await; + self.execute_session_locked(thread_key, input).await + }; + self.release_session_operation_lock(thread_key, operation_lock); + result + } + + async fn execute_session_locked( + &self, + thread_key: &ThreadKey, + input: ExecuteSessionInput, ) -> Result { let ExecuteSessionInput { idempotency_key, @@ -1225,7 +1340,25 @@ impl SessionRuntime { ) { self.execution_spans.lock().await.remove(execution_id); let error_message = error.to_string(); - let _ = self + let transition = self + .store + .fail_execution_if_active(execution_id, &error_message) + .await; + let execution = match transition { + Ok(Some(execution)) => execution, + Ok(None) => return, + Err(record_error) => { + warn!( + %thread_key, + execution_id, + error = %record_error, + "failed to transition active execution to failed" + ); + return; + } + }; + + if let Err(event_error) = self .store .append_event( thread_key, @@ -1237,37 +1370,38 @@ impl SessionRuntime { "error": error_message, }), ) - .await; - if let Ok(execution) = self - .store - .fail_execution(execution_id, &error_message) .await { - record_finished_execution_metric( - &self.store, - thread_key, - &execution, - "failed", - Some(runtime_error_failure_class(error)), - ) - .await; + warn!( + %thread_key, + execution_id, + error = %event_error, + "failed to append execution failure event" + ); } + record_finished_execution_metric( + &self.store, + thread_key, + &execution, + "failed", + Some(runtime_error_failure_class(error)), + ) + .await; } fn release_error_message( release_id: Option<&str>, thread_key: &ThreadKey, sandbox_id: Option<&str>, - execution_id: &str, ) -> String { match release_id { Some(release_id) => format!( - "thread released (release_id={release_id}, thread_key={}, sandbox_id={:?}, execution_id={execution_id})", + "thread released (release_id={release_id}, thread_key={}, sandbox_id={:?})", thread_key.as_str(), sandbox_id, ), None => format!( - "thread released (thread_key={}, sandbox_id={:?}, execution_id={execution_id})", + "thread released (thread_key={}, sandbox_id={:?})", thread_key.as_str(), sandbox_id, ), @@ -1469,6 +1603,16 @@ impl SessionRuntime { &self, request: EnsureSessionSandboxRequest<'_>, ) -> Result { + if self.draining.load(Ordering::SeqCst) { + return Err(SessionRuntimeError::Draining); + } + let _allocation_guard = self.sandbox_allocation_gate.read().await; + // Drain may have raised the flag while this task was waiting for a + // writer or an earlier reader. Recheck only after entering the gate. + if self.draining.load(Ordering::SeqCst) { + return Err(SessionRuntimeError::Draining); + } + let EnsureSessionSandboxRequest { thread_key, harness_type, @@ -1498,16 +1642,26 @@ impl SessionRuntime { let ensure_started = Instant::now(); let result = async { let persona_context = self.resolve_stored_persona(persona_id, harness_type)?; + let mut assignment_fence = existing_sandbox_id; if let Some(sandbox_id) = existing_sandbox_id { let id = SandboxId::new(sandbox_id); if !sandbox_capabilities_match(existing_sandbox_capabilities, desired_capabilities) { + if !self + .store + .clear_sandbox_id_if_matches(thread_key, sandbox_id) + .await? + { + return Err(SessionRuntimeError::BadRequest(format!( + "session sandbox changed while replacing {sandbox_id:?}; retry" + ))); + } + assignment_fence = None; self.sandbox_pipes.remove(sandbox_id); match self.sandbox_runtime.manager.stop(&id).await { Ok(()) | Err(SandboxError::NotFound(_)) => {} Err(error) => return Err(SessionRuntimeError::Sandbox(error)), } - self.store.update_sandbox_id(thread_key, None).await?; self.store .append_event( thread_key, @@ -1698,13 +1852,33 @@ impl SessionRuntime { span.record("centaur.sandbox_id", sandbox_id.as_str()); span.record("sandbox_id", sandbox_id.as_str()); let ready_duration = ensure_started.elapsed(); - self.store - .update_sandbox_assignment( + let assigned = self + .store + .assign_sandbox_to_active_execution( thread_key, + execution_id, + assignment_fence, sandbox_id.as_str(), desired_capabilities, ) .await?; + if assigned.is_none() { + let _ = self + .store + .mark_warm_sandbox_failed( + sandbox_id.as_str(), + "execution ended before warm sandbox assignment", + ) + .await; + let _ = self + .sandbox_runtime + .manager + .stop(&SandboxId::new(sandbox_id.as_str())) + .await; + return Err(SessionRuntimeError::BadRequest( + "execution ended before warm sandbox assignment".to_owned(), + )); + } self.store .append_event( thread_key, @@ -1766,9 +1940,22 @@ impl SessionRuntime { let ready_duration = ensure_started.elapsed(); span.record("centaur.sandbox_id", handle.id.as_str()); span.record("sandbox_id", handle.id.as_str()); - self.store - .update_sandbox_assignment(thread_key, handle.id.as_str(), desired_capabilities) + let assigned = self + .store + .assign_sandbox_to_active_execution( + thread_key, + execution_id, + assignment_fence, + handle.id.as_str(), + desired_capabilities, + ) .await?; + if assigned.is_none() { + let _ = self.sandbox_runtime.manager.stop(&handle.id).await; + return Err(SessionRuntimeError::BadRequest( + "execution ended before sandbox assignment".to_owned(), + )); + } self.record_sandbox_ready(SandboxReadyObservation { thread_key, execution_id, @@ -4174,6 +4361,7 @@ fn execution_duration(execution: &SessionExecution) -> Option { fn runtime_error_failure_class(error: &SessionRuntimeError) -> &'static str { match error { SessionRuntimeError::BadRequest(_) => "bad_request", + SessionRuntimeError::Draining => "draining", SessionRuntimeError::Store(_) => "store", SessionRuntimeError::Sandbox(SandboxError::NotFound(_)) => "sandbox_not_found", SessionRuntimeError::Sandbox(SandboxError::Unsupported { .. }) => "sandbox_unsupported", @@ -4985,6 +5173,8 @@ fn terminal_output_from_lines(lines: &[String]) -> Option { pub enum SessionRuntimeError { #[error("{0}")] BadRequest(String), + #[error("sandbox allocation is unavailable because runtime drain has started")] + Draining, #[error(transparent)] Store(#[from] SessionStoreError), #[error(transparent)] @@ -5969,17 +6159,20 @@ mod tests { } /// Integration tests for orphaned-execution adoption. They need a real -/// Postgres; set `SESSION_RUNTIME_TEST_DATABASE_URL` to run them (they skip -/// silently otherwise, mirroring `ABSURD_TEST_DATABASE_URL` in absurd-sdk). +/// Postgres; set `SESSION_RUNTIME_TEST_DATABASE_URL` to run them (CI treats +/// this suite as required). #[cfg(test)] mod adoption_tests { use std::{ - collections::BTreeSet, + collections::{BTreeMap, BTreeSet}, sync::atomic::{AtomicBool, AtomicUsize, Ordering}, }; use centaur_sandbox_core::{ObservedSandbox, SandboxHandle, SandboxIo, SandboxResult}; - use tokio::io::{AsyncWriteExt, DuplexStream}; + use tokio::{ + io::{AsyncWriteExt, DuplexStream}, + sync::{Notify, OnceCell}, + }; use super::*; @@ -5993,12 +6186,17 @@ mod adoption_tests { recorded_output: std::sync::Mutex>, open_count: AtomicUsize, status: std::sync::Mutex, + observed_statuses: std::sync::Mutex>, create_id: String, created_specs: std::sync::Mutex>, + block_create: AtomicBool, + create_started: Notify, + allow_create: Notify, resume_fails: AtomicBool, stopped: std::sync::Mutex>, proxy_ensures: std::sync::Mutex>, missing_on_stop: std::sync::Mutex>, + failing_on_stop: std::sync::Mutex>, } impl MockBackend { @@ -6008,12 +6206,17 @@ mod adoption_tests { recorded_output: std::sync::Mutex::new(recorded_output), open_count: AtomicUsize::new(0), status: std::sync::Mutex::new(status), + observed_statuses: std::sync::Mutex::new(BTreeMap::new()), create_id: "mock-sbx".to_owned(), created_specs: std::sync::Mutex::new(Vec::new()), + block_create: AtomicBool::new(false), + create_started: Notify::new(), + allow_create: Notify::new(), resume_fails: AtomicBool::new(false), stopped: std::sync::Mutex::new(Vec::new()), proxy_ensures: std::sync::Mutex::new(Vec::new()), missing_on_stop: std::sync::Mutex::new(BTreeSet::new()), + failing_on_stop: std::sync::Mutex::new(BTreeSet::new()), } } @@ -6030,7 +6233,22 @@ mod adoption_tests { } fn set_status(&self, status: SandboxStatus) { - *self.status.lock().unwrap() = status; + *self.status.lock().unwrap() = status.clone(); + for observed in self.observed_statuses.lock().unwrap().values_mut() { + *observed = status.clone(); + } + } + + fn block_create(&self) { + self.block_create.store(true, Ordering::SeqCst); + } + + async fn wait_for_create_started(&self) { + self.create_started.notified().await; + } + + fn allow_create(&self) { + self.allow_create.notify_one(); } fn fail_resume(&self) { @@ -6044,6 +6262,13 @@ mod adoption_tests { .insert(sandbox_id.to_owned()); } + fn fail_stop(&self, sandbox_id: &str) { + self.failing_on_stop + .lock() + .unwrap() + .insert(sandbox_id.to_owned()); + } + fn stopped(&self) -> Vec { self.stopped.lock().unwrap().clone() } @@ -6064,7 +6289,15 @@ mod adoption_tests { } async fn create(&self, spec: SandboxSpec) -> SandboxResult { + if self.block_create.load(Ordering::SeqCst) { + self.create_started.notify_one(); + self.allow_create.notified().await; + } self.created_specs.lock().unwrap().push(spec); + self.observed_statuses + .lock() + .unwrap() + .insert(self.create_id.clone(), SandboxStatus::Running); Ok(SandboxHandle::new( SandboxId::new(self.create_id.clone()), "mock", @@ -6089,6 +6322,9 @@ mod adoption_tests { } async fn status(&self, _id: &SandboxId) -> SandboxResult { + if let Some(status) = self.observed_statuses.lock().unwrap().get(_id.as_str()) { + return Ok(status.clone()); + } Ok(self.status.lock().unwrap().clone()) } @@ -6098,14 +6334,32 @@ mod adoption_tests { } async fn list_observed(&self) -> SandboxResult> { - Ok(Vec::new()) + Ok(self + .observed_statuses + .lock() + .unwrap() + .iter() + .map(|(id, status)| { + ObservedSandbox::new(SandboxId::new(id.clone()), "mock", status.clone()) + }) + .collect()) } async fn stop(&self, id: &SandboxId) -> SandboxResult<()> { if self.missing_on_stop.lock().unwrap().contains(id.as_str()) { return Err(SandboxError::NotFound(id.as_str().to_owned())); } + if self.failing_on_stop.lock().unwrap().contains(id.as_str()) { + return Err(SandboxError::backend(format!( + "mock stop failure for {}", + id.as_str() + ))); + } self.stopped.lock().unwrap().push(id.as_str().to_owned()); + self.observed_statuses + .lock() + .unwrap() + .insert(id.as_str().to_owned(), SandboxStatus::Stopped); Ok(()) } @@ -6173,11 +6427,20 @@ mod adoption_tests { eprintln!("skipping: SESSION_RUNTIME_TEST_DATABASE_URL not set"); return None; }; - let store = PgSessionStore::connect(&url) - .await - .expect("connect test db"); - store.run_migrations().await.expect("run migrations"); - Some(store) + static MIGRATIONS: OnceCell<()> = OnceCell::const_new(); + MIGRATIONS + .get_or_init(|| async { + let store = PgSessionStore::connect(&url) + .await + .expect("connect test db for migrations"); + store.run_migrations().await.expect("run migrations"); + }) + .await; + Some( + PgSessionStore::connect(&url) + .await + .expect("connect fresh test store after migrations"), + ) } async fn orphaned_execution( @@ -7014,7 +7277,7 @@ mod adoption_tests { let runtime = runtime_with(&store, backend.clone()); let outcome = runtime - .release_thread(&thread_key, Some("rel-test-1"), true) + .release_thread(&thread_key, Some("rel-test-1"), Some("sbx-release"), true) .await .expect("release thread"); @@ -7039,6 +7302,45 @@ mod adoption_tests { wait_for_event(&store, &thread_key, "session.released").await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn release_thread_reports_backend_stop_failure() { + let Some(store) = test_store().await else { + return; + }; + let _serial = TEST_LOCK.lock().await; + let thread_key = + ThreadKey::parse(format!("test:release-stop-{}", uuid::Uuid::new_v4())).unwrap(); + orphaned_execution(&store, &thread_key, Some("sbx-stop-failure"), true).await; + + let backend = Arc::new(MockBackend::new(SandboxStatus::Running, Vec::new())); + backend.fail_stop("sbx-stop-failure"); + let runtime = runtime_with(&store, backend.clone()); + + let outcome = runtime + .release_thread( + &thread_key, + Some("rel-stop-failure"), + Some("sbx-stop-failure"), + true, + ) + .await + .expect("release transaction still returns its stop report"); + + assert!(!outcome.sandbox_released); + assert!( + outcome + .sandbox_release_error + .as_deref() + .is_some_and(|error| error.contains("mock stop failure")) + ); + assert!(backend.stopped().is_empty()); + let session = store + .get_session(&thread_key) + .await + .expect("get released session"); + assert!(session.sandbox_id.is_none()); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn release_thread_rejects_active_execution_without_cancel() { let Some(store) = test_store().await else { @@ -7053,12 +7355,19 @@ mod adoption_tests { let runtime = runtime_with(&store, backend.clone()); let error = runtime - .release_thread(&thread_key, Some("rel-test-reject"), false) + .release_thread( + &thread_key, + Some("rel-test-reject"), + Some("sbx-release-reject"), + false, + ) .await .expect_err("release should reject active executions without cancellation"); assert!( - error.to_string().contains("pass cancel_inflight=true"), + error + .to_string() + .contains("retry with cancel_inflight=true"), "unexpected error: {error}" ); assert_eq!(backend.stopped(), Vec::::new()); @@ -7073,4 +7382,427 @@ mod adoption_tests { .expect("execution row"); assert_eq!(execution.status.as_ref(), "running"); } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn release_thread_rejects_a_stale_sandbox_fence_without_stopping_current_sandbox() { + let Some(store) = test_store().await else { + return; + }; + let _serial = TEST_LOCK.lock().await; + let thread_key = + ThreadKey::parse(format!("test:release-fence-{}", uuid::Uuid::new_v4())).unwrap(); + store + .create_or_get_session(&thread_key, &HarnessType::Codex, None, json!({})) + .await + .expect("create session"); + store + .update_sandbox_id(&thread_key, Some("sbx-current")) + .await + .expect("bind current sandbox"); + + let backend = Arc::new(MockBackend::new(SandboxStatus::Running, Vec::new())); + let runtime = runtime_with(&store, backend.clone()); + let error = runtime + .release_thread(&thread_key, Some("rel-stale"), Some("sbx-stale"), true) + .await + .expect_err("stale fence must reject release"); + + assert!(error.to_string().contains("caller fence")); + assert_eq!(backend.stopped(), Vec::::new()); + assert_eq!( + store + .get_session(&thread_key) + .await + .expect("current session") + .sandbox_id + .as_deref(), + Some("sbx-current") + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn release_thread_requires_a_fence_for_an_assigned_sandbox() { + let Some(store) = test_store().await else { + return; + }; + let _serial = TEST_LOCK.lock().await; + let thread_key = ThreadKey::parse(format!( + "test:release-fence-required-{}", + uuid::Uuid::new_v4() + )) + .unwrap(); + store + .create_or_get_session(&thread_key, &HarnessType::Codex, None, json!({})) + .await + .expect("create session"); + store + .update_sandbox_id(&thread_key, Some("sbx-current")) + .await + .expect("bind current sandbox"); + + let backend = Arc::new(MockBackend::new(SandboxStatus::Running, Vec::new())); + let runtime = runtime_with(&store, backend.clone()); + let error = runtime + .release_thread(&thread_key, Some("rel-no-fence"), None, true) + .await + .expect_err("assigned sandbox release must require a caller fence"); + + assert!( + error + .to_string() + .contains("expected_sandbox_id is required") + ); + assert_eq!(backend.stopped(), Vec::::new()); + assert_eq!( + store + .get_session(&thread_key) + .await + .expect("current session") + .sandbox_id + .as_deref(), + Some("sbx-current") + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn release_and_sandbox_assignment_race_has_exactly_one_winner() { + let Some(store) = test_store().await else { + return; + }; + let _serial = TEST_LOCK.lock().await; + let thread_key = + ThreadKey::parse(format!("test:release-race-{}", uuid::Uuid::new_v4())).unwrap(); + store + .create_or_get_session(&thread_key, &HarnessType::Codex, None, json!({})) + .await + .expect("create session"); + store + .update_sandbox_id(&thread_key, Some("sbx-old")) + .await + .expect("bind old sandbox"); + let execution_id = store + .create_execution(&thread_key, None, json!({})) + .await + .expect("create execution") + .execution + .execution_id; + store + .mark_execution_running(&execution_id) + .await + .expect("mark running"); + + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let assignment_store = store.clone(); + let assignment_thread = thread_key.clone(); + let assignment_execution = execution_id.clone(); + let assignment_barrier = barrier.clone(); + let assign = async move { + assignment_barrier.wait().await; + assignment_store + .assign_sandbox_to_active_execution( + &assignment_thread, + &assignment_execution, + Some("sbx-old"), + "sbx-new", + &default_capabilities(), + ) + .await + .expect("assignment decision") + }; + let release_store = store.clone(); + let release_thread = thread_key.clone(); + let release_barrier = barrier.clone(); + let release = async move { + release_barrier.wait().await; + release_store + .release_session_if_sandbox_matches( + &release_thread, + Some("sbx-old"), + true, + "concurrent release", + ) + .await + .expect("release decision") + }; + + let (assigned, released) = tokio::join!(assign, release); + match (assigned, released) { + (Some(_), ReleaseSessionResult::SandboxMismatch { current_sandbox_id }) => { + assert_eq!(current_sandbox_id.as_deref(), Some("sbx-new")); + assert_eq!( + store + .get_session(&thread_key) + .await + .expect("assigned session") + .sandbox_id + .as_deref(), + Some("sbx-new") + ); + } + (None, ReleaseSessionResult::Released { session, .. }) => { + assert_eq!(session.sandbox_id, None); + } + (assigned, released) => { + panic!("invalid race outcome: assigned={assigned:?}, released={released:?}") + } + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn cancelled_execution_cannot_publish_a_late_cold_sandbox() { + let Some(store) = test_store().await else { + return; + }; + let _serial = TEST_LOCK.lock().await; + let thread_key = + ThreadKey::parse(format!("test:late-cold-{}", uuid::Uuid::new_v4())).unwrap(); + store + .create_or_get_session(&thread_key, &HarnessType::Codex, None, json!({})) + .await + .expect("create session"); + let execution_id = store + .create_execution(&thread_key, None, json!({})) + .await + .expect("create execution") + .execution + .execution_id; + store + .cancel_execution_if_active(&execution_id, "release already won") + .await + .expect("cancel execution"); + + let backend = Arc::new(MockBackend::new(SandboxStatus::Running, Vec::new())); + let runtime = runtime_with(&store, backend.clone()); + let error = runtime + .ensure_session_sandbox(EnsureSessionSandboxRequest { + thread_key: &thread_key, + harness_type: &HarnessType::Codex, + persona_id: None, + existing_sandbox_id: None, + existing_sandbox_capabilities: None, + iron_control_principal: None, + desired_capabilities: &default_capabilities(), + execution_id: &execution_id, + }) + .await + .expect_err("cancelled execution cannot assign sandbox"); + + assert!(error.to_string().contains("execution ended")); + assert_eq!(backend.stopped(), vec!["mock-sbx".to_owned()]); + assert_eq!( + store + .get_session(&thread_key) + .await + .expect("session") + .sandbox_id, + None + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn release_winning_allocation_race_stays_cancelled_not_failed() { + let Some(store) = test_store().await else { + return; + }; + let _serial = TEST_LOCK.lock().await; + let thread_key = + ThreadKey::parse(format!("test:release-allocation-{}", uuid::Uuid::new_v4())).unwrap(); + store + .create_or_get_session(&thread_key, &HarnessType::Codex, None, json!({})) + .await + .expect("create session"); + + let backend = Arc::new(MockBackend::new(SandboxStatus::Running, Vec::new())); + backend.block_create(); + // Independent runtimes model the cross-replica race. They share the + // database CAS but intentionally do not share the in-process lock. + let allocator = runtime_with(&store, backend.clone()); + let releaser = runtime_with(&store, backend.clone()); + let allocator_thread = thread_key.clone(); + let allocation = tokio::spawn(async move { + allocator + .execute_session( + &allocator_thread, + ExecuteSessionInput { + idempotency_key: Some(format!("race-{}", uuid::Uuid::new_v4())), + metadata: None, + input_lines: vec![json!({"type": "user", "text": "race"}).to_string()], + idle_timeout_ms: None, + max_duration_ms: None, + }, + ) + .await + }); + + backend.wait_for_create_started().await; + let released = releaser + .release_thread(&thread_key, Some("release-wins"), None, true) + .await + .expect("release wins before allocation publishes"); + assert!(released.execution_cancelled); + assert_eq!(released.session.sandbox_id, None); + + backend.allow_create(); + let allocation_error = allocation + .await + .expect("allocation task") + .expect_err("cancelled execution cannot publish sandbox"); + assert!(allocation_error.to_string().contains("execution ended")); + + let execution = store + .latest_execution_for_thread(&thread_key) + .await + .expect("latest execution") + .expect("execution row"); + assert_eq!(execution.status, ExecutionStatus::Cancelled); + let session = store.get_session(&thread_key).await.expect("session row"); + assert_eq!(session.status.as_ref(), "idle"); + assert_eq!(session.sandbox_id, None); + assert_eq!(backend.stopped(), vec!["mock-sbx".to_owned()]); + assert!( + events(&store, &thread_key) + .await + .iter() + .all(|event| event.event_type != "session.execution_failed"), + "losing allocation must not emit a failure after release cancellation" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn drain_waits_for_inflight_allocation_and_rejects_new_allocations() { + let Some(store) = test_store().await else { + return; + }; + let _serial = TEST_LOCK.lock().await; + let first_thread = + ThreadKey::parse(format!("test:drain-first-{}", uuid::Uuid::new_v4())).unwrap(); + let second_thread = + ThreadKey::parse(format!("test:drain-second-{}", uuid::Uuid::new_v4())).unwrap(); + for thread_key in [&first_thread, &second_thread] { + store + .create_or_get_session(thread_key, &HarnessType::Codex, None, json!({})) + .await + .expect("create session"); + } + let first_execution = store + .create_execution(&first_thread, None, json!({})) + .await + .expect("first execution") + .execution + .execution_id; + let second_execution = store + .create_execution(&second_thread, None, json!({})) + .await + .expect("second execution") + .execution + .execution_id; + + let backend = Arc::new(MockBackend::new(SandboxStatus::Running, Vec::new())); + backend.block_create(); + let runtime = runtime_with(&store, backend.clone()); + let allocating_runtime = runtime.clone(); + let allocating_thread = first_thread.clone(); + let allocating_execution = first_execution.clone(); + let allocation = tokio::spawn(async move { + allocating_runtime + .ensure_session_sandbox(EnsureSessionSandboxRequest { + thread_key: &allocating_thread, + harness_type: &HarnessType::Codex, + persona_id: None, + existing_sandbox_id: None, + existing_sandbox_capabilities: None, + iron_control_principal: None, + desired_capabilities: &default_capabilities(), + execution_id: &allocating_execution, + }) + .await + }); + backend.wait_for_create_started().await; + + let draining_runtime = runtime.clone(); + let drain = tokio::spawn(async move { draining_runtime.drain().await }); + while !runtime.draining.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + + let rejected = runtime + .ensure_session_sandbox(EnsureSessionSandboxRequest { + thread_key: &second_thread, + harness_type: &HarnessType::Codex, + persona_id: None, + existing_sandbox_id: None, + existing_sandbox_capabilities: None, + iron_control_principal: None, + desired_capabilities: &default_capabilities(), + execution_id: &second_execution, + }) + .await + .expect_err("drain must reject a new allocation"); + assert!(matches!(rejected, SessionRuntimeError::Draining)); + + backend.allow_create(); + assert_eq!( + allocation + .await + .expect("allocation task") + .expect("in-flight allocation completes before inventory"), + "mock-sbx" + ); + let report = drain.await.expect("drain task").expect("drain succeeds"); + assert_eq!(report.stopped, vec!["mock-sbx".to_owned()]); + assert!(report.failed.is_empty()); + let observed = backend.list_observed().await.expect("post-drain inventory"); + assert!( + observed.iter().all(|sandbox| sandbox.status.is_terminal()), + "post-drain inventory still has a live sandbox: {observed:?}" + ); + assert_eq!(backend.created_specs().len(), 1); + + let post_drain = runtime + .ensure_session_sandbox(EnsureSessionSandboxRequest { + thread_key: &second_thread, + harness_type: &HarnessType::Codex, + persona_id: None, + existing_sandbox_id: None, + existing_sandbox_capabilities: None, + iron_control_principal: None, + desired_capabilities: &default_capabilities(), + execution_id: &second_execution, + }) + .await + .expect_err("drain gate remains permanently closed"); + assert!(matches!(post_drain, SessionRuntimeError::Draining)); + + for execution_id in [&first_execution, &second_execution] { + store + .fail_execution_if_active(execution_id, "test cleanup") + .await + .expect("terminalize test execution"); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn drain_permanently_pauses_warm_replenishment() { + let Some(store) = test_store().await else { + return; + }; + let _serial = TEST_LOCK.lock().await; + let backend = Arc::new(MockBackend::new(SandboxStatus::Running, Vec::new())); + let runtime = runtime_with_warm_pool( + &store, + backend.clone(), + format!("drain-pause-{}", uuid::Uuid::new_v4()), + ); + + runtime.drain().await.expect("drain runtime"); + runtime + .warm_pool + .as_ref() + .expect("warm pool") + .clone() + .spawn_replenisher(); + sleep(Duration::from_millis(50)).await; + + assert_eq!(backend.created_specs(), Vec::::new()); + } } diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/.checksums.sha384 b/services/api-rs/crates/centaur-session-sqlx/migrations/.checksums.sha384 new file mode 100644 index 000000000..b5a0ab59c --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/.checksums.sha384 @@ -0,0 +1,43 @@ +92b33283d76e4265c9fff457e81bb83a09e489a9a27e78dffcf29006eba7e850d8b1d80a3e6f1f08d728e2cfa2b2db15 services/api-rs/crates/centaur-session-sqlx/migrations/0001_session_control_plane.sql +d87e505d95c7e91c463f675961f87197bad6fd8e132d5e1fc021ea78a4f84a4a3ef38315ad8c51a8ebfaf988d4f0a923 services/api-rs/crates/centaur-session-sqlx/migrations/0002_session_event_notifications.sql +4a9f482eb8190f4c2abf599d1630e4f4d0f442b1f495533b223be64a4858854725a23d55b5db3df4cfb39b7e60cd4a33 services/api-rs/crates/centaur-session-sqlx/migrations/0003_session_iron_control_principal.sql +640d069dc8b5acf2d3a9bc62f1544e84f9968ed4aa041929c2dbb3ef3e79de5093338f87cae2d32c3f95314e1a0ddd3a services/api-rs/crates/centaur-session-sqlx/migrations/0004_session_warm_pool.sql +7485641d877d0ea09d06aee5ef78093fc2ec8a716f5b46b5850c217cf567cb3458f518c3b20521f678af862d7da1d971 services/api-rs/crates/centaur-session-sqlx/migrations/0005_session_handoff_idempotency.sql +f9e369500e6e3e140498797b083436260d8d88c46b5145b94cbdbbc8c693ad070d780966b87110922499b071a75940a6 services/api-rs/crates/centaur-session-sqlx/migrations/0006_session_persona_id.sql +f3e356ca2ffa0a9dc42cfafeed9ea507a30e1a402ce70b888b7f48f6ebde907e1e9c3a3b0b7be96110e0feef1741be30 services/api-rs/crates/centaur-session-sqlx/migrations/0007_absurd_workflows.sql +01a123b518bcbe7ba288f08f471bd2ee65a70e69aca2e56beb3849986880a6c8ab538dc17bf6b799d8b8a6bb5405ad75 services/api-rs/crates/centaur-session-sqlx/migrations/0008_absurd_complete_run_clears_failure.sql +06fb36fa25467808c5c74cd3df51389c10d8ede921bec55d5e4719ffa329ff0792e4f02635e2496889ba330b4477a753 services/api-rs/crates/centaur-session-sqlx/migrations/0009_absurd_await_event_task_guard.sql +2cc855ad49192675edfec7e460c1edeca83d95c7387db854c92c319dfd9957380a024b1a614387d8c384dc3db6a4c898 services/api-rs/crates/centaur-session-sqlx/migrations/0010_user_feedback.sql +af6802d40d2281fcee9afd4f44970a8a1d39b928ba5ff8b504f8cdf63a70f4257c5bbf73b9c80c581505dd7479fa28b8 services/api-rs/crates/centaur-session-sqlx/migrations/0011_slack_sync_tables.sql +4c5484e974feda89ced85376ababe05457ebc1e9d2756b835c39fcf9c010a318f22166988da9f82d6b10da4587d1de38 services/api-rs/crates/centaur-session-sqlx/migrations/0012_company_context_documents.sql +5a76b4f51b74d6da97ca7af81dd9baa7a414ddfef3514d176f24d954124f6ad7628c8ba4624814c4ee49890b1d6cdd20 services/api-rs/crates/centaur-session-sqlx/migrations/0013_google_drive_sync_tables.sql +f6e89154b724cc1433a158f6adf485767339ab2875538bc08db0726b8d8e65034d3bd8a73444862b93f6ac5fcd79c2dc services/api-rs/crates/centaur-session-sqlx/migrations/0014_google_calendar_sync_tables.sql +5226709400e6bab8e1c95cc4bae30bb927e5778c9f847697f73a241c1d4bb44ff3234ff6b5711af1e8f0d541125f3a82 services/api-rs/crates/centaur-session-sqlx/migrations/0015_linear_sync_tables.sql +47f2607b186daf18cf2623c57db836b4a8c41524b886f406de4ac69bc0c4cd531d82d3c43ae7817693e85d9b90f2c866 services/api-rs/crates/centaur-session-sqlx/migrations/0016_slack_context_rls.sql +8f9c444884532c79dc109c56bc63fc30b16f5367c3bad88d44383295ebac1de63688242a55fede8451767b62f34c8d4e services/api-rs/crates/centaur-session-sqlx/migrations/0017_slack_sync_message_attachments.sql +8b1bfaec97b359a782c0036289a7c63a1799555e32af208a61dbc8e7f51733452f644dc6599ffbc8032f822af4f29e5e services/api-rs/crates/centaur-session-sqlx/migrations/0018_slack_context_rls_admin_channels.sql +01c4a4daf4fcd3941d81947a89bafb079882c1195312d86fdf8d8cdfc66e9c54e06eea448b18bd44a3f579346376d271 services/api-rs/crates/centaur-session-sqlx/migrations/0019_company_context_public_slack_docs.sql +e510559d9fa76a5d561203a002d7ef253ae4bfdb4991d0b7593ad4aa7e3e45e3f9b2836be5309d9e5d446f7ea276e075 services/api-rs/crates/centaur-session-sqlx/migrations/0020_centaur_readonly_role.sql +4a794c94abd60a11b0f582c67fe556295bc24a7420188e41d6f5f8ac084619b45ed7a699b1cf26be72a83a0c74b4f527 services/api-rs/crates/centaur-session-sqlx/migrations/0021_centaur_readonly_role_only.sql +48290511f4040be3592da6355a49f9fa9c4b73f1fe98dd62de7f50d16546559211ce8881df585ed1d61041d123ca264d services/api-rs/crates/centaur-session-sqlx/migrations/0022_etl_context_rls.sql +2459805495164decff2d73452cfab4d74653d145da4de71b7b3f9f482e241fe09bed900f6f2d8500b0116ae1ea29fc67 services/api-rs/crates/centaur-session-sqlx/migrations/0023_drop_slack_context_rls_admin_channels.sql +d36a02d3cb1586cfa6311f5254da2a65c66238f65738b0dc94e277ae61d62485620e52e25c2e3ced6bd0e4c08ede41b6 services/api-rs/crates/centaur-session-sqlx/migrations/0024_centaur_readonly_rls_policies.sql +74e25f7193d9975eaca20ca8b31afbcb5e50616fc51414a097f3cc76ced278cfdcbe9f50b980a9f6dcc3db86bc7be83e services/api-rs/crates/centaur-session-sqlx/migrations/0025_session_event_execution_type_idx.sql +d998621b4d6db9d86692f3a853842ab239380ea61628f810722d371c390a1a44947a7d71f75153b6cb1a9997526651cc services/api-rs/crates/centaur-session-sqlx/migrations/0026_slack_archive_imports.sql +552d69ebfefb0e6c50f04f941b99f6484b2f927695f1ef65c228b903e13eb7c88341301ae5170b00c311f74d6f86f9ac services/api-rs/crates/centaur-session-sqlx/migrations/0027_drop_slack_archive_import_workspace.sql +3431ef06d7af8234f78d3b335487b10c4910d7a76ab44a2ff213d5e191a550b90802bfe7a7c217dc3effb2d9a6a15e68 services/api-rs/crates/centaur-session-sqlx/migrations/0028_slack_dm_sync_tables.sql +fd4aaae62dd5fd310ac028c335dfda19eb2169c199b738c068fb1ef0716426ff00623382f18be5f1a33ef4f7ebf109e3 services/api-rs/crates/centaur-session-sqlx/migrations/0029_slack_dm_context_documents.sql +c472c35a307682a84bee8172af27348457442b6d7598dab1eaa23ac8068c79d33c544fa078ded09b130792ab673f0142 services/api-rs/crates/centaur-session-sqlx/migrations/0030_slack_dm_conversation_context_documents.sql +2feeec8685e4e70ab831710247fe775c160fb97a7d5802cdfc9abaaebca914b5b6ddd49640cb5779bdf256d6595ea1b0 services/api-rs/crates/centaur-session-sqlx/migrations/0031_google_docs_oauth_sync_tables.sql +2acc60a0f3c8c9f8fafab4ae1ff394f1b3b7d7f2e088afc573ebe7e528b8a3501e1877508c2df53540fba51743535193 services/api-rs/crates/centaur-session-sqlx/migrations/0032_session_sandbox_capabilities.sql +875bf729aa1324d8db01e420825ca56f978a33f98ca73605c0a52d97a730ff7c625c27b6cf11caacb78f8062b9ad85b1 services/api-rs/crates/centaur-session-sqlx/migrations/0033_session_title.sql +94fd3890f9791d15a206570839139c8a1c13b6ff47e286635f16951eaf80090025a85dc158dd28c3d207de420662ae45 services/api-rs/crates/centaur-session-sqlx/migrations/0034_session_sandbox_activity.sql +f4e1eb86575901edea9ea540ec53354f4b159616080850fa79b8f68f91a3a6c592905af94d0b9e4299208418ad254775 services/api-rs/crates/centaur-session-sqlx/migrations/0035_session_execution_stdout_owner.sql +e4aed269a8eb82b921399de3fc399e7e2687c04519d9e581a5337b673f9ea2f237b71a1a5c49c5f62856682db971259d services/api-rs/crates/centaur-session-sqlx/migrations/0036_session_sandbox_api_server_capability.sql +0ffce7c31d8a104b6d7bbd6e56a8036325bc7cd8d1226eb671b08ed999583b9dd56059f60400fef2d687fc463c0c993f services/api-rs/crates/centaur-session-sqlx/migrations/0037_readonly_all_workflow_queues.sql +b1a1b0fe52fdd876bd53a96b2fff43568cc4631bf59712f5231f37f25cf209d9f72db0b5a3b3566cbd83b72b5e1861bf services/api-rs/crates/centaur-session-sqlx/migrations/0038_session_sandbox_repo_cache_access.sql +8a2d3e308204abe4fd807f9c9bebdc060a7c3c6e8950a4df6469bc07e973867486c0357921779a46a03a68cc6c828ec4 services/api-rs/crates/centaur-session-sqlx/migrations/0039_slack_private_channels.sql +65cbd5bafcfd4e124d51bd99cc501fee923e87882d1032dad34f4cfba3fd78b5d4869b61bb765f8d76310e520f44cac2 services/api-rs/crates/centaur-session-sqlx/migrations/0040_granola_sync_tables.sql +38f3d13f44fa29264529012118f6b3921de1cf7e8b75e0e3ffcd1b207124eb7e0f6bde7504cdf3ac51d29a99291a77b0 services/api-rs/crates/centaur-session-sqlx/migrations/0041_attio_sync_tables.sql +dcd05c897c50a15b8bc2342e76d520133ac2370c23e88e119d6a7ff4a615a88b5512209a33ee04c5070f4f4afeda020e services/api-rs/crates/centaur-session-sqlx/migrations/0042_centaur_readonly_slack_dm_rls.sql +fd353077080a2cbeaaa7242f5726415663baeb00b74dd1743db01cbeda51f0748714f0f59400907411bed45c52fcac2a services/api-rs/crates/centaur-session-sqlx/migrations/0043_session_sandbox_content_revision.sql diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0033_session_title.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0033_session_title.sql new file mode 100644 index 000000000..7e20b465e --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0033_session_title.sql @@ -0,0 +1,2 @@ +alter table sessions + add column if not exists title text; diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0034_session_sandbox_activity.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0034_session_sandbox_activity.sql new file mode 100644 index 000000000..eac1434ca --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0034_session_sandbox_activity.sql @@ -0,0 +1,21 @@ +alter table sessions + add column if not exists sandbox_last_active_at timestamptz; + +update sessions +set sandbox_last_active_at = coalesce(sandbox_last_active_at, updated_at, created_at) +where sandbox_id is not null; + +create index if not exists sessions_sandbox_activity_idx + on sessions (sandbox_last_active_at, thread_key) + where sandbox_id is not null; + +alter table session_warm_sandboxes + drop constraint if exists session_warm_sandboxes_status_supported; + +alter table session_warm_sandboxes + add constraint session_warm_sandboxes_status_supported + check (status in ('ready', 'claimed', 'evicting', 'failed')); + +create index if not exists session_warm_sandboxes_evicting_idx + on session_warm_sandboxes (updated_at, sandbox_id) + where status = 'evicting'; diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0035_session_execution_stdout_owner.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0035_session_execution_stdout_owner.sql new file mode 100644 index 000000000..9dd0c44b1 --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0035_session_execution_stdout_owner.sql @@ -0,0 +1,7 @@ +alter table session_executions + add column if not exists stdout_owner_id text, + add column if not exists stdout_owner_lease_expires_at timestamptz; + +create index if not exists session_executions_stdout_owner_lease_idx + on session_executions (stdout_owner_lease_expires_at) + where status in ('queued', 'running') and stdout_owner_id is not null; diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0036_session_sandbox_api_server_capability.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0036_session_sandbox_api_server_capability.sql new file mode 100644 index 000000000..f7faa61c5 --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0036_session_sandbox_api_server_capability.sql @@ -0,0 +1,7 @@ +alter table sessions + add column if not exists sandbox_api_server_enabled boolean; + +update sessions +set sandbox_api_server_enabled = true +where sandbox_observability_enabled is not null + and sandbox_api_server_enabled is null; diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0037_readonly_all_workflow_queues.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0037_readonly_all_workflow_queues.sql new file mode 100644 index 000000000..5a0095898 --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0037_readonly_all_workflow_queues.sql @@ -0,0 +1,91 @@ +select absurd.create_queue('centaur_workflows'); +select absurd.create_queue('centaur_workflows_slack_live'); +select absurd.create_queue('centaur_workflows_etl'); +select absurd.create_queue('centaur_workflows_etl_backfill'); + +create or replace view centaur_readonly_workflow_runs as +select + 'centaur_workflows'::text as queue_name, + r.run_id::text as run_id, + t.task_id::text as task_id, + t.task_name, + t.params ->> 'workflow_name' as workflow_name, + t.params ->> 'harness_type' as harness_type, + t.state, + t.attempts, + t.max_attempts, + t.enqueue_at as created_at, + t.first_started_at, + r.started_at, + r.completed_at, + r.failed_at, + r.available_at, + r.claimed_by is not null as claimed, + t.cancelled_at +from absurd.t_centaur_workflows t +left join absurd.r_centaur_workflows r on r.run_id = t.last_attempt_run +union all +select + 'centaur_workflows_slack_live'::text as queue_name, + r.run_id::text as run_id, + t.task_id::text as task_id, + t.task_name, + t.params ->> 'workflow_name' as workflow_name, + t.params ->> 'harness_type' as harness_type, + t.state, + t.attempts, + t.max_attempts, + t.enqueue_at as created_at, + t.first_started_at, + r.started_at, + r.completed_at, + r.failed_at, + r.available_at, + r.claimed_by is not null as claimed, + t.cancelled_at +from absurd.t_centaur_workflows_slack_live t +left join absurd.r_centaur_workflows_slack_live r on r.run_id = t.last_attempt_run +union all +select + 'centaur_workflows_etl'::text as queue_name, + r.run_id::text as run_id, + t.task_id::text as task_id, + t.task_name, + t.params ->> 'workflow_name' as workflow_name, + t.params ->> 'harness_type' as harness_type, + t.state, + t.attempts, + t.max_attempts, + t.enqueue_at as created_at, + t.first_started_at, + r.started_at, + r.completed_at, + r.failed_at, + r.available_at, + r.claimed_by is not null as claimed, + t.cancelled_at +from absurd.t_centaur_workflows_etl t +left join absurd.r_centaur_workflows_etl r on r.run_id = t.last_attempt_run +union all +select + 'centaur_workflows_etl_backfill'::text as queue_name, + r.run_id::text as run_id, + t.task_id::text as task_id, + t.task_name, + t.params ->> 'workflow_name' as workflow_name, + t.params ->> 'harness_type' as harness_type, + t.state, + t.attempts, + t.max_attempts, + t.enqueue_at as created_at, + t.first_started_at, + r.started_at, + r.completed_at, + r.failed_at, + r.available_at, + r.claimed_by is not null as claimed, + t.cancelled_at +from absurd.t_centaur_workflows_etl_backfill t +left join absurd.r_centaur_workflows_etl_backfill r on r.run_id = t.last_attempt_run; + +grant select on table centaur_readonly_workflow_runs to centaur_readonly; From 4f5834bc67d82fef0785e4b53762cfa62eba53f0 Mon Sep 17 00:00:00 2001 From: "fineas-bot[bot]" <258147136+fineas-bot[bot]@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:04:19 -0400 Subject: [PATCH 3/9] feat: add schema-forward rollback bridge (3/3) Reviewed emergency bridge for preserving the frozen forward migration ledger and workflow state during a staged rollback. This commit does not deploy or publish the bridge. --- ...0038_session_sandbox_repo_cache_access.sql | 10 + .../0039_slack_private_channels.sql | 130 ++++++++ .../migrations/0040_granola_sync_tables.sql | 277 ++++++++++++++++ .../migrations/0041_attio_sync_tables.sql | 131 ++++++++ .../0042_centaur_readonly_slack_dm_rls.sql | 139 ++++++++ .../0043_session_sandbox_content_revision.sql | 21 ++ .../crates/centaur-session-sqlx/src/lib.rs | 188 ++++++++++- .../crates/centaur-workflows/src/lib.rs | 313 ++++++++++++------ workflows/slack/archive_import.py | 8 +- workflows/slack/tests/test_archive_import.py | 5 + 10 files changed, 1101 insertions(+), 121 deletions(-) create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/0038_session_sandbox_repo_cache_access.sql create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/0039_slack_private_channels.sql create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/0040_granola_sync_tables.sql create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/0041_attio_sync_tables.sql create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/0042_centaur_readonly_slack_dm_rls.sql create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/0043_session_sandbox_content_revision.sql diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0038_session_sandbox_repo_cache_access.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0038_session_sandbox_repo_cache_access.sql new file mode 100644 index 000000000..a8e4dab30 --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0038_session_sandbox_repo_cache_access.sql @@ -0,0 +1,10 @@ +alter table sessions + add column if not exists sandbox_repo_cache_access text; + +update sessions +set sandbox_repo_cache_access = case + when sandbox_repo_cache_enabled then 'all' + else 'none' +end +where sandbox_repo_cache_access is null + and sandbox_repo_cache_enabled is not null; diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0039_slack_private_channels.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0039_slack_private_channels.sql new file mode 100644 index 000000000..eda032787 --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0039_slack_private_channels.sql @@ -0,0 +1,130 @@ +alter table slack_sync_channels + add column if not exists is_private boolean; + +-- Existing rows predate the dedicated privacy column. Trust an explicit +-- boolean from the stored Slack payload; when privacy is absent or malformed, +-- fail closed until a live Slack sync can classify the channel. +update slack_sync_channels +set is_private = case + when jsonb_typeof(raw_payload -> 'is_private') = 'boolean' + then (raw_payload ->> 'is_private')::boolean + else true +end; + +alter table slack_sync_channels + alter column is_private set default true, + alter column is_private set not null; + +create index if not exists idx_slack_sync_channels_private + on slack_sync_channels (is_private, channel_id); + +drop policy if exists centaur_readonly_slack_sync_channels_select + on slack_sync_channels; +create policy centaur_readonly_slack_sync_channels_select + on slack_sync_channels + for select + to centaur_readonly + using ( + not is_private + or channel_id = centaur_current_slack_channel_id() + ); + +drop policy if exists centaur_readonly_slack_sync_message_attachments_select + on slack_sync_message_attachments; +create policy centaur_readonly_slack_sync_message_attachments_select + on slack_sync_message_attachments + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_sync_channels channels + where channels.channel_id = slack_sync_message_attachments.channel_id + ) + ); + +drop policy if exists centaur_readonly_slack_sync_messages_select + on slack_sync_messages; +create policy centaur_readonly_slack_sync_messages_select + on slack_sync_messages + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_sync_channels channels + where channels.channel_id = slack_sync_messages.channel_id + ) + ); + +drop policy if exists centaur_readonly_company_context_documents_select + on company_context_documents; +create policy centaur_readonly_company_context_documents_select + on company_context_documents + for select + to centaur_readonly + using ( + source <> 'slack' + or exists ( + select 1 + from slack_sync_channels channels + where channels.channel_id = metadata ->> 'channel_id' + ) + ); + +-- Fineas company context intentionally exposes documents from public, +-- syncable Slack channels across channel-scoped principals. Keep direct access +-- to the principal's current channel (including a private channel), but never +-- use the Slack channel-id prefix as a privacy signal. +create or replace function centaur_slack_channel_is_public_syncable( + _schema name, + _channel_id text +) +returns boolean +language plpgsql +stable +security definer +set search_path = pg_catalog +as $$ +declare + public_syncable boolean; +begin + execute format( + 'select exists ( + select 1 + from %I.slack_sync_channels channels + where channels.channel_id = $1 + and channels.is_syncable + and not channels.is_private + )', + _schema + ) + into public_syncable + using _channel_id; + return coalesce(public_syncable, false); +end +$$; + +revoke all on function centaur_slack_channel_is_public_syncable(name, text) + from public; +grant execute on function centaur_slack_channel_is_public_syncable(name, text) + to centaur_slack_reader; + +drop policy if exists centaur_context_docs_reader_select + on company_context_documents; +create policy centaur_context_docs_reader_select + on company_context_documents + for select + to centaur_slack_reader + using ( + source <> 'slack' + or metadata ->> 'channel_id' = centaur_current_slack_channel_id() + or centaur_slack_channel_is_public_syncable( + current_schema(), + metadata ->> 'channel_id' + ) + ); + +-- The old helper treated a C-prefixed id as public and was executable by +-- PUBLIC. Its only policy dependency was replaced immediately above. +drop function if exists centaur_slack_channel_is_syncable(name, text); diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0040_granola_sync_tables.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0040_granola_sync_tables.sql new file mode 100644 index 000000000..20284fba3 --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0040_granola_sync_tables.sql @@ -0,0 +1,277 @@ +create extension if not exists pg_search; + +create table if not exists granola_sync_runs ( + run_id text primary key, + workflow_run_id text, + mode text not null default 'incremental', + status text not null, + scopes_requested jsonb not null default '[]'::jsonb, + scopes_synced jsonb not null default '[]'::jsonb, + scopes_failed jsonb not null default '[]'::jsonb, + notes_seen integer not null default 0, + notes_upserted integer not null default 0, + transcripts_seen integer not null default 0, + transcripts_upserted integer not null default 0, + started_at timestamptz not null default now(), + finished_at timestamptz, + error_text text not null default '', + metadata jsonb not null default '{}'::jsonb +); + +create index if not exists idx_granola_sync_runs_started + on granola_sync_runs (started_at desc); + +create table if not exists granola_sync_notes ( + note_id text primary key, + title text not null default '', + owner_id text not null default '', + owner_email text not null default '', + owner_name text not null default '', + attendees jsonb not null default '[]'::jsonb, + access_emails text[] not null default array[]::text[], + calendar_event jsonb not null default '{}'::jsonb, + summary_markdown text not null default '', + summary_text text not null default '', + transcript_text text not null default '', + transcript_payload jsonb not null default '[]'::jsonb, + url text not null default '', + content_text text not null default '', + content_hash text not null default '', + source_created_at timestamptz, + source_updated_at timestamptz, + raw_payload jsonb not null default '{}'::jsonb, + source_run_id text references granola_sync_runs(run_id) on delete set null, + first_seen_at timestamptz not null default now(), + last_seen_at timestamptz not null default now(), + last_error text not null default '', + updated_at timestamptz not null default now() +); + +create index if not exists idx_granola_sync_notes_source_updated + on granola_sync_notes (source_updated_at desc); + +create index if not exists idx_granola_sync_notes_owner + on granola_sync_notes (owner_email, source_created_at desc); + +create index if not exists idx_granola_sync_notes_access_emails + on granola_sync_notes using gin (access_emails); + +create index if not exists idx_granola_sync_notes_text + on granola_sync_notes + using gin (to_tsvector('english', coalesce(content_text, ''))); + +create table if not exists granola_context_documents ( + document_id text primary key, + note_id text not null references granola_sync_notes(note_id) on delete cascade, + title text not null default '', + body text not null default '', + url text not null default '', + owner_id text not null default '', + owner_email text not null default '', + owner_name text not null default '', + access_emails text[] not null default array[]::text[], + attendee_labels text[] not null default array[]::text[], + occurred_at timestamptz, + source_updated_at timestamptz, + content_hash text not null default '', + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + unique (note_id), + check (document_id <> ''), + check (note_id <> '') +); + +create index if not exists idx_granola_context_documents_note_time + on granola_context_documents (note_id, occurred_at desc); + +create index if not exists idx_granola_context_documents_owner_time + on granola_context_documents (owner_email, occurred_at desc); + +create index if not exists idx_granola_context_documents_access_emails + on granola_context_documents using gin (access_emails); + +create index if not exists idx_granola_context_documents_metadata + on granola_context_documents using gin (metadata); + +drop index if exists idx_granola_context_documents_bm25; + +create index idx_granola_context_documents_bm25 + on granola_context_documents + using bm25 ( + document_id, + note_id, + title, + body, + url, + owner_id, + owner_email, + owner_name, + occurred_at, + source_updated_at, + metadata + ) + with ( + key_field = 'document_id', + text_fields = '{ + "document_id": { + "tokenizer": {"type": "keyword"} + }, + "note_id": { + "tokenizer": {"type": "keyword"} + }, + "owner_id": { + "tokenizer": {"type": "keyword"} + }, + "owner_email": { + "tokenizer": {"type": "keyword"} + } + }' + ); + +create table if not exists granola_sync_checkpoints ( + scope_id text primary key, + watermark_time timestamptz, + last_run_id text references granola_sync_runs(run_id) on delete set null, + last_success_at timestamptz, + last_error text not null default '', + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +do $$ +declare + role_name text; +begin + foreach role_name in array array[ + 'centaur_slack_reader', + 'centaur_slack_admin', + 'centaur_readonly' + ] loop + if exists (select 1 from pg_roles where rolname = role_name) then + execute format( + 'grant select on %s to %I', + 'granola_sync_runs, granola_sync_notes, granola_context_documents, granola_sync_checkpoints', + role_name + ); + end if; + end loop; +end $$; + +alter table granola_sync_runs enable row level security; +alter table granola_sync_notes enable row level security; +alter table granola_context_documents enable row level security; +alter table granola_sync_checkpoints enable row level security; + +create or replace function centaur_current_slack_user_email() +returns text +language sql +stable +security definer +set search_path = public +as $$ + select coalesce( + lower(nullif(current_setting('centaur.user_email', true), '')), + ( + select lower(nullif(coalesce( + users.raw_payload #>> '{profile,email}', + users.raw_payload ->> 'email' + ), '')) + from slack_sync_users users + where users.team_id = centaur_current_slack_team_id() + and users.user_id = centaur_current_slack_user_id() + limit 1 + ) + ) +$$; + +create or replace function centaur_granola_current_user_can_read( + p_access_emails text[] +) +returns boolean +language sql +stable +as $$ + select coalesce( + centaur_current_slack_user_email() = any(coalesce(p_access_emails, array[]::text[])), + false + ) +$$; + +do $$ +declare + role_name text; +begin + foreach role_name in array array[ + 'centaur_slack_reader', + 'centaur_slack_admin', + 'centaur_readonly' + ] loop + if exists (select 1 from pg_roles where rolname = role_name) then + execute format( + 'grant execute on function centaur_current_slack_user_email() to %I', + role_name + ); + execute format( + 'grant execute on function centaur_granola_current_user_can_read(text[]) to %I', + role_name + ); + end if; + end loop; +end $$; + +drop policy if exists centaur_granola_runs_admin_select on granola_sync_runs; +drop policy if exists centaur_granola_runs_reader_select on granola_sync_runs; +create policy centaur_granola_runs_reader_select + on granola_sync_runs for select to centaur_slack_reader + using (false); +drop policy if exists centaur_readonly_granola_sync_runs_select on granola_sync_runs; +create policy centaur_readonly_granola_sync_runs_select + on granola_sync_runs for select to centaur_readonly using (false); + +drop policy if exists centaur_granola_notes_admin_select on granola_sync_notes; +drop policy if exists centaur_granola_notes_reader_select on granola_sync_notes; +create policy centaur_granola_notes_reader_select + on granola_sync_notes for select to centaur_slack_reader + using (centaur_granola_current_user_can_read(access_emails)); +drop policy if exists centaur_readonly_granola_sync_notes_select on granola_sync_notes; +create policy centaur_readonly_granola_sync_notes_select + on granola_sync_notes for select to centaur_readonly using (false); + +drop policy if exists centaur_granola_context_documents_admin_select + on granola_context_documents; +drop policy if exists centaur_granola_context_documents_reader_select + on granola_context_documents; +create policy centaur_granola_context_documents_reader_select + on granola_context_documents for select to centaur_slack_reader + using (centaur_granola_current_user_can_read(access_emails)); +drop policy if exists centaur_readonly_granola_context_documents_select + on granola_context_documents; +create policy centaur_readonly_granola_context_documents_select + on granola_context_documents for select to centaur_readonly using (false); + +drop policy if exists centaur_granola_checkpoints_admin_select + on granola_sync_checkpoints; +drop policy if exists centaur_granola_checkpoints_reader_select + on granola_sync_checkpoints; +create policy centaur_granola_checkpoints_reader_select + on granola_sync_checkpoints for select to centaur_slack_reader + using (false); +drop policy if exists centaur_readonly_granola_sync_checkpoints_select + on granola_sync_checkpoints; +create policy centaur_readonly_granola_sync_checkpoints_select + on granola_sync_checkpoints for select to centaur_readonly using (false); + +do $$ +begin + if exists (select 1 from pg_roles where rolname = 'centaur_slack_admin') then + create policy centaur_granola_runs_admin_select + on granola_sync_runs for select to centaur_slack_admin using (true); + create policy centaur_granola_notes_admin_select + on granola_sync_notes for select to centaur_slack_admin using (true); + create policy centaur_granola_context_documents_admin_select + on granola_context_documents for select to centaur_slack_admin using (true); + create policy centaur_granola_checkpoints_admin_select + on granola_sync_checkpoints for select to centaur_slack_admin using (true); + end if; +end $$; diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0041_attio_sync_tables.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0041_attio_sync_tables.sql new file mode 100644 index 000000000..442a374fa --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0041_attio_sync_tables.sql @@ -0,0 +1,131 @@ +create table if not exists attio_sync_runs ( + run_id text primary key, + workflow_run_id text, + mode text not null default 'incremental', + status text not null, + scopes_requested jsonb not null default '[]'::jsonb, + scopes_synced jsonb not null default '[]'::jsonb, + scopes_failed jsonb not null default '[]'::jsonb, + meetings_seen integer not null default 0, + meetings_upserted integer not null default 0, + call_recordings_seen integer not null default 0, + transcripts_upserted integer not null default 0, + started_at timestamptz not null default now(), + finished_at timestamptz, + error_text text not null default '', + metadata jsonb not null default '{}'::jsonb +); + +create index if not exists idx_attio_sync_runs_started + on attio_sync_runs (started_at desc); + +create table if not exists attio_sync_meetings ( + meeting_id text primary key, + title text not null default '', + description text not null default '', + url text not null default '', + linked_records jsonb not null default '[]'::jsonb, + participants jsonb not null default '[]'::jsonb, + organizer_id text not null default '', + organizer_name text not null default '', + organizer_email text not null default '', + call_recording_ids jsonb not null default '[]'::jsonb, + transcript_text text not null default '', + transcript_payload jsonb not null default '[]'::jsonb, + content_text text not null default '', + content_hash text not null default '', + started_at timestamptz, + ended_at timestamptz, + source_created_at timestamptz, + source_updated_at timestamptz, + raw_payload jsonb not null default '{}'::jsonb, + source_run_id text references attio_sync_runs(run_id) on delete set null, + first_seen_at timestamptz not null default now(), + last_seen_at timestamptz not null default now(), + last_error text not null default '', + updated_at timestamptz not null default now() +); + +create index if not exists idx_attio_sync_meetings_source_updated + on attio_sync_meetings (source_updated_at desc); + +create index if not exists idx_attio_sync_meetings_time + on attio_sync_meetings (started_at desc); + +create index if not exists idx_attio_sync_meetings_text + on attio_sync_meetings + using gin (to_tsvector('english', coalesce(content_text, ''))); + +create table if not exists attio_sync_checkpoints ( + scope_id text primary key, + watermark_time timestamptz, + last_run_id text references attio_sync_runs(run_id) on delete set null, + last_success_at timestamptz, + last_error text not null default '', + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +do $$ +declare + role_name text; +begin + foreach role_name in array array[ + 'centaur_slack_reader', + 'centaur_slack_admin', + 'centaur_readonly' + ] loop + if exists (select 1 from pg_roles where rolname = role_name) then + execute format( + 'grant select on %s to %I', + 'attio_sync_runs, attio_sync_meetings, attio_sync_checkpoints', + role_name + ); + end if; + end loop; +end $$; + +alter table attio_sync_runs enable row level security; +alter table attio_sync_meetings enable row level security; +alter table attio_sync_checkpoints enable row level security; + +drop policy if exists centaur_attio_runs_admin_select on attio_sync_runs; +drop policy if exists centaur_attio_runs_reader_select on attio_sync_runs; +create policy centaur_attio_runs_reader_select + on attio_sync_runs for select to centaur_slack_reader + using (false); +drop policy if exists centaur_readonly_attio_sync_runs_select on attio_sync_runs; +create policy centaur_readonly_attio_sync_runs_select + on attio_sync_runs for select to centaur_readonly using (true); + +drop policy if exists centaur_attio_meetings_admin_select on attio_sync_meetings; +drop policy if exists centaur_attio_meetings_reader_select on attio_sync_meetings; +create policy centaur_attio_meetings_reader_select + on attio_sync_meetings for select to centaur_slack_reader + using (false); +drop policy if exists centaur_readonly_attio_sync_meetings_select + on attio_sync_meetings; +create policy centaur_readonly_attio_sync_meetings_select + on attio_sync_meetings for select to centaur_readonly using (true); + +drop policy if exists centaur_attio_checkpoints_admin_select on attio_sync_checkpoints; +drop policy if exists centaur_attio_checkpoints_reader_select on attio_sync_checkpoints; +create policy centaur_attio_checkpoints_reader_select + on attio_sync_checkpoints for select to centaur_slack_reader + using (false); +drop policy if exists centaur_readonly_attio_sync_checkpoints_select + on attio_sync_checkpoints; +create policy centaur_readonly_attio_sync_checkpoints_select + on attio_sync_checkpoints for select to centaur_readonly using (true); + +do $$ +begin + if exists (select 1 from pg_roles where rolname = 'centaur_slack_admin') then + create policy centaur_attio_runs_admin_select + on attio_sync_runs for select to centaur_slack_admin using (true); + create policy centaur_attio_meetings_admin_select + on attio_sync_meetings for select to centaur_slack_admin using (true); + create policy centaur_attio_checkpoints_admin_select + on attio_sync_checkpoints for select to centaur_slack_admin using (true); + end if; +end $$; diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0042_centaur_readonly_slack_dm_rls.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0042_centaur_readonly_slack_dm_rls.sql new file mode 100644 index 000000000..4d0cf6dde --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0042_centaur_readonly_slack_dm_rls.sql @@ -0,0 +1,139 @@ +-- Keep centaur_readonly useful for public channel context while allowing a +-- principal that carries Slack identity settings to see only its own DMs. + +drop policy if exists centaur_readonly_slack_dm_sync_conversations_select + on slack_dm_sync_conversations; +create policy centaur_readonly_slack_dm_sync_conversations_select + on slack_dm_sync_conversations + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_dm_sync_conversation_members members + where members.home_team_id = slack_dm_sync_conversations.home_team_id + and members.home_team_id = centaur_current_slack_team_id() + and members.conversation_id = slack_dm_sync_conversations.conversation_id + and members.user_id = centaur_current_slack_user_id() + and members.is_current_member + ) + ); + +drop policy if exists centaur_readonly_slack_dm_sync_conversation_members_select + on slack_dm_sync_conversation_members; +create policy centaur_readonly_slack_dm_sync_conversation_members_select + on slack_dm_sync_conversation_members + for select + to centaur_readonly + using ( + home_team_id = centaur_current_slack_team_id() + and user_id = centaur_current_slack_user_id() + and is_current_member + ); + +drop policy if exists centaur_readonly_slack_dm_sync_messages_select + on slack_dm_sync_messages; +create policy centaur_readonly_slack_dm_sync_messages_select + on slack_dm_sync_messages + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_dm_sync_conversation_members members + where members.home_team_id = slack_dm_sync_messages.home_team_id + and members.home_team_id = centaur_current_slack_team_id() + and members.conversation_id = slack_dm_sync_messages.conversation_id + and members.user_id = centaur_current_slack_user_id() + and members.is_current_member + ) + ); + +drop policy if exists centaur_readonly_slack_dm_sync_message_attachments_select + on slack_dm_sync_message_attachments; +create policy centaur_readonly_slack_dm_sync_message_attachments_select + on slack_dm_sync_message_attachments + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_dm_sync_conversation_members members + where members.home_team_id = slack_dm_sync_message_attachments.home_team_id + and members.home_team_id = centaur_current_slack_team_id() + and members.conversation_id = slack_dm_sync_message_attachments.conversation_id + and members.user_id = centaur_current_slack_user_id() + and members.is_current_member + ) + ); + +drop policy if exists centaur_readonly_slack_dm_sync_checkpoints_select + on slack_dm_sync_checkpoints; +create policy centaur_readonly_slack_dm_sync_checkpoints_select + on slack_dm_sync_checkpoints + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_dm_sync_conversation_members members + where members.home_team_id = slack_dm_sync_checkpoints.home_team_id + and members.home_team_id = centaur_current_slack_team_id() + and members.conversation_id = slack_dm_sync_checkpoints.conversation_id + and members.user_id = centaur_current_slack_user_id() + and members.is_current_member + ) + ); + +-- Operational rows never belong in user-visible company context. +drop policy if exists centaur_readonly_slack_dm_sync_runs_select + on slack_dm_sync_runs; +create policy centaur_readonly_slack_dm_sync_runs_select + on slack_dm_sync_runs + for select + to centaur_readonly + using (false); + +drop policy if exists centaur_readonly_slack_dm_sync_backfill_jobs_select + on slack_dm_sync_backfill_jobs; +create policy centaur_readonly_slack_dm_sync_backfill_jobs_select + on slack_dm_sync_backfill_jobs + for select + to centaur_readonly + using (false); + +drop policy if exists centaur_readonly_slack_dm_context_documents_select + on slack_dm_context_documents; +create policy centaur_readonly_slack_dm_context_documents_select + on slack_dm_context_documents + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_dm_sync_conversation_members members + where members.home_team_id = slack_dm_context_documents.home_team_id + and members.home_team_id = centaur_current_slack_team_id() + and members.conversation_id = slack_dm_context_documents.conversation_id + and members.user_id = centaur_current_slack_user_id() + and members.is_current_member + ) + ); + +drop policy if exists centaur_readonly_slack_dm_conversation_context_documents_select + on slack_dm_conversation_context_documents; +create policy centaur_readonly_slack_dm_conversation_context_documents_select + on slack_dm_conversation_context_documents + for select + to centaur_readonly + using ( + exists ( + select 1 + from slack_dm_sync_conversation_members members + where members.home_team_id = slack_dm_conversation_context_documents.home_team_id + and members.home_team_id = centaur_current_slack_team_id() + and members.conversation_id = slack_dm_conversation_context_documents.conversation_id + and members.user_id = centaur_current_slack_user_id() + and members.is_current_member + ) + ); diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0043_session_sandbox_content_revision.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0043_session_sandbox_content_revision.sql new file mode 100644 index 000000000..74bf6f4d0 --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0043_session_sandbox_content_revision.sql @@ -0,0 +1,21 @@ +alter table sessions + add column if not exists sandbox_content_revision text; + +comment on column sessions.sandbox_content_revision is + 'Assignment-bound digest of the immutable deployment boot-content generation and sandbox ID; NULL on legacy assignments.'; + +create or replace view centaur_readonly_sessions as +select + thread_key, + sandbox_id, + harness_type, + harness_thread_id, + persona_id, + status, + metadata ->> 'source' as source, + metadata ->> 'platform' as platform, + metadata ->> 'thread_id' as external_thread_id, + created_at, + updated_at, + sandbox_content_revision +from sessions; diff --git a/services/api-rs/crates/centaur-session-sqlx/src/lib.rs b/services/api-rs/crates/centaur-session-sqlx/src/lib.rs index fc3b8daac..a965c44f1 100644 --- a/services/api-rs/crates/centaur-session-sqlx/src/lib.rs +++ b/services/api-rs/crates/centaur-session-sqlx/src/lib.rs @@ -37,6 +37,22 @@ pub struct ClaimExecutionResult { pub claimed: bool, } +/// Outcome of the transactional database fence used before stopping a +/// session sandbox. Locking the session row serializes release with sandbox +/// assignment, while the sandbox snapshot prevents an old release request +/// from clearing a newly assigned sandbox. +#[derive(Clone, Debug)] +pub enum ReleaseSessionResult { + Released { + session: Session, + cancelled_execution: Option, + }, + ActiveExecution(SessionExecution), + SandboxMismatch { + current_sandbox_id: Option, + }, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct IdleSandboxCandidate { pub thread_key: ThreadKey, @@ -686,6 +702,83 @@ impl PgSessionStore { row.try_into() } + /// Bind a sandbox only while the exact execution that allocated it is + /// still active and the session assignment has not crossed the caller's + /// fence. The lock order intentionally matches release: session first, + /// then execution. + pub async fn assign_sandbox_to_active_execution( + &self, + thread_key: &ThreadKey, + execution_id: &str, + expected_sandbox_id: Option<&str>, + sandbox_id: &str, + capabilities: &SandboxCapabilities, + ) -> Result, SessionStoreError> { + let mut tx = self.pool.begin().await?; + let current_sandbox_id = sqlx::query_scalar::<_, Option>( + r#" + select sandbox_id + from sessions + where thread_key = $1 + for update + "#, + ) + .bind(thread_key.as_str()) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| SessionStoreError::NotFound { + thread_key: thread_key.as_str().to_owned(), + })?; + if current_sandbox_id.as_deref() != expected_sandbox_id { + tx.commit().await?; + return Ok(None); + } + + let status = sqlx::query_scalar::<_, String>( + r#" + select status + from session_executions + where execution_id = $1 and thread_key = $2 + for update + "#, + ) + .bind(execution_id) + .bind(thread_key.as_str()) + .fetch_optional(&mut *tx) + .await?; + if !status.as_deref().is_some_and(|status| { + status == ExecutionStatus::Queued.as_ref() + || status == ExecutionStatus::Running.as_ref() + }) { + tx.commit().await?; + return Ok(None); + } + + let row = sqlx::query_as::<_, SessionRow>( + r#" + update sessions + set + sandbox_id = $3, + sandbox_repo_cache_enabled = $4, + sandbox_observability_enabled = $5, + updated_at = now() + where thread_key = $1 + and sandbox_id is not distinct from $2 + returning thread_key, sandbox_id, sandbox_repo_cache_enabled, sandbox_observability_enabled, harness_type, harness_thread_id, persona_id, status, iron_control_principal, created_at, updated_at + "#, + ) + .bind(thread_key.as_str()) + .bind(expected_sandbox_id) + .bind(sandbox_id) + .bind(capabilities.repo_cache_enabled) + .bind(capabilities.observability_enabled) + .fetch_optional(&mut *tx) + .await?; + let session = row.map(TryInto::try_into).transpose()?; + tx.commit().await?; + Ok(session) + } + pub async fn clear_sandbox_id_if_matches( &self, thread_key: &ThreadKey, @@ -873,34 +966,105 @@ impl PgSessionStore { row.try_into() } - pub async fn release_session( + pub async fn release_session_if_sandbox_matches( &self, thread_key: &ThreadKey, - ) -> Result { + expected_sandbox_id: Option<&str>, + cancel_inflight: bool, + cancellation_reason: &str, + ) -> Result { + let mut tx = self.pool.begin().await?; + let locked = sqlx::query_as::<_, SessionRow>( + r#" + select thread_key, sandbox_id, sandbox_repo_cache_enabled, sandbox_observability_enabled, harness_type, harness_thread_id, persona_id, status, iron_control_principal, created_at, updated_at + from sessions + where thread_key = $1 + for update + "#, + ) + .bind(thread_key.as_str()) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| SessionStoreError::NotFound { + thread_key: thread_key.as_str().to_owned(), + })?; + + if locked.sandbox_id.as_deref() != expected_sandbox_id { + let current_sandbox_id = locked.sandbox_id; + tx.commit().await?; + return Ok(ReleaseSessionResult::SandboxMismatch { current_sandbox_id }); + } + + let active = sqlx::query_as::<_, SessionExecutionRow>( + r#" + select execution_id, idempotency_key, thread_key, status, metadata, error, created_at, updated_at, started_at, completed_at + from session_executions + where thread_key = $1 and status in ($2, $3) + order by created_at desc, execution_id desc + limit 1 + for update + "#, + ) + .bind(thread_key.as_str()) + .bind(ExecutionStatus::Queued.as_ref()) + .bind(ExecutionStatus::Running.as_ref()) + .fetch_optional(&mut *tx) + .await?; + + if !cancel_inflight && let Some(active) = active { + let execution = active.try_into()?; + tx.commit().await?; + return Ok(ReleaseSessionResult::ActiveExecution(execution)); + } + + let cancelled_execution = if let Some(active) = active { + let row = sqlx::query_as::<_, SessionExecutionRow>( + r#" + update session_executions + set status = $2, + error = $3, + completed_at = coalesce(completed_at, now()), + updated_at = now() + where execution_id = $1 and status in ($4, $5) + returning execution_id, idempotency_key, thread_key, status, metadata, error, created_at, updated_at, started_at, completed_at + "#, + ) + .bind(active.execution_id) + .bind(ExecutionStatus::Cancelled.as_ref()) + .bind(cancellation_reason) + .bind(ExecutionStatus::Queued.as_ref()) + .bind(ExecutionStatus::Running.as_ref()) + .fetch_optional(&mut *tx) + .await?; + row.map(TryInto::try_into).transpose()? + } else { + None + }; + let row = sqlx::query_as::<_, SessionRow>( r#" update sessions set sandbox_id = null, sandbox_repo_cache_enabled = null, sandbox_observability_enabled = null, - status = $2, + status = $3, updated_at = now() where thread_key = $1 + and sandbox_id is not distinct from $2 returning thread_key, sandbox_id, sandbox_repo_cache_enabled, sandbox_observability_enabled, harness_type, harness_thread_id, persona_id, status, iron_control_principal, created_at, updated_at "#, ) .bind(thread_key.as_str()) + .bind(expected_sandbox_id) .bind(SessionStatus::Idle.as_ref()) - .fetch_optional(&self.pool) + .fetch_one(&mut *tx) .await?; - - let Some(row) = row else { - return Err(SessionStoreError::NotFound { - thread_key: thread_key.as_str().to_owned(), - }); - }; - - row.try_into() + let session = row.try_into()?; + tx.commit().await?; + Ok(ReleaseSessionResult::Released { + session, + cancelled_execution, + }) } async fn set_session_status( diff --git a/services/api-rs/crates/centaur-workflows/src/lib.rs b/services/api-rs/crates/centaur-workflows/src/lib.rs index 739c14e97..0e93bdf1f 100644 --- a/services/api-rs/crates/centaur-workflows/src/lib.rs +++ b/services/api-rs/crates/centaur-workflows/src/lib.rs @@ -8,8 +8,8 @@ use std::{ }; use absurd::{ - Client, ClientOptions, CreateQueueOptions, RetryKind, RetryStrategy, SpawnOptions, StepHandle, - TaskContext, TaskRegistrationOptions, Worker, WorkerOptions, + Client, ClientOptions, RetryKind, RetryStrategy, SpawnOptions, StepHandle, TaskContext, + TaskRegistrationOptions, Worker, WorkerOptions, }; use centaur_sandbox_core::SandboxSpec; use centaur_session_core::{HarnessType, MessageRole, SessionMessageInput, ThreadKey}; @@ -39,6 +39,13 @@ pub const WORKFLOW_SLACK_LIVE_QUEUE: &str = "centaur_workflows_slack_live"; pub const WORKFLOW_ETL_QUEUE: &str = "centaur_workflows_etl"; pub const WORKFLOW_ETL_BACKFILL_QUEUE: &str = "centaur_workflows_etl_backfill"; pub const WORKFLOW_SCHEDULE_QUEUE: &str = "centaur_workflow_schedules"; +const ROLLBACK_REQUIRED_WORKFLOW_QUEUES: [&str; 5] = [ + WORKFLOW_QUEUE, + WORKFLOW_SLACK_LIVE_QUEUE, + WORKFLOW_ETL_QUEUE, + WORKFLOW_ETL_BACKFILL_QUEUE, + WORKFLOW_SCHEDULE_QUEUE, +]; pub const WORKFLOW_TASK: &str = "centaur.workflow"; pub const WORKFLOW_SCHEDULE_TASK: &str = "centaur.workflow.schedule_tick"; const PYTHON_HOST_ENV: &str = "PYTHON_WORKFLOW_HOST_PATH"; @@ -52,6 +59,11 @@ const WORKFLOW_RECONCILE_INTERVAL_SECS_ENV: &str = "WORKFLOW_RECONCILE_INTERVAL_ const DEFAULT_WORKFLOW_RECONCILE_INTERVAL_SECS: u64 = 60; const WORKFLOW_ENABLE_MODE_ENV: &str = "WORKFLOW_ENABLE_MODE"; const WORKFLOW_ALLOWED_NAMES_ENV: &str = "WORKFLOW_ALLOWED_NAMES"; +/// This branch is an emergency schema-forward rollback bridge. Startup must +/// explicitly opt into preserving, rather than executing, workflow tasks +/// created by the forward runtime because their handler source may not exist +/// in the rollback overlay. +const ROLLBACK_BRIDGE_PAUSE_WORKFLOWS_ENV: &str = "CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS"; /// How many consecutive reconcile passes a workflow must be missing from /// discovery before its active tasks are cancelled. 0 disables reaping. const WORKFLOW_REAP_REMOVED_AFTER_TICKS_ENV: &str = "WORKFLOW_REAP_REMOVED_AFTER_TICKS"; @@ -93,11 +105,12 @@ struct WorkflowRuntimeInner { slack_live_client: Client, etl_client: Client, etl_backfill_client: Client, - _worker: Worker, - _slack_live_worker: Worker, - _etl_worker: Worker, - _etl_backfill_worker: Worker, - _schedule_worker: Worker, + _worker: Option, + _slack_live_worker: Option, + _etl_worker: Option, + _etl_backfill_worker: Option, + _schedule_worker: Option, + paused_for_rollback: bool, webhook_registry: Arc>>, schedule_registry: Arc>>, } @@ -420,6 +433,11 @@ impl WorkflowRuntime { session_runtime: SessionRuntime, workflow_host_sandbox: Option, ) -> Result { + // Validate before touching absurd queues. This rollback-only branch is + // unsafe if a missing or mistyped deployment value can start workers. + require_rollback_bridge_workflow_pause()?; + let paused_for_rollback = true; + let client = Client::from_pool_with_options( store.pool().clone(), ClientOptions { @@ -427,9 +445,6 @@ impl WorkflowRuntime { ..ClientOptions::default() }, )?; - client - .create_queue(Some(WORKFLOW_QUEUE), CreateQueueOptions::default()) - .await?; let slack_live_client = Client::from_pool_with_options( store.pool().clone(), ClientOptions { @@ -437,12 +452,6 @@ impl WorkflowRuntime { ..ClientOptions::default() }, )?; - slack_live_client - .create_queue( - Some(WORKFLOW_SLACK_LIVE_QUEUE), - CreateQueueOptions::default(), - ) - .await?; let etl_client = Client::from_pool_with_options( store.pool().clone(), ClientOptions { @@ -450,9 +459,6 @@ impl WorkflowRuntime { ..ClientOptions::default() }, )?; - etl_client - .create_queue(Some(WORKFLOW_ETL_QUEUE), CreateQueueOptions::default()) - .await?; let etl_backfill_client = Client::from_pool_with_options( store.pool().clone(), ClientOptions { @@ -460,12 +466,6 @@ impl WorkflowRuntime { ..ClientOptions::default() }, )?; - etl_backfill_client - .create_queue( - Some(WORKFLOW_ETL_BACKFILL_QUEUE), - CreateQueueOptions::default(), - ) - .await?; let schedule_client = Client::from_pool_with_options( store.pool().clone(), ClientOptions { @@ -473,9 +473,7 @@ impl WorkflowRuntime { ..ClientOptions::default() }, )?; - schedule_client - .create_queue(Some(WORKFLOW_SCHEDULE_QUEUE), CreateQueueOptions::default()) - .await?; + require_existing_rollback_workflow_queues(&client).await?; let discovery = discover_python_workflow_metadata() .await @@ -546,91 +544,110 @@ impl WorkflowRuntime { } }, )?; - let startup_schedules = schedule_registry - .read() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .clone(); - reconcile_schedules(&schedule_client, &startup_schedules).await?; - - let worker = client.start_worker(WorkerOptions { - worker_id: Some("centaur-api-rs-workflow-worker".to_owned()), - concurrency: worker_concurrency( - WORKFLOW_WORKER_CONCURRENCY_ENV, - DEFAULT_WORKFLOW_WORKER_CONCURRENCY, - ), - on_error: Some(Arc::new(|error| { - warn!(%error, "absurd workflow worker error"); - })), - ..WorkerOptions::default() + if !paused_for_rollback { + let startup_schedules = schedule_registry + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + reconcile_schedules(&schedule_client, &startup_schedules).await?; + } + + let worker = (!paused_for_rollback).then(|| { + client.start_worker(WorkerOptions { + worker_id: Some("centaur-api-rs-workflow-worker".to_owned()), + concurrency: worker_concurrency( + WORKFLOW_WORKER_CONCURRENCY_ENV, + DEFAULT_WORKFLOW_WORKER_CONCURRENCY, + ), + on_error: Some(Arc::new(|error| { + warn!(%error, "absurd workflow worker error"); + })), + ..WorkerOptions::default() + }) }); - let slack_live_worker = slack_live_client.start_worker(WorkerOptions { - worker_id: Some("centaur-api-rs-workflow-slack-live-worker".to_owned()), - concurrency: 1, - on_error: Some(Arc::new(|error| { - warn!(%error, "absurd workflow slack live worker error"); - })), - ..WorkerOptions::default() + let slack_live_worker = (!paused_for_rollback).then(|| { + slack_live_client.start_worker(WorkerOptions { + worker_id: Some("centaur-api-rs-workflow-slack-live-worker".to_owned()), + concurrency: 1, + on_error: Some(Arc::new(|error| { + warn!(%error, "absurd workflow slack live worker error"); + })), + ..WorkerOptions::default() + }) }); - let etl_worker = etl_client.start_worker(WorkerOptions { - worker_id: Some("centaur-api-rs-workflow-etl-worker".to_owned()), - concurrency: worker_concurrency( - WORKFLOW_ETL_WORKER_CONCURRENCY_ENV, - DEFAULT_WORKFLOW_ETL_WORKER_CONCURRENCY, - ), - on_error: Some(Arc::new(|error| { - warn!(%error, "absurd workflow etl worker error"); - })), - ..WorkerOptions::default() + let etl_worker = (!paused_for_rollback).then(|| { + etl_client.start_worker(WorkerOptions { + worker_id: Some("centaur-api-rs-workflow-etl-worker".to_owned()), + concurrency: worker_concurrency( + WORKFLOW_ETL_WORKER_CONCURRENCY_ENV, + DEFAULT_WORKFLOW_ETL_WORKER_CONCURRENCY, + ), + on_error: Some(Arc::new(|error| { + warn!(%error, "absurd workflow etl worker error"); + })), + ..WorkerOptions::default() + }) }); - let etl_backfill_worker = etl_backfill_client.start_worker(WorkerOptions { - worker_id: Some("centaur-api-rs-workflow-etl-backfill-worker".to_owned()), - concurrency: worker_concurrency( - WORKFLOW_ETL_BACKFILL_WORKER_CONCURRENCY_ENV, - DEFAULT_WORKFLOW_ETL_BACKFILL_WORKER_CONCURRENCY, - ), - on_error: Some(Arc::new(|error| { - warn!(%error, "absurd workflow etl backfill worker error"); - })), - ..WorkerOptions::default() + let etl_backfill_worker = (!paused_for_rollback).then(|| { + etl_backfill_client.start_worker(WorkerOptions { + worker_id: Some("centaur-api-rs-workflow-etl-backfill-worker".to_owned()), + concurrency: worker_concurrency( + WORKFLOW_ETL_BACKFILL_WORKER_CONCURRENCY_ENV, + DEFAULT_WORKFLOW_ETL_BACKFILL_WORKER_CONCURRENCY, + ), + on_error: Some(Arc::new(|error| { + warn!(%error, "absurd workflow etl backfill worker error"); + })), + ..WorkerOptions::default() + }) }); - let schedule_worker = schedule_client.start_worker(WorkerOptions { - worker_id: Some("centaur-api-rs-workflow-schedule-worker".to_owned()), - concurrency: worker_concurrency( - WORKFLOW_SCHEDULE_WORKER_CONCURRENCY_ENV, - DEFAULT_WORKFLOW_SCHEDULE_WORKER_CONCURRENCY, - ), - on_error: Some(Arc::new(|error| { - warn!(%error, "absurd workflow schedule worker error"); - })), - ..WorkerOptions::default() + let schedule_worker = (!paused_for_rollback).then(|| { + schedule_client.start_worker(WorkerOptions { + worker_id: Some("centaur-api-rs-workflow-schedule-worker".to_owned()), + concurrency: worker_concurrency( + WORKFLOW_SCHEDULE_WORKER_CONCURRENCY_ENV, + DEFAULT_WORKFLOW_SCHEDULE_WORKER_CONCURRENCY, + ), + on_error: Some(Arc::new(|error| { + warn!(%error, "absurd workflow schedule worker error"); + })), + ..WorkerOptions::default() + }) }); - info!( - queue = WORKFLOW_QUEUE, - task = WORKFLOW_TASK, - "started absurd workflow worker" - ); - info!( - queue = WORKFLOW_SLACK_LIVE_QUEUE, - task = WORKFLOW_TASK, - "started absurd workflow slack live worker" - ); - info!( - queue = WORKFLOW_ETL_QUEUE, - task = WORKFLOW_TASK, - "started absurd workflow etl worker" - ); - info!( - queue = WORKFLOW_ETL_BACKFILL_QUEUE, - task = WORKFLOW_TASK, - "started absurd workflow etl backfill worker" - ); - info!( - queue = WORKFLOW_SCHEDULE_QUEUE, - task = WORKFLOW_SCHEDULE_TASK, - "started absurd workflow schedule worker" - ); + if paused_for_rollback { + warn!( + env = ROLLBACK_BRIDGE_PAUSE_WORKFLOWS_ENV, + "rollback bridge is preserving absurd workflow rows without starting workers, schedules, or removed-workflow reaping" + ); + } else { + info!( + queue = WORKFLOW_QUEUE, + task = WORKFLOW_TASK, + "started absurd workflow worker" + ); + info!( + queue = WORKFLOW_SLACK_LIVE_QUEUE, + task = WORKFLOW_TASK, + "started absurd workflow slack live worker" + ); + info!( + queue = WORKFLOW_ETL_QUEUE, + task = WORKFLOW_TASK, + "started absurd workflow etl worker" + ); + info!( + queue = WORKFLOW_ETL_BACKFILL_QUEUE, + task = WORKFLOW_TASK, + "started absurd workflow etl backfill worker" + ); + info!( + queue = WORKFLOW_SCHEDULE_QUEUE, + task = WORKFLOW_SCHEDULE_TASK, + "started absurd workflow schedule worker" + ); + } - if let Some(interval) = workflow_reconcile_interval() { + if !paused_for_rollback && let Some(interval) = workflow_reconcile_interval() { spawn_workflow_metadata_reconciler( schedule_client.clone(), WorkflowQueueClients { @@ -656,16 +673,27 @@ impl WorkflowRuntime { _etl_worker: etl_worker, _etl_backfill_worker: etl_backfill_worker, _schedule_worker: schedule_worker, + paused_for_rollback, webhook_registry, schedule_registry, }), }) } + fn ensure_workflow_mutations_enabled(&self) -> Result<(), WorkflowRuntimeError> { + if self.inner.paused_for_rollback { + return Err(WorkflowRuntimeError::Disabled(format!( + "workflow mutation is paused by the rollback bridge ({ROLLBACK_BRIDGE_PAUSE_WORKFLOWS_ENV}=true)" + ))); + } + Ok(()) + } + pub async fn create_run( &self, request: CreateWorkflowRunRequest, ) -> Result { + self.ensure_workflow_mutations_enabled()?; let workflow_name = request.workflow_name.trim(); if workflow_name.is_empty() { return Err(WorkflowRuntimeError::BadRequest( @@ -904,6 +932,7 @@ impl WorkflowRuntime { } pub async fn cancel_run(&self, run_id: &str) -> Result<(), WorkflowRuntimeError> { + self.ensure_workflow_mutations_enabled()?; for (queue_name, client) in [ (WORKFLOW_QUEUE, &self.inner.client), (WORKFLOW_SLACK_LIVE_QUEUE, &self.inner.slack_live_client), @@ -967,6 +996,7 @@ impl WorkflowRuntime { event_name: &str, payload: Value, ) -> Result<(), WorkflowRuntimeError> { + self.ensure_workflow_mutations_enabled()?; self.inner .client .emit_event(event_name, payload.clone(), Some(WORKFLOW_QUEUE)) @@ -1025,6 +1055,29 @@ impl WorkflowRuntime { } } +async fn require_existing_rollback_workflow_queues( + client: &Client, +) -> Result<(), WorkflowRuntimeError> { + let existing = client + .list_queues() + .await? + .into_iter() + .collect::>(); + let missing = ROLLBACK_REQUIRED_WORKFLOW_QUEUES + .iter() + .filter(|queue| !existing.contains(**queue)) + .copied() + .collect::>(); + if missing.is_empty() { + return Ok(()); + } + + Err(WorkflowRuntimeError::Internal(format!( + "rollback bridge requires all forward absurd queues to exist before startup; missing: {}", + missing.join(", ") + ))) +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum WorkflowQueueClass { Standard, @@ -1802,6 +1855,26 @@ fn workflow_reconcile_interval() -> Option { (seconds > 0).then(|| Duration::from_secs(seconds)) } +/// Refuse to run this rollback-only binary unless the deployment explicitly +/// acknowledges that durable workflow processing must remain paused. +pub fn require_rollback_bridge_workflow_pause() -> Result<(), WorkflowRuntimeError> { + let raw = env::var(ROLLBACK_BRIDGE_PAUSE_WORKFLOWS_ENV).ok(); + if parse_rollback_bridge_pause(raw.as_deref()) { + return Ok(()); + } + + Err(WorkflowRuntimeError::Internal(format!( + "rollback bridge refuses to start unless {ROLLBACK_BRIDGE_PAUSE_WORKFLOWS_ENV}=true is explicitly configured" + ))) +} + +fn parse_rollback_bridge_pause(raw: Option<&str>) -> bool { + matches!( + raw.map(str::trim).map(str::to_ascii_lowercase).as_deref(), + Some("true") + ) +} + /// Resolve a worker concurrency from `env_name`, falling back to `default` when /// the value is unset, empty, non-numeric, or zero. fn worker_concurrency(env_name: &str, default: usize) -> usize { @@ -2782,7 +2855,10 @@ async fn run_python_workflow_host_local( .arg(&host_path) .stdin(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()); + .stderr(std::process::Stdio::piped()) + .env("WORKFLOW_RUN_ID", ctx.run_id()) + .env("WORKFLOW_TASK_ID", ctx.task_id()) + .env("WORKFLOW_NAME", input.workflow_name.clone()); if env::var_os("WORKFLOW_DIRS").is_none() { command.env("WORKFLOW_DIRS", default_workflow_dirs()); } @@ -3833,6 +3909,27 @@ mod tests { use super::*; use chrono::TimeZone; + #[test] + fn rollback_bridge_requires_workflow_pause_to_be_explicitly_enabled() { + for value in [Some("true"), Some(" TRUE ")] { + assert!(parse_rollback_bridge_pause(value), "value={value:?}"); + } + for value in [ + None, + Some(""), + Some("0"), + Some("false"), + Some(" no "), + Some("OFF"), + Some("1"), + Some("yes"), + Some("on"), + Some("tru"), + ] { + assert!(!parse_rollback_bridge_pause(value), "value={value:?}"); + } + } + #[test] fn parse_worker_concurrency_uses_override_or_default() { // Override wins. diff --git a/workflows/slack/archive_import.py b/workflows/slack/archive_import.py index 4d6480566..a7c50602e 100644 --- a/workflows/slack/archive_import.py +++ b/workflows/slack/archive_import.py @@ -700,11 +700,17 @@ def _api_base_url() -> str: def _request_archive_download_url(import_id: str) -> str: quoted_import_id = urllib.parse.quote(import_id, safe="") + headers = {"Content-Type": "application/json"} + workflow_run_id = _as_text(os.environ.get("WORKFLOW_RUN_ID")).strip() + workflow_task_id = _as_text(os.environ.get("WORKFLOW_TASK_ID")).strip() + if workflow_run_id and workflow_task_id: + headers["X-Centaur-Workflow-Run-Id"] = workflow_run_id + headers["X-Centaur-Workflow-Task-Id"] = workflow_task_id request = urllib.request.Request( f"{_api_base_url()}/api/admin/slack/archive-imports/{quoted_import_id}/download-url", data=b"", method="POST", - headers={"Content-Type": "application/json"}, + headers=headers, ) with urllib.request.urlopen(request, timeout=30) as response: payload = json.load(response) diff --git a/workflows/slack/tests/test_archive_import.py b/workflows/slack/tests/test_archive_import.py index 351fd2af2..f627c7b4b 100644 --- a/workflows/slack/tests/test_archive_import.py +++ b/workflows/slack/tests/test_archive_import.py @@ -391,6 +391,8 @@ def fake_urlopen(request, timeout): ) monkeypatch.setenv("CENTAUR_API_URL", "http://centaur-api-rs:8080/") + monkeypatch.setenv("WORKFLOW_RUN_ID", "run-123") + monkeypatch.setenv("WORKFLOW_TASK_ID", "task-456") monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) download_url = archive_import._request_archive_download_url("sai id/with/slash") @@ -404,6 +406,9 @@ def fake_urlopen(request, timeout): == "http://centaur-api-rs:8080/api/admin/slack/archive-imports/" "sai%20id%2Fwith%2Fslash/download-url" ) + headers = {name.lower(): value for name, value in request.header_items()} + assert headers["x-centaur-workflow-run-id"] == "run-123" + assert headers["x-centaur-workflow-task-id"] == "task-456" def test_download_archive_streams_api_presigned_url(tmp_path, monkeypatch): From a08bbe41c0bd2f7e5eab1cd17b99262ad28a1eef Mon Sep 17 00:00:00 2001 From: "fineas-bot[bot]" <258147136+fineas-bot[bot]@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:20:17 -0400 Subject: [PATCH 4/9] fix(ci): add audited publication tag trigger Allow a fresh, exact, contents-write attestation tag when Actions dispatch is unavailable. Preserve the manual path and all immutable publication gates. --- .../check-rollback-bridge-workflows.sh | 43 ++++++- ...est-rollback-bridge-publication-trigger.sh | 119 ++++++++++++++++++ .github/workflows/ci.yml | 1 + .github/workflows/publish-images.yml | 93 ++++++++++++-- .../operate/upstream-rollback-bridge.mdx | 35 +++--- .../md/operate/upstream-rollback-bridge.md | 35 +++--- 6 files changed, 284 insertions(+), 42 deletions(-) create mode 100644 .github/scripts/test-rollback-bridge-publication-trigger.sh diff --git a/.github/scripts/check-rollback-bridge-workflows.sh b/.github/scripts/check-rollback-bridge-workflows.sh index 6413ed135..ce144c1d6 100644 --- a/.github/scripts/check-rollback-bridge-workflows.sh +++ b/.github/scripts/check-rollback-bridge-workflows.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2016 # Workflow expressions and shell variables below are literal guards. set -euo pipefail IFS=$'\n\t' @@ -29,9 +30,29 @@ fi grep -Fq 'service: [api-rs, slackbotv2, linearbot, discordbot, teamsbot, agent, iron-proxy, console]' "$validator" || fail "PR validator must preserve the historical eight-image required-check matrix" -grep -q '^ workflow_dispatch:$' "$publisher" || fail "publisher must be manual-only" -if grep -Eq '^ (push|pull_request):' "$publisher"; then - fail "publisher must not run on push or pull request events" +grep -q '^ workflow_dispatch:$' "$publisher" || + fail "publisher must preserve the confirmed manual path" +expected_push_trigger=$' push:\n tags:\n - '\''rollback-bridge-publish-live-scope-verified-*'\''' +actual_push_trigger="$(awk ' + $0 == " push:" { capture = 1 } + capture && /^[^[:space:]]/ { exit } + capture { print } +' "$publisher")" +[[ "$actual_push_trigger" == "$expected_push_trigger" ]] || + fail "publisher push trigger must contain only the exact live-scope tag prefix" +if grep -Eq '^ (pull_request|create|schedule):' "$publisher"; then + fail "publisher must not run for branches, pull requests, create events, or schedules" +fi +expected_tag_pattern='^rollback-bridge-publish-live-scope-verified-([0-9a-f]{40})-forward-([0-9a-f]{40})-at-([1-9][0-9]{9})$' +actual_tag_pattern="$(awk -F "'" '/^[[:space:]]*tag_pattern=/{ print $2 }' "$publisher")" +[[ "$actual_tag_pattern" == "$expected_tag_pattern" ]] || + fail "publisher tag parser must require exact full SHAs and exactly ten timestamp digits" +valid_trigger_tag="rollback-bridge-publish-live-scope-verified-$(printf 'a%.0s' {1..40})-forward-$(printf 'b%.0s' {1..40})-at-1234567890" +oversized_trigger_tag="${valid_trigger_tag}1" +[[ "$valid_trigger_tag" =~ $actual_tag_pattern ]] || + fail "publisher tag parser rejected its exact valid trigger shape" +if [[ "$oversized_trigger_tag" =~ $actual_tag_pattern ]]; then + fail "publisher tag parser accepted an overflow-capable timestamp" fi grep -q '^ live_updater_scope_verified:$' "$publisher" || fail "publisher must require an explicit live updater-scope acknowledgement" @@ -42,6 +63,20 @@ grep -Fq 'none of the four exact bridge repositories is in image-list' "$publish if grep -qE 'image_updater_disabled|IMAGE_UPDATER_DISABLED' "$publisher"; then fail "publisher must not claim that global Image Updater disablement is a publication precondition" fi +for required_trigger_guard in \ + 'TRIGGER_REF_CREATED: ${{ github.event.created }}' \ + 'git cat-file -t "$TRIGGER_SHA"' \ + '"$bridge_commit" != "$TRIGGER_SHA"' \ + '"$tag_bridge_commit" != "$bridge_commit"' \ + '"$tag_forward_commit" != "$commit"' \ + '/git/ref/tags/${encoded_ref}' \ + "'.object.type'" \ + "'.object.sha'" \ + 'attested_at > now + 120' \ + 'now - attested_at > 900'; do + grep -Fq "$required_trigger_guard" "$publisher" || + fail "publisher is missing exact tag trigger guard: $required_trigger_guard" +done if grep -Eq 'kubectl|KUBECONFIG|kubeconfig' "$publisher"; then fail "publisher must not receive Kubernetes access" fi @@ -105,4 +140,6 @@ if [[ -n "$unexpected_placeholder_files" ]]; then fail "the unresolved forward commit placeholder may exist only in $pin" fi +bash .github/scripts/test-rollback-bridge-publication-trigger.sh + echo "rollback bridge workflow safety checks passed" diff --git a/.github/scripts/test-rollback-bridge-publication-trigger.sh b/.github/scripts/test-rollback-bridge-publication-trigger.sh new file mode 100644 index 000000000..d76308db4 --- /dev/null +++ b/.github/scripts/test-rollback-bridge-publication-trigger.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +publisher=.github/workflows/publish-images.yml +pin=.github/rollback-bridge-reviewed-forward-commit +scratch="$(mktemp -d -t rollback-bridge-trigger.XXXXXXXXXX)" +trap 'rm -rf "$scratch"' EXIT +gate_script="$scratch/release-gate.sh" + +awk ' + $0 == " - name: Require an exact reviewed publication trigger" { + in_step = 1 + next + } + in_step && $0 == " run: |" { + capture = 1 + next + } + capture && /^ [^[:space:]]/ { exit } + capture { + sub(/^ /, "") + print + } +' "$publisher" >"$gate_script" +# shellcheck disable=SC2016 # Extracted workflow shell is intentionally matched literally. +grep -qF 'case "$TRIGGER_EVENT_NAME" in' "$gate_script" || { + echo "could not extract rollback bridge publication release gate" >&2 + exit 1 +} + +bridge="$(git rev-parse HEAD)" +forward="$(tr -d '\r\n' < "$pin")" +now="$(date +%s)" + +curl() { + printf '{"ref":"%s","object":{"type":"%s","sha":"%s"}}\n' \ + "${MOCK_REF:?}" "${MOCK_OBJECT_TYPE:?}" "${MOCK_OBJECT_SHA:?}" +} +export -f curl + +run_gate() { + GITHUB_OUTPUT=/dev/null \ + TRIGGER_API_URL=https://api.github.invalid \ + TRIGGER_REPOSITORY=TipLink/centaur \ + GITHUB_API_TOKEN=test-token \ + bash "$gate_script" +} + +manual() { + TRIGGER_EVENT_NAME=workflow_dispatch \ + TRIGGER_REF=refs/heads/test \ + TRIGGER_REF_CREATED='' \ + TRIGGER_REF_NAME=test \ + TRIGGER_REF_TYPE=branch \ + TRIGGER_SHA="$bridge" \ + LIVE_UPDATER_SCOPE_VERIFIED="$1" \ + DISPATCH_FORWARD_COMMIT="$2" \ + run_gate +} + +push_tag() { + local tag="$1" + local created="$2" + TRIGGER_EVENT_NAME=push \ + TRIGGER_REF="refs/tags/${tag}" \ + TRIGGER_REF_CREATED="$created" \ + TRIGGER_REF_NAME="$tag" \ + TRIGGER_REF_TYPE=tag \ + TRIGGER_SHA="$bridge" \ + LIVE_UPDATER_SCOPE_VERIFIED='' \ + DISPATCH_FORWARD_COMMIT='' \ + run_gate +} + +expect_reject() { + local label="$1" + shift + if "$@" >"$scratch/${label}.out" 2>&1; then + echo "publication release gate unexpectedly accepted: $label" >&2 + exit 1 + fi +} + +manual true "$forward" +expect_reject manual-false manual false "$forward" +wrong_forward="$(printf '0%.0s' {1..40})" +expect_reject manual-wrong-forward manual true "$wrong_forward" + +valid="rollback-bridge-publish-live-scope-verified-${bridge}-forward-${forward}-at-${now}" +export MOCK_REF="refs/tags/${valid}" MOCK_OBJECT_TYPE=commit MOCK_OBJECT_SHA="$bridge" +push_tag "$valid" true + +export MOCK_OBJECT_TYPE=tag +expect_reject annotated-remote-ref push_tag "$valid" true +export MOCK_OBJECT_TYPE=commit +expect_reject reused-or-updated-ref push_tag "$valid" false +wrong_sha="$(printf '0%.0s' {1..40})" +export MOCK_OBJECT_SHA="$wrong_sha" +expect_reject wrong-remote-target push_tag "$valid" true +export MOCK_OBJECT_SHA="$bridge" + +wrong_forward_tag="rollback-bridge-publish-live-scope-verified-${bridge}-forward-${wrong_forward}-at-${now}" +export MOCK_REF="refs/tags/${wrong_forward_tag}" +expect_reject tag-wrong-forward push_tag "$wrong_forward_tag" true + +stale="rollback-bridge-publish-live-scope-verified-${bridge}-forward-${forward}-at-$((now - 901))" +export MOCK_REF="refs/tags/${stale}" +expect_reject stale-timestamp push_tag "$stale" true + +future="rollback-bridge-publish-live-scope-verified-${bridge}-forward-${forward}-at-$((now + 180))" +export MOCK_REF="refs/tags/${future}" +expect_reject future-timestamp push_tag "$future" true + +oversized="rollback-bridge-publish-live-scope-verified-${bridge}-forward-${forward}-at-10000000000" +export MOCK_REF="refs/tags/${oversized}" +expect_reject overflow-timestamp push_tag "$oversized" true + +echo "rollback bridge publication trigger tests passed" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 233600c21..65a346f03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,6 +79,7 @@ jobs: '^services/api-rs/' \ '^\.github/rollback-bridge-reviewed-forward-commit$' \ '^\.github/scripts/check-rollback-bridge-workflows\.sh$' \ + '^\.github/scripts/test-rollback-bridge-publication-trigger\.sh$' \ '^\.github/workflows/publish-images\.yml$' \ '^\.github/workflows/validate-images\.yml$' \ '^\.github/workflows/ci\.yml$' diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index 0d9580c8d..2e6ab21b8 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -12,6 +12,9 @@ on: description: Exact frozen reviewed-forward commit recorded by this bridge required: true type: string + push: + tags: + - 'rollback-bridge-publish-live-scope-verified-*' concurrency: # Tags are global registry state, so every dispatch must serialize across @@ -38,31 +41,100 @@ jobs: forward_commit: ${{ steps.forward_pin.outputs.commit }} env: DISPATCH_FORWARD_COMMIT: ${{ inputs.reviewed_forward_commit }} + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} LIVE_UPDATER_SCOPE_VERIFIED: ${{ inputs.live_updater_scope_verified }} + TRIGGER_API_URL: ${{ github.api_url }} + TRIGGER_EVENT_NAME: ${{ github.event_name }} + TRIGGER_REF: ${{ github.ref }} + TRIGGER_REF_CREATED: ${{ github.event.created }} + TRIGGER_REF_NAME: ${{ github.ref_name }} + TRIGGER_REF_TYPE: ${{ github.ref_type }} + TRIGGER_REPOSITORY: ${{ github.repository }} + TRIGGER_SHA: ${{ github.sha }} steps: - name: Checkout uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: persist-credentials: false - - name: Require live updater scope proof and the frozen forward commit + - name: Require an exact reviewed publication trigger id: forward_pin run: | set -euo pipefail IFS=$'\n\t' - if [[ "$LIVE_UPDATER_SCOPE_VERIFIED" != "true" ]]; then - echo "read the live child annotations and verify that none of the four exact bridge repositories is in image-list and every relevant allow-tags rule excludes reviewed-40 before publication" >&2 - exit 1 - fi commit="$(tr -d '\r\n' < .github/rollback-bridge-reviewed-forward-commit)" if [[ ! "$commit" =~ ^[0-9a-f]{40}$ ]]; then echo ".github/rollback-bridge-reviewed-forward-commit must contain the frozen lowercase 40-character commit SHA" >&2 exit 1 fi - if [[ "$DISPATCH_FORWARD_COMMIT" != "$commit" ]]; then - echo "dispatch reviewed_forward_commit does not match the bridge's frozen forward commit" >&2 + bridge_commit="$(git rev-parse HEAD)" + if [[ ! "$bridge_commit" =~ ^[0-9a-f]{40}$ ]]; then + echo "checked-out rollback bridge revision is not a full lowercase commit SHA" >&2 + exit 1 + fi + object_type="$(git cat-file -t "$TRIGGER_SHA" 2>/dev/null || true)" + if [[ "$object_type" != "commit" || "$bridge_commit" != "$TRIGGER_SHA" ]]; then + echo "publication trigger must resolve directly to the exact checked-out bridge commit" >&2 exit 1 fi + + case "$TRIGGER_EVENT_NAME" in + workflow_dispatch) + if [[ "$LIVE_UPDATER_SCOPE_VERIFIED" != "true" ]]; then + echo "read the live child annotations and verify that none of the four exact bridge repositories is in image-list and every relevant allow-tags rule excludes reviewed-40 before publication" >&2 + exit 1 + fi + if [[ "$DISPATCH_FORWARD_COMMIT" != "$commit" ]]; then + echo "dispatch reviewed_forward_commit does not match the bridge's frozen forward commit" >&2 + exit 1 + fi + ;; + push) + if [[ "$TRIGGER_REF_TYPE" != "tag" || "$TRIGGER_REF_CREATED" != "true" || + "$TRIGGER_REF" != "refs/tags/${TRIGGER_REF_NAME}" ]]; then + echo "publication tag trigger must be a newly created tag ref" >&2 + exit 1 + fi + tag_pattern='^rollback-bridge-publish-live-scope-verified-([0-9a-f]{40})-forward-([0-9a-f]{40})-at-([1-9][0-9]{9})$' + if [[ ! "$TRIGGER_REF_NAME" =~ $tag_pattern ]]; then + echo "publication tag does not match the exact live-scope attestation format" >&2 + exit 1 + fi + tag_bridge_commit="${BASH_REMATCH[1]}" + tag_forward_commit="${BASH_REMATCH[2]}" + attested_at="${BASH_REMATCH[3]}" + if [[ "$tag_bridge_commit" != "$bridge_commit" || + "$tag_forward_commit" != "$commit" ]]; then + echo "publication tag bridge or forward commit does not match the reviewed checkout" >&2 + exit 1 + fi + encoded_ref="$(jq -nr --arg value "$TRIGGER_REF_NAME" '$value|@uri')" + ref_json="$(curl --fail --silent --show-error \ + --header "Authorization: Bearer ${GITHUB_API_TOKEN}" \ + --header 'Accept: application/vnd.github+json' \ + --header 'X-GitHub-Api-Version: 2022-11-28' \ + "${TRIGGER_API_URL}/repos/${TRIGGER_REPOSITORY}/git/ref/tags/${encoded_ref}")" + if [[ "$(jq -er '.ref' <<<"$ref_json")" != "$TRIGGER_REF" || + "$(jq -er '.object.type' <<<"$ref_json")" != "commit" || + "$(jq -er '.object.sha' <<<"$ref_json")" != "$TRIGGER_SHA" ]]; then + echo "publication ref must be a lightweight tag directly targeting the trigger commit" >&2 + exit 1 + fi + now="$(date +%s)" + if ((attested_at > now + 120)); then + echo "publication tag live-scope timestamp is more than 120 seconds in the future" >&2 + exit 1 + fi + if ((now - attested_at > 900)); then + echo "publication tag live-scope proof is older than 900 seconds; rerun the read-only helper and create a new tag" >&2 + exit 1 + fi + ;; + *) + echo "unsupported rollback bridge publication trigger: $TRIGGER_EVENT_NAME" >&2 + exit 1 + ;; + esac echo "commit=$commit" >> "$GITHUB_OUTPUT" tag-absence-gate: @@ -119,10 +191,9 @@ jobs: # Build each image natively per platform (amd64 on x64 runners, arm64 on # arm runners), push by digest, and hand the digests to the merge job below - # which assembles the multi-arch manifest. This package-write workflow is - # manual-only and requires confirmation from a read of the live child - # annotations that the four repositories and reviewed-full namespace are - # outside Image Updater's scope. + # which assembles the multi-arch manifest. Package writes require either the + # confirmed manual path or the exact, + # short-lived contents-write tag attestation validated above. build: needs: tag-absence-gate runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} diff --git a/docs/pages/operate/upstream-rollback-bridge.mdx b/docs/pages/operate/upstream-rollback-bridge.mdx index c822d1bf7..a7cf9fdec 100644 --- a/docs/pages/operate/upstream-rollback-bridge.mdx +++ b/docs/pages/operate/upstream-rollback-bridge.mdx @@ -25,21 +25,28 @@ Fineas upstream-sync runbook in the four exact bridge repositories or the `reviewed-` tag namespace, and core review has frozen the exact forward commit. The current legacy updater manages only the separate Fineas overlay repository and its - allow-list accepts only deploy-shaped `sha-<7>` tags. The manual workflow - requires an explicit acknowledgement of that live scope proof and the exact - frozen commit as a matching dispatch input before it emits reviewed + allow-list accepts only deploy-shaped `sha-<7>` tags. A human with Actions + permission can use the manual path, which requires an explicit live-scope + acknowledgement and the exact frozen forward commit. A restricted + contents-write principal can instead create one new lightweight tag named + `rollback-bridge-publish-live-scope-verified--forward--at-` + immediately after the same read-only live check. The release gate requires + the tag object, checkout, event SHA, embedded bridge SHA, and frozen forward + SHA to agree. It allows 120 seconds of future clock skew and fails closed + once the attestation is 900 seconds old. If Actions queueing exceeds that + window, rerun the helper and create a new timestamped tag; never delete or + recreate an attestation ref. Both paths emit only `reviewed-` tags for exactly four linux/arm64 bridge runtime - rows: API, Slackbot v2, agent, and IronProxy. This namespace cannot match the - legacy updater's deploy-shaped `sha-<7>` tags; Console web/worker remains on - the forward image. Publication creates inert artifacts and a descriptor; it - is not rollout authorization. The infra runbook still requires global Image - Updater removal and the Argo automation freeze before consuming the - descriptor in Git or changing any Argo pin. The separate pull-request - workflow preserves the - historical `Publish Images` check context but has read-only repository - permissions, receives no registry credentials, and builds with `push: false`. - Push/tag events on this emergency branch do not publish; only the confirmed - manual dispatch can mutate GHCR. + rows: API, Slackbot v2, agent, and IronProxy. Other branch/tag events cannot + publish. This namespace cannot match the legacy updater's deploy-shaped + `sha-<7>` tags; Console web/worker remains on the forward image. The publisher + has no Kubernetes credentials. Publication creates inert artifacts and a + descriptor; it is not rollout authorization. The infra runbook still + requires global Image Updater removal and the Argo automation freeze before + consuming the descriptor in Git or changing any Argo pin. The separate + pull-request workflow preserves the historical `Publish Images` check + context but has read-only repository permissions, receives no registry + credentials, and builds with `push: false`. - Set `CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS=true` explicitly. The bridge refuses to start when the value is absent, false, or malformed. With the fence acknowledged, it starts no absurd workers, schedule ticks, metadata diff --git a/docs/public/md/operate/upstream-rollback-bridge.md b/docs/public/md/operate/upstream-rollback-bridge.md index c822d1bf7..a7cf9fdec 100644 --- a/docs/public/md/operate/upstream-rollback-bridge.md +++ b/docs/public/md/operate/upstream-rollback-bridge.md @@ -25,21 +25,28 @@ Fineas upstream-sync runbook in the four exact bridge repositories or the `reviewed-` tag namespace, and core review has frozen the exact forward commit. The current legacy updater manages only the separate Fineas overlay repository and its - allow-list accepts only deploy-shaped `sha-<7>` tags. The manual workflow - requires an explicit acknowledgement of that live scope proof and the exact - frozen commit as a matching dispatch input before it emits reviewed + allow-list accepts only deploy-shaped `sha-<7>` tags. A human with Actions + permission can use the manual path, which requires an explicit live-scope + acknowledgement and the exact frozen forward commit. A restricted + contents-write principal can instead create one new lightweight tag named + `rollback-bridge-publish-live-scope-verified--forward--at-` + immediately after the same read-only live check. The release gate requires + the tag object, checkout, event SHA, embedded bridge SHA, and frozen forward + SHA to agree. It allows 120 seconds of future clock skew and fails closed + once the attestation is 900 seconds old. If Actions queueing exceeds that + window, rerun the helper and create a new timestamped tag; never delete or + recreate an attestation ref. Both paths emit only `reviewed-` tags for exactly four linux/arm64 bridge runtime - rows: API, Slackbot v2, agent, and IronProxy. This namespace cannot match the - legacy updater's deploy-shaped `sha-<7>` tags; Console web/worker remains on - the forward image. Publication creates inert artifacts and a descriptor; it - is not rollout authorization. The infra runbook still requires global Image - Updater removal and the Argo automation freeze before consuming the - descriptor in Git or changing any Argo pin. The separate pull-request - workflow preserves the - historical `Publish Images` check context but has read-only repository - permissions, receives no registry credentials, and builds with `push: false`. - Push/tag events on this emergency branch do not publish; only the confirmed - manual dispatch can mutate GHCR. + rows: API, Slackbot v2, agent, and IronProxy. Other branch/tag events cannot + publish. This namespace cannot match the legacy updater's deploy-shaped + `sha-<7>` tags; Console web/worker remains on the forward image. The publisher + has no Kubernetes credentials. Publication creates inert artifacts and a + descriptor; it is not rollout authorization. The infra runbook still + requires global Image Updater removal and the Argo automation freeze before + consuming the descriptor in Git or changing any Argo pin. The separate + pull-request workflow preserves the historical `Publish Images` check + context but has read-only repository permissions, receives no registry + credentials, and builds with `push: false`. - Set `CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS=true` explicitly. The bridge refuses to start when the value is absent, false, or malformed. With the fence acknowledged, it starts no absurd workers, schedule ticks, metadata From a223a15259e0e8e3f5c94d8b7129d50ff2d57779 Mon Sep 17 00:00:00 2001 From: "fineas-bot[bot]" <258147136+fineas-bot[bot]@users.noreply.github.com> Date: Sun, 12 Jul 2026 10:11:30 -0400 Subject: [PATCH 5/9] fix: publish runnable rollback image digests Separate attested BuildKit indexes from runnable platform children, bind the descriptor to this run, and freeze the bridge rehearsal to the final reviewed forward head. --- .../rollback-bridge-reviewed-forward-commit | 2 +- .../check-rollback-bridge-workflows.sh | 33 ++++- .../scripts/resolve-runnable-image-digest.sh | 68 ++++++++++ .../test-resolve-runnable-image-digest.sh | 123 ++++++++++++++++++ .github/workflows/ci.yml | 6 +- .github/workflows/publish-images.yml | 100 +++++++++----- .../operate/upstream-rollback-bridge.mdx | 13 +- .../md/operate/upstream-rollback-bridge.md | 13 +- 8 files changed, 320 insertions(+), 38 deletions(-) create mode 100644 .github/scripts/resolve-runnable-image-digest.sh create mode 100644 .github/scripts/test-resolve-runnable-image-digest.sh diff --git a/.github/rollback-bridge-reviewed-forward-commit b/.github/rollback-bridge-reviewed-forward-commit index dd177d51c..5316b6a9e 100644 --- a/.github/rollback-bridge-reviewed-forward-commit +++ b/.github/rollback-bridge-reviewed-forward-commit @@ -1 +1 @@ -6847616b6fcbad3c0ba51fef25f8b97845fd5aec +8248db2d1dece81a2e91ed678068d2ab18d581cf diff --git a/.github/scripts/check-rollback-bridge-workflows.sh b/.github/scripts/check-rollback-bridge-workflows.sh index ce144c1d6..43265cbba 100644 --- a/.github/scripts/check-rollback-bridge-workflows.sh +++ b/.github/scripts/check-rollback-bridge-workflows.sh @@ -108,10 +108,18 @@ grep -Fq 'type=raw,value=reviewed-${{ github.sha }}' "$publisher" || fail "publisher must use the reviewed-full-commit tag namespace" grep -Fq 'tag="reviewed-${RELEASE_REVISION}"' "$publisher" || fail "release descriptor must use the reviewed-full-commit tag namespace" -grep -Fq 'pattern: digests-*-linux-arm64' "$publisher" || - fail "release descriptor must download this run's arm64 digest artifacts" -grep -Fq 'if [[ "$digest" != "$built_digest" ]]' "$publisher" || - fail "release descriptor must bind each tagged arm64 digest to this run's build artifact" +grep -Fq 'pattern: merge-index-digests-${{ matrix.image }}-*' "$publisher" || + fail "manifest merge must consume this run's attested per-platform index digests" +grep -Fq 'working-directory: ${{ runner.temp }}/merge-index-digests' "$publisher" || + fail "manifest merge must preserve the attested index digest inputs" +grep -Fq 'if [[ "${#merge_index_digest_files[@]}" -ne 2 ]]' "$publisher" || + fail "manifest merge must require both attested platform index digests" +grep -Fq 'pattern: runnable-digests-*-linux-arm64' "$publisher" || + fail "release descriptor must download this run's runnable arm64 child digests" +grep -Fq 'resolve-runnable-image-digest.sh' "$publisher" || + fail "build must resolve each runnable platform child from its attested index" +grep -Fq 'if [[ "$digest" != "$run_child_digest" ]]' "$publisher" || + fail "release descriptor must bind each tagged arm64 child to this run's runnable child" grep -Fq 'refusing to overwrite immutable reviewed tag' "$publisher" || fail "publisher must refuse to overwrite an existing reviewed tag" if grep -Eq 'type=sha|value=(latest|main|edge)|sha-\$\{|::7' "$publisher"; then @@ -129,8 +137,24 @@ actual_descriptor_rows="$(awk ' grep -Fq '.github/rollback-bridge-reviewed-forward-commit' "$ci" || fail "CI does not read the central reviewed forward commit pin" +mapfile -t integration_control_keys < <( + awk -F ': ' '/^[[:space:]]+CENTAUR_CONTROL_API_KEY: / { print $2 }' "$ci" +) +[[ "${#integration_control_keys[@]}" -eq 2 ]] || + fail "CI must configure exactly two forward integration control keys" +for integration_control_key in "${integration_control_keys[@]}"; do + [[ "${#integration_control_key}" -ge 32 ]] || + fail "CI contains a forward integration control key shorter than 32 bytes" +done +unset integration_control_key integration_control_keys grep -Fq '.github/rollback-bridge-reviewed-forward-commit' "$publisher" || fail "publisher does not read the central reviewed forward commit pin" +for resolver_path in \ + '^\.github/scripts/resolve-runnable-image-digest\.sh$' \ + '^\.github/scripts/test-resolve-runnable-image-digest\.sh$'; do + grep -Fq "$resolver_path" "$ci" || + fail "CI change detection does not cover $resolver_path" +done placeholder="__REVIEWED_FORWARD_""COMMIT_REQUIRED__" unexpected_placeholder_files="$( git grep -lF "$placeholder" -- .github services docs 2>/dev/null | @@ -141,5 +165,6 @@ if [[ -n "$unexpected_placeholder_files" ]]; then fi bash .github/scripts/test-rollback-bridge-publication-trigger.sh +bash .github/scripts/test-resolve-runnable-image-digest.sh echo "rollback bridge workflow safety checks passed" diff --git a/.github/scripts/resolve-runnable-image-digest.sh b/.github/scripts/resolve-runnable-image-digest.sh new file mode 100644 index 000000000..7f3b7abf8 --- /dev/null +++ b/.github/scripts/resolve-runnable-image-digest.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +image_index_ref=${1:-} +platform_os=${2:-} +platform_architecture=${3:-} + +if [[ ! "$image_index_ref" =~ @sha256:[0-9a-f]{64}$ ]]; then + echo "image index reference must end in a full sha256 digest" >&2 + exit 1 +fi +if [[ ! "$platform_os" =~ ^[a-z0-9]+$ || + ! "$platform_architecture" =~ ^[a-z0-9_]+$ ]]; then + echo "platform OS and architecture must be non-empty lowercase identifiers" >&2 + exit 1 +fi + +index_json="$(docker buildx imagetools inspect "$image_index_ref" --raw)" +if ! jq -e ' + type == "object" and + ( + .mediaType == "application/vnd.oci.image.index.v1+json" or + .mediaType == "application/vnd.docker.distribution.manifest.list.v2+json" + ) and + (.manifests | type == "array") +' <<<"$index_json" >/dev/null; then + echo "build output is not a parseable OCI/Docker image index: $image_index_ref" >&2 + exit 1 +fi + +runnable_digests=() +while IFS= read -r digest; do + runnable_digests+=("$digest") +done < <( + jq -r \ + --arg os "$platform_os" \ + --arg architecture "$platform_architecture" ' + .manifests[] + | select( + .platform.os == $os and + .platform.architecture == $architecture and + ( + .mediaType == "application/vnd.oci.image.manifest.v1+json" or + .mediaType == "application/vnd.docker.distribution.manifest.v2+json" + ) + ) + | .digest + ' <<<"$index_json" +) + +if [[ "${#runnable_digests[@]}" -ne 1 ]]; then + echo "expected exactly one runnable ${platform_os}/${platform_architecture} child in $image_index_ref; found ${#runnable_digests[@]}" >&2 + exit 1 +fi + +runnable_digest=${runnable_digests[0]} +if [[ ! "$runnable_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "index contains an invalid runnable child digest: $runnable_digest" >&2 + exit 1 +fi + +# Resolve the child independently so a malformed or inaccessible descriptor +# cannot be published as this run's deployable platform identity. +repository=${image_index_ref%@*} +docker buildx imagetools inspect "${repository}@${runnable_digest}" --raw >/dev/null + +printf '%s\n' "$runnable_digest" diff --git a/.github/scripts/test-resolve-runnable-image-digest.sh b/.github/scripts/test-resolve-runnable-image-digest.sh new file mode 100644 index 000000000..49b1c152c --- /dev/null +++ b/.github/scripts/test-resolve-runnable-image-digest.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +resolver=.github/scripts/resolve-runnable-image-digest.sh +scratch="$(mktemp -d -t runnable-image-digest.XXXXXXXXXX)" +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" + +cat >"$scratch/bin/docker" <<'MOCK_DOCKER' +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +if [[ "$#" -ne 5 || "$1" != "buildx" || "$2" != "imagetools" || + "$3" != "inspect" || "$5" != "--raw" ]]; then + echo "unexpected mocked docker invocation: $*" >&2 + exit 97 +fi + +case "$4" in + "$MOCK_INDEX_REF") + printf '%s\n' "$MOCK_INDEX_JSON" + ;; + "$MOCK_CHILD_REF") + if [[ "$MOCK_CHILD_PULLABLE" != "true" ]]; then + exit 98 + fi + printf '{"mediaType":"application/vnd.oci.image.manifest.v1+json"}\n' + ;; + *) + echo "unexpected mocked image reference: $4" >&2 + exit 99 + ;; +esac +MOCK_DOCKER +chmod +x "$scratch/bin/docker" + +digest_arm64="sha256:$(printf 'a%.0s' {1..64})" +digest_amd64="sha256:$(printf 'b%.0s' {1..64})" +digest_attestation="sha256:$(printf 'c%.0s' {1..64})" +MOCK_INDEX_REF="ghcr.io/tiplink/centaur/example@sha256:$(printf 'd%.0s' {1..64})" +export MOCK_INDEX_REF +export MOCK_CHILD_REF="ghcr.io/tiplink/centaur/example@${digest_arm64}" +export MOCK_CHILD_PULLABLE=true +export PATH="$scratch/bin:$PATH" + +make_index() { + jq -cn \ + --arg arm64 "$digest_arm64" \ + --arg amd64 "$digest_amd64" \ + --arg attestation "$digest_attestation" ' + { + mediaType: "application/vnd.oci.image.index.v1+json", + manifests: [ + { + mediaType: "application/vnd.oci.image.manifest.v1+json", + digest: $arm64, + platform: {os: "linux", architecture: "arm64"} + }, + { + mediaType: "application/vnd.oci.image.manifest.v1+json", + digest: $amd64, + platform: {os: "linux", architecture: "amd64"} + }, + { + mediaType: "application/vnd.oci.image.manifest.v1+json", + digest: $attestation, + platform: {os: "unknown", architecture: "unknown"}, + annotations: {"vnd.docker.reference.type": "attestation-manifest"} + } + ] + } + ' +} + +expect_reject() { + local label=$1 + shift + if "$@" >"$scratch/${label}.out" 2>&1; then + echo "runnable digest resolver unexpectedly accepted: $label" >&2 + exit 1 + fi +} + +MOCK_INDEX_JSON="$(make_index)" +export MOCK_INDEX_JSON +actual="$(bash "$resolver" "$MOCK_INDEX_REF" linux arm64)" +[[ "$actual" == "$digest_arm64" ]] || { + echo "resolver returned $actual instead of $digest_arm64" >&2 + exit 1 +} + +duplicate_json="$(jq --argjson duplicate "$(jq -c '.manifests[0]' <<<"$MOCK_INDEX_JSON")" \ + '.manifests += [$duplicate]' <<<"$MOCK_INDEX_JSON")" +export MOCK_INDEX_JSON="$duplicate_json" +expect_reject duplicate-platform-child bash "$resolver" "$MOCK_INDEX_REF" linux arm64 + +MOCK_INDEX_JSON="$(jq '.manifests |= map(select(.platform.architecture != "arm64"))' <<<"$(make_index)")" +export MOCK_INDEX_JSON +expect_reject missing-platform-child bash "$resolver" "$MOCK_INDEX_REF" linux arm64 + +MOCK_INDEX_JSON="$(jq '.manifests[0].digest = "sha256:not-a-digest"' <<<"$(make_index)")" +export MOCK_INDEX_JSON +expect_reject invalid-child-digest bash "$resolver" "$MOCK_INDEX_REF" linux arm64 + +export MOCK_INDEX_JSON='{"mediaType":"application/vnd.oci.image.manifest.v1+json"}' +expect_reject direct-manifest bash "$resolver" "$MOCK_INDEX_REF" linux arm64 + +MOCK_INDEX_JSON="$(jq '.mediaType = "application/example.index.v1+json"' <<<"$(make_index)")" +export MOCK_INDEX_JSON +expect_reject unsupported-index-media-type bash "$resolver" "$MOCK_INDEX_REF" linux arm64 + +MOCK_INDEX_JSON="$(make_index)" +export MOCK_INDEX_JSON +export MOCK_CHILD_PULLABLE=false +expect_reject inaccessible-child bash "$resolver" "$MOCK_INDEX_REF" linux arm64 +export MOCK_CHILD_PULLABLE=true + +expect_reject unpinned-index bash "$resolver" ghcr.io/tiplink/centaur/example:latest linux arm64 +expect_reject invalid-platform bash "$resolver" "$MOCK_INDEX_REF" 'Linux!' arm64 + +echo "runnable image digest resolver tests passed" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65a346f03..0406b02b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,6 +79,8 @@ jobs: '^services/api-rs/' \ '^\.github/rollback-bridge-reviewed-forward-commit$' \ '^\.github/scripts/check-rollback-bridge-workflows\.sh$' \ + '^\.github/scripts/resolve-runnable-image-digest\.sh$' \ + '^\.github/scripts/test-resolve-runnable-image-digest\.sh$' \ '^\.github/scripts/test-rollback-bridge-publication-trigger\.sh$' \ '^\.github/workflows/publish-images\.yml$' \ '^\.github/workflows/validate-images\.yml$' \ @@ -300,7 +302,7 @@ jobs: working-directory: reviewed-forward/services/api-rs env: BIND_ADDR: 127.0.0.1:18081 - CENTAUR_CONTROL_API_KEY: integration-control-key + CENTAUR_CONTROL_API_KEY: integration-control-key-0123456789abcdef DATABASE_URL: postgres://postgres:postgres@127.0.0.1:15432/centaur?sslmode=disable PYTHON_WORKFLOW_HOST_PATH: ${{ github.workspace }}/reviewed-forward/services/workflow-python/workflow_host.py PYTHON_WORKFLOW_HOST_PYTHON: python3 @@ -341,7 +343,7 @@ jobs: API_INTEGRATION_WORKFLOW_DIR: ${{ runner.temp }}/api-integration-workflows BIND_ADDR: 127.0.0.1:18080 CENTAUR_API_URL: http://127.0.0.1:18080 - CENTAUR_CONTROL_API_KEY: integration-control-key + CENTAUR_CONTROL_API_KEY: integration-control-key-0123456789abcdef CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS: "true" DATABASE_URL: postgres://postgres:postgres@127.0.0.1:15432/centaur?sslmode=disable ROLLBACK_BRIDGE_FORWARD_TEST_DATABASE_URL: postgres://postgres:postgres@127.0.0.1:15432/centaur?sslmode=disable diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index 2e6ab21b8..d06204d35 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -190,9 +190,11 @@ jobs: done # Build each image natively per platform (amd64 on x64 runners, arm64 on - # arm runners), push by digest, and hand the digests to the merge job below - # which assembles the multi-arch manifest. Package writes require either the - # confirmed manual path or the exact, + # arm runners), push by digest, and hand the attested index digests to the + # merge job below which assembles the multi-arch manifest. The build also + # records the exact runnable child selected from each index so the release + # descriptor can bind the final platform tag to this run. Package writes + # require either the confirmed manual path or the exact, # short-lived contents-write tag attestation validated above. build: needs: tag-absence-gate @@ -276,18 +278,47 @@ jobs: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.image }}:buildcache-${{ env.PLATFORM_SLUG }},mode=max type=gha,mode=max,scope=${{ matrix.image }}-${{ env.PLATFORM_SLUG }} - - name: Export digest + - name: Export attested index and runnable platform digests + env: + BUILD_INDEX_DIGEST: ${{ steps.build.outputs.digest }} + BUILD_IMAGE: ${{ matrix.image }} + BUILD_PLATFORM: ${{ matrix.platform }} run: | set -euo pipefail - mkdir -p ${{ runner.temp }}/digests - digest="${{ steps.build.outputs.digest }}" - touch "${{ runner.temp }}/digests/${digest#sha256:}" + IFS=$'\n\t' + if [[ ! "$BUILD_INDEX_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "invalid attested build index digest for $BUILD_IMAGE: $BUILD_INDEX_DIGEST" >&2 + exit 1 + fi + mkdir -p "${{ runner.temp }}/merge-index-digests" + touch "${{ runner.temp }}/merge-index-digests/${BUILD_INDEX_DIGEST#sha256:}" + + platform_os=${BUILD_PLATFORM%%/*} + platform_architecture=${BUILD_PLATFORM##*/} + runnable_digest="$(bash .github/scripts/resolve-runnable-image-digest.sh \ + "${REGISTRY}/${IMAGE_NAMESPACE}/${BUILD_IMAGE}@${BUILD_INDEX_DIGEST}" \ + "$platform_os" \ + "$platform_architecture")" + if [[ ! "$runnable_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "invalid runnable build digest for $BUILD_IMAGE ($BUILD_PLATFORM): $runnable_digest" >&2 + exit 1 + fi + mkdir -p "${{ runner.temp }}/runnable-digests" + touch "${{ runner.temp }}/runnable-digests/${runnable_digest#sha256:}" - - name: Upload digest + - name: Upload attested index digest for manifest merge uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: digests-${{ matrix.image }}-${{ env.PLATFORM_SLUG }} - path: ${{ runner.temp }}/digests/* + name: merge-index-digests-${{ matrix.image }}-${{ env.PLATFORM_SLUG }} + path: ${{ runner.temp }}/merge-index-digests/* + if-no-files-found: error + retention-days: 1 + + - name: Upload runnable platform child digest + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: runnable-digests-${{ matrix.image }}-${{ env.PLATFORM_SLUG }} + path: ${{ runner.temp }}/runnable-digests/* if-no-files-found: error retention-days: 1 @@ -304,11 +335,11 @@ jobs: - image: centaur-iron-proxy steps: - - name: Download digests + - name: Download attested index digests uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - pattern: digests-${{ matrix.image }}-* - path: ${{ runner.temp }}/digests + pattern: merge-index-digests-${{ matrix.image }}-* + path: ${{ runner.temp }}/merge-index-digests merge-multiple: true - name: Set up Docker Buildx @@ -335,7 +366,7 @@ jobs: org.opencontainers.image.source=${{ env.IMAGE_SOURCE }} - name: Create multi-arch manifest for ${{ matrix.image }} - working-directory: ${{ runner.temp }}/digests + working-directory: ${{ runner.temp }}/merge-index-digests env: GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} REVIEWED_TAG: reviewed-${{ github.sha }} @@ -386,9 +417,19 @@ jobs: tag_args+=("-t" "$tag") done + mapfile -t merge_index_digest_files < <(find . -maxdepth 1 -type f -print) + if [[ "${#merge_index_digest_files[@]}" -ne 2 ]]; then + echo "expected exactly two attested platform index digests for ${{ matrix.image }}; found ${#merge_index_digest_files[@]}" >&2 + exit 1 + fi image_refs=() - for digest in *; do - image_refs+=("${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.image }}@sha256:${digest}") + for digest_file in "${merge_index_digest_files[@]}"; do + digest="sha256:$(basename "$digest_file")" + if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "invalid attested platform index digest for ${{ matrix.image }}: $digest" >&2 + exit 1 + fi + image_refs+=("${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.image }}@${digest}") done docker buildx imagetools create "${tag_args[@]}" "${image_refs[@]}" @@ -403,11 +444,11 @@ jobs: runs-on: ubuntu-latest needs: merge steps: - - name: Download this run's linux/arm64 digests + - name: Download this run's runnable linux/arm64 child digests uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - pattern: digests-*-linux-arm64 - path: ${{ runner.temp }}/arm64-digests + pattern: runnable-digests-*-linux-arm64 + path: ${{ runner.temp }}/arm64-runnable-digests - name: Set up Docker Buildx uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 @@ -422,9 +463,10 @@ jobs: - name: Write release descriptor env: RELEASE_REVISION: ${{ github.sha }} - ARM64_DIGEST_ROOT: ${{ runner.temp }}/arm64-digests + ARM64_RUNNABLE_DIGEST_ROOT: ${{ runner.temp }}/arm64-runnable-digests run: | set -euo pipefail + IFS=$'\n\t' if [[ ! "$RELEASE_REVISION" =~ ^[0-9a-f]{40}$ ]]; then echo "invalid release revision: $RELEASE_REVISION" >&2 exit 1 @@ -447,19 +489,19 @@ jobs: echo "invalid arm64 digest for $image: $digest" >&2 exit 1 fi - artifact_dir="${ARM64_DIGEST_ROOT}/digests-${image}-linux-arm64" - mapfile -t built_digest_files < <(find "$artifact_dir" -maxdepth 1 -type f -print) - if [[ "${#built_digest_files[@]}" -ne 1 ]]; then - echo "expected one linux/arm64 digest artifact for $image; found ${#built_digest_files[@]}" >&2 + artifact_dir="${ARM64_RUNNABLE_DIGEST_ROOT}/runnable-digests-${image}-linux-arm64" + mapfile -t run_child_digest_files < <(find "$artifact_dir" -maxdepth 1 -type f -print) + if [[ "${#run_child_digest_files[@]}" -ne 1 ]]; then + echo "expected one runnable linux/arm64 child digest artifact for $image; found ${#run_child_digest_files[@]}" >&2 exit 1 fi - built_digest="sha256:$(basename "${built_digest_files[0]}")" - if [[ ! "$built_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "invalid built linux/arm64 digest for $image: $built_digest" >&2 + run_child_digest="sha256:$(basename "${run_child_digest_files[0]}")" + if [[ ! "$run_child_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "invalid runnable linux/arm64 child digest for $image: $run_child_digest" >&2 exit 1 fi - if [[ "$digest" != "$built_digest" ]]; then - echo "reviewed tag arm64 digest for $image does not match this run: tag=$digest built=$built_digest" >&2 + if [[ "$digest" != "$run_child_digest" ]]; then + echo "reviewed tag arm64 child for $image does not match this run: tag=$digest run_child=$run_child_digest" >&2 exit 1 fi # Prove the platform child itself is pullable. The multi-arch tag diff --git a/docs/pages/operate/upstream-rollback-bridge.mdx b/docs/pages/operate/upstream-rollback-bridge.mdx index a7cf9fdec..2ab0a145c 100644 --- a/docs/pages/operate/upstream-rollback-bridge.mdx +++ b/docs/pages/operate/upstream-rollback-bridge.mdx @@ -35,7 +35,11 @@ Fineas upstream-sync runbook in SHA to agree. It allows 120 seconds of future clock skew and fails closed once the attestation is 900 seconds old. If Actions queueing exceeds that window, rerun the helper and create a new timestamped tag; never delete or - recreate an attestation ref. Both paths emit only + recreate an attestation ref. The 900-second window bounds admission to the + package-writing workflow; it does not claim that a native multi-arch build + finishes inside that window. Hold the verified updater scope unchanged until + the descriptor completes. If it changes, supersede the run and repeat the + live check with a new immutable attestation ref. Both paths emit only `reviewed-` tags for exactly four linux/arm64 bridge runtime rows: API, Slackbot v2, agent, and IronProxy. Other branch/tag events cannot publish. This namespace cannot match the legacy updater's deploy-shaped @@ -47,6 +51,13 @@ Fineas upstream-sync runbook in pull-request workflow preserves the historical `Publish Images` check context but has read-only repository permissions, receives no registry credentials, and builds with `push: false`. +- Publication run `29175125487` at bridge head `a08bbe41` is superseded audit + evidence. Its BuildKit outputs were attested OCI indexes, while the final + tags resolved to runnable platform children; the old descriptor comparison + used the wrong identity and failed. The tags remain immutable, but that run + produced no approved descriptor and must never populate an infra lock. The + corrected publisher records merge-index and runnable-child digests + separately and requires the final arm64 child to equal this run's child. - Set `CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS=true` explicitly. The bridge refuses to start when the value is absent, false, or malformed. With the fence acknowledged, it starts no absurd workers, schedule ticks, metadata diff --git a/docs/public/md/operate/upstream-rollback-bridge.md b/docs/public/md/operate/upstream-rollback-bridge.md index a7cf9fdec..2ab0a145c 100644 --- a/docs/public/md/operate/upstream-rollback-bridge.md +++ b/docs/public/md/operate/upstream-rollback-bridge.md @@ -35,7 +35,11 @@ Fineas upstream-sync runbook in SHA to agree. It allows 120 seconds of future clock skew and fails closed once the attestation is 900 seconds old. If Actions queueing exceeds that window, rerun the helper and create a new timestamped tag; never delete or - recreate an attestation ref. Both paths emit only + recreate an attestation ref. The 900-second window bounds admission to the + package-writing workflow; it does not claim that a native multi-arch build + finishes inside that window. Hold the verified updater scope unchanged until + the descriptor completes. If it changes, supersede the run and repeat the + live check with a new immutable attestation ref. Both paths emit only `reviewed-` tags for exactly four linux/arm64 bridge runtime rows: API, Slackbot v2, agent, and IronProxy. Other branch/tag events cannot publish. This namespace cannot match the legacy updater's deploy-shaped @@ -47,6 +51,13 @@ Fineas upstream-sync runbook in pull-request workflow preserves the historical `Publish Images` check context but has read-only repository permissions, receives no registry credentials, and builds with `push: false`. +- Publication run `29175125487` at bridge head `a08bbe41` is superseded audit + evidence. Its BuildKit outputs were attested OCI indexes, while the final + tags resolved to runnable platform children; the old descriptor comparison + used the wrong identity and failed. The tags remain immutable, but that run + produced no approved descriptor and must never populate an infra lock. The + corrected publisher records merge-index and runnable-child digests + separately and requires the final arm64 child to equal this run's child. - Set `CENTAUR_ROLLBACK_BRIDGE_PAUSE_WORKFLOWS=true` explicitly. The bridge refuses to start when the value is absent, false, or malformed. With the fence acknowledged, it starts no absurd workers, schedule ticks, metadata From b613ff603a2a6195ef3cfa8c827ddc4d51d2b77b Mon Sep 17 00:00:00 2001 From: "fineas-bot[bot]" <258147136+fineas-bot[bot]@users.noreply.github.com> Date: Sun, 12 Jul 2026 10:42:20 -0400 Subject: [PATCH 6/9] fix: retain native browser support in rollback bridge Bind the bridge to the final reviewed forward head and preserve a tested native agent-browser executable on both amd64 and arm64. CodeQL baseline is unchanged. --- .../rollback-bridge-reviewed-forward-commit | 2 +- .../check-rollback-bridge-workflows.sh | 16 +++++++++++++ services/sandbox/Dockerfile | 23 +++++++++++++++---- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/.github/rollback-bridge-reviewed-forward-commit b/.github/rollback-bridge-reviewed-forward-commit index 5316b6a9e..a1be879a6 100644 --- a/.github/rollback-bridge-reviewed-forward-commit +++ b/.github/rollback-bridge-reviewed-forward-commit @@ -1 +1 @@ -8248db2d1dece81a2e91ed678068d2ab18d581cf +6a65eea2ebc7640e55d5b347bfa974aedb23620e diff --git a/.github/scripts/check-rollback-bridge-workflows.sh b/.github/scripts/check-rollback-bridge-workflows.sh index 43265cbba..94be3fadb 100644 --- a/.github/scripts/check-rollback-bridge-workflows.sh +++ b/.github/scripts/check-rollback-bridge-workflows.sh @@ -7,6 +7,7 @@ publisher=.github/workflows/publish-images.yml validator=.github/workflows/validate-images.yml ci=.github/workflows/ci.yml pin=.github/rollback-bridge-reviewed-forward-commit +sandbox_dockerfile=services/sandbox/Dockerfile fail() { echo "rollback bridge workflow safety check failed: $*" >&2 @@ -147,6 +148,21 @@ for integration_control_key in "${integration_control_keys[@]}"; do fail "CI contains a forward integration control key shorter than 32 bytes" done unset integration_control_key integration_control_keys + +grep -Fq 'ENV AGENT_BROWSER_EXECUTABLE_PATH=/home/agent/.local/bin/centaur-agent-browser-chromium' \ + "$sandbox_dockerfile" || + fail "rollback sandbox must expose the reviewed native browser path" +grep -Fq 'amd64)' "$sandbox_dockerfile" || + fail "rollback sandbox must bind agent-browser Chrome on amd64" +grep -Fq 'arm64)' "$sandbox_dockerfile" || + fail "rollback sandbox must bind Playwright Chromium on arm64" +grep -Fq -- "-type f -path '*/chrome-linux/headless_shell'" "$sandbox_dockerfile" || + fail "rollback sandbox arm64 path must use the installed Playwright shell" +grep -Fq 'ln -sf "$browser" "$AGENT_BROWSER_EXECUTABLE_PATH"' "$sandbox_dockerfile" || + fail "rollback sandbox must create the stable browser executable link" +grep -Fq '"$AGENT_BROWSER_EXECUTABLE_PATH" --version' "$sandbox_dockerfile" || + fail "rollback sandbox build must execute the selected browser" + grep -Fq '.github/rollback-bridge-reviewed-forward-commit' "$publisher" || fail "publisher does not read the central reviewed forward commit pin" for resolver_path in \ diff --git a/services/sandbox/Dockerfile b/services/sandbox/Dockerfile index 3cb522b6f..bf8bfc2ff 100644 --- a/services/sandbox/Dockerfile +++ b/services/sandbox/Dockerfile @@ -111,6 +111,7 @@ RUN set -eux; \ # ── Global npm CLIs (root) ────────────────────────────────────────────────── ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright +ENV AGENT_BROWSER_EXECUTABLE_PATH=/home/agent/.local/bin/centaur-agent-browser-chromium RUN --mount=type=cache,target=/root/.npm,sharing=locked \ npm install -g --prefer-online --force \ "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \ @@ -123,6 +124,7 @@ RUN --mount=type=cache,target=/root/.npm,sharing=locked \ && claude --version | grep -F "${CLAUDE_CODE_VERSION}" \ && codex --version | grep -F "${CODEX_VERSION}" \ && playwright --version | grep -F "${PLAYWRIGHT_VERSION}" \ + && agent-browser --version | grep -F "0.26.0" \ && pnpm --version | grep -F "${PNPM_VERSION}" # ── Browser automation deps (must run as root before USER agent) ───────────── @@ -181,10 +183,23 @@ RUN set -eux; \ rm -f /tmp/amp.sha256; \ amp --version >/dev/null -# ── agent-browser: download Chrome binary (runs as agent, deps already installed) ─ -RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \ - agent-browser install; \ - fi +# ── agent-browser: bind a native browser on both deployment architectures ─── +RUN set -eux; \ + mkdir -p "$HOME/.local/bin"; \ + case "$(dpkg --print-architecture)" in \ + amd64) \ + agent-browser install; \ + browser="$(find "$HOME/.agent-browser/browsers" \ + -type f -path '*/chrome-*/chrome' -print -quit)" ;; \ + arm64) \ + browser="$(find "$PLAYWRIGHT_BROWSERS_PATH" \ + -type f -path '*/chrome-linux/headless_shell' -print -quit)" ;; \ + *) echo unsupported-architecture >&2; exit 1 ;; \ + esac; \ + test -n "$browser"; \ + test -x "$browser"; \ + ln -sf "$browser" "$AGENT_BROWSER_EXECUTABLE_PATH"; \ + "$AGENT_BROWSER_EXECUTABLE_PATH" --version # ── Agent skills + workspace setup (rarely changes) ───────────────────────── RUN --mount=type=cache,target=/home/agent/.npm,uid=1001,gid=1001,sharing=locked \ From 2cd517085a263a8d7022b204acb12afda5e1edc6 Mon Sep 17 00:00:00 2001 From: "fineas-bot[bot]" <258147136+fineas-bot[bot]@users.noreply.github.com> Date: Sun, 12 Jul 2026 10:55:20 -0400 Subject: [PATCH 7/9] ci: validate rollback agent natively on arm64 Build the rollback agent on the native GitHub arm runner and execute its packaged Playwright/agent-browser contract before publication. CodeQL baseline is unchanged. --- .../check-rollback-bridge-workflows.sh | 8 ++++ .github/workflows/validate-images.yml | 42 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/.github/scripts/check-rollback-bridge-workflows.sh b/.github/scripts/check-rollback-bridge-workflows.sh index 94be3fadb..329ea645b 100644 --- a/.github/scripts/check-rollback-bridge-workflows.sh +++ b/.github/scripts/check-rollback-bridge-workflows.sh @@ -30,6 +30,14 @@ if grep -q 'docker/login-action' "$validator" || grep -q 'packages: write' "$val fi grep -Fq 'service: [api-rs, slackbotv2, linearbot, discordbot, teamsbot, agent, iron-proxy, console]' "$validator" || fail "PR validator must preserve the historical eight-image required-check matrix" +grep -q '^ agent-arm64:$' "$validator" || + fail "PR validator must build the rollback agent natively on arm64" +grep -Fq 'runs-on: ubuntu-24.04-arm' "$validator" || + fail "arm64 rollback validation must use the native GitHub arm runner" +grep -Fq 'centaur-agent:rollback-validate-linux-arm64' "$validator" || + fail "arm64 rollback validation must load and probe the packaged image" +grep -Fq '"$AGENT_BROWSER_EXECUTABLE_PATH" --version' "$validator" || + fail "arm64 rollback validation must execute the selected native browser" grep -q '^ workflow_dispatch:$' "$publisher" || fail "publisher must preserve the confirmed manual path" diff --git a/.github/workflows/validate-images.yml b/.github/workflows/validate-images.yml index bba42accf..3d0ce5185 100644 --- a/.github/workflows/validate-images.yml +++ b/.github/workflows/validate-images.yml @@ -86,3 +86,45 @@ jobs: RUST_BUILD_PROFILE=${{ env.RUST_BUILD_PROFILE }} cache-from: type=gha,scope=validate-${{ matrix.image }}-linux-amd64 cache-to: type=gha,mode=max,scope=validate-${{ matrix.image }}-linux-amd64 + + agent-arm64: + name: build (agent, linux/arm64) + runs-on: ubuntu-24.04-arm + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Build native arm64 rollback agent without registry credentials + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + file: services/sandbox/Dockerfile + target: sandbox + platforms: linux/arm64 + push: false + load: true + tags: centaur-agent:rollback-validate-linux-arm64 + build-args: | + RUST_BUILD_PROFILE=${{ env.RUST_BUILD_PROFILE }} + cache-from: type=gha,scope=validate-centaur-agent-linux-arm64 + cache-to: type=gha,mode=max,scope=validate-centaur-agent-linux-arm64 + + - name: Prove native browser contract in packaged arm64 image + run: | + set -euo pipefail + docker run --rm --entrypoint sh \ + centaur-agent:rollback-validate-linux-arm64 -ceu ' + test "$AGENT_BROWSER_EXECUTABLE_PATH" = \ + /home/agent/.local/bin/centaur-agent-browser-chromium + test -x "$AGENT_BROWSER_EXECUTABLE_PATH" + command -v agent-browser >/dev/null + command -v playwright >/dev/null + agent-browser --version | grep -F "0.26.0" + playwright --version | grep -F "Version 1.58.0" + "$AGENT_BROWSER_EXECUTABLE_PATH" --version >/dev/null + ' From 63ef84d040228e71ae1e1888051c97c4c6508ce4 Mon Sep 17 00:00:00 2001 From: "fineas-bot[bot]" <258147136+fineas-bot[bot]@users.noreply.github.com> Date: Sun, 12 Jul 2026 11:22:37 -0400 Subject: [PATCH 8/9] ci: bind rollback publication to reviewed runtime probes Enforce exact signed non-draft PR checks and probe the deployment-composed rollback agent on native amd64 and arm64. CodeQL remains an explicitly ignored inherited baseline. --- .../check-rollback-bridge-workflows.sh | 72 +++++- .../test-probe-rollback-agent-image.sh | 69 ++++++ ...est-rollback-bridge-publication-trigger.sh | 1 + .../test-verify-reviewed-rollback-release.sh | 130 ++++++++++ .../verify-reviewed-rollback-release.sh | 100 ++++++++ .github/workflows/ci.yml | 4 + .github/workflows/publish-images.yml | 6 + .github/workflows/validate-images.yml | 38 ++- .../operate/upstream-rollback-bridge.mdx | 17 +- .../md/operate/upstream-rollback-bridge.md | 17 +- scripts/probe-rollback-agent-image.sh | 230 ++++++++++++++++++ 11 files changed, 658 insertions(+), 26 deletions(-) create mode 100644 .github/scripts/test-probe-rollback-agent-image.sh create mode 100644 .github/scripts/test-verify-reviewed-rollback-release.sh create mode 100644 .github/scripts/verify-reviewed-rollback-release.sh create mode 100644 scripts/probe-rollback-agent-image.sh diff --git a/.github/scripts/check-rollback-bridge-workflows.sh b/.github/scripts/check-rollback-bridge-workflows.sh index 329ea645b..24d9ad95c 100644 --- a/.github/scripts/check-rollback-bridge-workflows.sh +++ b/.github/scripts/check-rollback-bridge-workflows.sh @@ -8,6 +8,7 @@ validator=.github/workflows/validate-images.yml ci=.github/workflows/ci.yml pin=.github/rollback-bridge-reviewed-forward-commit sandbox_dockerfile=services/sandbox/Dockerfile +agent_probe=scripts/probe-rollback-agent-image.sh fail() { echo "rollback bridge workflow safety check failed: $*" >&2 @@ -34,10 +35,50 @@ grep -q '^ agent-arm64:$' "$validator" || fail "PR validator must build the rollback agent natively on arm64" grep -Fq 'runs-on: ubuntu-24.04-arm' "$validator" || fail "arm64 rollback validation must use the native GitHub arm runner" +grep -Fq 'centaur-agent:rollback-validate-linux-amd64' "$validator" || + fail "amd64 rollback validation must load the packaged agent image" grep -Fq 'centaur-agent:rollback-validate-linux-arm64' "$validator" || - fail "arm64 rollback validation must load and probe the packaged image" -grep -Fq '"$AGENT_BROWSER_EXECUTABLE_PATH" --version' "$validator" || - fail "arm64 rollback validation must execute the selected native browser" + fail "arm64 rollback validation must load the packaged agent image" +grep -Fq "load: \${{ matrix.service == 'agent' }}" "$validator" || + fail "amd64 validation must load the final agent image" +if [[ "$(grep -Fc 'bash scripts/probe-rollback-agent-image.sh' "$validator")" -ne 2 ]]; then + fail "PR validator must probe exactly the native amd64 and arm64 rollback agent images" +fi +grep -Fq 'centaur-agent:rollback-validate-linux-amd64 linux/amd64' "$validator" || + fail "amd64 validation must run the deploy-composed rollback agent probe" +grep -Fq 'centaur-agent:rollback-validate-linux-arm64 linux/arm64' "$validator" || + fail "arm64 validation must run the deploy-composed rollback agent probe" +grep -q '^ image-validation-success:$' "$validator" || + fail "PR validator must expose an aggregate image validation check" +grep -q '^ name: Image validation success$' "$validator" || + fail "PR validator aggregate must preserve the exact publication check name" +grep -Fq 'ARM64_AGENT_RESULT: ${{ needs.agent-arm64.result }}' "$validator" || + fail "image validation aggregate must include the native arm64 agent result" +grep -Fq 'AMD64_BUILD_RESULT: ${{ needs.build.result }}' "$validator" || + fail "image validation aggregate must include the amd64 matrix and packaged agent result" + +for required_probe_contract in \ + '--network none' \ + 'EXPECTED_CODEX_VERSION=codex-cli 0.144.1' \ + 'EXPECTED_CLAUDE_VERSION=2.1.198 (Claude Code)' \ + 'EXPECTED_PLAYWRIGHT_VERSION=Version 1.58.0' \ + 'EXPECTED_AGENT_BROWSER_VERSION=0.26.0' \ + 'EXPECTED_HARNESS_SERVER_VERSION=harness-server 0.1.0' \ + '"$AGENT_BROWSER_EXECUTABLE_PATH" --version' \ + 'codex_config["model"] == "gpt-5.6-sol"' \ + 'codex_config["model_reasoning_effort"] == "medium"' \ + 'codex_config["plan_mode_reasoning_effort"] == "xhigh"' \ + '"max_concurrent_threads_per_session": 6' \ + 'codex_config["model_providers"] == {' \ + 'codex_config["projects"] == {"/": {"trust_level": "trusted"}}' \ + '["codex", "app-server", "--listen", "stdio://"]' \ + '"method": "initialize"'; do + grep -Fq -- "$required_probe_contract" "$agent_probe" || + fail "packaged rollback agent probe is missing contract: $required_probe_contract" +done +if grep -Fq '"method": "turn/start"' "$agent_probe"; then + fail "packaged rollback agent probe must initialize app-server without a model turn" +fi grep -q '^ workflow_dispatch:$' "$publisher" || fail "publisher must preserve the confirmed manual path" @@ -52,6 +93,12 @@ actual_push_trigger="$(awk ' if grep -Eq '^ (pull_request|create|schedule):' "$publisher"; then fail "publisher must not run for branches, pull requests, create events, or schedules" fi +for permission in 'actions: read' 'checks: read' 'pull-requests: read'; do + grep -q "^ ${permission}$" "$publisher" || + fail "publisher is missing required read permission: $permission" +done +grep -Fq 'bash .github/scripts/verify-reviewed-rollback-release.sh' "$publisher" || + fail "publisher must enforce the signed exact-head non-CodeQL PR release gate" expected_tag_pattern='^rollback-bridge-publish-live-scope-verified-([0-9a-f]{40})-forward-([0-9a-f]{40})-at-([1-9][0-9]{9})$' actual_tag_pattern="$(awk -F "'" '/^[[:space:]]*tag_pattern=/{ print $2 }' "$publisher")" [[ "$actual_tag_pattern" == "$expected_tag_pattern" ]] || @@ -146,9 +193,10 @@ actual_descriptor_rows="$(awk ' grep -Fq '.github/rollback-bridge-reviewed-forward-commit' "$ci" || fail "CI does not read the central reviewed forward commit pin" -mapfile -t integration_control_keys < <( - awk -F ': ' '/^[[:space:]]+CENTAUR_CONTROL_API_KEY: / { print $2 }' "$ci" -) +integration_control_keys=() +while IFS= read -r integration_control_key; do + integration_control_keys+=("$integration_control_key") +done < <(awk -F ': ' '/^[[:space:]]+CENTAUR_CONTROL_API_KEY: / { print $2 }' "$ci") [[ "${#integration_control_keys[@]}" -eq 2 ]] || fail "CI must configure exactly two forward integration control keys" for integration_control_key in "${integration_control_keys[@]}"; do @@ -173,11 +221,13 @@ grep -Fq '"$AGENT_BROWSER_EXECUTABLE_PATH" --version' "$sandbox_dockerfile" || grep -Fq '.github/rollback-bridge-reviewed-forward-commit' "$publisher" || fail "publisher does not read the central reviewed forward commit pin" -for resolver_path in \ +for required_ci_path in \ '^\.github/scripts/resolve-runnable-image-digest\.sh$' \ - '^\.github/scripts/test-resolve-runnable-image-digest\.sh$'; do - grep -Fq "$resolver_path" "$ci" || - fail "CI change detection does not cover $resolver_path" + '^\.github/scripts/test-resolve-runnable-image-digest\.sh$' \ + '^\.github/scripts/test-probe-rollback-agent-image\.sh$' \ + '^scripts/probe-rollback-agent-image\.sh$'; do + grep -Fq "$required_ci_path" "$ci" || + fail "CI change detection does not cover $required_ci_path" done placeholder="__REVIEWED_FORWARD_""COMMIT_REQUIRED__" unexpected_placeholder_files="$( @@ -189,6 +239,8 @@ if [[ -n "$unexpected_placeholder_files" ]]; then fi bash .github/scripts/test-rollback-bridge-publication-trigger.sh +bash .github/scripts/test-probe-rollback-agent-image.sh bash .github/scripts/test-resolve-runnable-image-digest.sh +bash .github/scripts/test-verify-reviewed-rollback-release.sh echo "rollback bridge workflow safety checks passed" diff --git a/.github/scripts/test-probe-rollback-agent-image.sh b/.github/scripts/test-probe-rollback-agent-image.sh new file mode 100644 index 000000000..46e98caf5 --- /dev/null +++ b/.github/scripts/test-probe-rollback-agent-image.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +probe=scripts/probe-rollback-agent-image.sh +scratch="$(mktemp -d -t rollback-agent-probe.XXXXXXXXXX)" +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" + +cat >"$scratch/bin/docker" <<'MOCK_DOCKER' +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +if [[ "${1:-}" == "image" && "${2:-}" == "inspect" ]]; then + [[ "${MOCK_IMAGE_PRESENT:?}" == "true" ]] + exit +fi +if [[ "${1:-}" == "run" ]]; then + printf '%s\n' "$@" >"${MOCK_DOCKER_ARGS:?}" + cat >"${MOCK_DOCKER_STDIN:?}" + exit +fi + +echo "unexpected mocked docker invocation: $*" >&2 +exit 97 +MOCK_DOCKER +chmod +x "$scratch/bin/docker" + +export PATH="$scratch/bin:$PATH" +export MOCK_DOCKER_ARGS="$scratch/docker-args" +export MOCK_DOCKER_STDIN="$scratch/docker-stdin" +export MOCK_IMAGE_PRESENT=true + +expect_reject() { + local label=$1 + shift + if "$@" >"$scratch/${label}.out" 2>&1; then + echo "rollback agent probe unexpectedly accepted: $label" >&2 + exit 1 + fi +} + +bash "$probe" centaur-agent:test linux/amd64 +grep -Fxq -- '--network' "$MOCK_DOCKER_ARGS" +grep -Fxq -- 'none' "$MOCK_DOCKER_ARGS" +grep -Fxq -- '--entrypoint' "$MOCK_DOCKER_ARGS" +grep -Fxq -- '/entrypoint.sh' "$MOCK_DOCKER_ARGS" +grep -Fxq -- 'EXPECTED_DEBIAN_ARCH=amd64' "$MOCK_DOCKER_ARGS" +grep -Fxq -- 'CODEX_MODEL=gpt-5.6-sol' "$MOCK_DOCKER_ARGS" +grep -Fxq -- 'CODEX_MODEL_REASONING_EFFORT=medium' "$MOCK_DOCKER_ARGS" +grep -Fq -- 'plan_mode_reasoning_effort = "xhigh"' "$MOCK_DOCKER_ARGS" +grep -Fq -- 'max_concurrent_threads_per_session = 6' "$MOCK_DOCKER_ARGS" +grep -Fq -- '"method": "initialize"' "$MOCK_DOCKER_STDIN" +if grep -Fq -- '"method": "turn/start"' "$MOCK_DOCKER_STDIN"; then + echo "rollback agent image probe must not start a model turn" >&2 + exit 1 +fi + +bash "$probe" centaur-agent:test linux/arm64 +grep -Fxq -- 'EXPECTED_DEBIAN_ARCH=arm64' "$MOCK_DOCKER_ARGS" + +expect_reject missing-image-argument bash "$probe" +expect_reject invalid-image-reference bash "$probe" '-unsafe' linux/amd64 +expect_reject invalid-platform bash "$probe" centaur-agent:test linux/s390x +export MOCK_IMAGE_PRESENT=false +expect_reject image-not-loaded bash "$probe" centaur-agent:test linux/amd64 + +echo "rollback agent packaged-image probe tests passed" diff --git a/.github/scripts/test-rollback-bridge-publication-trigger.sh b/.github/scripts/test-rollback-bridge-publication-trigger.sh index d76308db4..8c7dc6b8a 100644 --- a/.github/scripts/test-rollback-bridge-publication-trigger.sh +++ b/.github/scripts/test-rollback-bridge-publication-trigger.sh @@ -17,6 +17,7 @@ awk ' capture = 1 next } + capture && /^ - name:/ { exit } capture && /^ [^[:space:]]/ { exit } capture { sub(/^ /, "") diff --git a/.github/scripts/test-verify-reviewed-rollback-release.sh b/.github/scripts/test-verify-reviewed-rollback-release.sh new file mode 100644 index 000000000..91cadbb07 --- /dev/null +++ b/.github/scripts/test-verify-reviewed-rollback-release.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +verifier=.github/scripts/verify-reviewed-rollback-release.sh +scratch="$(mktemp -d -t reviewed-rollback-release.XXXXXXXXXX)" +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" + +MOCK_SHA="$(printf 'a%.0s' {1..40})" +export MOCK_SHA MOCK_MODE=valid + +cat >"$scratch/bin/git" <<'MOCK_GIT' +#!/usr/bin/env bash +set -euo pipefail +if [[ "$*" == "rev-parse HEAD" ]]; then + printf '%s\n' "${MOCK_SHA:?}" +else + echo "unexpected mocked git invocation: $*" >&2 + exit 97 +fi +MOCK_GIT + +cat >"$scratch/bin/curl" <<'MOCK_CURL' +#!/usr/bin/env bash +set -euo pipefail +url=${!#} +base='https://api.github.test/repos/TipLink/centaur' + +case "$url" in + "$base/git/commits/${MOCK_SHA}") + if [[ "${MOCK_MODE:?}" == "bad-signature" ]]; then + jq -cn --arg sha "$MOCK_SHA" '{sha:$sha,verification:{verified:false,reason:"unsigned"}}' + else + jq -cn --arg sha "$MOCK_SHA" '{sha:$sha,verification:{verified:true,reason:"valid"}}' + fi + ;; + "$base/commits/${MOCK_SHA}/pulls") + draft=false + if [[ "$MOCK_MODE" == "draft" ]]; then draft=true; fi + head_repo='TipLink/centaur' + if [[ "$MOCK_MODE" == "fork-pr" ]]; then head_repo='untrusted/centaur'; fi + base_ref=main + if [[ "$MOCK_MODE" == "wrong-base" ]]; then base_ref=release; fi + state=open + if [[ "$MOCK_MODE" == "closed-unmerged" ]]; then state=closed; fi + duplicate=false + if [[ "$MOCK_MODE" == "duplicate-pr" ]]; then duplicate=true; fi + jq -cn --arg sha "$MOCK_SHA" --argjson draft "$draft" --arg head_repo "$head_repo" \ + --arg base_ref "$base_ref" --arg state "$state" --argjson duplicate "$duplicate" \ + '[{number:78,state:$state,merged_at:null,draft:$draft,base:{ref:$base_ref,repo:{full_name:"TipLink/centaur"}},head:{sha:$sha,repo:{full_name:$head_repo}}}] | if $duplicate then . + . else . end' + ;; + "$base/commits/${MOCK_SHA}/check-runs?filter=latest&per_page=100") + ci=success + if [[ "$MOCK_MODE" == "failed-check" ]]; then ci=failure; fi + image_name='Image validation success' + if [[ "$MOCK_MODE" == "missing-check" ]]; then image_name='unrelated check'; fi + jq -cn --arg sha "$MOCK_SHA" --arg ci "$ci" --arg image_name "$image_name" '{check_runs:[ + {name:"CI success",head_sha:$sha,status:"completed",conclusion:$ci,completed_at:"2026-07-12T00:00:03Z",details_url:"https://github.com/TipLink/centaur/actions/runs/11/job/101",app:{slug:"github-actions"}}, + {name:"Console CI success",head_sha:$sha,status:"completed",conclusion:"success",completed_at:"2026-07-12T00:00:02Z",details_url:"https://github.com/TipLink/centaur/actions/runs/12/job/102",app:{slug:"github-actions"}}, + {name:$image_name,head_sha:$sha,status:"completed",conclusion:"success",completed_at:"2026-07-12T00:00:01Z",details_url:"https://github.com/TipLink/centaur/actions/runs/13/job/103",app:{slug:"github-actions"}}, + {name:"CodeQL",head_sha:$sha,status:"completed",conclusion:"failure",completed_at:"2026-07-12T00:00:04Z",details_url:"https://github.com/TipLink/centaur/actions/runs/14/job/104",app:{slug:"github-actions"}} + ]}' + ;; + "$base/actions/runs/11") + pr=78 + if [[ "$MOCK_MODE" == "wrong-pr-run" ]]; then pr=77; fi + event=pull_request + if [[ "$MOCK_MODE" == "wrong-event" ]]; then event=push; fi + run_sha="$MOCK_SHA" + if [[ "$MOCK_MODE" == "wrong-run-head" ]]; then run_sha="$(printf '0%.0s' {1..40})"; fi + jq -cn --arg sha "$run_sha" --arg event "$event" --argjson pr "$pr" '{head_sha:$sha,event:$event,status:"completed",conclusion:"success",path:".github/workflows/ci.yml",pull_requests:[{number:$pr,head:{sha:$sha}}]}' + ;; + "$base/actions/runs/12") + path='.github/workflows/console-ci.yml' + if [[ "$MOCK_MODE" == "wrong-workflow" ]]; then path='.github/workflows/codeql.yml'; fi + jq -cn --arg sha "$MOCK_SHA" --arg path "$path" '{head_sha:$sha,event:"pull_request",status:"completed",conclusion:"success",path:$path,pull_requests:[{number:78,head:{sha:$sha}}]}' + ;; + "$base/actions/runs/13") + jq -cn --arg sha "$MOCK_SHA" '{head_sha:$sha,event:"pull_request",status:"completed",conclusion:"success",path:".github/workflows/validate-images.yml",pull_requests:[{number:78,head:{sha:$sha}}]}' + ;; + *) + echo "unexpected mocked curl URL: $url" >&2 + exit 98 + ;; +esac +MOCK_CURL +chmod +x "$scratch/bin/git" "$scratch/bin/curl" + +export PATH="$scratch/bin:$PATH" +export GITHUB_API_TOKEN=not-a-real-token +export TRIGGER_API_URL=https://api.github.test +export TRIGGER_REPOSITORY=TipLink/centaur +export TRIGGER_SHA="$MOCK_SHA" + +expect_reject() { + local label=$1 + shift + if "$@" >"$scratch/${label}.out" 2>&1; then + echo "reviewed rollback release verifier unexpectedly accepted: $label" >&2 + exit 1 + fi +} + +bash "$verifier" >"$scratch/valid.out" +grep -qF 'OK reviewed signed rollback bridge PR #78' "$scratch/valid.out" + +for mode in \ + bad-signature \ + draft \ + fork-pr \ + wrong-base \ + closed-unmerged \ + duplicate-pr \ + failed-check \ + missing-check \ + wrong-workflow \ + wrong-pr-run \ + wrong-event \ + wrong-run-head; do + export MOCK_MODE="$mode" + expect_reject "$mode" bash "$verifier" +done + +export MOCK_MODE=valid +TRIGGER_SHA="$(printf 'b%.0s' {1..40})" +export TRIGGER_SHA +expect_reject wrong-checkout bash "$verifier" + +echo "reviewed rollback release gate tests passed" diff --git a/.github/scripts/verify-reviewed-rollback-release.sh b/.github/scripts/verify-reviewed-rollback-release.sh new file mode 100644 index 000000000..22814d8bc --- /dev/null +++ b/.github/scripts/verify-reviewed-rollback-release.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +required_env=( + GITHUB_API_TOKEN + TRIGGER_API_URL + TRIGGER_REPOSITORY + TRIGGER_SHA +) +for name in "${required_env[@]}"; do + if [[ -z "${!name:-}" ]]; then + echo "missing required environment variable: $name" >&2 + exit 2 + fi +done +unset name required_env + +if [[ ! "$TRIGGER_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ || + ! "$TRIGGER_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "invalid trigger repository or rollback bridge commit SHA" >&2 + exit 2 +fi +if [[ "$(git rev-parse HEAD)" != "$TRIGGER_SHA" ]]; then + echo "publication trigger SHA does not match the checked-out rollback bridge commit" >&2 + exit 1 +fi + +api_get() { + curl --fail --silent --show-error \ + --header "Authorization: Bearer ${GITHUB_API_TOKEN}" \ + --header 'Accept: application/vnd.github+json' \ + --header 'X-GitHub-Api-Version: 2022-11-28' \ + "$1" +} + +commit_json="$(api_get "${TRIGGER_API_URL}/repos/${TRIGGER_REPOSITORY}/git/commits/${TRIGGER_SHA}")" +if [[ "$(jq -r '.sha' <<<"$commit_json")" != "$TRIGGER_SHA" || + "$(jq -r '.verification.verified' <<<"$commit_json")" != "true" || + "$(jq -r '.verification.reason' <<<"$commit_json")" != "valid" ]]; then + echo "reviewed rollback bridge commit is not GitHub-signature verified" >&2 + exit 1 +fi + +pulls_json="$(api_get "${TRIGGER_API_URL}/repos/${TRIGGER_REPOSITORY}/commits/${TRIGGER_SHA}/pulls")" +reviewed_pr="$(jq -cer --arg sha "$TRIGGER_SHA" --arg repo "$TRIGGER_REPOSITORY" ' + [ .[] + | select(.base.ref == "main") + | select(.base.repo.full_name == $repo) + | select(.head.sha == $sha) + | select(.head.repo.full_name == $repo) + | select(.draft == false) + | select(.state == "open" or .merged_at != null) + ] + | if length == 1 then .[0] else error("expected exactly one ready or merged main PR at the rollback bridge SHA") end +' <<<"$pulls_json")" +reviewed_pr_number="$(jq -er '.number' <<<"$reviewed_pr")" + +checks_json="$(api_get "${TRIGGER_API_URL}/repos/${TRIGGER_REPOSITORY}/commits/${TRIGGER_SHA}/check-runs?filter=latest&per_page=100")" + +require_workflow_check() { + local check_name="$1" + local workflow_path="$2" + local check_json details_url run_id run_json + check_json="$(jq -cer --arg name "$check_name" --arg sha "$TRIGGER_SHA" ' + [.check_runs[] + | select(.name == $name and .head_sha == $sha and .app.slug == "github-actions")] + | sort_by(.completed_at // .started_at // "") + | if length > 0 then .[-1] else error("missing required GitHub Actions check") end + ' <<<"$checks_json")" + if [[ "$(jq -r '.status' <<<"$check_json")" != "completed" || + "$(jq -r '.conclusion' <<<"$check_json")" != "success" ]]; then + echo "required rollback bridge check is not successful: $check_name" >&2 + exit 1 + fi + details_url="$(jq -er '.details_url' <<<"$check_json")" + if [[ ! "$details_url" =~ /actions/runs/([0-9]+)/job/ ]]; then + echo "required rollback bridge check is not bound to an Actions run: $check_name" >&2 + exit 1 + fi + run_id="${BASH_REMATCH[1]}" + run_json="$(api_get "${TRIGGER_API_URL}/repos/${TRIGGER_REPOSITORY}/actions/runs/${run_id}")" + if [[ "$(jq -r '.head_sha' <<<"$run_json")" != "$TRIGGER_SHA" || + "$(jq -r '.event' <<<"$run_json")" != "pull_request" || + "$(jq -r '.status' <<<"$run_json")" != "completed" || + "$(jq -r '.conclusion' <<<"$run_json")" != "success" || + "$(jq -r '.path' <<<"$run_json")" != "$workflow_path" || + "$(jq -r --argjson pr "$reviewed_pr_number" --arg sha "$TRIGGER_SHA" ' + any(.pull_requests[]?; .number == $pr and .head.sha == $sha) + ' <<<"$run_json")" != "true" ]]; then + echo "required check is not a successful exact-head run of $workflow_path: $check_name" >&2 + exit 1 + fi +} + +require_workflow_check "CI success" ".github/workflows/ci.yml" +require_workflow_check "Console CI success" ".github/workflows/console-ci.yml" +require_workflow_check "Image validation success" ".github/workflows/validate-images.yml" + +echo "OK reviewed signed rollback bridge PR #${reviewed_pr_number} and exact-head non-CodeQL checks authorize publication" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0406b02b2..9587e39ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,8 +82,12 @@ jobs: '^\.github/scripts/resolve-runnable-image-digest\.sh$' \ '^\.github/scripts/test-resolve-runnable-image-digest\.sh$' \ '^\.github/scripts/test-rollback-bridge-publication-trigger\.sh$' \ + '^\.github/scripts/test-probe-rollback-agent-image\.sh$' \ + '^\.github/scripts/verify-reviewed-rollback-release\.sh$' \ + '^\.github/scripts/test-verify-reviewed-rollback-release\.sh$' \ '^\.github/workflows/publish-images\.yml$' \ '^\.github/workflows/validate-images\.yml$' \ + '^scripts/probe-rollback-agent-image\.sh$' \ '^\.github/workflows/ci\.yml$' set_output sandbox_config_tests \ '^services/sandbox/configure_codex_config\.py$' \ diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index d06204d35..1c1d92e74 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -24,8 +24,11 @@ concurrency: cancel-in-progress: false permissions: + actions: read + checks: read contents: read packages: write + pull-requests: read env: REGISTRY: ghcr.io @@ -137,6 +140,9 @@ jobs: esac echo "commit=$commit" >> "$GITHUB_OUTPUT" + - name: Require signed ready PR and exact-head non-CodeQL checks + run: bash .github/scripts/verify-reviewed-rollback-release.sh + tag-absence-gate: name: Prove reviewed tags do not already exist before publishing digests runs-on: ubuntu-latest diff --git a/.github/workflows/validate-images.yml b/.github/workflows/validate-images.yml index 3d0ce5185..ea56f271f 100644 --- a/.github/workflows/validate-images.yml +++ b/.github/workflows/validate-images.yml @@ -82,11 +82,17 @@ jobs: target: ${{ matrix.target }} platforms: ${{ matrix.platform }} push: false + load: ${{ matrix.service == 'agent' }} + tags: ${{ matrix.service == 'agent' && 'centaur-agent:rollback-validate-linux-amd64' || '' }} build-args: | RUST_BUILD_PROFILE=${{ env.RUST_BUILD_PROFILE }} cache-from: type=gha,scope=validate-${{ matrix.image }}-linux-amd64 cache-to: type=gha,mode=max,scope=validate-${{ matrix.image }}-linux-amd64 + - name: Probe deploy-composed rollback harness in packaged amd64 agent + if: matrix.service == 'agent' + run: bash scripts/probe-rollback-agent-image.sh centaur-agent:rollback-validate-linux-amd64 linux/amd64 + agent-arm64: name: build (agent, linux/arm64) runs-on: ubuntu-24.04-arm @@ -114,17 +120,25 @@ jobs: cache-from: type=gha,scope=validate-centaur-agent-linux-arm64 cache-to: type=gha,mode=max,scope=validate-centaur-agent-linux-arm64 - - name: Prove native browser contract in packaged arm64 image + - name: Probe deploy-composed rollback harness in packaged arm64 agent + run: bash scripts/probe-rollback-agent-image.sh centaur-agent:rollback-validate-linux-arm64 linux/arm64 + + image-validation-success: + name: Image validation success + runs-on: ubuntu-latest + if: always() + needs: + - build + - agent-arm64 + steps: + - name: Require every rollback image validation build to succeed + env: + AMD64_BUILD_RESULT: ${{ needs.build.result }} + ARM64_AGENT_RESULT: ${{ needs.agent-arm64.result }} run: | set -euo pipefail - docker run --rm --entrypoint sh \ - centaur-agent:rollback-validate-linux-arm64 -ceu ' - test "$AGENT_BROWSER_EXECUTABLE_PATH" = \ - /home/agent/.local/bin/centaur-agent-browser-chromium - test -x "$AGENT_BROWSER_EXECUTABLE_PATH" - command -v agent-browser >/dev/null - command -v playwright >/dev/null - agent-browser --version | grep -F "0.26.0" - playwright --version | grep -F "Version 1.58.0" - "$AGENT_BROWSER_EXECUTABLE_PATH" --version >/dev/null - ' + if [[ "$AMD64_BUILD_RESULT" != "success" || + "$ARM64_AGENT_RESULT" != "success" ]]; then + echo "rollback image validation failed: amd64=$AMD64_BUILD_RESULT arm64=$ARM64_AGENT_RESULT" >&2 + exit 1 + fi diff --git a/docs/pages/operate/upstream-rollback-bridge.mdx b/docs/pages/operate/upstream-rollback-bridge.mdx index 2ab0a145c..676649836 100644 --- a/docs/pages/operate/upstream-rollback-bridge.mdx +++ b/docs/pages/operate/upstream-rollback-bridge.mdx @@ -32,7 +32,10 @@ Fineas upstream-sync runbook in `rollback-bridge-publish-live-scope-verified--forward--at-` immediately after the same read-only live check. The release gate requires the tag object, checkout, event SHA, embedded bridge SHA, and frozen forward - SHA to agree. It allows 120 seconds of future clock skew and fails closed + SHA to agree. It also requires the exact GitHub-verified, same-repository, + non-draft bridge PR head and successful exact-head `CI success`, `Console CI + success`, and `Image validation success` runs; CodeQL is intentionally not a + publication gate. It allows 120 seconds of future clock skew and fails closed once the attestation is 900 seconds old. If Actions queueing exceeds that window, rerun the helper and create a new timestamped tag; never delete or recreate an attestation ref. The 900-second window bounds admission to the @@ -50,7 +53,17 @@ Fineas upstream-sync runbook in consuming the descriptor in Git or changing any Argo pin. The separate pull-request workflow preserves the historical `Publish Images` check context but has read-only repository permissions, receives no registry - credentials, and builds with `push: false`. + credentials, and builds with `push: false`. Its agent jobs load the final + image natively on both amd64 and arm64, run the real entrypoint with the exact + Fineas rollback Codex and Claude overlays, and then prove Codex `0.144.1`, + Claude Code `2.1.198`, Playwright `1.58.0`, agent-browser `0.26.0`, + `harness-server`, and the native browser executable. It also proves + `gpt-5.6-sol` at medium effort, plan effort `xhigh`, multi-agent v2 disabled + with cap 6, Sonnet 5 at high effort, and preservation of the exact OpenRouter + provider and root trust contract. The probe has + no network and sends only a Codex app-server `initialize` request; it never + starts a model turn. The aggregate `Image validation success` check fails if + either native packaged-image probe fails. - Publication run `29175125487` at bridge head `a08bbe41` is superseded audit evidence. Its BuildKit outputs were attested OCI indexes, while the final tags resolved to runnable platform children; the old descriptor comparison diff --git a/docs/public/md/operate/upstream-rollback-bridge.md b/docs/public/md/operate/upstream-rollback-bridge.md index 2ab0a145c..676649836 100644 --- a/docs/public/md/operate/upstream-rollback-bridge.md +++ b/docs/public/md/operate/upstream-rollback-bridge.md @@ -32,7 +32,10 @@ Fineas upstream-sync runbook in `rollback-bridge-publish-live-scope-verified--forward--at-` immediately after the same read-only live check. The release gate requires the tag object, checkout, event SHA, embedded bridge SHA, and frozen forward - SHA to agree. It allows 120 seconds of future clock skew and fails closed + SHA to agree. It also requires the exact GitHub-verified, same-repository, + non-draft bridge PR head and successful exact-head `CI success`, `Console CI + success`, and `Image validation success` runs; CodeQL is intentionally not a + publication gate. It allows 120 seconds of future clock skew and fails closed once the attestation is 900 seconds old. If Actions queueing exceeds that window, rerun the helper and create a new timestamped tag; never delete or recreate an attestation ref. The 900-second window bounds admission to the @@ -50,7 +53,17 @@ Fineas upstream-sync runbook in consuming the descriptor in Git or changing any Argo pin. The separate pull-request workflow preserves the historical `Publish Images` check context but has read-only repository permissions, receives no registry - credentials, and builds with `push: false`. + credentials, and builds with `push: false`. Its agent jobs load the final + image natively on both amd64 and arm64, run the real entrypoint with the exact + Fineas rollback Codex and Claude overlays, and then prove Codex `0.144.1`, + Claude Code `2.1.198`, Playwright `1.58.0`, agent-browser `0.26.0`, + `harness-server`, and the native browser executable. It also proves + `gpt-5.6-sol` at medium effort, plan effort `xhigh`, multi-agent v2 disabled + with cap 6, Sonnet 5 at high effort, and preservation of the exact OpenRouter + provider and root trust contract. The probe has + no network and sends only a Codex app-server `initialize` request; it never + starts a model turn. The aggregate `Image validation success` check fails if + either native packaged-image probe fails. - Publication run `29175125487` at bridge head `a08bbe41` is superseded audit evidence. Its BuildKit outputs were attested OCI indexes, while the final tags resolved to runnable platform children; the old descriptor comparison diff --git a/scripts/probe-rollback-agent-image.sh b/scripts/probe-rollback-agent-image.sh new file mode 100644 index 000000000..ac8fc0bdf --- /dev/null +++ b/scripts/probe-rollback-agent-image.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +image=${1:-} +platform=${2:-} + +if [[ ! "$image" =~ ^[A-Za-z0-9][A-Za-z0-9._/@:-]*$ ]]; then + echo "usage: $0 IMAGE linux/amd64|linux/arm64" >&2 + exit 2 +fi + +case "$platform" in + linux/amd64) + expected_debian_arch=amd64 + ;; + linux/arm64) + expected_debian_arch=arm64 + ;; + *) + echo "usage: $0 IMAGE linux/amd64|linux/arm64" >&2 + exit 2 + ;; +esac + +if ! command -v docker >/dev/null 2>&1; then + echo "missing required command: docker" >&2 + exit 1 +fi +if ! docker image inspect "$image" >/dev/null 2>&1; then + echo "rollback agent image is not loaded locally: $image" >&2 + exit 1 +fi + +# These are the exact Fineas rollback deployment deltas. The bridge image owns +# the legacy-compatible base config; entrypoint.sh must merge these values over +# that base without losing its OpenRouter provider or root trust declaration. +codex_overlay='model = "gpt-5.6-sol" +plan_mode_reasoning_effort = "xhigh" + +[features.multi_agent_v2] +enabled = false +max_concurrent_threads_per_session = 6' +claude_overlay='{"model":"claude-sonnet-5","effortLevel":"high","alwaysThinkingEnabled":false}' + +docker run --rm -i \ + --network none \ + --entrypoint /entrypoint.sh \ + --env "EXPECTED_DEBIAN_ARCH=${expected_debian_arch}" \ + --env 'EXPECTED_CODEX_VERSION=codex-cli 0.144.1' \ + --env 'EXPECTED_CLAUDE_VERSION=2.1.198 (Claude Code)' \ + --env 'EXPECTED_PLAYWRIGHT_VERSION=Version 1.58.0' \ + --env 'EXPECTED_AGENT_BROWSER_VERSION=0.26.0' \ + --env 'EXPECTED_HARNESS_SERVER_VERSION=harness-server 0.1.0' \ + --env CODEX_AUTH_MODE=access_token \ + --env CLAUDE_CODE_AUTH_MODE=access_token \ + --env CODEX_MODEL=gpt-5.6-sol \ + --env CODEX_MODEL_REASONING_EFFORT=medium \ + --env "CODEX_CONFIG_OVERLAY=${codex_overlay}" \ + --env CLAUDE_MODEL=claude-sonnet-5 \ + --env CLAUDE_CODE_EFFORT_LEVEL=high \ + --env "CLAUDE_SETTINGS_OVERLAY=${claude_overlay}" \ + --env GOOGLE_APPLICATION_CREDENTIALS=/tmp/centaur-no-network-adc.json \ + --env OPENAI_API_KEY= \ + --env CODEX_API_KEY= \ + --env OPENROUTER_API_KEY= \ + --env META_AI_API_KEY= \ + --env CENTAUR_TOOLS_URL= \ + "$image" /bin/bash -seu <<'CONTAINER_SCRIPT' +set -euo pipefail +IFS=$'\n\t' + +test "$(dpkg --print-architecture)" = "$EXPECTED_DEBIAN_ARCH" +test "$(codex --version)" = "$EXPECTED_CODEX_VERSION" +test "$(claude --version)" = "$EXPECTED_CLAUDE_VERSION" +test "$(playwright --version)" = "$EXPECTED_PLAYWRIGHT_VERSION" +test "$(harness-server --version)" = "$EXPECTED_HARNESS_SERVER_VERSION" + +agent_browser_version="$(agent-browser --version)" +case "$agent_browser_version" in + "$EXPECTED_AGENT_BROWSER_VERSION" | "agent-browser $EXPECTED_AGENT_BROWSER_VERSION") ;; + *) + echo "unexpected agent-browser version: $agent_browser_version" >&2 + exit 1 + ;; +esac + +for command in codex claude playwright agent-browser harness-server; do + command -v "$command" >/dev/null +done +test "${AGENT_BROWSER_EXECUTABLE_PATH:-}" = \ + /home/agent/.local/bin/centaur-agent-browser-chromium +test -x "$AGENT_BROWSER_EXECUTABLE_PATH" +"$AGENT_BROWSER_EXECUTABLE_PATH" --version >/dev/null + +test "${CODEX_MODEL:-}" = gpt-5.6-sol +test "${CODEX_MODEL_REASONING_EFFORT:-}" = medium +test "${CLAUDE_MODEL:-}" = claude-sonnet-5 +test "${CLAUDE_CODE_EFFORT_LEVEL:-}" = high +test -z "${CENTAUR_HARNESS_CONFIG_DIR:-}" + +python3 - <<'PY' +from __future__ import annotations + +import json +import os +import select +import subprocess +import time +import tomllib +from pathlib import Path + + +home = Path.home() +with (home / ".codex/config.toml").open("rb") as handle: + codex_config = tomllib.load(handle) +with (home / ".claude/settings.json").open(encoding="utf-8") as handle: + claude_settings = json.load(handle) + +assert codex_config["model"] == "gpt-5.6-sol" +assert codex_config["model_reasoning_effort"] == "medium" +assert codex_config["plan_mode_reasoning_effort"] == "xhigh" +assert codex_config["service_tier"] == "fast" + +features = codex_config["features"] +assert features["multi_agent"] is False +assert features["multi_agent_v2"] == { + "enabled": False, + "max_concurrent_threads_per_session": 6, +} +assert features["enable_fanout"] is False + +assert codex_config["model_providers"] == { + "openrouter": { + "name": "OpenRouter", + "base_url": "https://openrouter.ai/api/v1", + "env_key": "OPENROUTER_API_KEY", + "wire_api": "responses", + "requires_openai_auth": False, + } +} +assert codex_config["projects"] == {"/": {"trust_level": "trusted"}} + +assert claude_settings == { + "model": "claude-sonnet-5", + "effortLevel": "high", + "permissions": { + "defaultMode": "bypassPermissions", + "additionalDirectories": [ + "/home/agent/workspace", + "/home/agent/uploads", + ], + }, + "includeCoAuthoredBy": False, + "cleanupPeriodDays": 1, + "viewMode": "verbose", + "alwaysThinkingEnabled": False, +} + +feature_lines = subprocess.run( + ["codex", "features", "list"], + check=True, + capture_output=True, + text=True, +).stdout.splitlines() +feature_states = { + fields[0]: fields[-1] + for line in feature_lines + if len(fields := line.split()) >= 2 +} +assert feature_states.get("multi_agent") == "false" +assert feature_states.get("multi_agent_v2") == "false" + + +process = subprocess.Popen( + ["codex", "app-server", "--listen", "stdio://"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, +) +try: + assert process.stdin is not None + process.stdin.write( + json.dumps( + { + "id": 1, + "method": "initialize", + "params": { + "clientInfo": { + "name": "centaur-rollback-image-probe", + "title": None, + "version": "0", + }, + "capabilities": None, + }, + }, + separators=(",", ":"), + ) + + "\n" + ) + process.stdin.flush() + + assert process.stdout is not None + deadline = time.monotonic() + 10 + while True: + remaining = deadline - time.monotonic() + assert remaining > 0, "timed out waiting for Codex app-server initialize" + readable, _, _ = select.select([process.stdout], [], [], remaining) + assert readable, "timed out waiting for Codex app-server initialize" + line = process.stdout.readline() + assert line, f"Codex app-server exited during initialize: {process.poll()}" + message = json.loads(line) + if message.get("id") != 1: + continue + assert "error" not in message, message["error"] + assert "result" in message + break +finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + +print("deploy-composed rollback agent harness probe passed") +PY +CONTAINER_SCRIPT From 08fed5c267fd0d338a0c4923f893b1d4fa61bec5 Mon Sep 17 00:00:00 2001 From: "fineas-bot[bot]" <258147136+fineas-bot[bot]@users.noreply.github.com> Date: Sun, 12 Jul 2026 14:12:47 -0400 Subject: [PATCH 9/9] [upstream-sync] Retain scoped Slack ETL credentials in bridge Carry approved least-privilege Slack ETL rules through schema-forward rollback and advance the frozen reviewed-forward pin. --- .../rollback-bridge-reviewed-forward-commit | 2 +- docs/pages/operate/slack-etl.mdx | 5 +- docs/public/md/operate/slack-etl.md | 5 +- .../api-rs/crates/centaur-perms/src/tests.rs | 56 +++++++++++++++++++ tools/productivity/slack/pyproject.toml | 3 +- 5 files changed, 65 insertions(+), 6 deletions(-) diff --git a/.github/rollback-bridge-reviewed-forward-commit b/.github/rollback-bridge-reviewed-forward-commit index a1be879a6..7af433d65 100644 --- a/.github/rollback-bridge-reviewed-forward-commit +++ b/.github/rollback-bridge-reviewed-forward-commit @@ -1 +1 @@ -6a65eea2ebc7640e55d5b347bfa974aedb23620e +46d5932719cfcbb8a8067e68f98a4c7c2c20c853 diff --git a/docs/pages/operate/slack-etl.mdx b/docs/pages/operate/slack-etl.mdx index 1a9747de8..6f90599cc 100644 --- a/docs/pages/operate/slack-etl.mdx +++ b/docs/pages/operate/slack-etl.mdx @@ -39,8 +39,9 @@ posting to Slack. Create a Slack user token for ETL reads and store it as `SLACK_ETL_TOKEN` in the same secret source used by tools. The Slack tool declares it as an optional -HTTP secret for `slack.com` and `files.slack.com`; iron-proxy injects the real -value when the tool calls Slack. +HTTP secret scoped to the Slack Web API endpoints below and `GET` file downloads +from `files.slack.com`; iron-proxy injects the real value when the workflow +calls Slack. The token must be able to call: diff --git a/docs/public/md/operate/slack-etl.md b/docs/public/md/operate/slack-etl.md index 1a9747de8..6f90599cc 100644 --- a/docs/public/md/operate/slack-etl.md +++ b/docs/public/md/operate/slack-etl.md @@ -39,8 +39,9 @@ posting to Slack. Create a Slack user token for ETL reads and store it as `SLACK_ETL_TOKEN` in the same secret source used by tools. The Slack tool declares it as an optional -HTTP secret for `slack.com` and `files.slack.com`; iron-proxy injects the real -value when the tool calls Slack. +HTTP secret scoped to the Slack Web API endpoints below and `GET` file downloads +from `files.slack.com`; iron-proxy injects the real value when the workflow +calls Slack. The token must be able to call: diff --git a/services/api-rs/crates/centaur-perms/src/tests.rs b/services/api-rs/crates/centaur-perms/src/tests.rs index f93ad99a9..79febb1e2 100644 --- a/services/api-rs/crates/centaur-perms/src/tests.rs +++ b/services/api-rs/crates/centaur-perms/src/tests.rs @@ -1003,6 +1003,62 @@ fn real_slack_tool_parses_and_translates() { ), "expected the SLACK_BOT_TOKEN static secret" ); + + let etl_inputs = out + .inputs + .iter() + .filter_map(|input| match input { + SecretInput::Static(secret) if secret.name == "SLACK_ETL_TOKEN" => Some(secret), + _ => None, + }) + .collect::>(); + assert_eq!(etl_inputs.len(), 2); + + let slack_api = etl_inputs + .iter() + .find(|secret| { + secret + .rules + .iter() + .any(|rule| rule.host.as_deref() == Some("slack.com")) + }) + .expect("expected Slack Web API ETL token rules"); + let mut slack_hosts = slack_api + .rules + .iter() + .map(|rule| rule.host.as_deref().unwrap_or_default().to_owned()) + .collect::>(); + slack_hosts.sort(); + assert_eq!( + slack_hosts, + vec!["slack.com".to_owned(), "www.slack.com".to_owned()] + ); + for rule in &slack_api.rules { + assert_eq!(rule.http_methods, vec!["GET".to_owned(), "POST".to_owned()]); + assert_eq!( + rule.paths, + vec![ + "/api/conversations.list".to_owned(), + "/api/conversations.history".to_owned(), + "/api/conversations.replies".to_owned(), + "/api/users.list".to_owned(), + ] + ); + } + + let files = etl_inputs + .iter() + .find(|secret| { + secret + .rules + .iter() + .any(|rule| rule.host.as_deref() == Some("files.slack.com")) + }) + .expect("expected Slack file download ETL token rule"); + assert_eq!(files.rules.len(), 1); + assert_eq!(files.rules[0].host.as_deref(), Some("files.slack.com")); + assert_eq!(files.rules[0].http_methods, vec!["GET".to_owned()]); + assert!(files.rules[0].paths.is_empty()); } #[test] diff --git a/tools/productivity/slack/pyproject.toml b/tools/productivity/slack/pyproject.toml index c9ce75886..7c18c3440 100644 --- a/tools/productivity/slack/pyproject.toml +++ b/tools/productivity/slack/pyproject.toml @@ -32,5 +32,6 @@ optional_secrets = [ {type = "http", name = "SLACK_BOT_TOKEN", mode = "inject", inject_header = "Authorization", inject_formatter = "Bearer {{ .Value }}", hosts = ["slack.com"]}, {type = "http", name = "SLACK_SEARCH_TOKEN", mode = "inject", inject_header = "Authorization", inject_formatter = "Bearer {{ .Value }}", hosts = ["slack.com", "www.slack.com"], methods = ["POST"], paths = ["/api/assistant.search.context"]}, {type = "http", name = "SLACK_UPLOAD_TOKEN", mode = "inject", inject_header = "Authorization", inject_formatter = "Bearer {{ .Value }}", hosts = ["slack.com", "files.slack.com"]}, - {type = "http", name = "SLACK_ETL_TOKEN", mode = "inject", inject_header = "Authorization", inject_formatter = "Bearer {{ .Value }}", hosts = ["slack.com", "files.slack.com"]}, + {type = "http", name = "SLACK_ETL_TOKEN", mode = "inject", inject_header = "Authorization", inject_formatter = "Bearer {{ .Value }}", hosts = ["slack.com", "www.slack.com"], methods = ["GET", "POST"], paths = ["/api/conversations.list", "/api/conversations.history", "/api/conversations.replies", "/api/users.list"]}, + {type = "http", name = "SLACK_ETL_TOKEN", mode = "inject", inject_header = "Authorization", inject_formatter = "Bearer {{ .Value }}", hosts = ["files.slack.com"], methods = ["GET"]}, ]