diff --git a/.agents/skills/company-context/SKILL.md b/.agents/skills/company-context/SKILL.md index 7a2100266..c0685292d 100644 --- a/.agents/skills/company-context/SKILL.md +++ b/.agents/skills/company-context/SKILL.md @@ -1,11 +1,11 @@ --- name: company-context -description: "Use Centaur's indexed company context together with direct Slack, Linear, Google Docs, Drive, or Calendar searches when answering internal company-history, prior-decision, project-context, meeting-context, roadmap/status, or cross-source memory questions. Indexed context includes Slack channels, user-visible Slack DMs, Google Docs, Google Calendar, and Linear. Use for questions like what was discussed, decided, planned, mentioned, or documented internally, especially when the user did not name one exact source." +description: "Use Centaur's indexed company context together with direct Slack, Linear, Google Docs, Drive, Calendar, or Granola searches when answering internal company-history, prior-decision, project-context, meeting-context, roadmap/status, or cross-source memory questions. Indexed context includes Slack channels, user-visible Slack DMs, Google Docs, Google Calendar, Linear, and user-visible Granola notes. Use for questions like what was discussed, decided, planned, mentioned, or documented internally, especially when the user did not name one exact source." --- # Company Context -Use `company_context` as the first retrieval step for internal historical context. Its `search` command queries indexed company memory across enabled sources such as Slack channels, Google Docs (`--source docs`), Google Calendar, and Linear. It also has dedicated commands for user-visible Slack DMs and DM conversations. Always pair indexed results with the relevant direct source tools, then reconcile and collate both evidence sets before answering. +Use `company_context` as the first retrieval step for internal historical context. Its `search` command queries indexed company memory across enabled sources such as Slack channels, Google Docs (`--source docs`), Google Calendar, Linear, and user-visible Granola notes (`--source granola`). It also has dedicated commands for user-visible Slack DMs and DM conversations. Always pair indexed results with the relevant direct source tools, then reconcile and collate both evidence sets before answering. ## Default Workflow @@ -57,6 +57,7 @@ company_context search "QUERY" --source slack --limit 10 --json company_context search "QUERY" --source docs --source-type google_doc --limit 10 --json company_context search "QUERY" --source google_calendar --limit 10 --json company_context search "QUERY" --source linear --limit 10 --json +company_context search "QUERY" --source granola --limit 10 --json company_context search-dms "QUERY" --limit 10 --json ``` diff --git a/contrib/chart/Chart.yaml b/contrib/chart/Chart.yaml index efa9031d8..a03091a65 100644 --- a/contrib/chart/Chart.yaml +++ b/contrib/chart/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: centaur description: Helm chart for the trusted Centaur control plane type: application -version: 0.1.98 +version: 0.1.103 appVersion: "0.1.0" dependencies: - name: connect diff --git a/contrib/chart/templates/apirs.yaml b/contrib/chart/templates/apirs.yaml index 39d4458fa..81a11a9f0 100644 --- a/contrib/chart/templates/apirs.yaml +++ b/contrib/chart/templates/apirs.yaml @@ -127,6 +127,7 @@ (dict "name" "COMPANY_CONTEXT_DOCUMENTS_ENABLED" "value" (dig "companyContextDocuments" "enabled" true $apiRsEtl)) (dict "name" "COMPANY_CONTEXT_DOCUMENTS_INTERVAL_SECONDS" "value" (dig "companyContextDocuments" "intervalSeconds" 14400 $apiRsEtl)) (dict "name" "COMPANY_CONTEXT_DOCUMENTS_MAX_WINDOW_SECONDS" "value" (dig "companyContextDocuments" "maxWindowSeconds" 21600 $apiRsEtl)) + (dict "name" "COMPANY_CONTEXT_DOCUMENTS_BATCH_SIZE" "value" (dig "companyContextDocuments" "batchSize" 50 $apiRsEtl)) -}} {{- $apiRsEtlPassthroughNames := list -}} {{- range $env := $apiRsEtlEnv -}} diff --git a/contrib/chart/templates/console-worker.yaml b/contrib/chart/templates/console-worker.yaml index 739e8d05c..2419cdc22 100644 --- a/contrib/chart/templates/console-worker.yaml +++ b/contrib/chart/templates/console-worker.yaml @@ -164,6 +164,8 @@ spec: {{- if .Values.apiRs.enabled }} - name: CENTAUR_API_URL value: {{ printf "http://%s:%v" $apiRsName .Values.apiRs.port | quote }} + # Console OAuth sync workers submit normalized provider data to the + # in-cluster API; credentials themselves never leave this pod. - name: CENTAUR_CONSOLE_CENTAUR_API_URL value: {{ printf "http://%s:%v" $apiRsName .Values.apiRs.port | quote }} - name: CENTAUR_CONSOLE_CENTAUR_API_KEY diff --git a/contrib/chart/templates/console.yaml b/contrib/chart/templates/console.yaml index 3fc8a6841..1c748ed54 100644 --- a/contrib/chart/templates/console.yaml +++ b/contrib/chart/templates/console.yaml @@ -177,6 +177,12 @@ spec: secretKeyRef: name: {{ $secretEnv }} key: {{ printf "%sCENTAUR_JWT_SIGNING_SECRET" $prefix }} + # In-cluster console URL exactly as sandboxes dial it (the same + # value api-rs receives as IRON_CONTROL_URL). Console derives the + # sandbox entitlements injection-rule host from it, so the rule + # matches regardless of what Host the sync request carried. + - name: CENTAUR_CONSOLE_URL + value: {{ include "centaur.consoleUrl" . | quote }} {{- if $console.publicUrl }} - name: CENTAUR_CONSOLE_PUBLIC_URL value: {{ $console.publicUrl | quote }} @@ -184,6 +190,14 @@ spec: {{- if $mcpPublicUrl }} - name: CENTAUR_MCP_PUBLIC_URL value: {{ $mcpPublicUrl | quote }} +{{- end }} + - name: CENTAUR_CONSOLE_PASSWORD_LOGIN_ENABLED + value: {{ $console.passwordLoginEnabled | quote }} + - name: CENTAUR_CONSOLE_PUBLIC_SLACK_THREADS_ENABLED + value: {{ $console.publicSlackThreadsEnabled | quote }} +{{- with $console.ssoEmailDomains }} + - name: CENTAUR_CONSOLE_SSO_EMAIL_DOMAINS + value: {{ join "," . | quote }} {{- end }} {{- if $console.googleOauth.enabled }} # Google OAuth app credentials (sign-in + brokered token refresh). diff --git a/contrib/chart/templates/slackbotv2.yaml b/contrib/chart/templates/slackbotv2.yaml index 3f73158a4..47bb41476 100644 --- a/contrib/chart/templates/slackbotv2.yaml +++ b/contrib/chart/templates/slackbotv2.yaml @@ -89,12 +89,21 @@ spec: - name: SLACKBOT_TRIGGER_BOT_ALLOWLIST value: {{ .Values.slackbotv2.triggerBotAllowlist | quote }} {{- end }} -{{- range $name := tuple "CLAUDE_MODEL" "CODEX_MODEL" }} +{{- if .Values.slackbotv2.channelDefaults }} + # Per-channel default harness / model / provider / reasoning, keyed + # by Slack conversation id (each value an object of harness/model/ + # provider/reasoning fields, e.g. { harness: claude, model: opus }). + # Applied below an explicit/sticky per-thread flag and above the + # deployment/baked default, and forwarded onto the harness input line + # so it takes effect. + - name: SLACKBOTV2_CHANNEL_DEFAULTS + value: {{ .Values.slackbotv2.channelDefaults | toJson | quote }} +{{- end }} +{{- range $name := tuple "CLAUDE_MODEL" "CODEX_MODEL" "CODEX_MODEL_REASONING_EFFORT" }} {{- if and (hasKey $.Values.sandbox.extraEnv $name) (not (hasKey $.Values.slackbotv2.extraEnv $name)) }} - # Mirror the deployer's harness default-model override - # (sandbox.extraEnv) so the Slack Console-link line names the model - # sandboxes actually run. slackbotv2.extraEnv wins if it sets the - # same variable. + # Mirror deployer harness display settings (sandbox.extraEnv) so + # the Slack Console-link line names the model/effort sandboxes + # actually run. slackbotv2.extraEnv wins when explicitly set. - name: {{ $name }} value: {{ index $.Values.sandbox.extraEnv $name | toString | quote }} {{- end }} diff --git a/contrib/chart/values.schema.json b/contrib/chart/values.schema.json index b01846220..6daa70db1 100644 --- a/contrib/chart/values.schema.json +++ b/contrib/chart/values.schema.json @@ -387,7 +387,8 @@ "properties": { "enabled": { "type": "boolean" }, "intervalSeconds": { "type": "integer" }, - "maxWindowSeconds": { "type": "integer" } + "maxWindowSeconds": { "type": "integer" }, + "batchSize": { "type": "integer", "minimum": 1 } } } } diff --git a/contrib/chart/values.yaml b/contrib/chart/values.yaml index bf13ee363..40dfaa8ea 100644 --- a/contrib/chart/values.yaml +++ b/contrib/chart/values.yaml @@ -106,6 +106,18 @@ console: # When set, the slackbotv2 deployment also links the first assistant message # in a Slack thread to the Console session view; leave empty to omit the link. publicUrl: "" + # Break-glass email/password login. Disable when console is reachable from + # the public internet and SSO is configured. + passwordLoginEnabled: true + # When enabled, every authenticated Console user can browse conversations + # originating in public Slack channels. Private channels and DMs remain + # owner-only. Requires apiRs.etl.slack.enabled so channel privacy is synced; + # visibility fails closed without that catalog. Individual chats can still + # be shared by their owner from the Threads UI. + publicSlackThreadsEnabled: false + # Optional SSO admission policy. When non-empty, only identities whose email + # domain matches this list can sign in through Google or Slack. + ssoEmailDomains: [] image: repository: centaur-console tag: latest @@ -448,6 +460,7 @@ apiRs: enabled: true intervalSeconds: 14400 maxWindowSeconds: 21600 + batchSize: 50 # Reaper: stop sandboxes older than the max lifetime, regardless of whether # they are running or suspended. 0 disables the sweep. Interval must be >= 1. sandboxMaxLifetimeSecs: 259200 # 3 days @@ -511,6 +524,20 @@ slackbotv2: mcpPublicUrl: "http://localhost:3000" externalOrgAllowlist: "" triggerBotAllowlist: "" + # Per-channel default harness / model / provider / reasoning, keyed by Slack + # conversation id (the C…/G…/D… id). Each value is an object of optional + # harness / model / provider / reasoning fields, using the same vocabulary as + # the inline Slack flags (harness: claude|codex|amp, provider: bedrock|meta, + # model accepts Claude aliases like opus). A model is only meaningful within a + # harness, so name the harness alongside it. Applied when a message in that + # channel carries no explicit/sticky per-thread flag, below such a flag and + # above the deployment/baked harness default, and forwarded onto the harness + # input line so it takes effect. reasoning only affects the codex harness. + # Example: + # channelDefaults: + # C0ENG: { harness: claude, model: opus, reasoning: high } + # C0TRIAGE: { reasoning: low } + channelDefaults: {} metrics: # slackbotv2 serves Prometheus text metrics at /metrics. This flag only # controls scrape annotations for Prometheus/VictoriaMetrics-style discovery. diff --git a/docs/pages/extend/workflows-v2.mdx b/docs/pages/extend/workflows-v2.mdx index 16ae4c515..e8ae455db 100644 --- a/docs/pages/extend/workflows-v2.mdx +++ b/docs/pages/extend/workflows-v2.mdx @@ -110,6 +110,8 @@ result = await ctx.agent_turn( "Investigate this alert and return the next action.", thread_key=f"workflow:{ctx.run_id}:agent", harness="codex", + model="gpt-5.2", + reasoning="high", metadata={"workflow": WORKFLOW_NAME}, ) ``` @@ -118,6 +120,41 @@ The workflow host sandbox is separate from the agent sandbox. The workflow handler coordinates the run; the agent turn runs through the normal Centaur session runtime. +Production deployments should keep `WORKFLOW_HOST_SANDBOX=true` (the default). +`false` runs Python as a child process of api-rs for local development and is +not an isolation boundary. Centaur removes its known control, bot, feedback, +GitHub, Slack, and model-provider credentials from that child, but arbitrary +ambient process configuration can still be visible; use only trusted workflow +code in local mode. + +#### Pick the model and reasoning effort + +`ctx.agent_turn(...)` accepts optional `model`, `provider`, and `reasoning` +kwargs. They ride the turn exactly like the Slack `--model` / `--bedrock` / +`-rsn` flags: `model` selects the model within the harness, `reasoning` sets the +codex reasoning effort (`none`/`minimal`/`low`/`medium`/`high`/`xhigh`/`max`), +and `provider` selects the codex model provider. `provider` and `reasoning` only +affect the codex harness; claude and amp ignore them. `reasoning` also accepts +the `reasoning_effort` and `effort` aliases. When a kwarg is omitted the +deployment/baked harness default stands — dispatched turns are no longer pinned +to the deployment default. + +To set a default for **every** turn in a workflow, declare a module-level +`AGENT_DEFAULTS` dict. Explicit per-call kwargs override it key by key: + +```python +WORKFLOW_NAME = "nightly_report" +AGENT_DEFAULTS = {"harness": "codex", "model": "gpt-5.2", "reasoning": "high"} + +async def handler(inp: Input, ctx: WorkflowContext) -> dict[str, Any]: + await ctx.agent_turn("Draft the report.") # gpt-5.2 @ high + await ctx.agent_turn("Tidy formatting.", reasoning="low") # gpt-5.2 @ low +``` + +Keep `harness` and `model` together — a model is only meaningful within its +harness, and because kwargs override `AGENT_DEFAULTS` key by key, overriding one +without the other can strand a model on the wrong harness. + ### Declare webhook metadata in the workflow Expose a workflow through `WEBHOOKS`: @@ -151,12 +188,16 @@ Schedules can live beside the handler: ```python SCHEDULE = { "type": "cron", - "cron": "0 9 * * 1-5", + "cron": "0 9 * * MON-FRI", "timezone": "America/New_York", "input": {"profile": "default"}, } ``` +Write day-of-week as names (`MON-FRI`), not numbers: the parser is Quartz-style +(1 = Sunday), so a Unix-style `1-5` fires Sunday–Thursday. See +[Schedule a workflow](/extend/workflows#schedule-a-workflow) for details. + `api-rs` reconciles enabled schedule metadata into Absurd schedule tasks. ETL workflows can be routed to a separate queue so long-running sync jobs do not block normal workflow runs. diff --git a/docs/pages/extend/workflows.mdx b/docs/pages/extend/workflows.mdx index d5d6d46fd..5c4f630cd 100644 --- a/docs/pages/extend/workflows.mdx +++ b/docs/pages/extend/workflows.mdx @@ -70,6 +70,7 @@ async def handler(inp: Input, ctx: WorkflowContext) -> dict[str, Any]: | `ctx.sleep(name, duration)` | Suspend and resume later. | | `ctx.sleep_until(name, when)` | Resume at a specific time. | | `ctx.wait_for_event(name, event_type, correlation_id)` | Wait for an external event. | +| `ctx.start_workflow(workflow_name, input, idempotency_key=...)` | Queue a child workflow and continue immediately; returns its durable task/run identifiers. | | `ctx.wait_for_workflow(...)` | Wait for a child workflow to finish. | | `ctx.run_workflow(...)` | Start and wait in one call. | | `ctx.start_agent(...)` | Start an agent turn. | @@ -129,7 +130,7 @@ WORKFLOW_NAME = "daily_market_digest" SCHEDULE = { "type": "cron", - "cron": "0 9 * * 1-5", + "cron": "0 9 * * MON-FRI", "timezone": "America/New_York", "input": { "channel": "markets", @@ -144,11 +145,20 @@ Cron schedules use five fields: minute hour day-of-month month day-of-week ``` +:::warning[Day-of-week numbering is Quartz-style, not Unix crontab] +The schedule engine parses cron expressions with the Rust +[`cron` crate](https://github.com/zslayton/cron), which numbers days of week +1–7 with **1 = Sunday** (`0` is rejected). A Unix-style `1-5` therefore fires +Sunday–Thursday, not Monday–Friday. Always write day-of-week as names +(`MON`, `MON-FRI`, `SAT,SUN`) — they mean the same thing in every dialect. +::: + Examples: | Cron | Meaning | |------|---------| -| `0 9 * * 1-5` | 9:00 AM every weekday. | +| `0 9 * * MON-FRI` | 9:00 AM every weekday. | +| `0 9 * * 1-5` | 9:00 AM Sunday–Thursday (Quartz numbering — probably not what you meant). | | `*/15 * * * *` | Every 15 minutes. | | `30 6 * * *` | 6:30 AM every day. | | `0 0 1 * *` | Midnight on the first day of every month. | diff --git a/docs/pages/operate/slack-etl.mdx b/docs/pages/operate/slack-etl.mdx index 111dc557a..31de8dc3a 100644 --- a/docs/pages/operate/slack-etl.mdx +++ b/docs/pages/operate/slack-etl.mdx @@ -59,6 +59,27 @@ visible to that token. It does not sync DMs or Slackbot-only live thread events. Private channel rows are protected by RLS: `centaur_readonly` sees public channel data and the channel in `centaur.slack_channel_id`. +### User-scoped private-channel ingestion + +The console's Slack OAuth flow can ingest private channels through the same +user-scoped pipeline used for DMs. Add `groups:read` and `groups:history` to the +Slack OAuth app's allowed user scopes and have existing users consent again. +DM-only credentials continue syncing DMs while credentials with the new scopes +also request `private_channel` conversations. + +Every 10 minutes the console worker fans out across healthy Slack broker +credentials. Each credential lists the private channels visible to that user, +reads channel history, and fetches the complete `conversations.members` list. +Messages are deduplicated by Slack conversation and message timestamp when +multiple credentials can see the same channel. + +User-scoped private channels are stored with DMs and MPIMs in the neutral +`slack_private_sync_*` and `slack_private_*_context_documents` tables rather than +`company_context_documents`. RLS checks `(team_id, channel_id, user_id)` against +the reconciled membership list. A successful sync marks members omitted from +Slack's complete member list inactive; a partial or truncated member list is +never applied. + ## Enable the schedules Set `apiRs.etl.slack.enabled=true` in Helm values. The chart renders the @@ -90,10 +111,11 @@ apiRs: | `SLACK_RETENTION_ENABLED` | `true` | Allows the `slack_retention` schedule to run when at least one Slack retention TTL is positive. | | `SLACK_RETENTION_INTERVAL_MINUTES` | `60` | How often to prune Slack retention-managed rows. | | `SLACK_ETL_RETENTION_DAYS` | `0` | Deletes Slack ETL messages, derived Slack documents, and terminal ETL run/job rows older than this many days. `0` disables ETL retention. | -| `SLACK_DM_RETENTION_DAYS` | `0` | Deletes Slack DM messages, stale empty DM conversations, and terminal DM run/job rows older than this many days. `0` disables DM retention. | +| `SLACK_DM_RETENTION_DAYS` | `0` | Deletes user-scoped private Slack messages, stale empty conversations, and terminal run/job rows older than this many days. `0` disables retention. | | `COMPANY_CONTEXT_DOCUMENTS_ENABLED` | `true` | Enables projection from Slack sync rows into company context documents. | -| `COMPANY_CONTEXT_DOCUMENTS_INTERVAL_SECONDS` | `14400` | How often to project changed Slack rows into documents. | -| `COMPANY_CONTEXT_DOCUMENTS_MAX_WINDOW_SECONDS` | `21600` | Maximum source `updated_at` window projected by one company context documents run. | +| `COMPANY_CONTEXT_DOCUMENTS_INTERVAL_SECONDS` | `14400` | How often the coordinator claims stale projection scopes. | +| `COMPANY_CONTEXT_DOCUMENTS_MAX_WINDOW_SECONDS` | `21600` | Maximum source `updated_at` window claimed for one scope before it advances its watermark. | +| `COMPANY_CONTEXT_DOCUMENTS_BATCH_SIZE` | `50` | Maximum changed source rows processed by one per-scope child workflow. | Example exclusion list: @@ -278,7 +300,7 @@ setting alerts. | Channels are all skipped | Check `SLACK_ETL_EXCLUDED_CHANNEL_PATTERNS` for broad globs. | | Checkpoints show `missing_scope` or `not_allowed_token_type` | Add the missing Slack OAuth scope or use the expected user-token class. | | Backfill jobs keep failing | Inspect `slack_sync_backfill_jobs.last_error` and the corresponding `slack_sync_runs` row. | -| Documents lag behind messages | Check the `company_context_documents` workflow status and `company_context_projection_lag_seconds`. | +| Documents lag behind messages | Check `company_context_projection_checkpoints` for an expired lease or old watermark, then inspect the per-scope `company_context_documents` child workflow and `company_context_projection_lag_seconds`. | Keep the ETL token scoped to the channels and workspace data you actually want agents to retrieve. Synced rows and projected documents are deployment-wide diff --git a/docs/pages/reference/configuration.mdx b/docs/pages/reference/configuration.mdx index 5191c6a55..41f4b979c 100644 --- a/docs/pages/reference/configuration.mdx +++ b/docs/pages/reference/configuration.mdx @@ -137,6 +137,7 @@ Execution tuning: | `SLACK_FEEDBACK_COMMANDS`, `SLACK_FEEDBACK_ALLOWED_CHANNELS` | `slackbot.extraEnv`. | Feedback slash commands and optional channel allowlist. | | `SLACK_FEEDBACK_LINEAR_TEAM_ID`, `SLACK_FEEDBACK_LINEAR_PROJECT_ID` | `slackbot.extraEnv`. | Linear destination for feedback issues. | | `SLACKBOT_EXTERNAL_ORG_ALLOWLIST` | `slackbot.extraEnv`. | Slack team ids allowed for external org handoff. | +| `SLACKBOTV2_CHANNEL_DEFAULTS` | `slackbotv2.channelDefaults`. | Per-channel default harness / model / provider / reasoning as a JSON object keyed by Slack conversation id, where each value is an object of optional `harness`/`model`/`provider`/`reasoning` fields (same vocabulary as the inline flags, so `harness: claude`, `provider: bedrock`, and Claude model aliases like `opus` all work), e.g. `{"C0ENG":{"harness":"claude","model":"opus","reasoning":"high"},"C0TRIAGE":{"reasoning":"low"}}`. A model is only meaningful within a harness, so name the harness alongside it. Applied when a message in that channel carries no explicit/sticky per-thread flag (below such a flag, above the deployment/baked default) and forwarded onto the harness input line so it takes effect; setting the harness restarts a thread onto it like a `--claude`/`--codex` flag. `reasoning` only affects the codex harness. Malformed JSON and unrecognized field values are logged and ignored. | | `SLACK_TEAM_ID` | `slackbot.extraEnv`. | Workspace team ID (e.g. `T01ABCD2EFG`) used to rewrite `https://*.slack.com/archives/...` URLs in final-delivery messages into native `slack://channel?team=...` deep links that open in the Slack app. Leave unset to keep archive URLs unchanged. | | `COMMIT_SHA` | Build/deploy env. | Commit shown in Slackbot metadata. | @@ -204,6 +205,8 @@ Sandbox entrypoint and wrappers: | --- | --- | --- | | `CENTAUR_HARNESS_CONFIG_DIR`, `CENTAUR_HARNESS_ADAPTER` | Sandbox image or `sandbox.extraEnv`. | Authoritative harness config directory and optional adapter executable. The entrypoint copies this directory's Codex and Claude files instead of merging them with the image-baked `~/harness` defaults, so an override must carry every required provider, feature, and trust setting. Leave it unset to use the reviewed config packaged in the sandbox image. | | `CENTAUR_SKILL_DIRS` | Chart-rendered from `overlays.sources[*].skillsSubdir` (default `.agents/skills`) through `SESSION_SANDBOX_EXTRA_ENV`. | Ordered skill directories copied into the agent workspace. | +| `CENTAUR_TOOLS_AUTO_RELOAD` | `repoCache.autoReload` via api-rs tools config; defaults to `true`. | Enables repo-cache-backed auto-refresh of local tool shims and copied skills in running sandboxes. Runtime catalog only; secret grants/proxy credentials reconcile separately. | +| `CENTAUR_TOOLS_RELOAD_INTERVAL_SECONDS` | `sandbox.extraEnv`. | Poll interval for the repo-cache checkout watchdog. | | `AGENT_REPO`, `AGENT_PERSONA` | Runtime assignment metadata. | Workspace repo clone and persona prompt. | | `GOOGLE_APPLICATION_CREDENTIALS` | Sandbox entrypoint or `sandbox.extraEnv`. | Google ADC path; entrypoint creates a local stub when unset. | | `CODEX_API_KEY`, `CODEX_HOME`, `CODEX_CONTINUE_THREAD_ID` | `sandbox.extraEnv` or runtime resume. | Codex auth/config/resume behavior. | @@ -211,6 +214,8 @@ Sandbox entrypoint and wrappers: | `META_AI_API_KEY` | Secret mounted into api-rs. | Meta AI direct credential for Codex provider `responses` and Slack or Linear `--meta` selection. | | `CODEX_MODEL_REASONING_SUMMARY` | `sandbox.extraEnv`. | Sets `model_reasoning_summary` in the Codex config (`auto`, `concise`, `detailed`, `none`). Codex >= 0.139 emits no reasoning summaries unless this is set, so renderers show no thinking trace. | | `CODEX_MODEL_REASONING_EFFORT` | `sandbox.extraEnv`. | Overrides the codex `model_reasoning_effort` (baked into `harness/codex/config.toml`) by patching the per-sandbox `~/.codex/config.toml` at boot, without forking the image. One of `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`; an unknown value is ignored (the config default stands). | +| `CODEX_BEDROCK_REGION` | `sandbox.extraEnv`. | Opt-in switch and single source of truth for the Bedrock region. When set, the control plane registers the AWS SigV4 re-signing credential (scoped to the `bedrock` service and this region, upstream `bedrock-mantle..api.aws`), injects the placeholder `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` env so codex can sign requests iron-proxy re-signs with the real IAM keys, and pins codex's `amazon-bedrock` provider to this region at sandbox boot (so the in-sandbox client and the proxy agree). Unset disables Bedrock; defaults to `us-east-1`. See [Codex with Amazon Bedrock](/deploying-in-production#codex-with-amazon-bedrock). | +| `CODEX_BEDROCK_SESSION_TOKEN` | `sandbox.extraEnv`. | Set truthy when the Bedrock IAM credentials are temporary (STS) and carry a session token, so the `AWS_SESSION_TOKEN` placeholder is declared and injected. Omit for long-term IAM user keys. | | `CLAUDE_MODEL`, `CLAUDE_CONTINUE_SESSION_ID` | `sandbox.extraEnv` or runtime resume. | Claude model and resume behavior. | | `CLAUDE_CODE_AUTH_MODE` | `sandbox.extraEnv`. | Claude Code auth flow: `api_key` (default, uses `ANTHROPIC_API_KEY`) or `access_token` (Claude.ai Pro or Max via the brokered OAuth login). See [Claude Auth Modes](/deploying-in-production#claude-auth-modes). | | `DEPLOY_ENV`, `ENVIRONMENT`, `TRACEPARENT` | Deployment env or wrapper-generated. | Runtime environment and trace context. | @@ -240,6 +245,7 @@ Slack ETL workflows: | `SLACK_RETENTION_ENABLED`, `SLACK_RETENTION_INTERVAL_MINUTES`, `SLACK_ETL_RETENTION_DAYS`, `SLACK_DM_RETENTION_DAYS` | `apiRs.etl.slack.retention.*`. | Slack retention enablement, cadence, and separate public ETL/DM TTLs. | | `COMPANY_CONTEXT_DOCUMENTS_ENABLED` | `apiRs.etl.companyContextDocuments.enabled`. | Enables company-context projection when any ETL is on. | | `COMPANY_CONTEXT_DOCUMENTS_MAX_WINDOW_SECONDS` | `apiRs.etl.companyContextDocuments.maxWindowSeconds`. | Maximum source `updated_at` window projected by one company-context documents run. | +| `COMPANY_CONTEXT_DOCUMENTS_BATCH_SIZE` | `apiRs.etl.companyContextDocuments.batchSize`. | Maximum changed source rows handled by one per-scope company-context child workflow. | Google Workspace ETL workflows: diff --git a/docs/pages/reference/tool-directory.mdx b/docs/pages/reference/tool-directory.mdx index 17322fa7f..82ab45b28 100644 --- a/docs/pages/reference/tool-directory.mdx +++ b/docs/pages/reference/tool-directory.mdx @@ -34,9 +34,10 @@ These are broadly useful across most deployments and are good candidates to conf | `slack` | Search Slack, read threads, inspect channels/users, and send or upload messages | `SLACK_BOT_TOKEN`; optional: `SLACK_SEARCH_TOKEN`, `SLACK_ETL_TOKEN` | | `gsuite` | Use Gmail, Calendar, Drive, Docs, Sheets, Slides, and Google Analytics | `GOOGLE_TOKEN_JSON` | | `websearch` | Free web search via Parallel and deep research | None; `PARALLEL_API_KEY` for `deep_research`; `ANTHROPIC_API_KEY` for search synthesis | -| `company_context` | Search indexed company history, Slack DMs, and Google Docs | None | +| `company_context` | Search indexed company history, private Slack conversations, and Google Docs | None | | `grafana` | Query dashboards, alerts, VictoriaMetrics, VictoriaLogs, and annotations | `GRAFANA_URL`, `GRAFANA_API_KEY` | | `posthog` | Query product analytics, events, pageviews, breakdowns, and user agents | `POSTHOG_API_KEY`, `POSTHOG_PROJECT_ID` | +| `amplitude` | Query product analytics — event segmentation, funnels, retention, user activity, and taxonomy | `AMPLITUDE_API_KEY`, `AMPLITUDE_SECRET_KEY` | | `attio` | Work with CRM objects, records, lists, notes, tasks, calls, and meetings | `ATTIO_API_KEY` | | `pylon` | Read and manage support issues, accounts, contacts, teams, tags, and users | `PYLON_API_KEY` | @@ -59,10 +60,12 @@ These are broadly useful across most deployments and are good candidates to conf | Tool | Use | API key / credential | |---|---|---| +| `centaur-console` | Inspect the current sandbox's redacted permissions and capabilities | None | | `chart` | Render charts as PNG images for Slack or reports | None | | `demo` | Test tool hot-reload and basic tool plumbing | None | | `grafana` | Grafana dashboards, alerts, VictoriaMetrics, VictoriaLogs, and annotations | `GRAFANA_URL`, `GRAFANA_API_KEY` | | `posthog` | Product analytics through HogQL, events, pageviews, and breakdowns | `POSTHOG_API_KEY`, `POSTHOG_PROJECT_ID` | +| `amplitude` | Amplitude event segmentation, funnels, retention, user activity, realtime, and taxonomy | `AMPLITUDE_API_KEY`, `AMPLITUDE_SECRET_KEY` | | `profslice` | Extract Firefox Profiler data for analysis | None | | `reth` | Reth execution timing and performance metrics | None | | `reth-log-analyzer` | Parse Reth logs and generate performance graphs | None | @@ -73,8 +76,8 @@ These are broadly useful across most deployments and are good candidates to conf | Tool | Use | API key / credential | |---|---|---| -| `airtable` | Bases, schemas, tables, records, views, and URL parsing | `AIRTABLE_API_KEY` | -| `company_context` | Search indexed company history, Slack DMs, and Google Docs | None | +| `airtable` | Bases, schemas, tables, views, record reads, record writes, and URL parsing | `AIRTABLE_API_KEY` | +| `company_context` | Search indexed company history, private Slack conversations, and Google Docs | None | | `composio` | Execute tools from third-party services exposed through Composio | `COMPOSIO_API_KEY` | | `figma` | Extract Figma files, nodes, components, styles, and variables | `FIGMA_ACCESS_TOKEN` | | `granola` | Search and read Granola notes and transcripts | `GRANOLA_API_KEY` | diff --git a/docs/public/md/extend/workflows-v2.md b/docs/public/md/extend/workflows-v2.md index 610dfa06e..e8ae455db 100644 --- a/docs/public/md/extend/workflows-v2.md +++ b/docs/public/md/extend/workflows-v2.md @@ -110,6 +110,8 @@ result = await ctx.agent_turn( "Investigate this alert and return the next action.", thread_key=f"workflow:{ctx.run_id}:agent", harness="codex", + model="gpt-5.2", + reasoning="high", metadata={"workflow": WORKFLOW_NAME}, ) ``` @@ -125,6 +127,34 @@ GitHub, Slack, and model-provider credentials from that child, but arbitrary ambient process configuration can still be visible; use only trusted workflow code in local mode. +#### Pick the model and reasoning effort + +`ctx.agent_turn(...)` accepts optional `model`, `provider`, and `reasoning` +kwargs. They ride the turn exactly like the Slack `--model` / `--bedrock` / +`-rsn` flags: `model` selects the model within the harness, `reasoning` sets the +codex reasoning effort (`none`/`minimal`/`low`/`medium`/`high`/`xhigh`/`max`), +and `provider` selects the codex model provider. `provider` and `reasoning` only +affect the codex harness; claude and amp ignore them. `reasoning` also accepts +the `reasoning_effort` and `effort` aliases. When a kwarg is omitted the +deployment/baked harness default stands — dispatched turns are no longer pinned +to the deployment default. + +To set a default for **every** turn in a workflow, declare a module-level +`AGENT_DEFAULTS` dict. Explicit per-call kwargs override it key by key: + +```python +WORKFLOW_NAME = "nightly_report" +AGENT_DEFAULTS = {"harness": "codex", "model": "gpt-5.2", "reasoning": "high"} + +async def handler(inp: Input, ctx: WorkflowContext) -> dict[str, Any]: + await ctx.agent_turn("Draft the report.") # gpt-5.2 @ high + await ctx.agent_turn("Tidy formatting.", reasoning="low") # gpt-5.2 @ low +``` + +Keep `harness` and `model` together — a model is only meaningful within its +harness, and because kwargs override `AGENT_DEFAULTS` key by key, overriding one +without the other can strand a model on the wrong harness. + ### Declare webhook metadata in the workflow Expose a workflow through `WEBHOOKS`: @@ -158,12 +188,16 @@ Schedules can live beside the handler: ```python SCHEDULE = { "type": "cron", - "cron": "0 9 * * 1-5", + "cron": "0 9 * * MON-FRI", "timezone": "America/New_York", "input": {"profile": "default"}, } ``` +Write day-of-week as names (`MON-FRI`), not numbers: the parser is Quartz-style +(1 = Sunday), so a Unix-style `1-5` fires Sunday–Thursday. See +[Schedule a workflow](/extend/workflows#schedule-a-workflow) for details. + `api-rs` reconciles enabled schedule metadata into Absurd schedule tasks. ETL workflows can be routed to a separate queue so long-running sync jobs do not block normal workflow runs. diff --git a/docs/public/md/extend/workflows.md b/docs/public/md/extend/workflows.md index 687cc66a4..5c4f630cd 100644 --- a/docs/public/md/extend/workflows.md +++ b/docs/public/md/extend/workflows.md @@ -70,6 +70,7 @@ async def handler(inp: Input, ctx: WorkflowContext) -> dict[str, Any]: | `ctx.sleep(name, duration)` | Suspend and resume later. | | `ctx.sleep_until(name, when)` | Resume at a specific time. | | `ctx.wait_for_event(name, event_type, correlation_id)` | Wait for an external event. | +| `ctx.start_workflow(workflow_name, input, idempotency_key=...)` | Queue a child workflow and continue immediately; returns its durable task/run identifiers. | | `ctx.wait_for_workflow(...)` | Wait for a child workflow to finish. | | `ctx.run_workflow(...)` | Start and wait in one call. | | `ctx.start_agent(...)` | Start an agent turn. | @@ -94,10 +95,9 @@ These primitives compose into larger automations: ## Run a workflow The manual control API requires the trusted `CENTAUR_CONTROL_API_KEY` (or an -optional dedicated `WORKFLOW_API_KEY`). Agent tools use -a separate Console JWT lane: their workflow name must be listed in -`WORKFLOW_API_ALLOWED_NAMES`, and `input.thread_key` must belong to one of the -JWT's Slack upload channels. +optional dedicated `WORKFLOW_API_KEY`). Agent tools use a separate Console JWT +lane: their workflow name must be listed in `WORKFLOW_API_ALLOWED_NAMES`, and +`input.thread_key` must belong to one of the JWT's Slack upload channels. Create a run through the trusted operator lane: @@ -130,7 +130,7 @@ WORKFLOW_NAME = "daily_market_digest" SCHEDULE = { "type": "cron", - "cron": "0 9 * * 1-5", + "cron": "0 9 * * MON-FRI", "timezone": "America/New_York", "input": { "channel": "markets", @@ -145,11 +145,20 @@ Cron schedules use five fields: minute hour day-of-month month day-of-week ``` +:::warning[Day-of-week numbering is Quartz-style, not Unix crontab] +The schedule engine parses cron expressions with the Rust +[`cron` crate](https://github.com/zslayton/cron), which numbers days of week +1–7 with **1 = Sunday** (`0` is rejected). A Unix-style `1-5` therefore fires +Sunday–Thursday, not Monday–Friday. Always write day-of-week as names +(`MON`, `MON-FRI`, `SAT,SUN`) — they mean the same thing in every dialect. +::: + Examples: | Cron | Meaning | |------|---------| -| `0 9 * * 1-5` | 9:00 AM every weekday. | +| `0 9 * * MON-FRI` | 9:00 AM every weekday. | +| `0 9 * * 1-5` | 9:00 AM Sunday–Thursday (Quartz numbering — probably not what you meant). | | `*/15 * * * *` | Every 15 minutes. | | `30 6 * * *` | 6:30 AM every day. | | `0 0 1 * *` | Midnight on the first day of every month. | diff --git a/docs/public/md/operate/slack-etl.md b/docs/public/md/operate/slack-etl.md index 111dc557a..31de8dc3a 100644 --- a/docs/public/md/operate/slack-etl.md +++ b/docs/public/md/operate/slack-etl.md @@ -59,6 +59,27 @@ visible to that token. It does not sync DMs or Slackbot-only live thread events. Private channel rows are protected by RLS: `centaur_readonly` sees public channel data and the channel in `centaur.slack_channel_id`. +### User-scoped private-channel ingestion + +The console's Slack OAuth flow can ingest private channels through the same +user-scoped pipeline used for DMs. Add `groups:read` and `groups:history` to the +Slack OAuth app's allowed user scopes and have existing users consent again. +DM-only credentials continue syncing DMs while credentials with the new scopes +also request `private_channel` conversations. + +Every 10 minutes the console worker fans out across healthy Slack broker +credentials. Each credential lists the private channels visible to that user, +reads channel history, and fetches the complete `conversations.members` list. +Messages are deduplicated by Slack conversation and message timestamp when +multiple credentials can see the same channel. + +User-scoped private channels are stored with DMs and MPIMs in the neutral +`slack_private_sync_*` and `slack_private_*_context_documents` tables rather than +`company_context_documents`. RLS checks `(team_id, channel_id, user_id)` against +the reconciled membership list. A successful sync marks members omitted from +Slack's complete member list inactive; a partial or truncated member list is +never applied. + ## Enable the schedules Set `apiRs.etl.slack.enabled=true` in Helm values. The chart renders the @@ -90,10 +111,11 @@ apiRs: | `SLACK_RETENTION_ENABLED` | `true` | Allows the `slack_retention` schedule to run when at least one Slack retention TTL is positive. | | `SLACK_RETENTION_INTERVAL_MINUTES` | `60` | How often to prune Slack retention-managed rows. | | `SLACK_ETL_RETENTION_DAYS` | `0` | Deletes Slack ETL messages, derived Slack documents, and terminal ETL run/job rows older than this many days. `0` disables ETL retention. | -| `SLACK_DM_RETENTION_DAYS` | `0` | Deletes Slack DM messages, stale empty DM conversations, and terminal DM run/job rows older than this many days. `0` disables DM retention. | +| `SLACK_DM_RETENTION_DAYS` | `0` | Deletes user-scoped private Slack messages, stale empty conversations, and terminal run/job rows older than this many days. `0` disables retention. | | `COMPANY_CONTEXT_DOCUMENTS_ENABLED` | `true` | Enables projection from Slack sync rows into company context documents. | -| `COMPANY_CONTEXT_DOCUMENTS_INTERVAL_SECONDS` | `14400` | How often to project changed Slack rows into documents. | -| `COMPANY_CONTEXT_DOCUMENTS_MAX_WINDOW_SECONDS` | `21600` | Maximum source `updated_at` window projected by one company context documents run. | +| `COMPANY_CONTEXT_DOCUMENTS_INTERVAL_SECONDS` | `14400` | How often the coordinator claims stale projection scopes. | +| `COMPANY_CONTEXT_DOCUMENTS_MAX_WINDOW_SECONDS` | `21600` | Maximum source `updated_at` window claimed for one scope before it advances its watermark. | +| `COMPANY_CONTEXT_DOCUMENTS_BATCH_SIZE` | `50` | Maximum changed source rows processed by one per-scope child workflow. | Example exclusion list: @@ -278,7 +300,7 @@ setting alerts. | Channels are all skipped | Check `SLACK_ETL_EXCLUDED_CHANNEL_PATTERNS` for broad globs. | | Checkpoints show `missing_scope` or `not_allowed_token_type` | Add the missing Slack OAuth scope or use the expected user-token class. | | Backfill jobs keep failing | Inspect `slack_sync_backfill_jobs.last_error` and the corresponding `slack_sync_runs` row. | -| Documents lag behind messages | Check the `company_context_documents` workflow status and `company_context_projection_lag_seconds`. | +| Documents lag behind messages | Check `company_context_projection_checkpoints` for an expired lease or old watermark, then inspect the per-scope `company_context_documents` child workflow and `company_context_projection_lag_seconds`. | Keep the ETL token scoped to the channels and workspace data you actually want agents to retrieve. Synced rows and projected documents are deployment-wide diff --git a/docs/public/md/reference/configuration.md b/docs/public/md/reference/configuration.md index 6b3349324..41f4b979c 100644 --- a/docs/public/md/reference/configuration.md +++ b/docs/public/md/reference/configuration.md @@ -137,6 +137,7 @@ Execution tuning: | `SLACK_FEEDBACK_COMMANDS`, `SLACK_FEEDBACK_ALLOWED_CHANNELS` | `slackbot.extraEnv`. | Feedback slash commands and optional channel allowlist. | | `SLACK_FEEDBACK_LINEAR_TEAM_ID`, `SLACK_FEEDBACK_LINEAR_PROJECT_ID` | `slackbot.extraEnv`. | Linear destination for feedback issues. | | `SLACKBOT_EXTERNAL_ORG_ALLOWLIST` | `slackbot.extraEnv`. | Slack team ids allowed for external org handoff. | +| `SLACKBOTV2_CHANNEL_DEFAULTS` | `slackbotv2.channelDefaults`. | Per-channel default harness / model / provider / reasoning as a JSON object keyed by Slack conversation id, where each value is an object of optional `harness`/`model`/`provider`/`reasoning` fields (same vocabulary as the inline flags, so `harness: claude`, `provider: bedrock`, and Claude model aliases like `opus` all work), e.g. `{"C0ENG":{"harness":"claude","model":"opus","reasoning":"high"},"C0TRIAGE":{"reasoning":"low"}}`. A model is only meaningful within a harness, so name the harness alongside it. Applied when a message in that channel carries no explicit/sticky per-thread flag (below such a flag, above the deployment/baked default) and forwarded onto the harness input line so it takes effect; setting the harness restarts a thread onto it like a `--claude`/`--codex` flag. `reasoning` only affects the codex harness. Malformed JSON and unrecognized field values are logged and ignored. | | `SLACK_TEAM_ID` | `slackbot.extraEnv`. | Workspace team ID (e.g. `T01ABCD2EFG`) used to rewrite `https://*.slack.com/archives/...` URLs in final-delivery messages into native `slack://channel?team=...` deep links that open in the Slack app. Leave unset to keep archive URLs unchanged. | | `COMMIT_SHA` | Build/deploy env. | Commit shown in Slackbot metadata. | @@ -183,6 +184,7 @@ Kubernetes backend: | `KUBERNETES_SANDBOX_RUNTIME_CLASS_NAME`, `KUBERNETES_SANDBOX_SERVICE_ACCOUNT_NAME` | `sandbox.runtimeClassName`, `api.extraEnv`. | Pod runtime class and service account. | | `KUBERNETES_SANDBOX_CPU_LIMIT`, `KUBERNETES_SANDBOX_MEMORY_LIMIT`, `KUBERNETES_SANDBOX_CPU_REQUEST`, `KUBERNETES_SANDBOX_MEMORY_REQUEST` | `sandbox.resources.*`. | Sandbox pod resources. | | `KUBERNETES_SANDBOX_READY_TIMEOUT_S`, `KUBERNETES_ATTACH_LOG_TAIL_LINES` | `api.extraEnv`. | Sandbox readiness and attach diagnostics. | +| `SESSION_SANDBOX_RUNNING_LIMIT`, `SESSION_SANDBOX_HOT_IDLE_GRACE_SECS` | `apiRs.sandboxRunningLimit`, `apiRs.sandboxHotIdleGraceSecs`. | Capacity admission for running-like sandboxes; discards ready warm sandboxes first, then pauses least-recently-active idle sessions outside the grace window. | | `SESSION_SANDBOX_CLEANUP_INTERVAL_SECS`, `SESSION_SANDBOX_IDLE_CLEANUP_BACKSTOP_SECS` | `apiRs.sandboxCleanupIntervalSecs`, `apiRs.sandboxIdleCleanupBackstopSecs`. | DB-aware cleanup of unreferenced sandboxes and restart recovery for idle pauses. Persisted `idle_timeout_ms` is honored after restart; the backstop is the fallback for older execution rows without that metadata. | | `KUBERNETES_SANDBOX_EXTRA_ENV` | `sandbox.extraEnv`. | JSON list copied into each sandbox. | | `KUBERNETES_WORKFLOW_DIRS` | Chart-rendered from `overlays.sources[*].workflowsSubdir` (default `workflows`) using the sandbox repo-cache mount prefix. | Workflow-host sandbox discovery paths. | @@ -243,6 +245,7 @@ Slack ETL workflows: | `SLACK_RETENTION_ENABLED`, `SLACK_RETENTION_INTERVAL_MINUTES`, `SLACK_ETL_RETENTION_DAYS`, `SLACK_DM_RETENTION_DAYS` | `apiRs.etl.slack.retention.*`. | Slack retention enablement, cadence, and separate public ETL/DM TTLs. | | `COMPANY_CONTEXT_DOCUMENTS_ENABLED` | `apiRs.etl.companyContextDocuments.enabled`. | Enables company-context projection when any ETL is on. | | `COMPANY_CONTEXT_DOCUMENTS_MAX_WINDOW_SECONDS` | `apiRs.etl.companyContextDocuments.maxWindowSeconds`. | Maximum source `updated_at` window projected by one company-context documents run. | +| `COMPANY_CONTEXT_DOCUMENTS_BATCH_SIZE` | `apiRs.etl.companyContextDocuments.batchSize`. | Maximum changed source rows handled by one per-scope company-context child workflow. | Google Workspace ETL workflows: diff --git a/docs/public/md/reference/tool-directory.md b/docs/public/md/reference/tool-directory.md index b5c044f2f..82ab45b28 100644 --- a/docs/public/md/reference/tool-directory.md +++ b/docs/public/md/reference/tool-directory.md @@ -12,13 +12,13 @@ Centaur ships with a set of tool integrations under `tools/`. Deployments can en The repo inventory is not the same as a live deployment. To see what an agent can use in a running sandbox, ask it to run: ```bash -call tools +centaur-tools list ``` -To inspect a specific tool's methods and parameters: +To inspect a specific tool's CLI: ```bash -call discover linear +linear --help ``` The `API key / credential` column uses the secret names declared by each tool's `[tool.centaur]` config. `None` means the base tool declares no required tool-specific credential; optional credentials are called out separately. @@ -34,7 +34,7 @@ These are broadly useful across most deployments and are good candidates to conf | `slack` | Search Slack, read threads, inspect channels/users, and send or upload messages | `SLACK_BOT_TOKEN`; optional: `SLACK_SEARCH_TOKEN`, `SLACK_ETL_TOKEN` | | `gsuite` | Use Gmail, Calendar, Drive, Docs, Sheets, Slides, and Google Analytics | `GOOGLE_TOKEN_JSON` | | `websearch` | Free web search via Parallel and deep research | None; `PARALLEL_API_KEY` for `deep_research`; `ANTHROPIC_API_KEY` for search synthesis | -| `company_context` | Search indexed company history, Slack DMs, and Google Docs | None | +| `company_context` | Search indexed company history, private Slack conversations, and Google Docs | None | | `grafana` | Query dashboards, alerts, VictoriaMetrics, VictoriaLogs, and annotations | `GRAFANA_URL`, `GRAFANA_API_KEY` | | `posthog` | Query product analytics, events, pageviews, breakdowns, and user agents | `POSTHOG_API_KEY`, `POSTHOG_PROJECT_ID` | | `amplitude` | Query product analytics — event segmentation, funnels, retention, user activity, and taxonomy | `AMPLITUDE_API_KEY`, `AMPLITUDE_SECRET_KEY` | @@ -60,6 +60,7 @@ These are broadly useful across most deployments and are good candidates to conf | Tool | Use | API key / credential | |---|---|---| +| `centaur-console` | Inspect the current sandbox's redacted permissions and capabilities | None | | `chart` | Render charts as PNG images for Slack or reports | None | | `demo` | Test tool hot-reload and basic tool plumbing | None | | `grafana` | Grafana dashboards, alerts, VictoriaMetrics, VictoriaLogs, and annotations | `GRAFANA_URL`, `GRAFANA_API_KEY` | @@ -75,8 +76,8 @@ These are broadly useful across most deployments and are good candidates to conf | Tool | Use | API key / credential | |---|---|---| -| `airtable` | Bases, schemas, tables, records, views, and URL parsing | `AIRTABLE_API_KEY` | -| `company_context` | Search indexed company history, Slack DMs, and Google Docs | None | +| `airtable` | Bases, schemas, tables, views, record reads, record writes, and URL parsing | `AIRTABLE_API_KEY` | +| `company_context` | Search indexed company history, private Slack conversations, and Google Docs | None | | `composio` | Execute tools from third-party services exposed through Composio | `COMPOSIO_API_KEY` | | `figma` | Extract Figma files, nodes, components, styles, and variables | `FIGMA_ACCESS_TOKEN` | | `granola` | Search and read Granola notes and transcripts | `GRANOLA_API_KEY` | diff --git a/patches/chat@4.31.0.patch b/patches/chat@4.31.0.patch index 1001c40ea..9994167e8 100644 --- a/patches/chat@4.31.0.patch +++ b/patches/chat@4.31.0.patch @@ -16,9 +16,9 @@ index 6af9d7fec34adeb8a7092a3ba0077d9f327a44d7..252b031dd39e65cb6c85d3b0e5c23eb1 + pattern.lastIndex = lastIndex; + return matches; + }); -+ const hasAction = message.isMention || isDMForDedupe || isSubscribedForDedupe || matchesPatternForDedupe; ++ const hasIntrinsicAction = message.isMention || isDMForDedupe || isSubscribedForDedupe; + const hasHistoryPersistence = adapter.persistThreadHistory || adapter.persistMessageHistory; -+ const dedupeBucket = hasAction ? "action" : hasHistoryPersistence ? "history" : null; ++ const dedupeBucket = hasIntrinsicAction ? "action" : matchesPatternForDedupe || hasHistoryPersistence ? "history" : null; + if (!dedupeBucket) { + this.logger.debug("Skipping non-actionable message before dedupe", { + adapter: adapter.name, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 93ccf5341..a24ee414c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,7 +18,7 @@ patchedDependencies: hash: 69262b03c278ca9d4af0bed7b4e05f9d7a5a36d3d79fad31df2a85da4d349274 path: patches/@chat-adapter__state-pg@4.31.0.patch chat@4.31.0: - hash: 378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd + hash: 4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09 path: patches/chat@4.31.0.patch importers: @@ -88,7 +88,7 @@ importers: version: 4.31.0(patch_hash=69262b03c278ca9d4af0bed7b4e05f9d7a5a36d3d79fad31df2a85da4d349274)(zod@4.4.3) chat: specifier: ^4.31.0 - version: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + version: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) discord.js: specifier: ^14.25.1 version: 14.26.4 @@ -140,7 +140,7 @@ importers: version: 21.1.1 chat: specifier: ^4.31.0 - version: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + version: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) hono: specifier: ^4.12.18 version: 4.12.25 @@ -186,7 +186,7 @@ importers: version: 76.0.0(graphql@17.0.0) chat: specifier: ^4.31.0 - version: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + version: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) hono: specifier: ^4.12.18 version: 4.12.25 @@ -229,7 +229,7 @@ importers: version: 4.31.0(patch_hash=69262b03c278ca9d4af0bed7b4e05f9d7a5a36d3d79fad31df2a85da4d349274)(zod@4.4.3) chat: specifier: ^4.31.0 - version: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + version: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) hono: specifier: ^4.12.18 version: 4.12.25 @@ -278,7 +278,7 @@ importers: version: 4.31.0(zod@4.4.3) chat: specifier: ^4.31.0 - version: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + version: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) dotenv: specifier: ^17.3.1 version: 17.4.2 @@ -1960,7 +1960,7 @@ snapshots: '@chat-adapter/discord@4.31.0(patch_hash=8f4fbb770159f924570fbad681297fcb9f8f7a6353a705cfcb78e39ef9e3a3ab)(zod@4.4.3)': dependencies: '@chat-adapter/shared': 4.31.0(zod@4.4.3) - chat: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + chat: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) discord-api-types: 0.37.120 discord-interactions: 4.4.0 discord.js: 14.26.4 @@ -1976,7 +1976,7 @@ snapshots: '@chat-adapter/shared': 4.31.0(zod@4.4.3) '@octokit/auth-app': 8.2.0 '@octokit/rest': 22.0.1 - chat: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + chat: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) transitivePeerDependencies: - ai - supports-color @@ -1986,7 +1986,7 @@ snapshots: dependencies: '@chat-adapter/shared': 4.31.0(zod@4.4.3) '@linear/sdk': 76.0.0(graphql@17.0.0) - chat: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + chat: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) transitivePeerDependencies: - ai - graphql @@ -1995,7 +1995,7 @@ snapshots: '@chat-adapter/shared@4.31.0(zod@4.4.3)': dependencies: - chat: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + chat: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) transitivePeerDependencies: - ai - supports-color @@ -2006,7 +2006,7 @@ snapshots: '@chat-adapter/shared': 4.31.0(zod@4.4.3) '@slack/socket-mode': 2.0.7 '@slack/web-api': 7.17.0 - chat: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + chat: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) transitivePeerDependencies: - ai - bufferutil @@ -2017,7 +2017,7 @@ snapshots: '@chat-adapter/state-memory@4.31.0(zod@4.4.3)': dependencies: - chat: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + chat: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) transitivePeerDependencies: - ai - supports-color @@ -2025,7 +2025,7 @@ snapshots: '@chat-adapter/state-pg@4.31.0(patch_hash=69262b03c278ca9d4af0bed7b4e05f9d7a5a36d3d79fad31df2a85da4d349274)(zod@4.4.3)': dependencies: - chat: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + chat: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) pg: 8.21.0 transitivePeerDependencies: - ai @@ -2040,7 +2040,7 @@ snapshots: '@microsoft/teams.apps': 2.0.13 '@microsoft/teams.cards': 2.0.13 '@microsoft/teams.graph-endpoints': 2.0.13 - chat: 4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3) + chat: 4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3) transitivePeerDependencies: - ai - debug @@ -2737,7 +2737,7 @@ snapshots: character-entities@2.0.2: {} - chat@4.31.0(patch_hash=378e9d2b3a9218ea973cb4dd64af92652affe86f61676540d504151cdce421fd)(zod@4.4.3): + chat@4.31.0(patch_hash=4161c47c40a34b31907f6640b501681a38a84559f039fca820fecfcca04f2f09)(zod@4.4.3): dependencies: '@workflow/serde': 4.1.0-beta.2 mdast-util-to-string: 4.0.0 diff --git a/services/api-rs/crates/centaur-api-server/src/routes.rs b/services/api-rs/crates/centaur-api-server/src/routes.rs index 47f5b98f2..c4965393d 100644 --- a/services/api-rs/crates/centaur-api-server/src/routes.rs +++ b/services/api-rs/crates/centaur-api-server/src/routes.rs @@ -289,7 +289,7 @@ pub fn build_router_with_app_state(state: AppState) -> Router { ) .route( "/api/admin/slack/dm-sync/checkpoints", - get(list_slack_dm_sync_checkpoints), + get(list_slack_private_sync_checkpoints), ) .route( "/api/admin/slack/dm-sync/batch", @@ -303,6 +303,14 @@ pub fn build_router_with_app_state(state: AppState) -> Router { "/api/admin/google/docs-sync/batch", post(ingest_google_docs_sync_batch).layer(DefaultBodyLimit::disable()), ) + .route( + "/api/admin/granola/sync/checkpoint", + get(get_granola_sync_checkpoint), + ) + .route( + "/api/admin/granola/sync/batch", + post(ingest_granola_sync_batch).layer(DefaultBodyLimit::disable()), + ) .route("/api/webhooks/{slug}", any(invoke_workflow_webhook)) .layer( TraceLayer::new_for_http() @@ -717,6 +725,359 @@ async fn stream_events( Ok(Sse::new(stream).keep_alive(KeepAlive::default())) } +#[derive(Debug, Deserialize)] +struct GranolaSyncCheckpointQuery { + scope_id: String, +} + +#[derive(Debug, Serialize, sqlx::FromRow)] +struct GranolaSyncCheckpointResponse { + scope_id: String, + #[serde(with = "time::serde::rfc3339::option")] + watermark_time: Option, + #[serde(with = "time::serde::rfc3339::option")] + last_success_at: Option, + last_error: String, +} + +#[derive(Debug, Deserialize)] +struct GranolaSyncBatchRequest { + run: GranolaSyncRunPayload, + #[serde(default)] + notes: Vec, + #[serde(default)] + checkpoint: Option, +} + +#[derive(Debug, Deserialize)] +struct GranolaSyncRunPayload { + run_id: String, + #[serde(default = "default_granola_sync_mode")] + mode: String, + status: String, + scope_id: String, + broker_credential_id: String, + source_user_email: String, + #[serde(default)] + notes_seen: i32, + #[serde(default)] + notes_upserted: i32, + #[serde(default)] + transcripts_seen: i32, + #[serde(default)] + transcripts_upserted: i32, + #[serde(default)] + error_text: String, + #[serde(default)] + metadata: Value, +} + +#[derive(Debug, Deserialize)] +struct GranolaSyncNotePayload { + note_id: String, + #[serde(default)] + title: String, + #[serde(default)] + owner: Value, + #[serde(default)] + attendees: Value, + #[serde(default)] + calendar_event: Value, + #[serde(default)] + summary_markdown: String, + #[serde(default)] + summary_text: String, + #[serde(default)] + transcript: Value, + #[serde(default)] + url: String, + #[serde(default)] + source_created_at: Option, + #[serde(default)] + source_updated_at: Option, + #[serde(default)] + raw_payload: Value, +} + +#[derive(Debug, Deserialize)] +struct GranolaSyncCheckpointPayload { + scope_id: String, + #[serde(default)] + watermark_time: Option, +} + +async fn get_granola_sync_checkpoint( + State(state): State, + _authorization: AdminServiceAuthorization, + Query(query): Query, +) -> Result, ApiError> { + require_non_empty("scope_id", &query.scope_id)?; + let checkpoint = sqlx::query_as::<_, GranolaSyncCheckpointResponse>( + "SELECT scope_id, watermark_time, last_success_at, last_error \ + FROM granola_sync_checkpoints WHERE scope_id = $1", + ) + .bind(&query.scope_id) + .fetch_optional(&db_pool(&state)?) + .await?; + + Ok(Json(json!({ "ok": true, "checkpoint": checkpoint }))) +} + +async fn ingest_granola_sync_batch( + State(state): State, + _authorization: AdminServiceAuthorization, + Json(request): Json, +) -> Result, ApiError> { + validate_granola_sync_batch(&request)?; + let mut tx = db_pool(&state)?.begin().await?; + let run = &request.run; + let scope = json!({ + "scope_id": run.scope_id, + "broker_credential_id": run.broker_credential_id, + "source_user_email": run.source_user_email, + }); + let completed = run.status == "completed"; + let scopes_synced = if completed { + json!([scope.clone()]) + } else { + json!([]) + }; + let scopes_failed = if completed { + json!([]) + } else { + json!([{ "scope_id": run.scope_id, "reason": run.error_text }]) + }; + let finished_at = Some(OffsetDateTime::now_utc()); + let metadata = json!({ + "broker_credential_id": run.broker_credential_id, + "source_user_email": run.source_user_email, + "console_metadata": run.metadata.clone(), + }); + + sqlx::query( + "INSERT INTO granola_sync_runs (\ + run_id, mode, status, scopes_requested, scopes_synced, scopes_failed, \ + notes_seen, notes_upserted, transcripts_seen, transcripts_upserted, \ + finished_at, error_text, metadata\ + ) VALUES (\ + $1, $2, $3, $4::jsonb, $5::jsonb, $6::jsonb, $7, $8, $9, $10, $11, $12, $13::jsonb\ + ) ON CONFLICT (run_id) DO UPDATE SET \ + mode = EXCLUDED.mode, status = EXCLUDED.status, \ + scopes_requested = EXCLUDED.scopes_requested, scopes_synced = EXCLUDED.scopes_synced, \ + scopes_failed = EXCLUDED.scopes_failed, notes_seen = EXCLUDED.notes_seen, \ + notes_upserted = EXCLUDED.notes_upserted, transcripts_seen = EXCLUDED.transcripts_seen, \ + transcripts_upserted = EXCLUDED.transcripts_upserted, \ + finished_at = COALESCE(EXCLUDED.finished_at, granola_sync_runs.finished_at), \ + error_text = EXCLUDED.error_text, metadata = EXCLUDED.metadata", + ) + .bind(&run.run_id) + .bind(&run.mode) + .bind(&run.status) + .bind(json!([scope.clone()])) + .bind(scopes_synced) + .bind(scopes_failed) + .bind(run.notes_seen) + .bind(run.notes_upserted) + .bind(run.transcripts_seen) + .bind(run.transcripts_upserted) + .bind(finished_at) + .bind(&run.error_text) + .bind(metadata) + .execute(&mut *tx) + .await?; + + for note in &request.notes { + let owner_id = json_text(¬e.owner, "id").or_else(|| json_text(¬e.owner, "user_id")); + let owner_email = json_text(¬e.owner, "email").unwrap_or_default(); + let owner_name = json_text(¬e.owner, "name") + .or_else(|| json_text(¬e.owner, "display_name")) + .unwrap_or_default(); + let attendees = ensure_json_array("note.attendees", ¬e.attendees)?; + let transcript = ensure_json_array("note.transcript", ¬e.transcript)?; + ensure_json_object("note.calendar_event", ¬e.calendar_event)?; + let transcript_text = transcript + .iter() + .filter_map(|entry| json_text(entry, "text")) + .filter(|text| !text.trim().is_empty()) + .collect::>() + .join("\n"); + let content_text = [ + note.title.trim(), + note.summary_markdown.trim(), + note.summary_text.trim(), + transcript_text.trim(), + ] + .into_iter() + .filter(|part| !part.is_empty()) + .collect::>() + .join("\n"); + let access_emails = granola_access_emails(&run.source_user_email, &owner_email, attendees); + let content_hash = hex::encode(Sha256::digest(content_text.as_bytes())); + + sqlx::query( + "INSERT INTO granola_sync_notes (\ + note_id, title, owner_id, owner_email, owner_name, attendees, access_emails, \ + calendar_event, summary_markdown, summary_text, transcript_text, transcript_payload, \ + url, content_text, content_hash, source_created_at, source_updated_at, raw_payload, \ + source_run_id, last_seen_at, last_error, updated_at\ + ) VALUES (\ + $1, $2, $3, $4, $5, $6::jsonb, $7::text[], $8::jsonb, $9, $10, $11, $12::jsonb, \ + $13, $14, $15, $16::timestamptz, $17::timestamptz, $18::jsonb, $19, NOW(), '', NOW()\ + ) ON CONFLICT (note_id) DO UPDATE SET \ + title = EXCLUDED.title, owner_id = EXCLUDED.owner_id, owner_email = EXCLUDED.owner_email, \ + owner_name = EXCLUDED.owner_name, attendees = EXCLUDED.attendees, \ + access_emails = (SELECT COALESCE(array_agg(DISTINCT email ORDER BY email), ARRAY[]::text[]) \ + FROM unnest(granola_sync_notes.access_emails || EXCLUDED.access_emails) AS emails(email) \ + WHERE email <> ''), \ + calendar_event = EXCLUDED.calendar_event, summary_markdown = EXCLUDED.summary_markdown, \ + summary_text = EXCLUDED.summary_text, transcript_text = EXCLUDED.transcript_text, \ + transcript_payload = EXCLUDED.transcript_payload, url = EXCLUDED.url, \ + content_text = EXCLUDED.content_text, content_hash = EXCLUDED.content_hash, \ + source_created_at = COALESCE(EXCLUDED.source_created_at, granola_sync_notes.source_created_at), \ + source_updated_at = COALESCE(EXCLUDED.source_updated_at, granola_sync_notes.source_updated_at), \ + raw_payload = EXCLUDED.raw_payload, source_run_id = EXCLUDED.source_run_id, \ + last_seen_at = NOW(), last_error = '', updated_at = NOW()", + ) + .bind(¬e.note_id) + .bind(¬e.title) + .bind(owner_id.unwrap_or_default()) + .bind(&owner_email) + .bind(&owner_name) + .bind(¬e.attendees) + .bind(access_emails) + .bind(¬e.calendar_event) + .bind(¬e.summary_markdown) + .bind(¬e.summary_text) + .bind(transcript_text) + .bind(¬e.transcript) + .bind(¬e.url) + .bind(content_text) + .bind(content_hash) + .bind(¬e.source_created_at) + .bind(¬e.source_updated_at) + .bind(¬e.raw_payload) + .bind(&run.run_id) + .execute(&mut *tx) + .await?; + } + + if let Some(checkpoint) = &request.checkpoint { + let successful_at = completed.then(OffsetDateTime::now_utc); + sqlx::query( + "INSERT INTO granola_sync_checkpoints (\ + scope_id, watermark_time, last_run_id, last_success_at, last_error, updated_at\ + ) VALUES ($1, $2::timestamptz, $3, $4, $5, NOW()) \ + ON CONFLICT (scope_id) DO UPDATE SET \ + watermark_time = COALESCE(EXCLUDED.watermark_time, granola_sync_checkpoints.watermark_time), \ + last_run_id = EXCLUDED.last_run_id, \ + last_success_at = COALESCE(EXCLUDED.last_success_at, granola_sync_checkpoints.last_success_at), \ + last_error = EXCLUDED.last_error, updated_at = NOW()", + ) + .bind(&checkpoint.scope_id) + .bind(&checkpoint.watermark_time) + .bind(&run.run_id) + .bind(successful_at) + .bind(&run.error_text) + .execute(&mut *tx) + .await?; + } + + tx.commit().await?; + Ok(Json(json!({ + "ok": true, + "run_id": run.run_id, + "notes_ingested": request.notes.len(), + }))) +} + +fn default_granola_sync_mode() -> String { + "incremental".to_owned() +} + +fn validate_granola_sync_batch(request: &GranolaSyncBatchRequest) -> Result<(), ApiError> { + let run = &request.run; + require_non_empty("run.run_id", &run.run_id)?; + require_non_empty("run.status", &run.status)?; + require_non_empty("run.scope_id", &run.scope_id)?; + require_non_empty("run.broker_credential_id", &run.broker_credential_id)?; + require_non_empty("run.source_user_email", &run.source_user_email)?; + if run.scope_id != format!("oauth:{}", run.broker_credential_id) { + return Err(ApiError::BadRequest( + "run.scope_id must be the OAuth credential scope".to_owned(), + )); + } + if !matches!(run.status.as_str(), "completed" | "failed") { + return Err(ApiError::BadRequest( + "run.status must be completed or failed".to_owned(), + )); + } + ensure_json_object("run.metadata", &run.metadata)?; + for note in &request.notes { + require_non_empty("note.note_id", ¬e.note_id)?; + ensure_json_object("note.owner", ¬e.owner)?; + ensure_json_array("note.attendees", ¬e.attendees)?; + ensure_json_object("note.calendar_event", ¬e.calendar_event)?; + ensure_json_array("note.transcript", ¬e.transcript)?; + ensure_json_object("note.raw_payload", ¬e.raw_payload)?; + } + if let Some(checkpoint) = &request.checkpoint { + require_non_empty("checkpoint.scope_id", &checkpoint.scope_id)?; + if checkpoint.scope_id != run.scope_id { + return Err(ApiError::BadRequest( + "checkpoint.scope_id must match run.scope_id".to_owned(), + )); + } + } + Ok(()) +} + +fn ensure_json_object<'a>(field: &str, value: &'a Value) -> Result<&'a Value, ApiError> { + if value.is_object() { + Ok(value) + } else { + Err(ApiError::BadRequest(format!( + "{field} must be a JSON object" + ))) + } +} + +fn ensure_json_array<'a>(field: &str, value: &'a Value) -> Result<&'a Vec, ApiError> { + value + .as_array() + .ok_or_else(|| ApiError::BadRequest(format!("{field} must be a JSON array"))) +} + +fn json_text(value: &Value, key: &str) -> Option { + value + .get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|text| !text.is_empty()) + .map(ToOwned::to_owned) +} + +fn granola_access_emails( + source_user_email: &str, + owner_email: &str, + attendees: &[Value], +) -> Vec { + let mut emails = BTreeSet::new(); + for candidate in std::iter::once(source_user_email) + .chain(std::iter::once(owner_email)) + .chain( + attendees + .iter() + .filter_map(|attendee| attendee.get("email").and_then(Value::as_str)), + ) + { + let normalized = candidate.trim().to_ascii_lowercase(); + if !normalized.is_empty() { + emails.insert(normalized); + } + } + emails.into_iter().collect() +} + #[derive(Debug, Deserialize)] struct PresignSlackArchiveImportRequest { filename: String, @@ -1516,7 +1877,7 @@ async fn retry_slack_archive_import( )) } -async fn list_slack_dm_sync_checkpoints( +async fn list_slack_private_sync_checkpoints( State(state): State, _authorization: AdminServiceAuthorization, Query(query): Query, @@ -1525,7 +1886,7 @@ async fn list_slack_dm_sync_checkpoints( require_non_empty("broker_credential_id", &query.broker_credential_id)?; let rows = sqlx::query_as::<_, SlackDmSyncCheckpointResponse>( "SELECT broker_credential_id, home_team_id, conversation_id, watermark_ts \ - FROM slack_dm_sync_checkpoints \ + FROM slack_private_sync_checkpoints \ WHERE broker_credential_id = $1 \ AND ($2::text IS NULL OR home_team_id = $2) \ ORDER BY home_team_id, conversation_id", @@ -1558,7 +1919,7 @@ async fn ingest_slack_dm_sync_batch( for conversation in &request.conversations { sqlx::query( - "INSERT INTO slack_dm_sync_conversations (\ + "INSERT INTO slack_private_sync_conversations (\ home_team_id, conversation_id, conversation_type, is_archived, is_ext_shared, \ raw_payload, last_seen_at, updated_at\ ) VALUES ($1, $2, $3, $4, $5, $6::jsonb, NOW(), NOW()) \ @@ -1590,7 +1951,7 @@ async fn ingest_slack_dm_sync_batch( } for ((home_team_id, conversation_id), _) in conversations { sqlx::query( - "UPDATE slack_dm_sync_conversation_members \ + "UPDATE slack_private_sync_conversation_members \ SET is_current_member = false, updated_at = NOW() \ WHERE home_team_id = $1 AND conversation_id = $2", ) @@ -1603,7 +1964,7 @@ async fn ingest_slack_dm_sync_batch( for member in &request.members { sqlx::query( - "INSERT INTO slack_dm_sync_conversation_members (\ + "INSERT INTO slack_private_sync_conversation_members (\ home_team_id, conversation_id, user_id, user_team_id, is_external, \ is_current_member, raw_payload, last_seen_at, updated_at\ ) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, NOW(), NOW()) \ @@ -1634,7 +1995,7 @@ async fn ingest_slack_dm_sync_batch( None }; sqlx::query( - "INSERT INTO slack_dm_sync_messages (\ + "INSERT INTO slack_private_sync_messages (\ home_team_id, conversation_id, message_ts, occurred_at, thread_ts, \ parent_message_ts, is_thread_root, user_id, user_team_id, bot_id, \ message_type, message_subtype, text, permalink, reply_count, reply_users, \ @@ -1659,9 +2020,9 @@ async fn ingest_slack_dm_sync_batch( reply_count = EXCLUDED.reply_count, \ reply_users = EXCLUDED.reply_users, \ latest_reply_ts = EXCLUDED.latest_reply_ts, \ - thread_refreshed_at = COALESCE(EXCLUDED.thread_refreshed_at, slack_dm_sync_messages.thread_refreshed_at), \ + thread_refreshed_at = COALESCE(EXCLUDED.thread_refreshed_at, slack_private_sync_messages.thread_refreshed_at), \ raw_payload = EXCLUDED.raw_payload, \ - source_run_id = COALESCE(EXCLUDED.source_run_id, slack_dm_sync_messages.source_run_id), \ + source_run_id = COALESCE(EXCLUDED.source_run_id, slack_private_sync_messages.source_run_id), \ last_seen_at = NOW(), \ updated_at = NOW()", ) @@ -1691,7 +2052,7 @@ async fn ingest_slack_dm_sync_batch( for attachment in &request.attachments { sqlx::query( - "INSERT INTO slack_dm_sync_message_attachments (\ + "INSERT INTO slack_private_sync_message_attachments (\ home_team_id, conversation_id, message_ts, slack_file_id, name, title, \ mimetype, filetype, size_bytes, url_private, permalink, download_status, \ download_error, content_sha256, raw_payload, source_run_id, last_seen_at, updated_at\ @@ -1710,7 +2071,7 @@ async fn ingest_slack_dm_sync_batch( download_error = EXCLUDED.download_error, \ content_sha256 = EXCLUDED.content_sha256, \ raw_payload = EXCLUDED.raw_payload, \ - source_run_id = COALESCE(EXCLUDED.source_run_id, slack_dm_sync_message_attachments.source_run_id), \ + source_run_id = COALESCE(EXCLUDED.source_run_id, slack_private_sync_message_attachments.source_run_id), \ last_seen_at = NOW(), \ updated_at = NOW()", ) @@ -1741,14 +2102,14 @@ async fn ingest_slack_dm_sync_batch( None }; sqlx::query( - "INSERT INTO slack_dm_sync_checkpoints (\ + "INSERT INTO slack_private_sync_checkpoints (\ broker_credential_id, home_team_id, conversation_id, watermark_ts, \ last_run_id, last_success_at, last_error, updated_at\ ) VALUES ($1, $2, $3, $4, $5, $6, $7, NOW()) \ ON CONFLICT (broker_credential_id, home_team_id, conversation_id) DO UPDATE SET \ watermark_ts = EXCLUDED.watermark_ts, \ last_run_id = EXCLUDED.last_run_id, \ - last_success_at = COALESCE(EXCLUDED.last_success_at, slack_dm_sync_checkpoints.last_success_at), \ + last_success_at = COALESCE(EXCLUDED.last_success_at, slack_private_sync_checkpoints.last_success_at), \ last_error = EXCLUDED.last_error, \ updated_at = NOW()", ) @@ -2094,16 +2455,25 @@ impl FromRequestParts for AdminServiceAuthorization { parts: &mut Parts, _state: &AppState, ) -> Result { - let token = bearer_token(&parts.headers)?; - if token_matches_configured_env(token, ADMIN_API_SERVICE_KEY_ENVS) { - return Ok(Self); - } - Err(ApiError::Unauthorized( - "invalid admin service token".to_owned(), - )) + authorize_admin_service_with(&parts.headers, |token| { + token_matches_configured_env(token, ADMIN_API_SERVICE_KEY_ENVS) + }) } } +fn authorize_admin_service_with( + headers: &HeaderMap, + token_is_authorized: impl FnOnce(&str) -> bool, +) -> Result { + let token = bearer_token(headers)?; + if token_is_authorized(token) { + return Ok(AdminServiceAuthorization); + } + Err(ApiError::Unauthorized( + "invalid admin service token".to_owned(), + )) +} + impl FromRequestParts for ArchiveDownloadAuthorization { type Rejection = ApiError; @@ -2768,7 +3138,7 @@ async fn upsert_slack_dm_sync_run( None }; sqlx::query( - "INSERT INTO slack_dm_sync_runs (\ + "INSERT INTO slack_private_sync_runs (\ run_id, workflow_run_id, mode, status, broker_credential_id, source_user_id, \ home_team_id, conversations_requested, conversations_synced, conversations_failed, \ messages_fetched, messages_upserted, replies_fetched, replies_upserted, \ @@ -2790,7 +3160,7 @@ async fn upsert_slack_dm_sync_run( messages_upserted = EXCLUDED.messages_upserted, \ replies_fetched = EXCLUDED.replies_fetched, \ replies_upserted = EXCLUDED.replies_upserted, \ - finished_at = COALESCE(EXCLUDED.finished_at, slack_dm_sync_runs.finished_at), \ + finished_at = COALESCE(EXCLUDED.finished_at, slack_private_sync_runs.finished_at), \ error_text = EXCLUDED.error_text, \ metadata = EXCLUDED.metadata", ) @@ -3013,9 +3383,12 @@ fn validate_slack_dm_sync_batch(request: &SlackDmSyncBatchRequest) -> Result<(), "conversation.conversation_id", &conversation.conversation_id, )?; - if !matches!(conversation.conversation_type.as_str(), "im" | "mpim") { + if !matches!( + conversation.conversation_type.as_str(), + "im" | "mpim" | "private_channel" + ) { return Err(ApiError::BadRequest( - "conversation.conversation_type must be im or mpim".to_owned(), + "conversation.conversation_type must be im, mpim, or private_channel".to_owned(), )); } validate_json_shape("conversation.raw_payload", &conversation.raw_payload, true)?; @@ -3052,6 +3425,42 @@ fn validate_slack_dm_sync_batch(request: &SlackDmSyncBatchRequest) -> Result<(), Ok(()) } +#[cfg(test)] +mod slack_user_sync_tests { + use super::*; + + fn request_with_conversation_type(conversation_type: &str) -> SlackDmSyncBatchRequest { + SlackDmSyncBatchRequest { + run: None, + replace_memberships: false, + conversations: vec![SlackDmSyncConversationPayload { + home_team_id: "T123".to_owned(), + conversation_id: "G123".to_owned(), + conversation_type: conversation_type.to_owned(), + is_archived: false, + is_ext_shared: false, + raw_payload: json!({"name": "leadership"}), + }], + members: vec![], + messages: vec![], + attachments: vec![], + checkpoints: vec![], + } + } + + #[test] + fn accepts_private_channel_conversations() { + validate_slack_dm_sync_batch(&request_with_conversation_type("private_channel")).unwrap(); + } + + #[test] + fn rejects_public_channel_conversations() { + let error = validate_slack_dm_sync_batch(&request_with_conversation_type("public_channel")) + .unwrap_err(); + assert!(matches!(error, ApiError::BadRequest(_))); + } +} + fn validate_google_docs_sync_batch(request: &GoogleDocsSyncBatchRequest) -> Result<(), ApiError> { if let Some(run) = &request.run { require_non_empty("run.run_id", &run.run_id)?; @@ -3701,6 +4110,119 @@ mod workflow_api_tests { } } +#[cfg(test)] +mod granola_sync_tests { + use super::*; + + fn bearer_headers(token: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert( + axum::http::header::AUTHORIZATION, + format!("Bearer {token}").parse().unwrap(), + ); + headers + } + + fn batch() -> GranolaSyncBatchRequest { + GranolaSyncBatchRequest { + run: GranolaSyncRunPayload { + run_id: "granola_test".to_owned(), + mode: "incremental".to_owned(), + status: "completed".to_owned(), + scope_id: "oauth:bcr_123".to_owned(), + broker_credential_id: "bcr_123".to_owned(), + source_user_email: "owner@example.com".to_owned(), + notes_seen: 1, + notes_upserted: 1, + transcripts_seen: 1, + transcripts_upserted: 1, + error_text: String::new(), + metadata: json!({"source": "console"}), + }, + notes: vec![GranolaSyncNotePayload { + note_id: "meeting-1".to_owned(), + title: "Planning".to_owned(), + owner: json!({"email": "ada@example.com"}), + attendees: json!([ + {"email": "bob@example.com"}, + {"email": "OWNER@example.com"} + ]), + calendar_event: json!({}), + summary_markdown: "Ship it".to_owned(), + summary_text: "Ship it".to_owned(), + transcript: json!([{ "text": "Ada: ship it" }]), + url: String::new(), + source_created_at: None, + source_updated_at: None, + raw_payload: json!({"source": "granola_mcp"}), + }], + checkpoint: Some(GranolaSyncCheckpointPayload { + scope_id: "oauth:bcr_123".to_owned(), + watermark_time: Some("2026-07-08T12:00:00Z".to_owned()), + }), + } + } + + #[test] + fn validates_a_credential_scoped_granola_batch() { + validate_granola_sync_batch(&batch()).unwrap(); + } + + #[test] + fn granola_admin_authorization_rejects_missing_and_invalid_credentials() { + assert!(matches!( + authorize_admin_service_with(&HeaderMap::new(), |_| true), + Err(ApiError::Unauthorized(_)) + )); + assert!(matches!( + authorize_admin_service_with(&bearer_headers("invalid"), |_| false), + Err(ApiError::Unauthorized(_)) + )); + } + + #[test] + fn granola_admin_authorization_accepts_the_control_service_token() { + authorize_admin_service_with(&bearer_headers("control-token"), |token| { + token == "control-token" + }) + .unwrap(); + } + + #[test] + fn rejects_a_checkpoint_for_another_credential() { + let mut request = batch(); + request.checkpoint.as_mut().unwrap().scope_id = "oauth:bcr_other".to_owned(); + let error = validate_granola_sync_batch(&request).unwrap_err(); + assert!(matches!(error, ApiError::BadRequest(_))); + } + + #[test] + fn rejects_a_scope_that_does_not_belong_to_the_credential() { + let mut request = batch(); + request.run.scope_id = "oauth:bcr_other".to_owned(); + request.checkpoint.as_mut().unwrap().scope_id = "oauth:bcr_other".to_owned(); + let error = validate_granola_sync_batch(&request).unwrap_err(); + assert!(matches!(error, ApiError::BadRequest(_))); + } + + #[test] + fn access_is_granted_to_source_owner_and_attendees() { + let emails = granola_access_emails( + "Owner@example.com", + "ada@example.com", + batch().notes[0].attendees.as_array().unwrap(), + ); + assert_eq!( + emails, + vec![ + "ada@example.com".to_owned(), + "bob@example.com".to_owned(), + "owner@example.com".to_owned(), + ] + ); + } +} + #[cfg(test)] mod slack_archive_import_tests { use super::*; diff --git a/services/api-rs/crates/centaur-sandbox-agent-k8s/src/iron_proxy.rs b/services/api-rs/crates/centaur-sandbox-agent-k8s/src/iron_proxy.rs index eb7a895a0..6501f60f5 100644 --- a/services/api-rs/crates/centaur-sandbox-agent-k8s/src/iron_proxy.rs +++ b/services/api-rs/crates/centaur-sandbox-agent-k8s/src/iron_proxy.rs @@ -50,6 +50,7 @@ const PROXY_LOG_LEVEL: &str = "info"; // DSN. These are the deploy-level env vars iron-proxy reads for that listener. const PG_LISTENER_PORT: u16 = 5432; const CENTAUR_POSTGRES_DSN_ENV: &str = "CENTAUR_POSTGRES_DSN"; +const CENTAUR_CONSOLE_URL_ENV: &str = "CENTAUR_CONSOLE_URL"; const PG_LISTEN_ENV: &str = "IRON_PROXY_PG_LISTEN"; const PG_CLIENT_USER_ENV: &str = "IRON_PROXY_PG_CLIENT_USER"; const PG_CLIENT_PASSWORD_ENV: &str = "IRON_PROXY_PG_CLIENT_PASSWORD"; @@ -123,6 +124,7 @@ pub(crate) struct ResolvedIronProxy { proxy_host: String, proxy_pod_name: String, proxy_port: u16, + console_url: String, // iron-control principal OID this sandbox's proxy binds to. principal_id: String, // The single Postgres listener the proxy multiplexes all upstreams through, @@ -319,6 +321,12 @@ impl AgentSandboxBackend { proxy_host: iron_proxy_service_name(id), proxy_pod_name: new_iron_proxy_pod_name(id), proxy_port: PROXY_TUNNEL_PORT, + console_url: self + .config + .iron_control + .as_ref() + .map(|settings| settings.control_url.clone()) + .unwrap_or_default(), principal_id, pg, replace_placeholders, @@ -1115,6 +1123,9 @@ pub(crate) fn apply_proxy_env(spec: &mut SandboxSpec, resolved: &ResolvedIronPro ); set_missing_env(spec, CENTAUR_POSTGRES_DSN_ENV, &value); } + if !resolved.console_url.is_empty() { + set_missing_env(spec, CENTAUR_CONSOLE_URL_ENV, &resolved.console_url); + } } pub(crate) fn sandbox_ca_volume_mount_json() -> Value { @@ -1939,6 +1950,7 @@ mod tests { proxy_host: "asbx-test-iron-proxy".to_owned(), proxy_pod_name: "asbx-test-iron-proxy-1".to_owned(), proxy_port: 8080, + console_url: "http://console:3000".to_owned(), principal_id: "principal".to_owned(), pg: None, replace_placeholders: BTreeMap::new(), @@ -2647,4 +2659,20 @@ mod tests { ); } } + + #[test] + fn apply_proxy_env_adds_console_url() { + let mut spec = SandboxSpec::new("centaur-agent:latest"); + let mut resolved = resolved(); + resolved.console_url = "http://console:3000/".to_owned(); + + apply_proxy_env(&mut spec, &resolved); + + let value = spec + .env + .iter() + .find(|env| env.name == CENTAUR_CONSOLE_URL_ENV) + .map(|env| env.value.as_str()); + assert_eq!(value, Some("http://console:3000/")); + } } diff --git a/services/api-rs/crates/centaur-session-sqlx/MIGRATIONS.md b/services/api-rs/crates/centaur-session-sqlx/MIGRATIONS.md index 2e7237f12..bb1589f2c 100644 --- a/services/api-rs/crates/centaur-session-sqlx/MIGRATIONS.md +++ b/services/api-rs/crates/centaur-session-sqlx/MIGRATIONS.md @@ -18,6 +18,16 @@ contains the forward-only reconciliation for Fineas public Slack company context. Fork migration `0043` appends assignment-bound sandbox content revision tracking; it is deliberately backward compatible with older binaries that update `sandbox_id` without knowing the new nullable column. +Upstream versions `0043` through `0045` are shifted to TipLink versions `0044` +through `0046`: company-context projection checkpoints, Granola context +projection, and Slack private-channel OAuth synchronization respectively. + +Migration `0046` is not compatible with an ordinary overlapping rolling +deployment. It renames the live `slack_dm_*` relations and rebuilds both BM25 +indexes, so old API pods and already-running sandboxes must not continue using +the pre-migration relation names after it starts. Deploy this migration only +through a rehearsed zero-overlap cutover that accounts for existing sandboxes; +the prior image is not a functional rollback after the rename. The checksum manifests checked by `.github/scripts/check-migration-order.sh` lock the release migration tree. Append a manifest entry for a genuinely new diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/.checksums.sha384 b/services/api-rs/crates/centaur-session-sqlx/migrations/.checksums.sha384 index b5a0ab59c..4bda0651c 100644 --- a/services/api-rs/crates/centaur-session-sqlx/migrations/.checksums.sha384 +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/.checksums.sha384 @@ -41,3 +41,6 @@ b1a1b0fe52fdd876bd53a96b2fff43568cc4631bf59712f5231f37f25cf209d9f72db0b5a3b3566c 38f3d13f44fa29264529012118f6b3921de1cf7e8b75e0e3ffcd1b207124eb7e0f6bde7504cdf3ac51d29a99291a77b0 services/api-rs/crates/centaur-session-sqlx/migrations/0041_attio_sync_tables.sql dcd05c897c50a15b8bc2342e76d520133ac2370c23e88e119d6a7ff4a615a88b5512209a33ee04c5070f4f4afeda020e services/api-rs/crates/centaur-session-sqlx/migrations/0042_centaur_readonly_slack_dm_rls.sql fd353077080a2cbeaaa7242f5726415663baeb00b74dd1743db01cbeda51f0748714f0f59400907411bed45c52fcac2a services/api-rs/crates/centaur-session-sqlx/migrations/0043_session_sandbox_content_revision.sql +a16fe6d9871526af81c79c1cda40636e21068ed679dffcd6b473505aac0089fd8febf05b1b0607a062686434cf3f9a42 services/api-rs/crates/centaur-session-sqlx/migrations/0044_company_context_projection_checkpoints.sql +89615d78b09706339668e609f38ffeb205e8ab0a03f3ad3eeca8ba9f47938c5d110a6dfaadb36052f0d2a54bf08fb694 services/api-rs/crates/centaur-session-sqlx/migrations/0045_granola_context_projection.sql +8e1ee367cc62f1ffde0b4fad158291266ef78598ab613136822c9703d4fe049b883fd588328b894cb05480b0765b393e services/api-rs/crates/centaur-session-sqlx/migrations/0046_slack_private_channel_oauth_sync.sql diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0044_company_context_projection_checkpoints.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0044_company_context_projection_checkpoints.sql new file mode 100644 index 000000000..ebf79460f --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0044_company_context_projection_checkpoints.sql @@ -0,0 +1,48 @@ +create table if not exists company_context_projection_checkpoints ( + scope text primary key, + watermark timestamptz, + window_start timestamptz, + window_end timestamptz, + cursor_updated_at timestamptz, + cursor_key text not null default '', + lease_token text, + lease_expires_at timestamptz, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + check (window_end is null or window_start is null or window_end >= window_start) +); + +create index if not exists idx_company_context_projection_checkpoints_lease + on company_context_projection_checkpoints (lease_expires_at) + where lease_expires_at is not null; + +do $$ +declare + role_name text; +begin + foreach role_name in array array['centaur_slack_admin', 'centaur_readonly'] loop + if exists (select 1 from pg_roles where rolname = role_name) then + execute format( + 'grant select on company_context_projection_checkpoints to %I', + role_name + ); + end if; + end loop; +end $$; + +alter table company_context_projection_checkpoints enable row level security; + +drop policy if exists centaur_readonly_company_context_projection_checkpoints_select + on company_context_projection_checkpoints; +create policy centaur_readonly_company_context_projection_checkpoints_select + on company_context_projection_checkpoints for select to centaur_readonly using (true); + +do $$ +begin + if exists (select 1 from pg_roles where rolname = 'centaur_slack_admin') then + drop policy if exists centaur_slack_admin_company_context_projection_checkpoints_select + on company_context_projection_checkpoints; + create policy centaur_slack_admin_company_context_projection_checkpoints_select + on company_context_projection_checkpoints for select to centaur_slack_admin using (true); + end if; +end $$; diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0045_granola_context_projection.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0045_granola_context_projection.sql new file mode 100644 index 000000000..eee5f4dfb --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0045_granola_context_projection.sql @@ -0,0 +1,157 @@ +-- Granola's OAuth sync writes normalized source rows. Project those rows into +-- the dedicated search table in the same transaction, rather than routing +-- them through the generic company_context_documents projection. + +create or replace function centaur_refresh_granola_context_document( + p_note_id text +) +returns void +language sql +as $$ + with attendee_rows as ( + select + coalesce( + nullif(btrim(attendee ->> 'name'), ''), + nullif(btrim(attendee ->> 'display_name'), ''), + nullif(btrim(attendee ->> 'email'), ''), + nullif(btrim(attendee ->> 'id'), '') + ) as attendee_label + from granola_sync_notes notes + cross join lateral jsonb_array_elements(notes.attendees) attendee + where notes.note_id = p_note_id + ), + projected as ( + select + concat_ws(':', 'granola', notes.note_id) as document_id, + notes.note_id, + coalesce(nullif(notes.title, ''), 'Granola note') as title, + notes.content_text as body, + notes.url, + notes.owner_id, + notes.owner_email, + notes.owner_name, + notes.access_emails, + coalesce( + array_agg(distinct attendee_rows.attendee_label order by attendee_rows.attendee_label) + filter (where attendee_rows.attendee_label is not null), + array[]::text[] + ) as attendee_labels, + coalesce(notes.source_created_at, notes.source_updated_at) as occurred_at, + notes.source_updated_at, + jsonb_build_object( + 'source', 'granola', + 'source_type', 'granola_note', + 'note_id', notes.note_id, + 'owner_id', notes.owner_id, + 'owner_email', notes.owner_email, + 'owner_name', notes.owner_name, + 'attendee_count', jsonb_array_length(notes.attendees), + 'calendar_event', notes.calendar_event + ) as metadata + from granola_sync_notes notes + left join attendee_rows on true + where notes.note_id = p_note_id + group by + notes.note_id, + notes.title, + notes.content_text, + notes.url, + notes.owner_id, + notes.owner_email, + notes.owner_name, + notes.access_emails, + notes.source_created_at, + notes.source_updated_at, + notes.attendees, + notes.calendar_event + ), + hashed as ( + select + projected.*, + md5(concat_ws( + E'\\x1f', + title, + body, + url, + owner_id, + owner_email, + owner_name, + array_to_string(access_emails, E'\\x1e'), + array_to_string(attendee_labels, E'\\x1e'), + coalesce(occurred_at::text, ''), + coalesce(source_updated_at::text, ''), + metadata::text + )) as content_hash + from projected + ) + insert into granola_context_documents ( + document_id, + note_id, + title, + body, + url, + owner_id, + owner_email, + owner_name, + access_emails, + attendee_labels, + occurred_at, + source_updated_at, + content_hash, + metadata, + updated_at + ) + select + document_id, + note_id, + title, + body, + url, + owner_id, + owner_email, + owner_name, + access_emails, + attendee_labels, + occurred_at, + source_updated_at, + content_hash, + metadata, + now() + from hashed + on conflict (document_id) do update set + note_id = excluded.note_id, + title = excluded.title, + body = excluded.body, + url = excluded.url, + owner_id = excluded.owner_id, + owner_email = excluded.owner_email, + owner_name = excluded.owner_name, + access_emails = excluded.access_emails, + attendee_labels = excluded.attendee_labels, + occurred_at = excluded.occurred_at, + source_updated_at = excluded.source_updated_at, + content_hash = excluded.content_hash, + metadata = excluded.metadata, + updated_at = now() + where granola_context_documents.content_hash is distinct from excluded.content_hash; +$$; + +create or replace function centaur_refresh_granola_context_document_from_note() +returns trigger +language plpgsql +as $$ +begin + perform centaur_refresh_granola_context_document(new.note_id); + return new; +end; +$$; + +drop trigger if exists trg_granola_sync_notes_refresh_context + on granola_sync_notes; +create trigger trg_granola_sync_notes_refresh_context + after insert or update on granola_sync_notes + for each row + execute function centaur_refresh_granola_context_document_from_note(); + +select centaur_refresh_granola_context_document(note_id) +from granola_sync_notes; diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0046_slack_private_channel_oauth_sync.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0046_slack_private_channel_oauth_sync.sql new file mode 100644 index 000000000..b26af21b6 --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0046_slack_private_channel_oauth_sync.sql @@ -0,0 +1,354 @@ +-- Generalize the existing user-scoped DM store before adding private channels. +-- Table renames preserve all DM/MPIM rows, foreign keys, indexes, grants, RLS, +-- and triggers in place. +alter table slack_dm_sync_conversations + rename to slack_private_sync_conversations; +alter table slack_dm_sync_conversation_members + rename to slack_private_sync_conversation_members; +alter table slack_dm_sync_runs + rename to slack_private_sync_runs; +alter table slack_dm_sync_messages + rename to slack_private_sync_messages; +alter table slack_dm_sync_message_attachments + rename to slack_private_sync_message_attachments; +alter table slack_dm_sync_checkpoints + rename to slack_private_sync_checkpoints; +alter table slack_dm_sync_backfill_jobs + rename to slack_private_sync_backfill_jobs; +alter table slack_dm_context_documents + rename to slack_private_context_documents; +alter table slack_dm_conversation_context_documents + rename to slack_private_conversation_context_documents; + +-- ParadeDB ties BM25 metadata to the indexed relation name, so rebuild these +-- two indexes after the table rename instead of relying on the index OID alone. +drop index if exists idx_slack_dm_context_documents_bm25; +drop index if exists idx_slack_private_context_documents_bm25; +create index idx_slack_private_context_documents_bm25 + on slack_private_context_documents + using bm25 ( + document_id, + title, + body, + home_team_id, + conversation_id, + conversation_type, + user_id, + bot_id, + message_type, + message_subtype, + occurred_at, + source_updated_at, + metadata + ) + with ( + key_field = 'document_id', + text_fields = '{ + "document_id": { + "tokenizer": {"type": "keyword"} + }, + "home_team_id": { + "tokenizer": {"type": "keyword"} + }, + "conversation_id": { + "tokenizer": {"type": "keyword"} + }, + "user_id": { + "tokenizer": {"type": "keyword"} + }, + "bot_id": { + "tokenizer": {"type": "keyword"} + } + }' + ); + +drop index if exists idx_slack_dm_conversation_context_documents_bm25; +drop index if exists idx_slack_private_conversation_context_documents_bm25; +create index idx_slack_private_conversation_context_documents_bm25 + on slack_private_conversation_context_documents + using bm25 ( + document_id, + title, + body, + home_team_id, + conversation_id, + conversation_type, + last_seen_at, + source_updated_at, + metadata + ) + with ( + key_field = 'document_id', + text_fields = '{ + "document_id": { + "tokenizer": {"type": "keyword"} + }, + "home_team_id": { + "tokenizer": {"type": "keyword"} + }, + "conversation_id": { + "tokenizer": {"type": "keyword"} + } + }' + ); + +-- PL/pgSQL function bodies retain relation names as source text. Recreate any +-- existing projection function with the new table names so the triggers that +-- moved with the tables continue to work after the rename. +do $$ +declare + fn record; + old_definition text; + new_definition text; +begin + for fn in + select p.oid + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = 'public' + and p.prokind = 'f' + and pg_get_functiondef(p.oid) like '%slack_dm_%' + loop + old_definition := pg_get_functiondef(fn.oid); + new_definition := old_definition; + new_definition := replace(new_definition, + 'slack_dm_conversation_context_documents', + 'slack_private_conversation_context_documents'); + new_definition := replace(new_definition, + 'slack_dm_sync_conversation_members', + 'slack_private_sync_conversation_members'); + new_definition := replace(new_definition, + 'slack_dm_sync_message_attachments', + 'slack_private_sync_message_attachments'); + new_definition := replace(new_definition, + 'slack_dm_sync_backfill_jobs', + 'slack_private_sync_backfill_jobs'); + new_definition := replace(new_definition, + 'slack_dm_sync_conversations', + 'slack_private_sync_conversations'); + new_definition := replace(new_definition, + 'slack_dm_sync_checkpoints', + 'slack_private_sync_checkpoints'); + new_definition := replace(new_definition, + 'slack_dm_sync_messages', + 'slack_private_sync_messages'); + new_definition := replace(new_definition, + 'slack_dm_sync_runs', + 'slack_private_sync_runs'); + new_definition := replace(new_definition, + 'slack_dm_context_documents', + 'slack_private_context_documents'); + + if new_definition is distinct from old_definition then + execute new_definition; + end if; + end loop; +end +$$; + +alter table slack_private_sync_conversations + drop constraint if exists slack_dm_sync_conversations_conversation_type_check; + +alter table slack_private_sync_conversations + add constraint slack_private_sync_conversations_conversation_type_check + check (conversation_type in ('im', 'mpim', 'private_channel')); + +-- Centralize access checks for every user-scoped Slack conversation. A user +-- retains access until a successful membership reconciliation marks the row +-- inactive; incomplete membership responses are rejected by the console sync. +create or replace function centaur_can_read_slack_user_conversation( + p_home_team_id text, + p_conversation_id text +) +returns boolean +language sql +stable +security definer +set search_path = pg_catalog, public +as $$ + select exists ( + select 1 + from public.slack_private_sync_conversation_members members + where members.home_team_id = p_home_team_id + and members.conversation_id = p_conversation_id + and members.home_team_id = public.centaur_current_slack_team_id() + and members.user_id = public.centaur_current_slack_user_id() + and members.is_current_member + ) +$$; + +revoke all on function centaur_can_read_slack_user_conversation(text, text) from public; +grant execute on function centaur_can_read_slack_user_conversation(text, text) + to centaur_slack_reader, centaur_readonly; + +drop policy if exists centaur_slack_dm_conversations_reader_select + on slack_private_sync_conversations; +create policy centaur_slack_dm_conversations_reader_select + on slack_private_sync_conversations for select to centaur_slack_reader + using (centaur_can_read_slack_user_conversation(home_team_id, conversation_id)); + +drop policy if exists centaur_readonly_slack_dm_sync_conversations_select + on slack_private_sync_conversations; +create policy centaur_readonly_slack_dm_sync_conversations_select + on slack_private_sync_conversations for select to centaur_readonly + using (centaur_can_read_slack_user_conversation(home_team_id, conversation_id)); + +drop policy if exists centaur_slack_dm_members_reader_select + on slack_private_sync_conversation_members; +create policy centaur_slack_dm_members_reader_select + on slack_private_sync_conversation_members for select to centaur_slack_reader + using ( + user_id = centaur_current_slack_user_id() + and centaur_can_read_slack_user_conversation(home_team_id, conversation_id) + ); + +drop policy if exists centaur_readonly_slack_dm_sync_conversation_members_select + on slack_private_sync_conversation_members; +create policy centaur_readonly_slack_dm_sync_conversation_members_select + on slack_private_sync_conversation_members for select to centaur_readonly + using ( + user_id = centaur_current_slack_user_id() + and centaur_can_read_slack_user_conversation(home_team_id, conversation_id) + ); + +drop policy if exists centaur_slack_dm_messages_reader_select + on slack_private_sync_messages; +create policy centaur_slack_dm_messages_reader_select + on slack_private_sync_messages for select to centaur_slack_reader + using (centaur_can_read_slack_user_conversation(home_team_id, conversation_id)); + +drop policy if exists centaur_readonly_slack_dm_sync_messages_select + on slack_private_sync_messages; +create policy centaur_readonly_slack_dm_sync_messages_select + on slack_private_sync_messages for select to centaur_readonly + using (centaur_can_read_slack_user_conversation(home_team_id, conversation_id)); + +drop policy if exists centaur_slack_dm_attachments_reader_select + on slack_private_sync_message_attachments; +create policy centaur_slack_dm_attachments_reader_select + on slack_private_sync_message_attachments for select to centaur_slack_reader + using (centaur_can_read_slack_user_conversation(home_team_id, conversation_id)); + +drop policy if exists centaur_readonly_slack_dm_sync_message_attachments_select + on slack_private_sync_message_attachments; +create policy centaur_readonly_slack_dm_sync_message_attachments_select + on slack_private_sync_message_attachments for select to centaur_readonly + using (centaur_can_read_slack_user_conversation(home_team_id, conversation_id)); + +drop policy if exists centaur_slack_dm_checkpoints_reader_select + on slack_private_sync_checkpoints; +create policy centaur_slack_dm_checkpoints_reader_select + on slack_private_sync_checkpoints for select to centaur_slack_reader + using (centaur_can_read_slack_user_conversation(home_team_id, conversation_id)); + +drop policy if exists centaur_readonly_slack_dm_sync_checkpoints_select + on slack_private_sync_checkpoints; +create policy centaur_readonly_slack_dm_sync_checkpoints_select + on slack_private_sync_checkpoints for select to centaur_readonly + using (centaur_can_read_slack_user_conversation(home_team_id, conversation_id)); + +drop policy if exists centaur_slack_dm_context_documents_reader_select + on slack_private_context_documents; +create policy centaur_slack_dm_context_documents_reader_select + on slack_private_context_documents for select to centaur_slack_reader + using (centaur_can_read_slack_user_conversation(home_team_id, conversation_id)); + +drop policy if exists centaur_readonly_slack_dm_context_documents_select + on slack_private_context_documents; +create policy centaur_readonly_slack_dm_context_documents_select + on slack_private_context_documents for select to centaur_readonly + using (centaur_can_read_slack_user_conversation(home_team_id, conversation_id)); + +drop policy if exists centaur_slack_dm_conversation_context_documents_reader_select + on slack_private_conversation_context_documents; +create policy centaur_slack_dm_conversation_context_documents_reader_select + on slack_private_conversation_context_documents for select to centaur_slack_reader + using (centaur_can_read_slack_user_conversation(home_team_id, conversation_id)); + +drop policy if exists centaur_readonly_slack_dm_conversation_context_documents_select + on slack_private_conversation_context_documents; +create policy centaur_readonly_slack_dm_conversation_context_documents_select + on slack_private_conversation_context_documents for select to centaur_readonly + using (centaur_can_read_slack_user_conversation(home_team_id, conversation_id)); + +-- The existing projection triggers still populate the private conversation +-- tables. These BEFORE triggers give private-channel rows accurate titles and +-- metadata without duplicating the projection pipeline. +create or replace function centaur_label_slack_private_channel_message_document() +returns trigger +language plpgsql +as $$ +declare + channel_name text; +begin + if new.conversation_type <> 'private_channel' then + return new; + end if; + + select nullif(conversations.raw_payload ->> 'name', '') + into channel_name + from slack_private_sync_conversations conversations + where conversations.home_team_id = new.home_team_id + and conversations.conversation_id = new.conversation_id; + + new.title := 'Slack private channel: #' || coalesce(channel_name, new.conversation_id); + new.metadata := new.metadata || jsonb_build_object( + 'source', 'slack_private_channel', + 'channel_id', new.conversation_id, + 'channel_name', coalesce(channel_name, '') + ); + new.content_hash := md5(concat_ws( + E'\x1f', new.title, new.body, new.permalink, + coalesce(new.occurred_at::text, ''), new.metadata::text + )); + return new; +end; +$$; + +drop trigger if exists trg_label_slack_private_channel_message_document + on slack_private_context_documents; +create trigger trg_label_slack_private_channel_message_document + before insert or update on slack_private_context_documents + for each row + execute function centaur_label_slack_private_channel_message_document(); + +create or replace function centaur_label_slack_private_channel_conversation_document() +returns trigger +language plpgsql +as $$ +declare + channel_name text; +begin + if new.conversation_type <> 'private_channel' then + return new; + end if; + + select nullif(conversations.raw_payload ->> 'name', '') + into channel_name + from slack_private_sync_conversations conversations + where conversations.home_team_id = new.home_team_id + and conversations.conversation_id = new.conversation_id; + + new.title := 'Slack private channel: #' || coalesce(channel_name, new.conversation_id); + new.body := concat_ws(E'\n', channel_name, new.body); + new.metadata := new.metadata || jsonb_build_object( + 'source', 'slack_private_channel', + 'channel_id', new.conversation_id, + 'channel_name', coalesce(channel_name, '') + ); + new.content_hash := md5(concat_ws( + E'\x1f', new.title, new.body, + array_to_string(new.participant_user_ids, E'\x1e'), + array_to_string(new.participant_labels, E'\x1e'), + coalesce(new.last_seen_at::text, ''), new.metadata::text + )); + return new; +end; +$$; + +drop trigger if exists trg_label_slack_private_channel_conversation_document + on slack_private_conversation_context_documents; +create trigger trg_label_slack_private_channel_conversation_document + before insert or update on slack_private_conversation_context_documents + for each row + execute function centaur_label_slack_private_channel_conversation_document(); diff --git a/services/api-rs/crates/centaur-session-sqlx/tests/granola_context_projection.rs b/services/api-rs/crates/centaur-session-sqlx/tests/granola_context_projection.rs new file mode 100644 index 000000000..2636c5090 --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/tests/granola_context_projection.rs @@ -0,0 +1,331 @@ +use std::{ + env, + error::Error, + time::{SystemTime, UNIX_EPOCH}, +}; + +use sqlx::{Connection, Executor, PgConnection, Row}; + +const GRANOLA_SYNC_SQL: &str = include_str!("../migrations/0040_granola_sync_tables.sql"); +const GRANOLA_CONTEXT_PROJECTION_SQL: &str = + include_str!("../migrations/0045_granola_context_projection.sql"); + +#[tokio::test] +async fn granola_notes_project_into_their_dedicated_rls_protected_context_table() +-> Result<(), Box> { + let Some(database_url) = test_database_url() else { + return Ok(()); + }; + let mut conn = PgConnection::connect(&database_url).await?; + let schema = TestSchema::create(&mut conn).await?; + + let result = run_assertions(&mut conn, &schema.name).await; + schema.drop(&mut conn).await?; + result +} + +async fn run_assertions(conn: &mut PgConnection, schema: &str) -> Result<(), Box> { + set_search_path(conn, schema).await?; + create_roles(conn).await?; + create_slack_identity_helpers(conn).await?; + execute_migration(conn, &granola_sync_without_bm25()).await?; + sqlx::raw_sql( + r#" + insert into granola_sync_notes ( + note_id, title, owner_email, access_emails, content_text, source_created_at + ) values ( + 'note_backfilled', 'Existing note', 'alice@example.com', + array['alice@example.com'], 'Existing source data', '2026-07-13T09:00:00Z' + ); + "#, + ) + .execute(&mut *conn) + .await?; + execute_migration(conn, GRANOLA_CONTEXT_PROJECTION_SQL).await?; + grant_schema_usage(conn, schema).await?; + + let backfilled_document_id: String = sqlx::query_scalar( + "select document_id from granola_context_documents where note_id = 'note_backfilled'", + ) + .fetch_one(&mut *conn) + .await?; + assert_eq!(backfilled_document_id, "granola:note_backfilled"); + + sqlx::raw_sql( + r#" + insert into slack_sync_users (team_id, user_id, raw_payload) values + ('T_HOME', 'U_ALICE', '{"profile":{"email":"alice@example.com"}}'), + ('T_HOME', 'U_BOB', '{"profile":{"email":"bob@example.com"}}'); + + insert into granola_sync_notes ( + note_id, title, owner_id, owner_email, owner_name, attendees, + access_emails, calendar_event, content_text, source_created_at, + source_updated_at + ) values + ( + 'note_alice', 'Launch review', 'owner_alice', 'alice@example.com', 'Alice', + '[{"name":"Bob", "email":"bob@example.com"}]', + array['alice@example.com', 'bob@example.com'], + '{"title":"Launch review"}', 'Launch status and risks', + '2026-07-13T10:00:00Z', '2026-07-13T11:00:00Z' + ), + ( + 'note_bob', 'Budget review', 'owner_bob', 'bob@example.com', 'Bob', + '[]', array['bob@example.com'], '{}', 'Budget details', + '2026-07-13T12:00:00Z', '2026-07-13T13:00:00Z' + ); + "#, + ) + .execute(&mut *conn) + .await?; + + let projection = sqlx::query( + "select document_id, title, body, attendee_labels, access_emails, metadata \ + from granola_context_documents where note_id = 'note_alice'", + ) + .fetch_one(&mut *conn) + .await?; + assert_eq!( + projection.try_get::("document_id")?, + "granola:note_alice" + ); + assert_eq!(projection.try_get::("title")?, "Launch review"); + assert_eq!( + projection.try_get::("body")?, + "Launch status and risks" + ); + assert_eq!( + projection.try_get::, _>("attendee_labels")?, + vec!["Bob"] + ); + assert_eq!( + projection.try_get::, _>("access_emails")?, + vec!["alice@example.com", "bob@example.com"] + ); + assert_eq!( + projection + .try_get::("metadata")? + .get("source_type") + .and_then(serde_json::Value::as_str), + Some("granola_note") + ); + + sqlx::query( + "update granola_sync_notes set title = 'Launch decision', \ + content_text = 'Approved the launch', access_emails = array['alice@example.com'] \ + where note_id = 'note_alice'", + ) + .execute(&mut *conn) + .await?; + let updated = sqlx::query( + "select title, body, access_emails from granola_context_documents where note_id = 'note_alice'", + ) + .fetch_one(&mut *conn) + .await?; + assert_eq!(updated.try_get::("title")?, "Launch decision"); + assert_eq!(updated.try_get::("body")?, "Approved the launch"); + assert_eq!( + updated.try_get::, _>("access_emails")?, + vec!["alice@example.com"] + ); + + assert_visible_documents( + conn, + schema, + "U_ALICE", + &["granola:note_alice", "granola:note_backfilled"], + ) + .await?; + assert_visible_documents(conn, schema, "U_BOB", &["granola:note_bob"]).await?; + Ok(()) +} + +async fn assert_visible_documents( + conn: &mut PgConnection, + schema: &str, + user_id: &str, + expected: &[&str], +) -> Result<(), Box> { + let user_email = match user_id { + "U_ALICE" => "alice@example.com", + "U_BOB" => "bob@example.com", + _ => unreachable!("test only defines Alice and Bob"), + }; + sqlx::query("set role centaur_slack_reader") + .execute(&mut *conn) + .await?; + sqlx::query("select set_config('centaur.slack_team_id', 'T_HOME', false)") + .execute(&mut *conn) + .await?; + sqlx::query("select set_config('centaur.slack_user_id', $1, false)") + .bind(user_id) + .execute(&mut *conn) + .await?; + sqlx::query("select set_config('centaur.user_email', $1, false)") + .bind(user_email) + .execute(&mut *conn) + .await?; + let rows = + sqlx::query("select document_id from granola_context_documents order by document_id") + .fetch_all(&mut *conn) + .await?; + let actual = rows + .iter() + .map(|row| row.try_get::("document_id")) + .collect::, _>>()?; + assert_eq!( + actual, + expected + .iter() + .map(|document_id| (*document_id).to_owned()) + .collect::>() + ); + conn.execute("reset role").await?; + set_search_path(conn, schema).await?; + Ok(()) +} + +fn test_database_url() -> Option { + env::var("SESSION_SQLX_TEST_DATABASE_URL") + .or_else(|_| env::var("SESSION_RUNTIME_TEST_DATABASE_URL")) + .map_err(|_| { + eprintln!( + "skipping Granola context projection tests: set SESSION_SQLX_TEST_DATABASE_URL to a Postgres URL" + ); + }) + .ok() +} + +struct TestSchema { + name: String, +} + +impl TestSchema { + async fn create(conn: &mut PgConnection) -> Result> { + let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos(); + let name = format!("granola_context_{}_{}", std::process::id(), nanos); + conn.execute(format!(r#"create schema "{}""#, name).as_str()) + .await?; + Ok(Self { name }) + } + + async fn drop(self, conn: &mut PgConnection) -> Result<(), Box> { + conn.execute(format!(r#"drop schema if exists "{}" cascade"#, self.name).as_str()) + .await?; + Ok(()) + } +} + +async fn set_search_path(conn: &mut PgConnection, schema: &str) -> Result<(), sqlx::Error> { + conn.execute(format!(r#"set search_path to "{}", public"#, schema).as_str()) + .await?; + Ok(()) +} + +async fn create_roles(conn: &mut PgConnection) -> Result<(), sqlx::Error> { + sqlx::raw_sql( + r#" + do $$ + begin + if not exists (select 1 from pg_roles where rolname = 'centaur_slack_reader') then + create role centaur_slack_reader nologin; + end if; + end + $$; + "#, + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + +async fn create_slack_identity_helpers(conn: &mut PgConnection) -> Result<(), sqlx::Error> { + sqlx::raw_sql( + r#" + -- The Granola RLS helper intentionally pins its security-definer search + -- path to public, matching production. CI's disposable Postgres starts + -- without its source table and identity functions, so provide their + -- minimal public definitions. Local development already has the real + -- objects and is left untouched by the conditional setup below. + create table if not exists public.slack_sync_users ( + team_id text not null, + user_id text not null, + raw_payload jsonb not null default '{}'::jsonb, + primary key (team_id, user_id) + ); + + create table slack_sync_users ( + team_id text not null, + user_id text not null, + raw_payload jsonb not null default '{}'::jsonb, + primary key (team_id, user_id) + ); + + do $$ + begin + if to_regprocedure('public.centaur_current_slack_team_id()') is null then + execute $function$ + create function public.centaur_current_slack_team_id() + returns text language sql stable as $body$ + select nullif(current_setting('centaur.slack_team_id', true), '') + $body$ + $function$; + end if; + if to_regprocedure('public.centaur_current_slack_user_id()') is null then + execute $function$ + create function public.centaur_current_slack_user_id() + returns text language sql stable as $body$ + select nullif(current_setting('centaur.slack_user_id', true), '') + $body$ + $function$; + end if; + end + $$; + + create function centaur_current_slack_team_id() + returns text language sql stable as $$ + select nullif(current_setting('centaur.slack_team_id', true), '') + $$; + + create function centaur_current_slack_user_id() + returns text language sql stable as $$ + select nullif(current_setting('centaur.slack_user_id', true), '') + $$; + "#, + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + +async fn grant_schema_usage(conn: &mut PgConnection, schema: &str) -> Result<(), sqlx::Error> { + conn.execute( + format!( + r#"grant usage on schema "{}" to centaur_slack_reader"#, + schema + ) + .as_str(), + ) + .await?; + Ok(()) +} + +async fn execute_migration(conn: &mut PgConnection, sql: &str) -> Result<(), sqlx::Error> { + sqlx::raw_sql(sql).execute(&mut *conn).await?; + Ok(()) +} + +fn granola_sync_without_bm25() -> String { + let sql = GRANOLA_SYNC_SQL.replace( + "create extension if not exists pg_search;", + "-- search extension unavailable in this test database", + ); + let (before_bm25, rest) = sql + .split_once("drop index if exists idx_granola_context_documents_bm25;") + .expect("Granola migration should contain BM25 index block"); + let (_, after_bm25) = rest + .split_once("create table if not exists granola_sync_checkpoints") + .expect("Granola migration should create sync checkpoints after the BM25 index"); + + format!("{before_bm25}create table if not exists granola_sync_checkpoints{after_bm25}") +} diff --git a/services/api-rs/crates/centaur-session-sqlx/tests/slack_dm_context_rls.rs b/services/api-rs/crates/centaur-session-sqlx/tests/slack_dm_context_rls.rs index 8d10f0d23..67a12883d 100644 --- a/services/api-rs/crates/centaur-session-sqlx/tests/slack_dm_context_rls.rs +++ b/services/api-rs/crates/centaur-session-sqlx/tests/slack_dm_context_rls.rs @@ -14,17 +14,19 @@ const SLACK_DM_CONVERSATION_CONTEXT_DOCUMENTS_SQL: &str = include_str!("../migrations/0030_slack_dm_conversation_context_documents.sql"); const READONLY_DM_RLS_SQL: &str = include_str!("../migrations/0042_centaur_readonly_slack_dm_rls.sql"); +const SLACK_PRIVATE_CONVERSATIONS_SQL: &str = + include_str!("../migrations/0046_slack_private_channel_oauth_sync.sql"); const RLS_TABLES: &[&str] = &[ - "slack_dm_sync_conversations", - "slack_dm_sync_conversation_members", - "slack_dm_sync_messages", - "slack_dm_sync_message_attachments", - "slack_dm_sync_checkpoints", - "slack_dm_sync_runs", - "slack_dm_sync_backfill_jobs", - "slack_dm_context_documents", - "slack_dm_conversation_context_documents", + "slack_private_sync_conversations", + "slack_private_sync_conversation_members", + "slack_private_sync_messages", + "slack_private_sync_message_attachments", + "slack_private_sync_checkpoints", + "slack_private_sync_runs", + "slack_private_sync_backfill_jobs", + "slack_private_context_documents", + "slack_private_conversation_context_documents", ]; #[derive(Debug, PartialEq, Eq)] @@ -63,10 +65,12 @@ async fn run_rls_assertions(conn: &mut PgConnection, schema: &str) -> Result<(), execute_migration(conn, READONLY_DM_RLS_SQL).await?; grant_schema_usage(conn, schema).await?; - assert_rls_enabled(conn).await?; - assert_expected_policies(conn).await?; insert_fixture_rows(conn).await?; assert_projected_documents(conn).await?; + execute_slack_private_conversations_migration(conn, schema).await?; + insert_private_channel_fixture(conn).await?; + assert_rls_enabled(conn).await?; + assert_expected_policies(conn).await?; let user_a = visible_rows( conn, @@ -79,19 +83,30 @@ async fn run_rls_assertions(conn: &mut PgConnection, schema: &str) -> Result<(), assert_eq!( user_a, VisibleDmRows { - conversations: vec!["T_HOME:D_A".to_owned(), "T_HOME:G_MPIM".to_owned()], - members: vec!["T_HOME:D_A:U_A".to_owned(), "T_HOME:G_MPIM:U_A".to_owned()], + conversations: vec![ + "T_HOME:C_PRIVATE".to_owned(), + "T_HOME:D_A".to_owned(), + "T_HOME:G_MPIM".to_owned(), + ], + members: vec![ + "T_HOME:C_PRIVATE:U_A".to_owned(), + "T_HOME:D_A:U_A".to_owned(), + "T_HOME:G_MPIM:U_A".to_owned(), + ], messages: vec![ + "T_HOME:C_PRIVATE:1000.000005".to_owned(), "T_HOME:D_A:1000.000001".to_owned(), "T_HOME:G_MPIM:1000.000003".to_owned(), ], attachments: vec!["T_HOME:D_A:1000.000001:F_A".to_owned()], checkpoints: vec!["bcr_a:T_HOME:D_A".to_owned()], context_docs: vec![ + "slack_dm:T_HOME:C_PRIVATE:1000.000005".to_owned(), "slack_dm:T_HOME:D_A:1000.000001".to_owned(), "slack_dm:T_HOME:G_MPIM:1000.000003".to_owned(), ], conversation_context_docs: vec![ + "slack_dm_conversation:T_HOME:C_PRIVATE".to_owned(), "slack_dm_conversation:T_HOME:D_A".to_owned(), "slack_dm_conversation:T_HOME:G_MPIM".to_owned(), ], @@ -286,6 +301,14 @@ async fn execute_slack_dm_conversation_context_documents_migration( execute_migration(conn, &sql).await } +async fn execute_slack_private_conversations_migration( + conn: &mut PgConnection, + schema: &str, +) -> Result<(), sqlx::Error> { + let sql = slack_private_conversations_for_test_schema(schema); + execute_migration(conn, &sql).await +} + async fn pg_search_available(conn: &mut PgConnection) -> Result { sqlx::query_scalar( "select exists (select 1 from pg_available_extensions where name = 'pg_search')", @@ -331,6 +354,29 @@ fn slack_dm_conversation_context_documents_without_bm25() -> String { ) } +fn slack_private_conversations_for_test_schema(schema: &str) -> String { + let (before_bm25, rest) = SLACK_PRIVATE_CONVERSATIONS_SQL + .split_once("-- ParadeDB ties BM25 metadata") + .expect("Slack private migration should contain its BM25 index block"); + let (_, after_bm25) = rest + .split_once("-- PL/pgSQL function bodies retain relation names") + .expect("Slack private migration should recreate projection functions"); + let sql = format!( + "{before_bm25}-- BM25 indexes are omitted in the portable RLS test.\n\ + -- PL/pgSQL function bodies retain relation names{after_bm25}" + ); + let quoted_schema = format!("\"{}\"", schema.replace('"', "\"\"")); + sql.replace( + "n.nspname = 'public'", + format!("n.nspname = '{schema}'").as_str(), + ) + .replace("public.", format!("{quoted_schema}.").as_str()) + .replace( + "set search_path = pg_catalog, public", + format!("set search_path = pg_catalog, {quoted_schema}").as_str(), + ) +} + #[test] fn slack_dm_context_documents_test_migration_omits_bm25_when_extension_is_unavailable() { let sql = slack_dm_context_documents_without_bm25(); @@ -360,6 +406,48 @@ fn slack_dm_context_documents_test_migration_omits_bm25_when_extension_is_unavai ); } +#[test] +fn slack_private_conversation_migration_renames_every_user_scoped_table() { + for (old_name, new_name) in [ + ( + "slack_dm_sync_conversations", + "slack_private_sync_conversations", + ), + ( + "slack_dm_sync_conversation_members", + "slack_private_sync_conversation_members", + ), + ("slack_dm_sync_runs", "slack_private_sync_runs"), + ("slack_dm_sync_messages", "slack_private_sync_messages"), + ( + "slack_dm_sync_message_attachments", + "slack_private_sync_message_attachments", + ), + ( + "slack_dm_sync_checkpoints", + "slack_private_sync_checkpoints", + ), + ( + "slack_dm_sync_backfill_jobs", + "slack_private_sync_backfill_jobs", + ), + ( + "slack_dm_context_documents", + "slack_private_context_documents", + ), + ( + "slack_dm_conversation_context_documents", + "slack_private_conversation_context_documents", + ), + ] { + assert!( + SLACK_PRIVATE_CONVERSATIONS_SQL + .contains(format!("alter table {old_name}\n rename to {new_name};").as_str()), + "missing table rename from {old_name} to {new_name}" + ); + } +} + async fn assert_rls_enabled(conn: &mut PgConnection) -> Result<(), sqlx::Error> { for table in RLS_TABLES { let enabled: bool = sqlx::query_scalar( @@ -394,72 +482,75 @@ async fn assert_expected_policies(conn: &mut PgConnection) -> Result<(), sqlx::E for expected in [ ( - "slack_dm_sync_conversations", + "slack_private_sync_conversations", "centaur_slack_dm_conversations_reader_select", ), ( - "slack_dm_sync_conversation_members", + "slack_private_sync_conversation_members", "centaur_slack_dm_members_reader_select", ), ( - "slack_dm_sync_messages", + "slack_private_sync_messages", "centaur_slack_dm_messages_reader_select", ), ( - "slack_dm_sync_message_attachments", + "slack_private_sync_message_attachments", "centaur_slack_dm_attachments_reader_select", ), ( - "slack_dm_sync_checkpoints", + "slack_private_sync_checkpoints", "centaur_slack_dm_checkpoints_reader_select", ), - ("slack_dm_sync_runs", "centaur_slack_dm_runs_reader_select"), ( - "slack_dm_sync_backfill_jobs", + "slack_private_sync_runs", + "centaur_slack_dm_runs_reader_select", + ), + ( + "slack_private_sync_backfill_jobs", "centaur_slack_dm_backfill_jobs_reader_select", ), ( - "slack_dm_context_documents", + "slack_private_context_documents", "centaur_slack_dm_context_documents_reader_select", ), ( - "slack_dm_conversation_context_documents", + "slack_private_conversation_context_documents", "centaur_slack_dm_conversation_context_documents_reader_select", ), ( - "slack_dm_sync_conversations", + "slack_private_sync_conversations", "centaur_readonly_slack_dm_sync_conversations_select", ), ( - "slack_dm_sync_conversation_members", + "slack_private_sync_conversation_members", "centaur_readonly_slack_dm_sync_conversation_members_select", ), ( - "slack_dm_sync_messages", + "slack_private_sync_messages", "centaur_readonly_slack_dm_sync_messages_select", ), ( - "slack_dm_sync_message_attachments", + "slack_private_sync_message_attachments", "centaur_readonly_slack_dm_sync_message_attachments_select", ), ( - "slack_dm_sync_checkpoints", + "slack_private_sync_checkpoints", "centaur_readonly_slack_dm_sync_checkpoints_select", ), ( - "slack_dm_sync_runs", + "slack_private_sync_runs", "centaur_readonly_slack_dm_sync_runs_select", ), ( - "slack_dm_sync_backfill_jobs", + "slack_private_sync_backfill_jobs", "centaur_readonly_slack_dm_sync_backfill_jobs_select", ), ( - "slack_dm_context_documents", + "slack_private_context_documents", "centaur_readonly_slack_dm_context_documents_select", ), ( - "slack_dm_conversation_context_documents", + "slack_private_conversation_context_documents", "centaur_readonly_slack_dm_conversation_context_documents_select", ), ] { @@ -536,6 +627,30 @@ async fn insert_fixture_rows(conn: &mut PgConnection) -> Result<(), sqlx::Error> Ok(()) } +async fn insert_private_channel_fixture(conn: &mut PgConnection) -> Result<(), sqlx::Error> { + sqlx::raw_sql( + r#" + insert into slack_private_sync_conversations + (home_team_id, conversation_id, conversation_type) + values ('T_HOME', 'C_PRIVATE', 'private_channel'); + + insert into slack_private_sync_conversation_members + (home_team_id, conversation_id, user_id, is_current_member) + values + ('T_HOME', 'C_PRIVATE', 'U_A', true), + ('T_HOME', 'C_PRIVATE', 'U_C', false); + + insert into slack_private_sync_messages + (home_team_id, conversation_id, message_ts, text, source_run_id) + values + ('T_HOME', 'C_PRIVATE', '1000.000005', 'current member only', 'run_a'); + "#, + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + async fn assert_projected_documents(conn: &mut PgConnection) -> Result<(), sqlx::Error> { let row = sqlx::query( r#" @@ -614,41 +729,41 @@ async fn visible_rows( let rows = VisibleDmRows { conversations: text_array( &mut tx, - "select coalesce(array_agg(home_team_id || ':' || conversation_id order by home_team_id, conversation_id), '{}') from slack_dm_sync_conversations", + "select coalesce(array_agg(home_team_id || ':' || conversation_id order by home_team_id, conversation_id), '{}') from slack_private_sync_conversations", ) .await?, members: text_array( &mut tx, - "select coalesce(array_agg(home_team_id || ':' || conversation_id || ':' || user_id order by home_team_id, conversation_id, user_id), '{}') from slack_dm_sync_conversation_members", + "select coalesce(array_agg(home_team_id || ':' || conversation_id || ':' || user_id order by home_team_id, conversation_id, user_id), '{}') from slack_private_sync_conversation_members", ) .await?, messages: text_array( &mut tx, - "select coalesce(array_agg(home_team_id || ':' || conversation_id || ':' || message_ts order by home_team_id, conversation_id, message_ts), '{}') from slack_dm_sync_messages", + "select coalesce(array_agg(home_team_id || ':' || conversation_id || ':' || message_ts order by home_team_id, conversation_id, message_ts), '{}') from slack_private_sync_messages", ) .await?, attachments: text_array( &mut tx, - "select coalesce(array_agg(home_team_id || ':' || conversation_id || ':' || message_ts || ':' || slack_file_id order by home_team_id, conversation_id, message_ts, slack_file_id), '{}') from slack_dm_sync_message_attachments", + "select coalesce(array_agg(home_team_id || ':' || conversation_id || ':' || message_ts || ':' || slack_file_id order by home_team_id, conversation_id, message_ts, slack_file_id), '{}') from slack_private_sync_message_attachments", ) .await?, checkpoints: text_array( &mut tx, - "select coalesce(array_agg(broker_credential_id || ':' || home_team_id || ':' || conversation_id order by broker_credential_id, home_team_id, conversation_id), '{}') from slack_dm_sync_checkpoints", + "select coalesce(array_agg(broker_credential_id || ':' || home_team_id || ':' || conversation_id order by broker_credential_id, home_team_id, conversation_id), '{}') from slack_private_sync_checkpoints", ) .await?, context_docs: text_array( &mut tx, - "select coalesce(array_agg(document_id order by document_id), '{}') from slack_dm_context_documents", + "select coalesce(array_agg(document_id order by document_id), '{}') from slack_private_context_documents", ) .await?, conversation_context_docs: text_array( &mut tx, - "select coalesce(array_agg(document_id order by document_id), '{}') from slack_dm_conversation_context_documents", + "select coalesce(array_agg(document_id order by document_id), '{}') from slack_private_conversation_context_documents", ) .await?, - runs: count(&mut tx, "slack_dm_sync_runs").await?, - backfill_jobs: count(&mut tx, "slack_dm_sync_backfill_jobs").await?, + runs: count(&mut tx, "slack_private_sync_runs").await?, + backfill_jobs: count(&mut tx, "slack_private_sync_backfill_jobs").await?, }; tx.execute("reset role").await?; diff --git a/services/api-rs/crates/centaur-telemetry/src/lib.rs b/services/api-rs/crates/centaur-telemetry/src/lib.rs index dabbee79d..db9cce56a 100644 --- a/services/api-rs/crates/centaur-telemetry/src/lib.rs +++ b/services/api-rs/crates/centaur-telemetry/src/lib.rs @@ -73,7 +73,6 @@ pub const ETL_ITEMS_FAILED_TOTAL: &str = "etl_items_failed_total"; pub const ETL_BACKFILL_JOBS: &str = "etl_backfill_jobs"; pub const ETL_BACKFILL_JOB_AGE_SECONDS: &str = "etl_backfill_job_age_seconds"; pub const COMPANY_CONTEXT_DOCUMENTS_CHANGED_TOTAL: &str = "company_context_documents_changed_total"; -pub const COMPANY_CONTEXT_DOCUMENT_SIZE_CHARS: &str = "company_context_document_size_chars"; pub const COMPANY_CONTEXT_PROJECTION_LAG_SECONDS: &str = "company_context_projection_lag_seconds"; pub const WORKFLOW_QUEUE_TASKS: &str = "workflow_queue_tasks"; pub const WORKFLOW_QUEUE_TASKS_BY_WORKFLOW: &str = "workflow_queue_tasks_by_workflow"; @@ -127,9 +126,6 @@ const SESSION_FIRST_TOKEN_LATENCY_BUCKETS: &[f64] = &[ ]; const SANDBOX_STARTUP_DURATION_BUCKETS: &[f64] = &[0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0]; -const COMPANY_CONTEXT_DOCUMENT_SIZE_BUCKETS: &[f64] = &[ - 100.0, 500.0, 1_000.0, 5_000.0, 10_000.0, 25_000.0, 50_000.0, 100_000.0, 250_000.0, 500_000.0, -]; const SLACK_ARCHIVE_IMPORT_DURATION_BUCKETS: &[f64] = &[ 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1_200.0, 3_600.0, ]; @@ -266,10 +262,6 @@ pub fn prometheus_handle() -> Result { Matcher::Full(SANDBOX_STARTUP_DURATION_SECONDS.to_owned()), SANDBOX_STARTUP_DURATION_BUCKETS, )? - .set_buckets_for_metric( - Matcher::Full(COMPANY_CONTEXT_DOCUMENT_SIZE_CHARS.to_owned()), - COMPANY_CONTEXT_DOCUMENT_SIZE_BUCKETS, - )? .set_buckets_for_metric( Matcher::Full(SLACK_ARCHIVE_IMPORT_DURATION_SECONDS.to_owned()), SLACK_ARCHIVE_IMPORT_DURATION_BUCKETS, @@ -995,10 +987,6 @@ fn describe_metrics() { COMPANY_CONTEXT_DOCUMENTS_CHANGED_TOTAL, "Company context document changes observed by ETL workflows." ); - metrics::describe_histogram!( - COMPANY_CONTEXT_DOCUMENT_SIZE_CHARS, - "Company context document sizes in characters." - ); metrics::describe_gauge!( COMPANY_CONTEXT_PROJECTION_LAG_SECONDS, metrics::Unit::Seconds, diff --git a/services/api-rs/crates/centaur-workflows/src/lib.rs b/services/api-rs/crates/centaur-workflows/src/lib.rs index 51be071b2..c780572a7 100644 --- a/services/api-rs/crates/centaur-workflows/src/lib.rs +++ b/services/api-rs/crates/centaur-workflows/src/lib.rs @@ -627,6 +627,12 @@ impl WorkflowRuntime { schedule_client .create_queue(Some(WORKFLOW_SCHEDULE_QUEUE), CreateQueueOptions::default()) .await?; + let workflow_clients = WorkflowQueueClients { + standard: client.clone(), + slack_live: slack_live_client.clone(), + etl: etl_client.clone(), + etl_backfill: etl_backfill_client.clone(), + }; let discovery = discover_python_workflow_metadata() .await @@ -646,44 +652,81 @@ impl WorkflowRuntime { let task_session_runtime = session_runtime.clone(); let task_workflow_host_sandbox = workflow_host_sandbox.clone(); + let task_workflow_clients = workflow_clients.clone(); client.register_task(WORKFLOW_TASK, move |input: WorkflowTaskInput, ctx| { let session_runtime = task_session_runtime.clone(); let workflow_host_sandbox = task_workflow_host_sandbox.clone(); - async move { run_centaur_workflow(input, ctx, session_runtime, workflow_host_sandbox).await } + let workflow_clients = task_workflow_clients.clone(); + async move { + run_centaur_workflow( + input, + ctx, + session_runtime, + workflow_host_sandbox, + workflow_clients, + ) + .await + } })?; let slack_live_session_runtime = session_runtime.clone(); let slack_live_workflow_host_sandbox = workflow_host_sandbox.clone(); + let slack_live_workflow_clients = workflow_clients.clone(); slack_live_client.register_task(WORKFLOW_TASK, move |input: WorkflowTaskInput, ctx| { let session_runtime = slack_live_session_runtime.clone(); let workflow_host_sandbox = slack_live_workflow_host_sandbox.clone(); - async move { run_centaur_workflow(input, ctx, session_runtime, workflow_host_sandbox).await } + let workflow_clients = slack_live_workflow_clients.clone(); + async move { + run_centaur_workflow( + input, + ctx, + session_runtime, + workflow_host_sandbox, + workflow_clients, + ) + .await + } })?; let etl_session_runtime = session_runtime.clone(); let etl_workflow_host_sandbox = workflow_host_sandbox.clone(); + let etl_workflow_clients = workflow_clients.clone(); etl_client.register_task(WORKFLOW_TASK, move |input: WorkflowTaskInput, ctx| { let session_runtime = etl_session_runtime.clone(); let workflow_host_sandbox = etl_workflow_host_sandbox.clone(); - async move { run_centaur_workflow(input, ctx, session_runtime, workflow_host_sandbox).await } + let workflow_clients = etl_workflow_clients.clone(); + async move { + run_centaur_workflow( + input, + ctx, + session_runtime, + workflow_host_sandbox, + workflow_clients, + ) + .await + } })?; let etl_backfill_session_runtime = session_runtime.clone(); let etl_backfill_workflow_host_sandbox = workflow_host_sandbox.clone(); + let etl_backfill_workflow_clients = workflow_clients.clone(); etl_backfill_client.register_task( WORKFLOW_TASK, move |input: WorkflowTaskInput, ctx| { let session_runtime = etl_backfill_session_runtime.clone(); let workflow_host_sandbox = etl_backfill_workflow_host_sandbox.clone(); + let workflow_clients = etl_backfill_workflow_clients.clone(); async move { - run_centaur_workflow(input, ctx, session_runtime, workflow_host_sandbox).await + run_centaur_workflow( + input, + ctx, + session_runtime, + workflow_host_sandbox, + workflow_clients, + ) + .await } }, )?; let schedule_tick_client = schedule_client.clone(); - let workflow_clients_for_schedule = WorkflowQueueClients { - standard: client.clone(), - slack_live: slack_live_client.clone(), - etl: etl_client.clone(), - etl_backfill: etl_backfill_client.clone(), - }; + let workflow_clients_for_schedule = workflow_clients.clone(); let schedule_registry_for_task = schedule_registry.clone(); schedule_client.register_task_with( TaskRegistrationOptions::new(WORKFLOW_SCHEDULE_TASK), @@ -784,12 +827,7 @@ impl WorkflowRuntime { let metadata_reconciler = workflow_reconcile_interval().map(|interval| { spawn_workflow_metadata_reconciler( schedule_client.clone(), - WorkflowQueueClients { - standard: client.clone(), - slack_live: slack_live_client.clone(), - etl: etl_client.clone(), - etl_backfill: etl_backfill_client.clone(), - }, + workflow_clients, webhook_registry.clone(), schedule_registry.clone(), interval, @@ -2553,6 +2591,10 @@ fn next_schedule_time( } } +/// Prepends a seconds field so five-field crontab-style expressions parse with the +/// `cron` crate. Note the crate's day-of-week numbering is Quartz-style (1 = Sunday, +/// 7 = Saturday; 0 rejected), NOT Unix crontab — schedules should use day names +/// (`MON-FRI`) to avoid firing on the wrong days. fn normalize_cron_expression(expr: &str) -> String { let fields = expr.split_whitespace().collect::>(); if fields.len() == 5 { @@ -2567,11 +2609,18 @@ async fn run_centaur_workflow( ctx: TaskContext, session_runtime: SessionRuntime, workflow_host_sandbox: Option, + workflow_clients: WorkflowQueueClients, ) -> absurd::Result { let mut cleanup_guard = WorkflowSandboxCleanupGuard::new(session_runtime.clone(), ctx.run_id().to_owned()); - let result = - run_centaur_workflow_inner(input, ctx, session_runtime, workflow_host_sandbox).await; + let result = run_centaur_workflow_inner( + input, + ctx, + session_runtime, + workflow_host_sandbox, + workflow_clients, + ) + .await; if let Some(reason) = workflow_cleanup_reason(&result) { cleanup_guard.cleanup(reason).await; } else { @@ -2594,6 +2643,7 @@ async fn run_centaur_workflow_inner( ctx: TaskContext, session_runtime: SessionRuntime, workflow_host_sandbox: Option, + workflow_clients: WorkflowQueueClients, ) -> absurd::Result { let _heartbeat_guard = start_workflow_task_heartbeat(ctx.clone()) .await @@ -2689,6 +2739,9 @@ async fn run_centaur_workflow_inner( workflow_owned_thread: true, idle_timeout_ms, max_duration_ms, + model: None, + provider: None, + reasoning: None, }, ) .await @@ -2751,6 +2804,7 @@ async fn run_centaur_workflow_inner( ctx.clone(), session_runtime, workflow_host_sandbox, + workflow_clients, ) .await .map_err(absurd_error)?; @@ -2873,11 +2927,19 @@ async fn run_python_workflow_host( ctx: TaskContext, session_runtime: SessionRuntime, workflow_host_sandbox: Option, + workflow_clients: WorkflowQueueClients, ) -> Result { if let Some(sandbox) = workflow_host_sandbox { - return run_python_workflow_host_in_sandbox(input, ctx, session_runtime, sandbox).await; + return run_python_workflow_host_in_sandbox( + input, + ctx, + session_runtime, + sandbox, + workflow_clients, + ) + .await; } - run_python_workflow_host_local(input, ctx, session_runtime).await + run_python_workflow_host_local(input, ctx, session_runtime, workflow_clients).await } async fn start_workflow_task_heartbeat( @@ -2899,6 +2961,7 @@ async fn run_python_workflow_host_local( input: WorkflowTaskInput, ctx: TaskContext, session_runtime: SessionRuntime, + workflow_clients: WorkflowQueueClients, ) -> Result { let host_path = python_workflow_host_path(); let task_token = workflow_task_token(ctx.run_id(), ctx.task_id())?; @@ -3000,18 +3063,23 @@ async fn run_python_workflow_host_local( record_python_workflow_metric(&message); } Some(message_type) if message_type.starts_with("ctx.") => { - let response = - match handle_python_context_request(&message, &ctx, &session_runtime, &input) - .await - { - Ok(response) => response, - Err(error) => { - drop(stdin); - let _ = child.start_kill(); - let _ = child.wait().await; - return Err(error); - } - }; + let response = match handle_python_context_request( + &message, + &ctx, + &session_runtime, + &input, + &workflow_clients, + ) + .await + { + Ok(response) => response, + Err(error) => { + drop(stdin); + let _ = child.start_kill(); + let _ = child.wait().await; + return Err(error); + } + }; write_host_message(&mut stdin, &response).await?; } other => { @@ -3040,6 +3108,7 @@ async fn run_python_workflow_host_in_sandbox( ctx: TaskContext, session_runtime: SessionRuntime, sandbox: WorkflowHostSandboxRuntime, + workflow_clients: WorkflowQueueClients, ) -> Result { let task_token = workflow_task_token(ctx.run_id(), ctx.task_id())?; let mut spec = sandbox.spec.clone(); @@ -3078,6 +3147,7 @@ async fn run_python_workflow_host_in_sandbox( input, ctx, session_runtime, + workflow_clients, &mut stdin, io.stdout, stderr_task, @@ -3096,6 +3166,7 @@ async fn run_python_workflow_host_protocol( input: WorkflowTaskInput, ctx: TaskContext, session_runtime: SessionRuntime, + workflow_clients: WorkflowQueueClients, stdin: &mut W, stdout: R, stderr_task: JoinHandle, @@ -3155,8 +3226,14 @@ where record_python_workflow_metric(&message); } Some(message_type) if message_type.starts_with("ctx.") => { - let response = - handle_python_context_request(&message, &ctx, &session_runtime, &input).await?; + let response = handle_python_context_request( + &message, + &ctx, + &session_runtime, + &input, + &workflow_clients, + ) + .await?; write_host_message(stdin, &response).await?; } other => { @@ -3286,6 +3363,7 @@ async fn handle_python_context_request( ctx: &TaskContext, session_runtime: &SessionRuntime, input: &WorkflowTaskInput, + workflow_clients: &WorkflowQueueClients, ) -> Result { let request_id = message .get("request_id") @@ -3370,6 +3448,12 @@ async fn handle_python_context_request( Err(error) => Err(error.to_string()), } } + Some("ctx.workflow.start") => { + match start_python_child_workflow(message, input, workflow_clients).await { + Ok(value) => Ok(value), + Err(error) => Err(error.to_string()), + } + } Some("ctx.call_tool") => match call_python_workflow_tool(message).await { Ok(value) => Ok(value), Err(error) => Err(error.to_string()), @@ -3398,6 +3482,62 @@ async fn handle_python_context_request( }) } +async fn start_python_child_workflow( + message: &Value, + parent: &WorkflowTaskInput, + workflow_clients: &WorkflowQueueClients, +) -> Result { + let workflow_name = message + .get("workflow_name") + .and_then(Value::as_str) + .map(str::trim) + .filter(|name| !name.is_empty()) + .ok_or_else(|| { + WorkflowRuntimeError::BadRequest( + "ctx.workflow.start requires a non-empty workflow_name".to_owned(), + ) + })?; + WorkflowEnablement::from_env()?.ensure_enabled(workflow_name)?; + let child_input = message.get("input").cloned().unwrap_or_else(|| json!({})); + if !child_input.is_object() { + return Err(WorkflowRuntimeError::BadRequest( + "ctx.workflow.start input must be an object".to_owned(), + )); + } + let idempotency_key = message + .get("idempotency_key") + .and_then(Value::as_str) + .map(str::trim) + .filter(|key| !key.is_empty()) + .map(ToOwned::to_owned); + let target_client = match workflow_queue_class(workflow_name) { + WorkflowQueueClass::Standard => &workflow_clients.standard, + WorkflowQueueClass::SlackLive => &workflow_clients.slack_live, + WorkflowQueueClass::Etl => &workflow_clients.etl, + WorkflowQueueClass::EtlBackfill => &workflow_clients.etl_backfill, + }; + let spawn = target_client + .spawn( + WORKFLOW_TASK, + WorkflowTaskInput { + workflow_name: workflow_name.to_owned(), + input: child_input, + harness_type: parent.harness_type.clone(), + }, + SpawnOptions { + idempotency_key, + ..SpawnOptions::default() + }, + ) + .await?; + Ok(json!({ + "workflow_name": workflow_name, + "task_id": spawn.task_id, + "run_id": spawn.run_id, + "created": spawn.created, + })) +} + fn parse_python_duration_seconds(message: &Value) -> Result { let seconds = message .get("duration_seconds") @@ -3510,6 +3650,17 @@ async fn run_python_agent_turn( .and_then(Value::as_str) .map(ToOwned::to_owned) .unwrap_or_else(|| format!("absurd-workflow-agent-turn:{client_message_id}")); + // Optional per-turn harness knobs, mirroring the slackbot's `--model` / + // `--bedrock` / `-rsn` flags. `reasoning` accepts `reasoning_effort` and + // `effort` aliases so Python callers can use whichever reads best. + let model = first_str_arg(&args, &["model"]); + let provider = first_str_arg(&args, &["provider"]); + let reasoning = first_str_arg(&args, &["reasoning", "reasoning_effort", "effort"]); + // Record the model on the execution like the slackbot does, so Console + // readers can show what a workflow-dispatched turn ran on. + if let Some(model) = model.as_deref() { + object_insert(&mut execution_metadata, "model", json!(model)); + } let result = run_agent_session_turn( session_runtime, AgentTurnRequest { @@ -3525,12 +3676,24 @@ async fn run_python_agent_turn( workflow_owned_thread, idle_timeout_ms, max_duration_ms, + model, + provider, + reasoning, }, ) .await?; serde_json::to_value(result).map_err(WorkflowRuntimeError::from) } +/// Returns the first arg key that holds a non-empty (trimmed) string, owned. +fn first_str_arg(args: &Value, keys: &[&str]) -> Option { + keys.iter() + .filter_map(|key| args.get(*key).and_then(Value::as_str)) + .map(str::trim) + .find(|value| !value.is_empty()) + .map(ToOwned::to_owned) +} + fn parse_agent_harness(args: &Value) -> Result, WorkflowRuntimeError> { let Some(raw) = args .get("harness_type") @@ -3805,6 +3968,42 @@ struct AgentTurnRequest { workflow_owned_thread: bool, idle_timeout_ms: u64, max_duration_ms: u64, + // Optional per-turn model / provider / reasoning-effort overrides. When set + // they ride the execute input line exactly like the slackbot's per-turn + // `--model` / `--bedrock` / `-rsn` flags do (see slackbotv2's + // `toCodexInputLineWithStaged`), so the harness applies them to this turn; + // when `None` the deployment/baked harness default stands. `provider` and + // `reasoning` only affect the codex harness (claude/amp ignore them). + model: Option, + provider: Option, + reasoning: Option, +} + +/// Builds the single `type: "user"` execute input line for a workflow agent +/// turn, mirroring the blocks-protocol shape the harness parses +/// (`BlocksLine` in `harness-server`): optional top-level `model` / `provider` +/// / `reasoning` keys, then the `message.content` parts. api-rs enriches the +/// line with session/trace context before forwarding, so those keys are omitted +/// here. +fn agent_turn_input_line( + parts: &[Value], + model: Option<&str>, + provider: Option<&str>, + reasoning: Option<&str>, +) -> Result { + let mut line = serde_json::Map::new(); + line.insert("type".to_owned(), json!("user")); + if let Some(model) = model { + line.insert("model".to_owned(), json!(model)); + } + if let Some(provider) = provider { + line.insert("provider".to_owned(), json!(provider)); + } + if let Some(reasoning) = reasoning { + line.insert("reasoning".to_owned(), json!(reasoning)); + } + line.insert("message".to_owned(), json!({ "content": parts })); + serde_json::to_string(&Value::Object(line)) } async fn run_agent_session_turn( @@ -3824,6 +4023,9 @@ async fn run_agent_session_turn( workflow_owned_thread, idle_timeout_ms, max_duration_ms, + model, + provider, + reasoning, } = turn; let thread_key = ThreadKey::parse(thread_key)?; let mut session_metadata = session_metadata; @@ -3856,12 +4058,12 @@ async fn run_agent_session_turn( ExecuteSessionInput { idempotency_key: Some(execution_idempotency_key), metadata: Some(execution_metadata), - input_lines: vec![serde_json::to_string(&json!({ - "type": "user", - "message": { - "content": parts, - }, - }))?], + input_lines: vec![agent_turn_input_line( + &parts, + model.as_deref(), + provider.as_deref(), + reasoning.as_deref(), + )?], idle_timeout_ms: Some(idle_timeout_ms), max_duration_ms: Some(max_duration_ms), }, @@ -4078,6 +4280,47 @@ mod tests { } } + #[test] + fn agent_turn_input_line_omits_unset_harness_knobs() { + let parts = vec![json!({"type": "text", "text": "hi"})]; + let line = agent_turn_input_line(&parts, None, None, None).unwrap(); + let value: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(value.get("type"), Some(&json!("user"))); + assert_eq!(value.pointer("/message/content"), Some(&json!(parts))); + assert!(value.get("model").is_none()); + assert!(value.get("provider").is_none()); + assert!(value.get("reasoning").is_none()); + } + + #[test] + fn agent_turn_input_line_forwards_model_provider_reasoning() { + let parts = vec![json!({"type": "text", "text": "hi"})]; + let line = agent_turn_input_line( + &parts, + Some("claude-opus-4-8"), + Some("amazon-bedrock"), + Some("high"), + ) + .unwrap(); + let value: Value = serde_json::from_str(&line).unwrap(); + // Keys match the blocks-protocol shape the harness parses (BlocksLine). + assert_eq!(value.get("model"), Some(&json!("claude-opus-4-8"))); + assert_eq!(value.get("provider"), Some(&json!("amazon-bedrock"))); + assert_eq!(value.get("reasoning"), Some(&json!("high"))); + assert_eq!(value.pointer("/message/content"), Some(&json!(parts))); + } + + #[test] + fn first_str_arg_picks_first_non_empty_alias() { + let args = json!({"reasoning": " ", "reasoning_effort": " high ", "effort": "low"}); + assert_eq!( + first_str_arg(&args, &["reasoning", "reasoning_effort", "effort"]), + Some("high".to_owned()) + ); + assert_eq!(first_str_arg(&json!({}), &["model"]), None); + assert_eq!(first_str_arg(&json!({"model": " "}), &["model"]), None); + } + #[test] fn parse_worker_concurrency_uses_override_or_default() { // Override wins. @@ -4137,6 +4380,36 @@ mod tests { ); } + #[test] + fn cron_schedule_day_names_avoid_quartz_numbering() { + let named_days = normalize_schedule(json!({ + "workflow_name": "weekday_report", + "schedule_id": "named_weekdays", + "cron": "0 9 * * MON-FRI", + "timezone": "UTC", + "enabled": true, + })) + .unwrap(); + let numeric_days = normalize_schedule(json!({ + "workflow_name": "weekday_report", + "schedule_id": "numeric_days", + "cron": "0 9 * * 1-5", + "timezone": "UTC", + "enabled": true, + })) + .unwrap(); + let after_thursday = Utc.with_ymd_and_hms(2026, 7, 16, 10, 0, 0).unwrap(); + + assert_eq!( + next_schedule_time(&named_days, after_thursday).unwrap(), + Utc.with_ymd_and_hms(2026, 7, 17, 9, 0, 0).unwrap() + ); + assert_eq!( + next_schedule_time(&numeric_days, after_thursday).unwrap(), + Utc.with_ymd_and_hms(2026, 7, 19, 9, 0, 0).unwrap() + ); + } + #[test] fn interval_tick_reschedules_from_scheduled_time_without_drift() { let schedule = normalize_schedule(json!({ diff --git a/services/console/Gemfile b/services/console/Gemfile index d06ded0e5..f4844f410 100644 --- a/services/console/Gemfile +++ b/services/console/Gemfile @@ -28,6 +28,9 @@ gem "lograge" # Use Active Model has_secure_password [https://guides.rubyonrails.org/active_model_basics.html#securepassword] gem "bcrypt", "~> 3.1.7" +# JSON Web Tokens (HS256) for proxy sync, sandbox entitlement, and MCP tokens [https://github.com/jwt/ruby-jwt] +gem "jwt", "~> 3.1" + # Windows does not include zoneinfo files, so bundle the tzinfo-data gem gem "tzinfo-data", platforms: %i[ windows jruby ] diff --git a/services/console/Gemfile.lock b/services/console/Gemfile.lock index 15fdfa8b6..f2e038571 100644 --- a/services/console/Gemfile.lock +++ b/services/console/Gemfile.lock @@ -139,6 +139,8 @@ GEM hana (~> 1.3) regexp_parser (~> 2.0) simpleidn (~> 0.2) + jwt (3.2.0) + base64 language_server-protocol (3.17.0.5) lint_roller (1.1.0) logger (1.7.0) @@ -385,6 +387,7 @@ DEPENDENCIES importmap-rails jbuilder json_schemer (~> 2.3) + jwt (~> 3.1) lograge pg (~> 1.5) propshaft diff --git a/services/console/README.md b/services/console/README.md index 82aba7f82..e700edea3 100644 --- a/services/console/README.md +++ b/services/console/README.md @@ -96,7 +96,10 @@ The operator console always supports email and password sign-in. To add Google o | `CENTAUR_CONSOLE_GOOGLE_CLIENT_SECRET` | for Google | Google OAuth client secret for console login. | | `CENTAUR_CONSOLE_SLACK_CLIENT_ID` | for Slack | Slack OpenID Connect client ID for console login. | | `CENTAUR_CONSOLE_SLACK_CLIENT_SECRET` | for Slack | Slack OpenID Connect client secret for console login. | -| `CENTAUR_CONSOLE_BOOTSTRAP_ADMINS` | no | Comma- or whitespace-separated email allowlist. Matching users become active admins on first SSO login. Other SSO users become active non-admin operators and land on the console directly -- the deployment's network boundary is the access control, there is no approval queue. | +| `CENTAUR_CONSOLE_SSO_EMAIL_DOMAINS` | recommended for public exposure | Comma- or whitespace-separated domain allowlist for SSO users, for example `acme.com example.org`. Empty allows any IdP-authenticated email. | +| `CENTAUR_CONSOLE_PASSWORD_LOGIN_ENABLED` | no | Set to `false` to disable email and password sign-in. Defaults to enabled. | +| `CENTAUR_CONSOLE_PUBLIC_SLACK_THREADS_ENABLED` | no | Set to `true` to let every authenticated Console user browse public Slack channel conversations. Requires the Slack ETL channel catalog; access fails closed when it is unavailable. Private channels and DMs remain owner-only. Defaults to disabled. | +| `CENTAUR_CONSOLE_BOOTSTRAP_ADMINS` | no | Comma- or whitespace-separated email allowlist. Matching users become active admins on first SSO login. Other accepted SSO users become active non-admin operators and land on the console directly. | Register these callback URLs with the provider: diff --git a/services/console/app/assets/tailwind/application.css b/services/console/app/assets/tailwind/application.css index ce1059d30..6ee4ce74b 100644 --- a/services/console/app/assets/tailwind/application.css +++ b/services/console/app/assets/tailwind/application.css @@ -44,8 +44,11 @@ @apply mt-1 text-sm text-zinc-500; } + /* Stacks the title over its action buttons on narrow screens so long action + labels (e.g. "Add Principal" + "Cancel") stay fully on-screen; reverts to + the title/actions row from the sm breakpoint up. */ .console-page-header { - @apply mb-6 flex min-h-11 items-start justify-between gap-4; + @apply mb-6 flex min-h-11 flex-col gap-3 sm:flex-row sm:items-start sm:justify-between sm:gap-4; } .console-page-heading { diff --git a/services/console/app/controllers/api/base_controller.rb b/services/console/app/controllers/api/base_controller.rb index cfe528ea3..037281bc9 100644 --- a/services/console/app/controllers/api/base_controller.rb +++ b/services/console/app/controllers/api/base_controller.rb @@ -1,10 +1,10 @@ module Api class BaseController < ActionController::API + include ApiRequestSupport + before_action :authenticate_api_key! rescue_from ActiveRecord::RecordNotFound, with: :render_not_found - rescue_from ActionController::ParameterMissing, with: :render_bad_request - rescue_from ActionController::BadRequest, with: :render_bad_request attr_reader :current_api_key @@ -17,31 +17,18 @@ def current_user def authenticate_api_key! token = bearer_token @current_api_key = ApiKey.find_by_token(token) if token.present? - return if @current_api_key - - render_error(status: :unauthorized, message: "invalid or missing API key") - end - - def bearer_token - header = request.headers["Authorization"].to_s - return nil unless header.start_with?("Bearer ") - header.sub(/\ABearer\s+/, "").presence - end + unless current_api_key&.user&.active? + return render_error(status: :unauthorized, message: "invalid or missing API key") + end + return if current_api_key.user.admin? - def render_error(status:, message:, details: nil) - body = { error: { message: message } } - body[:error][:details] = details if details - render status: status, json: body + render_error(status: :forbidden, message: "API key owner is not an admin") end def render_not_found(e) render_error(status: :not_found, message: e.message) end - def render_bad_request(e) - render_error(status: :bad_request, message: e.message) - end - def render_validation_error(record) render_error(status: :unprocessable_entity, message: "validation failed", details: record.errors.as_json) diff --git a/services/console/app/controllers/api/proxy_base_controller.rb b/services/console/app/controllers/api/proxy_base_controller.rb index 940f96f33..7b657cc58 100644 --- a/services/console/app/controllers/api/proxy_base_controller.rb +++ b/services/console/app/controllers/api/proxy_base_controller.rb @@ -3,10 +3,9 @@ module Api # Api::BaseController (which authenticates a user-owned ApiKey), this # authenticates a Proxy's bearer token. class ProxyBaseController < ActionController::API - before_action :authenticate_proxy! + include ApiRequestSupport - rescue_from ActionController::ParameterMissing, with: :render_bad_request - rescue_from ActionController::BadRequest, with: :render_bad_request + before_action :authenticate_proxy! attr_reader :current_proxy @@ -19,21 +18,5 @@ def authenticate_proxy! render_error(status: :unauthorized, message: "invalid or missing proxy token") end - - def bearer_token - header = request.headers["Authorization"].to_s - return nil unless header.start_with?("Bearer ") - header.sub(/\ABearer\s+/, "").presence - end - - def render_error(status:, message:, details: nil) - body = { error: { message: message } } - body[:error][:details] = details if details - render status: status, json: body - end - - def render_bad_request(e) - render_error(status: :bad_request, message: e.message) - end end end diff --git a/services/console/app/controllers/api/v1/proxy_sync_controller.rb b/services/console/app/controllers/api/v1/proxy_sync_controller.rb index 286fb8de1..e1a9546a0 100644 --- a/services/console/app/controllers/api/v1/proxy_sync_controller.rb +++ b/services/console/app/controllers/api/v1/proxy_sync_controller.rb @@ -19,7 +19,9 @@ module V1 # yet. Each secret still carries its own per-secret `rules`. class ProxySyncController < Api::ProxyBaseController def create - snapshot = current_proxy.sync_config_snapshot + snapshot = current_proxy.sync_config_snapshot( + sandbox_entitlements_hosts: sandbox_entitlements_hosts + ) current_hash = snapshot[:config_hash] if params[:config_hash].presence == current_hash @@ -39,6 +41,16 @@ def create } end end + + private + + # Host for the entitlements secret's injection rule. It must match the + # host sandboxes dial, and that comes from the control URL: api-rs gets + # it as IRON_CONTROL_URL and the chart wires the same value here as + # CENTAUR_CONSOLE_URL, so both sides share one source of truth. + def sandbox_entitlements_hosts + [ Principal.host_from_url(ENV["CENTAUR_CONSOLE_URL"]) ] + end end end end diff --git a/services/console/app/controllers/api/v1/sandbox_oauth_apps_controller.rb b/services/console/app/controllers/api/v1/sandbox_oauth_apps_controller.rb new file mode 100644 index 000000000..994260f99 --- /dev/null +++ b/services/console/app/controllers/api/v1/sandbox_oauth_apps_controller.rb @@ -0,0 +1,51 @@ +module Api + module V1 + class SandboxOauthAppsController < ActionController::API + include ApiRequestSupport + + before_action :authenticate_sandbox_jwt! + + def index + apps = OauthApp.where(enabled: true).order(:slug, :id) + render json: { data: apps.map { |app| app_payload(app) } } + end + + private + + def authenticate_sandbox_jwt! + token = bearer_token + if token.blank? + return render_error(status: :unauthorized, message: "invalid or missing sandbox token") + end + + # This endpoint only needs to prove the caller has the same signed + # sandbox entitlement used by /sandbox/permissions. The listed start URLs + # are public OAuth entrypoints, so no proxy/principal claim check is + # needed here. + SandboxEntitlements::Jwt.decode(token) + rescue CentaurJwt::Hs256::VerificationError + render_error(status: :unauthorized, message: "invalid or missing sandbox token") + end + + def app_payload(app) + { + id: app.oid, + slug: app.slug, + description: app.description, + labels: app.labels, + provider: app.provider, + allowed_scopes: app.allowed_scopes, + start_url: oauth_start_url_for(app) + } + end + + def oauth_start_url_for(app) + URI.join(public_base_url, "/oauth/#{app.slug}/start").to_s + end + + def public_base_url + ConsoleEnv["PUBLIC_URL"].presence || request.base_url + end + end + end +end diff --git a/services/console/app/controllers/api/v1/sandbox_permissions_controller.rb b/services/console/app/controllers/api/v1/sandbox_permissions_controller.rb new file mode 100644 index 000000000..b83df4125 --- /dev/null +++ b/services/console/app/controllers/api/v1/sandbox_permissions_controller.rb @@ -0,0 +1,102 @@ +module Api + module V1 + class SandboxPermissionsController < ActionController::API + include ApiRequestSupport + + before_action :authenticate_sandbox_token! + + def show + principal = current_proxy.principal + unless principal + return render_error(status: :unauthorized, message: "sandbox token is no longer assigned") + end + + # Redacting the cached snapshot is equivalent to + # principal.effective_config (the snapshot stores the unredacted + # config) but skips the expensive per-request grant rebuild, under + # the same freshness model the proxy sync path accepts. + permissions = Principal.redact_live_secrets( + PrincipalSyncConfigSnapshot.fetch_for(principal).payload + ) + body = { + data: { + sandbox_id: sandbox_claims.fetch("sandbox_id"), + proxy_id: current_proxy.oid, + principal_id: principal.oid, + principal: principal_payload(principal), + capabilities: capabilities_payload(principal), + slack_channel_permissions: principal.slack_channel_permissions_payload, + oauth_credentials: oauth_credentials_payload(principal), + permissions: permissions + } + }.to_json + + response.headers["ETag"] = %("#{Digest::SHA256.hexdigest(body)}") + response.headers["Cache-Control"] = "no-store" + render json: body + end + + private + + attr_reader :current_proxy, :sandbox_claims + + def authenticate_sandbox_token! + token = bearer_token + if token.blank? + return render_error(status: :unauthorized, message: "invalid or missing sandbox token") + end + + # KeyError (signing secret unconfigured) is deliberately not rescued: + # that is a server fault and should surface as a 500, not a 401. + claims = SandboxEntitlements::Jwt.decode(token) + proxy = Proxy.find_by_oid(claims["proxy_id"]) + unless proxy&.assigned? && proxy.principal&.oid == claims["principal_id"] && + proxy.name == claims["sandbox_id"] + return render_error(status: :unauthorized, message: "invalid sandbox token") + end + + @sandbox_claims = claims + @current_proxy = proxy + rescue CentaurJwt::Hs256::VerificationError + render_error(status: :unauthorized, message: "invalid or missing sandbox token") + end + + def principal_payload(principal) + { + id: principal.oid, + namespace: principal.namespace, + foreign_id: principal.foreign_id, + name: principal.name + } + end + + def capabilities_payload(principal) + { + sandbox_repo_cache: principal.sandbox_repo_cache, + sandbox_observability_enabled: principal.sandbox_observability_enabled, + sandbox_api_server_enabled: principal.sandbox_api_server_enabled + } + end + + def oauth_credentials_payload(principal) + principal.granted_static_secrets + .includes(broker_credential: :oauth_app) + .filter_map(&:broker_credential) + .select(&:oauth_app) + .sort_by { |credential| [ credential.oauth_app.slug, credential.provider_email.to_s, credential.id ] } + .map do |credential| + { + id: credential.oid, + oauth_app_id: credential.oauth_app.oid, + slug: credential.oauth_app.slug, + provider: credential.oauth_app.provider, + provider_email: credential.provider_email, + provider_subject: credential.provider_subject, + status: credential.status, + scopes: credential.scopes + } + end + end + end + end +end diff --git a/services/console/app/controllers/application_controller.rb b/services/console/app/controllers/application_controller.rb index 71581ff7c..bb36b4e11 100644 --- a/services/console/app/controllers/application_controller.rb +++ b/services/console/app/controllers/application_controller.rb @@ -9,7 +9,7 @@ class ApplicationController < ActionController::Base # controllers don't each hand-roll a rescue. Mirrors Api::BaseController. rescue_from ActiveRecord::RecordNotFound, with: :render_not_found - helper_method :current_user, :acting_admin?, :descoped? + helper_method :current_user, :acting_admin?, :descoped?, :password_login_enabled? helper_method :public_base_url, :oauth_callback_redirect_uri # The public origin the console is reached at. Derived from the request by @@ -76,6 +76,10 @@ def descoped? false end + def password_login_enabled? + ConsoleAuth.password_login_enabled? + end + # The permission check console gates use instead of current_user.admin?: a # real admin who is not currently descoped. Keeping current_user untouched # means audit trails and data displays still see the true account. @@ -194,6 +198,10 @@ def console_sidebar_visible_thread_scope console_sidebar_console_thread_owner_sql, (console_sidebar_slack_thread_owner_sql(slack_owners) if slack_owners.any?) ].compact + if CentaurSession.public_slack_threads_enabled? + public_slack_sql = CentaurSession.public_slack_channel_sql + conditions << public_slack_sql if public_slack_sql + end return CentaurSession.where("1=0") if conditions.empty? @@ -209,10 +217,13 @@ def console_sidebar_direct_selected_threads(threads) thread_keys = console_sidebar_selected_thread_keys - threads.map(&:thread_key) return [] if thread_keys.empty? - # Resolve through the owner scope, not a raw find_by, so a directly linked - # thread only surfaces in the sidebar when the current user started it. This - # mirrors Console::ThreadsController#selected_session. - console_sidebar_visible_thread_scope.where(thread_key: thread_keys).to_a + visible = console_sidebar_visible_thread_scope.where(thread_key: thread_keys).to_a + missing_keys = thread_keys - visible.map(&:thread_key) + shared_keys = ThreadShare.where(thread_key: missing_keys).pluck(:thread_key) + shared = CentaurSession.where(thread_key: shared_keys).to_a + sessions_by_key = (visible + shared).index_by(&:thread_key) + + thread_keys.filter_map { |thread_key| sessions_by_key[thread_key] } end # The thread param carries up to PANEL_LIMIT comma-separated keys when the diff --git a/services/console/app/controllers/concerns/api_request_support.rb b/services/console/app/controllers/concerns/api_request_support.rb new file mode 100644 index 000000000..545235566 --- /dev/null +++ b/services/console/app/controllers/concerns/api_request_support.rb @@ -0,0 +1,29 @@ +# Shared plumbing for JSON API controllers: bearer-token extraction and the +# error envelope. Each including controller supplies its own authentication +# scheme on top (user ApiKey, proxy bearer token, sandbox entitlement JWT). +module ApiRequestSupport + extend ActiveSupport::Concern + + included do + rescue_from ActionController::ParameterMissing, with: :render_bad_request + rescue_from ActionController::BadRequest, with: :render_bad_request + end + + private + + def bearer_token + header = request.headers["Authorization"].to_s + return nil unless header.start_with?("Bearer ") + header.sub(/\ABearer\s+/, "").presence + end + + def render_error(status:, message:, details: nil) + body = { error: { message: message } } + body[:error][:details] = details if details + render status: status, json: body + end + + def render_bad_request(e) + render_error(status: :bad_request, message: e.message) + end +end diff --git a/services/console/app/controllers/console/threads_controller.rb b/services/console/app/controllers/console/threads_controller.rb index a7538c192..7bb5b7490 100644 --- a/services/console/app/controllers/console/threads_controller.rb +++ b/services/console/app/controllers/console/threads_controller.rb @@ -133,7 +133,8 @@ class Console::ThreadsController < ApplicationController :composer_agent_choices, :composer_default_agent_value, :composer_agents_json, - :thread_execution_active? + :thread_execution_active?, + :thread_owned? def index @query = params[:q].to_s.strip @@ -189,6 +190,65 @@ def sidebar render partial: "console/threads/sidebar_threads", layout: false end + # Single-panel transcript refresh, polled by thread_poller_controller.js + # while a turn is running in that panel. Renders only the panel's transcript + # stream (no layout) so an active thread never drags the rest of the console + # — other panes, composers, drafts — through a full Turbo visit. Resolves the + # key through the same readable scope as the page render. + def panel + thread_key = params[:thread_key].to_s.strip + session = readable_thread(thread_key) + if session.nil? + head :not_found + return + end + + @latest_executions = latest_executions_for([ session.thread_key ]) + panel = thread_panel_for(session) + active = thread_execution_active?(session.thread_key) + # The poller stops rescheduling once this header reports the turn is done, + # after swapping in the final transcript below. + response.set_header("X-Console-Execution-Active", active.to_s) + render partial: "console/threads/panel_transcript", + locals: { items: panel[:transcript_items], active: active }, + layout: false + rescue ActiveRecord::ActiveRecordError, PG::Error => e + Rails.logger.warn("console_threads_panel_refresh_failed error=#{e.class}: #{e.message}") + head :service_unavailable + end + + # Publishes a chat inside the authenticated Console boundary. Publication is + # stored in Console's own database rather than mutating api-rs session data; + # the Threads surface remains an observer of the durable transcript. + def share + thread_key = params[:thread_key].to_s.strip + session = owned_thread_scope.where(thread_key: thread_key).first + if session.nil? + respond_to do |format| + format.html { redirect_to console_threads_path, alert: "Chat not found." } + format.json { render json: { error: "Chat not found." }, status: :not_found } + end + return + end + + ThreadShare.create_or_find_by!(thread_key: session.thread_key) do |share| + share.created_by = current_user + end + share_url = console_threads_url(thread: session.thread_key) + respond_to do |format| + format.html do + redirect_to console_threads_path(thread: session.thread_key) + end + format.json { render json: { url: share_url } } + end + rescue ActiveRecord::ActiveRecordError, PG::Error => e + Rails.logger.warn("console_thread_share_failed error=#{e.class}: #{e.message}") + respond_to do |format| + format.html { redirect_to console_threads_path, alert: "Could not share the chat." } + format.json { render json: { error: "Could not share the chat." }, status: :service_unavailable } + end + end + private def api_client @@ -202,6 +262,15 @@ def thread_execution_active?(thread_key) execution.present? && %w[queued running executing].include?(execution.status.to_s) end + # Public and explicitly shared chats are read-only for non-owners. Keep the + # composer and write endpoint tied to the original owner scope. + def thread_owned?(session) + @thread_owned ||= {} + @thread_owned.fetch(session.thread_key) do |thread_key| + @thread_owned[thread_key] = owned_thread_scope.where(thread_key: thread_key).exists? + end + end + # Selector options as [label, value] pairs, the deploy's default model # first (pre-checked in the menu). The default comes from the same # env/config resolution the thread header uses, so the composer never @@ -267,8 +336,8 @@ def start_thread(prompt) def reply_to_thread(thread_key, prompt) # Resolve through the owner scope so a crafted thread_key cannot post into - # another user's chat — same rule the read side applies to ?thread=. - session = visible_thread_scope.where(thread_key: thread_key).first + # another user's chat, even when public or shared read access is allowed. + session = owned_thread_scope.where(thread_key: thread_key).first if session.nil? redirect_to console_threads_path, alert: "Chat not found." return @@ -329,13 +398,48 @@ def composer_input_line(thread_key, prompt, model:, effort:, client_message_id:) thread_key: thread_key, client_user_message_id: client_message_id, trace_metadata: { action: "execute", source: "console" }, - message: { role: "user", content: [ { type: "text", text: prompt } ] } + message: { + role: "user", + content: [ + { type: "text", text: console_requester_context }, + { type: "text", text: prompt } + ] + } } line[:model] = model if model.present? line[:reasoning] = effort if effort.present? line.to_json end + # Resolve the signed-in human through the same Slack profile custom-field + # path as slackbotv2, falling back to their Console display name/email. Keep + # this separate from the persisted prompt: it is harness execution context. + def console_requester_context + github_identity = SlackRequesterIdentity.resolve( + user_ids: slack_thread_owners_for_current_user.map(&:user_id) + ) + prompted_by = github_identity.handle.presence || + (current_user&.name.to_s.strip.presence || current_user&.email.to_s) + github_status = github_identity.handle.present? ? + "GitHub handle source: #{github_identity.source}\nGitHub handle verified: yes" : + "GitHub handle verified: no\nGitHub handle unavailable reason: #{github_identity.reason}" + <<~CONTEXT.strip + # Requester Context + + The Console user who prompted this turn is #{prompted_by}. + + ## GitHub PR Attribution + + If you create a GitHub PR for this request, the PR body MUST contain this standalone line: + Prompted by: #{prompted_by} + + #{github_status} + + The user message follows in the next content block. + --- + CONTEXT + end + # Follow-ups reuse the model the chat has been running on (mirrors the # display resolution in thread_model_label, minus the upcasing): last # execution's recorded model, session metadata, then the deploy default. @@ -425,11 +529,12 @@ def selected_session(session_scope, base_sessions) if @selected_thread_key.present? selected = base_sessions.find { |session| session.thread_key == @selected_thread_key } - # Resolve the key through the owner scope so a directly linked chat only - # loads when the current user started it. base_sessions is capped at - # THREAD_LIMIT, so this also recovers an owned thread beyond that window. + # Resolve the key through the readable scope so a directly linked chat only + # loads when it is visible to the current user. base_sessions is capped at + # THREAD_LIMIT, so this also recovers a visible thread beyond that window. selected ||= session_scope.where(thread_key: @selected_thread_key).first - # A directly requested key outside the owner scope renders as 404 rather + selected ||= explicitly_shared_thread(@selected_thread_key) + # A directly requested key outside the readable scope renders as 404 rather # than silently falling back to another chat, so nonexistent and # inaccessible chats are indistinguishable to the viewer. @thread_not_found = selected.nil? @@ -449,14 +554,14 @@ def requested_thread_keys params[:thread].to_s.split(",").map(&:strip).reject(&:blank?).uniq.first(PANEL_LIMIT) end - # Extra split-view panes resolve through the same owner scope as the primary - # thread, so a crafted ?thread= list cannot surface another user's thread. - # Unowned keys are dropped silently. + # Extra split-view panes resolve through the same readable scope as the + # primary thread. Inaccessible keys are dropped silently. def resolve_pane_sessions(session_scope, base_sessions) keys = @pane_thread_keys - [ @selected_session&.thread_key ] keys.filter_map do |key| base_sessions.find { |session| session.thread_key == key } || - session_scope.where(thread_key: key).first + session_scope.where(thread_key: key).first || + explicitly_shared_thread(key) end end @@ -493,6 +598,7 @@ def thread_panel_for(session) { session: session, thread_key: session.thread_key, + writable: thread_owned?(session), transcript_items: selected_transcript_items } end @@ -522,17 +628,41 @@ def load_selected_session_summaries(loaded_keys) end def visible_thread_scope + thread_scope(include_public_slack: true) + end + + def owned_thread_scope + thread_scope(include_public_slack: false) + end + + def thread_scope(include_public_slack:) slack_owners = slack_thread_owners_for_current_user conditions = [ console_thread_owner_sql, (slack_thread_owner_sql(slack_owners) if slack_owners.any?) ].compact + if include_public_slack && CentaurSession.public_slack_threads_enabled? + public_slack_sql = CentaurSession.public_slack_channel_sql + conditions << public_slack_sql if public_slack_sql + end return CentaurSession.where("1=0") if conditions.empty? CentaurSession.where(conditions.map { |condition| "(#{condition})" }.join(" OR ")) end + def readable_thread(thread_key) + return if thread_key.blank? + + visible_thread_scope.where(thread_key: thread_key).first || explicitly_shared_thread(thread_key) + end + + def explicitly_shared_thread(thread_key) + return unless ThreadShare.exists?(thread_key: thread_key) + + CentaurSession.where(thread_key: thread_key).first + end + def console_thread_owner_sql email = normalize_email(current_user&.email) return if email.blank? diff --git a/services/console/app/controllers/oauth/flows_controller.rb b/services/console/app/controllers/oauth/flows_controller.rb index fc45497a8..05324b34c 100644 --- a/services/console/app/controllers/oauth/flows_controller.rb +++ b/services/console/app/controllers/oauth/flows_controller.rb @@ -9,22 +9,18 @@ module Oauth # BrokerCredential linked to the OauthApp, then sends the user back to the # console Integrations page (or renders a result page on failure). # - # Deliberately unauthenticated -- a team member connects an integration by - # clicking a well-known link; there is no external app to integrate with, so - # there is no return_to or user key. Safety comes from: a credential is only - # minted after a successful consent + code exchange, and re-consent for the - # same (app, provider account) upserts the existing credential. All - # provider-specific behavior comes from the strategy (Oauth::Providers), which - # is derived from the app. + # Authenticated console consent flow. A team member connects an integration by + # clicking a console link; a credential is only minted after an active console + # session, successful consent, and code exchange. Re-consent for the same (app, + # provider account) upserts the existing credential. All provider-specific + # behavior comes from the strategy (Oauth::Providers), which is derived from + # the app. # # SECURITY: never logs the code, tokens, client_secret, or response bodies -- # only oids and error codes, like the rest of the Broker/Oauth subsystem. class FlowsController < ApplicationController layout "auth" - skip_before_action :require_login - skip_before_action :require_active_account - # The message_verifier purpose binding the signed state to this flow, the # state/cookie lifetime, and the encrypted cookie that ties a callback back to # the browser that started it. @@ -87,6 +83,7 @@ def callback end result = exchange_code(params[:code], flow["code_verifier"]) + validate_provider_result!(result) identity = @provider.identity_from(result, client_id: @app.client_id) @credential = upsert_credential(state, result, identity) enqueue_identity_enrichment(@credential) @@ -151,23 +148,22 @@ def exchange_code(code, code_verifier) code: code.to_s, redirect_uri: oauth_callback_redirect_uri(@app.slug), code_verifier: code_verifier.to_s, - require_refresh_token: @provider.refreshable? + require_refresh_token: provider_requires_refresh_token? ) end # Upserts one credential per (app, provider account). A new record gets its - # identity/endpoint fixed (and an auto-generated external_user_key, since the - # flow has no caller-supplied user); every consent (re)applies the rotating - # blob, including the freshly-exchanged access token so the credential is live - # immediately, and revives a dead credential. + # identity/endpoint fixed (and an auto-generated external_user_key); every + # consent (re)applies the rotating blob, including the freshly-exchanged + # access token so the credential is live immediately, and revives a dead + # credential. def upsert_credential(state, result, identity) BrokerCredential.transaction do credential = BrokerCredential.find_or_initialize_by(oauth_app: @app, provider_subject: identity[:subject]) - # When the consenting browser carries a signed-in console session, - # remember which user connected this account. The Integrations page - # matches on it, so the card flips to "Connected" even when the - # provider account's email differs from the console login email. - # Never overwritten: the first linked user keeps the credential. + # Remember which user connected this account. The Integrations page + # matches on it, so the card flips to "Connected" even when the provider + # account's email differs from the console login email. Never + # overwritten: the first linked user keeps the credential. credential.created_by ||= current_user if credential.new_record? credential.namespace = @app.credential_namespace @@ -178,7 +174,7 @@ def upsert_credential(state, result, identity) end now = Time.current - expires_in = result.expires_in&.positive? ? result.expires_in : BrokerCredential::DEFAULT_EXPIRES_IN_SECONDS + refreshable_result = provider_refreshable_result?(result) credential.assign_attributes( provider_email: identity[:email], # Store exactly what the IdP granted, so the refresh POST re-requests it. @@ -186,17 +182,40 @@ def upsert_credential(state, result, identity) labels: credential_labels(credential, identity), refresh_token: result.refresh_token, access_token: result.access_token, - expires_at: now + expires_in, + expires_at: credential_expires_at(result, now: now, refreshable_result: refreshable_result), last_refresh: now, failure_count: 0, dead: false, dead_reason: nil ) - credential.next_attempt_at = @provider.refreshable? ? credential.compute_next_attempt_at(now: now) : nil + credential.next_attempt_at = refreshable_result ? credential.compute_next_attempt_at(now: now) : nil credential.save! ensure_wrapping_secret(credential) credential end end + def validate_provider_result!(result) + @provider.validate_result!(result) if @provider.respond_to?(:validate_result!) + end + + def provider_requires_refresh_token? + return @provider.require_refresh_token? if @provider.respond_to?(:require_refresh_token?) + + @provider.refreshable? + end + + def provider_refreshable_result?(result) + return @provider.refreshable_result?(result) if @provider.respond_to?(:refreshable_result?) + + @provider.refreshable? + end + + def credential_expires_at(result, now:, refreshable_result:) + return now + result.expires_in if result.expires_in&.positive? + return nil unless refreshable_result + + now + BrokerCredential::DEFAULT_EXPIRES_IN_SECONDS + end + def granted_scopes(result, state) return Array(state["scopes"]) if result.scope.blank? @provider.parse_granted_scopes(result.scope) diff --git a/services/console/app/controllers/session_oauth_controller.rb b/services/console/app/controllers/session_oauth_controller.rb index 56ae5ad6f..8564c4ec7 100644 --- a/services/console/app/controllers/session_oauth_controller.rb +++ b/services/console/app/controllers/session_oauth_controller.rb @@ -71,6 +71,9 @@ def callback rescue Broker::ExchangeError => e Rails.logger.error { "console login exchange failed (#{@key}): #{e.reason}" } redirect_to login_path, alert: "Sign in failed. Please try again." + rescue User::SsoEmailDomainNotAllowed + Rails.logger.warn { "console login rejected by SSO email domain allowlist (#{@key})" } + redirect_to login_path, alert: "That email domain is not allowed to access the console." rescue ActiveRecord::RecordInvalid => e Rails.logger.error { "console login provisioning failed: #{e.record.errors.full_messages.to_sentence}" } redirect_to login_path, alert: "Sign in failed while setting up your account." diff --git a/services/console/app/controllers/sessions_controller.rb b/services/console/app/controllers/sessions_controller.rb index 3e56e73ab..6aeb57172 100644 --- a/services/console/app/controllers/sessions_controller.rb +++ b/services/console/app/controllers/sessions_controller.rb @@ -1,7 +1,8 @@ # Quick session-cookie login for the operator console. Authenticates an existing # User (has_secure_password) by email + password and stores their id in the -# session. No registration, password reset, or rate limiting: this is an internal -# gate, not a public auth system. +# session. No registration, password reset, or rate limiting: deployments that +# expose the console publicly should disable this break-glass path with +# CENTAUR_CONSOLE_PASSWORD_LOGIN_ENABLED=false. class SessionsController < ApplicationController layout "auth" @@ -24,6 +25,11 @@ def pending end def create + unless password_login_enabled? + flash.now[:alert] = "Email and password sign in is disabled." + return render :new, status: :not_found + end + user = User.find_by(email: params[:email].to_s.strip.downcase) unless user&.authenticate(params[:password]) flash.now[:alert] = "Invalid email or password." diff --git a/services/console/app/helpers/application_helper.rb b/services/console/app/helpers/application_helper.rb index 8aa7bddbc..ba02e44f9 100644 --- a/services/console/app/helpers/application_helper.rb +++ b/services/console/app/helpers/application_helper.rb @@ -188,6 +188,11 @@ def console_icon(name, classes: "size-4") classes, "M12 3.75 19.5 6v5.25c0 4.207-2.765 8.04-7.5 9-4.735-.96-7.5-4.793-7.5-9V6L12 3.75Zm3.75 6-4.5 4.5-2.25-2.25" ) + when "share" + outline_icon( + classes, + "M12 16.5V3m0 0L7.5 7.5M12 3l4.5 4.5M6.75 10.5h-.75A2.25 2.25 0 0 0 3.75 12.75v6A2.25 2.25 0 0 0 6 21h12a2.25 2.25 0 0 0 2.25-2.25v-6A2.25 2.25 0 0 0 18 10.5h-.75" + ) when "slack" tag.svg( tag.path( @@ -220,6 +225,8 @@ def console_icon(name, classes: "size-4") classes, "M9.75 10.5a3.75 3.75 0 1 1 7.5 0 3.75 3.75 0 0 1-7.5 0ZM4.5 18.75a6.75 6.75 0 0 1 13.5 0M18 8.25a3 3 0 0 1 0 6M19.5 18.75a5.25 5.25 0 0 0-2.25-4.307" ) + when "menu" + outline_icon(classes, "M3.75 6.75h16.5M3.75 12h16.5M3.75 17.25h16.5") end end diff --git a/services/console/app/javascript/controllers/thread_poller_controller.js b/services/console/app/javascript/controllers/thread_poller_controller.js new file mode 100644 index 000000000..cd7ee88ad --- /dev/null +++ b/services/console/app/javascript/controllers/thread_poller_controller.js @@ -0,0 +1,78 @@ +import { Controller } from "@hotwired/stimulus" + +// Polls one thread panel's transcript while its turn is running and swaps only +// that panel's transcript stream in place (ThreadsController#panel). Replaces +// the old whole-page Turbo refresh, so a running thread never re-renders the +// other panes, their composers, or in-progress drafts. +export default class extends Controller { + static targets = ["transcript"] + static values = { + url: String, + active: Boolean, + interval: { type: Number, default: 4000 } + } + + connect() { + // A composer submit hands control to the server-side redirect; polling in + // that window could paint a pre-submit transcript over the optimistic + // user bubble (see _composer.html.erb). + this.section = this.element.closest("section") + this.onSubmit = () => this.cancel() + this.section?.addEventListener("submit", this.onSubmit) + if (this.activeValue) this.schedule() + } + + disconnect() { + this.section?.removeEventListener("submit", this.onSubmit) + this.cancel() + } + + schedule() { + this.cancel() + this.timer = window.setTimeout(() => this.refresh(), this.intervalValue) + } + + cancel() { + if (this.timer) window.clearTimeout(this.timer) + this.timer = null + } + + async refresh() { + try { + const response = await fetch(this.urlValue, { + credentials: "same-origin", + headers: { "Accept": "text/html" } + }) + if (response.ok) { + const active = response.headers.get("X-Console-Execution-Active") === "true" + this.swap(await response.text()) + // The final transcript (turn result included) just rendered; a + // composer submit re-renders the page and restarts the poller. + if (!active) return + } + } catch { + // Transient network error; retry on the next tick. + } + this.schedule() + } + + swap(html) { + if (html === this.lastHtml) return + this.lastHtml = html + + // Keep the reader's place: reopen the disclosures they had expanded and + // stay pinned to the bottom when they were already there. The transcript + // is append-mostly, so positional indexes are stable across swaps. + const openIndexes = new Set() + this.transcriptTarget.querySelectorAll("details").forEach((details, index) => { + if (details.open) openIndexes.add(index) + }) + const pinned = this.element.scrollHeight - this.element.scrollTop - this.element.clientHeight < 48 + + this.transcriptTarget.innerHTML = html + this.transcriptTarget.querySelectorAll("details").forEach((details, index) => { + if (openIndexes.has(index)) details.open = true + }) + if (pinned) this.element.scrollTop = this.element.scrollHeight + } +} diff --git a/services/console/app/javascript/controllers/thread_share_controller.js b/services/console/app/javascript/controllers/thread_share_controller.js new file mode 100644 index 000000000..4f152a489 --- /dev/null +++ b/services/console/app/javascript/controllers/thread_share_controller.js @@ -0,0 +1,89 @@ +import { Controller } from "@hotwired/stimulus" + +export default class extends Controller { + static targets = ["dialog", "form", "copyButton", "error"] + static values = { url: String } + + open(event) { + event.preventDefault() + this.reset() + this.dialogTarget.showModal() + } + + cancel(event) { + event.preventDefault() + this.dialogTarget.close() + } + + closeFromBackdrop(event) { + if (event.target !== this.dialogTarget) return + + const bounds = this.dialogTarget.getBoundingClientRect() + const inside = event.clientX >= bounds.left && event.clientX <= bounds.right && + event.clientY >= bounds.top && event.clientY <= bounds.bottom + if (!inside) this.dialogTarget.close() + } + + async copyLink(event) { + event.preventDefault() + if (this.copyButtonTarget.disabled) return + + this.copyButtonTarget.disabled = true + this.copyButtonTarget.textContent = "Copying…" + this.errorTarget.hidden = true + + try { + // Invoke clipboard access directly from the click gesture. Only publish + // after copying succeeds so a denied clipboard permission does not make + // the chat public as a side effect. + try { + await this.writeToClipboard(this.urlValue) + } catch { + throw new Error("Could not copy the link.") + } + const response = await fetch(this.formTarget.action, { + method: "POST", + body: new FormData(this.formTarget), + credentials: "same-origin", + headers: { + "Accept": "application/json", + "X-CSRF-Token": document.querySelector("meta[name='csrf-token']")?.content ?? "" + } + }) + if (!response.ok) { + const payload = await response.json().catch(() => ({})) + throw new Error(payload.error || "Could not share the chat.") + } + + this.copyButtonTarget.textContent = "Copied" + window.setTimeout(() => this.dialogTarget.close(), 600) + } catch (error) { + this.errorTarget.textContent = error.message || "Could not copy the link." + this.errorTarget.hidden = false + this.copyButtonTarget.disabled = false + this.copyButtonTarget.textContent = "Copy link" + } + } + + writeToClipboard(text) { + if (navigator.clipboard?.writeText) return navigator.clipboard.writeText(text) + + const input = document.createElement("textarea") + input.value = text + input.setAttribute("readonly", "") + input.style.position = "fixed" + input.style.opacity = "0" + document.body.appendChild(input) + input.select() + const copied = document.execCommand("copy") + input.remove() + return copied ? Promise.resolve() : Promise.reject(new Error("Could not copy the link.")) + } + + reset() { + this.copyButtonTarget.disabled = false + this.copyButtonTarget.textContent = "Copy link" + this.errorTarget.hidden = true + this.errorTarget.textContent = "" + } +} diff --git a/services/console/app/jobs/granola/poll_sync_job.rb b/services/console/app/jobs/granola/poll_sync_job.rb new file mode 100644 index 000000000..973f87fef --- /dev/null +++ b/services/console/app/jobs/granola/poll_sync_job.rb @@ -0,0 +1,23 @@ +module Granola + class PollSyncJob < ApplicationJob + queue_as :default + + def perform(oauth_app_slug = Granola::SyncCredential.oauth_app_slug) + credentials = BrokerCredential + .includes(:oauth_app) + .joins(:oauth_app) + .where(dead: false) + .where(oauth_apps: { + provider: Oauth::Providers::Granola::KEY, + slug: oauth_app_slug, + enabled: true + }) + + credentials.find_each do |credential| + next unless Granola::SyncCredential.syncable?(credential, oauth_app_slug: oauth_app_slug) + + Granola::SyncCredentialJob.perform_later(credential.id) + end + end + end +end diff --git a/services/console/app/jobs/granola/sync_credential_job.rb b/services/console/app/jobs/granola/sync_credential_job.rb new file mode 100644 index 000000000..8a77f90b4 --- /dev/null +++ b/services/console/app/jobs/granola/sync_credential_job.rb @@ -0,0 +1,12 @@ +module Granola + class SyncCredentialJob < ApplicationJob + queue_as :default + + def perform(credential_id) + credential = BrokerCredential.includes(:oauth_app).find_by(id: credential_id) + return unless Granola::SyncCredential.syncable?(credential) + + Granola::SyncCredential.new(credential).call + end + end +end diff --git a/services/console/app/models/centaur_session.rb b/services/console/app/models/centaur_session.rb index fdfd27e84..8eaf79128 100644 --- a/services/console/app/models/centaur_session.rb +++ b/services/console/app/models/centaur_session.rb @@ -2,6 +2,12 @@ class CentaurSession < CentaurSessionRecord self.table_name = "sessions" self.primary_key = "thread_key" + SLACK_SOURCE_SQL = <<~SQL.squish.freeze + thread_key LIKE 'slack:%' + OR metadata ->> 'platform' = 'slack' + OR metadata ->> 'source' = 'slackbotv2' + SQL + has_many :messages, class_name: "CentaurSessionMessage", foreign_key: :thread_key, @@ -20,6 +26,33 @@ class CentaurSession < CentaurSessionRecord scope :recent_first, -> { order(Arel.sql("coalesce(updated_at, created_at) desc"), :thread_key) } + def self.public_slack_threads_enabled? + ActiveModel::Type::Boolean.new.cast(ConsoleEnv.fetch("PUBLIC_SLACK_THREADS_ENABLED", false)) + end + + # Slack channel ID prefixes do not encode privacy: modern private channels + # can also start with C. Treat the synchronized channel catalog as a positive + # public allowlist, and fail closed until that catalog is available. + def self.public_slack_channel_sql + required_columns = %i[is_private is_syncable] + return unless connection.data_source_exists?(:slack_sync_channels) + return unless required_columns.all? { |column| connection.column_exists?(:slack_sync_channels, column) } + + <<~SQL.squish + (#{SLACK_SOURCE_SQL}) + AND EXISTS ( + SELECT 1 + FROM slack_sync_channels + WHERE slack_sync_channels.is_private = false + AND slack_sync_channels.is_syncable = true + AND slack_sync_channels.channel_id IN ( + split_part(sessions.thread_key, ':', 2), + split_part(sessions.thread_key, ':', 3) + ) + ) + SQL + end + def readonly? = true def metadata_hash diff --git a/services/console/app/models/mcp_oauth_client.rb b/services/console/app/models/mcp_oauth_client.rb index 38649610f..15af8c873 100644 --- a/services/console/app/models/mcp_oauth_client.rb +++ b/services/console/app/models/mcp_oauth_client.rb @@ -66,7 +66,11 @@ def scopes_supported def self.allowed_redirect_uri?(value) uri = URI.parse(value.to_s) - uri.scheme == "http" && loopback_host?(uri.host) + return false unless uri.host.present? + return false if uri.fragment.present? + return false if value.to_s.include?("*") + + uri.scheme == "https" || (uri.scheme == "http" && loopback_host?(uri.host)) rescue URI::InvalidURIError false end diff --git a/services/console/app/models/principal.rb b/services/console/app/models/principal.rb index 0a458d784..4b4c9257d 100644 --- a/services/console/app/models/principal.rb +++ b/services/console/app/models/principal.rb @@ -297,10 +297,7 @@ def slack_channel_ids_for(permission) def api_server_hosts configured = ENV["CENTAUR_API_SERVER_PROXY_HOSTS"].to_s.split(",") from_url = self.class.host_from_url(ENV["CENTAUR_API_URL"]) - (configured + [ from_url, "centaur-api-rs", "api" ]) - .map { |host| host.to_s.strip.downcase.delete_suffix(".") } - .reject(&:blank?) - .uniq + self.class.normalize_hosts(configured + [ from_url ]) end def proxy_transforms_for(served) @@ -321,6 +318,15 @@ def self.host_from_url(value) nil end + # Canonical form for hosts used in iron-proxy injection rules, so rule + # matching never hinges on case, whitespace, or a trailing dot. + def self.normalize_hosts(hosts) + Array(hosts) + .map { |host| host.to_s.strip.downcase.delete_suffix(".") } + .reject(&:blank?) + .uniq + end + # Cross-type conflict resolution. The wire protocol applies the `secrets` array # (static secrets) before the `transforms` array (gcp_auth, aws_auth, hmac_sign, # oauth_token), so the proxy's last-transform-wins cannot let a direct static diff --git a/services/console/app/models/proxy.rb b/services/console/app/models/proxy.rb index cf7645b7b..d5119b05d 100644 --- a/services/console/app/models/proxy.rb +++ b/services/console/app/models/proxy.rb @@ -3,6 +3,7 @@ class Proxy < ApplicationRecord TOKEN_PREFIX = "iprx_".freeze TOKEN_FORMAT = /\Aiprx_[0-9a-f]{64}\z/ + SANDBOX_ENTITLEMENTS_PATH_PATTERN = "/api/v1/sandbox/*".freeze attr_readonly :bearer_token_hash attr_accessor :token @@ -45,14 +46,17 @@ def sync_config principal&.effective_config(redact_secrets: false) || Principal::EMPTY_CONFIG end - def sync_config_snapshot + def sync_config_snapshot(sandbox_entitlements_hosts: []) config = principal ? PrincipalSyncConfigSnapshot.fetch_for(principal).payload : Principal::EMPTY_CONFIG + config = with_sandbox_entitlements_secret(config, sandbox_entitlements_hosts: sandbox_entitlements_hosts) { config_hash: config_hash_for(config), config: config } end - # Opaque, deterministic fingerprint of the delivered config. The proxy treats - # this as an ETag: it echoes its current hash on each sync and only re-applies - # config when the hash changes. + # Opaque, deterministic fingerprint of the base (principal-derived) config. + # Note this is not necessarily the hash the proxy echoes on sync: the sync + # path hashes the delivered config, which also folds in the per-proxy + # sandbox entitlements secret when one is configured (see + # #sync_config_snapshot). def config_hash # The principal identity and assignment time are folded in so that any # assignment change forces a refresh, even a swap between principals whose @@ -85,8 +89,38 @@ def self.canonicalize(value) end end + def sandbox_entitlements_secret(hosts:) + rules = Principal.normalize_hosts(hosts) + .map do |host| + { + "host" => host, + "methods" => [ "GET" ], + "paths" => [ SANDBOX_ENTITLEMENTS_PATH_PATTERN ] + } + end + return nil if rules.empty? + + token = SandboxEntitlements::Jwt.encode_for_proxy(self) + return nil if token.blank? + + { + "source" => { "type" => "control_plane", "value" => token }, + "inject" => { "header" => "Authorization", "formatter" => "Bearer {{ .Value }}" }, + "rules" => rules + } + end + private + def with_sandbox_entitlements_secret(config, sandbox_entitlements_hosts:) + secret = sandbox_entitlements_secret(hosts: sandbox_entitlements_hosts) + return config unless secret + + config.deep_dup.tap do |copy| + copy["secrets"] = Array(copy["secrets"]) + [ secret ] + end + end + # Stamp (or clear) the assignment time whenever principal_id changes, so the # column always reflects the current assignment. def stamp_principal_assignment diff --git a/services/console/app/models/thread_share.rb b/services/console/app/models/thread_share.rb new file mode 100644 index 000000000..39746e404 --- /dev/null +++ b/services/console/app/models/thread_share.rb @@ -0,0 +1,5 @@ +class ThreadShare < ApplicationRecord + belongs_to :created_by, class_name: "User" + + validates :thread_key, presence: true, length: { maximum: 512 } +end diff --git a/services/console/app/models/user.rb b/services/console/app/models/user.rb index 82dd0aa57..568c20d47 100644 --- a/services/console/app/models/user.rb +++ b/services/console/app/models/user.rb @@ -1,4 +1,6 @@ class User < ApplicationRecord + class SsoEmailDomainNotAllowed < StandardError; end + oid_prefix "usr" # validations: false because SSO-only users have no password. The password @@ -46,6 +48,8 @@ def revoke_mcp_oauth_refresh_tokens! # verified email is on the bootstrap allowlist). +identity+ is the provider # strategy's { subject:, email:, email_verified:, name: } hash. def self.link_or_provision(provider:, identity:) + raise SsoEmailDomainNotAllowed unless ConsoleAuth.sso_email_allowed?(identity[:email]) + transaction do user = if (existing = UserIdentity.find_by(provider: provider, subject: identity[:subject])) @@ -82,10 +86,9 @@ def self.identity_attributes(provider:, identity:) end private_class_method :identity_attributes - # Attributes for a brand-new SSO user: everyone is provisioned active -- the - # console is only reachable on the internal network, so a completed SSO login - # is sufficient and there is no admin-approval queue. Admin additionally - # requires a bootstrap-allowlisted, IdP-verified email. + # Attributes for a brand-new SSO user: everyone is provisioned active once the + # IdP identity has passed the configured SSO admission policy. Admin + # additionally requires a bootstrap-allowlisted, IdP-verified email. def self.provisioned_attributes(identity) admin = identity[:email_verified] == true && ConsoleAuth.bootstrap_admin?(identity[:email]) { email: identity[:email], name: identity[:name], status: :active, admin: admin } diff --git a/services/console/app/services/centaur_api_client.rb b/services/console/app/services/centaur_api_client.rb index aac4761a1..e54dac98b 100644 --- a/services/console/app/services/centaur_api_client.rb +++ b/services/console/app/services/centaur_api_client.rb @@ -1,7 +1,7 @@ +require "cgi" require "json" require "net/http" require "uri" -require "cgi" class CentaurApiClient Response = Struct.new(:status, :body, keyword_init: true) @@ -69,6 +69,14 @@ def ingest_google_docs_sync_batch(payload) post("/api/admin/google/docs-sync/batch", payload) end + def get_granola_sync_checkpoint(scope_id:) + get("/api/admin/granola/sync/checkpoint", scope_id: scope_id) + end + + def ingest_granola_sync_batch(payload) + post("/api/admin/granola/sync/batch", payload) + end + def create_session(thread_key:, harness_type:, metadata: {}, persona_id: nil, on_harness_conflict: "reject") payload = { @@ -145,6 +153,7 @@ def request_headers def parse_body(body) return {} if body.blank? + JSON.parse(body) rescue JSON::ParserError { "raw" => body.to_s } @@ -165,8 +174,8 @@ def net_http_request(method:, url:, body:, headers:, timeout:) http.use_ssl = uri.scheme == "https" http.open_timeout = timeout http.read_timeout = timeout - res = http.request(request) - Response.new(status: res.code.to_i, body: res.body.to_s) + response = http.request(request) + Response.new(status: response.code.to_i, body: response.body.to_s) end def escape_path(value) diff --git a/services/console/app/services/granola/sync_credential.rb b/services/console/app/services/granola/sync_credential.rb new file mode 100644 index 000000000..a3b180fdb --- /dev/null +++ b/services/console/app/services/granola/sync_credential.rb @@ -0,0 +1,349 @@ +require "cgi" +require "date" +require "json" +require "net/http" +require "time" +require "uri" + +module Granola + # Syncs one user's connected Granola MCP account. This runs in the console so + # the OAuth access token never leaves the control plane; only normalized note + # data is sent to api-rs for storage and RLS-protected access. + class SyncCredential + MCP_URL = "https://mcp.granola.ai/mcp" + DEFAULT_INITIAL_LOOKBACK_DAYS = 365 + # The MCP service currently advertises an average limit of about 100 + # requests/minute. One run issues a list call, a batched detail call, and + # at most one transcript call per note, so fifty keeps a normal run well + # inside that envelope. + DEFAULT_MAX_NOTES = 50 + WATERMARK_OVERLAP_SECONDS = 5 * 60 + + MEETING_RE = /[^"]+)"\s+title="(?[^"]*)"\s+date="(?<date>[^"]*)">(?<body>.*?)<\/meeting>/m + PARTICIPANTS_RE = /<known_participants>(?<participants>.*?)<\/known_participants>/m + SUMMARY_RE = /<summary>(?<summary>.*?)<\/summary>/m + PARTICIPANT_RE = /(?<name>[^,<]+?)\s*<(?<email>[^>]+)>/ + MCP_DATE_RE = /\A(?<date>\w+ \d+, \d+ \d+:\d+ [AP]M) GMT(?<offset>[+-]\d+)?\z/ + + GranolaApiError = Class.new(StandardError) + + class << self + attr_accessor :mcp_http + + def oauth_app_slug + ConsoleEnv["GRANOLA_SYNC_OAUTH_APP_SLUG"].presence || "granola" + end + + def syncable?(credential, oauth_app_slug: self.oauth_app_slug) + credential.present? && !credential.dead? && credential.access_token.present? && + credential.oauth_app&.provider == Oauth::Providers::Granola::KEY && + credential.oauth_app&.slug == oauth_app_slug && credential.oauth_app.enabled? + end + + def initial_lookback_days + positive_int(ConsoleEnv["GRANOLA_SYNC_INITIAL_LOOKBACK_DAYS"], DEFAULT_INITIAL_LOOKBACK_DAYS) + end + + def max_notes + positive_int(ConsoleEnv["GRANOLA_SYNC_MAX_NOTES"], DEFAULT_MAX_NOTES) + end + + def positive_int(value, default) + parsed = value.to_i + parsed.positive? ? parsed : default + end + end + + def initialize(credential, api_client: CentaurApiClient.new, mcp_http: nil) + @credential = credential + @api_client = api_client + @mcp_http = mcp_http || self.class.mcp_http + @run_id = "granola_#{SecureRandom.hex(16)}" + @rpc_id = 0 + @source_user_email = credential.provider_email.to_s.strip.downcase + end + + def call + account = parse_account(mcp_tool("get_account_info")) + @source_user_email = account["email"].to_s.strip.downcase.presence || @source_user_email + raise GranolaApiError, "Granola account did not provide an email" if @source_user_email.blank? + + checkpoint = load_checkpoint + notes = sync_notes(checkpoint) + @api_client.ingest_granola_sync_batch(success_batch(notes, checkpoint)) + rescue StandardError => error + record_failure(error) + raise + end + + private + + def scope_id + "oauth:#{@credential.oid}" + end + + def load_checkpoint + @api_client.get_granola_sync_checkpoint(scope_id: scope_id).fetch("checkpoint") + end + + def sync_notes(checkpoint) + meetings = parse_meetings( + mcp_tool( + "list_meetings", + "time_range" => "custom", + "custom_start" => range_start(checkpoint), + "custom_end" => Time.current.utc.to_date.iso8601 + ) + ).first(self.class.max_notes) + + details = parse_meetings( + mcp_tool("get_meetings", "meeting_ids" => meetings.map { |meeting| meeting.fetch("id") }) + ).index_by { |meeting| meeting.fetch("id") } + + meetings.filter_map do |meeting| + detailed = details.fetch(meeting.fetch("id"), meeting) + transcript_text = meeting_transcript(detailed.fetch("id")) + normalize_note(detailed, transcript_text) + end + end + + def meeting_transcript(meeting_id) + mcp_tool("get_meeting_transcript", "meeting_id" => meeting_id) + rescue GranolaApiError => error + # Transcripts are only available on paid Granola plans. Keep syncing the + # note metadata when that optional tool is unavailable or access is + # denied, rather than dropping the entire user's sync. + Rails.logger.info do + "Granola transcript unavailable for #{meeting_id} on credential #{@credential.oid}: " \ + "#{error.message}" + end + "" + end + + def success_batch(notes, checkpoint) + transcript_count = notes.count { |note| note["transcript"].present? } + watermark_time = notes.filter_map { |note| parse_time(note["source_updated_at"]) }.max + watermark_time ||= parse_time(checkpoint&.fetch("watermark_time", nil)) + watermark_time ||= Time.current.utc + + { + run: { + run_id: @run_id, + mode: "incremental", + status: "completed", + scope_id: scope_id, + broker_credential_id: @credential.oid, + source_user_email: @source_user_email, + notes_seen: notes.length, + notes_upserted: notes.length, + transcripts_seen: notes.length, + transcripts_upserted: transcript_count, + metadata: run_metadata + }, + notes: notes, + checkpoint: { + scope_id: scope_id, + watermark_time: watermark_time.iso8601 + } + } + end + + def record_failure(error) + return if @source_user_email.blank? + + @api_client.ingest_granola_sync_batch( + run: { + run_id: @run_id, + mode: "incremental", + status: "failed", + scope_id: scope_id, + broker_credential_id: @credential.oid, + source_user_email: @source_user_email, + error_text: "#{error.class}: #{error.message}".truncate(2_000), + metadata: run_metadata + }, + notes: [], + checkpoint: { scope_id: scope_id } + ) + rescue StandardError => report_error + Rails.logger.warn do + "Granola sync failure could not be recorded for credential #{@credential.oid}: " \ + "#{report_error.class}: #{report_error.message}" + end + end + + def run_metadata + { + "oauth_app_slug" => @credential.oauth_app&.slug, + "credential_id" => @credential.oid, + "provider_subject" => @credential.provider_subject.to_s + } + end + + def range_start(checkpoint) + watermark = parse_time(checkpoint&.fetch("watermark_time", nil)) + start_time = watermark ? watermark - WATERMARK_OVERLAP_SECONDS : self.class.initial_lookback_days.days.ago + start_time.utc.to_date.iso8601 + end + + def parse_account(text) + JSON.parse(text) + rescue JSON::ParserError + raise GranolaApiError, "Granola MCP returned an invalid account response" + end + + def normalize_note(meeting, transcript_text) + transcript = transcript_text.to_s.strip + date = parse_mcp_date(meeting["date"]) + { + "note_id" => meeting.fetch("id"), + "title" => meeting["title"].to_s, + "owner" => meeting.fetch("owner", {}), + "attendees" => meeting.fetch("attendees", []), + "calendar_event" => {}, + "summary_markdown" => meeting["summary_markdown"].to_s, + "summary_text" => meeting["summary_markdown"].to_s, + "transcript" => transcript.present? ? [ { "speaker" => { "source" => "transcript" }, "text" => transcript } ] : [], + "url" => "", + "source_created_at" => date, + "source_updated_at" => date, + "raw_payload" => { + "meeting" => meeting, + "transcript_available" => transcript.present?, + "source" => "granola_mcp" + } + } + end + + def parse_meetings(text) + text.to_s.scan(MEETING_RE).filter_map do |id, title, date, body| + participants = participant_list(body) + owner = participants.find { |participant| participant["name"].include?("(note creator)") } || participants.first || {} + owner = owner.merge( + "name" => owner.fetch("name", "").sub("(note creator)", "").split(" from ", 2).first.strip + ) unless owner.empty? + summary_match = body.match(SUMMARY_RE) + { + "id" => CGI.unescapeHTML(id), + "title" => CGI.unescapeHTML(title), + "date" => CGI.unescapeHTML(date), + "owner" => owner, + "attendees" => participants, + "summary_markdown" => CGI.unescapeHTML(summary_match&.[](:summary).to_s.strip) + } + end + end + + def participant_list(body) + participants = CGI.unescapeHTML(body.match(PARTICIPANTS_RE)&.[](:participants).to_s) + participants.scan(PARTICIPANT_RE).map do |name, email| + { "name" => CGI.unescapeHTML(name).strip, "email" => email.strip.downcase } + end + end + + def parse_mcp_date(value) + match = value.to_s.match(MCP_DATE_RE) + return nil unless match + + offset = format("%+03d00", (match[:offset].presence || "+0").to_i) + Time.strptime("#{match[:date]} #{offset}", "%b %d, %Y %I:%M %p %z").iso8601 + rescue ArgumentError + nil + end + + def parse_time(value) + return if value.blank? + + Time.iso8601(value) + rescue ArgumentError + nil + end + + def mcp_tool(name, arguments = {}) + if @mcp_http + return @mcp_http.call(tool: name, arguments: arguments, access_token: @credential.access_token).to_s + end + + initialize_mcp_session unless @mcp_initialized + response = mcp_request( + "tools/call", + { name: name, arguments: arguments }, + session_id: @mcp_session_id + ) + payload = decode_mcp_response(response) + raise GranolaApiError, "Granola MCP returned #{payload['error']}" if payload["error"] + + result = payload.fetch("result", {}) + raise GranolaApiError, "Granola MCP tool #{name} failed" if result["isError"] + + Array(result["content"]) + .filter_map { |content| content["text"] if content["type"] == "text" } + .join("\n") + end + + def initialize_mcp_session + response = mcp_request( + "initialize", + { + protocolVersion: "2025-03-26", + capabilities: {}, + clientInfo: { name: "centaur-console", version: "1.0" } + } + ) + payload = decode_mcp_response(response) + raise GranolaApiError, "Granola MCP returned #{payload['error']}" if payload["error"] + raise GranolaApiError, "Granola MCP did not acknowledge initialization" unless payload["result"] + + @mcp_session_id = response["mcp-session-id"].presence + send_mcp_initialized_notification if @mcp_session_id + @mcp_initialized = true + end + + def send_mcp_initialized_notification + mcp_request("notifications/initialized", {}, session_id: @mcp_session_id, notification: true) + rescue GranolaApiError => error + Rails.logger.warn do + "Granola MCP initialization notification failed for credential #{@credential.oid}: " \ + "#{error.message}" + end + end + + def mcp_request(method, params, session_id: nil, notification: false) + @rpc_id += 1 unless notification + uri = URI.parse(MCP_URL) + request = Net::HTTP::Post.new(uri) + request["Authorization"] = "Bearer #{@credential.access_token}" + request["Content-Type"] = "application/json" + request["Accept"] = "application/json, text/event-stream" + request["MCP-Protocol-Version"] = "2025-03-26" if session_id.present? + request["MCP-Session-Id"] = session_id if session_id.present? + payload = { jsonrpc: "2.0", method: method, params: params } + payload[:id] = @rpc_id unless notification + request.body = payload.to_json + + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + http.open_timeout = 30 + http.read_timeout = 60 + response = http.request(request) + unless response.code.to_i.between?(200, 299) + raise GranolaApiError, "Granola MCP returned HTTP #{response.code}" + end + + response + end + + def decode_mcp_response(response) + body = response.body.to_s + if response["content-type"].to_s.start_with?("text/event-stream") + payload = body.each_line.filter_map { |line| line.delete_prefix("data: ").strip if line.start_with?("data: ") }.last + raise GranolaApiError, "Granola MCP returned an empty event stream" if payload.blank? + + body = payload + end + JSON.parse(body) + rescue JSON::ParserError + raise GranolaApiError, "Granola MCP returned malformed JSON" + end + end +end diff --git a/services/console/app/services/slack_dm/sync_credential.rb b/services/console/app/services/slack_dm/sync_credential.rb index dafe755e5..c6b079c66 100644 --- a/services/console/app/services/slack_dm/sync_credential.rb +++ b/services/console/app/services/slack_dm/sync_credential.rb @@ -4,7 +4,12 @@ module SlackDm class SyncCredential - REQUIRED_SCOPES = %w[im:read im:history mpim:read mpim:history].freeze + DM_REQUIRED_SCOPES = %w[im:read im:history].freeze + MPIM_REQUIRED_SCOPES = %w[mpim:read mpim:history].freeze + PRIVATE_CHANNEL_REQUIRED_SCOPES = %w[groups:read groups:history].freeze + REQUIRED_SCOPES = ( + DM_REQUIRED_SCOPES + MPIM_REQUIRED_SCOPES + PRIVATE_CHANNEL_REQUIRED_SCOPES + ).freeze AUTH_TEST_ENDPOINT = "https://slack.com/api/auth.test" CONVERSATIONS_LIST_ENDPOINT = "https://slack.com/api/conversations.list" @@ -22,7 +27,16 @@ def oauth_app_slug end def required_scopes_granted?(scopes) - (REQUIRED_SCOPES - Array(scopes)).empty? + supported_conversation_types(scopes).any? + end + + def supported_conversation_types(scopes) + granted = Array(scopes) + types = [] + types << "im" if (DM_REQUIRED_SCOPES - granted).empty? + types << "mpim" if (MPIM_REQUIRED_SCOPES - granted).empty? + types << "private_channel" if (PRIVATE_CHANNEL_REQUIRED_SCOPES - granted).empty? + types end end @@ -111,9 +125,12 @@ def empty_batch(home_team_id, source_user_id) end def list_conversations + types = self.class.supported_conversation_types(@credential.scopes) + raise SlackApiError, "Slack credential has no supported conversation scopes" if types.empty? + each_page( CONVERSATIONS_LIST_ENDPOINT, - { "types" => "im,mpim", "exclude_archived" => "false", "limit" => list_page_size }, + { "types" => types.join(","), "exclude_archived" => "false", "limit" => list_page_size }, max_pages: list_max_pages ).flat_map { |page| Array(page["channels"]) } end @@ -122,7 +139,7 @@ def normalize_conversation(conversation, home_team_id, batch) batch[:conversations] << { home_team_id: home_team_id, conversation_id: conversation.fetch("id"), - conversation_type: conversation["is_mpim"] ? "mpim" : "im", + conversation_type: conversation_type(conversation), is_archived: conversation["is_archived"] == true, is_ext_shared: conversation["is_ext_shared"] == true, raw_payload: conversation @@ -151,16 +168,32 @@ def conversation_members(conversation) return members.uniq end + complete = true pages = each_page( CONVERSATIONS_MEMBERS_ENDPOINT, { "channel" => conversation.fetch("id"), "limit" => members_page_size }, max_pages: members_max_pages - ) + ) do |_page, truncated| + complete = false if truncated + end + unless complete + raise SlackApiError, + "Slack membership pagination truncated for #{conversation.fetch('id')}" + end + members = pages.flat_map { |page| Array(page["members"]) }.compact members << @credential.provider_subject if @credential.provider_subject.present? members.uniq end + def conversation_type(conversation) + return "mpim" if conversation["is_mpim"] + return "im" if conversation["is_im"] + return "private_channel" if conversation["is_private"] + + raise SlackApiError, "Unsupported Slack conversation #{conversation['id']}" + end + def sync_history(conversation, home_team_id, checkpoint, batch) conversation_id = conversation.fetch("id") max_message_ts = checkpoint diff --git a/services/console/app/services/slack_requester_identity.rb b/services/console/app/services/slack_requester_identity.rb new file mode 100644 index 000000000..dd101e128 --- /dev/null +++ b/services/console/app/services/slack_requester_identity.rb @@ -0,0 +1,68 @@ +require "json" +require "net/http" +require "uri" + +# Resolves a Console user's GitHub handle through the same authoritative source +# as slackbotv2: the GitHub custom field on the human requester's Slack profile. +class SlackRequesterIdentity + Result = Data.define(:handle, :source, :reason) + DEFAULT_API_URL = "https://slack.com/api".freeze + GITHUB_LABEL = /github/i + GITHUB_URL = %r{github\.com/([A-Za-z0-9-]{1,39})(?:[/?#]|$)}i + GITHUB_PREFIX = /github\s*[:=]\s*@?([A-Za-z0-9-]{1,39})/i + GITHUB_HANDLE = /\A[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?\z/ + + def self.resolve(user_ids:) + token = ENV["CENTAUR_CONSOLE_SLACK_BOT_TOKEN"].presence || ENV["SLACK_BOT_TOKEN"].presence + return Result.new(handle: nil, source: nil, reason: "Slack bot token is not configured") if token.blank? + + resolver = new(token: token, api_url: ENV["SLACK_API_URL"].presence || DEFAULT_API_URL) + Array(user_ids).filter_map { |user_id| resolver.resolve(user_id) }.first || + Result.new(handle: nil, source: nil, reason: "no GitHub custom field found on Slack profile") + end + + def initialize(token:, api_url:) + @token = token + @api_url = api_url.to_s.delete_suffix("/") + end + + def resolve(user_id) + payload = slack_get("users.profile.get", user: user_id, include_labels: "true") + return nil unless payload["ok"] == true && payload["profile"].is_a?(Hash) + + github_field(payload["profile"]) + rescue StandardError => e + Rails.logger.warn("console_slack_requester_identity_lookup_failed error=#{e.class}") + nil + end + + private + + def slack_get(method, params) + uri = URI("#{@api_url}/#{method}") + uri.query = URI.encode_www_form(params) + request = Net::HTTP::Get.new(uri) + request["Authorization"] = "Bearer #{@token}" + request["Accept"] = "application/json" + response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", + open_timeout: 2, read_timeout: 5) { |http| http.request(request) } + JSON.parse(response.body) + end + + def github_field(profile) + fields = profile["fields"].is_a?(Hash) ? profile["fields"] : {} + fields.each_value do |field| + next unless field.is_a?(Hash) + label = field["label"].presence || field["alt"].to_s + value = field["value"].to_s.strip + next unless label.match?(GITHUB_LABEL) || value.match?(GITHUB_LABEL) + + login = value[GITHUB_URL, 1] || value[GITHUB_PREFIX, 1] || (value if label.match?(GITHUB_LABEL)) + login = login.to_s.delete_prefix("@") + next unless login.match?(GITHUB_HANDLE) + + return Result.new(handle: "@#{login}", source: "Slack profile custom field \"#{label}\"", reason: nil) + end + nil + end +end diff --git a/services/console/app/views/console/threads/_composer.html.erb b/services/console/app/views/console/threads/_composer.html.erb index 6caee67f9..78d14229e 100644 --- a/services/console/app/views/console/threads/_composer.html.erb +++ b/services/console/app/views/console/threads/_composer.html.erb @@ -181,10 +181,41 @@ try { return JSON.parse(picker.dataset.agents || "{}"); } catch { return {}; } }; + // Keep the popover inside the window: it normally hangs below the pill, + // but docked composers sit near the bottom of the viewport, so flip it + // above when the space below can't fit it — and cap its height (the + // menu scrolls) when neither side can. Runs on every level switch since + // the levels differ in height. + const positionMenu = (picker) => { + const menu = picker.querySelector("[role=menu]"); + const trigger = picker.querySelector("[aria-haspopup=menu]"); + if (!menu || !trigger || menu.hidden) return; + const margin = 8; + menu.classList.remove("console-composer-model-menu--above"); + menu.style.maxHeight = ""; + const pill = trigger.getBoundingClientRect(); + const rect = menu.getBoundingClientRect(); + const gap = rect.top - pill.bottom; + const below = window.innerHeight - rect.top - margin; + const above = pill.top - gap - margin; + if (rect.height <= below) return; + if (above > below) { + menu.classList.add("console-composer-model-menu--above"); + if (rect.height > above) menu.style.maxHeight = `${Math.max(above, 0)}px`; + } else { + menu.style.maxHeight = `${Math.max(below, 0)}px`; + } + }; + + window.addEventListener("resize", () => { + document.querySelectorAll("[data-console-model-picker]").forEach(positionMenu); + }); + const showLevel = (picker, name) => { picker.querySelectorAll("[data-picker-level]").forEach((level) => { level.hidden = level.dataset.pickerLevel !== name; }); + positionMenu(picker); }; function closePicker(picker) { diff --git a/services/console/app/views/console/threads/_panel_transcript.html.erb b/services/console/app/views/console/threads/_panel_transcript.html.erb new file mode 100644 index 000000000..a3e4134b6 --- /dev/null +++ b/services/console/app/views/console/threads/_panel_transcript.html.erb @@ -0,0 +1,5 @@ +<%# Poll payload for one thread panel: the transcript stream plus the thinking + indicator while the turn is still running. Swapped into the panel's scroll + container by thread_poller_controller.js — see ThreadsController#panel. %> +<%= render "console/threads/transcript", items: items %> +<%= render "console/threads/thinking_indicator" if active %> diff --git a/services/console/app/views/console/threads/_thread_menu.html.erb b/services/console/app/views/console/threads/_thread_menu.html.erb new file mode 100644 index 000000000..46da5dcfb --- /dev/null +++ b/services/console/app/views/console/threads/_thread_menu.html.erb @@ -0,0 +1,47 @@ +<div class="console-thread-share-wrap" + data-controller="thread-share" + data-thread-share-url-value="<%= console_threads_url(thread: session.thread_key) %>"> + <button type="button" + class="console-thread-share-trigger" + data-action="thread-share#open" + aria-label="Share chat" + title="Share chat"> + <%= console_icon("share", classes: "size-4") %> + </button> + + <dialog class="console-share-dialog" + aria-label="Share chat" + data-thread-share-target="dialog" + data-action="cancel->thread-share#cancel click->thread-share#closeFromBackdrop"> + <%= form_with url: console_thread_share_path, + method: :post, + data: { + thread_share_target: "form", + action: "submit->thread-share#copyLink" + } do %> + <%= hidden_field_tag :thread_key, session.thread_key %> + <div class="console-share-dialog-copy"> + <h2>Share chat</h2> + <p> + Anyone with access to Centaur Console will be able to view this chat. + </p> + <p class="console-share-dialog-error" + role="alert" + data-thread-share-target="error" + hidden></p> + </div> + <div class="console-share-dialog-actions"> + <button type="button" + class="btn-secondary" + data-action="thread-share#cancel"> + Cancel + </button> + <button type="submit" + class="btn-primary" + data-thread-share-target="copyButton"> + Copy link + </button> + </div> + <% end %> + </dialog> +</div> diff --git a/services/console/app/views/console/threads/_thread_panel.html.erb b/services/console/app/views/console/threads/_thread_panel.html.erb index a78dfede9..3560a75d7 100644 --- a/services/console/app/views/console/threads/_thread_panel.html.erb +++ b/services/console/app/views/console/threads/_thread_panel.html.erb @@ -7,7 +7,7 @@ <% remaining_keys = panels.map { |other| other[:thread_key] } - [ panel[:thread_key] ] %> <% close_path = console_threads_path(thread: remaining_keys.join(",")) %> <% if panel[:new_chat] %> - <section class="flex min-h-0 min-w-0 flex-col overflow-hidden rounded-xl border border-ink-700 bg-ink-900/35" + <section class="flex min-h-[75vh] min-w-0 flex-col overflow-hidden rounded-xl border border-ink-700 bg-ink-900/35 md:min-h-0" data-thread-panel="<%= panel[:thread_key] %>"> <header class="flex shrink-0 items-center gap-2 border-b border-ink-700 px-4 py-2.5"> <div class="min-w-0 flex-1"> @@ -31,7 +31,7 @@ </div> </section> <% else %> -<section class="flex min-h-0 min-w-0 flex-col overflow-hidden rounded-xl border border-ink-700 bg-ink-900/35" +<section class="flex min-h-[75vh] min-w-0 flex-col overflow-hidden rounded-xl border border-ink-700 bg-ink-900/35 md:min-h-0" data-thread-panel="<%= session.thread_key %>"> <header class="flex shrink-0 items-center gap-2 border-b border-ink-700 px-4 py-2.5"> <div class="min-w-0 flex-1"> @@ -53,6 +53,7 @@ <%= local_time(session.updated_at || session.created_at, relative: true, format: :compact) %> </div> </div> + <%= render "console/threads/thread_menu", session: session if panel[:writable] %> <a href="<%= close_path %>" class="console-panel-close" aria-label="Close panel" @@ -60,14 +61,20 @@ <%= console_icon("x-mark", classes: "size-4") %> </a> </header> - <div class="console-transcript-scroll min-h-0 flex-1 overflow-y-auto px-4 py-4"> - <div class="flex flex-col gap-5"> - <%= render "console/threads/transcript", items: panel[:transcript_items] %> - <%= render "console/threads/thinking_indicator" if thread_execution_active?(session.thread_key) %> + <div class="console-transcript-scroll min-h-0 flex-1 overflow-y-auto px-4 py-4" + data-controller="thread-poller" + data-thread-poller-url-value="<%= console_thread_panel_path(thread_key: session.thread_key) %>" + data-thread-poller-active-value="<%= thread_execution_active?(session.thread_key) %>"> + <div class="flex flex-col gap-5" data-thread-poller-target="transcript"> + <%= render "console/threads/panel_transcript", + items: panel[:transcript_items], + active: thread_execution_active?(session.thread_key) %> </div> </div> - <div class="console-composer-dock console-composer-dock--panel"> - <%= render "console/threads/composer", mode: :thread, session: session %> - </div> + <% if panel[:writable] %> + <div class="console-composer-dock console-composer-dock--panel"> + <%= render "console/threads/composer", mode: :thread, session: session %> + </div> + <% end %> </section> <% end %> diff --git a/services/console/app/views/console/threads/index.html.erb b/services/console/app/views/console/threads/index.html.erb index ea5b74894..97a244bb1 100644 --- a/services/console/app/views/console/threads/index.html.erb +++ b/services/console/app/views/console/threads/index.html.erb @@ -9,13 +9,15 @@ <div class="flex min-h-0 flex-1 overflow-hidden bg-ink-950/15"> <% if @thread_panels.size > 1 %> + <%# On phones the multi-pane grid can't fit side by side, so it stacks into + a single scrolling column; the md+ tracks below restore the split grid. %> <% grid_classes = case @thread_panels.size - when 2 then "grid-cols-2" - when 3 then "grid-cols-3" - else "grid-cols-2 grid-rows-2" + when 2 then "grid-cols-1 md:grid-cols-2" + when 3 then "grid-cols-1 md:grid-cols-3" + else "grid-cols-1 md:grid-cols-2 md:grid-rows-2" end %> - <section class="grid min-h-0 min-w-0 flex-1 gap-3 overflow-hidden pb-3 <%= grid_classes %>"> + <section class="grid min-h-0 min-w-0 flex-1 gap-3 overflow-y-auto overflow-x-hidden pb-3 md:overflow-hidden <%= grid_classes %>"> <% @thread_panels.each do |panel| %> <%= render "console/threads/thread_panel", panel: panel, panels: @thread_panels %> <% end %> @@ -26,6 +28,11 @@ <div class="console-thread-detail-header"> <div class="console-thread-content"> <% selected_thread_title = thread_title(@selected_session) %> + <% if thread_owned?(@selected_session) %> + <% thread_actions = capture do %> + <%= render "console/threads/thread_menu", session: @selected_session %> + <% end %> + <% end %> <% thread_meta = capture do %> <div class="flex min-w-0 items-center gap-2"> <span class="truncate"><%= thread_source_label(@selected_session) %></span> @@ -44,6 +51,7 @@ title: selected_thread_title, title_attr: selected_thread_title, subtitle: thread_meta, + actions: thread_actions, title_class: "truncate text-base font-semibold text-zinc-100", subtitle_class: "mt-0.5 text-xs text-zinc-500", class: "mb-0 min-h-16 items-center" %> @@ -62,11 +70,17 @@ </div> </div> <% else %> - <div id="thread-transcript-scroll" class="console-transcript-scroll min-h-0 flex-1 overflow-y-auto py-6"> + <div id="thread-transcript-scroll" class="console-transcript-scroll min-h-0 flex-1 overflow-y-auto py-6" + <% if @selected_session %> + data-controller="thread-poller" + data-thread-poller-url-value="<%= console_thread_panel_path(thread_key: @selected_session.thread_key) %>" + data-thread-poller-active-value="<%= thread_execution_active?(@selected_session.thread_key) %>" + <% end %>> <% if @selected_session %> - <div class="console-thread-content flex flex-col gap-6"> - <%= render "console/threads/transcript", items: @selected_transcript_items %> - <%= render "console/threads/thinking_indicator" if thread_execution_active?(@selected_session.thread_key) %> + <div class="console-thread-content flex flex-col gap-6" data-thread-poller-target="transcript"> + <%= render "console/threads/panel_transcript", + items: @selected_transcript_items, + active: thread_execution_active?(@selected_session.thread_key) %> </div> <% else %> <div class="flex h-full items-center justify-center px-6"> @@ -80,7 +94,7 @@ <% end %> </div> - <% if @selected_session %> + <% if @selected_session && thread_owned?(@selected_session) %> <div class="console-composer-dock"> <div class="console-thread-content"> <%= render "console/threads/composer", mode: :thread, session: @selected_session %> @@ -91,27 +105,8 @@ </section> <% end %> - <% if @thread_panels.any? { |panel| thread_execution_active?(panel[:thread_key]) } %> - <%# The console has no event stream; while a turn is running in any open - pane, refresh the transcript every few seconds. Skipped whenever a - composer holds a draft so a reload never eats typed text. %> - <script> - (() => { - const refresh = () => { - const drafting = Array.from(document.querySelectorAll("[data-console-composer-input]")) - .some((input) => input.value.trim() !== "" || document.activeElement === input); - if (drafting) { - setTimeout(refresh, 4000); - return; - } - if (window.Turbo) { - window.Turbo.visit(window.location.href, { action: "replace" }); - } else { - window.location.reload(); - } - }; - setTimeout(refresh, 4000); - })(); - </script> - <% end %> + <%# Live updates while a turn runs are per-panel: each transcript scroll + container carries a thread-poller Stimulus controller that polls + ThreadsController#panel and swaps only its own transcript, so activity + in one pane never re-renders the others (or eats composer drafts). %> </div> diff --git a/services/console/app/views/layouts/console.html.erb b/services/console/app/views/layouts/console.html.erb index b8b61cc01..308d14d00 100644 --- a/services/console/app/views/layouts/console.html.erb +++ b/services/console/app/views/layouts/console.html.erb @@ -367,6 +367,87 @@ color: #f4f4f5; } + .console-thread-share-wrap { + position: relative; + flex: 0 0 auto; + } + + .console-thread-share-trigger { + display: grid; + height: 1.75rem; + width: 1.75rem; + cursor: pointer; + place-items: center; + border-radius: 0.5rem; + color: #71717a; + transition: background 120ms ease, color 120ms ease; + } + + .console-thread-share-trigger:hover { + background: rgba(255, 255, 255, 0.08); + color: #f4f4f5; + } + + .console-share-dialog { + width: min(28rem, calc(100vw - 2rem)); + margin: auto; + border: 1px solid rgba(255, 255, 255, 0.12); + border-radius: 0.875rem; + background: #101012; + color: #d4d4d8; + padding: 0; + box-shadow: 0 24px 70px rgba(0, 0, 0, 0.58); + } + + .console-share-dialog::backdrop { + background: rgba(0, 0, 0, 0.66); + backdrop-filter: blur(2px); + } + + .console-share-dialog-copy { + padding: 1.5rem 1.5rem 1.25rem; + } + + .console-share-dialog-copy h2 { + color: #f4f4f5; + font-size: 1rem; + font-weight: 600; + } + + .console-share-dialog-copy > p:not(.console-share-dialog-error) { + margin-top: 0.75rem; + color: #a1a1aa; + font-size: 0.875rem; + line-height: 1.5; + } + + .console-share-dialog-error { + margin-top: 0.75rem; + color: #fca5a5; + font-size: 0.75rem; + } + + .console-share-dialog-actions { + display: flex; + justify-content: flex-end; + gap: 0.75rem; + border-top: 1px solid rgba(255, 255, 255, 0.08); + padding: 1rem 1.5rem; + } + + .console-share-dialog-actions .btn-primary, + .console-share-dialog-actions .btn-secondary { + min-width: 6.5rem; + padding-top: 0.5rem; + padding-bottom: 0.5rem; + text-align: center; + } + + .console-share-dialog-actions .btn-primary:disabled { + cursor: wait; + opacity: 0.6; + } + .console-thinking { min-width: 0; padding: 0.125rem 0; @@ -917,13 +998,22 @@ } /* The picker sits just left of the send button, so the popover hangs - from its right edge. */ + from its right edge. It scrolls when the composer script caps its + height to keep it inside the viewport. */ .console-composer-model-menu { top: calc(100% + 0.4rem); right: 0; bottom: auto; left: auto; width: 15rem; + overflow-y: auto; + } + + /* Flipped by the composer script when the space below the pill can't + fit the menu (docked composers sit at the bottom of the window). */ + .console-composer-model-menu--above { + top: auto; + bottom: calc(100% + 0.4rem); } /* Root rows of the picker menu: setting name left, current value and a @@ -1025,15 +1115,25 @@ gap: 0.25rem; margin-bottom: 1.5rem; border-bottom: 1px solid rgba(255, 255, 255, 0.08); + /* Too many tabs to fit a phone width: scroll them horizontally rather + than clipping the trailing ones off the edge. */ + overflow-x: auto; + scrollbar-width: none; + } + + .console-control-tabs::-webkit-scrollbar { + display: none; } .console-control-tab { margin-bottom: -1px; + flex: 0 0 auto; padding: 0.625rem 0.75rem; border-bottom: 1px solid transparent; color: #a1a1aa; font-size: 0.875rem; font-weight: 500; + white-space: nowrap; text-decoration: none; transition: border-color 120ms ease, color 120ms ease; } @@ -1178,6 +1278,42 @@ color: #24272b; } + html[data-console-theme="light"] .console-thread-share-trigger { + color: #7a7f86; + } + + html[data-console-theme="light"] .console-thread-share-trigger:hover { + background: #e8e9e3; + color: #24272b; + } + + html[data-console-theme="light"] .console-share-dialog { + border-color: #d8d8d0; + background: #ffffff; + color: #565c63; + box-shadow: 0 24px 70px rgba(31, 36, 40, 0.2); + } + + html[data-console-theme="light"] .console-share-dialog::backdrop { + background: rgba(36, 39, 43, 0.38); + } + + html[data-console-theme="light"] .console-share-dialog-copy h2 { + color: #24272b; + } + + html[data-console-theme="light"] .console-share-dialog-copy > p:not(.console-share-dialog-error) { + color: #6f757c; + } + + html[data-console-theme="light"] .console-share-dialog-actions { + border-top-color: #e5e5de; + } + + html[data-console-theme="light"] .console-share-dialog-error { + color: #b42318; + } + html[data-console-theme="light"] .console-thinking-summary { color: #565c63; } @@ -1509,23 +1645,120 @@ color: #1c1f23; } - @media (max-width: 760px) { + /* The mobile top bar (hamburger + brand) and the drawer scrim are hidden + on desktop; the media query below turns them on for narrow viewports. */ + .console-mobile-bar { + display: none; + flex: 0 0 auto; + position: sticky; + top: 0; + z-index: 30; + height: 3.25rem; + align-items: center; + gap: 0.75rem; + padding: 0 0.75rem; + border-bottom: 1px solid rgba(255, 255, 255, 0.07); + background: rgba(7, 7, 8, 0.92); + backdrop-filter: blur(8px); + } + + .console-mobile-nav-toggle { + display: grid; + height: 2.25rem; + width: 2.25rem; + flex: 0 0 auto; + cursor: pointer; + place-items: center; + border-radius: 0.625rem; + color: #d4d4d8; + transition: background 120ms ease, color 120ms ease; + } + + .console-mobile-nav-toggle:hover { + background: rgba(255, 255, 255, 0.06); + color: #f4f4f5; + } + + .console-mobile-brand { + display: flex; + min-width: 0; + align-items: center; + color: #f4f4f5; + text-decoration: none; + } + + .console-mobile-brand img { + height: 1.4rem; + width: auto; + } + + .console-scrim { + display: none; + position: fixed; + inset: 0; + z-index: 40; + background: rgba(0, 0, 0, 0.55); + opacity: 0; + pointer-events: none; + transition: opacity 200ms ease; + } + + html[data-console-theme="light"] .console-mobile-bar { + border-bottom-color: #deded7; + background: rgba(251, 251, 248, 0.92); + } + + html[data-console-theme="light"] .console-mobile-nav-toggle { + color: #5f656c; + } + + html[data-console-theme="light"] .console-mobile-nav-toggle:hover { + background: #edede8; + color: #24272b; + } + + html[data-console-theme="light"] .console-mobile-brand { + color: #24272b; + } + + html[data-console-theme="light"] .console-mobile-brand img { + filter: invert(1) drop-shadow(0 0 14px rgba(40, 194, 106, 0.14)); + } + + /* Mobile: the 18rem rail squeezes the content to a sliver, so below this + breakpoint the sidebar becomes an off-canvas drawer toggled from the + top bar. The main content then gets the full viewport width, and the + drawer keeps its full labels/brand/account row (no cryptic icon-only + rail). */ + @media (max-width: 768px) { + .console-mobile-bar { + display: flex; + } + .console-sidebar { - width: 11rem; + position: fixed; + top: 0; + bottom: 0; + left: 0; + z-index: 50; + width: 17rem; + max-width: 82vw; + transform: translateX(-100%); + transition: transform 200ms ease; } - .console-brand-lockup, - .console-account-details { - display: none; + .console-nav-open .console-sidebar { + transform: translateX(0); + box-shadow: 0 24px 60px rgba(0, 0, 0, 0.5); } - .console-brand-glyph { - display: grid; + .console-scrim { + display: block; } - .console-sidebar-top { - justify-content: center; - padding: 0; + .console-nav-open .console-scrim { + opacity: 1; + pointer-events: auto; } .console-main-page { @@ -1533,9 +1766,8 @@ } .console-main-thread-frame { - padding: 1rem 1rem 0; + padding: 0.85rem 0.85rem 0; } - } </style> </head> @@ -1724,6 +1956,20 @@ </aside> <main class="console-main <%= "console-main-threads" if threads_view %>"> + <div class="console-mobile-bar"> + <button type="button" + id="console-mobile-nav-toggle" + class="console-mobile-nav-toggle" + aria-label="Open navigation" + aria-controls="console-shell" + aria-expanded="false" + data-console-nav-toggle> + <%= console_icon("menu", classes: "size-5") %> + </button> + <a href="<%= root_path %>" class="console-mobile-brand" aria-label="Centaur Console"> + <%= image_tag "centaur-lockup-white.svg", alt: "Centaur", class: "glow", width: 497, height: 127 %> + </a> + </div> <div class="<%= threads_view ? "console-main-thread-frame" : "console-main-page" %>"> <% if flash[:notice] %> <div class="mb-6 rounded border border-ink-600 bg-ink-850/60 px-4 py-3 text-sm text-zinc-400"><%= flash[:notice] %></div> @@ -1734,9 +1980,71 @@ <%= yield %> </div> </main> + + <div class="console-scrim" data-console-nav-scrim aria-hidden="true"></div> </div> <script> + (() => { + // Mobile navigation drawer. Below the CSS breakpoint the sidebar is + // off-canvas; the hamburger in the top bar toggles a `console-nav-open` + // class on the shell that slides it in over a scrim. The drawer closes + // on scrim click, Escape, following any nav link, and after every Turbo + // navigation so it never lingers over the freshly loaded page. + const shell = document.getElementById("console-shell"); + const toggle = document.querySelector("[data-console-nav-toggle]"); + const scrim = document.querySelector("[data-console-nav-scrim]"); + const sidebar = shell && shell.querySelector(".console-sidebar"); + if (!shell || !toggle || !sidebar) return; + + const OPEN_CLASS = "console-nav-open"; + // Mirrors the CSS breakpoint: only in the off-canvas range does a + // closed drawer need to be pulled out of the tab / assistive-tech order. + const mobileQuery = window.matchMedia("(max-width: 768px)"); + + const isOpen = () => shell.classList.contains(OPEN_CLASS); + + // Off-canvas and closed => hide the sidebar from keyboard and assistive + // tech: it sits before the page in DOM order, so leaving it merely + // translated off-screen makes users tab through invisible controls + // first. Desktop and the open drawer stay fully interactive. + const syncInert = () => { + sidebar.inert = mobileQuery.matches && !isOpen(); + }; + + const setOpen = (open) => { + shell.classList.toggle(OPEN_CLASS, open); + toggle.setAttribute("aria-expanded", String(open)); + syncInert(); + }; + + toggle.addEventListener("click", () => setOpen(!isOpen())); + if (scrim) scrim.addEventListener("click", () => setOpen(false)); + + document.addEventListener("keydown", (event) => { + if (event.key === "Escape" && isOpen()) setOpen(false); + }); + + // Only real navigations should dismiss the drawer. Links (section nav, + // thread rows, the brand) qualify; in-drawer buttons that open a menu + // or reveal more rows (account menu, theme toggle, "show more") must + // not, or they'd slam the drawer shut before doing anything. Form + // buttons that submit (sign out, view as operator) trigger a full + // navigation, which the turbo listeners below already close on. + shell.addEventListener("click", (event) => { + if (!isOpen()) return; + if (event.target.closest(".console-sidebar a[href]")) { + setOpen(false); + } + }); + + mobileQuery.addEventListener("change", syncInert); + document.addEventListener("turbo:visit", () => setOpen(false)); + document.addEventListener("turbo:load", () => setOpen(false)); + + syncInert(); + })(); + (() => { const accountButton = document.getElementById("console-account-menu-button"); const accountMenu = document.getElementById("console-account-menu"); diff --git a/services/console/app/views/sessions/new.html.erb b/services/console/app/views/sessions/new.html.erb index ca3e219f8..e73a51d50 100644 --- a/services/console/app/views/sessions/new.html.erb +++ b/services/console/app/views/sessions/new.html.erb @@ -28,27 +28,31 @@ <% end %> </div> - <div class="my-6 flex items-center gap-3 text-xs uppercase tracking-wider text-zinc-600"> - <span class="h-px flex-1 bg-ink-600"></span> - or sign in with email - <span class="h-px flex-1 bg-ink-600"></span> - </div> + <% if password_login_enabled? %> + <div class="my-6 flex items-center gap-3 text-xs uppercase tracking-wider text-zinc-600"> + <span class="h-px flex-1 bg-ink-600"></span> + or sign in with email + <span class="h-px flex-1 bg-ink-600"></span> + </div> + <% end %> <% end %> - <%= form_with url: login_path, method: :post do |f| %> - <div class="space-y-5"> - <div> - <%= f.label :email, class: label %> - <%= f.email_field :email, class: input, placeholder: "you@example.com", autocomplete: "username", autofocus: true %> - </div> - <div> - <%= f.label :password, class: label %> - <%= f.password_field :password, class: input, autocomplete: "current-password" %> + <% if password_login_enabled? %> + <%= form_with url: login_path, method: :post do |f| %> + <div class="space-y-5"> + <div> + <%= f.label :email, class: label %> + <%= f.email_field :email, class: input, placeholder: "you@example.com", autocomplete: "username", autofocus: true %> + </div> + <div> + <%= f.label :password, class: label %> + <%= f.password_field :password, class: input, autocomplete: "current-password" %> + </div> </div> - </div> - <div class="mt-6"> - <%= f.submit "Sign in", class: "w-full cursor-pointer rounded border border-centaur-500/40 bg-centaur-500/10 px-4 py-2 text-sm text-centaur-300 transition-colors hover:bg-centaur-500/20 hover:text-centaur-200" %> - </div> + <div class="mt-6"> + <%= f.submit "Sign in", class: "w-full cursor-pointer rounded border border-centaur-500/40 bg-centaur-500/10 px-4 py-2 text-sm text-centaur-300 transition-colors hover:bg-centaur-500/20 hover:text-centaur-200" %> + </div> + <% end %> <% end %> </div> diff --git a/services/console/config/environments/production.rb b/services/console/config/environments/production.rb index 005460e82..f5dbeec77 100644 --- a/services/console/config/environments/production.rb +++ b/services/console/config/environments/production.rb @@ -1,4 +1,5 @@ require "active_support/core_ext/integer/time" +require "uri" Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. @@ -24,15 +25,11 @@ # Store uploaded files on the local file system (see config/storage.yml for options). config.active_storage.service = :local - # Assume all access to the app is happening through a SSL-terminating reverse proxy. + # Do not assume or force SSL here: in-cluster callers reach the service over + # plain HTTP. Public TLS enforcement belongs at the ingress/proxy layer. # config.assume_ssl = true - - # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. # config.force_ssl = true - # Skip http-to-https redirect for the default health check endpoint. - # config.ssl_options = { redirect: { exclude: ->(request) { request.path == "/up" } } } - # Log to STDOUT as single-line JSON with the current request id as a default log tag. config.log_tags = [ :request_id ] config.logger = ActiveSupport::TaggedLogging.logger(STDOUT) @@ -48,6 +45,7 @@ # request. The Raw formatter emits a hash that JsonLogFormatter merges into # the JSON log entry. config.lograge.enabled = true + config.lograge.base_controller_class = %w[ActionController::Base ActionController::API] config.lograge.formatter = Lograge::Formatters::Raw.new config.lograge.custom_payload do |controller| { request_id: controller.request.request_id } @@ -92,12 +90,18 @@ # Only use :id for inspections in production. config.active_record.attributes_for_inspect = [ :id ] - # Enable DNS rebinding protection and other `Host` header attacks. - # config.hosts = [ - # "example.com", # Allow requests from example.com - # /.*\.example\.com/ # Allow requests from subdomains like `www.example.com` - # ] - # + # Enable DNS rebinding protection and other `Host` header attacks. Public + # deployments should set CENTAUR_CONSOLE_PUBLIC_URL; add any extra internal + # health-check or ingress hosts with CENTAUR_CONSOLE_ALLOWED_HOSTS. + public_url = ConsoleEnv["PUBLIC_URL"].presence + internal_url = ConsoleEnv["URL"].presence + allowed_hosts = ConsoleEnv["ALLOWED_HOSTS"].to_s.split(/[,\s]+/).map(&:strip).reject(&:blank?) + [ public_url, internal_url ].compact.each do |url| + host = URI.parse(url).host + allowed_hosts << host if host.present? + end + config.hosts.concat(allowed_hosts.uniq) if allowed_hosts.any? + # Skip DNS rebinding protection for the default health check endpoint. - # config.host_authorization = { exclude: ->(request) { request.path == "/up" } } + config.host_authorization = { exclude: ->(request) { request.path == "/up" } } end diff --git a/services/console/config/recurring.yml b/services/console/config/recurring.yml index aa87b7a06..3f4a86739 100644 --- a/services/console/config/recurring.yml +++ b/services/console/config/recurring.yml @@ -24,6 +24,8 @@ production: schedule: every 10 minutes google_docs_poll_sync: class: "GoogleDocs::PollSyncJob" + granola_poll_sync: + class: "Granola::PollSyncJob" schedule: every 30 minutes prune_principal_sync_config_snapshots: class: "PrunePrincipalSyncConfigSnapshotsJob" diff --git a/services/console/config/routes.rb b/services/console/config/routes.rb index b8d27235f..f3ffb97d9 100644 --- a/services/console/config/routes.rb +++ b/services/console/config/routes.rb @@ -47,6 +47,11 @@ get "console/principals/:id", to: "console#principal", as: :console_principal namespace :console do resources :threads, only: %i[index create] + post "threads/share", to: "threads#share", as: :thread_share + # Single-panel transcript refresh polled by thread_poller_controller.js + # while a turn is running. thread_key rides as a query param: keys carry + # colons and dots a path segment would mangle. + get "threads/panel", to: "threads#panel", as: :thread_panel resources :workflows, only: %i[index show] do member do post :run, action: :force_start @@ -204,12 +209,16 @@ # Called by iron-proxy instances (proxy bearer auth, not ApiKey auth). post "proxy/sync", to: "proxy_sync#create" + + # Called from inside sandboxes through their assigned iron-proxy. The + # proxy injects a short-lived sandbox entitlement JWT scoped to these paths. + get "sandbox/permissions", to: "sandbox_permissions#show" + get "sandbox/oauth_apps", to: "sandbox_oauth_apps#index" end end - # Public OAuth consent flow, keyed by the app's well-known slug - # (/oauth/google/start). Deliberately unauthenticated: a team member clicks the - # link to connect an integration; the provider is derived from the app. + # OAuth consent flow, keyed by the app's well-known slug (/oauth/google/start). + # Requires an active console session; the provider is derived from the app. get "oauth/:slug/start", to: "oauth/flows#start", as: :oauth_start get "oauth/:slug/callback", to: "oauth/flows#callback", as: :oauth_callback diff --git a/services/console/db/migrate/.checksums.sha256 b/services/console/db/migrate/.checksums.sha256 index f2185c0a2..ce7ec61c9 100644 --- a/services/console/db/migrate/.checksums.sha256 +++ b/services/console/db/migrate/.checksums.sha256 @@ -56,3 +56,4 @@ aee4f8181e9ad68a949e3ad1e21be9ed9b8ef84bd0eb171a2c782e53bc9ef4e1 services/conso f2e59517094a049919adfcc48e84bf9f1569f13ccbf43a14ccfe4da346f22a09 services/console/db/migrate/20260709223000_backfill_slack_channel_permissions_from_labels.rb 43387034d3afe8942416311cdad834a2f0e001722884aa4086d2db2e8dabdac3 services/console/db/migrate/20260711182055_create_system_settings.rb 357a1338a43d5f7ef1cede938fb947ece878903e18ea0357b6eb73e60d99c25b services/console/db/migrate/20260711190035_add_sandbox_repo_cache_to_principals.rb +8f59287ac283dd7c10eeec43a4d2b9e5406c8b298af24951a02d494e3ffabdbf services/console/db/migrate/20260714175437_create_thread_shares.rb diff --git a/services/console/db/migrate/20260714175437_create_thread_shares.rb b/services/console/db/migrate/20260714175437_create_thread_shares.rb new file mode 100644 index 000000000..33f403c65 --- /dev/null +++ b/services/console/db/migrate/20260714175437_create_thread_shares.rb @@ -0,0 +1,12 @@ +class CreateThreadShares < ActiveRecord::Migration[8.1] + def change + create_table :thread_shares do |t| + t.string :thread_key, null: false, limit: 512 + t.references :created_by, null: false, foreign_key: { to_table: :users } + + t.timestamps + end + + add_index :thread_shares, :thread_key, unique: true + end +end diff --git a/services/console/db/schema.rb b/services/console/db/schema.rb index 22aeeff31..3fefa7206 100644 --- a/services/console/db/schema.rb +++ b/services/console/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_07_11_190035) do +ActiveRecord::Schema[8.1].define(version: 2026_07_14_175437) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -436,6 +436,15 @@ t.index ["singleton"], name: "index_system_settings_on_singleton", unique: true end + create_table "thread_shares", force: :cascade do |t| + t.datetime "created_at", null: false + t.bigint "created_by_id", null: false + t.string "thread_key", limit: 512, null: false + t.datetime "updated_at", null: false + t.index ["created_by_id"], name: "index_thread_shares_on_created_by_id" + t.index ["thread_key"], name: "index_thread_shares_on_thread_key", unique: true + end + create_table "user_identities", force: :cascade do |t| t.datetime "created_at", null: false t.string "email" @@ -511,6 +520,7 @@ add_foreign_key "slack_channel_permissions", "principals" add_foreign_key "static_secrets", "broker_credentials" add_foreign_key "static_secrets", "users", column: "created_by_id" + add_foreign_key "thread_shares", "users", column: "created_by_id" add_foreign_key "user_identities", "users" add_foreign_key "users", "users", column: "approved_by_id" end diff --git a/services/console/docs/API.md b/services/console/docs/API.md index b063099a9..75d1a5607 100644 --- a/services/console/docs/API.md +++ b/services/console/docs/API.md @@ -1,6 +1,6 @@ # iron-control API -`iron-control` exposes a JSON API under `/api/v1`. Every resource endpoint requires API key authentication. The single exception is `POST /api/v1/proxy/sync`, which `iron-proxy` instances call with a proxy bearer token. +`iron-control` exposes a JSON API under `/api/v1`. Resource endpoints require API key authentication. `POST /api/v1/proxy/sync` uses proxy bearer authentication, and sandbox read endpoints use the sandbox entitlement JWT injected by `iron-proxy`. - [Authentication](#authentication) - [Conventions](#conventions) @@ -1073,6 +1073,53 @@ Returns `201`. The `client_secret` is never echoed back: | `PUT`/`PATCH` | `/api/v1/oauth_apps/:id` | [Upsert](#upsert-put--patch) by OID or slug. Omitted fields are preserved; `client_secret` is only changed when supplied. | | `DELETE` | `/api/v1/oauth_apps/:id` | Delete. Returns `204`; `404` if missing. Returns `409` while the app still has minted credentials (delete or unlink them first). | +### Sandbox Start URLs + +`GET /api/v1/sandbox/oauth_apps` + +Returns enabled OAuth apps and the console URLs a sandbox user can open to start consent. Authenticate with the same sandbox entitlement JWT as `GET /api/v1/sandbox/permissions`. The token signature, issuer, audience, and expiry are verified. Proxy and principal claims are not checked because these URLs are not sensitive. + +```json +{ + "data": [ + { + "id": "oap_...", + "slug": "google", + "description": "Gmail", + "labels": {}, + "provider": "google", + "allowed_scopes": ["https://www.googleapis.com/auth/gmail.readonly"], + "start_url": "https://<iron-control>/oauth/google/start" + } + ] +} +``` + +### Sandbox OAuth Credential Metadata + +`GET /api/v1/sandbox/permissions` + +The sandbox permissions response includes an `oauth_credentials` array with non-secret metadata for OAuth-flow credentials currently granted to the sandbox principal. Use it to confirm that a user completed consent for the expected app and personal email. + +```json +{ + "data": { + "oauth_credentials": [ + { + "id": "bcr_...", + "oauth_app_id": "oap_...", + "slug": "google", + "provider": "google", + "provider_email": "person@example.com", + "provider_subject": "google-subject", + "status": "live", + "scopes": ["https://www.googleapis.com/auth/gmail.readonly"] + } + ] + } +} +``` + ## OAuth consent flow The consent flow turns a team member's OAuth consent into a managed broker credential. It runs on iron-control's own domain and is deliberately unauthenticated: the member reaches it with a single well-known link keyed by the app's `slug`. There is no external app to integrate with, so the start endpoint takes no `user` or `return_to`: after consent the member lands on an iron-control result page, and the credential's `external_user_key` is generated automatically. Safety comes from the consent itself (a credential is only created after a successful code exchange) and upsert-on-reconsent (re-consenting for the same provider account updates the existing credential instead of creating a new one). @@ -1118,7 +1165,7 @@ A tampered, expired, or missing flow state or cookie renders an error page with | Google | `google` | | Slack | `slack` | -Slack OAuth apps should have token rotation enabled so the callback receives a refresh token for the broker refresh loop. +Slack OAuth apps may use token rotation or long-lived tokens. When token rotation is enabled, the callback stores the returned refresh token and the broker refresh loop keeps the access token fresh. When Slack returns a long-lived token without a refresh token or expiry, the credential is stored without scheduling broker refresh. Slack OAuth apps should use normal Slack API scopes such as `channels:history`, not Sign in with Slack scopes such as `openid`, `email`, or `profile`. ## Principals diff --git a/services/console/lib/api_server/jwt.rb b/services/console/lib/api_server/jwt.rb index 15f027bda..932ae447b 100644 --- a/services/console/lib/api_server/jwt.rb +++ b/services/console/lib/api_server/jwt.rb @@ -1,5 +1,3 @@ -require "zlib" - module ApiServer module Jwt DEFAULT_AUDIENCE = "centaur-api".freeze @@ -13,38 +11,32 @@ def encode_for_principal(principal, now: Time.current) upload_channels = principal.slack_upload_channel_ids download_channels = principal.slack_download_channel_ids history_channels = principal.slack_history_channel_ids - signing_secret = ENV["CENTAUR_JWT_SIGNING_SECRET"].to_s - return nil if signing_secret.blank? - - issued_at = window_start_for(principal, now.to_i) - expires_at = issued_at + DEFAULT_TTL_SECONDS - CentaurJwt::Hs256.encode( - { - "iss" => issuer, + CentaurJwt::WindowedToken.encode( + subject_oid: principal.oid, + audience: audience, + issuer: issuer, + window_seconds: DEFAULT_WINDOW_SECONDS, + ttl_seconds: DEFAULT_TTL_SECONDS, + now: now, + claims: { "sub" => principal.oid, - "aud" => audience, - "iat" => issued_at, - "exp" => expires_at, "slack" => { "upload_channels" => upload_channels, "download_channels" => download_channels, "history_channels" => history_channels } - }, - signing_secret: signing_secret + } ) end - # Rotation boundaries are offset per principal (deterministically, from - # the oid) so the fleet's tokens don't all roll over — and force snapshot - # rebuilds — at the same instant. + # Kept for callers that reason about rotation boundaries directly + # (snapshot staleness checks, tests). def window_start_for(principal, timestamp) - offset = rotation_offset(principal) - timestamp - ((timestamp - offset) % DEFAULT_WINDOW_SECONDS) + CentaurJwt::WindowedToken.window_start(principal.oid, timestamp, window_seconds: DEFAULT_WINDOW_SECONDS) end def rotation_offset(principal) - Zlib.crc32(principal.oid.to_s) % DEFAULT_WINDOW_SECONDS + CentaurJwt::WindowedToken.rotation_offset(principal.oid, window_seconds: DEFAULT_WINDOW_SECONDS) end def audience diff --git a/services/console/lib/centaur_jwt/hs256.rb b/services/console/lib/centaur_jwt/hs256.rb index cef627be5..98816088b 100644 --- a/services/console/lib/centaur_jwt/hs256.rb +++ b/services/console/lib/centaur_jwt/hs256.rb @@ -1,23 +1,35 @@ -require "base64" -require "json" -require "openssl" +require "jwt" module CentaurJwt module Hs256 + VerificationError = Class.new(StandardError) + module_function def encode(payload, signing_secret:) signing_secret = signing_secret.to_s raise KeyError, "CENTAUR_JWT_SIGNING_SECRET is not configured" if signing_secret.blank? - header = { "alg" => "HS256", "typ" => "JWT" } - signing_input = [ base64url_json(header), base64url_json(payload) ].join(".") - signature = OpenSSL::HMAC.digest("SHA256", signing_secret, signing_input) - "#{signing_input}.#{Base64.urlsafe_encode64(signature, padding: false)}" + JWT.encode(payload, signing_secret, "HS256", { "typ" => "JWT" }) end - def base64url_json(value) - Base64.urlsafe_encode64(JSON.generate(value), padding: false) + def decode(token, signing_secret:, aud: nil, iss: nil) + signing_secret = signing_secret.to_s + raise KeyError, "CENTAUR_JWT_SIGNING_SECRET is not configured" if signing_secret.blank? + + payload, _header = JWT.decode( + token.to_s, + signing_secret, + true, + algorithm: "HS256", + verify_iss: iss.present?, + iss: iss, + verify_aud: aud.present?, + aud: aud + ) + payload + rescue JWT::DecodeError => e + raise VerificationError, e.message end end end diff --git a/services/console/lib/centaur_jwt/windowed_token.rb b/services/console/lib/centaur_jwt/windowed_token.rb new file mode 100644 index 000000000..af8d4f925 --- /dev/null +++ b/services/console/lib/centaur_jwt/windowed_token.rb @@ -0,0 +1,40 @@ +require "zlib" + +module CentaurJwt + # Tokens that rotate on a fixed window rather than per request, so the token + # bytes — and any synced proxy config that embeds them — stay stable within + # a window and the sync hash short-circuit keeps working. Rotation + # boundaries are offset per subject (deterministically, from its oid) so a + # fleet's tokens don't all roll over — and force config re-pushes — at the + # same instant. + module WindowedToken + module_function + + # Returns nil when the signing secret is unconfigured, matching the + # callers' contract of quietly omitting the token from synced config. + def encode(subject_oid:, audience:, issuer:, window_seconds:, ttl_seconds:, claims:, now: Time.current) + signing_secret = ENV["CENTAUR_JWT_SIGNING_SECRET"].to_s + return nil if signing_secret.blank? + + issued_at = window_start(subject_oid, now.to_i, window_seconds: window_seconds) + CentaurJwt::Hs256.encode( + { + "iss" => issuer, + "aud" => audience, + "iat" => issued_at, + "exp" => issued_at + ttl_seconds + }.merge(claims), + signing_secret: signing_secret + ) + end + + def window_start(oid, timestamp, window_seconds:) + offset = rotation_offset(oid, window_seconds: window_seconds) + timestamp - ((timestamp - offset) % window_seconds) + end + + def rotation_offset(oid, window_seconds:) + Zlib.crc32(oid.to_s) % window_seconds + end + end +end diff --git a/services/console/lib/console_auth.rb b/services/console/lib/console_auth.rb index c8d66e5b1..43af64ccf 100644 --- a/services/console/lib/console_auth.rb +++ b/services/console/lib/console_auth.rb @@ -8,6 +8,14 @@ # credentials.console_auth.<provider>.client_id/secret (fallback) # A provider is offered on the login page only when both are present. # +# SSO email domains are optional. When configured, every SSO login must use an +# email address under one of these domains: +# CENTAUR_CONSOLE_SSO_EMAIL_DOMAINS="acme.com example.org" +# +# Password login is a break-glass fallback and can be disabled for public +# deployments: +# CENTAUR_CONSOLE_PASSWORD_LOGIN_ENABLED=false +# # Bootstrap admins are matched by email and become active + admin on first login # (the first admin needs no existing approver): # CENTAUR_CONSOLE_BOOTSTRAP_ADMINS="me@acme.com, you@acme.com" (ENV) @@ -31,6 +39,24 @@ def configured?(provider) def client_id(provider) = setting(provider, "client_id") def client_secret(provider) = setting(provider, "client_secret") + def password_login_enabled? + raw = ConsoleEnv["PASSWORD_LOGIN_ENABLED"] + raw.nil? ? true : boolean_setting(raw) + end + + def sso_email_allowed?(email) + domains = sso_email_domains + return true if domains.empty? + + domain = email.to_s.strip.downcase.split("@", 2).last + domains.include?(domain) + end + + def sso_email_domains + raw = ConsoleEnv["SSO_EMAIL_DOMAINS"].presence + raw.to_s.split(/[,\s]+/).map { |domain| domain.strip.downcase }.reject(&:empty?).uniq + end + def bootstrap_admin?(email) normalized = email.to_s.strip.downcase return false if normalized.empty? @@ -54,4 +80,8 @@ def setting(provider, field) def credentials_dig(*path) Rails.application.credentials.dig(:console_auth, *path) end + + def boolean_setting(value) + ActiveModel::Type::Boolean.new.cast(value) + end end diff --git a/services/console/lib/oauth/providers/slack.rb b/services/console/lib/oauth/providers/slack.rb index affd9aea9..797d0b2ca 100644 --- a/services/console/lib/oauth/providers/slack.rb +++ b/services/console/lib/oauth/providers/slack.rb @@ -23,6 +23,22 @@ def authorization_scope_param = "user_scope" def scope_separator = "," def extra_authorization_params = {} def refreshable? = true + # Slack app token rotation is deployment-specific. When token rotation is + # enabled Slack returns a refresh_token and the broker loop keeps it fresh; + # when it is disabled Slack returns long-lived xoxp/xoxb tokens without a + # refresh_token, which should be stored without scheduling refresh. + def require_refresh_token? = false + def refreshable_result?(result) = result.refresh_token.present? + + def validate_result!(result) + return unless result.refresh_token.blank? && result.expires_in.present? + + raise Broker::ExchangeError.new( + "token endpoint returned expiring Slack token without refresh_token", + stage: "oauth", + code: "missing_refresh_token" + ) + end def parse_granted_scopes(scope) scope.to_s.split(/[,\s]+/).reject(&:blank?) @@ -41,6 +57,16 @@ def identity_from(result, client_id:) } end + bot_user_id = result.response&.dig("bot_user_id") + if bot_user_id.present? + return { + subject: bot_user_id, + email: nil, + name: slack_bot_name(result.response), + team_id: slack_team_id(result.response) + } + end + Login::IdToken.identity(result.id_token, client_id: client_id, valid_issuers: VALID_ISSUERS) .slice(:subject, :email, :name) @@ -57,6 +83,10 @@ def slack_team_id(response) response.dig("team", "id").presence || response.dig("authed_user", "team_id").presence end + + def slack_bot_name(response) + response.dig("team", "name").presence || response["bot_user_id"].presence + end end end end diff --git a/services/console/lib/sandbox_entitlements/jwt.rb b/services/console/lib/sandbox_entitlements/jwt.rb new file mode 100644 index 000000000..5ba690524 --- /dev/null +++ b/services/console/lib/sandbox_entitlements/jwt.rb @@ -0,0 +1,56 @@ +module SandboxEntitlements + module Jwt + DEFAULT_AUDIENCE = "centaur-console-sandbox-entitlements".freeze + DEFAULT_ISSUER = "centaur-console".freeze + # The token is low-sensitivity (scoped to read-only sandbox endpoints). The + # permissions endpoint re-validates the proxy -> principal binding against + # the database on every request, so reassignment revokes permissions access + # immediately regardless of exp. Rotation is + # therefore infrequent: it exists to bound the lifetime of a leaked token, + # not to enforce freshness. Keep the window long — every rotation changes + # the synced config hash and forces a full config re-push to the proxy. + # The TTL must comfortably exceed the window: iat is floored to the window + # start, so a token delivered late in a window carries TTL - WINDOW of + # remaining validity, which must cover proxy sync stalls. + DEFAULT_WINDOW_SECONDS = 1.day.to_i + DEFAULT_TTL_SECONDS = 3.days.to_i + + module_function + + def encode_for_proxy(proxy, now: Time.current) + return nil unless proxy.assigned? + + CentaurJwt::WindowedToken.encode( + subject_oid: proxy.oid, + audience: audience, + issuer: issuer, + window_seconds: DEFAULT_WINDOW_SECONDS, + ttl_seconds: DEFAULT_TTL_SECONDS, + now: now, + claims: { + "sub" => proxy.name, + "sandbox_id" => proxy.name, + "proxy_id" => proxy.oid, + "principal_id" => proxy.principal&.oid + } + ) + end + + def decode(token) + CentaurJwt::Hs256.decode( + token, + signing_secret: ENV["CENTAUR_JWT_SIGNING_SECRET"].to_s, + aud: audience, + iss: issuer + ) + end + + def audience + ENV["CENTAUR_SANDBOX_ENTITLEMENTS_JWT_AUDIENCE"].presence || DEFAULT_AUDIENCE + end + + def issuer + ENV["CENTAUR_SANDBOX_ENTITLEMENTS_JWT_ISSUER"].presence || DEFAULT_ISSUER + end + end +end diff --git a/services/console/test/controllers/api/v1/api_keys_controller_test.rb b/services/console/test/controllers/api/v1/api_keys_controller_test.rb index deba31ae8..0b3f396df 100644 --- a/services/console/test/controllers/api/v1/api_keys_controller_test.rb +++ b/services/console/test/controllers/api/v1/api_keys_controller_test.rb @@ -30,6 +30,17 @@ def json_body assert_response :unauthorized end + test "rejects API keys for disabled users" do + get api_v1_api_keys_url, headers: auth_headers("iak_disabled-token") + assert_response :unauthorized + end + + test "rejects API keys for non-admin users" do + get api_v1_api_keys_url, headers: auth_headers("iak_member-token") + assert_response :forbidden + assert_equal "API key owner is not an admin", json_body.dig("error", "message") + end + test "GET index lists only the caller's keys and never returns plaintext token" do get api_v1_api_keys_url, headers: auth_headers assert_response :ok diff --git a/services/console/test/controllers/api/v1/proxy_sync_controller_test.rb b/services/console/test/controllers/api/v1/proxy_sync_controller_test.rb index d09ca6804..cbcefc86d 100644 --- a/services/console/test/controllers/api/v1/proxy_sync_controller_test.rb +++ b/services/console/test/controllers/api/v1/proxy_sync_controller_test.rb @@ -52,6 +52,48 @@ def json_body refute body.key?("ingest_token") end + test "sync includes a scoped sandbox entitlements token when signing is configured" do + with_env( + "CENTAUR_JWT_SIGNING_SECRET" => "test-secret", + "CENTAUR_CONSOLE_URL" => "http://centaur-console:3000" + ) do + post api_v1_proxy_sync_url, params: {}.to_json, headers: auth_headers + end + assert_response :ok + + entry = json_body.fetch("secrets").find do |secret| + secret.dig("inject", "header") == "Authorization" && + secret.fetch("rules").any? { |rule| rule["paths"] == [ Proxy::SANDBOX_ENTITLEMENTS_PATH_PATTERN ] } + end + + refute_nil entry + assert_equal "Bearer {{ .Value }}", entry.dig("inject", "formatter") + assert_equal( + { "host" => "centaur-console", "methods" => [ "GET" ], "paths" => [ Proxy::SANDBOX_ENTITLEMENTS_PATH_PATTERN ] }, + entry.fetch("rules").first + ) + + claims = jwt_payload(entry.dig("source", "value")) + assert_equal @proxy.name, claims.fetch("sandbox_id") + assert_equal @proxy.oid, claims.fetch("proxy_id") + assert_equal @proxy.principal.oid, claims.fetch("principal_id") + assert_equal "centaur-console-sandbox-entitlements", claims.fetch("aud") + assert_equal SandboxEntitlements::Jwt::DEFAULT_TTL_SECONDS, claims.fetch("exp") - claims.fetch("iat") + end + + test "sync omits the sandbox entitlements token when the console URL is not configured" do + with_env("CENTAUR_JWT_SIGNING_SECRET" => "test-secret", "CENTAUR_CONSOLE_URL" => nil) do + post api_v1_proxy_sync_url, params: {}.to_json, headers: auth_headers + end + assert_response :ok + + entry = json_body.fetch("secrets").find do |secret| + secret.fetch("rules", []).any? { |rule| rule["paths"] == [ Proxy::SANDBOX_ENTITLEMENTS_PATH_PATTERN ] } + end + + assert_nil entry + end + test "cold sync stores an encrypted principal snapshot" do assert_difference -> { PrincipalSyncConfigSnapshot.count }, 1 do post api_v1_proxy_sync_url, params: {}.to_json, headers: auth_headers @@ -361,7 +403,9 @@ def json_body test "an unassigned proxy syncs an empty config with unassigned status" do unassigned_token = "iprx_#{'c' * 64}" - post api_v1_proxy_sync_url, params: {}.to_json, headers: auth_headers(unassigned_token) + with_env("CENTAUR_JWT_SIGNING_SECRET" => "test-secret") do + post api_v1_proxy_sync_url, params: {}.to_json, headers: auth_headers(unassigned_token) + end assert_response :ok body = json_body @@ -409,4 +453,21 @@ def json_body entry = json_body.fetch("secrets").find { |s| s.dig("source", "value") == "token-2" } refute_nil entry end + + def jwt_payload(token) + _header, payload, _signature = token.split(".") + JSON.parse(Base64.urlsafe_decode64(payload)) + end + + def with_env(values) + previous = values.keys.to_h { |key| [ key, ENV[key] ] } + values.each do |key, value| + value.nil? ? ENV.delete(key) : ENV[key] = value + end + yield + ensure + previous.each do |key, value| + value.nil? ? ENV.delete(key) : ENV[key] = value + end + end end diff --git a/services/console/test/controllers/api/v1/sandbox_oauth_apps_controller_test.rb b/services/console/test/controllers/api/v1/sandbox_oauth_apps_controller_test.rb new file mode 100644 index 000000000..746f1eb24 --- /dev/null +++ b/services/console/test/controllers/api/v1/sandbox_oauth_apps_controller_test.rb @@ -0,0 +1,87 @@ +require "test_helper" + +module Api + module V1 + class SandboxOauthAppsControllerTest < ActionDispatch::IntegrationTest + setup do + @proxy = proxies(:acme_proxy) + end + + test "returns enabled OAuth app start URLs for a valid sandbox token" do + with_env( + "CENTAUR_JWT_SIGNING_SECRET" => "test-secret", + "CENTAUR_CONSOLE_PUBLIC_URL" => "https://console.example.test" + ) do + get "/api/v1/sandbox/oauth_apps", headers: auth_headers(token_for(@proxy)) + end + assert_response :ok + + data = json_body.fetch("data") + slugs = data.map { |app| app.fetch("slug") } + assert_equal slugs.sort, slugs + assert_includes slugs, "google" + refute_includes slugs, "google-disabled" + + google = data.find { |app| app.fetch("slug") == "google" } + assert_equal oauth_apps(:acme_google).oid, google.fetch("id") + assert_equal "google", google.fetch("provider") + assert_equal({ "team" => "comms" }, google.fetch("labels")) + assert_equal oauth_apps(:acme_google).allowed_scopes, google.fetch("allowed_scopes") + assert_equal "https://console.example.test/oauth/google/start", google.fetch("start_url") + refute google.key?("client_id") + end + + test "does not reject a valid token when proxy claims are stale" do + with_env("CENTAUR_JWT_SIGNING_SECRET" => "test-secret") do + token = token_for(@proxy) + @proxy.update!(principal: principals(:globex_user)) + + get "/api/v1/sandbox/oauth_apps", headers: auth_headers(token) + end + + assert_response :ok + end + + test "rejects requests without a sandbox token" do + get "/api/v1/sandbox/oauth_apps" + assert_response :unauthorized + end + + test "rejects expired sandbox tokens" do + with_env("CENTAUR_JWT_SIGNING_SECRET" => "test-secret") do + token = token_for(@proxy, now: (SandboxEntitlements::Jwt::DEFAULT_TTL_SECONDS + 1.hour).seconds.ago) + + get "/api/v1/sandbox/oauth_apps", headers: auth_headers(token) + end + + assert_response :unauthorized + end + + private + + def auth_headers(token) + { "Authorization" => "Bearer #{token}" } + end + + def token_for(proxy, now: Time.current) + SandboxEntitlements::Jwt.encode_for_proxy(proxy, now: now) + end + + def json_body + JSON.parse(response.body) + end + + def with_env(values) + previous = values.keys.to_h { |key| [ key, ENV[key] ] } + values.each do |key, value| + value.nil? ? ENV.delete(key) : ENV[key] = value + end + yield + ensure + previous.each do |key, value| + value.nil? ? ENV.delete(key) : ENV[key] = value + end + end + end + end +end diff --git a/services/console/test/controllers/api/v1/sandbox_permissions_controller_test.rb b/services/console/test/controllers/api/v1/sandbox_permissions_controller_test.rb new file mode 100644 index 000000000..228843126 --- /dev/null +++ b/services/console/test/controllers/api/v1/sandbox_permissions_controller_test.rb @@ -0,0 +1,134 @@ +require "test_helper" + +module Api + module V1 + class SandboxPermissionsControllerTest < ActionDispatch::IntegrationTest + setup do + @proxy = proxies(:acme_proxy) + SecretSource.create!( + source_type: "control_plane", + secret: "s3cr3t-db-pass", + static_secret: static_secrets(:db_password_replace) + ) + SlackChannelPermission.create!( + principal: @proxy.principal, + channel_id: "C0123456789", + channel_name: "general", + upload_enabled: true, + history_enabled: true + ) + end + + test "returns redacted sandbox permissions for a valid sandbox token" do + credential = BrokerCredential.create!( + namespace: @proxy.principal.namespace, + foreign_id: "google-personal", + name: "Google - Personal User", + token_endpoint: "https://oauth2.googleapis.com/token", + oauth_app: oauth_apps(:acme_google), + provider_email: "person@example.com", + provider_subject: "google-sub-1", + scopes: [ "https://www.googleapis.com/auth/gmail.readonly" ], + refresh_token: "refresh-token", + access_token: "access-token", + expires_at: 1.hour.from_now, + last_refresh: Time.current + ) + secret = StaticSecret.new( + namespace: @proxy.principal.namespace, + name: "Google - Personal User token", + broker_credential: credential, + inject_config: { "header" => "Authorization", "formatter" => "Bearer {{ .Value }}" } + ) + secret.build_source(source_type: "token_broker", config: { "credential_id" => credential.oid }) + secret.rules.build(host: "www.googleapis.com", position: 0) + secret.save! + Grant.create!(principal: @proxy.principal, static_secret: secret, created_by: users(:acme_admin)) + + with_env("CENTAUR_JWT_SIGNING_SECRET" => "test-secret") do + get "/api/v1/sandbox/permissions", headers: auth_headers(token_for(@proxy)) + end + assert_response :ok + + data = json_body.fetch("data") + assert_equal @proxy.name, data.fetch("sandbox_id") + assert_equal @proxy.oid, data.fetch("proxy_id") + assert_equal @proxy.principal.oid, data.fetch("principal_id") + assert_equal @proxy.principal.namespace, data.dig("principal", "namespace") + assert_equal @proxy.principal.sandbox_repo_cache, data.dig("capabilities", "sandbox_repo_cache") + assert_equal 1, data.fetch("slack_channel_permissions").length + assert_equal [ + { + "id" => credential.oid, + "oauth_app_id" => oauth_apps(:acme_google).oid, + "slug" => "google", + "provider" => "google", + "provider_email" => "person@example.com", + "provider_subject" => "google-sub-1", + "status" => "live", + "scopes" => [ "https://www.googleapis.com/auth/gmail.readonly" ] + } + ], data.fetch("oauth_credentials") + + entry = data.dig("permissions", "secrets").find { |secret| secret.dig("source", "type") == "control_plane" } + refute_nil entry + assert_equal "[redacted]", entry.dig("source", "value") + refute_includes response.body, "s3cr3t-db-pass" + assert_equal "no-store", response.headers["Cache-Control"] + assert_match(/\A"[0-9a-f]{64}"\z/, response.headers["ETag"]) + end + + test "rejects requests without a sandbox token" do + get "/api/v1/sandbox/permissions" + assert_response :unauthorized + end + + test "rejects tokens after proxy assignment changes" do + with_env("CENTAUR_JWT_SIGNING_SECRET" => "test-secret") do + token = token_for(@proxy) + @proxy.update!(principal: principals(:globex_user)) + + get "/api/v1/sandbox/permissions", headers: auth_headers(token) + end + + assert_response :unauthorized + end + + test "rejects expired sandbox tokens" do + with_env("CENTAUR_JWT_SIGNING_SECRET" => "test-secret") do + token = token_for(@proxy, now: (SandboxEntitlements::Jwt::DEFAULT_TTL_SECONDS + 1.hour).seconds.ago) + + get "/api/v1/sandbox/permissions", headers: auth_headers(token) + end + + assert_response :unauthorized + end + + private + + def auth_headers(token) + { "Authorization" => "Bearer #{token}" } + end + + def token_for(proxy, now: Time.current) + SandboxEntitlements::Jwt.encode_for_proxy(proxy, now: now) + end + + def json_body + JSON.parse(response.body) + end + + def with_env(values) + previous = values.keys.to_h { |key| [ key, ENV[key] ] } + values.each do |key, value| + value.nil? ? ENV.delete(key) : ENV[key] = value + end + yield + ensure + previous.each do |key, value| + value.nil? ? ENV.delete(key) : ENV[key] = value + end + end + end + end +end diff --git a/services/console/test/controllers/console/threads_controller_test.rb b/services/console/test/controllers/console/threads_controller_test.rb index 14e58c8ba..d5f14cf7a 100644 --- a/services/console/test/controllers/console/threads_controller_test.rb +++ b/services/console/test/controllers/console/threads_controller_test.rb @@ -101,6 +101,122 @@ class Console::ThreadsControllerTest < ActionDispatch::IntegrationTest assert_select "body", text: /Chat not found/ end + test "public Slack channel threads are readable by every console user only when enabled" do + skip_unless_session_table + skip_unless_slack_channel_table + + public_channel_id = "C#{SecureRandom.hex(6).upcase}" + private_channel_id = "C#{SecureRandom.hex(6).upcase}" + removed_channel_id = "C#{SecureRandom.hex(6).upcase}" + public_thread_key = "slack:#{public_channel_id}:#{SecureRandom.hex(6)}" + private_thread_key = "slack:#{private_channel_id}:#{SecureRandom.hex(6)}" + removed_thread_key = "slack:#{removed_channel_id}:#{SecureRandom.hex(6)}" + insert_slack_sync_channel(public_channel_id, is_private: false) + insert_slack_sync_channel(private_channel_id, is_private: true) + insert_slack_sync_channel(removed_channel_id, is_private: false, is_syncable: false) + insert_slack_session(public_thread_key, slack_user_id: "U_OTHER", slack_user_name: "someone-else") + insert_slack_session(private_thread_key, slack_user_id: "U_OTHER", slack_user_name: "someone-else") + insert_slack_session(removed_thread_key, slack_user_id: "U_OTHER", slack_user_name: "someone-else") + + with_env( + "CENTAUR_CONSOLE_PUBLIC_SLACK_THREADS_ENABLED" => nil, + "IRON_CONTROL_PUBLIC_SLACK_THREADS_ENABLED" => nil + ) do + get console_threads_url(thread: public_thread_key) + assert_response :not_found + end + + with_env("CENTAUR_CONSOLE_PUBLIC_SLACK_THREADS_ENABLED" => "true") do + get console_threads_url(thread: public_thread_key) + assert_response :ok + assert_select ".console-thread-detail-header", count: 1 + assert_select "textarea[name=prompt]", count: 0 + + get console_threads_url(thread: private_thread_key) + assert_response :not_found + + get console_threads_url(thread: removed_thread_key) + assert_response :not_found + end + end + + test "sharing publishes a direct read-only link from an in-page copy dialog" do + skip_unless_session_table + + thread_key = "console:shared-#{SecureRandom.hex(6)}" + insert_console_session(thread_key) + + get console_threads_url(thread: thread_key) + + assert_response :ok + assert_select "button.console-thread-share-trigger[aria-label=?][data-action=?]", + "Share chat", "thread-share#open", count: 1 do + assert_select "svg", count: 1 + end + assert_select ".console-thread-menu", count: 0 + assert_select "button[data-turbo-confirm]", count: 0 + assert_select "dialog.console-share-dialog[data-thread-share-target=dialog]" do + assert_select "h2", text: "Share chat" + assert_select "p", text: "Anyone with access to Centaur Console will be able to view this chat." + assert_select "form[action=?][data-action*=?]", console_thread_share_path, "thread-share#copyLink" do + assert_select "input[name=thread_key][value=?]", thread_key + assert_select "button.btn-secondary[type=button]", text: "Cancel" + assert_select "button.btn-primary[type=submit]", text: "Copy link" + end + end + + post console_thread_share_url, params: { thread_key: thread_key }, as: :json + + assert_response :ok + assert_equal console_threads_url(thread: thread_key), response.parsed_body.fetch("url") + assert_equal @operator, ThreadShare.find_by!(thread_key: thread_key).created_by + + post console_thread_share_url, params: { thread_key: thread_key } + + assert_redirected_to console_threads_path(thread: thread_key) + assert_nil flash[:notice] + assert_equal 1, ThreadShare.where(thread_key: thread_key).count + + delete logout_url + post login_url, params: { email: users(:member_user).email, password: "password123456" } + get console_threads_url(thread: thread_key) + + assert_response :ok + assert_select ".console-thread-detail-header", count: 1 + assert_select "textarea[name=prompt]", count: 0 + end + + test "a user cannot share a chat they cannot read" do + skip_unless_session_table + + thread_key = "slack:G0PRIVATE12:#{SecureRandom.hex(6)}" + insert_slack_session(thread_key, slack_user_id: "U_OTHER", slack_user_name: "someone-else") + + post console_thread_share_url, params: { thread_key: thread_key } + + assert_redirected_to console_threads_path + assert_equal "Chat not found.", flash[:alert] + assert_not ThreadShare.exists?(thread_key: thread_key) + end + + test "a non-owner cannot persistently share a deployment-public Slack thread" do + skip_unless_session_table + skip_unless_slack_channel_table + + channel_id = "C#{SecureRandom.hex(6).upcase}" + thread_key = "slack:#{channel_id}:#{SecureRandom.hex(6)}" + insert_slack_sync_channel(channel_id, is_private: false) + insert_slack_session(thread_key, slack_user_id: "U_OTHER", slack_user_name: "someone-else") + + with_env("CENTAUR_CONSOLE_PUBLIC_SLACK_THREADS_ENABLED" => "true") do + post console_thread_share_url, params: { thread_key: thread_key } + end + + assert_redirected_to console_threads_path + assert_equal "Chat not found.", flash[:alert] + assert_not ThreadShare.exists?(thread_key: thread_key) + end + test "slack assistant-role messages from the current Slack user render as user authored" do controller = Console::ThreadsController.new controller.define_singleton_method(:current_slack_user_ids) { [ "u123" ] } @@ -495,6 +611,33 @@ class Console::ThreadsControllerTest < ActionDispatch::IntegrationTest refute_includes sql, "slack_user_id" end + test "public Slack thread visibility defaults off and never expands the owner scope" do + controller = threads_controller_for(@operator) + + with_env( + "CENTAUR_CONSOLE_PUBLIC_SLACK_THREADS_ENABLED" => nil, + "IRON_CONTROL_PUBLIC_SLACK_THREADS_ENABLED" => nil + ) do + refute_includes controller.send(:visible_thread_scope).to_sql, "slack_sync_channels" + end + + with_env("CENTAUR_CONSOLE_PUBLIC_SLACK_THREADS_ENABLED" => "true") do + if slack_channel_privacy_catalog_available? + assert_includes controller.send(:visible_thread_scope).to_sql, "slack_sync_channels" + end + refute_includes controller.send(:owned_thread_scope).to_sql, "slack_sync_channels" + end + end + + test "public Slack visibility fails closed without the synchronized channel catalog" do + connection = CentaurSession.connection + replacement = ->(_table) { false } + + with_singleton_method(connection, :data_source_exists?, replacement) do + assert_nil CentaurSession.public_slack_channel_sql + end + end + test "visible thread scope matches Slack threads by user id when the credential has no team" do app = oauth_apps(:acme_slack) app.update!(client_secret: "slack-secret", labels: {}) @@ -573,6 +716,18 @@ class Console::ThreadsControllerTest < ActionDispatch::IntegrationTest assert_includes sql, "ussoonly" end + test "sidebar includes public Slack threads only when the deploy setting is enabled" do + controller = threads_controller_for(@operator) + + with_env("CENTAUR_CONSOLE_PUBLIC_SLACK_THREADS_ENABLED" => "true") do + sql = controller.send(:console_sidebar_visible_thread_scope).to_sql + + if slack_channel_privacy_catalog_available? + assert_includes sql, "slack_sync_channels" + end + end + end + test "selected session resolves a directly linked thread only within the owner scope" do controller = Console::ThreadsController.new owned_thread = SelectedSession.new(thread_key: "slack:C123:1782339173.755169") @@ -649,6 +804,65 @@ class Console::ThreadsControllerTest < ActionDispatch::IntegrationTest assert_select "[data-console-thinking-indicator]", count: 0 end + test "an active thread wires a per-panel poller instead of a full-page refresh" do + skip_unless_session_table + thread_key = "console:poller-active-#{SecureRandom.hex(6)}" + insert_console_session(thread_key) + insert_session_execution(thread_key, status: "running") + + get console_threads_url(thread: thread_key) + + assert_response :ok + assert_select "[data-controller=thread-poller][data-thread-poller-active-value=true]", count: 1 + assert_select "[data-thread-poller-url-value=?]", + console_thread_panel_path(thread_key: thread_key), + count: 1 + # The old behavior re-rendered the whole console with a Turbo visit while + # any pane was executing; that script must stay gone. + assert_no_match "Turbo.visit(window.location.href", response.body + end + + test "panel poll renders one thread's transcript with the active header" do + skip_unless_session_table + thread_key = "console:poller-panel-#{SecureRandom.hex(6)}" + insert_console_session(thread_key) + insert_session_message(thread_key, index: 1) + insert_session_execution(thread_key, status: "running") + + get console_thread_panel_url(thread_key: thread_key) + + assert_response :ok + assert_equal "true", response.headers["X-Console-Execution-Active"] + assert_select "[data-console-thinking-indicator]", count: 1 + assert_match "message 1", response.body + # Transcript stream only: no layout, no composer, no panel chrome. + assert_select "textarea[name=prompt]", count: 0 + assert_select "[data-thread-panel]", count: 0 + end + + test "panel poll reports inactive once the execution completes" do + skip_unless_session_table + thread_key = "console:poller-done-#{SecureRandom.hex(6)}" + insert_console_session(thread_key) + insert_session_execution(thread_key, status: "completed") + + get console_thread_panel_url(thread_key: thread_key) + + assert_response :ok + assert_equal "false", response.headers["X-Console-Execution-Active"] + assert_select "[data-console-thinking-indicator]", count: 0 + end + + test "panel poll is scoped to threads the current user can read" do + skip_unless_session_table + thread_key = "slack:C0POLL:#{SecureRandom.hex(6)}" + insert_slack_session(thread_key, slack_user_id: "U_OTHER", slack_user_name: "someone-else") + + get console_thread_panel_url(thread_key: thread_key) + + assert_response :not_found + end + test "a new sentinel pane opens a composer panel alongside a thread" do skip_unless_session_table insert_console_session("console:with-new-pane") @@ -712,10 +926,21 @@ class Console::ThreadsControllerTest < ActionDispatch::IntegrationTest end test "starting a chat creates a session, appends the prompt, and executes it" do + @operator.update!(name: "Ada Admin") + UserIdentity.create!(user: @operator, provider: "slack", subject: "UADA") client = RecordingApiClient.new - with_composer(client: client) do - post console_threads_url, - params: { prompt: "Reply with PONG.", model: "claude-opus-4-8" } + identity = SlackRequesterIdentity::Result.new( + handle: "@ada", source: 'Slack profile custom field "GitHub"', reason: nil + ) + test_case = self + with_singleton_method(SlackRequesterIdentity, :resolve, ->(user_ids:) { + test_case.assert_includes user_ids, "uada" + identity + }) do + with_composer(client: client) do + post console_threads_url, + params: { prompt: "Reply with PONG.", model: "claude-opus-4-8" } + end end assert_equal %i[create_session append_session_messages execute_session], client.calls.map(&:first) @@ -744,7 +969,12 @@ class Console::ThreadsControllerTest < ActionDispatch::IntegrationTest assert_equal create[:thread_key], line["thread_key"] assert_equal "claude-opus-4-8", line["model"] assert_equal message[:client_message_id], line["client_user_message_id"] - assert_equal "Reply with PONG.", line.dig("message", "content", 0, "text") + requester_context = line.dig("message", "content", 0, "text") + assert_includes requester_context, "# Requester Context" + assert_includes requester_context, "Prompted by: @ada" + assert_includes requester_context, 'GitHub handle source: Slack profile custom field "GitHub"' + assert_includes requester_context, "GitHub handle verified: yes" + assert_equal "Reply with PONG.", line.dig("message", "content", 1, "text") assert_redirected_to console_threads_path(thread: create[:thread_key]) end @@ -1464,6 +1694,36 @@ def skip_unless_session_table skip("api-rs session tables are unavailable") unless CentaurSession.connection.data_source_exists?("sessions") end + def skip_unless_slack_channel_table + return if slack_channel_privacy_catalog_available? + + skip("Slack channel privacy catalog is unavailable") + end + + def slack_channel_privacy_catalog_available? + return false unless CentaurSession.connection.data_source_exists?(:slack_sync_channels) + + %i[is_private is_syncable].all? do |column| + CentaurSession.connection.column_exists?(:slack_sync_channels, column) + end + end + + def insert_slack_sync_channel(channel_id, is_private:, is_syncable: true) + connection = CentaurSession.connection + connection.execute(<<~SQL.squish) + insert into slack_sync_channels (channel_id, channel_name, is_private, is_syncable) + values ( + #{connection.quote(channel_id)}, + #{connection.quote(channel_id.downcase)}, + #{connection.quote(is_private)}, + #{connection.quote(is_syncable)} + ) + on conflict (channel_id) do update set + is_private = excluded.is_private, + is_syncable = excluded.is_syncable + SQL + end + def insert_slack_session(thread_key, slack_user_id:, slack_user_name:) metadata = { source: "slackbotv2", @@ -1577,4 +1837,13 @@ def insert_session(thread_key, metadata) ) SQL end + + def with_singleton_method(object, method_name, replacement) + singleton = object.singleton_class + original = singleton.instance_method(method_name) + singleton.define_method(method_name, replacement) + yield + ensure + singleton.define_method(method_name, original) + end end diff --git a/services/console/test/controllers/mcp/oauth_controller_test.rb b/services/console/test/controllers/mcp/oauth_controller_test.rb index d991b8a26..aad94b628 100644 --- a/services/console/test/controllers/mcp/oauth_controller_test.rb +++ b/services/console/test/controllers/mcp/oauth_controller_test.rb @@ -57,12 +57,28 @@ class OauthControllerTest < ActionDispatch::IntegrationTest assert_equal "mcp:tools", body.fetch("scope") end - test "dynamic client registration rejects non-loopback redirect URIs" do + test "dynamic client registration creates a hosted HTTPS client" do + assert_difference -> { McpOauthClient.count }, 1 do + post "/mcp/oauth/register", + params: { + client_name: "Claude", + redirect_uris: [ "https://claude.ai/api/mcp/auth_callback" ], + scope: "mcp:tools" + }, + as: :json + end + + assert_response :created + body = JSON.parse(response.body) + assert_equal [ "https://claude.ai/api/mcp/auth_callback" ], body.fetch("redirect_uris") + end + + test "dynamic client registration rejects non-loopback plain HTTP redirect URIs" do assert_no_difference -> { McpOauthClient.count } do post "/mcp/oauth/register", params: { client_name: "Attacker", - redirect_uris: [ "https://evil.example/callback" ], + redirect_uris: [ "http://evil.example/callback" ], scope: "mcp:tools" }, as: :json @@ -72,14 +88,14 @@ class OauthControllerTest < ActionDispatch::IntegrationTest assert_equal "invalid_client_metadata", JSON.parse(response.body).fetch("error") end - test "authorize rejects non-loopback redirect URIs even when already stored" do + test "authorize rejects non-loopback plain HTTP redirect URIs even when already stored" do client = create_client - client.update_column(:redirect_uris, [ "https://evil.example/callback" ]) + client.update_column(:redirect_uris, [ "http://evil.example/callback" ]) post login_url, params: { email: @operator.email, password: "password123456" } assert_no_difference -> { McpOauthAuthorizationCode.count } do get "/mcp/oauth/authorize", - params: authorize_params(client).merge(redirect_uri: "https://evil.example/callback") + params: authorize_params(client).merge(redirect_uri: "http://evil.example/callback") end assert_response :bad_request diff --git a/services/console/test/controllers/oauth/flows_controller_test.rb b/services/console/test/controllers/oauth/flows_controller_test.rb index ec30bd5ab..78ce9530f 100644 --- a/services/console/test/controllers/oauth/flows_controller_test.rb +++ b/services/console/test/controllers/oauth/flows_controller_test.rb @@ -3,7 +3,7 @@ module Oauth # Covers the consent flow end to end: /oauth/:slug/start builds the IdP redirect # and binds the browser; /oauth/:slug/callback exchanges the code, upserts a - # BrokerCredential, and renders an iron-control result page. The IdP is faked by + # BrokerCredential, and renders a console result page. The IdP is faked by # swapping the controller's exchange_client_factory for a client wrapped around # an HTTP double returning a canned token response. class FlowsControllerTest < ActionDispatch::IntegrationTest @@ -22,6 +22,8 @@ class FlowsControllerTest < ActionDispatch::IntegrationTest oauth_apps(:acme_github).update!(client_secret: "github-secret") oauth_apps(:acme_attio).update!(client_secret: "attio-secret") oauth_apps(:acme_linear).update!(client_secret: "linear-secret") + @user = users(:member_user) + sign_in @user clear_enqueued_jobs end @@ -75,6 +77,33 @@ def slack_token_body(sub: "U0R7MFMJM", scope: "chat:write", id_token_value: nil, }.merge(overrides).to_json end + def slack_bot_token_body(**overrides) + { + ok: true, + access_token: "xoxb-non-rotating-bot", + token_type: "bot", + scope: "commands,chat:write", + bot_user_id: "U0BOTUSER", + app_id: "A0APP", + team: { id: "TACME", name: "Acme" } + }.merge(overrides).to_json + end + + def slack_static_user_token_body + slack_token_body( + access_token: "xoxb-non-rotating-bot", + refresh_token: nil, + expires_in: nil, + authed_user: { + id: "USTATIC", + user: "static-grace", + access_token: "xoxp-non-rotating-user", + scope: "chat:write", + token_type: "user" + } + ) + end + def github_token_body(scope: "repo,read:user", **overrides) { access_token: "gho-user-token", @@ -104,6 +133,10 @@ def sign_in(user) post login_url, params: { email: user.email, password: "password123456" } end + def sign_out + delete logout_url + end + # Runs /start and returns the state extracted from the IdP redirect (the flow # cookie is set in the shared integration cookie jar as a side effect). def start_flow(slug: "google", **params) @@ -210,19 +243,23 @@ def start_flow(slug: "google", **params) assert_includes scopes, "write" end - test "start works without any session" do + test "start redirects signed-out users to login" do + sign_out + get oauth_start_url(slug: "google") + assert_response :redirect + assert_redirected_to login_path assert_nil session[:user_id] end - test "start works with a pending console session" do + test "start redirects pending console users to the pending page" do sign_in users(:pending_user) get oauth_start_url(slug: "google") assert_response :redirect - assert_equal "accounts.google.com", URI.parse(response.location).host + assert_redirected_to pending_path end test "start 404s an unknown slug" do @@ -273,7 +310,7 @@ def start_flow(slug: "google", **params) assert_equal "AT", cred.access_token assert_equal "RT", cred.refresh_token assert cred.next_attempt_at.present? - assert_nil cred.created_by + assert_equal @user, cred.created_by end test "callback happy path supports Slack user tokens" do @@ -296,11 +333,58 @@ def start_flow(slug: "google", **params) assert_equal %w[chat:write], cred.scopes assert_equal "xoxe.xoxp-1-user", cred.access_token assert_equal "xoxe-1-refresh", cred.refresh_token + assert cred.next_attempt_at.present? assert_equal "TACME", cred.labels["slack_team_id"] assert_equal [ "slack.com" ], cred.static_secret.rules.map(&:host) assert_equal "Slack – grace token", cred.static_secret.name end + test "callback happy path supports non-rotating Slack user tokens" do + state = start_flow(slug: "slack", scopes: "chat:write") + stub_exchange(status: 200, body: slack_static_user_token_body) + + assert_difference -> { BrokerCredential.count } => 1 do + get oauth_callback_url(slug: "slack"), params: { state: state, code: "auth-code" } + end + assert_redirected_to console_integrations_path + + app = oauth_apps(:acme_slack) + cred = BrokerCredential.find_by(oauth_app: app, provider_subject: "USTATIC") + assert_equal "Slack – static-grace", cred.name + assert_equal %w[chat:write], cred.scopes + assert_equal "xoxp-non-rotating-user", cred.access_token + assert_nil cred.refresh_token + assert_nil cred.expires_at + assert_nil cred.next_attempt_at + refute_includes BrokerCredential.refreshable, cred + end + + test "callback happy path supports non-rotating Slack bot tokens" do + state = start_flow(slug: "slack", scopes: "chat:write") + stub_exchange(status: 200, body: slack_bot_token_body) + + assert_difference -> { BrokerCredential.count } => 1 do + get oauth_callback_url(slug: "slack"), params: { state: state, code: "auth-code" } + end + assert_redirected_to console_integrations_path + + app = oauth_apps(:acme_slack) + cred = BrokerCredential.find_by(oauth_app: app, provider_subject: "U0BOTUSER") + assert_equal "acme", cred.namespace + assert_equal "slack-slack-u0botuser", cred.foreign_id + assert_equal "Slack – Acme", cred.name + assert_equal "https://slack.com/api/oauth.v2.access", cred.token_endpoint + assert_nil cred.provider_email + assert_equal %w[commands chat:write], cred.scopes + assert_equal "xoxb-non-rotating-bot", cred.access_token + assert_nil cred.refresh_token + assert_nil cred.expires_at + assert_nil cred.next_attempt_at + assert_equal "TACME", cred.labels["slack_team_id"] + assert_equal [ "slack.com" ], cred.static_secret.rules.map(&:host) + refute_includes BrokerCredential.refreshable, cred + end + test "callback happy path supports Attio workspace tokens" do state = start_flow(slug: "attio", scopes: "record_permission:read") stub_exchange(status: 200, body: attio_token_body) @@ -400,7 +484,7 @@ def start_flow(slug: "google", **params) assert_equal cred, secret.broker_credential # first-class link to the credential assert_equal cred.namespace, secret.namespace assert_nil secret.foreign_id # found by association, so no collidable foreign_id - assert_nil secret.created_by # the unauthenticated flow has no operator + assert_nil secret.created_by # the wrapping secret is not owned by an operator assert_equal({ "header" => "Authorization", "formatter" => "Bearer {{ .Value }}" }, secret.inject_config) assert_equal "token_broker", secret.source.source_type assert_equal cred.oid, secret.source.config["credential_id"] @@ -463,20 +547,31 @@ def start_flow(slug: "google", **params) assert_select "a.btn-secondary[href=?]", "http://www.example.com/oauth/slack/start", text: "Reconnect" end - test "callback works with a disabled console session" do + test "callback redirects signed-out users to login and mints nothing" do + state = start_flow + sign_out + stub_exchange(status: 200, body: token_body) + + assert_no_difference -> { BrokerCredential.count } do + get oauth_callback_url(slug: "google"), params: { state: state, code: "auth-code" } + end + + assert_redirected_to login_path + assert_nil session[:user_id] + end + + test "callback rejects a disabled console session before minting" do user = users(:member_user) - sign_in user state = start_flow user.update!(status: :disabled) stub_exchange(status: 200, body: token_body) - assert_difference -> { BrokerCredential.count }, 1 do + assert_no_difference -> { BrokerCredential.count } do get oauth_callback_url(slug: "google"), params: { state: state, code: "auth-code" } end - assert_redirected_to console_integrations_path - assert_match(/connected/, flash[:notice]) - assert_equal user.id, session[:user_id] + assert_redirected_to login_path + assert_nil session[:user_id] end test "re-consent for the same account updates the existing credential and revives a dead one" do diff --git a/services/console/test/controllers/session_oauth_controller_test.rb b/services/console/test/controllers/session_oauth_controller_test.rb index 7ba73379c..d16416e34 100644 --- a/services/console/test/controllers/session_oauth_controller_test.rb +++ b/services/console/test/controllers/session_oauth_controller_test.rb @@ -8,7 +8,8 @@ class SessionOauthControllerTest < ActionDispatch::IntegrationTest GOOGLE_CLIENT_ID = "google-login-client-id".freeze ENV_KEYS = %w[ - CENTAUR_CONSOLE_GOOGLE_CLIENT_ID CENTAUR_CONSOLE_GOOGLE_CLIENT_SECRET CENTAUR_CONSOLE_BOOTSTRAP_ADMINS + CENTAUR_CONSOLE_GOOGLE_CLIENT_ID CENTAUR_CONSOLE_GOOGLE_CLIENT_SECRET + CENTAUR_CONSOLE_BOOTSTRAP_ADMINS CENTAUR_CONSOLE_SSO_EMAIL_DOMAINS ].freeze setup do @@ -114,6 +115,27 @@ def run_callback(sub:, email:, provider: "google", **token_overrides) assert_equal [ [ "google", "new-sub" ] ], user.user_identities.pluck(:provider, :subject) end + test "callback provisions a user inside the configured SSO domain allowlist" do + ENV["CENTAUR_CONSOLE_SSO_EMAIL_DOMAINS"] = "example.com acme.example" + assert_difference -> { User.count }, 1 do + run_callback(sub: "allowed-sub", email: "newcomer@example.com") + end + assert_redirected_to console_threads_path + assert_equal User.find_by!(email: "newcomer@example.com").id, session[:user_id] + end + + test "callback rejects a user outside the configured SSO domain allowlist" do + ENV["CENTAUR_CONSOLE_SSO_EMAIL_DOMAINS"] = "acme.example" + assert_no_difference -> { User.count } do + assert_no_difference -> { UserIdentity.count } do + run_callback(sub: "outside-sub", email: "newcomer@example.com") + end + end + assert_redirected_to login_path + assert_equal "That email domain is not allowed to access the console.", flash[:alert] + assert_nil session[:user_id] + end + test "callback makes a bootstrap-allowlisted email active and admin" do run_callback(sub: "boss-sub", email: "boss@acme.example") assert_redirected_to console_principals_path diff --git a/services/console/test/controllers/sessions_controller_test.rb b/services/console/test/controllers/sessions_controller_test.rb index f2edd6699..e98bda74e 100644 --- a/services/console/test/controllers/sessions_controller_test.rb +++ b/services/console/test/controllers/sessions_controller_test.rb @@ -9,12 +9,32 @@ class SessionsControllerTest < ActionDispatch::IntegrationTest assert_select "form[action=?]", login_path end + test "GET new hides the password form when password login is disabled" do + ENV["CENTAUR_CONSOLE_PASSWORD_LOGIN_ENABLED"] = "false" + get login_url + assert_response :ok + assert_select "form[action=?]", login_path, count: 0 + assert_select "input[name=?]", "email", count: 0 + ensure + ENV.delete("CENTAUR_CONSOLE_PASSWORD_LOGIN_ENABLED") + end + test "valid credentials sign in and redirect to the console" do post login_url, params: { email: @operator.email, password: "password123456" } assert_redirected_to console_principals_path assert_equal @operator.id, session[:user_id] end + test "password login rejects credentials when disabled" do + ENV["CENTAUR_CONSOLE_PASSWORD_LOGIN_ENABLED"] = "false" + post login_url, params: { email: @operator.email, password: "password123456" } + assert_response :not_found + assert_nil session[:user_id] + assert_select "div", /Email and password sign in is disabled/ + ensure + ENV.delete("CENTAUR_CONSOLE_PASSWORD_LOGIN_ENABLED") + end + test "a non-admin lands on the threads view after login" do member = users(:member_user) post login_url, params: { email: member.email, password: "password123456" } diff --git a/services/console/test/fixtures/api_keys.yml b/services/console/test/fixtures/api_keys.yml index 25c8a4d3e..12b16e20f 100644 --- a/services/console/test/fixtures/api_keys.yml +++ b/services/console/test/fixtures/api_keys.yml @@ -12,3 +12,13 @@ globex_ci_key: user: globex_admin name: ci token_hash: <%= Digest::SHA256.hexdigest("iak_globex-ci-token") %> + +member_key: + user: member_user + name: member + token_hash: <%= Digest::SHA256.hexdigest("iak_member-token") %> + +disabled_key: + user: disabled_user + name: disabled + token_hash: <%= Digest::SHA256.hexdigest("iak_disabled-token") %> diff --git a/services/console/test/jobs/granola/jobs_test.rb b/services/console/test/jobs/granola/jobs_test.rb new file mode 100644 index 000000000..a1e800a29 --- /dev/null +++ b/services/console/test/jobs/granola/jobs_test.rb @@ -0,0 +1,50 @@ +require "test_helper" + +module Granola + class JobsTest < ActiveJob::TestCase + def create_credential(app:, access_token: "token", dead: false) + BrokerCredential.create!( + oauth_app: app, + namespace: "acme", + foreign_id: "granola-job-#{SecureRandom.hex(6)}", + token_endpoint: app.provider_strategy.token_endpoint, + access_token: access_token, + refresh_token: "refresh", + last_refresh: Time.current, + expires_at: 1.hour.from_now, + scopes: %w[meetings:read], + provider_subject: "granola-subject-#{SecureRandom.hex(4)}", + provider_email: "person@example.com", + dead: dead + ) + end + + def create_granola_app(enabled: true, slug: "granola") + OauthApp.create!( + provider: "granola", + slug: "#{slug}-#{SecureRandom.hex(6)}", + client_id: "granola-client", + client_secret: "granola-secret", + allowed_scopes: %w[meetings:read], + credential_namespace: "acme", + enabled: enabled, + created_by: users(:acme_admin) + ) + end + + test "poll job only enqueues live credentials for the configured Granola app" do + expected_app = create_granola_app(slug: "granola-sync") + expected = create_credential(app: expected_app) + create_credential(app: expected_app, dead: true) + create_credential(app: create_granola_app(slug: "granola-disabled", enabled: false)) + create_credential(app: create_granola_app(slug: "another-granola-app")) + + PollSyncJob.perform_now(expected_app.slug) + + enqueued_ids = enqueued_jobs + .select { |job| job[:job] == SyncCredentialJob } + .map { |job| job[:args].first } + assert_equal [ expected.id ], enqueued_ids + end + end +end diff --git a/services/console/test/jobs/slack_dm/jobs_test.rb b/services/console/test/jobs/slack_dm/jobs_test.rb index 7e804d153..c69af1af4 100644 --- a/services/console/test/jobs/slack_dm/jobs_test.rb +++ b/services/console/test/jobs/slack_dm/jobs_test.rb @@ -34,10 +34,11 @@ def slack_credential( ) end - test "PollSyncJob enqueues credentials for the configured Slack OAuth app with required scopes" do + test "PollSyncJob enqueues credentials with any supported private conversation scopes" do app = slack_app good = slack_credential(app: app) - missing_scope = slack_credential(app: app, scopes: %w[im:read im:history]) + dm_only = slack_credential(app: app, scopes: SlackDm::SyncCredential::DM_REQUIRED_SCOPES) + missing_scope = slack_credential(app: app, scopes: %w[chat:write]) no_token = slack_credential(app: app, access_token: nil) other_app = slack_app(slug: "other-slack") other = slack_credential(app: other_app) @@ -48,6 +49,7 @@ def slack_credential( .select { |job| job[:job] == SlackDm::SyncCredentialJob } .map { |job| job[:args].first } assert_includes enqueued_ids, good.id + assert_includes enqueued_ids, dm_only.id refute_includes enqueued_ids, missing_scope.id refute_includes enqueued_ids, no_token.id refute_includes enqueued_ids, other.id diff --git a/services/console/test/lib/centaur_jwt/hs256_test.rb b/services/console/test/lib/centaur_jwt/hs256_test.rb index be63064d8..f1b4d7db5 100644 --- a/services/console/test/lib/centaur_jwt/hs256_test.rb +++ b/services/console/test/lib/centaur_jwt/hs256_test.rb @@ -16,4 +16,67 @@ class CentaurJwtHs256Test < ActiveSupport::TestCase expected = OpenSSL::HMAC.digest("SHA256", "test-secret", "#{header}.#{payload}") assert_equal Base64.urlsafe_encode64(expected, padding: false), signature end + + test "decode verifies signature issuer audience and expiry" do + token = CentaurJwt::Hs256.encode( + { "iss" => "issuer", "aud" => "audience", "sub" => "x", "exp" => 1.hour.from_now.to_i }, + signing_secret: "test-secret" + ) + + payload = CentaurJwt::Hs256.decode( + token, + signing_secret: "test-secret", + iss: "issuer", + aud: "audience" + ) + + assert_equal "x", payload.fetch("sub") + end + + test "decode rejects invalid signatures" do + token = CentaurJwt::Hs256.encode({ "sub" => "x" }, signing_secret: "test-secret") + tampered = token.sub(/\.[^.]+\z/, ".bad") + + assert_raises(CentaurJwt::Hs256::VerificationError) do + CentaurJwt::Hs256.decode(tampered, signing_secret: "test-secret") + end + end + + test "decode rejects structurally malformed tokens" do + [ + "W10.e30.AAAA", # header segment decodes to a JSON array + "bnVsbA.e30.AAAA", # header segment decodes to JSON null + "not-a-jwt", + "" + ].each do |token| + assert_raises(CentaurJwt::Hs256::VerificationError, token.inspect) do + CentaurJwt::Hs256.decode(token, signing_secret: "test-secret") + end + end + end + + test "decode rejects issuer and audience mismatches" do + token = CentaurJwt::Hs256.encode( + { "iss" => "issuer", "aud" => "audience", "exp" => 1.hour.from_now.to_i }, + signing_secret: "test-secret" + ) + + assert_raises(CentaurJwt::Hs256::VerificationError) do + CentaurJwt::Hs256.decode(token, signing_secret: "test-secret", iss: "other", aud: "audience") + end + assert_raises(CentaurJwt::Hs256::VerificationError) do + CentaurJwt::Hs256.decode(token, signing_secret: "test-secret", iss: "issuer", aud: "other") + end + end + + test "decode rejects expired tokens" do + token = CentaurJwt::Hs256.encode( + { "sub" => "x", "exp" => 1.hour.ago.to_i }, + signing_secret: "test-secret" + ) + + assert_raises(CentaurJwt::Hs256::VerificationError) do + CentaurJwt::Hs256.decode(token, signing_secret: "test-secret") + end + end end diff --git a/services/console/test/lib/oauth/providers/slack_test.rb b/services/console/test/lib/oauth/providers/slack_test.rb index 63210fc39..9d94086e3 100644 --- a/services/console/test/lib/oauth/providers/slack_test.rb +++ b/services/console/test/lib/oauth/providers/slack_test.rb @@ -55,6 +55,38 @@ def valid_claims(**overrides) assert_equal "ada", identity[:name] end + test "uses Slack bot user id when no authed user token or id token is returned" do + result = result_with( + claims: valid_claims, + id_token: nil, + response: { + "team" => { "id" => "T12345", "name" => "Acme" }, + "bot_user_id" => "U0BOTUSER" + } + ) + + identity = strategy.identity_from(result, client_id: CLIENT_ID) + assert_equal "U0BOTUSER", identity[:subject] + assert_equal "Acme", identity[:name] + assert_equal "T12345", identity[:team_id] + end + + test "only schedules refresh when Slack returned a refresh token" do + rotating = result_with(claims: valid_claims, refresh_token: "RT") + static = result_with(claims: valid_claims, refresh_token: nil) + + refute strategy.require_refresh_token? + assert strategy.refreshable_result?(rotating) + refute strategy.refreshable_result?(static) + end + + test "rejects expiring Slack responses without a refresh token" do + result = result_with(claims: valid_claims, refresh_token: nil, expires_in: 43_200) + + err = assert_raises(Broker::ExchangeError) { strategy.validate_result!(result) } + assert_equal "missing_refresh_token", err.code + end + test "aud mismatch raises an oauth exchange error" do result = result_with(claims: valid_claims("aud" => "someone-else")) err = assert_raises(Broker::ExchangeError) { strategy.identity_from(result, client_id: CLIENT_ID) } diff --git a/services/console/test/models/mcp_oauth_client_test.rb b/services/console/test/models/mcp_oauth_client_test.rb index 86d22956e..e18e0af69 100644 --- a/services/console/test/models/mcp_oauth_client_test.rb +++ b/services/console/test/models/mcp_oauth_client_test.rb @@ -1,18 +1,28 @@ require "test_helper" class McpOauthClientTest < ActiveSupport::TestCase - test "allowed redirect URI accepts only localhost and loopback IP literals" do + test "allowed redirect URI accepts HTTPS and plain HTTP loopback redirects" do + assert McpOauthClient.allowed_redirect_uri?("https://claude.ai/api/mcp/auth_callback") + assert McpOauthClient.allowed_redirect_uri?("https://example.com/callback") assert McpOauthClient.allowed_redirect_uri?("http://localhost:49152/callback") assert McpOauthClient.allowed_redirect_uri?("http://127.0.0.1:49152/callback") assert McpOauthClient.allowed_redirect_uri?("http://127.1.2.3:49152/callback") assert McpOauthClient.allowed_redirect_uri?("http://[::1]:49152/callback") - refute McpOauthClient.allowed_redirect_uri?("https://127.0.0.1/callback") refute McpOauthClient.allowed_redirect_uri?("http://127.evil.com/callback") refute McpOauthClient.allowed_redirect_uri?("http://127.0.0.1.evil.com/callback") refute McpOauthClient.allowed_redirect_uri?("http://localhost.evil.com/callback") end + test "allowed redirect URI rejects wildcard redirects" do + refute McpOauthClient.allowed_redirect_uri?("https://*.example.com/callback") + refute McpOauthClient.allowed_redirect_uri?("https://example.com/*") + end + + test "allowed redirect URI rejects fragments" do + refute McpOauthClient.allowed_redirect_uri?("https://example.com/callback#token") + end + test "redirect matching rejects attacker controlled 127-looking hostnames" do client = McpOauthClient.create!( name: "Amp", diff --git a/services/console/test/models/principal_sync_config_snapshot_test.rb b/services/console/test/models/principal_sync_config_snapshot_test.rb index 4e6ed9826..5c097cade 100644 --- a/services/console/test/models/principal_sync_config_snapshot_test.rb +++ b/services/console/test/models/principal_sync_config_snapshot_test.rb @@ -48,7 +48,10 @@ def while_rebuild_lock_held end test "fetch_for rebuilds api server JWT snapshots when the jwt window advances" do - with_env("CENTAUR_JWT_SIGNING_SECRET" => "test-secret") do + with_env( + "CENTAUR_JWT_SIGNING_SECRET" => "test-secret", + "CENTAUR_API_URL" => "http://api.internal:8080" + ) do SlackChannelPermission.create!( principal: @principal, channel_id: "C0123456789", diff --git a/services/console/test/models/principal_test.rb b/services/console/test/models/principal_test.rb index c9d372247..ced8d06b9 100644 --- a/services/console/test/models/principal_test.rb +++ b/services/console/test/models/principal_test.rb @@ -170,7 +170,7 @@ def default_attrs(overrides = {}) refute_nil entry assert_equal "Bearer {{ .Value }}", entry.dig("inject", "formatter") - assert_includes entry.fetch("rules"), { "host" => "api.internal" } + assert_equal [ { "host" => "api.internal" } ], entry.fetch("rules") claims = jwt_payload(entry.dig("source", "value")) assert_equal "centaur-console", claims.fetch("iss") @@ -199,7 +199,11 @@ def default_attrs(overrides = {}) end test "api-enabled non-Slack principal receives a subject-only API JWT" do - with_env("CENTAUR_JWT_SIGNING_SECRET" => "test-secret") do + with_env( + "CENTAUR_JWT_SIGNING_SECRET" => "test-secret", + "CENTAUR_API_URL" => "http://api.internal:8080", + "CENTAUR_API_SERVER_PROXY_HOSTS" => nil + ) do principal = Principal.create!( default_attrs(namespace: "acme", foreign_id: "github-subject-only") ) diff --git a/services/console/test/models/thread_share_test.rb b/services/console/test/models/thread_share_test.rb new file mode 100644 index 000000000..f80b5d782 --- /dev/null +++ b/services/console/test/models/thread_share_test.rb @@ -0,0 +1,17 @@ +require "test_helper" + +class ThreadShareTest < ActiveSupport::TestCase + test "the database enforces globally unique thread keys" do + ThreadShare.create!(thread_key: "console:shared", created_by: users(:acme_admin)) + duplicate = ThreadShare.new(thread_key: "console:shared", created_by: users(:member_user)) + + assert_raises(ActiveRecord::RecordNotUnique) { duplicate.save! } + end + + test "thread keys use the session API maximum length" do + share = ThreadShare.new(thread_key: "x" * 513, created_by: users(:acme_admin)) + + assert_not share.valid? + assert_includes share.errors[:thread_key], "is too long (maximum is 512 characters)" + end +end diff --git a/services/console/test/models/user_test.rb b/services/console/test/models/user_test.rb index e71a52961..ed5be3c0a 100644 --- a/services/console/test/models/user_test.rb +++ b/services/console/test/models/user_test.rb @@ -116,6 +116,30 @@ def identity(overrides = {}) assert_equal [ [ "google", "sub-1" ] ], user.user_identities.pluck(:provider, :subject) end + test "link_or_provision rejects SSO emails outside the configured domain allowlist" do + ENV["CENTAUR_CONSOLE_SSO_EMAIL_DOMAINS"] = "acme.example example.org" + assert_no_difference [ "User.count", "UserIdentity.count" ] do + assert_raises(User::SsoEmailDomainNotAllowed) do + User.link_or_provision(provider: "google", + identity: identity(subject: "outside-sub", email: "newcomer@example.com")) + end + end + ensure + ENV.delete("CENTAUR_CONSOLE_SSO_EMAIL_DOMAINS") + end + + test "link_or_provision allows SSO emails inside the configured domain allowlist" do + ENV["CENTAUR_CONSOLE_SSO_EMAIL_DOMAINS"] = "acme.example example.org" + user = nil + assert_difference -> { User.count }, 1 do + user = User.link_or_provision(provider: "google", + identity: identity(subject: "inside-sub", email: "worker@acme.example")) + end + assert user.active? + ensure + ENV.delete("CENTAUR_CONSOLE_SSO_EMAIL_DOMAINS") + end + test "link_or_provision returns the existing user for a returning identity" do existing = user_identities(:acme_admin_google) user = nil @@ -126,6 +150,19 @@ def identity(overrides = {}) assert_equal existing.user, user end + test "link_or_provision rejects a returning identity outside the configured domain allowlist" do + ENV["CENTAUR_CONSOLE_SSO_EMAIL_DOMAINS"] = "globex.example" + existing = user_identities(:acme_admin_google) + assert_no_difference [ "User.count", "UserIdentity.count" ] do + assert_raises(User::SsoEmailDomainNotAllowed) do + User.link_or_provision(provider: existing.provider, + identity: identity(subject: existing.subject, email: existing.email)) + end + end + ensure + ENV.delete("CENTAUR_CONSOLE_SSO_EMAIL_DOMAINS") + end + test "link_or_provision links a new identity to an existing user by verified email" do target = users(:globex_admin) user = nil diff --git a/services/console/test/services/granola/sync_credential_test.rb b/services/console/test/services/granola/sync_credential_test.rb new file mode 100644 index 000000000..353b328b3 --- /dev/null +++ b/services/console/test/services/granola/sync_credential_test.rb @@ -0,0 +1,130 @@ +require "test_helper" + +module Granola + class SyncCredentialTest < ActiveSupport::TestCase + class FakeApiClient + attr_reader :batches + + def initialize(checkpoint: nil) + @checkpoint = checkpoint + @batches = [] + end + + def get_granola_sync_checkpoint(scope_id:) + { "ok" => true, "checkpoint" => @checkpoint&.merge("scope_id" => scope_id) } + end + + def ingest_granola_sync_batch(payload) + @batches << payload + { "ok" => true } + end + end + + def granola_app + @granola_app ||= OauthApp.create!( + provider: "granola", + slug: "granola-sync-#{SecureRandom.hex(6)}", + client_id: "granola-client", + client_secret: "granola-secret", + allowed_scopes: %w[meetings:read], + credential_namespace: "acme", + created_by: users(:acme_admin) + ) + end + + def credential + @credential ||= BrokerCredential.create!( + oauth_app: granola_app, + namespace: "acme", + foreign_id: "granola-sync-#{SecureRandom.hex(6)}", + token_endpoint: Oauth::Providers::Granola::TOKEN_ENDPOINT, + access_token: "granola-access-token", + refresh_token: "granola-refresh-token", + last_refresh: Time.current, + expires_at: 1.hour.from_now, + scopes: %w[meetings:read], + provider_subject: "granola-user-1", + provider_email: "owner@example.com" + ) + end + + test "syncs the connected user's notes into a credential-scoped batch" do + api_client = FakeApiClient.new( + checkpoint: { "watermark_time" => "2026-07-08T12:00:00Z" } + ) + mcp_http = lambda do |tool:, arguments:, access_token:| + assert_equal "granola-access-token", access_token + + case tool + when "get_account_info" + { email: "Owner@Example.com", workspace: "Acme" }.to_json + when "list_meetings" + assert_equal "custom", arguments.fetch("time_range") + assert arguments.fetch("custom_start") <= arguments.fetch("custom_end") + meeting_xml + when "get_meetings" + assert_equal [ "meeting-1" ], arguments.fetch("meeting_ids") + meeting_xml + when "get_meeting_transcript" + assert_equal "meeting-1", arguments.fetch("meeting_id") + "Ada: Ship the Granola sync." + else + flunk "unexpected Granola MCP tool #{tool}" + end + end + + SyncCredential.new(credential, api_client: api_client, mcp_http: mcp_http).call + + batch = api_client.batches.fetch(0) + assert_equal "completed", batch[:run][:status] + assert_equal "oauth:#{credential.oid}", batch[:run][:scope_id] + assert_equal credential.oid, batch[:run][:broker_credential_id] + assert_equal "owner@example.com", batch[:run][:source_user_email] + assert_equal "oauth:#{credential.oid}", batch[:checkpoint][:scope_id] + + note = batch[:notes].fetch(0) + assert_equal "meeting-1", note["note_id"] + assert_equal "Planning", note["title"] + assert_equal "ada@example.com", note["owner"]["email"] + assert_equal "Ada", note["owner"]["name"] + assert_equal [ "ada@example.com", "bob@example.com" ], note["attendees"].pluck("email") + assert_equal "2026-07-08T17:30:00+02:00", note["source_updated_at"] + assert_equal "Ada: Ship the Granola sync.", note["transcript"].first["text"] + end + + test "records an API failure against the same OAuth credential scope" do + api_client = FakeApiClient.new + mcp_http = lambda do |tool:, **| + case tool + when "get_account_info" + { email: "owner@example.com" }.to_json + when "list_meetings" + raise SyncCredential::GranolaApiError, "rate limited" + else + flunk "unexpected Granola MCP tool #{tool}" + end + end + + assert_raises(SyncCredential::GranolaApiError) do + SyncCredential.new(credential, api_client: api_client, mcp_http: mcp_http).call + end + + failed = api_client.batches.fetch(0) + assert_equal "failed", failed[:run][:status] + assert_equal "oauth:#{credential.oid}", failed[:run][:scope_id] + assert_includes failed[:run][:error_text], "rate limited" + end + + private + + def meeting_xml + <<~XML + <meeting id="meeting-1" title="Planning" date="Jul 8, 2026 5:30 PM GMT+2"> + <known_participants>Ada (note creator) from Acme <ada@example.com> + Bob <bob@example.com></known_participants> + <summary>Ship the Granola sync.</summary> + </meeting> + XML + end + end +end diff --git a/services/console/test/services/slack_dm/sync_credential_test.rb b/services/console/test/services/slack_dm/sync_credential_test.rb index d6ad7cb06..b6bf4bf90 100644 --- a/services/console/test/services/slack_dm/sync_credential_test.rb +++ b/services/console/test/services/slack_dm/sync_credential_test.rb @@ -83,6 +83,7 @@ def credential when SlackDm::SyncCredential::AUTH_TEST_ENDPOINT { "ok" => true, "team_id" => "T123", "user_id" => "U_ME" } when SlackDm::SyncCredential::CONVERSATIONS_LIST_ENDPOINT + assert_equal "im,mpim,private_channel", params["types"] { "ok" => true, "channels" => [ @@ -176,5 +177,104 @@ def credential assert_equal "F123", batch[:attachments].first[:slack_file_id] assert_equal "1700000000.000002", batch[:checkpoints].first[:watermark_ts] end + + test "sync ingests private channels and their complete member list" do + api_client = FakeApiClient.new + slack_http = lambda do |endpoint:, params:, access_token:| + assert_equal "xoxp-live", access_token + case endpoint + when SlackDm::SyncCredential::AUTH_TEST_ENDPOINT + { "ok" => true, "team_id" => "T123", "user_id" => "U_ME" } + when SlackDm::SyncCredential::CONVERSATIONS_LIST_ENDPOINT + assert_equal "im,mpim,private_channel", params["types"] + { + "ok" => true, + "channels" => [ + { + "id" => "G123", + "name" => "leadership", + "is_private" => true, + "is_archived" => false + } + ], + "response_metadata" => { "next_cursor" => "" } + } + when SlackDm::SyncCredential::CONVERSATIONS_MEMBERS_ENDPOINT + assert_equal "G123", params["channel"] + { + "ok" => true, + "members" => %w[U_ME U_OTHER], + "response_metadata" => { "next_cursor" => "" } + } + when SlackDm::SyncCredential::CONVERSATIONS_HISTORY_ENDPOINT + { + "ok" => true, + "messages" => [ + { + "type" => "message", + "ts" => "1700000001.000001", + "user" => "U_OTHER", + "text" => "private roadmap" + } + ], + "response_metadata" => { "next_cursor" => "" } + } + else + flunk "unexpected Slack endpoint #{endpoint}" + end + end + + SlackDm::SyncCredential.new( + credential, + api_client: api_client, + slack_api_http: slack_http + ).call + + batch = api_client.batch + assert_equal "private_channel", batch[:conversations].first[:conversation_type] + assert_equal "leadership", batch[:conversations].first[:raw_payload]["name"] + assert_equal %w[U_ME U_OTHER], batch[:members].map { |member| member[:user_id] } + assert_equal "private roadmap", batch[:messages].first[:text] + end + + test "sync never replaces membership from truncated pagination" do + env_key = "CENTAUR_CONSOLE_SLACK_DM_SYNC_MEMBERS_MAX_PAGES" + previous = ENV[env_key] + ENV[env_key] = "1" + api_client = FakeApiClient.new + slack_http = lambda do |endpoint:, params:, access_token:| + assert_equal "xoxp-live", access_token + case endpoint + when SlackDm::SyncCredential::AUTH_TEST_ENDPOINT + { "ok" => true, "team_id" => "T123", "user_id" => "U_ME" } + when SlackDm::SyncCredential::CONVERSATIONS_LIST_ENDPOINT + { + "ok" => true, + "channels" => [ { "id" => "G123", "is_private" => true } ], + "response_metadata" => { "next_cursor" => "" } + } + when SlackDm::SyncCredential::CONVERSATIONS_MEMBERS_ENDPOINT + { + "ok" => true, + "members" => [ "U_ME" ], + "response_metadata" => { "next_cursor" => "more" } + } + else + flunk "unexpected Slack endpoint #{endpoint} with #{params}" + end + end + + error = assert_raises(SlackDm::SyncCredential::SlackApiError) do + SlackDm::SyncCredential.new( + credential, + api_client: api_client, + slack_api_http: slack_http + ).call + end + assert_match "membership pagination truncated", error.message + assert_nil api_client.batch + ensure + previous.nil? ? ENV.delete(env_key) : ENV[env_key] = previous + end end end diff --git a/services/console/test/services/slack_requester_identity_test.rb b/services/console/test/services/slack_requester_identity_test.rb new file mode 100644 index 000000000..4278d2d6a --- /dev/null +++ b/services/console/test/services/slack_requester_identity_test.rb @@ -0,0 +1,30 @@ +require "test_helper" + +class SlackRequesterIdentityTest < ActiveSupport::TestCase + test "resolves a verified GitHub handle from the requester's labeled Slack profile field" do + response = Struct.new(:body).new({ + ok: true, + profile: { fields: { "XfGithub" => { label: "GitHub", value: "https://github.com/ada" } } } + }.to_json) + http = Object.new + http.define_singleton_method(:request) { |_request| response } + + with_singleton_method(Net::HTTP, :start, ->(*_args, **_options, &block) { block.call(http) }) do + result = SlackRequesterIdentity.new(token: "xoxb-test", api_url: "https://slack.test/api").resolve("UADA") + + assert_equal "@ada", result.handle + assert_equal 'Slack profile custom field "GitHub"', result.source + end + end + + private + + def with_singleton_method(object, method_name, replacement) + singleton = object.singleton_class + original = singleton.instance_method(method_name) + singleton.define_method(method_name, replacement) + yield + ensure + singleton.define_method(method_name, original) + end +end diff --git a/services/sandbox/SYSTEM_PROMPT.md b/services/sandbox/SYSTEM_PROMPT.md index 98bcccad0..7123889a4 100644 --- a/services/sandbox/SYSTEM_PROMPT.md +++ b/services/sandbox/SYSTEM_PROMPT.md @@ -85,8 +85,8 @@ |For other cargo commands, prefer the repository's pinned/default toolchain unless the repo or user asks for nightly. [GitHub PR Attribution] -|When opening a GitHub PR for a Slack request, attribute the requester in the PR body with one standalone `Prompted by: ...` line. -|Use the [Requester Context] block when present: prefer the verified GitHub handle resolved from the requester's Slack profile; if none is configured, use the requester's Slack display name or username. +|When opening a GitHub PR, attribute the requester in the PR body with one standalone `Prompted by: ...` line. +|Use the [Requester Context] block when present. For Slack, prefer the verified GitHub handle resolved from the requester's Slack profile; otherwise use the exact `Prompted by:` value supplied by the requesting surface. |If [Requester Context] provides an exact `Prompted by:` line, copy that line exactly into the PR body. |Do not infer a GitHub username from a Slack name, email, or thread history. The credited prompter is the user who prompted the current turn, not necessarily the Slack thread root author. @@ -178,6 +178,15 @@ | notion search "meeting notes" | vlogs query 'level:error AND _stream:{service="api"}' --limit 20 +[Personal OAuth app connections] +|When a user asks how to connect, authorize, sign in, link, or use their personal account for OAuth-backed apps in a Centaur DM, first fetch the live configured start URLs with `centaur-console oauth-apps`. +|This applies to apps such as Google, Granola, Attio, Linear, Slack, and GitHub. The endpoint returns only apps configured and enabled for this deployment. +|Use the returned `start_url` for the matching app. Do not invent OAuth links, hard-code `/oauth/<slug>/start`, or assume an app is configured because the tool exists. +|Tell the user to open the returned start URL, complete the provider consent flow in their browser, then come back to the DM. After they return, validate the connection with `centaur-console permissions`: look in `oauth_credentials` for the requested app/provider and the user's personal `provider_email`. +|If `oauth_credentials` contains that app/provider and personal email, tell the user the account is connected and that Centaur can use their personal connected account where the relevant tool or workflow supports user-scoped credentials. +|If the credential is not present yet, ask the user to confirm which email they used in the provider consent flow or to retry the returned start URL. Do not claim the account is connected until `centaur-console permissions` shows the matching email. +|If the requested app is missing from the endpoint response, say that it is not currently configured for self-service connection in this deployment. If the endpoint call fails, say you cannot retrieve connection links right now and include the tool error briefly. + [Tool discovery — discover before you call] |IMPORTANT: Before using any unfamiliar tool CLI, run `<tool> --help` to see commands, parameters, and descriptions. |This tells you exactly which command to use and avoids redundant calls. diff --git a/services/sandbox/test_system_prompt.py b/services/sandbox/test_system_prompt.py index d166610f1..b12d862d2 100644 --- a/services/sandbox/test_system_prompt.py +++ b/services/sandbox/test_system_prompt.py @@ -53,5 +53,18 @@ def test_model_and_harness_switching_answer_guidance_is_present(self) -> None: self.assertIn("`--bedrock` selects Codex with the Bedrock provider", prompt) self.assertIn("`-rsn <effort>` sets Codex reasoning effort", prompt) + def test_personal_oauth_app_connection_guidance_is_present(self) -> None: + prompt = SYSTEM_PROMPT.read_text() + + self.assertIn("[Personal OAuth app connections]", prompt) + self.assertIn("centaur-console oauth-apps", prompt) + self.assertIn("Google, Granola, Attio, Linear, Slack, and GitHub", prompt) + self.assertIn("Use the returned `start_url`", prompt) + self.assertIn("Do not invent OAuth links", prompt) + self.assertIn("validate the connection with `centaur-console permissions`", prompt) + self.assertIn("look in `oauth_credentials`", prompt) + self.assertIn("personal `provider_email`", prompt) + self.assertIn("Centaur can use their personal connected account", prompt) + if __name__ == "__main__": unittest.main() diff --git a/services/slackbotv2/src/channel-defaults.ts b/services/slackbotv2/src/channel-defaults.ts new file mode 100644 index 000000000..b5379c26f --- /dev/null +++ b/services/slackbotv2/src/channel-defaults.ts @@ -0,0 +1,90 @@ +/** + * Per-channel default harness / model / provider / reasoning. Loaded from the + * `SLACKBOTV2_CHANNEL_DEFAULTS` env var: JSON keyed by Slack conversation id, + * each value an object normalized like the inline flags (see + * `normalizeHarnessOverrides`): + * + * SLACKBOTV2_CHANNEL_DEFAULTS='{ + * "C0ENG": {"harness": "claude", "model": "opus", "reasoning": "high"}, + * "C0TRIAGE": {"reasoning": "low"}, + * "C0BEDROCK": {"provider": "bedrock", "model": "gpt-5.2"} + * }' + * + * Fields are independent. Precedence (in index.ts): per-thread override, then + * channel default, then deployment default. Setting `harness` restarts a thread + * onto it like `--claude`/`--codex`; `reasoning` only affects codex. + */ + +import { normalizeHarnessOverrides, type HarnessOverrides } from './overrides' + +export type ChannelDefaults = Record<string, HarnessOverrides> + +/** + * Parses `SLACKBOTV2_CHANNEL_DEFAULTS` into a channel→overrides map (empty for + * unset input). Never throws — bad JSON or entries are skipped and reported via + * `onError`. + */ +export function parseChannelDefaults( + raw: string | undefined, + onError?: (message: string) => void +): ChannelDefaults { + const trimmed = raw?.trim() + if (!trimmed) return {} + let parsed: unknown + try { + parsed = JSON.parse(trimmed) + } catch (error) { + onError?.(`invalid JSON: ${error instanceof Error ? error.message : String(error)}`) + return {} + } + if (!isPlainObject(parsed)) { + onError?.('expected a JSON object keyed by channel id') + return {} + } + const result: ChannelDefaults = {} + for (const [channelId, rawEntry] of Object.entries(parsed)) { + const key = channelId.trim() + if (!key) continue + if (!isPlainObject(rawEntry)) { + onError?.(`channel ${key}: expected an object of harness/model/provider/reasoning fields`) + continue + } + const overrides = normalizeHarnessOverrides(rawEntry, message => onError?.(`channel ${key}: ${message}`)) + if (!overrides.harnessType && !overrides.model && !overrides.provider && !overrides.reasoning) { + onError?.(`channel ${key}: no usable harness/model/provider/reasoning fields`) + continue + } + result[key] = overrides + } + return result +} + +function isPlainObject(value: unknown): value is Record<string, unknown> { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +/** + * Extracts the Slack conversation id from a thread key of the shape + * `slack:CHANNEL[:THREAD_TS]` (or `slack:TEAM:CHANNEL:…`), mirroring the + * classification in session-api's `slackConversationId`: the first segment + * after the namespace whose first character is `C`, `G`, or `D`. + */ +export function channelIdFromThreadId(threadId: string): string | undefined { + const segments = threadId.split(':').slice(1) + for (const segment of segments) { + const first = segment.charAt(0) + if (first === 'C' || first === 'G' || first === 'D') return segment + } + return undefined +} + +/** Resolves the channel default for a thread, or undefined when none applies. */ +export function resolveChannelDefault( + defaults: ChannelDefaults | undefined, + threadId: string +): HarnessOverrides | undefined { + if (!defaults) return undefined + const channelId = channelIdFromThreadId(threadId) + if (!channelId) return undefined + return defaults[channelId] +} diff --git a/services/slackbotv2/src/console-session-link.ts b/services/slackbotv2/src/console-session-link.ts index e713a0a44..fef78272c 100644 --- a/services/slackbotv2/src/console-session-link.ts +++ b/services/slackbotv2/src/console-session-link.ts @@ -35,6 +35,16 @@ const BAKED_DEFAULT_MODELS: Record<string, string | undefined> = { : undefined } +const BAKED_CODEX_EFFORT = + typeof (codexConfig as { model_reasoning_effort?: unknown }).model_reasoning_effort === 'string' + ? (codexConfig as { model_reasoning_effort: string }).model_reasoning_effort + : undefined + +const BAKED_CODEX_SPEED = + typeof (codexConfig as { service_tier?: unknown }).service_tier === 'string' + ? (codexConfig as { service_tier: string }).service_tier + : undefined + /** Slack mrkdwn requires `&`, `<`, `>` to be escaped in free text. */ function escapeSlackMrkdwn(text: string): string { return text.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>') @@ -76,6 +86,14 @@ export function defaultModelForHarness( return configured?.[key]?.trim() || BAKED_DEFAULT_MODELS[key] } +export function defaultCodexEffort(configured?: string): string | undefined { + return configured?.trim() || BAKED_CODEX_EFFORT +} + +export function defaultCodexSpeed(configured?: string): string | undefined { + return configured?.trim() || BAKED_CODEX_SPEED +} + /** * Builds the Console session URL for a Slack thread key, or undefined when no * Console base URL is configured (in which case no link/block should render). @@ -98,7 +116,7 @@ export type SlackContextBlock = { } /** - * Builds the "Open chat in Console · {MODEL} · {Harness}" context block, or + * Builds the Slack context block with model, harness, effort, and speed, or * undefined when no Console base URL is configured (a bare "Open chat in * Console" with no link is pointless, so the whole block is skipped). The * model id is uppercased for display. @@ -108,6 +126,8 @@ export function buildConsoleSessionContextBlock(params: { threadKey: string harnessType?: string | null model?: string | null + effort?: string | null + speed?: string | null }): SlackContextBlock | undefined { const url = consoleSessionUrl(params.consoleBaseUrl, params.threadKey) if (!url) return undefined @@ -116,6 +136,10 @@ export function buildConsoleSessionContextBlock(params: { if (model) segments.push(escapeSlackMrkdwn(model.toUpperCase())) const harness = harnessDisplayName(params.harnessType) if (harness) segments.push(escapeSlackMrkdwn(harness)) + const effort = params.effort?.trim() + if (effort) segments.push(`Effort: ${escapeSlackMrkdwn(titleCase(effort))}`) + const speed = params.speed?.trim() + if (speed) segments.push(`Speed: ${escapeSlackMrkdwn(titleCase(speed))}`) // Middot (U+00B7) with a space on each side, matching the bot's other // context lines. return { diff --git a/services/slackbotv2/src/index.ts b/services/slackbotv2/src/index.ts index 26b05a053..d06062472 100644 --- a/services/slackbotv2/src/index.ts +++ b/services/slackbotv2/src/index.ts @@ -25,7 +25,11 @@ import { } from '@centaur/rendering' import { conflateChatSdkStream } from './conflate' import { observeSeconds, slackbotMetrics } from './metrics' -import { renderSlackDisplayText, slackMessagePromptText } from './slack-display-text' +import { + renderSlackDisplayText, + slackMessagePromptText, + slackRichTextMentionsUser +} from './slack-display-text' import { slackUserIdForMessage } from './slack-user' import { collectInitialContext, @@ -42,9 +46,12 @@ import { } from './session-api' import { buildConsoleSessionContextBlock, + defaultCodexEffort, + defaultCodexSpeed, defaultModelForHarness, type SlackContextBlock } from './console-session-link' +import { channelIdFromThreadId, resolveChannelDefault } from './channel-defaults' import { extractMessageOverrides } from './overrides' import { isAllowedSlackMessage, isAllowedSlackWebhookBody } from './slack-events' import { isSlackStopCommand } from './stop-command' @@ -185,6 +192,17 @@ function stickyOverrideValue( return stringValue(state[key]) } +// Like stickyOverrideValue but keeps an explicit `null` — the tombstone a +// harness switch writes to clear the old model/provider — so callers can tell +// "cleared" (null) from "never set" (undefined). +function stickyOverrideRaw( + state: SlackbotV2ThreadState, + update: StickyThreadOverrides | undefined, + key: keyof StickyThreadOverrides +): string | null | undefined { + return update && Object.prototype.hasOwnProperty.call(update, key) ? update[key] : state[key] +} + export function createSlackbotV2(options: SlackbotV2Options): SlackbotV2 { const userName = options.userName ?? 'centaur' const logger = options.logger ?? noopLogger @@ -221,8 +239,30 @@ export function createSlackbotV2(options: SlackbotV2Options): SlackbotV2 { const ambientSlackChannelIds = ambientSlackChannelIdSet(options) + // Slack does not classify mentions inside Block Kit or legacy attachments as + // app_mention events. Alertmanager uses attachment.pretext. Ambient channels + // also execute root messages without mentions, so handle both cases in one + // callback to avoid double execution when a rich mention is ambient. + chat.onNewMessage(/[\s\S]*/, async (thread, message) => { + const richMention = slackRichTextMentionsUser(message.raw, options.botUserId) + const isAmbientMessage = isAmbientSlackChannelMessage(message, ambientSlackChannelIds) + if (!richMention && !isAmbientMessage) return + if (!(await isAllowedSlackMessage(message, options, logger))) return + if (richMention) message.isMention = true + lateSlackFiles.rememberFilelessMention(thread, message) + await handleSlackMessageHandoff(thread, message, { + assistantStatusRequested: true, + mode: 'execute', + options, + state, + subscribe: true, + trigger: richMention ? 'new_mention' : 'ambient_channel_message' + }) + }) + chat.onSubscribedMessage(async (thread, message) => { if (!(await isAllowedSlackMessage(message, options, logger))) return + if (slackRichTextMentionsUser(message.raw, options.botUserId)) message.isMention = true lateSlackFiles.rememberFilelessMention(thread, message) const isAmbientMessage = isAmbientSlackChannelMessage(message, ambientSlackChannelIds) const shouldExecute = message.isMention === true || isAmbientMessage @@ -235,22 +275,6 @@ export function createSlackbotV2(options: SlackbotV2Options): SlackbotV2 { }) }) - if (ambientSlackChannelIds.size > 0) { - chat.onNewMessage(/[\s\S]*/, async (thread, message) => { - if (!isAmbientSlackChannelMessage(message, ambientSlackChannelIds)) return - if (!isAllowedSlackMessage(message, options, logger)) return - lateSlackFiles.rememberFilelessMention(thread, message) - await handleSlackMessageHandoff(thread, message, { - assistantStatusRequested: true, - mode: 'execute', - options, - state, - subscribe: true, - trigger: 'ambient_channel_message' - }) - }) - } - const app = new Hono() app.get('/health', c => c.json({ ok: true, service: 'slackbotv2' })) app.get('/metrics', c => @@ -352,7 +376,7 @@ function isAmbientSlackChannelMessage( const raw = isJsonObject(message.raw) ? message.raw : undefined if (raw && stringValue(raw.type) !== 'message') return false - const channelId = stringValue(raw?.channel) ?? slackChannelFromThreadId(message.threadId) + const channelId = stringValue(raw?.channel) ?? channelIdFromThreadId(message.threadId) return Boolean(channelId && allowedChannels.has(channelId)) } @@ -362,14 +386,6 @@ function ambientSlackChannelIdSet(options: SlackbotV2Options): ReadonlySet<strin ) } -function slackChannelFromThreadId(threadId: string): string | undefined { - const parts = threadId.split(':') - if (parts[0] !== 'slack') return undefined - if (parts.length === 4) return parts[2] || undefined - if (parts.length === 3) return parts[1] || undefined - return undefined -} - function splitEnvList(value: string | undefined): string[] { return (value ?? '') .split(/[\s,]+/) @@ -790,20 +806,44 @@ async function syncThreadMessageToSession( // (`slack:CHANNEL:THREAD_TS`) is the exact value sent to the session API as // `thread_key`, which the Console indexes by. const isFirstAssistantMessage = shouldStartExecution && executedMessageIds.size === 0 + // Channel default: below a per-thread flag, above the deployment default, and + // (unlike it) ridden on the input line to take effect. harness/model/provider + // are sticky (effectiveOverrides); reasoning is per-turn. + const channelDefault = resolveChannelDefault(input.options.channelDefaults, thread.id) + const resolvedHarnessType = effectiveOverrides.harnessType ?? channelDefault?.harnessType + // A `null` sticky model/provider is a tombstone from a harness switch: honor + // it, don't re-pair a stale channel default with the new harness. Only + // `undefined` (never set) falls through to the channel default. + const resolvedModel = + stickyOverrideRaw(state, stickyOverridesUpdate, 'model') === null + ? undefined + : effectiveOverrides.model ?? channelDefault?.model + const resolvedProvider = + stickyOverrideRaw(state, stickyOverridesUpdate, 'provider') === null + ? undefined + : effectiveOverrides.provider ?? channelDefault?.provider + const resolvedReasoning = overrides.reasoning ?? channelDefault?.reasoning const effectiveHarnessType = - effectiveOverrides.harnessType ?? input.options.defaultHarnessType ?? 'codex' - // Without an explicit --model/--opus/... override the harness runs its + resolvedHarnessType ?? input.options.defaultHarnessType ?? 'codex' + // Without an explicit override or channel default the harness runs its // configured default (CLAUDE_MODEL/CODEX_MODEL, else the baked harness // config); show and record that instead of dropping the model entirely. const effectiveModel = - effectiveOverrides.model ?? + resolvedModel ?? defaultModelForHarness(effectiveHarnessType, input.options.harnessDefaultModels) + const effectiveEffort = + effectiveHarnessType === 'codex' + ? resolvedReasoning ?? defaultCodexEffort(input.options.codexDefaultReasoningEffort) + : undefined + const effectiveSpeed = effectiveHarnessType === 'codex' ? defaultCodexSpeed() : undefined const consoleSessionBlock = isFirstAssistantMessage ? buildConsoleSessionContextBlock({ consoleBaseUrl: input.options.consolePublicUrl, threadKey: thread.id, harnessType: effectiveHarnessType, - model: effectiveModel + model: effectiveModel, + effort: effectiveEffort, + speed: effectiveSpeed }) : undefined if (overrides.harnessType || overrides.model || overrides.provider || overrides.reasoning) { @@ -871,12 +911,12 @@ async function syncThreadMessageToSession( executeMessage: shouldStartExecution ? serializedMessage : undefined, // Sticky harness changes only apply when a message starts an execution; // restarting the thread out from under an active execution would kill it. - harnessType: shouldStartExecution ? effectiveOverrides.harnessType : undefined, + harnessType: shouldStartExecution ? resolvedHarnessType : undefined, messages: messagesToAppend, - model: shouldStartExecution ? effectiveOverrides.model : undefined, + model: shouldStartExecution ? resolvedModel : undefined, metadataModel: shouldStartExecution ? effectiveModel : undefined, - provider: shouldStartExecution ? effectiveOverrides.provider : undefined, - reasoning: overrides.reasoning, + provider: shouldStartExecution ? resolvedProvider : undefined, + reasoning: resolvedReasoning, onEventId: eventId => { lastEventId = Math.max(lastEventId, eventId) }, diff --git a/services/slackbotv2/src/overrides.ts b/services/slackbotv2/src/overrides.ts index 1bfcd105f..ba143a9e3 100644 --- a/services/slackbotv2/src/overrides.ts +++ b/services/slackbotv2/src/overrides.ts @@ -20,14 +20,22 @@ * shortcuts imply the codex harness. */ -export type MessageOverrides = { - cleanedText: string +/** + * A resolved bundle of harness knobs (harness + model/provider/reasoning), all + * optional. Shared by the inline flag parser and per-channel defaults so both + * speak the same vocabulary. + */ +export type HarnessOverrides = { harnessType?: string model?: string provider?: string reasoning?: string } +export type MessageOverrides = HarnessOverrides & { + cleanedText: string +} + // Flag name -> HarnessType wire value (serde lowercase of the Rust enum). const HARNESS_FLAGS: Record<string, string> = { amp: 'amp', @@ -158,6 +166,57 @@ export function extractMessageOverrides(text: string): MessageOverrides { } } +/** + * Object-shaped counterpart to {@link extractMessageOverrides}: normalizes a + * `{ harness, model, provider, reasoning }` config through the same vocabulary + * as the flag parser (harness/provider/model aliases; a provider implies its + * harness, like `--bedrock`). Fields are independent; unrecognized harness / + * provider / reasoning values are reported via `onError` and dropped. + */ +export function normalizeHarnessOverrides( + raw: { harness?: unknown; model?: unknown; provider?: unknown; reasoning?: unknown }, + onError?: (message: string) => void +): HarnessOverrides { + let harnessType: string | undefined + let model: string | undefined + let provider: string | undefined + let reasoning: string | undefined + + const harnessRaw = cleanString(raw.harness) + if (harnessRaw) { + harnessType = HARNESS_FLAGS[harnessRaw.toLowerCase()] + if (!harnessType) onError?.(`unknown harness "${harnessRaw}"`) + } + + const providerRaw = cleanString(raw.provider) + if (providerRaw) { + const mapping = PROVIDER_FLAGS[providerRaw.toLowerCase()] + if (mapping) { + provider = mapping.provider + harnessType ??= mapping.harnessType // a provider implies its harness, like --bedrock + } else { + onError?.(`unknown provider "${providerRaw}"`) + } + } + + const modelRaw = cleanString(raw.model) + if (modelRaw) model = CLAUDE_MODEL_ALIASES[modelRaw.toLowerCase()] ?? modelRaw + + const reasoningRaw = cleanString(raw.reasoning) + if (reasoningRaw) { + reasoning = REASONING_EFFORTS[reasoningRaw.toLowerCase()] + if (!reasoning) onError?.(`unknown reasoning effort "${reasoningRaw}"`) + } + + return { harnessType, model, provider, reasoning } +} + +function cleanString(value: unknown): string | undefined { + if (typeof value !== 'string') return undefined + const trimmed = value.trim() + return trimmed === '' ? undefined : trimmed +} + function flagPattern(flag: string): RegExp { return new RegExp(`(?:^|\\s)--${flag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?=\\s|$)`, 'i') } diff --git a/services/slackbotv2/src/server.ts b/services/slackbotv2/src/server.ts index 14edb993d..0bcfd9a25 100644 --- a/services/slackbotv2/src/server.ts +++ b/services/slackbotv2/src/server.ts @@ -1,4 +1,5 @@ import { createSlackbotV2, type SlackbotV2Options } from './index' +import { parseChannelDefaults } from './channel-defaults' const port = numberEnv('PORT', 3002) const apiUrl = stringEnv('CENTAUR_API_URL', 'http://127.0.0.1:8080') @@ -32,7 +33,11 @@ const options: SlackbotV2Options = { activitySummaryStatusEnabled: booleanEnv('SLACKBOTV2_ACTIVITY_SUMMARY_STATUS_ENABLED', false), botToken, botUserId: optionalEnv('SLACK_BOT_USER_ID'), + channelDefaults: parseChannelDefaults(optionalEnv('SLACKBOTV2_CHANNEL_DEFAULTS'), reason => + consoleLogger.warn('slackbotv2 SLACKBOTV2_CHANNEL_DEFAULTS', { reason }) + ), consolePublicUrl: optionalEnv('CENTAUR_CONSOLE_PUBLIC_URL'), + codexDefaultReasoningEffort: optionalEnv('CODEX_MODEL_REASONING_EFFORT'), defaultHarnessType: optionalEnv('SLACKBOTV2_DEFAULT_HARNESS'), // Same env vars deployers use to override the sandbox harness model // (sandbox.extraEnv); the chart mirrors them here so displayed defaults diff --git a/services/slackbotv2/src/slack-display-text.ts b/services/slackbotv2/src/slack-display-text.ts index 51e37df08..c83562a35 100644 --- a/services/slackbotv2/src/slack-display-text.ts +++ b/services/slackbotv2/src/slack-display-text.ts @@ -7,6 +7,18 @@ export type SlackDisplayText = { text: string } +export function slackRichTextMentionsUser(raw: unknown, userId: string | undefined): boolean { + if (!userId) return false + const mention = new RegExp(`<@${escapeRegExp(userId)}(?:\\|[^>]*)?>`, 'i') + for (const record of slackMessageRecords(raw)) { + for (const key of ['blocks', 'attachments']) { + const value = record[key] + if (Array.isArray(value) && richValueMentionsUser(value, userId, mention)) return true + } + } + return false +} + const MAX_RAW_DISPLAY_TEXT_CHARS = 24_000 type UnknownRecord = Record<string, unknown> @@ -167,6 +179,22 @@ function collectRawAttachmentLines(records: UnknownRecord[]): string[] { return lines } +function richValueMentionsUser(value: unknown, userId: string, mention: RegExp): boolean { + if (typeof value === 'string') return mention.test(value) + if (Array.isArray(value)) { + return value.some(item => richValueMentionsUser(item, userId, mention)) + } + if (!isRecord(value)) return false + if (value.type === 'user' && stringField(value.user_id).toUpperCase() === userId.toUpperCase()) { + return true + } + return Object.values(value).some(item => richValueMentionsUser(item, userId, mention)) +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + function collectSlackAttachmentText(value: unknown, lines: string[]): void { if (!isRecord(value)) return collectStringFields(value, lines, ['fallback', 'pretext', 'title', 'text']) diff --git a/services/slackbotv2/src/slack-events.ts b/services/slackbotv2/src/slack-events.ts index 0a5a57a2a..86c54ad57 100644 --- a/services/slackbotv2/src/slack-events.ts +++ b/services/slackbotv2/src/slack-events.ts @@ -138,6 +138,15 @@ async function isAllowedTriggerBotMessage( stringValue(event.user), stringValue(event.bot_profile?.user_id) ) + const allowedBotIds = new Set( + allowlist + .map(entry => entry.trim()) + .filter(entry => entry.startsWith('bot:')) + .map(entry => entry.slice('bot:'.length)) + .filter(isSlackBotId) + ) + if ([...botIds].some(botId => allowedBotIds.has(botId))) return true + const allowedUserIds = new Set(allowlist.map(entry => entry.trim()).filter(isSlackMemberId)) if (!allowedUserIds.size) return false if ([...botUserIds].some(userId => allowedUserIds.has(userId))) return true @@ -158,6 +167,10 @@ function isSlackMemberId(value: string): boolean { return /^[UW][A-Z0-9]+$/i.test(value) } +function isSlackBotId(value: string): boolean { + return /^B[A-Z0-9]+$/i.test(value) +} + async function resolveTriggerBotIdentity( botId: string, options: SlackbotV2Options, diff --git a/services/slackbotv2/src/types.ts b/services/slackbotv2/src/types.ts index 505fb0b29..18a9a77ee 100644 --- a/services/slackbotv2/src/types.ts +++ b/services/slackbotv2/src/types.ts @@ -2,6 +2,7 @@ import type { RustSessionStreamEvent } from '@centaur/harness-events' import type { CodexAppServerToChatStreamOptions } from '@centaur/rendering' import type { Attachment, Chat, Logger, StateAdapter } from 'chat' import type { Hono } from 'hono' +import type { ChannelDefaults } from './channel-defaults' import type { SlackDisplayTextSource } from './slack-display-text' export type JsonPrimitive = string | number | boolean | null @@ -121,6 +122,13 @@ export type SlackbotV2Options = { * the block entirely. */ consolePublicUrl?: string + /** Codex effort displayed in Slack when no per-turn `-rsn` override is set. */ + codexDefaultReasoningEffort?: string + /** + * Per-channel default harness/model/provider/reasoning, keyed by Slack + * conversation id (SLACKBOTV2_CHANNEL_DEFAULTS). See channel-defaults.ts. + */ + channelDefaults?: ChannelDefaults /** * Harness for new threads when no --claude/--amp/--codex flag is given * (HarnessType wire value: codex | amp | claudecode). Defaults to codex. diff --git a/services/slackbotv2/test/channel-defaults.test.ts b/services/slackbotv2/test/channel-defaults.test.ts new file mode 100644 index 000000000..0a73c375f --- /dev/null +++ b/services/slackbotv2/test/channel-defaults.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from 'bun:test' +import { + channelIdFromThreadId, + parseChannelDefaults, + resolveChannelDefault +} from '../src/channel-defaults' + +describe('parseChannelDefaults', () => { + test('returns an empty map for unset or blank input', () => { + expect(parseChannelDefaults(undefined)).toEqual({}) + expect(parseChannelDefaults('')).toEqual({}) + expect(parseChannelDefaults(' ')).toEqual({}) + }) + + test('normalizes each channel object through the shared flag vocabulary', () => { + const parsed = parseChannelDefaults( + JSON.stringify({ + C0ENG: { harness: 'claude', model: 'opus', reasoning: 'high' }, + C0TRIAGE: { harness: 'codex', reasoning: 'low' }, + C0BEDROCK: { provider: 'bedrock', model: 'gpt-5.2' } + }) + ) + expect(parsed).toEqual({ + // `claude` -> wire harness, `opus` -> full model id. + C0ENG: { harnessType: 'claudecode', model: 'claude-opus-4-8', reasoning: 'high' }, + C0TRIAGE: { harnessType: 'codex', reasoning: 'low' }, + // A provider shortcut implies its harness, mirroring `--bedrock`. + C0BEDROCK: { harnessType: 'codex', model: 'gpt-5.2', provider: 'amazon-bedrock' } + }) + }) + + test('allows reasoning alone, with no harness or model', () => { + expect(parseChannelDefaults(JSON.stringify({ C0TRIAGE: { reasoning: 'low' } }))).toEqual({ + C0TRIAGE: { reasoning: 'low' } + }) + }) + + test('expands a model alias but leaves the harness to the explicit field', () => { + // Like `--model opus` (not `--opus`): fields are independent, so a model + // with no `harness` inherits the thread/deployment harness rather than one + // guessed from the model name. + expect( + parseChannelDefaults(JSON.stringify({ C0A: { model: 'opus' }, C0B: { model: 'gpt-5.2' } })) + ).toEqual({ + C0A: { model: 'claude-opus-4-8' }, + C0B: { model: 'gpt-5.2' } + }) + }) + + test('reports unknown field values and skips an entry that resolves to nothing', () => { + const reasons: string[] = [] + const parsed = parseChannelDefaults( + JSON.stringify({ + C0BAD: { harness: 'gpt', reasoning: 'turbo' }, + C0OK: { harness: 'codex' } + }), + reason => reasons.push(reason) + ) + expect(parsed).toEqual({ C0OK: { harnessType: 'codex' } }) + expect(reasons.some(r => r.includes('C0BAD') && r.includes('unknown harness'))).toBe(true) + expect(reasons.some(r => r.includes('C0BAD') && r.includes('unknown reasoning'))).toBe(true) + expect(reasons.some(r => r.includes('C0BAD') && r.includes('no usable'))).toBe(true) + }) + + test('reports and skips a non-object entry', () => { + const reasons: string[] = [] + const parsed = parseChannelDefaults( + JSON.stringify({ C0ENG: '--claude', C0OK: { harness: 'codex' } }), + reason => reasons.push(reason) + ) + expect(parsed).toEqual({ C0OK: { harnessType: 'codex' } }) + expect(reasons.some(r => r.includes('C0ENG') && r.includes('expected an object'))).toBe(true) + }) + + test('reports and ignores invalid JSON without throwing', () => { + const reasons: string[] = [] + expect(parseChannelDefaults('{not json', reason => reasons.push(reason))).toEqual({}) + expect(reasons).toHaveLength(1) + expect(reasons[0]).toContain('invalid JSON') + }) + + test('reports and ignores a non-object top level', () => { + const reasons: string[] = [] + expect(parseChannelDefaults('["C0ENG"]', reason => reasons.push(reason))).toEqual({}) + expect(reasons[0]).toContain('object') + }) +}) + +describe('channelIdFromThreadId', () => { + test('extracts the channel segment from a slack thread key', () => { + expect(channelIdFromThreadId('slack:C0ENG:1700000000.0001')).toBe('C0ENG') + expect(channelIdFromThreadId('slack:T0TEAM:C0ENG:1700000000.0001')).toBe('C0ENG') + expect(channelIdFromThreadId('slack:D0DM')).toBe('D0DM') + expect(channelIdFromThreadId('slack:G0GROUP:ts')).toBe('G0GROUP') + }) + + test('returns undefined when no conversation segment is present', () => { + expect(channelIdFromThreadId('web:t1')).toBeUndefined() + expect(channelIdFromThreadId('slack')).toBeUndefined() + }) +}) + +describe('resolveChannelDefault', () => { + const defaults = { C0ENG: { harnessType: 'claudecode', model: 'claude-opus-4-8' } } + + test('returns the default for a matching channel', () => { + expect(resolveChannelDefault(defaults, 'slack:C0ENG:1700000000.0001')).toEqual({ + harnessType: 'claudecode', + model: 'claude-opus-4-8' + }) + }) + + test('returns undefined for an unmapped channel or missing config', () => { + expect(resolveChannelDefault(defaults, 'slack:C0OTHER:ts')).toBeUndefined() + expect(resolveChannelDefault(undefined, 'slack:C0ENG:ts')).toBeUndefined() + expect(resolveChannelDefault(defaults, 'web:t1')).toBeUndefined() + }) +}) diff --git a/services/slackbotv2/test/chat-sdk-emulate.test.ts b/services/slackbotv2/test/chat-sdk-emulate.test.ts index 9ed22dcb5..e43214f3e 100644 --- a/services/slackbotv2/test/chat-sdk-emulate.test.ts +++ b/services/slackbotv2/test/chat-sdk-emulate.test.ts @@ -24,6 +24,7 @@ import { import { clearRequesterIdentityCacheForTests } from '../src/session-api' import { slackbotMetrics } from '../src/metrics' import claudeSettings from '../../../harness/claude/settings.json' +import codexConfig from '../../../harness/codex/config.toml' const BOT_TOKEN = 'xoxb-slackbotv2-emulate' const USER_TOKEN = 'xoxp-slackbotv2-user' @@ -175,6 +176,103 @@ describe('slackbotv2', () => { expect(codexApi.executes[0]?.threadKey).toBe(threadKey(message.ts)) }) + it('denies an unallowlisted bot-authored message in an ambient channel', async () => { + bot = createTestBot({ ambientSlackChannelIds: [CHANNEL_ID] }) + const message = await postUserMessage('Unapproved bot ambient message.') + const waits: Promise<unknown>[] = [] + const response = await bot.app.request( + '/api/webhooks/slack', + signedSlackEvent({ + event_id: 'Ev-slackbotv2-ambient-bot-denied', + event: { + type: 'message', + bot_id: 'BOTHERBOT', + channel: CHANNEL_ID, + subtype: 'bot_message', + team: TEAM_ID, + ts: message.ts, + text: 'Unapproved bot ambient message.', + user: 'UOTHERBOT', + username: 'otherbot' + } + }), + {}, + waitUntilContext(waits) + ) + + expect(response.status).toBe(200) + await Promise.all(waits) + expect(codexApi.appends).toHaveLength(0) + expect(codexApi.executes).toHaveLength(0) + }) + + it('allows an approved bot-authored message in an ambient channel', async () => { + bot = createTestBot({ + ambientSlackChannelIds: [CHANNEL_ID], + triggerBotAllowlist: ['UOTHERBOT'] + }) + const message = await postUserMessage('Approved bot ambient message.') + const waits: Promise<unknown>[] = [] + const response = await bot.app.request( + '/api/webhooks/slack', + signedSlackEvent({ + event_id: 'Ev-slackbotv2-ambient-bot-allowed', + event: { + type: 'message', + app_id: 'AOTHERBOT', + bot_id: 'BOTHERBOT', + bot_profile: { + app_id: 'AOTHERBOT', + id: 'BOTHERBOT', + user_id: 'UOTHERBOT' + }, + channel: CHANNEL_ID, + subtype: 'bot_message', + team: TEAM_ID, + ts: message.ts, + text: 'Approved bot ambient message.', + user: 'UOTHERBOT', + username: 'otherbot' + } + }), + {}, + waitUntilContext(waits) + ) + + expect(response.status).toBe(200) + await Promise.all(waits) + expect(codexApi.appends).toHaveLength(1) + expect(codexApi.executes).toHaveLength(1) + }) + + it('executes a rich mention in an ambient channel exactly once', async () => { + bot = createTestBot({ ambientSlackChannelIds: [CHANNEL_ID] }) + const message = await postUserMessage('') + const waits: Promise<unknown>[] = [] + const response = await bot.app.request( + '/api/webhooks/slack', + signedSlackEvent({ + event_id: 'Ev-slackbotv2-ambient-rich-mention', + event: { + type: 'message', + attachments: [{ pretext: `<@${BOT_USER_ID}> investigate the ambient alert` }], + channel: CHANNEL_ID, + team: TEAM_ID, + ts: message.ts, + text: '', + user: USER_ID + } + }), + {}, + waitUntilContext(waits) + ) + + expect(response.status).toBe(200) + await Promise.all(waits) + expect(codexApi.appends).toHaveLength(1) + expect(codexApi.executes).toHaveLength(1) + }) + it('dedupes an ambient root retry after the first delivery subscribes the thread', async () => { bot = createTestBot({ ambientSlackChannelIds: [CHANNEL_ID] }) const message = await postUserMessage('Execute this ambient message exactly once.') @@ -751,10 +849,7 @@ describe('slackbotv2', () => { .filter(text => text.includes('Open chat in Console')) const parent = await postUserMessage('Default model thread context.') - const mention = await postUserMessage( - `<@${BOT_USER_ID}> --claude what is your current model?`, - parent.ts - ) + const mention = await postUserMessage(`<@${BOT_USER_ID}> what is your current model?`, parent.ts) const waits: Promise<unknown>[] = [] const response = await bot.app.request( '/api/webhooks/slack', @@ -767,7 +862,7 @@ describe('slackbotv2', () => { team: TEAM_ID, ts: mention.ts, thread_ts: parent.ts, - text: `<@${BOT_USER_ID}> --claude what is your current model?` + text: `<@${BOT_USER_ID}> what is your current model?` } }), {}, @@ -778,18 +873,222 @@ describe('slackbotv2', () => { const blocks = consoleBlockTexts(slackApi.calls) expect(blocks).toHaveLength(1) - expect(blocks[0]).toContain('Claude Code') - expect(blocks[0]).toContain(claudeSettings.model.toUpperCase()) + expect(blocks[0]).toContain('Codex') + expect(blocks[0]).toContain(codexConfig.model.toUpperCase()) + expect(blocks[0]).toContain('Effort: Low') + expect(blocks[0]).toContain('Speed: Fast') // The effective (default) model is recorded in execution metadata for the // Console, but never forwarded to the harness — only explicit overrides // ride the input lines. expect(codexApi.executes).toHaveLength(1) const executeBody = codexApi.executes[0]!.body - expect(executeBody.metadata.model).toBe(claudeSettings.model) + expect(executeBody.metadata.model).toBe(codexConfig.model) expect(JSON.parse(executeBody.input_lines.at(-1)!).model).toBeUndefined() }) + it('shows a channel-default Codex reasoning effort in the Console context block', async () => { + bot = createTestBot({ + channelDefaults: { [CHANNEL_ID]: { reasoning: 'high' } }, + consolePublicUrl: 'https://console.example.dev' + }) + + const parent = await postUserMessage('Channel reasoning context.') + const mention = await postUserMessage(`<@${BOT_USER_ID}> use the channel effort`, parent.ts) + const waits: Promise<unknown>[] = [] + const response = await bot.app.request( + '/api/webhooks/slack', + signedSlackEvent({ + event_id: 'Ev-slackbotv2-console-link-channel-reasoning', + event: { + type: 'app_mention', + user: USER_ID, + channel: CHANNEL_ID, + team: TEAM_ID, + ts: mention.ts, + thread_ts: parent.ts, + text: `<@${BOT_USER_ID}> use the channel effort` + } + }), + {}, + waitUntilContext(waits) + ) + expect(response.status).toBe(200) + await Promise.all(waits) + + const blocks = slackApi.calls + .filter(call => call.method === 'chat.stopStream') + .flatMap(call => (Array.isArray(call.body.blocks) ? (call.body.blocks as unknown[]) : [])) + .map(block => JSON.stringify(block)) + .filter(text => text.includes('Open chat in Console')) + expect(blocks).toHaveLength(1) + expect(blocks[0]).toContain('Effort: High') + + expect(codexApi.executes).toHaveLength(1) + expect(JSON.parse(codexApi.executes[0]!.body.input_lines.at(-1)!).reasoning).toBe('high') + }) + + it('forwards a per-channel default harness + model + reasoning onto the turn', async () => { + const sharedState = createMemoryState() + await sharedState.connect() + bot = createTestBot({ + state: sharedState, + // The channel pins the harness and its model together (as `--claude + // --model opus -rsn high` would parse), so the pair can't mismatch. + channelDefaults: { + [CHANNEL_ID]: { harnessType: 'claudecode', model: 'claude-opus-4-8', reasoning: 'high' } + } + }) + + const parent = await postUserMessage('Channel default thread context.') + const mention = await postUserMessage(`<@${BOT_USER_ID}> investigate this`, parent.ts) + const waits: Promise<unknown>[] = [] + const response = await bot.app.request( + '/api/webhooks/slack', + signedSlackEvent({ + event_id: 'Ev-slackbotv2-channel-default', + event: { + type: 'app_mention', + user: USER_ID, + channel: CHANNEL_ID, + team: TEAM_ID, + ts: mention.ts, + thread_ts: parent.ts, + text: `<@${BOT_USER_ID}> investigate this` + } + }), + {}, + waitUntilContext(waits) + ) + expect(response.status).toBe(200) + await Promise.all(waits) + + // No explicit flags, but the channel default selects the harness and rides + // the model/reasoning onto the input line (unlike the deployment/baked + // default) and is recorded for the Console. + expect(codexApi.creates.map(create => create.body.harness_type)).toEqual(['claudecode']) + expect(codexApi.executes).toHaveLength(1) + const executeBody = codexApi.executes[0]!.body + const inputLine = JSON.parse(executeBody.input_lines.at(-1)!) as Record<string, unknown> + expect(inputLine.model).toBe('claude-opus-4-8') + expect(inputLine.reasoning).toBe('high') + expect(executeBody.metadata.model).toBe('claude-opus-4-8') + }) + + it('lets an explicit per-thread flag override the per-channel default harness + model', async () => { + const sharedState = createMemoryState() + await sharedState.connect() + bot = createTestBot({ + state: sharedState, + channelDefaults: { + [CHANNEL_ID]: { harnessType: 'claudecode', model: 'claude-opus-4-8', reasoning: 'high' } + } + }) + + const parent = await postUserMessage('Channel default override thread context.') + const mention = await postUserMessage( + `<@${BOT_USER_ID}> --codex --model gpt-5.4 -rsn low go`, + parent.ts + ) + const waits: Promise<unknown>[] = [] + const response = await bot.app.request( + '/api/webhooks/slack', + signedSlackEvent({ + event_id: 'Ev-slackbotv2-channel-default-override', + event: { + type: 'app_mention', + user: USER_ID, + channel: CHANNEL_ID, + team: TEAM_ID, + ts: mention.ts, + thread_ts: parent.ts, + text: `<@${BOT_USER_ID}> --codex --model gpt-5.4 -rsn low go` + } + }), + {}, + waitUntilContext(waits) + ) + expect(response.status).toBe(200) + await Promise.all(waits) + + // Explicit --codex/--model/-rsn beat every field of the channel default. + expect(codexApi.creates.map(create => create.body.harness_type)).toEqual(['codex']) + expect(codexApi.executes).toHaveLength(1) + const inputLine = JSON.parse(codexApi.executes[0]!.body.input_lines.at(-1)!) as Record< + string, + unknown + > + expect(inputLine.model).toBe('gpt-5.4') + expect(inputLine.reasoning).toBe('low') + }) + + it('a harness-only override does not drag the channel default model onto the new harness', async () => { + const sharedState = createMemoryState() + await sharedState.connect() + bot = createTestBot({ + state: sharedState, + channelDefaults: { + [CHANNEL_ID]: { harnessType: 'claudecode', model: 'claude-opus-4-8', reasoning: 'high' } + } + }) + + const parent = await postUserMessage('Harness-switch thread context.') + // Switch to codex with no model. The channel's Claude model must NOT ride + // onto codex; switching harness clears the previous harness's model. + const first = await postUserMessage(`<@${BOT_USER_ID}> --codex look into this`, parent.ts) + const firstWaits: Promise<unknown>[] = [] + await bot.app.request( + '/api/webhooks/slack', + signedSlackEvent({ + event_id: 'Ev-slackbotv2-channel-harness-switch-1', + event: { + type: 'app_mention', + user: USER_ID, + channel: CHANNEL_ID, + team: TEAM_ID, + ts: first.ts, + thread_ts: parent.ts, + text: `<@${BOT_USER_ID}> --codex look into this` + } + }), + {}, + waitUntilContext(firstWaits) + ) + await Promise.all(firstWaits) + + // A follow-up with no flags: the cleared model must stay cleared (the sticky + // tombstone persists), not resurface from the channel default. + const second = await postUserMessage(`<@${BOT_USER_ID}> keep going`, parent.ts) + const secondWaits: Promise<unknown>[] = [] + await bot.app.request( + '/api/webhooks/slack', + signedSlackEvent({ + event_id: 'Ev-slackbotv2-channel-harness-switch-2', + event: { + type: 'app_mention', + user: USER_ID, + channel: CHANNEL_ID, + team: TEAM_ID, + ts: second.ts, + thread_ts: parent.ts, + text: `<@${BOT_USER_ID}> keep going` + } + }), + {}, + waitUntilContext(secondWaits) + ) + await Promise.all(secondWaits) + + // Both turns run on codex (the second inherits the sticky harness); neither + // forwards a model — the channel's Claude model never leaks onto codex. + expect(codexApi.creates.map(c => c.body.harness_type)).toEqual(['codex', 'codex']) + expect(codexApi.executes).toHaveLength(2) + for (const execute of codexApi.executes) { + const line = JSON.parse(execute.body.input_lines.at(-1)!) as Record<string, unknown> + expect(line.model).toBeUndefined() + } + }) + it('includes all preceding Slack thread messages for a first mid-thread mention', async () => { const parent = await postUserMessage('Root context for the thread.') const firstReply = await postUserMessage('First preceding reply.', parent.ts) @@ -4616,6 +4915,78 @@ describe('slackbotv2', () => { recipient_user_id: 'UOTHERBOT' }) ) + + bot = createTestBot({ triggerBotAllowlist: ['UOTHERBOT'] }) + codexApi.reset() + slackApi.reset() + const richBotMessage = await postUserMessage('') + const richBotWaits: Promise<unknown>[] = [] + const richBotResponse = await bot.app.request( + '/api/webhooks/slack', + signedSlackEvent({ + event_id: 'Ev-slackbotv2-bot-attachment-mention-allowed', + event: { + type: 'message', + app_id: 'AOTHERBOT', + attachments: [ + { + pretext: `<@${BOT_USER_ID}> investigate`, + title: ':red_circle: Validator stalled', + text: '*Cluster:* stg-na\n*Tenant:* luganodes' + } + ], + bot_id: 'BOTHERBOT', + bot_profile: { + app_id: 'AOTHERBOT', + id: 'BOTHERBOT', + user_id: 'UOTHERBOT' + }, + channel: CHANNEL_ID, + subtype: 'bot_message', + team: TEAM_ID, + text: '', + ts: richBotMessage.ts, + username: 'otherbot' + } + }), + {}, + waitUntilContext(richBotWaits) + ) + expect(richBotResponse.status).toBe(200) + await Promise.all(richBotWaits) + expect(codexApi.appends).toHaveLength(1) + expect(codexApi.executes).toHaveLength(1) + expect(sessionMessageTexts(codexApi.appends[0]!.body.messages).join('\n')).toContain( + 'Validator stalled' + ) + + bot = createTestBot() + codexApi.reset() + const deniedRichBotMessage = await postUserMessage('') + const deniedRichBotWaits: Promise<unknown>[] = [] + const deniedRichBotResponse = await bot.app.request( + '/api/webhooks/slack', + signedSlackEvent({ + event_id: 'Ev-slackbotv2-bot-attachment-mention-denied', + event: { + type: 'message', + attachments: [{ pretext: `<@${BOT_USER_ID}> investigate` }], + bot_id: 'BOTHERBOT', + channel: CHANNEL_ID, + subtype: 'bot_message', + team: TEAM_ID, + text: '', + ts: deniedRichBotMessage.ts, + username: 'otherbot' + } + }), + {}, + waitUntilContext(deniedRichBotWaits) + ) + expect(deniedRichBotResponse.status).toBe(200) + await Promise.all(deniedRichBotWaits) + expect(codexApi.appends).toHaveLength(0) + expect(codexApi.executes).toHaveLength(0) }) }) diff --git a/services/slackbotv2/test/console-session-link.test.ts b/services/slackbotv2/test/console-session-link.test.ts index 3e67fa77f..f6de94e38 100644 --- a/services/slackbotv2/test/console-session-link.test.ts +++ b/services/slackbotv2/test/console-session-link.test.ts @@ -2,6 +2,8 @@ import { describe, expect, test } from 'bun:test' import { buildConsoleSessionContextBlock, consoleSessionUrl, + defaultCodexEffort, + defaultCodexSpeed, defaultModelForHarness, harnessDisplayName } from '../src/console-session-link' @@ -85,12 +87,14 @@ describe('consoleSessionUrl', () => { }) describe('buildConsoleSessionContextBlock', () => { - test('builds a context block with uppercased model then harness, middot separated', () => { + test('builds a context block with model, harness, effort, and speed', () => { const block = buildConsoleSessionContextBlock({ consoleBaseUrl: 'https://console.centaur.dev', threadKey: 'slack:C123:1700000000.000100', harnessType: 'codex', - model: 'gpt-5.2' + model: 'gpt-5.2', + effort: 'xhigh', + speed: 'fast' }) expect(block).toEqual({ type: 'context', @@ -98,7 +102,7 @@ describe('buildConsoleSessionContextBlock', () => { { type: 'mrkdwn', text: - '<https://console.centaur.dev/console/threads?thread=slack%3AC123%3A1700000000.000100|Open chat in Console> · GPT-5.2 · Codex' + '<https://console.centaur.dev/console/threads?thread=slack%3AC123%3A1700000000.000100|Open chat in Console> · GPT-5.2 · Codex · Effort: Xhigh · Speed: Fast' } ] }) @@ -126,3 +130,15 @@ describe('buildConsoleSessionContextBlock', () => { ).toBeUndefined() }) }) + +describe('Codex display defaults', () => { + test('reads effort and speed from the baked Codex config', () => { + expect(defaultCodexEffort()).toBe('low') + expect(defaultCodexSpeed()).toBe('fast') + }) + + test('allows deployment-configured defaults to override baked values', () => { + expect(defaultCodexEffort('high')).toBe('high') + expect(defaultCodexSpeed('flex')).toBe('flex') + }) +}) diff --git a/services/slackbotv2/test/overrides.test.ts b/services/slackbotv2/test/overrides.test.ts index d09130f9f..5e111393b 100644 --- a/services/slackbotv2/test/overrides.test.ts +++ b/services/slackbotv2/test/overrides.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'bun:test' import { SlackFormatConverter } from '@chat-adapter/slack' -import { extractMessageOverrides } from '../src/overrides' +import { extractMessageOverrides, normalizeHarnessOverrides } from '../src/overrides' describe('extractMessageOverrides', () => { test('returns text untouched without flags', () => { @@ -281,6 +281,60 @@ describe('extractMessageOverrides', () => { }) }) +// normalizeHarnessOverrides is the object-shaped sibling of +// extractMessageOverrides: config fields resolve through the SAME vocabulary +// tables as the inline flags, so a channel default and a Slack flag validate +// identically. +describe('normalizeHarnessOverrides', () => { + test('resolves harness / model / provider / reasoning through the flag vocabulary', () => { + expect( + normalizeHarnessOverrides({ harness: 'claude', model: 'opus', reasoning: 'hi' }) + ).toEqual({ + harnessType: 'claudecode', + model: 'claude-opus-4-8', + provider: undefined, + reasoning: 'high' + }) + }) + + test('a provider shortcut implies its harness, like --bedrock', () => { + expect(normalizeHarnessOverrides({ provider: 'bedrock', model: 'gpt-5.2' })).toEqual({ + harnessType: 'codex', + model: 'gpt-5.2', + provider: 'amazon-bedrock', + reasoning: undefined + }) + }) + + test('expands a model alias but does not imply a harness (fields are independent)', () => { + // Like `--model opus` (not `--opus`): the alias expands, harness is left to + // the explicit `harness` field / thread / deployment default. + expect(normalizeHarnessOverrides({ model: 'opus' })).toEqual({ + harnessType: undefined, + model: 'claude-opus-4-8', + provider: undefined, + reasoning: undefined + }) + }) + + test('reports and drops unrecognized enum-like values', () => { + const errors: string[] = [] + const result = normalizeHarnessOverrides( + { harness: 'gpt', provider: 'openai', reasoning: 'turbo' }, + message => errors.push(message) + ) + expect(result).toEqual({ + harnessType: undefined, + model: undefined, + provider: undefined, + reasoning: undefined + }) + expect(errors.some(e => e.includes('unknown harness'))).toBe(true) + expect(errors.some(e => e.includes('unknown provider'))).toBe(true) + expect(errors.some(e => e.includes('unknown reasoning effort'))).toBe(true) + }) +}) + // The adapter's plain-text extraction feeds extractMessageOverrides. The // unpatched @chat-adapter/slack flattened the parsed AST with // mdast-util-to-string, which joins sibling paragraphs with NO separator — diff --git a/services/slackbotv2/test/slack-display-text.test.ts b/services/slackbotv2/test/slack-display-text.test.ts new file mode 100644 index 000000000..d363f94e5 --- /dev/null +++ b/services/slackbotv2/test/slack-display-text.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'bun:test' +import { slackRichTextMentionsUser } from '../src/slack-display-text' + +const BOT_USER_ID = 'U0ANX3AM5RR' +const MENTION = `<@${BOT_USER_ID}> investigate` + +describe('Slack rich-text mentions', () => { + for (const [name, raw] of [ + ['attachment pretext', { attachments: [{ pretext: MENTION }] }], + ['attachment fallback', { attachments: [{ fallback: MENTION }] }], + ['attachment title', { attachments: [{ title: MENTION }] }], + ['attachment text', { attachments: [{ text: MENTION }] }], + ['attachment field', { attachments: [{ fields: [{ value: MENTION }] }] }], + [ + 'attachment block', + { attachments: [{ blocks: [{ type: 'section', text: { type: 'mrkdwn', text: MENTION } }] }] } + ], + ['top-level block', { blocks: [{ type: 'section', text: { type: 'mrkdwn', text: MENTION } }] }], + [ + 'Block Kit user element', + { + blocks: [ + { + type: 'rich_text', + elements: [ + { type: 'rich_text_section', elements: [{ type: 'user', user_id: BOT_USER_ID }] } + ] + } + ] + } + ], + ['labeled mention', { attachments: [{ pretext: `<@${BOT_USER_ID}|centaur> investigate` }] }] + ] as const) { + it(`recognizes ${name}`, () => { + expect(slackRichTextMentionsUser(raw, BOT_USER_ID)).toBe(true) + }) + } + + it('does not infer a mention from top-level text or plain display text', () => { + expect(slackRichTextMentionsUser({ text: MENTION }, BOT_USER_ID)).toBe(false) + expect(slackRichTextMentionsUser({ attachments: [{ pretext: `@${BOT_USER_ID} investigate` }] }, BOT_USER_ID)).toBe(false) + }) + + it('requires the exact configured bot user', () => { + expect(slackRichTextMentionsUser({ attachments: [{ pretext: MENTION }] }, 'UOTHER')).toBe(false) + expect(slackRichTextMentionsUser({ attachments: [{ pretext: MENTION }] }, undefined)).toBe(false) + }) +}) diff --git a/services/slackbotv2/test/slack-events.test.ts b/services/slackbotv2/test/slack-events.test.ts index dd717bc0c..f468dc5b6 100644 --- a/services/slackbotv2/test/slack-events.test.ts +++ b/services/slackbotv2/test/slack-events.test.ts @@ -87,6 +87,20 @@ describe('Slack trigger bot allowlist', () => { expect(await isAllowedSlackMessage(botMessage('BOTHER'), config, logger)).toBe(false) }) + it('allows an explicitly scoped bot identifier without an identity lookup', async () => { + let requests = 0 + const config = { + ...options(async () => { + requests += 1 + return Response.json({ ok: true }) + }), + triggerBotAllowlist: ['bot:BCHANNELBOT'] + } + + expect(await isAllowedSlackMessage(botMessage('BCHANNELBOT'), config, logger)).toBe(true) + expect(requests).toBe(0) + }) + it('does not treat bot or app identifiers as public allowlist entries', async () => { let requests = 0 const config = { diff --git a/services/workflow-python/api/workflow_engine.py b/services/workflow-python/api/workflow_engine.py index 389a6e399..3fd351436 100644 --- a/services/workflow-python/api/workflow_engine.py +++ b/services/workflow-python/api/workflow_engine.py @@ -25,12 +25,17 @@ def __init__( task_id: str, workflow_name: str, pool: Any = None, + agent_defaults: dict[str, Any] | None = None, ) -> None: self._rpc = rpc self.run_id = run_id self.task_id = task_id self.workflow_name = workflow_name self._pool = pool + # Module-level `AGENT_DEFAULTS` (e.g. {"model": ..., "reasoning": ...}) + # applied to every ctx.agent_turn as a per-workflow default; explicit + # per-call kwargs always win. See agent_turn(). + self._agent_defaults = dict(agent_defaults or {}) self.tools = WorkflowTools(WorkflowToolManager(self._rpc)) def log(self, event: str, **fields: Any) -> None: @@ -97,7 +102,9 @@ async def sleep_until(self, name: str, when: dt.datetime) -> None: ) async def agent_turn(self, text: str | None = None, **kwargs: Any) -> Any: - args = dict(kwargs) + # Per-workflow AGENT_DEFAULTS (model / provider / reasoning / harness, + # ...) form the base; explicit per-call kwargs override them key by key. + args = {**self._agent_defaults, **kwargs} if text is not None: args.setdefault("text", text) return await self._rpc.request({"type": "ctx.agent_turn", "args": args}) @@ -112,6 +119,23 @@ async def run_agent(self, *args: Any, text: str | None = None, **kwargs: Any) -> async def start_agent(self, *args: Any, text: str | None = None, **kwargs: Any) -> Any: return await self.run_agent(*args, text=text, **kwargs) + async def start_workflow( + self, + workflow_name: str, + input: dict[str, Any] | None = None, + *, + idempotency_key: str | None = None, + ) -> dict[str, Any]: + """Queue another workflow and return its durable task identifiers.""" + request: dict[str, Any] = { + "type": "ctx.workflow.start", + "workflow_name": workflow_name, + "input": input or {}, + } + if idempotency_key: + request["idempotency_key"] = idempotency_key + return await self._rpc.request(request) + async def call_tool(self, tool: str, method: str, args: dict[str, Any] | None = None) -> Any: return await WorkflowToolManager(self._rpc).call_tool_raw(tool, method, args or {}) diff --git a/services/workflow-python/tests/test_workflow_host.py b/services/workflow-python/tests/test_workflow_host.py index 71f26727b..36ed47178 100644 --- a/services/workflow-python/tests/test_workflow_host.py +++ b/services/workflow-python/tests/test_workflow_host.py @@ -4,6 +4,7 @@ import importlib.util import os import sys +import tempfile import types import unittest from pathlib import Path @@ -59,6 +60,13 @@ async def request(self, payload): } if message_type == "ctx.agent_turn": return payload["args"] + if message_type == "ctx.workflow.start": + return { + "workflow_name": payload["workflow_name"], + "task_id": "task-child", + "run_id": "run-child", + "created": True, + } if message_type == "ctx.sleep": return {"slept": True} raise AssertionError(f"unexpected request {payload}") @@ -162,6 +170,73 @@ def test_run_agent_accepts_positional_step_name_with_text(self) -> None: self.assertEqual(result, {"name": "draft_summary", "text": "summarize this"}) + def test_agent_turn_applies_workflow_agent_defaults(self) -> None: + host = load_workflow_host() + rpc = RequestRpc() + ctx = host.WorkflowContext( + rpc, + run_id="run-123", + task_id="task-456", + workflow_name="sample", + agent_defaults={"model": "claude-opus-4-8", "reasoning": "high"}, + ) + + result = asyncio.run(ctx.agent_turn("do the thing")) + + self.assertEqual( + result, + {"model": "claude-opus-4-8", "reasoning": "high", "text": "do the thing"}, + ) + + def test_agent_turn_per_call_kwargs_override_agent_defaults(self) -> None: + host = load_workflow_host() + rpc = RequestRpc() + ctx = host.WorkflowContext( + rpc, + run_id="run-123", + task_id="task-456", + workflow_name="sample", + agent_defaults={"model": "claude-opus-4-8", "reasoning": "high"}, + ) + + result = asyncio.run(ctx.agent_turn("cheap step", reasoning="low")) + + self.assertEqual( + result, + {"model": "claude-opus-4-8", "reasoning": "low", "text": "cheap step"}, + ) + + def test_start_workflow_enqueues_durable_child_with_idempotency_key(self) -> None: + host = load_workflow_host() + rpc = RequestRpc() + ctx = host.WorkflowContext( + rpc, + run_id="run-123", + task_id="task-456", + workflow_name="sample", + ) + + result = asyncio.run( + ctx.start_workflow( + "company_context_documents", + {"scope": "slack_thread"}, + idempotency_key="company-context:slack-thread:42", + ) + ) + + self.assertEqual(result["task_id"], "task-child") + self.assertEqual( + rpc.requests, + [ + { + "type": "ctx.workflow.start", + "workflow_name": "company_context_documents", + "input": {"scope": "slack_thread"}, + "idempotency_key": "company-context:slack-thread:42", + } + ], + ) + def test_create_pool_retries_transient_connection_failure(self) -> None: host = load_workflow_host() calls = [] @@ -247,6 +322,70 @@ async def create_pool(): self.assertTrue(rpc.drained) self.assertTrue(pool.closed) + def test_run_workflow_threads_agent_defaults_into_context(self) -> None: + host = load_workflow_host() + rpc = RequestRpc() + + async def handler(inp, ctx): + return await ctx.agent_turn("do the thing") + + registered = host.RegisteredWorkflow( + workflow_name="sample_workflow", + source_path="workflows/sample.py", + handler=handler, + input_cls=None, + webhooks=None, + schedule=None, + agent_defaults={"model": "claude-opus-4-8", "reasoning": "high"}, + ) + + async def create_pool(): + return None + + with ( + patch.object( + host, + "discover_workflows", + return_value={"sample_workflow": registered}, + ), + patch.object(host, "create_pool", create_pool), + ): + payload = asyncio.run( + host.run_workflow( + { + "type": "workflow.start", + "workflow_name": "sample_workflow", + "run_id": "run-123", + "task_id": "task-456", + "input": {}, + }, + rpc, + ) + ) + + self.assertEqual( + payload["result"], + {"model": "claude-opus-4-8", "reasoning": "high", "text": "do the thing"}, + ) + + def test_load_workflow_file_reads_agent_defaults(self) -> None: + host = load_workflow_host() + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "defaults_workflow.py" + path.write_text( + "WORKFLOW_NAME = 'defaults_workflow'\n" + "AGENT_DEFAULTS = {'model': 'claude-opus-4-8', 'reasoning': 'high'}\n" + "def handler(inp, ctx):\n" + " return None\n" + ) + registered = host.load_workflow_file(path) + + assert registered is not None + self.assertEqual( + registered.agent_defaults, + {"model": "claude-opus-4-8", "reasoning": "high"}, + ) + if __name__ == "__main__": unittest.main() diff --git a/services/workflow-python/workflow_host.py b/services/workflow-python/workflow_host.py index eac67f2a4..528dc16e9 100644 --- a/services/workflow-python/workflow_host.py +++ b/services/workflow-python/workflow_host.py @@ -84,6 +84,7 @@ class RegisteredWorkflow: input_cls: type | None webhooks: Any schedule: Any + agent_defaults: dict[str, Any] | None = None def workflow_dirs() -> list[Path]: @@ -142,6 +143,9 @@ def load_workflow_file(path: Path) -> RegisteredWorkflow | None: handler = getattr(module, "handler", None) if not isinstance(workflow_name, str) or not callable(handler): return None + agent_defaults = getattr(module, "AGENT_DEFAULTS", None) + if not isinstance(agent_defaults, dict): + agent_defaults = None return RegisteredWorkflow( workflow_name=workflow_name, source_path=str(path), @@ -149,6 +153,7 @@ def load_workflow_file(path: Path) -> RegisteredWorkflow | None: input_cls=getattr(module, "Input", None), webhooks=getattr(module, "WEBHOOKS", None), schedule=getattr(module, "SCHEDULE", None), + agent_defaults=agent_defaults, ) @@ -335,6 +340,7 @@ async def run_workflow(message: dict[str, Any], rpc: RpcClient) -> dict[str, Any task_id=str(message.get("task_id") or ""), workflow_name=workflow_name, pool=pool, + agent_defaults=registered.agent_defaults, ) previous_metric_rpc = metrics.get_metric_rpc() metrics.set_metric_rpc(rpc) diff --git a/tools/infra/centaur-console/__init__.py b/tools/infra/centaur-console/__init__.py new file mode 100644 index 000000000..32e93a64b --- /dev/null +++ b/tools/infra/centaur-console/__init__.py @@ -0,0 +1 @@ +"""Centaur console sandbox-permissions tool.""" diff --git a/tools/infra/centaur-console/cli.py b/tools/infra/centaur-console/cli.py new file mode 100644 index 000000000..1e8d106be --- /dev/null +++ b/tools/infra/centaur-console/cli.py @@ -0,0 +1,80 @@ +"""CLI for centaur-console sandbox permission introspection.""" + +from __future__ import annotations + +import json + +import typer +from dotenv import load_dotenv +from rich.console import Console + +load_dotenv() + +app = typer.Typer( + name="centaur-console", + help="Inspect the current sandbox's centaur-console permissions", +) +console = Console() + + +def get_client( + url: str | None = None, + bearer_token: str | None = None, +): + from .client import ConsoleClient + + return ConsoleClient(url=url, bearer_token=bearer_token) + + +@app.command("permissions") +def permissions( + url: str | None = typer.Option(None, "--url", help="centaur-console base URL"), + bearer_token: str | None = typer.Option( + None, + "--bearer-token", + help="Local/debug bearer token override", + envvar="CENTAUR_CONSOLE_BEARER_TOKEN", + ), +): + """Print the current sandbox's redacted permissions as JSON.""" + with get_client(url=url, bearer_token=bearer_token) as client: + result = client.sandbox_permissions() + console.print_json(json.dumps(result, default=str)) + + +@app.command("oauth-apps") +def oauth_apps( + url: str | None = typer.Option(None, "--url", help="centaur-console base URL"), + bearer_token: str | None = typer.Option( + None, + "--bearer-token", + help="Local/debug bearer token override", + envvar="CENTAUR_CONSOLE_BEARER_TOKEN", + ), +): + """Print enabled OAuth apps and their consent start URLs as JSON.""" + with get_client(url=url, bearer_token=bearer_token) as client: + result = client.sandbox_oauth_apps() + console.print_json(json.dumps({"data": result}, default=str)) + + +@app.command() +def health( + url: str | None = typer.Option(None, "--url", help="centaur-console base URL"), + bearer_token: str | None = typer.Option( + None, + "--bearer-token", + help="Local/debug bearer token override", + envvar="CENTAUR_CONSOLE_BEARER_TOKEN", + ), +): + """Assert the sandbox permissions endpoint is reachable and authorized.""" + with get_client(url=url, bearer_token=bearer_token) as client: + payload = client.health() + print(json.dumps(payload, indent=2, default=str)) + if not payload.get("ok"): + raise typer.Exit(1) + + +if __name__ == "__main__": + app() diff --git a/tools/infra/centaur-console/client.py b/tools/infra/centaur-console/client.py new file mode 100644 index 000000000..28f63075b --- /dev/null +++ b/tools/infra/centaur-console/client.py @@ -0,0 +1,139 @@ +"""Client for centaur-console sandbox-scoped permission introspection.""" + +from __future__ import annotations + +import os +from typing import Any + +import httpx + +SANDBOX_PERMISSIONS_PATH = "/api/v1/sandbox/permissions" +SANDBOX_OAUTH_APPS_PATH = "/api/v1/sandbox/oauth_apps" + + +class ConsoleClient: + """Read the current sandbox's redacted permissions from centaur-console.""" + + def __init__( + self, + url: str | None = None, + bearer_token: str | None = None, + timeout: float = 30.0, + transport: httpx.BaseTransport | None = None, + ): + self._url = url + self._bearer_token = bearer_token + self.timeout = timeout + self._transport = transport + self._client: httpx.Client | None = None + + @property + def base_url(self) -> str: + # Non-secret endpoint config. Sandboxes receive this from api-rs. + url = (self._url or os.getenv("CENTAUR_CONSOLE_URL", "http://centaur-console:3000")).strip().rstrip("/") # noqa: TID251 + if url and not url.startswith(("http://", "https://")): + url = f"http://{url}" + return url + + def _headers(self) -> dict[str, str]: + headers = {"Accept": "application/json"} + # Optional local/debug override. In sandboxes, iron-proxy injects the + # scoped Authorization header for this endpoint. + bearer = (self._bearer_token or os.getenv("CENTAUR_CONSOLE_BEARER_TOKEN", "")).strip() # noqa: TID251 + if bearer: + headers["Authorization"] = f"Bearer {bearer}" + return headers + + @property + def client(self) -> httpx.Client: + if self._client is None: + self._client = httpx.Client( + base_url=self.base_url, + headers=self._headers(), + timeout=self.timeout, + transport=self._transport, + ) + return self._client + + def sandbox_permissions(self) -> dict[str, Any]: + """Return the current sandbox's redacted permissions payload.""" + response = self.client.get(SANDBOX_PERMISSIONS_PATH) + try: + response.raise_for_status() + except httpx.HTTPStatusError as exc: + detail = _response_error_detail(exc.response) + raise RuntimeError(f"centaur-console permissions request failed: {detail}") from exc + + payload = response.json() + data = payload.get("data") + if not isinstance(data, dict): + raise RuntimeError("centaur-console permissions response did not include a data object") + return data + + def permissions(self) -> dict[str, Any]: + """Alias for tool bridge calls.""" + return self.sandbox_permissions() + + def sandbox_oauth_apps(self) -> list[dict[str, Any]]: + """Return enabled OAuth apps with user-facing consent start URLs.""" + response = self.client.get(SANDBOX_OAUTH_APPS_PATH) + try: + response.raise_for_status() + except httpx.HTTPStatusError as exc: + detail = _response_error_detail(exc.response) + raise RuntimeError(f"centaur-console OAuth apps request failed: {detail}") from exc + + payload = response.json() + data = payload.get("data") + if not isinstance(data, list): + raise RuntimeError("centaur-console OAuth apps response did not include a data array") + return data + + def oauth_apps(self) -> list[dict[str, Any]]: + """Alias for tool bridge calls.""" + return self.sandbox_oauth_apps() + + def health(self) -> dict[str, Any]: + """Assert the sandbox permissions endpoint is reachable and authorized.""" + try: + data = self.sandbox_permissions() + return { + "ok": True, + "tool": "centaur-console", + "error": None, + "details": { + "sandbox_id": data.get("sandbox_id"), + "principal_id": data.get("principal_id"), + "proxy_id": data.get("proxy_id"), + }, + } + except Exception as exc: + return { + "ok": False, + "tool": "centaur-console", + "error": str(exc), + "details": {}, + } + + def close(self) -> None: + if self._client: + self._client.close() + self._client = None + + def __enter__(self) -> ConsoleClient: + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + +def _response_error_detail(response: httpx.Response) -> str: + try: + body = response.json() + except ValueError: + body = response.text + return f"HTTP {response.status_code}: {body}" + + +def _client() -> ConsoleClient: + return ConsoleClient() diff --git a/tools/infra/centaur-console/pyproject.toml b/tools/infra/centaur-console/pyproject.toml new file mode 100644 index 000000000..4795d8567 --- /dev/null +++ b/tools/infra/centaur-console/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "centaur-console" +description = "Centaur console sandbox permission introspection" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [ + "httpx>=0.27.0", + "typer>=0.12.0", + "rich>=13.0.0", + "python-dotenv>=1.0.0", +] + +[project.scripts] +centaur-console = "centaur_tool_centaur_console.cli:app" + +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.hatch.build.targets.wheel.sources] +"." = "centaur_tool_centaur_console" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.centaur] +module = "client.py" +hosts = [] +secrets = [] diff --git a/tools/infra/centaur-console/test_client.py b/tools/infra/centaur-console/test_client.py new file mode 100644 index 000000000..2126bc2a0 --- /dev/null +++ b/tools/infra/centaur-console/test_client.py @@ -0,0 +1,115 @@ +import httpx +import pytest +from client import SANDBOX_OAUTH_APPS_PATH, SANDBOX_PERMISSIONS_PATH, ConsoleClient + + +def json_response(payload, status_code=200): + return httpx.Response(status_code, json=payload) + + +def make_client(handler, *, bearer_token=None): + return ConsoleClient( + url="http://centaur-console:3000", + bearer_token=bearer_token, + transport=httpx.MockTransport(handler), + ) + + +def test_sandbox_permissions_fetches_and_unwraps_data(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" + assert request.url.path == SANDBOX_PERMISSIONS_PATH + assert request.headers["Accept"] == "application/json" + return json_response( + { + "data": { + "sandbox_id": "sandbox-1", + "principal_id": "prn_123", + "permissions": {"secrets": []}, + } + } + ) + + result = make_client(handler).sandbox_permissions() + + assert result["sandbox_id"] == "sandbox-1" + assert result["principal_id"] == "prn_123" + assert result["permissions"] == {"secrets": []} + + +def test_sandbox_permissions_sends_debug_bearer_token_when_provided(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.headers["Authorization"] == "Bearer test-token" + return json_response({"data": {"sandbox_id": "sandbox-1"}}) + + assert make_client(handler, bearer_token="test-token").permissions()["sandbox_id"] == "sandbox-1" + + +def test_sandbox_permissions_wraps_http_errors(): + def handler(_request: httpx.Request) -> httpx.Response: + return json_response({"error": {"message": "invalid sandbox token"}}, status_code=401) + + with pytest.raises(RuntimeError, match="HTTP 401"): + make_client(handler).sandbox_permissions() + + +def test_sandbox_oauth_apps_fetches_and_unwraps_data(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" + assert request.url.path == SANDBOX_OAUTH_APPS_PATH + assert request.headers["Accept"] == "application/json" + return json_response( + { + "data": [ + { + "slug": "google", + "provider": "google", + "start_url": "https://console.example/oauth/google/start", + } + ] + } + ) + + result = make_client(handler).sandbox_oauth_apps() + + assert result == [ + { + "slug": "google", + "provider": "google", + "start_url": "https://console.example/oauth/google/start", + } + ] + + +def test_sandbox_oauth_apps_wraps_http_errors(): + def handler(_request: httpx.Request) -> httpx.Response: + return json_response({"error": {"message": "invalid sandbox token"}}, status_code=401) + + with pytest.raises(RuntimeError, match="HTTP 401"): + make_client(handler).sandbox_oauth_apps() + + +def test_health_returns_identity_details(): + def handler(_request: httpx.Request) -> httpx.Response: + return json_response( + { + "data": { + "sandbox_id": "sandbox-1", + "proxy_id": "proxy-1", + "principal_id": "principal-1", + } + } + ) + + result = make_client(handler).health() + + assert result == { + "ok": True, + "tool": "centaur-console", + "error": None, + "details": { + "sandbox_id": "sandbox-1", + "proxy_id": "proxy-1", + "principal_id": "principal-1", + }, + } diff --git a/tools/productivity/airtable/cli.py b/tools/productivity/airtable/cli.py index e319d0e5c..0788e6dce 100644 --- a/tools/productivity/airtable/cli.py +++ b/tools/productivity/airtable/cli.py @@ -20,7 +20,7 @@ def health(): client = _client() try: - details = client.preflight_access() + details = client.health() payload = {"ok": True, "tool": "airtable", "error": None, "details": details} except Exception as exc: payload = {"ok": False, "tool": "airtable", "error": str(exc), "details": {}} @@ -40,6 +40,30 @@ def _print(data: object) -> None: console.print_json(json.dumps(data, default=str)) +def _json_object(value: str) -> dict: + parsed = json.loads(value) + if not isinstance(parsed, dict): + raise typer.BadParameter("Expected a JSON object.") + return parsed + + +def _json_object_list(value: str) -> list[dict]: + parsed = json.loads(value) + if not isinstance(parsed, list) or not all(isinstance(item, dict) for item in parsed): + raise typer.BadParameter("Expected a JSON array of objects.") + return parsed + + +def _comma_list(value: str) -> list[str]: + return [item.strip() for item in value.split(",") if item.strip()] + + +@app.command() +def whoami() -> None: + """Show the current Airtable API key identity.""" + _print(AirtableClient().whoami()) + + @app.command() def bases(limit: int = typer.Option(100, "--limit", "-n")) -> None: """List visible Airtable bases.""" @@ -69,5 +93,127 @@ def from_url(url: str, max_records: int = typer.Option(50, "--max-records", "-n" _print(AirtableClient().snapshot_from_url(url, max_records=max_records)) +@app.command() +def create_record( + base_id: str, + table: str, + fields: str = typer.Option(..., "--fields", help="Record fields as a JSON object."), + typecast: bool = typer.Option(False, "--typecast", help="Let Airtable coerce field values."), +) -> None: + """Create one record.""" + _print(AirtableClient().create_record(base_id, table, _json_object(fields), typecast=typecast)) + + +@app.command() +def create_records( + base_id: str, + table: str, + records: str = typer.Option( + ..., + "--records", + help="Records as a JSON array. Items may be field objects or objects with a fields property.", + ), + typecast: bool = typer.Option(False, "--typecast", help="Let Airtable coerce field values."), +) -> None: + """Create records.""" + _print( + AirtableClient().create_records( + base_id, table, _json_object_list(records), typecast=typecast + ) + ) + + +@app.command() +def update_record( + base_id: str, + table: str, + record_id: str, + fields: str = typer.Option(..., "--fields", help="Updated fields as a JSON object."), + typecast: bool = typer.Option(False, "--typecast", help="Let Airtable coerce field values."), + replace: bool = typer.Option(False, "--replace", help="Replace all writable fields with PUT."), +) -> None: + """Update one record.""" + _print( + AirtableClient().update_record( + base_id, + table, + record_id, + _json_object(fields), + typecast=typecast, + replace=replace, + ) + ) + + +@app.command() +def update_records( + base_id: str, + table: str, + records: str = typer.Option( + ..., + "--records", + help="Updates as a JSON array of objects with id and fields properties.", + ), + typecast: bool = typer.Option(False, "--typecast", help="Let Airtable coerce field values."), + replace: bool = typer.Option(False, "--replace", help="Replace all writable fields with PUT."), +) -> None: + """Update records.""" + _print( + AirtableClient().update_records( + base_id, + table, + _json_object_list(records), + typecast=typecast, + replace=replace, + ) + ) + + +@app.command() +def upsert_records( + base_id: str, + table: str, + records: str = typer.Option( + ..., + "--records", + help="Records as a JSON array. Items may be field objects or objects with a fields property.", + ), + merge_fields: str = typer.Option( + ..., + "--merge-fields", + help="Comma-separated field names for Airtable performUpsert matching.", + ), + typecast: bool = typer.Option(False, "--typecast", help="Let Airtable coerce field values."), +) -> None: + """Create or update records by merge fields.""" + _print( + AirtableClient().upsert_records( + base_id, + table, + _json_object_list(records), + fields_to_merge_on=_comma_list(merge_fields), + typecast=typecast, + ) + ) + + +@app.command() +def delete_record(base_id: str, table: str, record_id: str) -> None: + """Delete one record.""" + _print(AirtableClient().delete_record(base_id, table, record_id)) + + +@app.command() +def delete_records( + base_id: str, + table: str, + record_ids: str = typer.Option( + ..., "--record-ids", help="Comma-separated Airtable record IDs." + ), +) -> None: + """Delete records.""" + _print(AirtableClient().delete_records(base_id, table, _comma_list(record_ids))) + + if __name__ == "__main__": app() diff --git a/tools/productivity/airtable/client.py b/tools/productivity/airtable/client.py index 9f1a2d4f1..171358f21 100644 --- a/tools/productivity/airtable/client.py +++ b/tools/productivity/airtable/client.py @@ -1,4 +1,4 @@ -"""Airtable API client for bases, schemas, views, and records.""" +"""Airtable API client for bases, schemas, views, and record writes.""" from __future__ import annotations @@ -57,6 +57,12 @@ def _compact_record(record: dict[str, Any], fields: list[str] | None = None) -> } +def _compact_records( + records: list[dict[str, Any]], fields: list[str] | None = None +) -> list[dict[str, Any]]: + return [_compact_record(record, fields) for record in records] + + def _match_text(value: Any, query: str) -> bool: if value is None: return False @@ -73,6 +79,47 @@ def _path_part(value: str) -> str: return quote(value, safe="") +def _clamp_batch_size(value: int) -> int: + return max(1, min(value, 10)) + + +def _record_batch( + records: list[dict[str, Any]], batch_size: int = 10 +) -> list[list[dict[str, Any]]]: + size = _clamp_batch_size(batch_size) + return [records[index : index + size] for index in range(0, len(records), size)] + + +def _record_id_batch(record_ids: list[str], batch_size: int = 10) -> list[list[str]]: + size = _clamp_batch_size(batch_size) + return [record_ids[index : index + size] for index in range(0, len(record_ids), size)] + + +def _normalize_create_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + normalized: list[dict[str, Any]] = [] + for record in records: + fields = record.get("fields") if "fields" in record else record + if not isinstance(fields, dict): + raise ValueError( + "Each record must be a field mapping or an object with a 'fields' mapping." + ) + normalized.append({"fields": fields}) + return normalized + + +def _normalize_update_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + normalized: list[dict[str, Any]] = [] + for record in records: + record_id = record.get("id") + fields = record.get("fields") + if not isinstance(record_id, str) or not record_id: + raise ValueError("Each record update must include a non-empty 'id'.") + if not isinstance(fields, dict): + raise ValueError("Each record update must include a 'fields' mapping.") + normalized.append({"id": record_id, "fields": fields}) + return normalized + + def _airtable_host(host: str | None) -> str: return (host or "").split(":", 1)[0].lower() @@ -174,7 +221,9 @@ def _raise_for_error(self, response: httpx.Response) -> None: response.raise_for_status() except httpx.HTTPStatusError as exc: detail = error_message or exc.response.text - raise RuntimeError(f"Airtable API error: {exc.response.status_code} - {detail}") from exc + raise RuntimeError( + f"Airtable API error: {exc.response.status_code} - {detail}" + ) from exc def _request( self, @@ -252,6 +301,15 @@ def whoami(self) -> dict[str, Any]: """Return the Airtable user/workspace identity for this API key.""" return self._request("GET", f"{META_URL}/whoami") + def current_user(self) -> dict[str, Any]: + """Return a privacy-minimized identity for the current Airtable API key.""" + return _minimal_identity(self.whoami()) + + def health(self) -> dict[str, Any]: + """Check Airtable authentication and report the current API key identity.""" + whoami = self.whoami() + return {"current_user": {"id": whoami.get("id")}} + def preflight_access( self, url: str | None = None, @@ -382,7 +440,8 @@ def preflight_access( return { "ok": False, "status": "bad_url" if url else "bad_target", - "message": probe_error_message or "Airtable could not resolve the requested base, table, or view.", + "message": probe_error_message + or "Airtable could not resolve the requested base, table, or view.", "target": target, "auth": { "attempted": True, @@ -514,9 +573,10 @@ def find_tables(self, base_id: str, query: str, limit: int = 20) -> list[dict[st needle = query.lower() matches: list[dict[str, Any]] = [] for table in self.list_tables(base_id): - table_hit = needle in str(table.get("name", "")).lower() or needle in str( - table.get("id", "") - ).lower() + table_hit = ( + needle in str(table.get("name", "")).lower() + or needle in str(table.get("id", "")).lower() + ) view_hits = [ view for view in table.get("views", []) @@ -629,6 +689,163 @@ def search_records( "records": matches, } + def create_record( + self, + base_id: str, + table: str, + fields: dict[str, Any], + typecast: bool = False, + ) -> dict[str, Any]: + """Create one Airtable record. + + `table` may be a table ID or table name. `fields` maps Airtable field names to values. + The API key needs Airtable's `data.records:write` scope for the target base. + """ + data = self._request( + "POST", + f"{BASE_URL}/{_path_part(base_id)}/{_path_part(table)}", + json={"fields": fields, "typecast": typecast}, + ) + return _compact_record(data) + + def create_records( + self, + base_id: str, + table: str, + records: list[dict[str, Any]], + typecast: bool = False, + ) -> dict[str, Any]: + """Create Airtable records in batches of up to 10 records per request.""" + normalized = _normalize_create_records(records) + created: list[dict[str, Any]] = [] + for batch in _record_batch(normalized): + data = self._request( + "POST", + f"{BASE_URL}/{_path_part(base_id)}/{_path_part(table)}", + json={"records": batch, "typecast": typecast}, + ) + created.extend(data.get("records", [])) + return { + "base_id": base_id, + "table": table, + "count": len(created), + "records": _compact_records(created), + } + + def update_record( + self, + base_id: str, + table: str, + record_id: str, + fields: dict[str, Any], + typecast: bool = False, + replace: bool = False, + ) -> dict[str, Any]: + """Update one Airtable record. + + By default this partially updates fields with PATCH. Set `replace=True` to use PUT. + """ + method = "PUT" if replace else "PATCH" + data = self._request( + method, + f"{BASE_URL}/{_path_part(base_id)}/{_path_part(table)}/{_path_part(record_id)}", + json={"fields": fields, "typecast": typecast}, + ) + return _compact_record(data) + + def update_records( + self, + base_id: str, + table: str, + records: list[dict[str, Any]], + typecast: bool = False, + replace: bool = False, + ) -> dict[str, Any]: + """Update Airtable records in batches of up to 10 records per request.""" + normalized = _normalize_update_records(records) + method = "PUT" if replace else "PATCH" + updated: list[dict[str, Any]] = [] + for batch in _record_batch(normalized): + data = self._request( + method, + f"{BASE_URL}/{_path_part(base_id)}/{_path_part(table)}", + json={"records": batch, "typecast": typecast}, + ) + updated.extend(data.get("records", [])) + return { + "base_id": base_id, + "table": table, + "count": len(updated), + "records": _compact_records(updated), + } + + def upsert_records( + self, + base_id: str, + table: str, + records: list[dict[str, Any]], + fields_to_merge_on: list[str], + typecast: bool = False, + ) -> dict[str, Any]: + """Create or update records using Airtable's `performUpsert` merge fields.""" + if not fields_to_merge_on: + raise ValueError("fields_to_merge_on must include at least one field name.") + normalized = _normalize_create_records(records) + upserted: list[dict[str, Any]] = [] + created_record_ids: list[str] = [] + updated_record_ids: list[str] = [] + for batch in _record_batch(normalized): + data = self._request( + "PATCH", + f"{BASE_URL}/{_path_part(base_id)}/{_path_part(table)}", + json={ + "records": batch, + "typecast": typecast, + "performUpsert": {"fieldsToMergeOn": fields_to_merge_on}, + }, + ) + upserted.extend(data.get("records", [])) + created_record_ids.extend(data.get("createdRecords", [])) + updated_record_ids.extend(data.get("updatedRecords", [])) + return { + "base_id": base_id, + "table": table, + "count": len(upserted), + "records": _compact_records(upserted), + "createdRecords": created_record_ids, + "updatedRecords": updated_record_ids, + } + + def delete_record(self, base_id: str, table: str, record_id: str) -> dict[str, Any]: + """Delete one Airtable record.""" + return self._request( + "DELETE", + f"{BASE_URL}/{_path_part(base_id)}/{_path_part(table)}/{_path_part(record_id)}", + ) + + def delete_records( + self, + base_id: str, + table: str, + record_ids: list[str], + ) -> dict[str, Any]: + """Delete Airtable records in batches of up to 10 record IDs per request.""" + deleted: list[dict[str, Any]] = [] + for batch in _record_id_batch(record_ids): + params = [("records[]", record_id) for record_id in batch] + data = self._request( + "DELETE", + f"{BASE_URL}/{_path_part(base_id)}/{_path_part(table)}", + params=params, + ) + deleted.extend(data.get("records", [])) + return { + "base_id": base_id, + "table": table, + "count": len(deleted), + "records": deleted, + } + def snapshot_from_url(self, url: str, max_records: int = 50) -> dict[str, Any]: """Return a compact table-shaped snapshot for an Airtable table/view URL.""" data = self.records_from_url(url, max_records=max_records) diff --git a/tools/productivity/airtable/pyproject.toml b/tools/productivity/airtable/pyproject.toml index 02fdfa58a..69a67b600 100644 --- a/tools/productivity/airtable/pyproject.toml +++ b/tools/productivity/airtable/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "airtable" -description = "Airtable API client for bases, schemas, views, and records" +description = "Airtable API client for bases, schemas, views, records, and record writes" version = "0.1.0" requires-python = ">=3.11" dependencies = [ diff --git a/tools/productivity/airtable/test_client.py b/tools/productivity/airtable/test_client.py index 488f27638..67ae2c316 100644 --- a/tools/productivity/airtable/test_client.py +++ b/tools/productivity/airtable/test_client.py @@ -1,15 +1,20 @@ from __future__ import annotations +import importlib import importlib.util +import json import sys from pathlib import Path import httpx +import pytest +from typer.testing import CliRunner REPO_ROOT = Path(__file__).resolve().parents[3] sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "tools" / "productivity")) -from centaur_sdk import ToolContext, reset_tool_context, set_tool_context +from centaur_sdk import ToolContext, reset_tool_context, set_tool_context # noqa: E402 CLIENT_PATH = REPO_ROOT / "tools" / "productivity" / "airtable" / "client.py" @@ -23,6 +28,12 @@ def _load_airtable_module(): return module +def _load_airtable_cli_module(): + for module_name in ("airtable.cli", "airtable.client", "airtable"): + sys.modules.pop(module_name, None) + return importlib.import_module("airtable.cli") + + def _mock_client(client, handler) -> None: client._client.close() client._client = httpx.Client( @@ -36,7 +47,9 @@ def test_client_factory_loads_without_secret_and_preflight_reports_missing_secre token = set_tool_context(ToolContext(name="airtable", secrets={"AIRTABLE_API_KEY": ""})) try: client = module._client() - result = client.preflight_access(url="https://airtable.com/appBase123/tblTable456/viwView789") + result = client.preflight_access( + url="https://airtable.com/appBase123/tblTable456/viwView789" + ) client.close() finally: reset_tool_context(token) @@ -46,6 +59,115 @@ def test_client_factory_loads_without_secret_and_preflight_reports_missing_secre assert result["probe"]["attempted"] is False +def test_health_raises_missing_secret_without_network_call() -> None: + module = _load_airtable_module() + token = set_tool_context(ToolContext(name="airtable", secrets={"AIRTABLE_API_KEY": ""})) + try: + client = module._client() + with pytest.raises(RuntimeError, match="AIRTABLE_API_KEY not set"): + client.health() + finally: + client.close() + reset_tool_context(token) + + +def test_health_reports_current_user_from_whoami() -> None: + module = _load_airtable_module() + client = module.AirtableClient(api_key="test-key") + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" + assert request.url.path == "/v0/meta/whoami" + return httpx.Response( + 200, + request=request, + json={ + "id": "usr123", + "email": "ada@example.com", + "scopes": ["schema.bases:read", "data.records:read"], + }, + ) + + _mock_client(client, handler) + try: + result = client.health() + finally: + client.close() + + assert result == {"current_user": {"id": "usr123"}} + assert "email" not in result["current_user"] + + +def test_health_raises_invalid_token_from_whoami() -> None: + module = _load_airtable_module() + client = module.AirtableClient(api_key="test-key") + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" + assert request.url.path == "/v0/meta/whoami" + return httpx.Response( + 401, + request=request, + json={ + "error": { + "type": "AUTHENTICATION_REQUIRED", + "message": "Authentication required", + } + }, + ) + + _mock_client(client, handler) + try: + with pytest.raises(RuntimeError, match="AIRTABLE_API_KEY is missing or invalid"): + client.health() + finally: + client.close() + + +def test_cli_health_command_reports_current_user(monkeypatch) -> None: + cli = _load_airtable_cli_module() + + class FakeClient: + def health(self) -> dict: + return {"current_user": {"id": "usr123"}} + + def close(self) -> None: + pass + + monkeypatch.setattr(sys.modules["airtable.client"], "_client", lambda: FakeClient()) + + result = CliRunner().invoke(cli.app, ["health"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["ok"] is True + assert payload["tool"] == "airtable" + assert payload["details"]["current_user"] == {"id": "usr123"} + + +def test_cli_whoami_command_reports_current_user(monkeypatch) -> None: + cli = _load_airtable_cli_module() + + class FakeClient: + def whoami(self) -> dict: + return { + "id": "usr123", + "email": "ada@example.com", + "scopes": ["schema.bases:read"], + } + + monkeypatch.setattr(cli, "AirtableClient", FakeClient) + + whoami = CliRunner().invoke(cli.app, ["whoami"]) + + assert whoami.exit_code == 0, whoami.output + assert json.loads(whoami.output) == { + "id": "usr123", + "email": "ada@example.com", + "scopes": ["schema.bases:read"], + } + + def test_preflight_access_reports_invalid_token_from_whoami() -> None: module = _load_airtable_module() client = module.AirtableClient(api_key="test-key") @@ -81,7 +203,9 @@ def test_preflight_access_reports_missing_base_scope_from_probe() -> None: def handler(request: httpx.Request) -> httpx.Response: if request.method == "GET" and request.url.path == "/v0/meta/whoami": - return httpx.Response(200, request=request, json={"id": "usr123", "scopes": ["data.records:read"]}) + return httpx.Response( + 200, request=request, json={"id": "usr123", "scopes": ["data.records:read"]} + ) if request.method == "GET" and request.url.path == "/v0/appBase123/tblTable456": assert request.url.params.get("pageSize") == "1" return httpx.Response( @@ -148,7 +272,9 @@ def handler(request: httpx.Request) -> httpx.Response: _mock_client(client, handler) try: - result = client.preflight_access(url="https://airtable.com/appBase123/tblTable456/viwView789") + result = client.preflight_access( + url="https://airtable.com/appBase123/tblTable456/viwView789" + ) finally: client.close() @@ -156,3 +282,140 @@ def handler(request: httpx.Request) -> httpx.Response: assert result["status"] == "ok" assert result["probe"]["type"] == "records" assert result["probe"]["details"] == {"record_count": 1, "has_more": True} + + +def test_create_record_posts_fields_and_compacts_response() -> None: + module = _load_airtable_module() + client = module.AirtableClient(api_key="test-key") + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert request.url.path == "/v0/appBase123/tblTable456" + assert json.loads(request.content) == { + "fields": {"Name": "Ada", "Done": True}, + "typecast": True, + } + return httpx.Response( + 200, + request=request, + json={ + "id": "rec1", + "createdTime": "2026-07-13T00:00:00.000Z", + "fields": {"Name": "Ada", "Done": True}, + }, + ) + + _mock_client(client, handler) + try: + result = client.create_record( + "appBase123", + "tblTable456", + {"Name": "Ada", "Done": True}, + typecast=True, + ) + finally: + client.close() + + assert result == { + "id": "rec1", + "createdTime": "2026-07-13T00:00:00.000Z", + "fields": {"Name": "Ada", "Done": True}, + } + + +def test_update_records_batches_and_uses_patch() -> None: + module = _load_airtable_module() + client = module.AirtableClient(api_key="test-key") + calls: list[list[dict]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "PATCH" + assert request.url.path == "/v0/appBase123/tblTable456" + body = json.loads(request.content) + calls.append(body["records"]) + assert body["typecast"] is False + return httpx.Response(200, request=request, json={"records": body["records"]}) + + records = [{"id": f"rec{index}", "fields": {"Index": index}} for index in range(11)] + _mock_client(client, handler) + try: + result = client.update_records("appBase123", "tblTable456", records) + finally: + client.close() + + assert [len(call) for call in calls] == [10, 1] + assert result["count"] == 11 + assert result["records"][0] == { + "id": "rec0", + "createdTime": None, + "fields": {"Index": 0}, + } + + +def test_upsert_records_sends_perform_upsert() -> None: + module = _load_airtable_module() + client = module.AirtableClient(api_key="test-key") + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "PATCH" + assert request.url.path == "/v0/appBase123/Table Name" + assert json.loads(request.content) == { + "records": [{"fields": {"External ID": "ext-1", "Name": "Ada"}}], + "typecast": False, + "performUpsert": {"fieldsToMergeOn": ["External ID"]}, + } + return httpx.Response( + 200, + request=request, + json={ + "records": [{"id": "rec1", "fields": {"External ID": "ext-1", "Name": "Ada"}}], + "createdRecords": ["rec1"], + "updatedRecords": [], + }, + ) + + _mock_client(client, handler) + try: + result = client.upsert_records( + "appBase123", + "Table Name", + [{"External ID": "ext-1", "Name": "Ada"}], + fields_to_merge_on=["External ID"], + ) + finally: + client.close() + + assert result["createdRecords"] == ["rec1"] + assert result["updatedRecords"] == [] + assert result["records"][0]["id"] == "rec1" + + +def test_delete_records_batches_query_params() -> None: + module = _load_airtable_module() + client = module.AirtableClient(api_key="test-key") + calls: list[list[str]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "DELETE" + assert request.url.path == "/v0/appBase123/tblTable456" + record_ids = request.url.params.get_list("records[]") + calls.append(record_ids) + return httpx.Response( + 200, + request=request, + json={"records": [{"id": record_id, "deleted": True} for record_id in record_ids]}, + ) + + _mock_client(client, handler) + try: + result = client.delete_records( + "appBase123", + "tblTable456", + [f"rec{index}" for index in range(12)], + ) + finally: + client.close() + + assert [len(call) for call in calls] == [10, 2] + assert result["count"] == 12 + assert result["records"][-1] == {"id": "rec11", "deleted": True} diff --git a/tools/productivity/company_context/cli.py b/tools/productivity/company_context/cli.py index db56a8620..50fcbacf6 100644 --- a/tools/productivity/company_context/cli.py +++ b/tools/productivity/company_context/cli.py @@ -17,7 +17,7 @@ app = typer.Typer( name="company_context", - help="Search indexed company history, Slack DMs, and Google Docs.", + help="Search indexed company history, Slack DMs, Google Docs, and Granola notes.", ) @@ -75,7 +75,7 @@ def search( source: str | None = typer.Option( None, "--source", - help="Filter by source. Use 'docs' for Google Docs.", + help="Filter by source. Use 'docs' for Google Docs or 'granola' for Granola notes.", ), source_type: str | None = typer.Option(None, "--source-type", help="Filter by source type."), occurred_after: str | None = typer.Option( @@ -86,7 +86,7 @@ def search( ), json_output: bool = typer.Option(False, "--json", help="Output raw JSON."), ) -> None: - """Search indexed company context documents, including Google Docs with --source docs.""" + """Search indexed company context, including Google Docs and Granola notes.""" result = CompanyContextClient().search( query=query, limit=limit, @@ -211,7 +211,7 @@ def list_documents( source: str | None = typer.Option( None, "--source", - help="Filter by source. Use 'docs' for Google Docs.", + help="Filter by source. Use 'docs' for Google Docs or 'granola' for Granola notes.", ), source_type: str | None = typer.Option(None, "--source-type", help="Filter by source type."), occurred_after: str | None = typer.Option( @@ -222,7 +222,7 @@ def list_documents( ), json_output: bool = typer.Option(False, "--json", help="Output raw JSON."), ) -> None: - """List indexed company context documents, including Google Docs with --source docs.""" + """List indexed company context documents, including Google Docs and Granola notes.""" result = CompanyContextClient().list_documents( limit=limit, source=source, @@ -263,7 +263,7 @@ def read_document( ), json_output: bool = typer.Option(False, "--json", help="Output raw JSON."), ) -> None: - """Read a company context document returned by search, including Google Docs chunks.""" + """Read a company context document returned by search, including Granola notes.""" result = CompanyContextClient().read_document( document_id=document_id, max_chars=max_chars, @@ -291,7 +291,7 @@ def latest_date( source: str | None = typer.Option( None, "--source", - help="Filter by source. Use 'docs' for Google Docs.", + help="Filter by source. Use 'docs' for Google Docs or 'granola' for Granola notes.", ), source_type: str | None = typer.Option(None, "--source-type", help="Filter by source type."), ) -> None: diff --git a/tools/productivity/company_context/client.py b/tools/productivity/company_context/client.py index 340932abf..899005771 100644 --- a/tools/productivity/company_context/client.py +++ b/tools/productivity/company_context/client.py @@ -456,7 +456,7 @@ def _granola_doc_summary(row: Any) -> dict[str, Any]: def _dm_document_summary(row: Any) -> dict[str, Any]: - """Return the common metadata we expose for Slack DM context records.""" + """Return metadata for user-scoped Slack conversation context records.""" metadata = _as_dict(_row_value(row, "metadata", {})) conversation_type = str(_row_value(row, "conversation_type", "")) conversation_id = str(_row_value(row, "conversation_id", "")) @@ -473,7 +473,11 @@ def _dm_document_summary(row: Any) -> dict[str, Any]: "title": str(_row_value(row, "title", "")), "url": str(_row_value(row, "permalink", "")), "author_name": user_id or bot_id, - "access_scope": "slack_dm", + "access_scope": ( + "slack_private_channel" + if conversation_type == "private_channel" + else "slack_dm" + ), "occurred_at": _isoformat(_row_value(row, "occurred_at")), "source_updated_at": _isoformat(_row_value(row, "source_updated_at")), "conversation_id": conversation_id, @@ -521,6 +525,7 @@ def _include_slack_dms_source(source: str | None, source_type: str | None) -> bo SLACK_DM_SOURCE, "slack_im", "slack_mpim", + "slack_private_channel", "slack_dm_conversation", ) @@ -925,8 +930,14 @@ async def _latest_slack_dms_for_connection( return self._empty_latest_date_result(source=source, source_type=source_type) message_conversation_type = None - include_messages = source_type in (None, SLACK_DM_SOURCE, "slack_im", "slack_mpim") - if source_type in ("slack_im", "slack_mpim"): + include_messages = source_type in ( + None, + SLACK_DM_SOURCE, + "slack_im", + "slack_mpim", + "slack_private_channel", + ) + if source_type in ("slack_im", "slack_mpim", "slack_private_channel"): message_conversation_type = source_type.removeprefix("slack_") include_conversations = source_type in (None, SLACK_DM_SOURCE, "slack_dm_conversation") @@ -942,7 +953,7 @@ async def _latest_slack_dms_for_connection( MAX(source_updated_at) AS latest_source_updated_at, MAX(occurred_at) AS latest_occurred_at, COUNT(*)::bigint AS document_count - FROM slack_dm_context_documents + FROM slack_private_context_documents WHERE ($1::text IS NULL OR conversation_type = $1) """, message_conversation_type, @@ -962,7 +973,7 @@ async def _latest_slack_dms_for_connection( MAX(source_updated_at) AS latest_source_updated_at, MAX(last_seen_at) AS latest_occurred_at, COUNT(*)::bigint AS document_count - FROM slack_dm_conversation_context_documents + FROM slack_private_conversation_context_documents """, ) conversations = self._latest_date_result_from_row( @@ -1124,7 +1135,7 @@ async def _search_dm_conversations_async( participant_count, metadata, paradedb.score(document_id) AS score - FROM slack_dm_conversation_context_documents + FROM slack_private_conversation_context_documents WHERE {_search_where_clause(len(terms))} ORDER BY paradedb.score(document_id) DESC, last_seen_at DESC NULLS LAST, @@ -1219,7 +1230,7 @@ async def _search_dms_async( source_updated_at, metadata, paradedb.score(document_id) AS score - FROM slack_dm_context_documents + FROM slack_private_context_documents WHERE {_search_where_clause(len(terms))} AND (${conversation_id_param}::text IS NULL OR conversation_id = ${conversation_id_param}) @@ -1271,7 +1282,7 @@ def search_dms( occurred_after: str | datetime | None = None, occurred_before: str | datetime | None = None, ) -> dict: - """Search Slack DM and group DM context visible to the current Slack user.""" + """Search private Slack context visible to the current Slack user.""" normalized_query = query.strip() if not normalized_query: return {"status": "error", "error": "query cannot be empty"} diff --git a/tools/productivity/company_context/pyproject.toml b/tools/productivity/company_context/pyproject.toml index c55f63c46..4b528cafa 100644 --- a/tools/productivity/company_context/pyproject.toml +++ b/tools/productivity/company_context/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "company_context" -description = "Search indexed company history, user-visible Slack DMs, and Google Docs. Prefer this over direct Slack or Drive search for historical queries. Current sources: Slack, Slack DMs, Google Docs, Google Calendar, Linear." +description = "Search indexed company history, user-visible Slack DMs, Google Docs, and Granola notes. Prefer this over direct Slack, Drive, or Granola search for historical queries. Current sources: Slack, Slack DMs, Google Docs, Google Calendar, Linear, Granola." version = "0.1.0" requires-python = ">=3.11" dependencies = [ diff --git a/tools/productivity/company_context/tests/test_client.py b/tools/productivity/company_context/tests/test_client.py index 4bbc9e30c..388a83659 100644 --- a/tools/productivity/company_context/tests/test_client.py +++ b/tools/productivity/company_context/tests/test_client.py @@ -748,7 +748,7 @@ async def fake_connect(*args, **kwargs): ], } query, args = fake.fetch_calls[0] - assert "FROM slack_dm_conversation_context_documents" in query + assert "FROM slack_private_conversation_context_documents" in query assert "title ||| $1::text::pdb.boost(8) OR body ||| $1::text::pdb.boost(2)" in query assert "OR (title ||| $2::text::pdb.boost(4) OR body ||| $2::text)" in query assert "LIMIT $3" in query @@ -831,7 +831,7 @@ async def fake_connect(*args, **kwargs): ], } query, args = fake.fetch_calls[0] - assert "FROM slack_dm_context_documents" in query + assert "FROM slack_private_context_documents" in query assert "title ||| $1::text::pdb.boost(8) OR body ||| $1::text::pdb.boost(2)" in query assert "OR (title ||| $2::text::pdb.boost(4) OR body ||| $2::text)" in query assert "OR (title ||| $3::text::pdb.boost(4) OR body ||| $3::text)" in query @@ -845,6 +845,22 @@ async def fake_connect(*args, **kwargs): assert fake.closed is True +def test_private_channel_documents_use_private_access_scope(): + summary = company_context_client._dm_document_summary( + { + "document_id": "slack_dm:T_HOME:G123:1770000000.000000", + "conversation_id": "G123", + "conversation_type": "private_channel", + "message_ts": "1770000000.000000", + "title": "Slack private channel: #leadership", + "metadata": {"channel_name": "leadership"}, + } + ) + + assert summary["source_type"] == "slack_private_channel" + assert summary["access_scope"] == "slack_private_channel" + + def test_search_dms_applies_occurred_at_filters(monkeypatch): fake = _FakeConnection(rows=[]) @@ -1071,8 +1087,8 @@ async def fake_connect(*args, **kwargs): "latest_occurred_at": "2026-05-10T10:00:00+00:00", } assert len(fake.fetchrow_calls) == 2 - assert "FROM slack_dm_context_documents" in fake.fetchrow_calls[0][0] - assert "FROM slack_dm_conversation_context_documents" in fake.fetchrow_calls[1][0] + assert "FROM slack_private_context_documents" in fake.fetchrow_calls[0][0] + assert "FROM slack_private_conversation_context_documents" in fake.fetchrow_calls[1][0] assert fake.closed is True @@ -1102,11 +1118,40 @@ async def fake_connect(*args, **kwargs): assert result["latest_date"] == "2026-05-09T15:30:00+00:00" assert len(fake.fetchrow_calls) == 1 query, args = fake.fetchrow_calls[0] - assert "FROM slack_dm_context_documents" in query + assert "FROM slack_private_context_documents" in query assert args == ("im",) assert fake.closed is True +def test_latest_date_can_filter_private_channel_messages(monkeypatch): + fake = _FakeConnection( + fetchrow_rows=[ + { + "latest_date": dt.datetime(2026, 5, 9, 15, 30, tzinfo=dt.UTC), + "latest_source_updated_at": dt.datetime(2026, 5, 9, 15, 30, tzinfo=dt.UTC), + "latest_occurred_at": dt.datetime(2026, 5, 8, 14, 0, tzinfo=dt.UTC), + "document_count": 12, + }, + ] + ) + + async def fake_connect(*args, **kwargs): + return fake + + monkeypatch.setattr(company_context_client.asyncpg, "connect", fake_connect) + + result = CompanyContextClient("postgresql://example").latest_date( + source="slack_dm", + source_type="slack_private_channel", + ) + + assert result["document_count"] == 12 + query, args = fake.fetchrow_calls[0] + assert "FROM slack_private_context_documents" in query + assert args == ("private_channel",) + assert fake.closed is True + + def test_read_document_returns_full_content_by_default(monkeypatch): body = "x" * 2_500 fake = _FakeConnection( diff --git a/tools/productivity/slack/cli.py b/tools/productivity/slack/cli.py index c4f2f3e26..3c9ed9d3b 100644 --- a/tools/productivity/slack/cli.py +++ b/tools/productivity/slack/cli.py @@ -362,7 +362,7 @@ def thread( """ import sys - from .client import get_thread_replies_proxy + from .client import get_thread_replies_page, get_thread_replies_proxy channel_id, thread_ts = _parse_thread_ref(permalink) @@ -376,9 +376,20 @@ def thread( latest=latest, inclusive=inclusive, ) - except (RuntimeError, ValueError) as e: - stderr_console.print(f"[red]Error: {e}[/]") - raise typer.Exit(1) from e + except (RuntimeError, ValueError): + try: + page = get_thread_replies_page( + channel_id, + thread_ts, + limit=limit, + cursor=cursor, + oldest=oldest, + latest=latest, + inclusive=inclusive, + ) + except (RuntimeError, ValueError) as direct_error: + stderr_console.print(f"[red]Error: {direct_error}[/]") + raise typer.Exit(1) from direct_error messages = page.get("messages", []) diff --git a/tools/productivity/slack/client.py b/tools/productivity/slack/client.py index ca0fa0e5e..288f0bc60 100644 --- a/tools/productivity/slack/client.py +++ b/tools/productivity/slack/client.py @@ -749,11 +749,16 @@ def _fetch_channel_history_for_search( limit: int, user_cache: dict[str, str], ) -> list[dict]: - """Fetch history for a single search fallback channel through the proxy.""" + """Fetch history for a single search fallback channel.""" try: response = self.get_channel_history_proxy(channel_id, limit=limit) except (RuntimeError, ValueError): - return [] + return self._fetch_direct_channel_history_for_search( + channel_id, + channel_name, + limit, + user_cache, + ) messages = [] for msg in response.get("messages", []): @@ -768,6 +773,54 @@ def _fetch_channel_history_for_search( return messages + _MAX_SEARCH_DIRECT_THREADS = 10 + + def _fetch_direct_channel_history_for_search( + self, + channel_id: str, + channel_name: str, + limit: int, + user_cache: dict[str, str], + ) -> list[dict]: + """Fetch direct Slack history and expand a bounded number of threads.""" + try: + page = self.get_channel_history_page(channel_id, limit=limit) + except (RuntimeError, ValueError): + return [] + + messages = [{**msg, "channel": channel_name} for msg in page.get("messages", [])] + seen_timestamps = {message.get("timestamp") for message in messages} + expanded_threads = 0 + + for message in list(messages): + if expanded_threads >= self._MAX_SEARCH_DIRECT_THREADS: + break + if int(message.get("reply_count") or 0) <= 0: + continue + + thread_ts = message.get("thread_ts") or message.get("timestamp") + if not thread_ts: + continue + + try: + thread_page = self.get_thread_replies_page( + channel=channel_id, + thread_ts=thread_ts, + limit=min(limit, self._DEFAULT_THREAD_REPLY_LIMIT), + ) + except (RuntimeError, ValueError): + continue + + expanded_threads += 1 + for reply in thread_page.get("messages", []): + timestamp = reply.get("timestamp") + if not timestamp or timestamp in seen_timestamps: + continue + seen_timestamps.add(timestamp) + messages.append({**reply, "channel": channel_name}) + + return messages + _MAX_SEARCH_CHANNELS = 50 # Max channels to search when no filter specified def _rank_channels_for_query(self, channels: list[dict], query_terms: list[str]) -> list[dict]: diff --git a/tools/productivity/slack/tests/test_cli.py b/tools/productivity/slack/tests/test_cli.py index 1a390a4aa..27005b3dc 100644 --- a/tools/productivity/slack/tests/test_cli.py +++ b/tools/productivity/slack/tests/test_cli.py @@ -462,7 +462,13 @@ def fake_get_thread_replies_proxy(*args, **kwargs): "has_more": False, } - fake_client = types.SimpleNamespace(get_thread_replies_proxy=fake_get_thread_replies_proxy) + def fake_get_thread_replies_page(*args, **kwargs): + raise AssertionError("direct thread fallback should not be used") + + fake_client = types.SimpleNamespace( + get_thread_replies_page=fake_get_thread_replies_page, + get_thread_replies_proxy=fake_get_thread_replies_proxy, + ) monkeypatch.setitem(sys.modules, "slack.client", fake_client) result = CliRunner().invoke( @@ -492,6 +498,53 @@ def fake_get_thread_replies_proxy(*args, **kwargs): ] +def test_thread_falls_back_to_direct_client_when_api_server_fails(monkeypatch) -> None: + proxy_calls = [] + direct_calls = [] + + def fake_get_thread_replies_proxy(*args, **kwargs): + proxy_calls.append((args, kwargs)) + raise RuntimeError("proxy unavailable") + + def fake_get_thread_replies_page(*args, **kwargs): + direct_calls.append((args, kwargs)) + return { + "messages": [{"user": "alice", "text": "root"}], + "has_more": False, + "window": {"oldest": None, "latest": None, "inclusive": True}, + } + + fake_client = types.SimpleNamespace( + get_thread_replies_page=fake_get_thread_replies_page, + get_thread_replies_proxy=fake_get_thread_replies_proxy, + ) + monkeypatch.setitem(sys.modules, "slack.client", fake_client) + + result = CliRunner().invoke( + app, + [ + "thread", + "C1234567890:1780000000.000000", + "--limit", + "10", + ], + ) + + expected_call = ( + ("C1234567890", "1780000000.000000"), + { + "limit": 10, + "cursor": None, + "oldest": None, + "latest": None, + "inclusive": True, + }, + ) + assert result.exit_code == 0 + assert proxy_calls == [expected_call] + assert direct_calls == [expected_call] + + def test_thread_direct_calls_direct_client(monkeypatch) -> None: calls = [] diff --git a/tools/productivity/slack/tests/test_client.py b/tools/productivity/slack/tests/test_client.py index 8606bd9a0..83ce0062b 100644 --- a/tools/productivity/slack/tests/test_client.py +++ b/tools/productivity/slack/tests/test_client.py @@ -1051,6 +1051,69 @@ def history_proxy(channel_id: str, **kwargs): assert results[0]["user_id"] == "UGZCSQTPE" +def test_search_messages_falls_back_to_direct_history_and_threads_when_proxy_fails() -> None: + client, fake_web_client = _make_client() + client._get_user_cache = lambda: {"U1": "alice", "U2": "bob"} # type: ignore[method-assign] + client._resolve_channel = lambda channel: channel # type: ignore[method-assign] + + def fail_proxy(*args, **kwargs): + raise RuntimeError("proxy unavailable") + + client.get_channel_history_proxy = fail_proxy # type: ignore[method-assign] + fake_web_client.history_pages = [ + { + "messages": [ + { + "user": "U1", + "text": "root without the query", + "ts": "100.000000", + "thread_ts": "100.000000", + "reply_count": 1, + } + ], + "response_metadata": {"next_cursor": ""}, + } + ] + fake_web_client.reply_pages = [ + { + "messages": [ + { + "user": "U1", + "text": "root without the query", + "ts": "100.000000", + "thread_ts": "100.000000", + }, + { + "user": "U2", + "text": "needle is in the direct thread reply", + "ts": "100.000001", + "thread_ts": "100.000000", + }, + ], + "response_metadata": {"next_cursor": ""}, + } + ] + + results = client.search_messages( + "needle", + channels=["C123456789"], + messages_per_channel=25, + ) + + assert fake_web_client.history_calls == [{"channel": "C123456789", "limit": 25}] + assert fake_web_client.reply_calls == [ + { + "channel": "C123456789", + "ts": "100.000000", + "limit": 25, + "inclusive": True, + } + ] + assert len(results) == 1 + assert results[0]["text"] == "needle is in the direct thread reply" + assert results[0]["channel"] == "C123456789" + + def test_list_channels_returns_cache_when_slack_rate_limited() -> None: client, fake_web_client = _make_client() cached_channels = [{"id": "C123", "name": "cached", "is_private": False}] diff --git a/workflows/company_context_documents.py b/workflows/company_context_documents.py index c9c785eae..cff544c72 100644 --- a/workflows/company_context_documents.py +++ b/workflows/company_context_documents.py @@ -11,7 +11,6 @@ from api.runtime_control import canonical_json, decode_jsonb from workflows.company_context_metrics import ( - observe_company_context_document_size, record_company_context_documents_changed, set_company_context_projection_lag, ) @@ -27,6 +26,8 @@ DEFAULT_SYNC_INTERVAL_SECONDS = 4 * 60 * 60 DEFAULT_WATERMARK_OVERLAP_SECONDS = 60 DEFAULT_MAX_WINDOW_SECONDS = 6 * 60 * 60 +DEFAULT_BATCH_SIZE = 50 +DEFAULT_SCOPE_LEASE_SECONDS = 20 * 60 MIN_THREAD_MESSAGES = 5 FALSE_ENV_VALUES = {"0", "false", "no", "off"} SLACK_MENTION_RE = re.compile(r"<@([A-Z0-9]+)>") @@ -100,6 +101,9 @@ class Input: since: str | None = None watermark_overlap_seconds: int = DEFAULT_WATERMARK_OVERLAP_SECONDS max_window_seconds: int | None = None + scope: str | None = None + lease_token: str | None = None + batch_size: int | None = None metadata: dict[str, Any] = field(default_factory=dict) @@ -267,39 +271,6 @@ def _emit_company_context_counter_baselines(enabled_sources: list[str]) -> None: ) -async def _emit_company_context_document_size_snapshot( - pool, - enabled_sources: list[str], -) -> None: - """Observe current projected document sizes so the corpus p95 panel has data.""" - if not enabled_sources: - return - rows = await pool.fetch( - "SELECT source, source_type, LENGTH(COALESCE(body, '')) AS body_chars " - "FROM company_context_documents " - "WHERE source = ANY($1::text[]) " - "ORDER BY source, source_type, document_id", - enabled_sources, - ) - seen_source_types: set[tuple[str, str]] = set() - for row in rows: - source = str(row["source"] or "") - source_type = str(row["source_type"] or "") - if not source or not source_type: - continue - seen_source_types.add((source, source_type)) - observe_company_context_document_size( - source, - source_type, - int(row["body_chars"] or 0), - ) - - for source in enabled_sources: - for source_type in COMPANY_CONTEXT_SOURCE_TYPES.get(source, ()): - if (source, source_type) not in seen_source_types: - observe_company_context_document_size(source, source_type, 0) - - async def _emit_etl_scope_metrics(pool, enabled_sources: list[str]) -> None: """Publish source scope health gauges used by the Grafana overview row.""" for source in enabled_sources: @@ -343,6 +314,31 @@ def _emit_company_context_projection_lag( set_company_context_projection_lag(source, lag_seconds) +async def _emit_projection_lag_from_checkpoints( + pool, + enabled_scopes: dict[str, str], +) -> None: + """Publish the oldest completed scope watermark as each source's lag.""" + rows = await pool.fetch( + "SELECT scope, watermark FROM company_context_projection_checkpoints " + "WHERE scope = ANY($1::text[])", + list(enabled_scopes), + ) + source_watermarks: dict[str, dt.datetime | None] = {} + for row in rows: + source = enabled_scopes.get(str(row["scope"])) + watermark = row["watermark"] + if source is None or not isinstance(watermark, dt.datetime): + continue + watermark = watermark.astimezone(dt.timezone.utc) + current = source_watermarks.get(source) + if current is None or watermark < current: + source_watermarks[source] = watermark + _emit_company_context_projection_lag( + sorted(set(enabled_scopes.values())), source_watermarks + ) + + async def _load_slack_lookup_maps(pool) -> tuple[dict[str, str], dict[str, str]]: """Load Slack user/channel name maps for document rendering.""" user_rows = await pool.fetch( @@ -1470,333 +1466,576 @@ async def _delete_document(pool, document_id: str) -> bool: return status.endswith(" 1") -async def handler(inp: Input, ctx: WorkflowContext) -> dict[str, Any]: - """Project changed sync rows into embeddable company context documents.""" - if not ( - _source_enabled() - and _env_flag_enabled("COMPANY_CONTEXT_DOCUMENTS_ENABLED", default=True) - ): - ctx.log("company_context_documents_skipped_disabled") - return {"status": "skipped", "reason": "company_context_documents_disabled"} +def _batch_size(value: int | str | None = None) -> int: + """Return a bounded source-row page size for one projection workflow.""" + configured = ( + value + if value is not None + else os.getenv("COMPANY_CONTEXT_DOCUMENTS_BATCH_SIZE") + ) + return min(_positive_int(configured, DEFAULT_BATCH_SIZE), 250) + + +def _enabled_scopes() -> dict[str, str]: + """Return the durable projection scopes enabled for this deployment.""" + scopes: dict[str, str] = {} + if _env_flag_enabled("SLACK_ETL_ENABLED"): + scopes.update( + { + "slack_channel_day": "slack", + "slack_thread": "slack", + "slack_attachment": "slack", + } + ) + if _env_flag_enabled("GOOGLE_DRIVE_ETL_ENABLED"): + scopes["google_doc"] = "google_drive" + if _env_flag_enabled("GOOGLE_CALENDAR_ETL_ENABLED"): + scopes["calendar_event"] = "google_calendar" + if _env_flag_enabled("LINEAR_ETL_ENABLED"): + # Comments have their own cursor because updating a comment does not update + # the parent issue row's synced timestamp. + scopes.update({"linear_issue": "linear", "linear_comment": "linear"}) + if _env_flag_enabled("ATTIO_ETL_ENABLED"): + scopes["attio_meeting"] = "attio" + return scopes + + +def _page_where( + column: str, + key_expression: str, + *, + window_start: dt.datetime | None, + window_end: dt.datetime, + cursor_updated_at: dt.datetime | None, + cursor_key: str, + base: tuple[str, ...] = (), +) -> tuple[str, list[Any]]: + """Build a stable `(updated_at, key)` keyset page predicate.""" + clauses = list(base) + args: list[Any] = [] + if window_start is not None: + args.append(window_start) + clauses.append(f"{column} > ${len(args)}") + args.append(window_end) + clauses.append(f"{column} <= ${len(args)}") + if cursor_updated_at is not None: + args.extend((cursor_updated_at, cursor_key)) + clauses.append( + f"({column} > ${len(args) - 1} OR " + f"({column} = ${len(args) - 1} AND {key_expression} > ${len(args)}))" + ) + return " AND ".join(clauses), args - explicit_since = _parse_datetime(inp.since) - last_watermark = explicit_since or await _latest_successful_watermark( - ctx._pool, ctx.run_id + +async def _fetch_page( + pool, + *, + table: str, + column: str, + key_expression: str, + key_alias: str, + window_start: dt.datetime | None, + window_end: dt.datetime, + cursor_updated_at: dt.datetime | None, + cursor_key: str, + batch_size: int, + base: tuple[str, ...] = (), +) -> list[Any]: + """Fetch one keyset page, always including its durable cursor columns.""" + where_sql, args = _page_where( + column, + key_expression, + window_start=window_start, + window_end=window_end, + cursor_updated_at=cursor_updated_at, + cursor_key=cursor_key, + base=base, ) - overlap_seconds = _nonnegative_int( - inp.watermark_overlap_seconds, - DEFAULT_WATERMARK_OVERLAP_SECONDS, + args.append(batch_size) + return list( + await pool.fetch( + f"SELECT *, {column} AS projection_updated_at, {key_expression} AS {key_alias} " + f"FROM {table} WHERE {where_sql} " + f"ORDER BY {column}, {key_expression} LIMIT ${len(args)}", + *args, + ) ) - since = ( - last_watermark - dt.timedelta(seconds=overlap_seconds) - if last_watermark is not None - else None + + +def _cursor_from_page(rows: list[Any]) -> tuple[dt.datetime | None, str]: + """Extract the next durable cursor from a source-row page.""" + if not rows: + return None, "" + row = rows[-1] + updated_at = row["projection_updated_at"] + return ( + updated_at.astimezone(dt.timezone.utc) + if isinstance(updated_at, dt.datetime) + else None, + str(row["projection_key"] or ""), ) - now = dt.datetime.now(dt.timezone.utc) - max_window_seconds = _max_window_seconds(inp.max_window_seconds) - batch_until = _batch_until(since, now, max_window_seconds) - - slack_enabled = _env_flag_enabled("SLACK_ETL_ENABLED") - google_drive_enabled = _env_flag_enabled("GOOGLE_DRIVE_ETL_ENABLED") - google_calendar_enabled = _env_flag_enabled("GOOGLE_CALENDAR_ETL_ENABLED") - linear_enabled = _env_flag_enabled("LINEAR_ETL_ENABLED") - attio_enabled = _env_flag_enabled("ATTIO_ETL_ENABLED") - enabled_sources = [ - source - for source, enabled in ( - ("slack", slack_enabled), - ("google_drive", google_drive_enabled), - ("google_calendar", google_calendar_enabled), - ("linear", linear_enabled), - ("attio", attio_enabled), + + +async def _load_scope_page( + pool, + scope: str, + *, + window_start: dt.datetime | None, + window_end: dt.datetime, + cursor_updated_at: dt.datetime | None, + cursor_key: str, + batch_size: int, +) -> list[Any]: + """Load one bounded source-row page for a projection scope.""" + if scope == "slack_channel_day": + return await _fetch_page( + pool, + table="slack_sync_messages", + column="updated_at", + key_expression="channel_id || ':' || message_ts", + key_alias="projection_key", + window_start=window_start, + window_end=window_end, + cursor_updated_at=cursor_updated_at, + cursor_key=cursor_key, + batch_size=batch_size, + base=("occurred_at IS NOT NULL",), ) - if enabled - ] - _emit_company_context_counter_baselines(enabled_sources) - changed = { - "channel_days": [], - "threads": [], - "attachments": [], - "changed_messages": 0, - "changed_attachments": 0, - "max_updated_at": None, - } - users_by_id: dict[str, str] = {} - channels_by_id: dict[str, str] = {} - if slack_enabled: - users_by_id, channels_by_id = await _load_slack_lookup_maps(ctx._pool) - changed = await _load_changed_message_keys(ctx._pool, since, batch_until) - drive_changed = { - "files": [], - "changed_files": 0, - "max_updated_at": None, - } - if google_drive_enabled: - drive_changed = await _load_changed_drive_files(ctx._pool, since, batch_until) - calendar_changed = { - "events": [], - "changed_events": 0, - "max_updated_at": None, - } - if google_calendar_enabled: - calendar_changed = await _load_changed_calendar_events( - ctx._pool, since, batch_until + if scope == "slack_thread": + return await _fetch_page( + pool, + table="slack_sync_messages", + column="updated_at", + key_expression="channel_id || ':' || message_ts", + key_alias="projection_key", + window_start=window_start, + window_end=window_end, + cursor_updated_at=cursor_updated_at, + cursor_key=cursor_key, + batch_size=batch_size, + base=("thread_ts IS NOT NULL", "thread_ts <> ''"), ) - linear_changed = { - "issues": [], - "changed_issues": 0, - "max_updated_at": None, - } - if linear_enabled: - linear_changed = await _load_changed_linear_issues( - ctx._pool, since, batch_until + if scope == "slack_attachment": + return await _fetch_page( + pool, + table="slack_sync_message_attachments", + column="updated_at", + key_expression="channel_id || ':' || message_ts || ':' || slack_file_id", + key_alias="projection_key", + window_start=window_start, + window_end=window_end, + cursor_updated_at=cursor_updated_at, + cursor_key=cursor_key, + batch_size=batch_size, ) - attio_changed = { - "meetings": [], - "changed_meetings": 0, - "max_updated_at": None, - } - if attio_enabled: - attio_changed = await _load_changed_attio_meetings( - ctx._pool, since, batch_until + if scope == "google_doc": + return await _fetch_page( + pool, + table="google_drive_sync_files", + column="updated_at", + key_expression="file_id", + key_alias="projection_key", + window_start=window_start, + window_end=window_end, + cursor_updated_at=cursor_updated_at, + cursor_key=cursor_key, + batch_size=batch_size, + base=("last_error = ''", "trashed = FALSE"), ) - - documents_upserted = 0 - documents_deleted = 0 - for channel_id, day in changed["channel_days"]: - messages = await _load_channel_day_messages(ctx._pool, channel_id, day) - document = _channel_day_document( - channel_id=channel_id, - day=day, - messages=messages, - users_by_id=users_by_id, - channels_by_id=channels_by_id, + if scope == "calendar_event": + return await _fetch_page( + pool, + table="google_calendar_sync_events", + column="updated_at", + key_expression="calendar_id || ':' || event_id", + key_alias="projection_key", + window_start=window_start, + window_end=window_end, + cursor_updated_at=cursor_updated_at, + cursor_key=cursor_key, + batch_size=batch_size, + base=("last_error = ''",), ) - if document is None: - if await _delete_document( - ctx._pool, - f"slack:channel_day:{channel_id}:{day.isoformat()}", - ): - documents_deleted += 1 - record_company_context_documents_changed( - "slack", - "slack_channel_day", - "deleted", - ) - continue - observe_company_context_document_size( - "slack", - str(document["source_type"]), - len(str(document["body"] or "")), + if scope == "linear_issue": + return await _fetch_page( + pool, + table="linear_sync_issues", + column="updated_at", + key_expression="issue_id", + key_alias="projection_key", + window_start=window_start, + window_end=window_end, + cursor_updated_at=cursor_updated_at, + cursor_key=cursor_key, + batch_size=batch_size, + base=("last_error = ''",), ) - action = await _upsert_document(ctx._pool, document) - record_company_context_documents_changed( - "slack", - str(document["source_type"]), - action, + if scope == "linear_comment": + return await _fetch_page( + pool, + table="linear_sync_comments", + column="updated_at", + key_expression="comment_id", + key_alias="projection_key", + window_start=window_start, + window_end=window_end, + cursor_updated_at=cursor_updated_at, + cursor_key=cursor_key, + batch_size=batch_size, + base=("last_error = ''",), ) - if action in {"inserted", "updated"}: - documents_upserted += 1 - - for channel_id, thread_ts in changed["threads"]: - messages = await _load_thread_messages(ctx._pool, channel_id, thread_ts) - document = _thread_document( - channel_id=channel_id, - thread_ts=thread_ts, - messages=messages, - users_by_id=users_by_id, - channels_by_id=channels_by_id, + if scope == "attio_meeting": + return await _fetch_page( + pool, + table="attio_sync_meetings", + column="updated_at", + key_expression="meeting_id", + key_alias="projection_key", + window_start=window_start, + window_end=window_end, + cursor_updated_at=cursor_updated_at, + cursor_key=cursor_key, + batch_size=batch_size, + base=("last_error = ''",), ) + raise ValueError(f"unknown company context projection scope: {scope}") + + +async def _project_scope_page( + pool, + scope: str, + rows: list[Any], +) -> tuple[int, int]: + """Project one source-row page and return inserted/updated and deleted counts.""" + upserted = 0 + deleted = 0 + users_by_id: dict[str, str] = {} + channels_by_id: dict[str, str] = {} + if scope.startswith("slack_"): + users_by_id, channels_by_id = await _load_slack_lookup_maps(pool) + + async def save( + document: dict[str, Any] | None, source: str, source_type: str + ) -> None: + nonlocal upserted if document is None: - if await _delete_document( - ctx._pool, f"slack:thread:{channel_id}:{thread_ts}" - ): - documents_deleted += 1 - record_company_context_documents_changed( + return + action = await _upsert_document(pool, document) + record_company_context_documents_changed(source, source_type, action) + if action in {"inserted", "updated"}: + upserted += 1 + + if scope == "slack_channel_day": + keys = {(str(row["channel_id"]), row["occurred_at"].date()) for row in rows} + for channel_id, day in keys: + document = _channel_day_document( + channel_id=channel_id, + day=day, + messages=await _load_channel_day_messages(pool, channel_id, day), + users_by_id=users_by_id, + channels_by_id=channels_by_id, + ) + if document is None: + if await _delete_document( + pool, f"slack:channel_day:{channel_id}:{day.isoformat()}" + ): + deleted += 1 + record_company_context_documents_changed( + "slack", "slack_channel_day", "deleted" + ) + else: + await save(document, "slack", "slack_channel_day") + elif scope == "slack_thread": + keys = {(str(row["channel_id"]), str(row["thread_ts"])) for row in rows} + for channel_id, thread_ts in keys: + document = _thread_document( + channel_id=channel_id, + thread_ts=thread_ts, + messages=await _load_thread_messages(pool, channel_id, thread_ts), + users_by_id=users_by_id, + channels_by_id=channels_by_id, + ) + if document is None: + if await _delete_document( + pool, f"slack:thread:{channel_id}:{thread_ts}" + ): + deleted += 1 + record_company_context_documents_changed( + "slack", "slack_thread", "deleted" + ) + else: + await save(document, "slack", "slack_thread") + elif scope == "slack_attachment": + for row in rows: + attachment = await pool.fetchrow( + "SELECT a.*, c.channel_name, m.occurred_at, m.thread_ts, m.parent_message_ts, " + "m.user_id, u.user_name, u.real_name, u.display_name, m.text, " + "m.permalink AS message_permalink " + "FROM slack_sync_message_attachments a " + "JOIN slack_sync_messages m ON m.channel_id = a.channel_id AND m.message_ts = a.message_ts " + "LEFT JOIN slack_sync_channels c ON c.channel_id = a.channel_id " + "LEFT JOIN slack_sync_users u ON u.user_id = m.user_id " + "WHERE a.channel_id = $1 AND a.message_ts = $2 AND a.slack_file_id = $3", + row["channel_id"], + row["message_ts"], + row["slack_file_id"], + ) + if attachment: + await save( + _slack_attachment_document( + attachment, + users_by_id=users_by_id, + channels_by_id=channels_by_id, + ), "slack", - "slack_thread", - "deleted", + "slack_attachment", ) - continue - observe_company_context_document_size( - "slack", - str(document["source_type"]), - len(str(document["body"] or "")), - ) - action = await _upsert_document(ctx._pool, document) - record_company_context_documents_changed( - "slack", - str(document["source_type"]), - action, - ) - if action in {"inserted", "updated"}: - documents_upserted += 1 + elif scope == "google_doc": + for row in rows: + await save(_drive_document(row), "google_drive", "google_doc") + elif scope == "calendar_event": + for row in rows: + event = await pool.fetchrow( + "SELECT e.*, c.summary AS calendar_summary, c.time_zone " + "FROM google_calendar_sync_events e " + "LEFT JOIN google_calendar_sync_calendars c ON c.calendar_id = e.calendar_id " + "WHERE e.calendar_id = $1 AND e.event_id = $2", + row["calendar_id"], + row["event_id"], + ) + if event is None: + continue + if str(event["status"] or "") == "cancelled": + if await _delete_document(pool, _calendar_event_document_id(event)): + deleted += 1 + record_company_context_documents_changed( + "google_calendar", "calendar_event", "deleted" + ) + else: + await save( + _calendar_event_document(event), "google_calendar", "calendar_event" + ) + elif scope in {"linear_issue", "linear_comment"}: + issue_ids = {str(row["issue_id"]) for row in rows} + for issue_id in issue_ids: + issue = await pool.fetchrow( + "SELECT i.*, (" + " SELECT MAX(COALESCE(c.source_updated_at, c.source_edited_at, c.updated_at)) " + " FROM linear_sync_comments c WHERE c.issue_id = i.issue_id AND c.last_error = ''" + ") AS comments_source_updated_at " + "FROM linear_sync_issues i WHERE i.issue_id = $1 AND i.last_error = ''", + issue_id, + ) + if issue: + await save( + _linear_issue_document( + issue, await _load_linear_issue_comments(pool, issue_id) + ), + "linear", + "linear_issue", + ) + elif scope == "attio_meeting": + for row in rows: + await save(_attio_meeting_document(row), "attio", "attio_meeting") + return upserted, deleted - for row in changed["attachments"]: - document = _slack_attachment_document( - row, - users_by_id=users_by_id, - channels_by_id=channels_by_id, - ) - if document is None: - continue - observe_company_context_document_size( - "slack", - str(document["source_type"]), - len(str(document["body"] or "")), - ) - action = await _upsert_document(ctx._pool, document) - record_company_context_documents_changed( - "slack", - str(document["source_type"]), - action, - ) - if action in {"inserted", "updated"}: - documents_upserted += 1 - for row in drive_changed["files"]: - document = _drive_document(row) - if document is None: - continue - observe_company_context_document_size( - "google_drive", - str(document["source_type"]), - len(str(document["body"] or "")), - ) - action = await _upsert_document(ctx._pool, document) - record_company_context_documents_changed( - "google_drive", - str(document["source_type"]), - action, - ) - if action in {"inserted", "updated"}: - documents_upserted += 1 +async def _claim_scope( + pool, + *, + scope: str, + seed_watermark: dt.datetime | None, + overlap_seconds: int, + max_window_seconds: int, +) -> Any | None: + """Start or reclaim one scope window; only one child chain owns it at a time.""" + now = dt.datetime.now(dt.timezone.utc) + initial_start = ( + seed_watermark - dt.timedelta(seconds=overlap_seconds) + if seed_watermark is not None + else None + ) + initial_end = _batch_until(initial_start, now, max_window_seconds) or now + token = hashlib.sha256(f"{scope}:{now.isoformat()}".encode()).hexdigest() + return await pool.fetchrow( + "INSERT INTO company_context_projection_checkpoints " + "(scope, watermark, window_start, window_end, cursor_updated_at, cursor_key, lease_token, lease_expires_at) " + "VALUES ($1, $2, $3, $4, NULL, '', $5, NOW() + ($6::text || ' seconds')::interval) " + "ON CONFLICT (scope) DO UPDATE SET " + "watermark = CASE WHEN company_context_projection_checkpoints.window_end IS NULL " + " THEN COALESCE(company_context_projection_checkpoints.watermark, EXCLUDED.watermark) " + " ELSE company_context_projection_checkpoints.watermark END, " + "window_start = CASE WHEN company_context_projection_checkpoints.window_end IS NULL " + " THEN COALESCE(company_context_projection_checkpoints.watermark - ($7::text || ' seconds')::interval, EXCLUDED.window_start) " + " ELSE company_context_projection_checkpoints.window_start END, " + "window_end = CASE WHEN company_context_projection_checkpoints.window_end IS NULL " + " THEN LEAST(NOW(), COALESCE(company_context_projection_checkpoints.watermark - ($7::text || ' seconds')::interval, EXCLUDED.window_start) + ($8::text || ' seconds')::interval) " + " ELSE company_context_projection_checkpoints.window_end END, " + "cursor_updated_at = CASE WHEN company_context_projection_checkpoints.window_end IS NULL THEN NULL ELSE company_context_projection_checkpoints.cursor_updated_at END, " + "cursor_key = CASE WHEN company_context_projection_checkpoints.window_end IS NULL THEN '' ELSE company_context_projection_checkpoints.cursor_key END, " + "lease_token = EXCLUDED.lease_token, lease_expires_at = EXCLUDED.lease_expires_at, updated_at = NOW() " + "WHERE company_context_projection_checkpoints.lease_expires_at IS NULL " + " OR company_context_projection_checkpoints.lease_expires_at < NOW() " + "RETURNING scope, lease_token, window_start, window_end", + scope, + seed_watermark, + initial_start, + initial_end, + token, + str(DEFAULT_SCOPE_LEASE_SECONDS), + str(overlap_seconds), + str(max_window_seconds), + ) - for row in calendar_changed["events"]: - if str(row["status"] or "") == "cancelled": - if await _delete_document(ctx._pool, _calendar_event_document_id(row)): - documents_deleted += 1 - record_company_context_documents_changed( - "google_calendar", - "calendar_event", - "deleted", - ) - continue - document = _calendar_event_document(row) - if document is None: - continue - observe_company_context_document_size( - "google_calendar", - str(document["source_type"]), - len(str(document["body"] or "")), - ) - action = await _upsert_document(ctx._pool, document) - record_company_context_documents_changed( - "google_calendar", - str(document["source_type"]), - action, - ) - if action in {"inserted", "updated"}: - documents_upserted += 1 - for row in linear_changed["issues"]: - comments = await _load_linear_issue_comments(ctx._pool, str(row["issue_id"])) - document = _linear_issue_document(row, comments) - if document is None: - continue - observe_company_context_document_size( - "linear", - str(document["source_type"]), - len(str(document["body"] or "")), +async def _read_owned_scope(pool, scope: str, lease_token: str) -> Any | None: + return await pool.fetchrow( + "SELECT scope, watermark, window_start, window_end, cursor_updated_at, cursor_key " + "FROM company_context_projection_checkpoints " + "WHERE scope = $1 AND lease_token = $2 AND lease_expires_at > NOW()", + scope, + lease_token, + ) + + +async def _finish_scope_window(pool, scope: str, lease_token: str) -> None: + await pool.execute( + "UPDATE company_context_projection_checkpoints SET watermark = window_end, " + "window_start = NULL, window_end = NULL, cursor_updated_at = NULL, cursor_key = '', " + "lease_token = NULL, lease_expires_at = NULL, updated_at = NOW() " + "WHERE scope = $1 AND lease_token = $2", + scope, + lease_token, + ) + + +async def _advance_scope_cursor( + pool, + scope: str, + lease_token: str, + cursor_updated_at: dt.datetime, + cursor_key: str, +) -> None: + await pool.execute( + "UPDATE company_context_projection_checkpoints SET cursor_updated_at = $3, cursor_key = $4, " + "lease_expires_at = NOW() + ($5::text || ' seconds')::interval, updated_at = NOW() " + "WHERE scope = $1 AND lease_token = $2", + scope, + lease_token, + cursor_updated_at, + cursor_key, + str(DEFAULT_SCOPE_LEASE_SECONDS), + ) + + +async def _run_scope_batch( + inp: Input, ctx: WorkflowContext, scope: str +) -> dict[str, Any]: + lease_token = str(inp.lease_token or "") + checkpoint = await _read_owned_scope(ctx._pool, scope, lease_token) + if checkpoint is None: + return {"status": "skipped", "scope": scope, "reason": "lease_not_owned"} + window_end = checkpoint["window_end"] + if not isinstance(window_end, dt.datetime): + return {"status": "skipped", "scope": scope, "reason": "no_active_window"} + rows = await _load_scope_page( + ctx._pool, + scope, + window_start=checkpoint["window_start"], + window_end=window_end, + cursor_updated_at=checkpoint["cursor_updated_at"], + cursor_key=str(checkpoint["cursor_key"] or ""), + batch_size=_batch_size(inp.batch_size), + ) + upserted, deleted = await _project_scope_page(ctx._pool, scope, rows) + if len(rows) < _batch_size(inp.batch_size): + await _finish_scope_window(ctx._pool, scope, lease_token) + continuation = None + else: + cursor_updated_at, cursor_key = _cursor_from_page(rows) + if cursor_updated_at is None or not cursor_key: + raise RuntimeError(f"missing durable cursor for {scope} projection page") + await _advance_scope_cursor( + ctx._pool, scope, lease_token, cursor_updated_at, cursor_key ) - action = await _upsert_document(ctx._pool, document) - record_company_context_documents_changed( - "linear", - str(document["source_type"]), - action, + continuation = await ctx.start_workflow( + WORKFLOW_NAME, + { + "scope": scope, + "lease_token": lease_token, + "batch_size": _batch_size(inp.batch_size), + }, + idempotency_key=f"company-context:{scope}:{window_end.isoformat()}:{cursor_updated_at.isoformat()}:{cursor_key}", ) - if action in {"inserted", "updated"}: - documents_upserted += 1 + return { + "status": "completed", + "scope": scope, + "source_rows": len(rows), + "documents_upserted": upserted, + "documents_deleted": deleted, + "window_end": window_end.isoformat(), + "continuation": continuation, + } - for row in attio_changed["meetings"]: - document = _attio_meeting_document(row) - if document is None: - continue - observe_company_context_document_size( - "attio", - str(document["source_type"]), - len(str(document["body"] or "")), + +async def handler(inp: Input, ctx: WorkflowContext) -> dict[str, Any]: + """Fan out bounded, durable projection pages for each enabled source scope.""" + if not ( + _source_enabled() + and _env_flag_enabled("COMPANY_CONTEXT_DOCUMENTS_ENABLED", default=True) + ): + ctx.log("company_context_documents_skipped_disabled") + return {"status": "skipped", "reason": "company_context_documents_disabled"} + + enabled_scopes = _enabled_scopes() + if inp.scope: + if inp.scope not in enabled_scopes: + return {"status": "skipped", "scope": inp.scope, "reason": "scope_disabled"} + result = await _run_scope_batch(inp, ctx, inp.scope) + ctx.log("company_context_documents_scope_completed", **result) + return result + + overlap_seconds = _nonnegative_int( + inp.watermark_overlap_seconds, + DEFAULT_WATERMARK_OVERLAP_SECONDS, + ) + seed_watermark = _parse_datetime(inp.since) or await _latest_successful_watermark( + ctx._pool, ctx.run_id + ) + started: list[dict[str, Any]] = [] + for scope in enabled_scopes: + checkpoint = await _claim_scope( + ctx._pool, + scope=scope, + seed_watermark=seed_watermark, + overlap_seconds=overlap_seconds, + max_window_seconds=_max_window_seconds(inp.max_window_seconds), ) - action = await _upsert_document(ctx._pool, document) - record_company_context_documents_changed( - "attio", - str(document["source_type"]), - action, + if checkpoint is None: + continue + child = await ctx.start_workflow( + WORKFLOW_NAME, + { + "scope": str(checkpoint["scope"]), + "lease_token": str(checkpoint["lease_token"]), + "batch_size": _batch_size(inp.batch_size), + }, + idempotency_key=( + f"company-context:{checkpoint['scope']}:{checkpoint['window_end'].isoformat()}:" + f"{checkpoint['lease_token']}" + ), ) - if action in {"inserted", "updated"}: - documents_upserted += 1 + started.append({"scope": str(checkpoint["scope"]), "child": child}) - if batch_until is not None: - watermark = batch_until - else: - watermark_candidates = [ - value - for value in ( - changed["max_updated_at"], - drive_changed["max_updated_at"], - calendar_changed["max_updated_at"], - linear_changed["max_updated_at"], - attio_changed["max_updated_at"], - last_watermark, - ) - if value is not None - ] - watermark = max(watermark_candidates) if watermark_candidates else None - source_watermarks = { - "slack": watermark - if batch_until is not None - else changed["max_updated_at"] or last_watermark, - "google_drive": watermark - if batch_until is not None - else drive_changed["max_updated_at"] or last_watermark, - "google_calendar": watermark - if batch_until is not None - else calendar_changed["max_updated_at"] or last_watermark, - "linear": watermark - if batch_until is not None - else linear_changed["max_updated_at"] or last_watermark, - "attio": watermark - if batch_until is not None - else attio_changed["max_updated_at"] or last_watermark, - } - remaining_lag_seconds = ( - max((now - watermark).total_seconds(), 0.0) if watermark is not None else None - ) - _emit_company_context_projection_lag(enabled_sources, source_watermarks) + enabled_sources = sorted(set(enabled_scopes.values())) + _emit_company_context_counter_baselines(enabled_sources) + await _emit_projection_lag_from_checkpoints(ctx._pool, enabled_scopes) await _emit_etl_scope_metrics(ctx._pool, enabled_sources) - await _emit_company_context_document_size_snapshot(ctx._pool, enabled_sources) result = { "status": "completed", - "changed_messages": changed["changed_messages"], - "changed_slack_attachments": changed["changed_attachments"], - "changed_drive_files": drive_changed["changed_files"], - "changed_calendar_events": calendar_changed["changed_events"], - "changed_linear_issues": linear_changed["changed_issues"], - "changed_attio_meetings": attio_changed["changed_meetings"], - "channel_day_documents": len(changed["channel_days"]), - "thread_candidates": len(changed["threads"]), - "slack_attachment_documents": len(changed["attachments"]), - "drive_documents": len(drive_changed["files"]), - "calendar_event_documents": len(calendar_changed["events"]), - "linear_issue_documents": len(linear_changed["issues"]), - "attio_meeting_documents": len(attio_changed["meetings"]), - "documents_upserted": documents_upserted, - "documents_deleted": documents_deleted, - "since": since.isoformat() if since else None, - "batch_until": batch_until.isoformat() if batch_until else None, - "max_window_seconds": max_window_seconds, - "remaining_lag_seconds": remaining_lag_seconds, - "watermark": watermark.isoformat() if watermark else None, + "started_scopes": started, + "batch_size": _batch_size(inp.batch_size), + "max_window_seconds": _max_window_seconds(inp.max_window_seconds), } - ctx.log("company_context_documents_completed", **result) + ctx.log("company_context_documents_coordinator_completed", **result) return result diff --git a/workflows/company_context_metrics.py b/workflows/company_context_metrics.py index 3a0174ccd..b50e55d8d 100644 --- a/workflows/company_context_metrics.py +++ b/workflows/company_context_metrics.py @@ -1,20 +1,6 @@ from __future__ import annotations -from api.metrics import increment_metric, observe_histogram, set_gauge - - -_COMPANY_CONTEXT_DOCUMENT_SIZE_BUCKETS = [ - 100, - 500, - 1_000, - 5_000, - 10_000, - 25_000, - 50_000, - 100_000, - 250_000, - 500_000, -] +from api.metrics import increment_metric, set_gauge def record_company_context_documents_changed( @@ -32,16 +18,6 @@ def record_company_context_documents_changed( ) -def observe_company_context_document_size(source: str, source_type: str, chars: int) -> None: - observe_histogram( - "company_context_document_size_chars", - max(chars, 0), - _COMPANY_CONTEXT_DOCUMENT_SIZE_BUCKETS, - source=source, - source_type=source_type, - ) - - def set_company_context_projection_lag(source: str, projection_lag_s: float) -> None: set_gauge( "company_context_projection_lag_seconds", diff --git a/workflows/granola_sync.py b/workflows/granola_sync.py deleted file mode 100644 index 0fca41c17..000000000 --- a/workflows/granola_sync.py +++ /dev/null @@ -1,757 +0,0 @@ -"""Workflow: sync Granola notes and transcripts into Postgres.""" - -from __future__ import annotations - -import datetime as dt -import hashlib -import os -from dataclasses import dataclass, field -from typing import Any, Protocol - -from api.runtime_control import canonical_json -from workflows.etl_metrics import ( - record_etl_items_failed, - record_etl_items_seen, - record_etl_items_upserted, - set_etl_active_scopes, - set_etl_failed_scopes, - set_etl_scope_sync_freshness_seconds, -) -from api.workflow_engine import WorkflowContext -from workflows.slack.shared import env_flag_enabled, positive_int - -WORKFLOW_NAME = "granola_sync" -DEFAULT_SYNC_INTERVAL_SECONDS = 4 * 60 * 60 -DEFAULT_PAGE_SIZE = 30 -DEFAULT_WATERMARK_OVERLAP_SECONDS = 5 * 60 -WORKSPACE_SCOPE = "workspace" - - -SCHEDULE = { - "schedule_id": "granola_sync", - "interval_seconds": positive_int( - os.getenv("GRANOLA_SYNC_INTERVAL_SECONDS"), - DEFAULT_SYNC_INTERVAL_SECONDS, - ), - "enabled": env_flag_enabled("GRANOLA_ETL_ENABLED", default=False), - "no_delivery": True, -} - - -@dataclass -class Input: - """Runtime options for a manual Granola sync workflow run.""" - - since: str | None = None - limit: int = DEFAULT_PAGE_SIZE - max_notes: int | None = None - include_transcripts: bool = True - watermark_overlap_seconds: int = DEFAULT_WATERMARK_OVERLAP_SECONDS - metadata: dict[str, Any] = field(default_factory=dict) - - -class GranolaSyncClient(Protocol): - """Small adapter protocol used by the Granola ETL workflow.""" - - async def list_notes( - self, - page_size: int = DEFAULT_PAGE_SIZE, - cursor: str | None = None, - created_before: str | None = None, - created_after: str | None = None, - updated_after: str | None = None, - ) -> dict[str, Any]: ... - - async def get_note( - self, note_id: str, include_transcript: bool = False - ) -> dict[str, Any]: ... - - -class GranolaToolClient: - """Granola client backed by the workflow tool bridge.""" - - def __init__(self, ctx: WorkflowContext) -> None: - self._ctx = ctx - - async def list_notes( - self, - page_size: int = DEFAULT_PAGE_SIZE, - cursor: str | None = None, - created_before: str | None = None, - created_after: str | None = None, - updated_after: str | None = None, - ) -> dict[str, Any]: - result = await self._ctx.call_tool( - "granola", - "list_notes", - { - "page_size": page_size, - "cursor": cursor, - "created_before": created_before, - "created_after": created_after, - "updated_after": updated_after, - }, - ) - return result if isinstance(result, dict) else {} - - async def get_note( - self, note_id: str, include_transcript: bool = False - ) -> dict[str, Any]: - result = await self._ctx.call_tool( - "granola", - "get_note", - {"note_id": note_id, "include_transcript": include_transcript}, - ) - return result if isinstance(result, dict) else {} - - -def _client(ctx: WorkflowContext) -> GranolaSyncClient: - return GranolaToolClient(ctx) - - -def _parse_datetime(value: str | None) -> dt.datetime | None: - if not value: - return None - try: - parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError: - return None - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=dt.timezone.utc) - return parsed.astimezone(dt.timezone.utc) - - -def _source_datetime(payload: dict[str, Any], *keys: str) -> dt.datetime | None: - for key in keys: - parsed = _parse_datetime(str(payload.get(key) or "")) - if parsed is not None: - return parsed - return None - - -def _rfc3339(value: dt.datetime) -> str: - return value.astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z") - - -def _text_value(value: Any) -> str: - return str(value or "") - - -def _note_url(note: dict[str, Any]) -> str: - return _text_value(note.get("url") or note.get("permalink") or note.get("web_url")) - - -def _normalized_email(value: Any) -> str: - return str(value or "").strip().lower() - - -def _access_emails(owner: dict[str, Any], attendees: list[Any]) -> list[str]: - emails: list[str] = [] - - def add(value: Any) -> None: - email = _normalized_email(value) - if email and email not in emails: - emails.append(email) - - add(owner.get("email")) - for attendee in attendees: - if isinstance(attendee, dict): - add(attendee.get("email")) - return emails - - -def _json_object(value: Any) -> dict[str, Any]: - return value if isinstance(value, dict) else {} - - -def _json_array(value: Any) -> list[Any]: - return value if isinstance(value, list) else [] - - -def _format_time(value: dt.datetime | None) -> str: - if not value: - return "unknown time" - return value.astimezone(dt.timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") - - -def _named_entry(value: Any) -> str: - if isinstance(value, dict): - name = _text_value(value.get("name") or value.get("display_name")).strip() - email = _text_value(value.get("email")).strip() - if name and email: - return f"{name} <{email}>" - return name or email - return _text_value(value).strip() - - -def _named_entries(value: Any) -> list[str]: - labels: list[str] = [] - for entry in _json_array(value): - label = _named_entry(entry) - if label and label not in labels: - labels.append(label) - return labels - - -def _content_hash(*parts: Any) -> str: - return hashlib.sha256(canonical_json(parts).encode("utf-8")).hexdigest() - - -def _workflow_run_id_to_sync_run_id(workflow_run_id: str) -> str: - safe_run_id = "".join(char if char.isalnum() else "_" for char in workflow_run_id) - return f"granola_sync_{safe_run_id}" - - -def _scope_ref(scope_id: str, reason: str | None = None) -> dict[str, str]: - result = {"scope_id": scope_id} - if reason: - result["reason"] = reason - return result - - -def _failure_reason(error: str) -> str: - lowered = error.lower() - if "rate" in lowered or "429" in lowered: - return "rate_limited" - if ( - "401" in lowered - or "403" in lowered - or "auth" in lowered - or "permission" in lowered - ): - return "permission_error" - if "database" in lowered or "postgres" in lowered: - return "write_error" - return "api_error" - - -def _transcript_text(transcript: Any) -> str: - lines: list[str] = [] - for utterance in _json_array(transcript): - if not isinstance(utterance, dict): - continue - speaker = _json_object(utterance.get("speaker")) - speaker_name = ( - _text_value(speaker.get("name")) - or _text_value(speaker.get("email")) - or _text_value(speaker.get("source")) - or "Unknown" - ) - text = _text_value(utterance.get("text")).strip() - if text: - lines.append(f"{speaker_name}: {text}") - return "\n".join(lines) - - -def _granola_context_document( - *, - note: dict[str, Any], - note_id: str, - title: str, - owner: dict[str, Any], - attendees: list[Any], - access_emails: list[str], - calendar_event: dict[str, Any], - transcript: list[Any], - transcript_text: str, - summary_markdown: str, - summary_text: str, - source_created_at: dt.datetime | None, - source_updated_at: dt.datetime | None, -) -> dict[str, Any]: - owner_id = _text_value(owner.get("id") or owner.get("user_id")) - owner_email = _text_value(owner.get("email")) - owner_name = _text_value(owner.get("name") or owner.get("display_name")) - owner_label = _named_entry(owner) - attendee_labels = _named_entries(attendees) - document_title = title.strip() or "Untitled Granola note" - url = _note_url(note) - summary = summary_markdown.strip() or summary_text.strip() - - lines = [ - f"# {document_title}", - "", - "- Source: Granola", - f"- Created: {_format_time(source_created_at)}", - f"- Updated: {_format_time(source_updated_at)}", - ] - if owner_label: - lines.append(f"- Owner: {owner_label}") - if attendee_labels: - lines.append(f"- Attendees: {', '.join(attendee_labels)}") - if url: - lines.append(f"- URL: {url}") - if summary: - lines.extend(["", "## Summary", summary]) - if transcript_text.strip(): - lines.extend(["", "## Transcript", transcript_text.strip()]) - - body = "\n".join(lines).strip() - metadata = { - "source": "granola", - "note_id": note_id, - "owner_id": owner_id, - "owner_email": owner_email, - "owner_name": owner_name, - "access_emails": access_emails, - "attendees": attendees, - "attendee_labels": attendee_labels, - "calendar_event": calendar_event, - "transcript_payload": transcript, - "has_summary": bool(summary), - "has_transcript": bool(transcript_text.strip()), - "raw_payload": note, - } - return { - "document_id": f"granola:note:{note_id}", - "note_id": note_id, - "title": document_title, - "body": body, - "url": url, - "owner_id": owner_id, - "owner_email": owner_email, - "owner_name": owner_name, - "access_emails": access_emails, - "attendee_labels": attendee_labels, - "occurred_at": source_created_at or source_updated_at, - "source_updated_at": source_updated_at, - "content_hash": _content_hash(document_title, body, url, metadata), - "metadata": metadata, - } - - -async def _load_checkpoint(pool, scope_id: str) -> dict[str, Any] | None: - row = await pool.fetchrow( - "SELECT watermark_time, last_error FROM granola_sync_checkpoints " - "WHERE scope_id = $1", - scope_id, - ) - return dict(row) if row else None - - -async def _update_checkpoint_success( - pool, - *, - scope_id: str, - watermark_time: dt.datetime | None, - run_id: str, -) -> None: - await pool.execute( - "INSERT INTO granola_sync_checkpoints (" - "scope_id, watermark_time, last_run_id, last_success_at, last_error, updated_at" - ") VALUES ($1, $2, $3, NOW(), '', NOW()) " - "ON CONFLICT (scope_id) DO UPDATE SET " - "watermark_time = COALESCE(EXCLUDED.watermark_time, " - "granola_sync_checkpoints.watermark_time), " - "last_run_id = EXCLUDED.last_run_id, " - "last_success_at = NOW(), " - "last_error = '', " - "updated_at = NOW()", - scope_id, - watermark_time, - run_id, - ) - - -async def _update_checkpoint_failure( - pool, - *, - scope_id: str, - run_id: str, - error: str, -) -> None: - await pool.execute( - "INSERT INTO granola_sync_checkpoints (" - "scope_id, last_run_id, last_error, updated_at" - ") VALUES ($1, $2, $3, NOW()) " - "ON CONFLICT (scope_id) DO UPDATE SET " - "last_run_id = EXCLUDED.last_run_id, " - "last_error = EXCLUDED.last_error, " - "updated_at = NOW()", - scope_id, - run_id, - error, - ) - - -async def _emit_checkpoint_metrics(pool) -> None: - """Publish Granola workspace checkpoint health for the ETL overview.""" - row = await pool.fetchrow( - "SELECT COUNT(*) AS active_scopes, " - "COUNT(*) FILTER (WHERE last_error <> '') AS failed_scopes, " - "COALESCE(" - " EXTRACT(EPOCH FROM NOW() - MIN(last_success_at) " - " FILTER (WHERE last_success_at IS NOT NULL)" - " ), " - " 0" - ") AS freshness_seconds " - "FROM granola_sync_checkpoints" - ) - set_etl_active_scopes("granola", int(row["active_scopes"] or 0) if row else 0) - set_etl_failed_scopes("granola", int(row["failed_scopes"] or 0) if row else 0) - set_etl_scope_sync_freshness_seconds( - "granola", - float(row["freshness_seconds"] or 0.0) if row else 0.0, - ) - - -async def _record_run_start( - pool, - *, - run_id: str, - workflow_run_id: str, - scopes_requested: list[dict[str, str]], - metadata: dict[str, Any], -) -> None: - await pool.execute( - "INSERT INTO granola_sync_runs (" - "run_id, workflow_run_id, mode, status, scopes_requested, metadata" - ") VALUES ($1, $2, 'incremental', 'running', $3::jsonb, $4::jsonb) " - "ON CONFLICT (run_id) DO UPDATE SET " - "workflow_run_id = EXCLUDED.workflow_run_id, " - "status = 'running', " - "scopes_requested = EXCLUDED.scopes_requested, " - "scopes_synced = '[]'::jsonb, " - "scopes_failed = '[]'::jsonb, " - "notes_seen = 0, " - "notes_upserted = 0, " - "transcripts_seen = 0, " - "transcripts_upserted = 0, " - "finished_at = NULL, " - "error_text = '', " - "metadata = EXCLUDED.metadata", - run_id, - workflow_run_id, - canonical_json(scopes_requested), - canonical_json(metadata), - ) - - -async def _record_run_finish( - pool, - *, - run_id: str, - status: str, - scopes_synced: list[dict[str, str]], - scopes_failed: list[dict[str, str]], - counts: dict[str, int], - error_text: str = "", -) -> None: - await pool.execute( - "UPDATE granola_sync_runs SET " - "status = $2, scopes_synced = $3::jsonb, scopes_failed = $4::jsonb, " - "notes_seen = $5, notes_upserted = $6, transcripts_seen = $7, " - "transcripts_upserted = $8, finished_at = NOW(), error_text = $9 " - "WHERE run_id = $1", - run_id, - status, - canonical_json(scopes_synced), - canonical_json(scopes_failed), - counts.get("notes_seen", 0), - counts.get("notes_upserted", 0), - counts.get("transcripts_seen", 0), - counts.get("transcripts_upserted", 0), - error_text, - ) - - -async def _upsert_context_document(pool, document: dict[str, Any]) -> None: - await pool.execute( - "INSERT INTO granola_context_documents (" - "document_id, note_id, title, body, url, owner_id, owner_email, owner_name, " - "access_emails, attendee_labels, occurred_at, source_updated_at, content_hash, " - "metadata, updated_at" - ") VALUES (" - "$1, $2, $3, $4, $5, $6, $7, $8, $9::text[], $10::text[], $11, $12, $13, " - "$14::jsonb, NOW()" - ") ON CONFLICT (document_id) DO UPDATE SET " - "note_id = EXCLUDED.note_id, " - "title = EXCLUDED.title, " - "body = EXCLUDED.body, " - "url = EXCLUDED.url, " - "owner_id = EXCLUDED.owner_id, " - "owner_email = EXCLUDED.owner_email, " - "owner_name = EXCLUDED.owner_name, " - "access_emails = EXCLUDED.access_emails, " - "attendee_labels = EXCLUDED.attendee_labels, " - "occurred_at = EXCLUDED.occurred_at, " - "source_updated_at = EXCLUDED.source_updated_at, " - "content_hash = EXCLUDED.content_hash, " - "metadata = EXCLUDED.metadata, " - "updated_at = NOW()", - document["document_id"], - document["note_id"], - document["title"], - document["body"], - document["url"], - document["owner_id"], - document["owner_email"], - document["owner_name"], - document["access_emails"], - document["attendee_labels"], - document["occurred_at"], - document["source_updated_at"], - document["content_hash"], - canonical_json(document["metadata"]), - ) - - -async def _upsert_note( - pool, - *, - note: dict[str, Any], - run_id: str, -) -> tuple[dt.datetime | None, bool]: - note_id = _text_value(note.get("id") or note.get("note_id")) - owner = _json_object(note.get("owner")) - attendees = _json_array(note.get("attendees")) - access_emails = _access_emails(owner, attendees) - calendar_event = _json_object(note.get("calendar_event")) - transcript = _json_array(note.get("transcript")) - transcript_text = _transcript_text(transcript) - summary_markdown = _text_value(note.get("summary_markdown")) - summary_text = _text_value(note.get("summary_text")) - title = _text_value(note.get("title")) - content_text = "\n".join( - part - for part in (title, summary_markdown, summary_text, transcript_text) - if part.strip() - ) - source_created_at = _source_datetime(note, "created_at", "createdAt") - source_updated_at = ( - _source_datetime(note, "updated_at", "updatedAt") or source_created_at - ) - context_document = _granola_context_document( - note=note, - note_id=note_id, - title=title, - owner=owner, - attendees=attendees, - access_emails=access_emails, - calendar_event=calendar_event, - transcript=transcript, - transcript_text=transcript_text, - summary_markdown=summary_markdown, - summary_text=summary_text, - source_created_at=source_created_at, - source_updated_at=source_updated_at, - ) - await pool.execute( - "INSERT INTO granola_sync_notes (" - "note_id, title, owner_id, owner_email, owner_name, attendees, access_emails, " - "calendar_event, summary_markdown, summary_text, transcript_text, transcript_payload, " - "url, content_text, content_hash, source_created_at, source_updated_at, raw_payload, " - "source_run_id, last_seen_at, last_error, updated_at" - ") VALUES (" - "$1, $2, $3, $4, $5, $6::jsonb, $7::text[], $8::jsonb, $9, $10, " - "$11, $12::jsonb, $13, $14, $15, $16, $17, $18::jsonb, $19, NOW(), '', NOW()" - ") ON CONFLICT (note_id) DO UPDATE SET " - "title = EXCLUDED.title, " - "owner_id = EXCLUDED.owner_id, " - "owner_email = EXCLUDED.owner_email, " - "owner_name = EXCLUDED.owner_name, " - "attendees = EXCLUDED.attendees, " - "access_emails = EXCLUDED.access_emails, " - "calendar_event = EXCLUDED.calendar_event, " - "summary_markdown = EXCLUDED.summary_markdown, " - "summary_text = EXCLUDED.summary_text, " - "transcript_text = EXCLUDED.transcript_text, " - "transcript_payload = EXCLUDED.transcript_payload, " - "url = EXCLUDED.url, " - "content_text = EXCLUDED.content_text, " - "content_hash = EXCLUDED.content_hash, " - "source_created_at = EXCLUDED.source_created_at, " - "source_updated_at = EXCLUDED.source_updated_at, " - "raw_payload = EXCLUDED.raw_payload, " - "source_run_id = EXCLUDED.source_run_id, " - "last_seen_at = NOW(), " - "last_error = '', " - "updated_at = NOW()", - note_id, - title, - _text_value(owner.get("id") or owner.get("user_id")), - _text_value(owner.get("email")), - _text_value(owner.get("name") or owner.get("display_name")), - canonical_json(attendees), - access_emails, - canonical_json(calendar_event), - summary_markdown, - summary_text, - transcript_text, - canonical_json(transcript), - _text_value(note.get("url") or note.get("permalink")), - content_text, - _content_hash(content_text), - source_created_at, - source_updated_at, - canonical_json(note), - run_id, - ) - await _upsert_context_document(pool, context_document) - return source_updated_at, bool(transcript) - - -async def _sync_workspace( - *, - client: GranolaSyncClient, - pool, - page_size: int, - updated_after: dt.datetime | None, - max_notes: int | None, - include_transcripts: bool, - run_id: str, -) -> tuple[int, int, int, int, dt.datetime | None]: - seen = 0 - upserted = 0 - transcripts_seen = 0 - transcripts_upserted = 0 - watermark: dt.datetime | None = None - cursor: str | None = None - updated_after_arg = _rfc3339(updated_after) if updated_after else None - - while True: - page = await client.list_notes( - page_size=page_size, - cursor=cursor, - updated_after=updated_after_arg, - ) - notes = [ - note - for note in page.get("notes", []) or [] - if isinstance(note, dict) and (note.get("id") or note.get("note_id")) - ] - if max_notes is not None: - notes = notes[: max(max_notes - seen, 0)] - seen += len(notes) - record_etl_items_seen("granola", WORKSPACE_SCOPE, "note", len(notes)) - - for note_ref in notes: - note_id = _text_value(note_ref.get("id") or note_ref.get("note_id")) - note = ( - await client.get_note(note_id, include_transcript=include_transcripts) - if note_id - else note_ref - ) - if not isinstance(note, dict): - note = note_ref - note.setdefault("id", note_id) - source_updated_at, has_transcript = await _upsert_note( - pool, note=note, run_id=run_id - ) - upserted += 1 - record_etl_items_upserted("granola", WORKSPACE_SCOPE, "note", 1) - if include_transcripts: - transcripts_seen += 1 - if has_transcript: - transcripts_upserted += 1 - record_etl_items_upserted( - "granola", WORKSPACE_SCOPE, "transcript", 1 - ) - if source_updated_at and ( - watermark is None or source_updated_at > watermark - ): - watermark = source_updated_at - - if max_notes is not None and seen >= max_notes: - break - cursor = ( - _text_value(page.get("cursor") or page.get("next_cursor")).strip() or None - ) - if not page.get("hasMore") or not cursor: - break - - return seen, upserted, transcripts_seen, transcripts_upserted, watermark - - -async def handler(inp: Input, ctx: WorkflowContext) -> dict[str, Any]: - """Sync changed Granola notes into raw sync tables.""" - if not env_flag_enabled("GRANOLA_ETL_ENABLED", default=False): - ctx.log("granola_sync_skipped_disabled") - return {"status": "skipped", "reason": "granola_etl_disabled"} - - page_size = min(positive_int(inp.limit, DEFAULT_PAGE_SIZE), DEFAULT_PAGE_SIZE) - overlap_seconds = max(int(inp.watermark_overlap_seconds), 0) - run_id = _workflow_run_id_to_sync_run_id(ctx.run_id) - scopes_requested = [_scope_ref(WORKSPACE_SCOPE)] - - await _record_run_start( - ctx._pool, - run_id=run_id, - workflow_run_id=ctx.run_id, - scopes_requested=scopes_requested, - metadata={ - **inp.metadata, - "page_size": page_size, - "max_notes": inp.max_notes, - "include_transcripts": inp.include_transcripts, - }, - ) - - client = _client(ctx) - explicit_since = _parse_datetime(inp.since) - checkpoint = await _load_checkpoint(ctx._pool, WORKSPACE_SCOPE) - watermark = explicit_since - if watermark is None and checkpoint and checkpoint.get("watermark_time"): - watermark = checkpoint["watermark_time"].astimezone(dt.timezone.utc) - if watermark is not None: - watermark = watermark - dt.timedelta(seconds=overlap_seconds) - - synced: list[dict[str, str]] = [] - failed: list[dict[str, str]] = [] - counts = { - "notes_seen": 0, - "notes_upserted": 0, - "transcripts_seen": 0, - "transcripts_upserted": 0, - } - try: - ( - counts["notes_seen"], - counts["notes_upserted"], - counts["transcripts_seen"], - counts["transcripts_upserted"], - successful_watermark, - ) = await _sync_workspace( - client=client, - pool=ctx._pool, - page_size=page_size, - updated_after=watermark, - max_notes=inp.max_notes, - include_transcripts=inp.include_transcripts, - run_id=run_id, - ) - await _update_checkpoint_success( - ctx._pool, - scope_id=WORKSPACE_SCOPE, - watermark_time=successful_watermark, - run_id=run_id, - ) - synced.append(_scope_ref(WORKSPACE_SCOPE)) - except Exception as exc: - error = str(exc) - failed.append(_scope_ref(WORKSPACE_SCOPE, error)) - record_etl_items_failed( - "granola", WORKSPACE_SCOPE, "scope", _failure_reason(error) - ) - await _update_checkpoint_failure( - ctx._pool, - scope_id=WORKSPACE_SCOPE, - run_id=run_id, - error=error, - ) - ctx.log("granola_sync_scope_failed", scope_id=WORKSPACE_SCOPE, error=error) - - status = "completed" if not failed else "failed" - error_text = "" if not failed else "Granola workspace sync failed" - await _record_run_finish( - ctx._pool, - run_id=run_id, - status=status, - scopes_synced=synced, - scopes_failed=failed, - counts=counts, - error_text=error_text, - ) - await _emit_checkpoint_metrics(ctx._pool) - - return {"status": status, "run_id": run_id, **counts} diff --git a/workflows/slack/retention.py b/workflows/slack/retention.py index 83fb92a3a..b83eca403 100644 --- a/workflows/slack/retention.py +++ b/workflows/slack/retention.py @@ -180,12 +180,12 @@ async def prune_slack_dm(pool, *, retention_days: int, dry_run: bool = False) -> "messages": await _count_or_delete( pool, count_sql=( - "SELECT COUNT(*) FROM slack_dm_sync_messages " + "SELECT COUNT(*) FROM slack_private_sync_messages " "WHERE occurred_at < NOW() - make_interval(days => $1)" ), delete_sql=( "WITH deleted AS (" - " DELETE FROM slack_dm_sync_messages " + " DELETE FROM slack_private_sync_messages " " WHERE occurred_at < NOW() - make_interval(days => $1) " " RETURNING 1" ") SELECT COUNT(*) FROM deleted" @@ -196,20 +196,20 @@ async def prune_slack_dm(pool, *, retention_days: int, dry_run: bool = False) -> "conversations": await _count_or_delete( pool, count_sql=( - "SELECT COUNT(*) FROM slack_dm_sync_conversations c " + "SELECT COUNT(*) FROM slack_private_sync_conversations c " "WHERE c.last_seen_at < NOW() - make_interval(days => $1) " " AND NOT EXISTS (" - " SELECT 1 FROM slack_dm_sync_messages m " + " SELECT 1 FROM slack_private_sync_messages m " " WHERE m.home_team_id = c.home_team_id " " AND m.conversation_id = c.conversation_id" " )" ), delete_sql=( "WITH deleted AS (" - " DELETE FROM slack_dm_sync_conversations c " + " DELETE FROM slack_private_sync_conversations c " " WHERE c.last_seen_at < NOW() - make_interval(days => $1) " " AND NOT EXISTS (" - " SELECT 1 FROM slack_dm_sync_messages m " + " SELECT 1 FROM slack_private_sync_messages m " " WHERE m.home_team_id = c.home_team_id " " AND m.conversation_id = c.conversation_id" " ) " @@ -222,13 +222,13 @@ async def prune_slack_dm(pool, *, retention_days: int, dry_run: bool = False) -> "backfill_jobs": await _count_or_delete( pool, count_sql=( - "SELECT COUNT(*) FROM slack_dm_sync_backfill_jobs " + "SELECT COUNT(*) FROM slack_private_sync_backfill_jobs " "WHERE status IN ('completed', 'failed') " " AND updated_at < NOW() - make_interval(days => $1)" ), delete_sql=( "WITH deleted AS (" - " DELETE FROM slack_dm_sync_backfill_jobs " + " DELETE FROM slack_private_sync_backfill_jobs " " WHERE status IN ('completed', 'failed') " " AND updated_at < NOW() - make_interval(days => $1) " " RETURNING 1" @@ -240,13 +240,13 @@ async def prune_slack_dm(pool, *, retention_days: int, dry_run: bool = False) -> "runs": await _count_or_delete( pool, count_sql=( - "SELECT COUNT(*) FROM slack_dm_sync_runs " + "SELECT COUNT(*) FROM slack_private_sync_runs " "WHERE status <> 'running' " " AND COALESCE(finished_at, started_at) < NOW() - make_interval(days => $1)" ), delete_sql=( "WITH deleted AS (" - " DELETE FROM slack_dm_sync_runs " + " DELETE FROM slack_private_sync_runs " " WHERE status <> 'running' " " AND COALESCE(finished_at, started_at) " " < NOW() - make_interval(days => $1) " diff --git a/workflows/slack/tests/test_retention.py b/workflows/slack/tests/test_retention.py index a36e3781d..740708ea1 100644 --- a/workflows/slack/tests/test_retention.py +++ b/workflows/slack/tests/test_retention.py @@ -126,11 +126,11 @@ def test_prune_slack_dm_dry_run_counts_expected_tables(): "runs": 8, } sql = "\n".join(call[0] for call in pool.calls) - assert "SELECT COUNT(*) FROM slack_dm_sync_messages" in sql - assert "SELECT COUNT(*) FROM slack_dm_sync_conversations" in sql + assert "SELECT COUNT(*) FROM slack_private_sync_messages" in sql + assert "SELECT COUNT(*) FROM slack_private_sync_conversations" in sql assert "NOT EXISTS" in sql - assert "SELECT COUNT(*) FROM slack_dm_sync_backfill_jobs" in sql - assert "SELECT COUNT(*) FROM slack_dm_sync_runs" in sql + assert "SELECT COUNT(*) FROM slack_private_sync_backfill_jobs" in sql + assert "SELECT COUNT(*) FROM slack_private_sync_runs" in sql assert "DELETE FROM" not in sql diff --git a/workflows/tests/test_company_context_documents_attachments.py b/workflows/tests/test_company_context_documents_attachments.py index 185e093a2..2f6172e6a 100644 --- a/workflows/tests/test_company_context_documents_attachments.py +++ b/workflows/tests/test_company_context_documents_attachments.py @@ -25,7 +25,6 @@ def _load_projection_module(): company_context_metrics = types.ModuleType("workflows.company_context_metrics") for name in ( - "observe_company_context_document_size", "record_company_context_documents_changed", "set_company_context_projection_lag", ): @@ -112,10 +111,15 @@ def __init__(self) -> None: self.run_id = "run_123" self._pool = object() self.logs: list[tuple[str, dict]] = [] + self.children: list[tuple[str, dict, str | None]] = [] def log(self, message, **fields): self.logs.append((message, fields)) + async def start_workflow(self, workflow_name, input, *, idempotency_key=None): + self.children.append((workflow_name, input, idempotency_key)) + return {"task_id": f"task_{len(self.children)}", "created": True} + def test_latest_successful_watermark_reads_absurd_etl_queue(): pool = FakeWatermarkPool() @@ -159,26 +163,19 @@ def test_load_changed_message_keys_applies_upper_batch_bound(): assert args == (since, until) -def test_handler_advances_empty_bounded_window(monkeypatch): +def test_coordinator_starts_one_child_per_enabled_scope(monkeypatch): last_watermark = dt.datetime(2026, 6, 18, 22, 59, 36, tzinfo=dt.UTC) - seen_bounds: dict[str, dt.datetime | None] = {} + claimed_scopes: list[str] = [] async def latest_watermark(_pool, _run_id): return last_watermark - async def load_slack_lookup_maps(_pool): - return {}, {} - - async def load_changed_message_keys(_pool, since, until=None): - seen_bounds["since"] = since - seen_bounds["until"] = until + async def claim_scope(_pool, *, scope, **_kwargs): + claimed_scopes.append(scope) return { - "channel_days": [], - "threads": [], - "attachments": [], - "changed_messages": 0, - "changed_attachments": 0, - "max_updated_at": None, + "scope": scope, + "lease_token": f"lease_{scope}", + "window_end": dt.datetime(2026, 6, 18, 23, 58, 36, tzinfo=dt.UTC), } async def noop_async(*_args, **_kwargs): @@ -190,17 +187,8 @@ async def noop_async(*_args, **_kwargs): monkeypatch.setenv("LINEAR_ETL_ENABLED", "false") monkeypatch.setenv("COMPANY_CONTEXT_DOCUMENTS_ENABLED", "true") monkeypatch.setattr(projection, "_latest_successful_watermark", latest_watermark) - monkeypatch.setattr(projection, "_load_slack_lookup_maps", load_slack_lookup_maps) - monkeypatch.setattr( - projection, - "_load_changed_message_keys", - load_changed_message_keys, - ) - monkeypatch.setattr( - projection, - "_emit_company_context_document_size_snapshot", - noop_async, - ) + monkeypatch.setattr(projection, "_claim_scope", claim_scope) + monkeypatch.setattr(projection, "_emit_projection_lag_from_checkpoints", noop_async) monkeypatch.setattr(projection, "_emit_etl_scope_metrics", noop_async) ctx = FakeWorkflowContext() @@ -211,14 +199,59 @@ async def noop_async(*_args, **_kwargs): ) ) - expected_since = dt.datetime(2026, 6, 18, 22, 58, 36, tzinfo=dt.UTC) - expected_until = dt.datetime(2026, 6, 18, 23, 58, 36, tzinfo=dt.UTC) - assert seen_bounds == {"since": expected_since, "until": expected_until} - assert result["changed_messages"] == 0 - assert result["batch_until"] == expected_until.isoformat() - assert result["watermark"] == expected_until.isoformat() - assert result["remaining_lag_seconds"] is not None - assert ctx.logs[-1][0] == "company_context_documents_completed" + assert claimed_scopes == ["slack_channel_day", "slack_thread", "slack_attachment"] + assert [child[1]["scope"] for child in ctx.children] == claimed_scopes + assert all(child[0] == "company_context_documents" for child in ctx.children) + assert all("company-context:slack_" in str(child[2]) for child in ctx.children) + assert result["started_scopes"] + assert result["batch_size"] == projection.DEFAULT_BATCH_SIZE + assert ctx.logs[-1][0] == "company_context_documents_coordinator_completed" + + +def test_scope_batch_advances_cursor_before_starting_one_continuation(monkeypatch): + window_end = dt.datetime(2026, 6, 18, 23, 58, 36, tzinfo=dt.UTC) + cursor_at = dt.datetime(2026, 6, 18, 23, 0, tzinfo=dt.UTC) + advanced: list[tuple] = [] + + async def read_owned_scope(_pool, _scope, _lease_token): + return { + "window_start": dt.datetime(2026, 6, 18, 22, 58, 36, tzinfo=dt.UTC), + "window_end": window_end, + "cursor_updated_at": None, + "cursor_key": "", + } + + async def load_scope_page(*_args, **_kwargs): + return [ + {"projection_updated_at": cursor_at, "projection_key": "doc_a"}, + {"projection_updated_at": cursor_at, "projection_key": "doc_b"}, + ] + + async def project_scope_page(*_args, **_kwargs): + return 2, 0 + + async def advance_scope_cursor(*args): + advanced.append(args) + + monkeypatch.setattr(projection, "_read_owned_scope", read_owned_scope) + monkeypatch.setattr(projection, "_load_scope_page", load_scope_page) + monkeypatch.setattr(projection, "_project_scope_page", project_scope_page) + monkeypatch.setattr(projection, "_advance_scope_cursor", advance_scope_cursor) + monkeypatch.setattr(projection, "_batch_size", lambda _value=None: 2) + + ctx = FakeWorkflowContext() + result = asyncio.run( + projection._run_scope_batch( + projection.Input(scope="google_doc", lease_token="lease_1", batch_size=2), + ctx, + "google_doc", + ) + ) + + assert advanced == [(ctx._pool, "google_doc", "lease_1", cursor_at, "doc_b")] + assert ctx.children[0][1]["scope"] == "google_doc" + assert ctx.children[0][1]["lease_token"] == "lease_1" + assert result["continuation"]["created"] is True def test_etl_scope_metrics_no_longer_emit_slack_scope_gauges(monkeypatch): diff --git a/workflows/tests/test_granola_sync.py b/workflows/tests/test_granola_sync.py deleted file mode 100644 index 1a7373227..000000000 --- a/workflows/tests/test_granola_sync.py +++ /dev/null @@ -1,219 +0,0 @@ -from __future__ import annotations - -import asyncio -import datetime as dt -import importlib -import json -import sys -import types -from pathlib import Path - - -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - - -def _install_workflow_stubs() -> None: - api_module = sys.modules.get("api") or types.ModuleType("api") - runtime_control = sys.modules.get("api.runtime_control") or types.ModuleType( - "api.runtime_control" - ) - runtime_control.canonical_json = lambda value: json.dumps(value, sort_keys=True) - - etl_metrics = types.ModuleType("workflows.etl_metrics") - for name in ( - "record_etl_items_failed", - "record_etl_items_seen", - "record_etl_items_upserted", - "set_etl_active_scopes", - "set_etl_failed_scopes", - "set_etl_scope_sync_freshness_seconds", - ): - setattr(etl_metrics, name, lambda *_args, **_kwargs: None) - - workflow_engine = types.ModuleType("api.workflow_engine") - workflow_engine.WorkflowContext = object - - slack_shared = types.ModuleType("workflows.slack.shared") - slack_shared.env_flag_enabled = lambda _name, default=True: default - slack_shared.positive_int = lambda value, default: ( - int(value) if value is not None and int(value) > 0 else default - ) - - api_module.runtime_control = runtime_control - api_module.workflow_engine = workflow_engine - sys.modules.setdefault("api", api_module) - sys.modules["api.runtime_control"] = runtime_control - sys.modules["api.workflow_engine"] = workflow_engine - sys.modules["workflows.etl_metrics"] = etl_metrics - sys.modules["workflows.slack.shared"] = slack_shared - - -def _load(name: str): - _install_workflow_stubs() - return importlib.import_module(name) - - -def test_granola_transcript_text_uses_speaker_identity(): - granola = _load("workflows.granola_sync") - - text = granola._transcript_text( - [ - {"speaker": {"name": "Alice"}, "text": "Hello"}, - {"speaker": {"email": "bob@example.com"}, "text": "Ship it"}, - {"speaker": {}, "text": ""}, - ] - ) - - assert text == "Alice: Hello\nbob@example.com: Ship it" - - -def test_granola_access_emails_include_owner_and_attendees_once(): - granola = _load("workflows.granola_sync") - - emails = granola._access_emails( - {"email": "Alice@Example.com "}, - [ - {"email": "bob@example.com"}, - {"email": "alice@example.com"}, - {"name": "No Email"}, - ], - ) - - assert emails == ["alice@example.com", "bob@example.com"] - - -def test_granola_context_document_is_user_scoped_to_owner_and_attendees(): - granola = _load("workflows.granola_sync") - note = { - "id": "not_123", - "title": "Launch review", - "web_url": "https://app.granola.ai/notes/not_123", - "owner": {"id": "usr_1", "name": "Alice", "email": "Alice@Example.com"}, - "attendees": [ - {"name": "Bob", "email": "bob@example.com"}, - {"name": "Alice", "email": "alice@example.com"}, - ], - "summary_markdown": "We agreed to ship.", - "transcript": [ - {"speaker": {"name": "Alice"}, "text": "Let's ship."}, - ], - "created_at": "2026-07-01T10:00:00Z", - "updated_at": "2026-07-01T10:30:00Z", - } - owner = granola._json_object(note["owner"]) - attendees = granola._json_array(note["attendees"]) - transcript = granola._json_array(note["transcript"]) - access_emails = granola._access_emails(owner, attendees) - - document = granola._granola_context_document( - note=note, - note_id="not_123", - title="Launch review", - owner=owner, - attendees=attendees, - access_emails=access_emails, - calendar_event={}, - transcript=transcript, - transcript_text=granola._transcript_text(transcript), - summary_markdown="We agreed to ship.", - summary_text="", - source_created_at=dt.datetime(2026, 7, 1, 10, tzinfo=dt.UTC), - source_updated_at=dt.datetime(2026, 7, 1, 10, 30, tzinfo=dt.UTC), - ) - - assert document["document_id"] == "granola:note:not_123" - assert document["note_id"] == "not_123" - assert document["url"] == "https://app.granola.ai/notes/not_123" - assert document["access_emails"] == ["alice@example.com", "bob@example.com"] - assert document["attendee_labels"] == [ - "Bob <bob@example.com>", - "Alice <alice@example.com>", - ] - assert "## Summary\nWe agreed to ship." in document["body"] - assert "## Transcript\nAlice: Let's ship." in document["body"] - assert document["metadata"]["access_emails"] == [ - "alice@example.com", - "bob@example.com", - ] - assert document["metadata"]["has_transcript"] is True - - -def test_granola_checkpoint_metrics_use_workspace_checkpoint_health(monkeypatch): - granola = _load("workflows.granola_sync") - calls: dict[str, list[tuple]] = { - "active": [], - "failed": [], - "freshness": [], - } - monkeypatch.setattr( - granola, - "set_etl_active_scopes", - lambda *args: calls["active"].append(args), - ) - monkeypatch.setattr( - granola, - "set_etl_failed_scopes", - lambda *args: calls["failed"].append(args), - ) - monkeypatch.setattr( - granola, - "set_etl_scope_sync_freshness_seconds", - lambda *args: calls["freshness"].append(args), - ) - - class FakePool: - def __init__(self) -> None: - self.fetchrow_calls: list[tuple[str, tuple]] = [] - - async def fetchrow(self, query, *args): - self.fetchrow_calls.append((query, args)) - return { - "active_scopes": 1, - "failed_scopes": 1, - "freshness_seconds": 123.5, - } - - pool = FakePool() - asyncio.run(granola._emit_checkpoint_metrics(pool)) - - assert len(pool.fetchrow_calls) == 1 - assert "FROM granola_sync_checkpoints" in pool.fetchrow_calls[0][0] - assert calls == { - "active": [("granola", 1)], - "failed": [("granola", 1)], - "freshness": [("granola", 123.5)], - } - - -def test_granola_sync_emits_checkpoint_metrics_after_a_failed_attempt(monkeypatch): - granola = _load("workflows.granola_sync") - monkeypatch.setattr(granola, "env_flag_enabled", lambda *_args, **_kwargs: True) - monkeypatch.setattr(granola, "_client", lambda _ctx: object()) - - async def noop(*_args, **_kwargs): - return None - - async def fail_sync(*_args, **_kwargs): - raise RuntimeError("Granola API unavailable") - - emitted: list[object] = [] - - async def record_metrics(pool): - emitted.append(pool) - - monkeypatch.setattr(granola, "_record_run_start", noop) - monkeypatch.setattr(granola, "_load_checkpoint", noop) - monkeypatch.setattr(granola, "_sync_workspace", fail_sync) - monkeypatch.setattr(granola, "_update_checkpoint_failure", noop) - monkeypatch.setattr(granola, "_record_run_finish", noop) - monkeypatch.setattr(granola, "_emit_checkpoint_metrics", record_metrics) - - pool = object() - context = types.SimpleNamespace( - run_id="run-123", _pool=pool, log=lambda *_args, **_kwargs: None - ) - - result = asyncio.run(granola.handler(granola.Input(), context)) - - assert result["status"] == "failed" - assert emitted == [pool]