feat(0056): route ops alarms to prices' own Slack channel via Chatbot

infra/envs/production.json

@@ -23,7 +23,11 @@
   "opsAlarms": {
     "sdexPushFreshnessSeconds": 604800,
     "mtlsNotAfterDaysThreshold": 30,
-    "ledgerProcessorLagSeconds": 120
+    "ledgerProcessorLagSeconds": 120,
+    "slack": {
+      "workspaceIdSsmParam": "/prices/production/slack-workspace-id",
+      "channelIdSsmParam": "/prices/production/slack-channel-id"
+    }
   },
   "ledgerProcessor": {
     "memoryMb": 512,

infra/src/lib/stacks/observability-stack.ts

@@ -1,8 +1,11 @@
 import * as cdk from 'aws-cdk-lib';
+import * as chatbot from 'aws-cdk-lib/aws-chatbot';
 import * as cloudwatch from 'aws-cdk-lib/aws-cloudwatch';
 import * as cw_actions from 'aws-cdk-lib/aws-cloudwatch-actions';
+import * as iam from 'aws-cdk-lib/aws-iam';
 import * as sns from 'aws-cdk-lib/aws-sns';
 import * as subscriptions from 'aws-cdk-lib/aws-sns-subscriptions';
+import * as ssm from 'aws-cdk-lib/aws-ssm';
 import type { Construct } from 'constructs';
 
 import type { EnvironmentConfig } from '../types.js';
@@ -71,6 +74,11 @@ export class ObservabilityStack extends cdk.Stack {
    * `config.opsAlarms.notificationEmail`.
    */
   public readonly opsAlarmsTopic: sns.Topic;
+  /**
+   * AWS Chatbot Slack subscription for the ops-alarms topic (task 0056).
+   * Created only when `config.opsAlarms.slack` is set; undefined otherwise.
+   */
+  public readonly opsAlarmsSlackChannel?: chatbot.SlackChannelConfiguration;
   /** SDEX push-freshness alarm (§5.6 / Tranche-1 AC #5). */
   public readonly sdexPushFreshnessAlarm: cloudwatch.Alarm;
   /** mTLS client-cert expiry alarm (§7 / §11.4). */
@@ -202,11 +210,56 @@ export class ObservabilityStack extends cdk.Stack {
         new subscriptions.EmailSubscription(config.opsAlarms.notificationEmail),
       );
     }
+
+    // AWS Chatbot → Slack (task 0056). When `opsAlarms.slack` is configured we
+    // route the ops-alarms topic to a Slack channel, matching how BE delivers
+    // its own CloudWatch alarms (its `${env}-soroban-explorer-alarms` topic →
+    // Chatbot → Slack) — the team's real ops surface, no email/mailing list.
+    // Reuses BE's channel: the workspace/channel IDs are read at deploy from the
+    // SSM params `opsAlarms.slack.{workspaceIdSsmParam,channelIdSsmParam}` (kept
+    // out of this public repo), which point at BE's existing
+    // `/soroban-explorer/{env}/slack-*` values in the shared account. The Slack
+    // workspace is already authorized in AWS Chatbot for BE, so no extra console
+    // step is needed here — the named SSM params just have to exist before
+    // deploying Observability. Omit the config to leave the topic
+    // subscriber-less (managed manually in SNS).
+    if (config.opsAlarms.slack) {
+      const slackWorkspaceId = ssm.StringParameter.valueForStringParameter(
+        this,
+        config.opsAlarms.slack.workspaceIdSsmParam,
+      );
+      const slackChannelId = ssm.StringParameter.valueForStringParameter(
+        this,
+        config.opsAlarms.slack.channelIdSsmParam,
+      );
+      this.opsAlarmsSlackChannel = new chatbot.SlackChannelConfiguration(
+        this,
+        'OpsAlarmsSlackChannel',
+        {
+          slackChannelConfigurationName: opsAlarmsTopicName(config.envName),
+          slackWorkspaceId,
+          slackChannelId,
+          notificationTopics: [this.opsAlarmsTopic],
+          role: new iam.Role(this, 'OpsAlarmsChatbotRole', {
+            assumedBy: new iam.ServicePrincipal('chatbot.amazonaws.com'),
+            managedPolicies: [
+              iam.ManagedPolicy.fromAwsManagedPolicyName(
+                'CloudWatchReadOnlyAccess',
+              ),
+            ],
+          }),
+        },
+      );
+    }
+
     const snsAction = new cw_actions.SnsAction(this.opsAlarmsTopic);
 
+    // Every alarm gets both an ALARM and an OK action on the ops topic, so the
+    // Slack channel sees the recovery as well as the breach (task 0056).
     // The enrichment backlog alarm (task 0026) shipped without an action; wire
     // it to the ops topic now that 0056 owns the alarm-routing.
     this.enrichmentBacklogAlarm.addAlarmAction(snsAction);
+    this.enrichmentBacklogAlarm.addOkAction(snsAction);
 
     // SDEX push freshness (§5.6 / Tranche-1 AC #5). The backfill-freshness-probe
     // republishes `sdex_archive`'s push age (seconds) as Prices/Backfill
@@ -240,6 +293,7 @@ export class ObservabilityStack extends cdk.Stack {
       },
     );
     this.sdexPushFreshnessAlarm.addAlarmAction(snsAction);
+    this.sdexPushFreshnessAlarm.addOkAction(snsAction);
 
     // mTLS cert expiry (§7 / §11.4). The mtls-notafter-probe publishes the
     // minimum days-to-NotAfter across the ingestion + api client certs; alarm
@@ -263,6 +317,7 @@ export class ObservabilityStack extends cdk.Stack {
       treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
     });
     this.mtlsNotAfterAlarm.addAlarmAction(snsAction);
+    this.mtlsNotAfterAlarm.addOkAction(snsAction);
 
     // -----------------------------------------------------------------
     // Live ledger-processor health (task 0056 finding B). The core ingestion
@@ -318,6 +373,7 @@ export class ObservabilityStack extends cdk.Stack {
       },
     );
     this.ledgerProcessorLagAlarm.addAlarmAction(snsAction);
+    this.ledgerProcessorLagAlarm.addOkAction(snsAction);
 
     // Hard errors: the ledger-processor Lambda throwing (a crash, not a graceful
     // per-item batch failure). Any invocation error over 5 min pages.
@@ -344,6 +400,7 @@ export class ObservabilityStack extends cdk.Stack {
       },
     );
     this.ledgerProcessorErrorAlarm.addAlarmAction(snsAction);
+    this.ledgerProcessorErrorAlarm.addOkAction(snsAction);
 
     // Poison-pill / permanent-failure doorbells: under reportBatchItemFailures a
     // handler that keeps failing one item re-drives it (no Lambda Error) until
@@ -374,6 +431,7 @@ export class ObservabilityStack extends cdk.Stack {
       },
     );
     this.ledgerProcessorDlqAlarm.addAlarmAction(snsAction);
+    this.ledgerProcessorDlqAlarm.addOkAction(snsAction);
 
     // Total ingestion halt: the lag / errors / DLQ alarms above all key on the
     // *presence* of enqueued or failed messages, so a producer-side stop (BE's
@@ -411,6 +469,7 @@ export class ObservabilityStack extends cdk.Stack {
       },
     );
     this.ledgerProcessorNoInvocationsAlarm.addAlarmAction(snsAction);
+    this.ledgerProcessorNoInvocationsAlarm.addOkAction(snsAction);
 
     new cdk.CfnOutput(this, 'OpsAlarmsTopicArn', {
       value: this.opsAlarmsTopic.topicArn,

infra/src/lib/types.ts

@@ -150,6 +150,27 @@ export interface EnvironmentConfig {
      * past it).
      */
     readonly ledgerProcessorLagSeconds: number;
+    /**
+     * Optional AWS Chatbot → Slack routing for the ops-alarms topic (task 0056).
+     * When set, `ObservabilityStack` subscribes `prices-{env}-ops-alarms` to a
+     * Slack channel via a `SlackChannelConfiguration`, so alarms land in Slack —
+     * matching how BE routes its own CloudWatch alarms (no ops email/mailing
+     * list). Prices has its **own** channel (`#stellar-prices-api-bot`), not BE's,
+     * so these point at **prices-owned** params
+     * (`/prices/{env}/slack-{workspace,channel}-id`) per the SSM ownership split.
+     * The workspace ID is the same shared Slack workspace BE already authorized in
+     * AWS Chatbot (only the channel differs), so no re-authorization is needed.
+     * Omit to leave the topic subscriber-less (managed manually in SNS).
+     *
+     * Values are SSM Parameter *names* (plain String, not credentials), resolved
+     * at deploy — the workspace/channel IDs stay out of this public repo. The
+     * named params must exist before deploying Observability, else synth/deploy
+     * fails on the lookup.
+     */
+    readonly slack?: {
+      readonly workspaceIdSsmParam: string;
+      readonly channelIdSsmParam: string;
+    };
   };
 
   // Ledger Processor ingest (consumed by IngestStack — task 0038)
@@ -350,6 +371,20 @@ export function validateConfig(config: EnvironmentConfig): void {
         `opsAlarms.ledgerProcessorLagSeconds must be a positive integer (seconds), got: ${ops.ledgerProcessorLagSeconds}`,
       );
     }
+    if (ops.slack !== undefined) {
+      const isSsmName = (v: unknown): boolean =>
+        typeof v === 'string' && v.startsWith('/') && v.length > 1;
+      if (
+        typeof ops.slack !== 'object' ||
+        ops.slack === null ||
+        !isSsmName(ops.slack.workspaceIdSsmParam) ||
+        !isSsmName(ops.slack.channelIdSsmParam)
+      ) {
+        errors.push(
+          'opsAlarms.slack, when set, must be { workspaceIdSsmParam, channelIdSsmParam } with absolute SSM parameter names (leading "/")',
+        );
+      }
+    }
     // Require a real local@domain.tld shape, not just a stray '@': a value like
     // '@' or 'ops@' passes an includes('@') check but yields an undeliverable
     // SNS subscription (a silent notification black hole).

lore/1-tasks/active/0056_FEATURE_cloudwatch-alarms-push-freshness-mtls-notafter.md

@@ -65,6 +65,35 @@ history:
       + production.json. fmt/clippy/eslint/prettier clean. Task stays active:
       deploy Observability + ops-topic subscribe + both fire-tests remain
       operational.
+  - date: 2026-07-08
+    status: active
+    who: okarcz
+    note: >
+      Ops routing decided + wired: AWS Chatbot → Slack, reusing BE's channel
+      (BE has no ops email — it routes CloudWatch alarms to Slack via Chatbot).
+      Added optional `opsAlarms.slack` config; ObservabilityStack now creates a
+      SlackChannelConfiguration (prices-{env}-ops-alarms) on the ops topic,
+      reading BE's existing `/soroban-explorer/{env}/slack-{workspace,channel}-id`
+      SSM params in the shared account. Synth-verified; tsc/eslint/prettier clean.
+      Supersedes the manual email-subscribe as the chosen route (email path kept
+      as fallback). Deploy of Observability (with the SSM params present) makes it
+      live.
+  - date: 2026-07-08
+    status: active
+    who: okarcz
+    note: >
+      Slack routing DEPLOYED to production + verified. Changed target from BE's
+      shared channel to prices' OWN channel #stellar-prices-api-bot: opsAlarms.slack
+      now reads prices-owned SSM params /prices/production/slack-{workspace,channel}-id
+      (workspace T83HLEDJN — same shared workspace, so no Chatbot re-auth; channel
+      C0BFWLMFQ9G). Created the two SSM params, deployed Prices-production-Observability
+      (SlackChannelConfiguration + Chatbot role + 4 ledger-processor alarms all live).
+      Added addOkAction to all 7 alarms so recoveries notify too. Smoke-tested via
+      set-alarm-state on ledger-processor-errors: both a breach and a recovery landed
+      in #stellar-prices-api-bot. Verified config with `aws chatbot
+      describe-slack-channel-configurations --region us-east-2` (Chatbot API is
+      us-east-2-only). Task stays active: the freshness + mTLS fire-tests +
+      notes/G-alarm-fire-test.md remain operational.
 ---
 
 # CloudWatch alarms — SDEX push freshness + mTLS NotAfter
@@ -152,13 +181,38 @@ In the 0011 CDK app, add:
 
 ### Step 3.5: Post-deploy — subscribe the ops topic (operational)
 
+> **Routing chosen + DEPLOYED (2026-07-08): AWS Chatbot → Slack, prices' OWN
+> channel `#stellar-prices-api-bot`.** BE has **no ops email** — it routes all its
+> CloudWatch alarms to Slack via an `AWS Chatbot SlackChannelConfiguration`
+> (`${env}-soroban-explorer-alarms` topic → Slack), with the workspace/channel
+> IDs in SSM (`/soroban-explorer/{env}/slack-{workspace,channel}-id`). We match
+> that mechanism but route to a **separate prices channel**, not BE's:
+> `opsAlarms.slack` in `production.json` wires a `SlackChannelConfiguration`
+> (`prices-{env}-ops-alarms`) on our ops topic, reading **prices-owned** SSM params
+> (`/prices/{env}/slack-{workspace,channel}-id`) per the SSM ownership split — the
+> workspace ID is the same shared workspace BE already authorized in Chatbot
+> (`T83HLEDJN`; only the channel differs, `C0BFWLMFQ9G`), so no re-authorization.
+> Prices owning its own params decouples us from BE's param lifecycle.
+> **Prereq:** the prices SSM params must exist before deploying Observability
+> (create them with `aws ssm put-parameter`; the deploy fails on the lookup if not).
+> **Status: deployed to production 2026-07-08 + smoke-tested** — forcing
+> `prices-production-ledger-processor-errors` into ALARM then OK posted **both** a
+> 🚨 breach and a ✅ recovery to `#stellar-prices-api-bot` (all 7 alarms carry both
+> `addAlarmAction` + `addOkAction` → ops topic, so recoveries notify too).
+> The email path (`notificationEmail` / `aws sns subscribe`) below remains
+> supported as a fallback but is not the chosen route.
+>
+> _Earlier draft (superseded same day): reuse BE's existing channel by reading
+> BE's `/soroban-explorer/...` params. Changed to a dedicated prices channel +
+> prices-owned params at the operator's request — keeps prices alarms off BE's
+> ops surface and off BE's param ownership._
+
 **By design, the `prices-{env}-ops-alarms` topic ships with no
-subscribers.** `config.opsAlarms.notificationEmail` is left unset in
+email subscribers.** `config.opsAlarms.notificationEmail` is left unset in
 `envs/production.json` on purpose: subscriptions are managed directly
 in SNS (email / Slack / PagerDuty) so the on-call roster can change
-without a redeploy. The consequence is that a fresh deploy routes every
-alarm to a topic **no one is listening to** until an operator subscribes
-an address — the alarms fire correctly but deliver nowhere.
+without a redeploy. With the Slack route above wired, the topic is no longer a
+black hole; the email checklist below is only needed if Slack is *not* used.
 
 Deploy checklist — **must be done once per env, immediately after the
 first deploy, before the fire-tests below can pass:**