From 24345c0cfd98cb076307e8f2ca3d8dcbd45be7e4 Mon Sep 17 00:00:00 2001 From: karczuRF Date: Wed, 8 Jul 2026 01:36:18 +0200 Subject: [PATCH 1/2] feat(lore-0056): route ops alarms to Slack via Chatbot, reusing BE's channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BE has no ops email — it delivers all CloudWatch alarms to Slack through an AWS Chatbot SlackChannelConfiguration (its ${env}-soroban-explorer-alarms topic → Slack), with workspace/channel IDs in SSM. Match that instead of inventing an email/mailing list for prices. - Add optional opsAlarms.slack = { workspaceIdSsmParam, channelIdSsmParam } (+ validation) to the config type. - ObservabilityStack creates a SlackChannelConfiguration named prices-{env}-ops-alarms on the ops topic when the config is present, reading the workspace/channel IDs from SSM at deploy (kept out of this public repo) and attaching a chatbot IAM role with CloudWatchReadOnlyAccess (mirrors BE). - production.json points at BE's existing /soroban-explorer/production/slack-{workspace,channel}-id params in the shared account, so prices alarms land in BE's existing ops channel — one surface, no click-to-confirm, no new mailing list. Workspace already authorized in Chatbot for BE. - Config is optional/non-breaking: omit to leave the topic subscriber-less. The email path (notificationEmail / aws sns subscribe) stays as a fallback. Synth-verified (SlackChannelConfiguration prices-production-ops-alarms → OpsAlarmsTopic, SSM lookups for both IDs); tsc/eslint/prettier clean. --- infra/envs/production.json | 6 ++- infra/src/lib/stacks/observability-stack.ts | 50 +++++++++++++++++++ infra/src/lib/types.ts | 33 ++++++++++++ ...tch-alarms-push-freshness-mtls-notafter.md | 35 +++++++++++-- 4 files changed, 119 insertions(+), 5 deletions(-) diff --git a/infra/envs/production.json b/infra/envs/production.json index 33d2ffd..719b6a9 100644 --- a/infra/envs/production.json +++ b/infra/envs/production.json @@ -23,7 +23,11 @@ "opsAlarms": { "sdexPushFreshnessSeconds": 604800, "mtlsNotAfterDaysThreshold": 30, - "ledgerProcessorLagSeconds": 120 + "ledgerProcessorLagSeconds": 120, + "slack": { + "workspaceIdSsmParam": "/soroban-explorer/production/slack-workspace-id", + "channelIdSsmParam": "/soroban-explorer/production/slack-channel-id" + } }, "ledgerProcessor": { "memoryMb": 512, diff --git a/infra/src/lib/stacks/observability-stack.ts b/infra/src/lib/stacks/observability-stack.ts index 49c5850..213f93c 100644 --- a/infra/src/lib/stacks/observability-stack.ts +++ b/infra/src/lib/stacks/observability-stack.ts @@ -1,8 +1,11 @@ import * as cdk from 'aws-cdk-lib'; +import * as chatbot from 'aws-cdk-lib/aws-chatbot'; import * as cloudwatch from 'aws-cdk-lib/aws-cloudwatch'; import * as cw_actions from 'aws-cdk-lib/aws-cloudwatch-actions'; +import * as iam from 'aws-cdk-lib/aws-iam'; import * as sns from 'aws-cdk-lib/aws-sns'; import * as subscriptions from 'aws-cdk-lib/aws-sns-subscriptions'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; import type { Construct } from 'constructs'; import type { EnvironmentConfig } from '../types.js'; @@ -71,6 +74,11 @@ export class ObservabilityStack extends cdk.Stack { * `config.opsAlarms.notificationEmail`. */ public readonly opsAlarmsTopic: sns.Topic; + /** + * AWS Chatbot Slack subscription for the ops-alarms topic (task 0056). + * Created only when `config.opsAlarms.slack` is set; undefined otherwise. + */ + public readonly opsAlarmsSlackChannel?: chatbot.SlackChannelConfiguration; /** SDEX push-freshness alarm (§5.6 / Tranche-1 AC #5). */ public readonly sdexPushFreshnessAlarm: cloudwatch.Alarm; /** mTLS client-cert expiry alarm (§7 / §11.4). */ @@ -202,6 +210,48 @@ export class ObservabilityStack extends cdk.Stack { new subscriptions.EmailSubscription(config.opsAlarms.notificationEmail), ); } + + // AWS Chatbot → Slack (task 0056). When `opsAlarms.slack` is configured we + // route the ops-alarms topic to a Slack channel, matching how BE delivers + // its own CloudWatch alarms (its `${env}-soroban-explorer-alarms` topic → + // Chatbot → Slack) — the team's real ops surface, no email/mailing list. + // Reuses BE's channel: the workspace/channel IDs are read at deploy from the + // SSM params `opsAlarms.slack.{workspaceIdSsmParam,channelIdSsmParam}` (kept + // out of this public repo), which point at BE's existing + // `/soroban-explorer/{env}/slack-*` values in the shared account. The Slack + // workspace is already authorized in AWS Chatbot for BE, so no extra console + // step is needed here — the named SSM params just have to exist before + // deploying Observability. Omit the config to leave the topic + // subscriber-less (managed manually in SNS). + if (config.opsAlarms.slack) { + const slackWorkspaceId = ssm.StringParameter.valueForStringParameter( + this, + config.opsAlarms.slack.workspaceIdSsmParam, + ); + const slackChannelId = ssm.StringParameter.valueForStringParameter( + this, + config.opsAlarms.slack.channelIdSsmParam, + ); + this.opsAlarmsSlackChannel = new chatbot.SlackChannelConfiguration( + this, + 'OpsAlarmsSlackChannel', + { + slackChannelConfigurationName: opsAlarmsTopicName(config.envName), + slackWorkspaceId, + slackChannelId, + notificationTopics: [this.opsAlarmsTopic], + role: new iam.Role(this, 'OpsAlarmsChatbotRole', { + assumedBy: new iam.ServicePrincipal('chatbot.amazonaws.com'), + managedPolicies: [ + iam.ManagedPolicy.fromAwsManagedPolicyName( + 'CloudWatchReadOnlyAccess', + ), + ], + }), + }, + ); + } + const snsAction = new cw_actions.SnsAction(this.opsAlarmsTopic); // The enrichment backlog alarm (task 0026) shipped without an action; wire diff --git a/infra/src/lib/types.ts b/infra/src/lib/types.ts index 64b8c61..1bb474c 100644 --- a/infra/src/lib/types.ts +++ b/infra/src/lib/types.ts @@ -150,6 +150,25 @@ export interface EnvironmentConfig { * past it). */ readonly ledgerProcessorLagSeconds: number; + /** + * Optional AWS Chatbot → Slack routing for the ops-alarms topic (task 0056). + * When set, `ObservabilityStack` subscribes `prices-{env}-ops-alarms` to a + * Slack channel via a `SlackChannelConfiguration`, so alarms land in Slack — + * matching how BE routes its own CloudWatch alarms (no ops email/mailing + * list). Reuses BE's existing channel by reading the same SSM params + * (`/soroban-explorer/{env}/slack-{workspace,channel}-id`) in the shared + * account; the Slack workspace is already authorized in AWS Chatbot for BE. + * Omit to leave the topic subscriber-less (managed manually in SNS). + * + * Values are SSM Parameter *names* (plain String, not credentials), resolved + * at deploy — the workspace/channel IDs stay out of this public repo. The + * named params must exist before deploying Observability, else synth/deploy + * fails on the lookup. + */ + readonly slack?: { + readonly workspaceIdSsmParam: string; + readonly channelIdSsmParam: string; + }; }; // Ledger Processor ingest (consumed by IngestStack — task 0038) @@ -350,6 +369,20 @@ export function validateConfig(config: EnvironmentConfig): void { `opsAlarms.ledgerProcessorLagSeconds must be a positive integer (seconds), got: ${ops.ledgerProcessorLagSeconds}`, ); } + if (ops.slack !== undefined) { + const isSsmName = (v: unknown): boolean => + typeof v === 'string' && v.startsWith('/') && v.length > 1; + if ( + typeof ops.slack !== 'object' || + ops.slack === null || + !isSsmName(ops.slack.workspaceIdSsmParam) || + !isSsmName(ops.slack.channelIdSsmParam) + ) { + errors.push( + 'opsAlarms.slack, when set, must be { workspaceIdSsmParam, channelIdSsmParam } with absolute SSM parameter names (leading "/")', + ); + } + } // Require a real local@domain.tld shape, not just a stray '@': a value like // '@' or 'ops@' passes an includes('@') check but yields an undeliverable // SNS subscription (a silent notification black hole). diff --git a/lore/1-tasks/active/0056_FEATURE_cloudwatch-alarms-push-freshness-mtls-notafter.md b/lore/1-tasks/active/0056_FEATURE_cloudwatch-alarms-push-freshness-mtls-notafter.md index 18e955f..72b6939 100644 --- a/lore/1-tasks/active/0056_FEATURE_cloudwatch-alarms-push-freshness-mtls-notafter.md +++ b/lore/1-tasks/active/0056_FEATURE_cloudwatch-alarms-push-freshness-mtls-notafter.md @@ -65,6 +65,19 @@ history: + production.json. fmt/clippy/eslint/prettier clean. Task stays active: deploy Observability + ops-topic subscribe + both fire-tests remain operational. + - date: 2026-07-08 + status: active + who: okarcz + note: > + Ops routing decided + wired: AWS Chatbot → Slack, reusing BE's channel + (BE has no ops email — it routes CloudWatch alarms to Slack via Chatbot). + Added optional `opsAlarms.slack` config; ObservabilityStack now creates a + SlackChannelConfiguration (prices-{env}-ops-alarms) on the ops topic, + reading BE's existing `/soroban-explorer/{env}/slack-{workspace,channel}-id` + SSM params in the shared account. Synth-verified; tsc/eslint/prettier clean. + Supersedes the manual email-subscribe as the chosen route (email path kept + as fallback). Deploy of Observability (with the SSM params present) makes it + live. --- # CloudWatch alarms — SDEX push freshness + mTLS NotAfter @@ -152,13 +165,27 @@ In the 0011 CDK app, add: ### Step 3.5: Post-deploy — subscribe the ops topic (operational) +> **Routing chosen (2026-07-08): AWS Chatbot → Slack, reusing BE's channel +> (PR #97 follow-up).** BE has **no ops email** — it routes all its CloudWatch +> alarms to Slack via an `AWS Chatbot SlackChannelConfiguration` +> (`${env}-soroban-explorer-alarms` topic → Slack), with the workspace/channel +> IDs in SSM (`/soroban-explorer/{env}/slack-{workspace,channel}-id`). We now +> match that: `opsAlarms.slack` in `production.json` wires a +> `SlackChannelConfiguration` (`prices-{env}-ops-alarms`) on our ops topic, +> reading **BE's same SSM params** in the shared account, so prices alarms land +> in BE's existing ops channel — one surface, no email/mailing list, no +> click-to-confirm. The Slack workspace is already authorized in Chatbot for BE. +> **Prereq:** the named SSM params must exist before deploying Observability +> (they do — BE's alarms depend on them); the deploy fails on the lookup if not. +> The email path (`notificationEmail` / `aws sns subscribe`) below remains +> supported as a fallback but is not the chosen route. + **By design, the `prices-{env}-ops-alarms` topic ships with no -subscribers.** `config.opsAlarms.notificationEmail` is left unset in +email subscribers.** `config.opsAlarms.notificationEmail` is left unset in `envs/production.json` on purpose: subscriptions are managed directly in SNS (email / Slack / PagerDuty) so the on-call roster can change -without a redeploy. The consequence is that a fresh deploy routes every -alarm to a topic **no one is listening to** until an operator subscribes -an address — the alarms fire correctly but deliver nowhere. +without a redeploy. With the Slack route above wired, the topic is no longer a +black hole; the email checklist below is only needed if Slack is *not* used. Deploy checklist — **must be done once per env, immediately after the first deploy, before the fire-tests below can pass:** From bf2898b96d76e902fccbd1e2856324533595f753 Mon Sep 17 00:00:00 2001 From: karczuRF Date: Wed, 8 Jul 2026 13:07:10 +0200 Subject: [PATCH 2/2] feat(lore-0056): route ops alarms to prices' own Slack channel Point opsAlarms.slack at prices-owned SSM params (/prices/{env}/slack-{workspace,channel}-id) so alarms land in the dedicated #stellar-prices-api-bot channel rather than BE's ops channel, on the same shared Slack workspace (no Chatbot re-auth). Add addOkAction to all 7 alarms so recoveries notify alongside breaches. Deployed to production and smoke-tested: forcing ledger-processor-errors ALARM->OK posted both a breach and a recovery to the channel. --- infra/envs/production.json | 4 +- infra/src/lib/stacks/observability-stack.ts | 9 ++++ infra/src/lib/types.ts | 8 +-- ...tch-alarms-push-freshness-mtls-notafter.md | 49 ++++++++++++++----- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/infra/envs/production.json b/infra/envs/production.json index 719b6a9..b7123c8 100644 --- a/infra/envs/production.json +++ b/infra/envs/production.json @@ -25,8 +25,8 @@ "mtlsNotAfterDaysThreshold": 30, "ledgerProcessorLagSeconds": 120, "slack": { - "workspaceIdSsmParam": "/soroban-explorer/production/slack-workspace-id", - "channelIdSsmParam": "/soroban-explorer/production/slack-channel-id" + "workspaceIdSsmParam": "/prices/production/slack-workspace-id", + "channelIdSsmParam": "/prices/production/slack-channel-id" } }, "ledgerProcessor": { diff --git a/infra/src/lib/stacks/observability-stack.ts b/infra/src/lib/stacks/observability-stack.ts index 213f93c..d360b80 100644 --- a/infra/src/lib/stacks/observability-stack.ts +++ b/infra/src/lib/stacks/observability-stack.ts @@ -254,9 +254,12 @@ export class ObservabilityStack extends cdk.Stack { const snsAction = new cw_actions.SnsAction(this.opsAlarmsTopic); + // Every alarm gets both an ALARM and an OK action on the ops topic, so the + // Slack channel sees the recovery as well as the breach (task 0056). // The enrichment backlog alarm (task 0026) shipped without an action; wire // it to the ops topic now that 0056 owns the alarm-routing. this.enrichmentBacklogAlarm.addAlarmAction(snsAction); + this.enrichmentBacklogAlarm.addOkAction(snsAction); // SDEX push freshness (§5.6 / Tranche-1 AC #5). The backfill-freshness-probe // republishes `sdex_archive`'s push age (seconds) as Prices/Backfill @@ -290,6 +293,7 @@ export class ObservabilityStack extends cdk.Stack { }, ); this.sdexPushFreshnessAlarm.addAlarmAction(snsAction); + this.sdexPushFreshnessAlarm.addOkAction(snsAction); // mTLS cert expiry (§7 / §11.4). The mtls-notafter-probe publishes the // minimum days-to-NotAfter across the ingestion + api client certs; alarm @@ -313,6 +317,7 @@ export class ObservabilityStack extends cdk.Stack { treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, }); this.mtlsNotAfterAlarm.addAlarmAction(snsAction); + this.mtlsNotAfterAlarm.addOkAction(snsAction); // ----------------------------------------------------------------- // Live ledger-processor health (task 0056 finding B). The core ingestion @@ -368,6 +373,7 @@ export class ObservabilityStack extends cdk.Stack { }, ); this.ledgerProcessorLagAlarm.addAlarmAction(snsAction); + this.ledgerProcessorLagAlarm.addOkAction(snsAction); // Hard errors: the ledger-processor Lambda throwing (a crash, not a graceful // per-item batch failure). Any invocation error over 5 min pages. @@ -394,6 +400,7 @@ export class ObservabilityStack extends cdk.Stack { }, ); this.ledgerProcessorErrorAlarm.addAlarmAction(snsAction); + this.ledgerProcessorErrorAlarm.addOkAction(snsAction); // Poison-pill / permanent-failure doorbells: under reportBatchItemFailures a // handler that keeps failing one item re-drives it (no Lambda Error) until @@ -424,6 +431,7 @@ export class ObservabilityStack extends cdk.Stack { }, ); this.ledgerProcessorDlqAlarm.addAlarmAction(snsAction); + this.ledgerProcessorDlqAlarm.addOkAction(snsAction); // Total ingestion halt: the lag / errors / DLQ alarms above all key on the // *presence* of enqueued or failed messages, so a producer-side stop (BE's @@ -461,6 +469,7 @@ export class ObservabilityStack extends cdk.Stack { }, ); this.ledgerProcessorNoInvocationsAlarm.addAlarmAction(snsAction); + this.ledgerProcessorNoInvocationsAlarm.addOkAction(snsAction); new cdk.CfnOutput(this, 'OpsAlarmsTopicArn', { value: this.opsAlarmsTopic.topicArn, diff --git a/infra/src/lib/types.ts b/infra/src/lib/types.ts index 1bb474c..20f4aed 100644 --- a/infra/src/lib/types.ts +++ b/infra/src/lib/types.ts @@ -155,9 +155,11 @@ export interface EnvironmentConfig { * When set, `ObservabilityStack` subscribes `prices-{env}-ops-alarms` to a * Slack channel via a `SlackChannelConfiguration`, so alarms land in Slack — * matching how BE routes its own CloudWatch alarms (no ops email/mailing - * list). Reuses BE's existing channel by reading the same SSM params - * (`/soroban-explorer/{env}/slack-{workspace,channel}-id`) in the shared - * account; the Slack workspace is already authorized in AWS Chatbot for BE. + * list). Prices has its **own** channel (`#stellar-prices-api-bot`), not BE's, + * so these point at **prices-owned** params + * (`/prices/{env}/slack-{workspace,channel}-id`) per the SSM ownership split. + * The workspace ID is the same shared Slack workspace BE already authorized in + * AWS Chatbot (only the channel differs), so no re-authorization is needed. * Omit to leave the topic subscriber-less (managed manually in SNS). * * Values are SSM Parameter *names* (plain String, not credentials), resolved diff --git a/lore/1-tasks/active/0056_FEATURE_cloudwatch-alarms-push-freshness-mtls-notafter.md b/lore/1-tasks/active/0056_FEATURE_cloudwatch-alarms-push-freshness-mtls-notafter.md index 72b6939..6622c51 100644 --- a/lore/1-tasks/active/0056_FEATURE_cloudwatch-alarms-push-freshness-mtls-notafter.md +++ b/lore/1-tasks/active/0056_FEATURE_cloudwatch-alarms-push-freshness-mtls-notafter.md @@ -78,6 +78,22 @@ history: Supersedes the manual email-subscribe as the chosen route (email path kept as fallback). Deploy of Observability (with the SSM params present) makes it live. + - date: 2026-07-08 + status: active + who: okarcz + note: > + Slack routing DEPLOYED to production + verified. Changed target from BE's + shared channel to prices' OWN channel #stellar-prices-api-bot: opsAlarms.slack + now reads prices-owned SSM params /prices/production/slack-{workspace,channel}-id + (workspace T83HLEDJN — same shared workspace, so no Chatbot re-auth; channel + C0BFWLMFQ9G). Created the two SSM params, deployed Prices-production-Observability + (SlackChannelConfiguration + Chatbot role + 4 ledger-processor alarms all live). + Added addOkAction to all 7 alarms so recoveries notify too. Smoke-tested via + set-alarm-state on ledger-processor-errors: both a breach and a recovery landed + in #stellar-prices-api-bot. Verified config with `aws chatbot + describe-slack-channel-configurations --region us-east-2` (Chatbot API is + us-east-2-only). Task stays active: the freshness + mTLS fire-tests + + notes/G-alarm-fire-test.md remain operational. --- # CloudWatch alarms — SDEX push freshness + mTLS NotAfter @@ -165,20 +181,31 @@ In the 0011 CDK app, add: ### Step 3.5: Post-deploy — subscribe the ops topic (operational) -> **Routing chosen (2026-07-08): AWS Chatbot → Slack, reusing BE's channel -> (PR #97 follow-up).** BE has **no ops email** — it routes all its CloudWatch -> alarms to Slack via an `AWS Chatbot SlackChannelConfiguration` +> **Routing chosen + DEPLOYED (2026-07-08): AWS Chatbot → Slack, prices' OWN +> channel `#stellar-prices-api-bot`.** BE has **no ops email** — it routes all its +> CloudWatch alarms to Slack via an `AWS Chatbot SlackChannelConfiguration` > (`${env}-soroban-explorer-alarms` topic → Slack), with the workspace/channel -> IDs in SSM (`/soroban-explorer/{env}/slack-{workspace,channel}-id`). We now -> match that: `opsAlarms.slack` in `production.json` wires a -> `SlackChannelConfiguration` (`prices-{env}-ops-alarms`) on our ops topic, -> reading **BE's same SSM params** in the shared account, so prices alarms land -> in BE's existing ops channel — one surface, no email/mailing list, no -> click-to-confirm. The Slack workspace is already authorized in Chatbot for BE. -> **Prereq:** the named SSM params must exist before deploying Observability -> (they do — BE's alarms depend on them); the deploy fails on the lookup if not. +> IDs in SSM (`/soroban-explorer/{env}/slack-{workspace,channel}-id`). We match +> that mechanism but route to a **separate prices channel**, not BE's: +> `opsAlarms.slack` in `production.json` wires a `SlackChannelConfiguration` +> (`prices-{env}-ops-alarms`) on our ops topic, reading **prices-owned** SSM params +> (`/prices/{env}/slack-{workspace,channel}-id`) per the SSM ownership split — the +> workspace ID is the same shared workspace BE already authorized in Chatbot +> (`T83HLEDJN`; only the channel differs, `C0BFWLMFQ9G`), so no re-authorization. +> Prices owning its own params decouples us from BE's param lifecycle. +> **Prereq:** the prices SSM params must exist before deploying Observability +> (create them with `aws ssm put-parameter`; the deploy fails on the lookup if not). +> **Status: deployed to production 2026-07-08 + smoke-tested** — forcing +> `prices-production-ledger-processor-errors` into ALARM then OK posted **both** a +> 🚨 breach and a ✅ recovery to `#stellar-prices-api-bot` (all 7 alarms carry both +> `addAlarmAction` + `addOkAction` → ops topic, so recoveries notify too). > The email path (`notificationEmail` / `aws sns subscribe`) below remains > supported as a fallback but is not the chosen route. +> +> _Earlier draft (superseded same day): reuse BE's existing channel by reading +> BE's `/soroban-explorer/...` params. Changed to a dedicated prices channel + +> prices-owned params at the operator's request — keeps prices alarms off BE's +> ops surface and off BE's param ownership._ **By design, the `prices-{env}-ops-alarms` topic ships with no email subscribers.** `config.opsAlarms.notificationEmail` is left unset in