From 3befcee44b5e95b3fc2257b09b91fb03dbe7c96e Mon Sep 17 00:00:00 2001 From: Darkdruce Date: Fri, 26 Jun 2026 13:09:05 +0000 Subject: [PATCH] feat(monitoring): add failure alerting and escalation chain for cron job failures Closes #759 --- .../services/cron-failure-tracker.service.ts | 137 ++++++++++++++ .../monitoring/cron-failure-tracker.test.ts | 172 ++++++++++++++++++ supabase/migrations/014_cron_job_failures.sql | 18 ++ 3 files changed, 327 insertions(+) create mode 100644 apps/backend/src/services/cron-failure-tracker.service.ts create mode 100644 apps/backend/tests/monitoring/cron-failure-tracker.test.ts create mode 100644 supabase/migrations/014_cron_job_failures.sql diff --git a/apps/backend/src/services/cron-failure-tracker.service.ts b/apps/backend/src/services/cron-failure-tracker.service.ts new file mode 100644 index 00000000..5072e8cc --- /dev/null +++ b/apps/backend/src/services/cron-failure-tracker.service.ts @@ -0,0 +1,137 @@ +/** + * Cron Job Failure Tracker Service + * + * Persists per-job failure state in Supabase so the escalation chain + * survives server restarts. + * + * Escalation: + * - 3 consecutive failures → Slack webhook alert (SLACK_WEBHOOK_URL) + * - 6 consecutive failures → email alert (console.error [EMAIL_ALERT]) + * - Successful run → resets consecutive_failures to 0 + * + * Issue: #759 + */ + +import { NextRequest, NextResponse } from 'next/server'; +import { createClient } from '@/lib/supabase/server'; + +const SLACK_ALERT_THRESHOLD = 3; +const EMAIL_ALERT_THRESHOLD = 6; + +export class CronFailureTrackerService { + /** + * Record a successful cron run — clears the consecutive failure count. + */ + async recordSuccess(jobName: string): Promise { + const supabase = createClient(); + await supabase.from('cron_job_failures').upsert( + { + job_name: jobName, + consecutive_failures: 0, + last_success_at: new Date().toISOString(), + last_error: null, + updated_at: new Date().toISOString(), + }, + { onConflict: 'job_name' } + ); + } + + /** + * Record a failed cron run, increment the counter, and trigger alerts + * at the configured thresholds. + */ + async recordFailure(jobName: string, error: string): Promise { + const supabase = createClient(); + + // Fetch current count + const { data: existing } = await supabase + .from('cron_job_failures') + .select('consecutive_failures') + .eq('job_name', jobName) + .single(); + + const newCount = (existing?.consecutive_failures ?? 0) + 1; + + await supabase.from('cron_job_failures').upsert( + { + job_name: jobName, + consecutive_failures: newCount, + last_failure_at: new Date().toISOString(), + last_error: error, + updated_at: new Date().toISOString(), + }, + { onConflict: 'job_name' } + ); + + await this._escalate(jobName, newCount, error); + } + + /** + * Wraps a cron route handler. + * Calls recordFailure on thrown exceptions or non-2xx responses, + * recordSuccess on 2xx responses. + */ + wrapCronHandler( + jobName: string, + handler: (req: NextRequest) => Promise + ): (req: NextRequest) => Promise { + return async (req: NextRequest): Promise => { + try { + const response = await handler(req); + if (response.status >= 400) { + await this.recordFailure(jobName, `HTTP ${response.status}`); + } else { + await this.recordSuccess(jobName); + } + return response; + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + await this.recordFailure(jobName, msg); + return NextResponse.json({ error: msg }, { status: 500 }); + } + }; + } + + // ── Private ────────────────────────────────────────────────────────────── + + private async _escalate(jobName: string, count: number, error: string): Promise { + if (count === SLACK_ALERT_THRESHOLD) { + await this._sendSlackAlert(jobName, count, error); + } else if (count === EMAIL_ALERT_THRESHOLD) { + await this._sendSlackAlert(jobName, count, error); + this._sendEmailAlert(jobName, count, error); + } + } + + private async _sendSlackAlert( + jobName: string, + count: number, + error: string + ): Promise { + const url = process.env.SLACK_WEBHOOK_URL; + if (!url) return; + + try { + await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + text: `🚨 Cron job *${jobName}* has failed *${count}* consecutive times.\nError: ${error}`, + }), + }); + } catch (err) { + console.error('[cron-failure-tracker] Failed to send Slack alert', err); + } + } + + private _sendEmailAlert(jobName: string, count: number, error: string): void { + console.error('[CRON_EMAIL_ALERT]', { + jobName, + consecutiveFailures: count, + error, + timestamp: new Date().toISOString(), + }); + } +} + +export const cronFailureTrackerService = new CronFailureTrackerService(); diff --git a/apps/backend/tests/monitoring/cron-failure-tracker.test.ts b/apps/backend/tests/monitoring/cron-failure-tracker.test.ts new file mode 100644 index 00000000..0f241e5a --- /dev/null +++ b/apps/backend/tests/monitoring/cron-failure-tracker.test.ts @@ -0,0 +1,172 @@ +// @vitest-environment node +/** + * Tests for CronFailureTrackerService (#759) + * + * Covers: + * - recordSuccess clears failure count + * - recordFailure increments count + * - Slack alert triggered at 3 consecutive failures + * - Email alert triggered at 6 consecutive failures + * - Auto-resolve: after success following failures, count resets + * - wrapCronHandler calls recordFailure on exception + * - wrapCronHandler calls recordSuccess on 200 response + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { NextRequest, NextResponse } from 'next/server'; + +// ── Supabase mock ───────────────────────────────────────────────────────────── + +let mockConsecutiveFailures = 0; + +function makeSupabaseMock() { + return { + from: vi.fn().mockReturnValue({ + upsert: vi.fn().mockResolvedValue({ error: null }), + select: vi.fn().mockReturnThis(), + eq: vi.fn().mockReturnThis(), + single: vi.fn().mockResolvedValue({ + data: { consecutive_failures: mockConsecutiveFailures }, + error: null, + }), + }), + }; +} + +vi.mock('@/lib/supabase/server', () => ({ + createClient: vi.fn(), +})); + +import { createClient } from '@/lib/supabase/server'; +const mockCreateClient = vi.mocked(createClient); + +// ── Tests ───────────────────────────────────────────────────────────────────── + +describe('CronFailureTrackerService', () => { + const ORIGINAL_ENV = { ...process.env }; + + beforeEach(() => { + vi.resetModules(); + mockConsecutiveFailures = 0; + mockCreateClient.mockImplementation(() => makeSupabaseMock() as any); + process.env.SLACK_WEBHOOK_URL = 'https://hooks.slack.com/test'; + }); + + afterEach(() => { + process.env = { ...ORIGINAL_ENV }; + vi.restoreAllMocks(); + }); + + async function load() { + const mod = await import('@/services/cron-failure-tracker.service'); + return new mod.CronFailureTrackerService(); + } + + it('recordSuccess upserts with consecutive_failures = 0', async () => { + const svc = await load(); + const mock = makeSupabaseMock(); + mockCreateClient.mockReturnValue(mock as any); + await svc.recordSuccess('health-check'); + expect(mock.from).toHaveBeenCalledWith('cron_job_failures'); + const upsertCall = mock.from().upsert as any; + expect(upsertCall).toHaveBeenCalledWith( + expect.objectContaining({ consecutive_failures: 0, job_name: 'health-check' }), + { onConflict: 'job_name' } + ); + }); + + it('recordFailure increments consecutive_failures', async () => { + mockConsecutiveFailures = 2; + const svc = await load(); + const mock = makeSupabaseMock(); + mockCreateClient.mockReturnValue(mock as any); + + await svc.recordFailure('sync-status', 'timeout'); + + const upsertCall = mock.from().upsert as any; + expect(upsertCall).toHaveBeenCalledWith( + expect.objectContaining({ consecutive_failures: 3 }), + { onConflict: 'job_name' } + ); + }); + + it('sends Slack alert at 3 consecutive failures', async () => { + mockConsecutiveFailures = 2; // next failure = 3 + const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue(new Response()); + const svc = await load(); + await svc.recordFailure('health-check', 'boom'); + expect(fetchSpy).toHaveBeenCalledWith( + 'https://hooks.slack.com/test', + expect.objectContaining({ method: 'POST' }) + ); + }); + + it('does not send Slack alert below threshold', async () => { + mockConsecutiveFailures = 0; // next failure = 1 + const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue(new Response()); + const svc = await load(); + await svc.recordFailure('health-check', 'err'); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('sends email (console.error) alert at 6 consecutive failures', async () => { + mockConsecutiveFailures = 5; // next failure = 6 + vi.spyOn(global, 'fetch').mockResolvedValue(new Response()); + const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const svc = await load(); + await svc.recordFailure('health-check', 'critical'); + expect(consoleSpy).toHaveBeenCalledWith( + '[CRON_EMAIL_ALERT]', + expect.objectContaining({ consecutiveFailures: 6 }) + ); + }); + + it('auto-resolves: recordSuccess after failures resets count', async () => { + mockConsecutiveFailures = 5; + const svc = await load(); + const mock = makeSupabaseMock(); + mockCreateClient.mockReturnValue(mock as any); + await svc.recordSuccess('health-check'); + expect(mock.from().upsert).toHaveBeenCalledWith( + expect.objectContaining({ consecutive_failures: 0 }), + { onConflict: 'job_name' } + ); + }); + + it('wrapCronHandler calls recordFailure on thrown exception', async () => { + const svc = await load(); + const recordFailureSpy = vi.spyOn(svc, 'recordFailure').mockResolvedValue(); + const badHandler = vi.fn().mockRejectedValue(new Error('crash')); + const wrapped = svc.wrapCronHandler('bad-job', badHandler); + const res = await wrapped( + new NextRequest('http://localhost/api/cron/bad-job', { method: 'GET' }) + ); + expect(res.status).toBe(500); + expect(recordFailureSpy).toHaveBeenCalledWith('bad-job', 'crash'); + }); + + it('wrapCronHandler calls recordSuccess on 200 response', async () => { + const svc = await load(); + const recordSuccessSpy = vi.spyOn(svc, 'recordSuccess').mockResolvedValue(); + const goodHandler = vi.fn().mockResolvedValue(NextResponse.json({ ok: true })); + const wrapped = svc.wrapCronHandler('good-job', goodHandler); + const res = await wrapped( + new NextRequest('http://localhost/api/cron/good-job', { method: 'GET' }) + ); + expect(res.status).toBe(200); + expect(recordSuccessSpy).toHaveBeenCalledWith('good-job'); + }); + + it('wrapCronHandler calls recordFailure on non-2xx response', async () => { + const svc = await load(); + const recordFailureSpy = vi.spyOn(svc, 'recordFailure').mockResolvedValue(); + const errHandler = vi.fn().mockResolvedValue( + NextResponse.json({ error: 'oops' }, { status: 500 }) + ); + const wrapped = svc.wrapCronHandler('err-job', errHandler); + await wrapped( + new NextRequest('http://localhost/api/cron/err-job', { method: 'GET' }) + ); + expect(recordFailureSpy).toHaveBeenCalledWith('err-job', 'HTTP 500'); + }); +}); diff --git a/supabase/migrations/014_cron_job_failures.sql b/supabase/migrations/014_cron_job_failures.sql new file mode 100644 index 00000000..15e12d57 --- /dev/null +++ b/supabase/migrations/014_cron_job_failures.sql @@ -0,0 +1,18 @@ +-- Migration 014: Cron Job Failure Tracking +-- +-- Stores per-job failure state so the alerting system survives server restarts. +-- Each row represents one cron job identified by a unique job_name. +-- +-- Issue: #759 — Cron Job Failure Alerting System + +CREATE TABLE IF NOT EXISTS cron_job_failures ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + job_name TEXT NOT NULL UNIQUE, + consecutive_failures INT NOT NULL DEFAULT 0, + last_failure_at TIMESTAMPTZ, + last_success_at TIMESTAMPTZ, + last_error TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS cron_job_failures_job_name_idx ON cron_job_failures (job_name);