feat(monitoring): add failure alerting and escalation chain for cron job failures

apps/backend/src/services/cron-failure-tracker.service.ts

@@ -0,0 +1,137 @@
+/**
+ * Cron Job Failure Tracker Service
+ *
+ * Persists per-job failure state in Supabase so the escalation chain
+ * survives server restarts.
+ *
+ * Escalation:
+ *   - 3 consecutive failures → Slack webhook alert (SLACK_WEBHOOK_URL)
+ *   - 6 consecutive failures → email alert (console.error [EMAIL_ALERT])
+ *   - Successful run        → resets consecutive_failures to 0
+ *
+ * Issue: #759
+ */
+
+import { NextRequest, NextResponse } from 'next/server';
+import { createClient } from '@/lib/supabase/server';
+
+const SLACK_ALERT_THRESHOLD = 3;
+const EMAIL_ALERT_THRESHOLD = 6;
+
+export class CronFailureTrackerService {
+    /**
+     * Record a successful cron run — clears the consecutive failure count.
+     */
+    async recordSuccess(jobName: string): Promise<void> {
+        const supabase = createClient();
+        await supabase.from('cron_job_failures').upsert(
+            {
+                job_name: jobName,
+                consecutive_failures: 0,
+                last_success_at: new Date().toISOString(),
+                last_error: null,
+                updated_at: new Date().toISOString(),
+            },
+            { onConflict: 'job_name' }
+        );
+    }
+
+    /**
+     * Record a failed cron run, increment the counter, and trigger alerts
+     * at the configured thresholds.
+     */
+    async recordFailure(jobName: string, error: string): Promise<void> {
+        const supabase = createClient();
+
+        // Fetch current count
+        const { data: existing } = await supabase
+            .from('cron_job_failures')
+            .select('consecutive_failures')
+            .eq('job_name', jobName)
+            .single();
+
+        const newCount = (existing?.consecutive_failures ?? 0) + 1;
+
+        await supabase.from('cron_job_failures').upsert(
+            {
+                job_name: jobName,
+                consecutive_failures: newCount,
+                last_failure_at: new Date().toISOString(),
+                last_error: error,
+                updated_at: new Date().toISOString(),
+            },
+            { onConflict: 'job_name' }
+        );
+
+        await this._escalate(jobName, newCount, error);
+    }
+
+    /**
+     * Wraps a cron route handler.
+     * Calls recordFailure on thrown exceptions or non-2xx responses,
+     * recordSuccess on 2xx responses.
+     */
+    wrapCronHandler(
+        jobName: string,
+        handler: (req: NextRequest) => Promise<NextResponse>
+    ): (req: NextRequest) => Promise<NextResponse> {
+        return async (req: NextRequest): Promise<NextResponse> => {
+            try {
+                const response = await handler(req);
+                if (response.status >= 400) {
+                    await this.recordFailure(jobName, `HTTP ${response.status}`);
+                } else {
+                    await this.recordSuccess(jobName);
+                }
+                return response;
+            } catch (err: unknown) {
+                const msg = err instanceof Error ? err.message : String(err);
+                await this.recordFailure(jobName, msg);
+                return NextResponse.json({ error: msg }, { status: 500 });
+            }
+        };
+    }
+
+    // ── Private ──────────────────────────────────────────────────────────────
+
+    private async _escalate(jobName: string, count: number, error: string): Promise<void> {
+        if (count === SLACK_ALERT_THRESHOLD) {
+            await this._sendSlackAlert(jobName, count, error);
+        } else if (count === EMAIL_ALERT_THRESHOLD) {
+            await this._sendSlackAlert(jobName, count, error);
+            this._sendEmailAlert(jobName, count, error);
+        }
+    }
+
+    private async _sendSlackAlert(
+        jobName: string,
+        count: number,
+        error: string
+    ): Promise<void> {
+        const url = process.env.SLACK_WEBHOOK_URL;
+        if (!url) return;
+
+        try {
+            await fetch(url, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                    text: `🚨 Cron job *${jobName}* has failed *${count}* consecutive times.\nError: ${error}`,
+                }),
+            });
+        } catch (err) {
+            console.error('[cron-failure-tracker] Failed to send Slack alert', err);
+        }
+    }
+
+    private _sendEmailAlert(jobName: string, count: number, error: string): void {
+        console.error('[CRON_EMAIL_ALERT]', {
+            jobName,
+            consecutiveFailures: count,
+            error,
+            timestamp: new Date().toISOString(),
+        });
+    }
+}
+
+export const cronFailureTrackerService = new CronFailureTrackerService();

apps/backend/tests/monitoring/cron-failure-tracker.test.ts

@@ -0,0 +1,172 @@
+// @vitest-environment node
+/**
+ * Tests for CronFailureTrackerService (#759)
+ *
+ * Covers:
+ *   - recordSuccess clears failure count
+ *   - recordFailure increments count
+ *   - Slack alert triggered at 3 consecutive failures
+ *   - Email alert triggered at 6 consecutive failures
+ *   - Auto-resolve: after success following failures, count resets
+ *   - wrapCronHandler calls recordFailure on exception
+ *   - wrapCronHandler calls recordSuccess on 200 response
+ */
+
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { NextRequest, NextResponse } from 'next/server';
+
+// ── Supabase mock ─────────────────────────────────────────────────────────────
+
+let mockConsecutiveFailures = 0;
+
+function makeSupabaseMock() {
+    return {
+        from: vi.fn().mockReturnValue({
+            upsert: vi.fn().mockResolvedValue({ error: null }),
+            select: vi.fn().mockReturnThis(),
+            eq: vi.fn().mockReturnThis(),
+            single: vi.fn().mockResolvedValue({
+                data: { consecutive_failures: mockConsecutiveFailures },
+                error: null,
+            }),
+        }),
+    };
+}
+
+vi.mock('@/lib/supabase/server', () => ({
+    createClient: vi.fn(),
+}));
+
+import { createClient } from '@/lib/supabase/server';
+const mockCreateClient = vi.mocked(createClient);
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe('CronFailureTrackerService', () => {
+    const ORIGINAL_ENV = { ...process.env };
+
+    beforeEach(() => {
+        vi.resetModules();
+        mockConsecutiveFailures = 0;
+        mockCreateClient.mockImplementation(() => makeSupabaseMock() as any);
+        process.env.SLACK_WEBHOOK_URL = 'https://hooks.slack.com/test';
+    });
+
+    afterEach(() => {
+        process.env = { ...ORIGINAL_ENV };
+        vi.restoreAllMocks();
+    });
+
+    async function load() {
+        const mod = await import('@/services/cron-failure-tracker.service');
+        return new mod.CronFailureTrackerService();
+    }
+
+    it('recordSuccess upserts with consecutive_failures = 0', async () => {
+        const svc = await load();
+        const mock = makeSupabaseMock();
+        mockCreateClient.mockReturnValue(mock as any);
+        await svc.recordSuccess('health-check');
+        expect(mock.from).toHaveBeenCalledWith('cron_job_failures');
+        const upsertCall = mock.from().upsert as any;
+        expect(upsertCall).toHaveBeenCalledWith(
+            expect.objectContaining({ consecutive_failures: 0, job_name: 'health-check' }),
+            { onConflict: 'job_name' }
+        );
+    });
+
+    it('recordFailure increments consecutive_failures', async () => {
+        mockConsecutiveFailures = 2;
+        const svc = await load();
+        const mock = makeSupabaseMock();
+        mockCreateClient.mockReturnValue(mock as any);
+
+        await svc.recordFailure('sync-status', 'timeout');
+
+        const upsertCall = mock.from().upsert as any;
+        expect(upsertCall).toHaveBeenCalledWith(
+            expect.objectContaining({ consecutive_failures: 3 }),
+            { onConflict: 'job_name' }
+        );
+    });
+
+    it('sends Slack alert at 3 consecutive failures', async () => {
+        mockConsecutiveFailures = 2; // next failure = 3
+        const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue(new Response());
+        const svc = await load();
+        await svc.recordFailure('health-check', 'boom');
+        expect(fetchSpy).toHaveBeenCalledWith(
+            'https://hooks.slack.com/test',
+            expect.objectContaining({ method: 'POST' })
+        );
+    });
+
+    it('does not send Slack alert below threshold', async () => {
+        mockConsecutiveFailures = 0; // next failure = 1
+        const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue(new Response());
+        const svc = await load();
+        await svc.recordFailure('health-check', 'err');
+        expect(fetchSpy).not.toHaveBeenCalled();
+    });
+
+    it('sends email (console.error) alert at 6 consecutive failures', async () => {
+        mockConsecutiveFailures = 5; // next failure = 6
+        vi.spyOn(global, 'fetch').mockResolvedValue(new Response());
+        const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+        const svc = await load();
+        await svc.recordFailure('health-check', 'critical');
+        expect(consoleSpy).toHaveBeenCalledWith(
+            '[CRON_EMAIL_ALERT]',
+            expect.objectContaining({ consecutiveFailures: 6 })
+        );
+    });
+
+    it('auto-resolves: recordSuccess after failures resets count', async () => {
+        mockConsecutiveFailures = 5;
+        const svc = await load();
+        const mock = makeSupabaseMock();
+        mockCreateClient.mockReturnValue(mock as any);
+        await svc.recordSuccess('health-check');
+        expect(mock.from().upsert).toHaveBeenCalledWith(
+            expect.objectContaining({ consecutive_failures: 0 }),
+            { onConflict: 'job_name' }
+        );
+    });
+
+    it('wrapCronHandler calls recordFailure on thrown exception', async () => {
+        const svc = await load();
+        const recordFailureSpy = vi.spyOn(svc, 'recordFailure').mockResolvedValue();
+        const badHandler = vi.fn().mockRejectedValue(new Error('crash'));
+        const wrapped = svc.wrapCronHandler('bad-job', badHandler);
+        const res = await wrapped(
+            new NextRequest('http://localhost/api/cron/bad-job', { method: 'GET' })
+        );
+        expect(res.status).toBe(500);
+        expect(recordFailureSpy).toHaveBeenCalledWith('bad-job', 'crash');
+    });
+
+    it('wrapCronHandler calls recordSuccess on 200 response', async () => {
+        const svc = await load();
+        const recordSuccessSpy = vi.spyOn(svc, 'recordSuccess').mockResolvedValue();
+        const goodHandler = vi.fn().mockResolvedValue(NextResponse.json({ ok: true }));
+        const wrapped = svc.wrapCronHandler('good-job', goodHandler);
+        const res = await wrapped(
+            new NextRequest('http://localhost/api/cron/good-job', { method: 'GET' })
+        );
+        expect(res.status).toBe(200);
+        expect(recordSuccessSpy).toHaveBeenCalledWith('good-job');
+    });
+
+    it('wrapCronHandler calls recordFailure on non-2xx response', async () => {
+        const svc = await load();
+        const recordFailureSpy = vi.spyOn(svc, 'recordFailure').mockResolvedValue();
+        const errHandler = vi.fn().mockResolvedValue(
+            NextResponse.json({ error: 'oops' }, { status: 500 })
+        );
+        const wrapped = svc.wrapCronHandler('err-job', errHandler);
+        await wrapped(
+            new NextRequest('http://localhost/api/cron/err-job', { method: 'GET' })
+        );
+        expect(recordFailureSpy).toHaveBeenCalledWith('err-job', 'HTTP 500');
+    });
+});

supabase/migrations/014_cron_job_failures.sql

@@ -0,0 +1,18 @@
+-- Migration 014: Cron Job Failure Tracking
+--
+-- Stores per-job failure state so the alerting system survives server restarts.
+-- Each row represents one cron job identified by a unique job_name.
+--
+-- Issue: #759 — Cron Job Failure Alerting System
+
+CREATE TABLE IF NOT EXISTS cron_job_failures (
+    id                  UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    job_name            TEXT NOT NULL UNIQUE,
+    consecutive_failures INT NOT NULL DEFAULT 0,
+    last_failure_at     TIMESTAMPTZ,
+    last_success_at     TIMESTAMPTZ,
+    last_error          TEXT,
+    updated_at          TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS cron_job_failures_job_name_idx ON cron_job_failures (job_name);