From 92eab0d67beffb99bd6c3461799bff9fcbaaff18 Mon Sep 17 00:00:00 2001 From: Mao Nakamoto Date: Sun, 30 Aug 2026 08:36:21 +0000 Subject: [PATCH] fix(frontier-digest): stop the proposals phase blocking the cron's HTTP response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fc-cron@frontier-digest.service was failing with curl exit 28 (120s timeout, 0 bytes received): the route awaited runFrontierProposals() synchronously, and that phase's model calls retry across a multi-vendor fallback chain (lib/groq.ts's fallback: true default), each burning its full per-link timeout on a degraded link — easily exceeding the cron wrapper's fixed `curl -m 120` even though ingest+digest (the public-facing half) finish in well under 30s. Confirmed live: production's /frontier page already showed today's real digest while the systemd unit still sat failed, proving the digest half was fast and the proposals phase was what blocked the response. Detach the proposals call (fire-and-forget into a new logProposalsOutcome helper) so the cron gets its response as soon as the digest is saved. Proposals still run to completion and get logged via logDebug exactly as before — just asynchronously, which is safe here since this is a persistent systemd-managed server, not serverless. No box-side change needed; deploying this alone fixes it. Co-Authored-By: Claude Sonnet 5 --- src/app/api/crons/frontier-digest/route.ts | 75 +++++++++++++--------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/src/app/api/crons/frontier-digest/route.ts b/src/app/api/crons/frontier-digest/route.ts index 7ba69885..f34d8ec4 100644 --- a/src/app/api/crons/frontier-digest/route.ts +++ b/src/app/api/crons/frontier-digest/route.ts @@ -10,7 +10,7 @@ import { type NextRequest, NextResponse } from "next/server"; import { requireCronAuth } from "@/lib/cron-auth"; import { logDebug } from "@/db/queries/debug-logs"; -import { runFrontierDigest, runFrontierProposals, type RunProposalsResult } from "@/lib/frontier/run"; +import { runFrontierDigest, runFrontierProposals, type RunFrontierResult, type RunProposalsResult } from "@/lib/frontier/run"; /** * How loud each generator outcome is. The distinction that matters: a fault in @@ -29,6 +29,30 @@ function outcomeLevel(p: RunProposalsResult): "info" | "warn" | "error" { return "info"; } +// Persist the proposals-phase outcome. Until this existed the whole diagnosis +// lived in the systemd journal, which on this box holds ONE day of this unit — +// so the loop could (and did) go two months surfacing nothing with no +// recoverable record of why. +async function logProposalsOutcome(r: RunFrontierResult, proposals: RunProposalsResult | { error: string }): Promise { + await logDebug({ + source: "crons/frontier-digest", + level: "error" in proposals ? "error" : outcomeLevel(proposals), + message: "error" in proposals + ? `frontier proposals THREW: ${proposals.error}` + : proposals.skipped + ? `no proposals attempted: ${proposals.skipped}` + : `generation=${proposals.generation} returned=${proposals.returned ?? 0} drafted=${proposals.drafted} surfaced=${proposals.surfaced}`, + meta: { + digestDate: r.saved.digestDate, + items: r.itemCount, + sourcesOk: r.sourcesOk, + sourcesFailed: r.sourcesFailed, + model: r.saved.model, + proposals, + }, + }); +} + // Ingestion fans out to several external feeds + a Groq call; give it room. export const maxDuration = 60; @@ -38,36 +62,25 @@ export async function GET(req: NextRequest) { try { const r = await runFrontierDigest(); - // Second half: draft + critique self-improvement proposals from the digest. - // Best-effort — a proposal failure must not fail the public digest. - let proposals: RunProposalsResult | { error: string }; - try { - proposals = await runFrontierProposals(r.saved); - } catch (e) { - proposals = { error: e instanceof Error ? e.message : "unknown" }; - } - // Persist the outcome. Until this existed the whole diagnosis lived in the - // systemd journal, which on this box holds ONE day of this unit — so the - // loop could (and did) go two months surfacing nothing with no recoverable - // record of why. The response body is written once and read by nobody. - await logDebug({ - source: "crons/frontier-digest", - level: "error" in proposals ? "error" : outcomeLevel(proposals), - message: "error" in proposals - ? `frontier proposals THREW: ${proposals.error}` - : proposals.skipped - ? `no proposals attempted: ${proposals.skipped}` - : `generation=${proposals.generation} returned=${proposals.returned ?? 0} drafted=${proposals.drafted} surfaced=${proposals.surfaced}`, - meta: { - digestDate: r.saved.digestDate, - items: r.itemCount, - sourcesOk: r.sourcesOk, - sourcesFailed: r.sourcesFailed, - model: r.saved.model, - proposals, - }, - }); + // Second half: draft + critique self-improvement proposals from the + // digest. Best-effort in two senses now — a proposal failure must not + // fail the public digest (unchanged), AND a slow model chain must not + // hold the cron's HTTP response hostage past fc-cron.sh's `curl -m 120`. + // generateProposals and each judge in the panel retry across a + // multi-vendor chain (lib/groq.ts's `fallback: true` default), burning + // their FULL per-link timeout on every degraded link before moving to + // the next — a single sluggish link can push this phase past two + // minutes on its own, even though ingest+digest above finish in well + // under 30s. Observed 2026-08-30: today's digest saved and served fine, + // but fc-cron@frontier-digest.service still failed on a 120s curl + // timeout with 0 bytes received — the response was still blocked on + // this phase. Not awaited: this is a long-running systemd process, not + // serverless, so the promise keeps running after the response is sent + // and logs its own outcome via logProposalsOutcome. + void runFrontierProposals(r.saved) + .catch((e): { error: string } => ({ error: e instanceof Error ? e.message : "unknown" })) + .then((proposals) => logProposalsOutcome(r, proposals)); return NextResponse.json({ ok: true, @@ -77,7 +90,7 @@ export async function GET(req: NextRequest) { sourcesOk: r.sourcesOk, sourcesFailed: r.sourcesFailed, model: r.saved.model, - proposals, + proposals: "queued — outcome logged separately (source=crons/frontier-digest)", }); } catch (err) { const message = err instanceof Error ? err.message : "unknown";