fix(release): self-heal a lost CDN rebuild and surface tap push errors

The CDN manifest every installed client polls was kept correct by a single
fire-and-forget webhook POST. On 0.18.0 that POST hit four consecutive 60s
connect timeouts, the job only warned, and the consistency gate then polled a
manifest nobody had asked to rebuild -- the release stayed invisible until a
human fired the webhook by hand.

The consistency poll now re-fires the deploy trigger every 8 attempts while the
CDN is behind, so a lost trigger heals inside the job that gates on it.

The Homebrew tap bump has failed twice with its real cause swallowed by
stdio: 'ignore'. Git's stderr now reaches the log, with the token redacted.

Author: elkaix

.changeset/reliable-release-channel.md

@@ -0,0 +1,5 @@
+---
+"@pymodel/pythinker-code": patch
+---
+
+Keep releases visible in the update channel when a CDN rebuild request is temporarily lost.

.github/workflows/release.yml

@@ -264,11 +264,14 @@ jobs:
           # nothing — an earlier version of this job was deleted because a
           # curl exit-28 timeout failed the 0.5.0 release. verify-cdn-release
           # polls the manifest and is the gate that fails loudly.
+          # On 0.18.0, one connect consumed the full 60-second budget. A short
+          # connect timeout turns the same wall-clock budget into more attempts
+          # during an outage instead of waiting on connections never made.
           status=$(curl -sS -o /dev/stderr -w '%{http_code}' -X POST "$WEBHOOK" \
             -H 'Content-Type: application/json' \
             -H 'X-GitHub-Event: push' \
             -d '{"ref":"refs/heads/main"}' \
-            --retry 3 --retry-all-errors --retry-delay 10 --max-time 60) || status=000
+            --connect-timeout 15 --max-time 45 --retry 5 --retry-all-errors --retry-delay 15) || status=000
           case "$status" in
             2*) echo "CDN redeploy triggered (HTTP $status)." ;;
             *) echo "::warning::CDN redeploy webhook returned HTTP $status — verify-cdn-release will catch a stale CDN." ;;
@@ -282,7 +285,7 @@ jobs:
   # publish hid the one case where the version and the published artifacts
   # diverge — and every client polled the CDN for a release that never existed.
   verify-cdn-release:
-    timeout-minutes: 15
+    timeout-minutes: 20
     name: Verify release consistency
     needs:
       - release
@@ -303,6 +306,8 @@ jobs:
           node-version-file: .nvmrc
 
       - name: Verify release consistency
+        env:
+          DOKPLOY_CDN_DEPLOY_WEBHOOK: ${{ secrets.DOKPLOY_CDN_DEPLOY_WEBHOOK }}
         run: node scripts/release/verify-release-consistency.mjs
 
   update-brew-tap:

apps/pythinker-code/test/scripts/release/cdn-consistency.test.ts

@@ -111,6 +111,125 @@ describe('pollCdnUntilCaughtUp', () => {
     expect(result).toMatchObject({ ok: true, reason: 'match', attempts: 3 });
   });
 
+  it('re-triggers on the configured cadence while the CDN is behind', async () => {
+    const { now, sleep } = fakeClock();
+    let retriggerCalls = 0;
+    const result = await pollCdnUntilCaughtUp({
+      ...base,
+      now,
+      sleep,
+      fetchImpl: scriptedFetch([
+        () => manifest('0.12.0'),
+        () => manifest('0.12.0'),
+        () => manifest('0.12.0'),
+        () => manifest('0.12.0'),
+        () => manifest('0.12.0'),
+        () => manifest('0.12.0'),
+        () => manifest('0.13.0'),
+      ]),
+      retrigger: async () => {
+        retriggerCalls += 1;
+      },
+      retriggerEveryAttempts: 3,
+    });
+
+    expect(retriggerCalls).toBe(2);
+    expect(result).toMatchObject({ ok: true, attempts: 7, retriggers: 2 });
+  });
+
+  it('does not re-trigger when the first attempt matches', async () => {
+    const { now, sleep } = fakeClock();
+    let retriggerCalls = 0;
+    const result = await pollCdnUntilCaughtUp({
+      ...base,
+      now,
+      sleep,
+      fetchImpl: scriptedFetch([() => manifest('0.13.0')]),
+      retrigger: async () => {
+        retriggerCalls += 1;
+      },
+      retriggerEveryAttempts: 1,
+    });
+
+    expect(retriggerCalls).toBe(0);
+    expect(result.retriggers).toBe(0);
+  });
+
+  it('does not re-trigger when the CDN is ahead', async () => {
+    const { now, sleep } = fakeClock();
+    let retriggerCalls = 0;
+    const result = await pollCdnUntilCaughtUp({
+      ...base,
+      now,
+      sleep,
+      fetchImpl: scriptedFetch([() => manifest('0.14.0')]),
+      retrigger: async () => {
+        retriggerCalls += 1;
+      },
+      retriggerEveryAttempts: 1,
+    });
+
+    expect(retriggerCalls).toBe(0);
+    expect(result).toMatchObject({ reason: 'ahead', retriggers: 0 });
+  });
+
+  it('keeps polling when a re-trigger throws', async () => {
+    const { now, sleep } = fakeClock();
+    let retriggerCalls = 0;
+    const result = await pollCdnUntilCaughtUp({
+      ...base,
+      now,
+      sleep,
+      fetchImpl: scriptedFetch([() => manifest('0.12.0'), () => manifest('0.13.0')]),
+      retrigger: async () => {
+        retriggerCalls += 1;
+        throw new Error('trigger failed');
+      },
+      retriggerEveryAttempts: 1,
+    });
+
+    expect(retriggerCalls).toBe(1);
+    expect(result).toMatchObject({ ok: true, attempts: 2, retriggers: 1 });
+  });
+
+  it('re-triggers while the CDN is unreachable', async () => {
+    const { now, sleep } = fakeClock();
+    let retriggerCalls = 0;
+    const result = await pollCdnUntilCaughtUp({
+      ...base,
+      now,
+      sleep,
+      fetchImpl: scriptedFetch([
+        () => {
+          throw new Error('ECONNREFUSED');
+        },
+        () => manifest('0.13.0'),
+      ]),
+      retrigger: async () => {
+        retriggerCalls += 1;
+      },
+      retriggerEveryAttempts: 1,
+    });
+
+    expect(retriggerCalls).toBe(1);
+    expect(result).toMatchObject({ ok: true, retriggers: 1 });
+  });
+
+  it('reports re-trigger attempts when the budget expires', async () => {
+    const { now, sleep } = fakeClock();
+    const result = await pollCdnUntilCaughtUp({
+      ...base,
+      budgetMs: 45_000,
+      now,
+      sleep,
+      fetchImpl: scriptedFetch([() => manifest('0.12.0')]),
+      retrigger: async () => {},
+      retriggerEveryAttempts: 1,
+    });
+
+    expect(result).toMatchObject({ reason: 'timeout', attempts: 3, retriggers: 2 });
+  });
+
   it('treats an unreachable CDN as lag rather than a failure', async () => {
     const { now, sleep } = fakeClock();
     const result = await pollCdnUntilCaughtUp({

scripts/release/cdn-consistency.mjs

@@ -67,12 +67,16 @@ async function readCdnVersion(fetchImpl, url) {
  * half-written, and both that and plain lag resolve by waiting, so only the
  * budget decides. 'ahead' returns at once — more waiting cannot fix a manifest
  * that names a release npm does not have.
+ *
+ * A periodic re-trigger heals a lost deploy request. This poll is the only
+ * pipeline stage that both knows the CDN is still behind and is still running.
  */
 export async function pollCdnUntilCaughtUp(options) {
-  const { fetchImpl, sleep, now, url, npmLatest, budgetMs, intervalMs } = options;
+  const { fetchImpl, sleep, now, url, npmLatest, budgetMs, intervalMs, retrigger, retriggerEveryAttempts } = options;
   const deadline = now() + budgetMs;
   let cdnVersion = null;
   let attempts = 0;
+  let retriggers = 0;
 
   for (;;) {
     attempts += 1;
@@ -87,12 +91,25 @@ export async function pollCdnUntilCaughtUp(options) {
       // Deliberately swallowed: an unreachable CDN is lag, not a gate failure.
     }
 
-    if (classification === 'match') return { ok: true, reason: 'match', cdnVersion, attempts };
-    if (classification === 'ahead') return { ok: false, reason: 'ahead', cdnVersion, attempts };
+    if (classification === 'match') return { ok: true, reason: 'match', cdnVersion, attempts, retriggers };
+    if (classification === 'ahead') return { ok: false, reason: 'ahead', cdnVersion, attempts, retriggers };
 
     // Stop before a sleep that would run past the budget rather than after it.
     if (now() + intervalMs >= deadline) {
-      return { ok: false, reason: 'timeout', cdnVersion, attempts };
+      return { ok: false, reason: 'timeout', cdnVersion, attempts, retriggers };
+    }
+    if (
+      typeof retrigger === 'function' &&
+      Number.isInteger(retriggerEveryAttempts) &&
+      retriggerEveryAttempts > 0 &&
+      attempts % retriggerEveryAttempts === 0
+    ) {
+      retriggers += 1;
+      try {
+        await retrigger();
+      } catch {
+        // Deliberately swallowed: a failed trigger is lag, not a gate failure.
+      }
     }
     await sleep(intervalMs);
   }

scripts/release/update-brew-formula.mjs

@@ -4,6 +4,11 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 
+function redactGitOutput(value, token) {
+  const redacted = String(value ?? '').replaceAll(/\/\/x-access-token:[^@\s]*@/gu, '//***@');
+  return token.length >= 8 ? redacted.replaceAll(token, '***') : redacted;
+}
+
 async function main() {
   const packageJson = JSON.parse(readFileSync(new URL('../../apps/pythinker-code/package.json', import.meta.url), 'utf8'));
   const version = packageJson.version;
@@ -22,10 +27,13 @@ async function main() {
   try {
     try {
       execFileSync('git', ['clone', `https://x-access-token:${token}@github.com/PyModel/homebrew-tap.git`, tapDir], {
-        stdio: 'ignore',
+        stdio: 'pipe',
       });
-    } catch {
-      throw new Error('Failed to clone Homebrew tap');
+    } catch (error) {
+      const stderr = redactGitOutput(error.stderr, token).trim();
+      const stdout = redactGitOutput(error.stdout, token).trim();
+      const message = redactGitOutput(error.message, token).trim();
+      throw new Error(`Failed to clone Homebrew tap: ${stderr || stdout || message}`, { cause: error });
     }
 
     const formulaPath = join(tapDir, 'Formula/pythinker-code.rb');
@@ -62,9 +70,12 @@ async function main() {
       { cwd: tapDir, stdio: 'inherit' },
     );
     try {
-      execFileSync('git', ['push', 'origin', 'main'], { cwd: tapDir, stdio: 'ignore' });
-    } catch {
-      throw new Error('Failed to push Homebrew tap');
+      execFileSync('git', ['push', 'origin', 'main'], { cwd: tapDir, stdio: 'pipe' });
+    } catch (error) {
+      const stderr = redactGitOutput(error.stderr, token).trim();
+      const stdout = redactGitOutput(error.stdout, token).trim();
+      const message = redactGitOutput(error.message, token).trim();
+      throw new Error(`Failed to push Homebrew tap: ${stderr || stdout || message}`, { cause: error });
     }
   } finally {
     rmSync(tapDir, { recursive: true, force: true });

scripts/release/verify-release-consistency.mjs

@@ -8,10 +8,9 @@ const SEMVER = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[
 
 const CDN_MANIFEST_URL = 'https://code.pythinker.com/pythinker-code/latest.json';
 
-// A Dokploy rebuild serves the new manifest in roughly two minutes. The budget
-// stays well under the job's own timeout-minutes so a stale CDN is reported
-// here rather than killed by the runner.
-const CDN_POLL_BUDGET_MS = 600_000;
+// The budget covers detecting a lost trigger and completing a fresh rebuild,
+// while staying under the job timeout so this gate can report a stale CDN.
+const CDN_POLL_BUDGET_MS = 900_000;
 const CDN_POLL_INTERVAL_MS = 15_000;
 
 function fail(reason) {
@@ -59,6 +58,34 @@ try {
 }
 if (!gitTags.trim().split('\n').includes(releaseTag)) fail(`missing git tag ${releaseTag}`);
 
+const webhook = process.env.DOKPLOY_CDN_DEPLOY_WEBHOOK;
+let retrigger;
+if (typeof webhook === 'string' && webhook.length > 0) {
+  if (!webhook.startsWith('https://')) {
+    console.error('warning: DOKPLOY_CDN_DEPLOY_WEBHOOK is not an https:// URL; CDN rebuild requests are disabled');
+  } else {
+    retrigger = async () => {
+      try {
+        const response = await fetch(webhook, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'X-GitHub-Event': 'push',
+          },
+          body: '{"ref":"refs/heads/main"}',
+          signal: AbortSignal.timeout(30_000),
+        });
+        if (!response.ok) throw new Error(`HTTP ${response.status}`);
+        console.log(`CDN rebuild request returned HTTP ${response.status}`);
+      } catch (error) {
+        const message = error instanceof Error ? error.message.replaceAll(webhook, '***') : 'unknown error';
+        console.error(`CDN rebuild request failed: ${message}`);
+        throw error;
+      }
+    };
+  }
+}
+
 // The CDN manifest is what every installed client polls for updates. A version
 // it advertises that npm does not have sends all of them into an install that
 // cannot succeed; a version it never catches up to hides the release entirely.
@@ -75,6 +102,8 @@ const cdnPoll = await pollCdnUntilCaughtUp({
   npmLatest: distTags.latest,
   budgetMs: CDN_POLL_BUDGET_MS,
   intervalMs: CDN_POLL_INTERVAL_MS,
+  retrigger,
+  retriggerEveryAttempts: 8,
 });
 
 if (cdnPoll.reason === 'ahead') {
@@ -87,11 +116,15 @@ if (!cdnPoll.ok) {
   fail(
     `CDN never caught up with npm within ${CDN_POLL_BUDGET_MS / 1000}s ` +
       `(cdn=${cdnPoll.cdnVersion ?? 'unreachable'} latest=${distTags.latest}, ` +
-      `${cdnPoll.attempts} attempts) — every installed client polls this manifest, ` +
+      `${cdnPoll.attempts} attempt(s), ${cdnPoll.retriggers} rebuild request(s)) — ` +
+      'every installed client polls this manifest, ' +
       'so the release stays invisible until the site rebuilds',
   );
 }
 
-console.log(`CDN matches npm (${cdnPoll.cdnVersion}) after ${cdnPoll.attempts} attempt(s)`);
+console.log(
+  `CDN matches npm (${cdnPoll.cdnVersion}) after ${cdnPoll.attempts} attempt(s), ` +
+    `${cdnPoll.retriggers} rebuild request(s)`,
+);
 
 console.log(`consistency OK: latest=${distTags.latest} beta=${distTags.beta ?? '-'} dev=${distTags.dev ?? '-'}`);