stop paging a human about upstream flapping

.github/workflows/fetch-deepseek-docs.yml

@@ -31,16 +31,32 @@ jobs:
       - name: Fetch docs
         run: uv run scripts/fetcher.py
 
+      # Two different questions. "Did the tree move?" decides whether there
+      # is anything to commit. "Is there news?" decides whether to spend an
+      # agent call and wake a phone -- and those came apart badly: 19 of the
+      # last 30 commits here changed nothing but content/.metadata.json,
+      # where a page's entry flips between a title and a transient "served
+      # fallback shell" error as upstream wobbles. Every one of those flips
+      # ran triage, committed, and pushed "DeepSeek docs updated" to Eric's
+      # phone for zero documentation changes. A notification channel that
+      # cries wolf 19 times in 30 is not a notification channel.
       - name: Detect changes
         id: diff
         run: |
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
           if [ -z "$(git status --porcelain)" ]; then
             echo "changed=false" >> "$GITHUB_OUTPUT"
+            echo "newsworthy=false" >> "$GITHUB_OUTPUT"
             echo "Mirror is up to date."
+          elif [ -z "$(git status --porcelain -- ':!content/.metadata.json')" ]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            echo "newsworthy=false" >> "$GITHUB_OUTPUT"
+            echo "Bookkeeping only: fetch-state metadata moved, no content changed."
+            git status --short | head -30
           else
             echo "changed=true" >> "$GITHUB_OUTPUT"
+            echo "newsworthy=true" >> "$GITHUB_OUTPUT"
             git status --short | head -30
           fi
 
@@ -50,7 +66,7 @@ jobs:
       # PR) lives in the deterministic Publish step below. Runs Claude Code
       # on the DeepSeek API via the Anthropic-compatible endpoint.
       - name: Agent triage (deepseek-v4-flash)
-        if: steps.diff.outputs.changed == 'true'
+        if: steps.diff.outputs.newsworthy == 'true'
         uses: anthropics/claude-code-action@v1.0.168
         env:
           ANTHROPIC_BASE_URL: https://api.deepseek.com/anthropic
@@ -118,6 +134,7 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           BARK_SERVER: ${{ secrets.BARK_SERVER }}
           BARK_KEY: ${{ secrets.BARK_KEY }}
+          NEWSWORTHY: ${{ steps.diff.outputs.newsworthy }}
         run: |
           set -euo pipefail
           D=.decision
@@ -126,13 +143,28 @@ jobs:
           # pages are captured from the dirty tree before anything commits.
           PAGES=$(git status --porcelain content/ | awk '{print $NF}' | head -12)
           notify() {  # $1 = link target (commit or PR)
+            # Bookkeeping commits get no push. The mirror stays honest in
+            # git; the phone only rings when a document actually changed.
+            [ "${NEWSWORTHY:-true}" = "true" ] || return 0
             [ -n "${BARK_SERVER:-}" ] && [ -n "${BARK_KEY:-}" ] || return 0
             jq -n --arg k "$BARK_KEY" --arg t "DeepSeek docs updated" \
               --arg b "${TITLE}"$'\n\n'"${PAGES}" --arg u "$1" \
               '{device_key:$k, title:$t, body:$b, url:$u, group:"DeepSeek Docs"}' \
             | curl -sS -X POST "$BARK_SERVER/push" -H 'Content-Type: application/json' -d @- \
             || echo "::warning::bark notification failed (sync itself is fine)"
           }
+          # No triage ran for a metadata-only run, so do not go looking for
+          # a decision and do not warn about its absence -- that is the
+          # expected path now, not a degraded one.
+          if [ "${NEWSWORTHY:-true}" != "true" ]; then
+            git add -A
+            git commit -q -m "chore: refresh upstream fetch-state metadata" \
+              -m "No document content changed; content/.metadata.json only."
+            git push -q
+            echo "Bookkeeping commit pushed; no triage, no notification."
+            exit 0
+          fi
+
           SIGNAL=$(tr -d '[:space:]' < "$D/signal" 2>/dev/null || echo "")
           TITLE=$(head -1 "$D/title" 2>/dev/null || echo "")
           if [ -z "$SIGNAL" ] || [ -z "$TITLE" ]; then
@@ -203,6 +235,25 @@ jobs:
             exit 1
           fi
 
+      # The question a green pipeline and a fresh main cannot answer: not
+      # "is the mirror updating?" but "is anything in it no longer being
+      # fetched?" Aggregate freshness hides a single page that stopped
+      # coming back, because the other 273 keep moving. Runs every time,
+      # including on no-change runs, since a page going dead does not
+      # announce itself with a diff.
+      - name: Sources that stopped fetching
+        if: success()
+        run: |
+          out=$(python3 scripts/stale_sources.py) || true
+          if [ -n "$out" ]; then
+            echo "::warning::sources unfetched for 7+ days -- the mirror is holding fossils"
+            printf '%s\n' "$out"
+            { echo "### Sources that stopped fetching"; echo '```'; printf '%s\n' "$out"; echo '```'; } \
+              >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "every source fetched within the last 7 days"
+          fi
+
       # Same pattern as claude-code-docs: a failed sync run pushes to the
       # phone instead of rotting silently in the Actions tab.
       - name: Alert on failure

scripts/fetcher.py

@@ -460,14 +460,31 @@ def build_llms_full() -> None:
 
 
 def write_manifest(results: dict) -> None:
+    """Persist per-page fetch state, and remember how long an error has held.
+
+    Upstream serves a fallback shell for a page now and then, so a page's
+    entry flips error -> title -> error across runs. That flapping is not
+    news, but the manifest is the only record of it, so a page that goes
+    dead for good looks exactly like a page having a bad afternoon.
+    `error_since` is what tells them apart: set on the first failing run,
+    carried across consecutive failures, dropped the moment the page comes
+    back. news250120 has carried one since 2026-08-08 and nobody noticed,
+    because 19 of the last 30 commits here were this file flapping.
+    """
     manifest_path = CONTENT / ".metadata.json"
     manifest = {}
     if manifest_path.exists():
         manifest = json.loads(manifest_path.read_text())
+    previous = manifest.get("files", {})
     any_changed = any(r.get("changed") for r in results.values())
-    manifest.setdefault("files", {}).update(
-        {k: {kk: vv for kk, vv in v.items() if kk != "changed"}
-         for k, v in results.items()})
+    today = date.today().isoformat()
+    entries = {}
+    for k, v in results.items():
+        entry = {kk: vv for kk, vv in v.items() if kk != "changed"}
+        if "error" in entry:
+            entry["error_since"] = previous.get(k, {}).get("error_since", today)
+        entries[k] = entry
+    manifest.setdefault("files", {}).update(entries)
     if any_changed or "updated" not in manifest:
         manifest["updated"] = date.today().isoformat()
     manifest["site"] = SITE

scripts/stale_sources.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Report pages this mirror still holds but no longer successfully fetches.
+
+A mirror can commit every six hours, keep a green pipeline and a fresh
+`main`, and still be quietly wrong: the aggregate looks alive because most
+pages update, while one page has been serving a fallback shell for weeks
+and the copy in the repo is a fossil nobody flagged. Freshness of the whole
+cannot see staleness of a part.
+
+Upstream flaps, so a single failing run means nothing. `error_since` in
+content/.metadata.json records when a page's error first appeared and
+survives only while it keeps failing; anything holding one past the
+threshold has stopped being a blip and become a fact about the mirror.
+
+The threshold is measured, not guessed. Replaying every manifest commit
+from 2026-08-02 to 09-02 for the three flappiest pages (news250120 en and
+zh-cn, news1226 en) gives error spells of 0, 1, 3, 4 and 5 days -- 27
+spells, longest 5. Seven days would have fired zero times across that
+month, so it stays silent on upstream having a bad week and speaks only
+when a page has actually stopped coming back.
+
+Exit 0 and print nothing when every source is healthy.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import date
+from pathlib import Path
+
+MANIFEST = Path(__file__).resolve().parent.parent / "content" / ".metadata.json"
+DEFAULT_DAYS = 7
+
+
+def stale(manifest: dict, days: int, today: date) -> list[tuple[int, str, str]]:
+    out = []
+    for path, meta in sorted(manifest.get("files", {}).items()):
+        if not isinstance(meta, dict) or "error" not in meta:
+            continue
+        since = meta.get("error_since")
+        if not since:
+            continue
+        try:
+            age = (today - date.fromisoformat(since)).days
+        except ValueError:
+            continue
+        if age >= days:
+            out.append((age, path, meta["error"]))
+    return sorted(out, reverse=True)
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--days", type=int, default=DEFAULT_DAYS,
+                    help=f"flag errors at least this old (default {DEFAULT_DAYS})")
+    ap.add_argument("--manifest", type=Path, default=MANIFEST)
+    ap.add_argument("--github-output", action="store_true",
+                    help="also emit stale=<count> for GITHUB_OUTPUT")
+    args = ap.parse_args()
+
+    if not args.manifest.exists():
+        print(f"no manifest at {args.manifest}", file=sys.stderr)
+        return 0
+    manifest = json.loads(args.manifest.read_text())
+    rows = stale(manifest, args.days, date.today())
+
+    for age, path, err in rows:
+        print(f"{age:>4}d  {path}\n        {err}")
+    if args.github_output:
+        print(f"stale={len(rows)}")
+    if rows:
+        print(f"\n{len(rows)} source(s) unfetched for {args.days}+ days",
+              file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())