diff --git a/.github/workflows/fetch-deepseek-docs.yml b/.github/workflows/fetch-deepseek-docs.yml index 2ece7b1..651c878 100644 --- a/.github/workflows/fetch-deepseek-docs.yml +++ b/.github/workflows/fetch-deepseek-docs.yml @@ -31,6 +31,15 @@ jobs: - name: Fetch docs run: uv run scripts/fetcher.py + # Two different questions. "Did the tree move?" decides whether there + # is anything to commit. "Is there news?" decides whether to spend an + # agent call and wake a phone -- and those came apart badly: 19 of the + # last 30 commits here changed nothing but content/.metadata.json, + # where a page's entry flips between a title and a transient "served + # fallback shell" error as upstream wobbles. Every one of those flips + # ran triage, committed, and pushed "DeepSeek docs updated" to Eric's + # phone for zero documentation changes. A notification channel that + # cries wolf 19 times in 30 is not a notification channel. - name: Detect changes id: diff run: | @@ -38,9 +47,16 @@ jobs: git config user.email "github-actions[bot]@users.noreply.github.com" if [ -z "$(git status --porcelain)" ]; then echo "changed=false" >> "$GITHUB_OUTPUT" + echo "newsworthy=false" >> "$GITHUB_OUTPUT" echo "Mirror is up to date." + elif [ -z "$(git status --porcelain -- ':!content/.metadata.json')" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "newsworthy=false" >> "$GITHUB_OUTPUT" + echo "Bookkeeping only: fetch-state metadata moved, no content changed." + git status --short | head -30 else echo "changed=true" >> "$GITHUB_OUTPUT" + echo "newsworthy=true" >> "$GITHUB_OUTPUT" git status --short | head -30 fi @@ -50,7 +66,7 @@ jobs: # PR) lives in the deterministic Publish step below. Runs Claude Code # on the DeepSeek API via the Anthropic-compatible endpoint. - name: Agent triage (deepseek-v4-flash) - if: steps.diff.outputs.changed == 'true' + if: steps.diff.outputs.newsworthy == 'true' uses: anthropics/claude-code-action@v1.0.168 env: ANTHROPIC_BASE_URL: https://api.deepseek.com/anthropic @@ -118,6 +134,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} BARK_SERVER: ${{ secrets.BARK_SERVER }} BARK_KEY: ${{ secrets.BARK_KEY }} + NEWSWORTHY: ${{ steps.diff.outputs.newsworthy }} run: | set -euo pipefail D=.decision @@ -126,6 +143,9 @@ jobs: # pages are captured from the dirty tree before anything commits. PAGES=$(git status --porcelain content/ | awk '{print $NF}' | head -12) notify() { # $1 = link target (commit or PR) + # Bookkeeping commits get no push. The mirror stays honest in + # git; the phone only rings when a document actually changed. + [ "${NEWSWORTHY:-true}" = "true" ] || return 0 [ -n "${BARK_SERVER:-}" ] && [ -n "${BARK_KEY:-}" ] || return 0 jq -n --arg k "$BARK_KEY" --arg t "DeepSeek docs updated" \ --arg b "${TITLE}"$'\n\n'"${PAGES}" --arg u "$1" \ @@ -133,6 +153,18 @@ jobs: | curl -sS -X POST "$BARK_SERVER/push" -H 'Content-Type: application/json' -d @- \ || echo "::warning::bark notification failed (sync itself is fine)" } + # No triage ran for a metadata-only run, so do not go looking for + # a decision and do not warn about its absence -- that is the + # expected path now, not a degraded one. + if [ "${NEWSWORTHY:-true}" != "true" ]; then + git add -A + git commit -q -m "chore: refresh upstream fetch-state metadata" \ + -m "No document content changed; content/.metadata.json only." + git push -q + echo "Bookkeeping commit pushed; no triage, no notification." + exit 0 + fi + SIGNAL=$(tr -d '[:space:]' < "$D/signal" 2>/dev/null || echo "") TITLE=$(head -1 "$D/title" 2>/dev/null || echo "") if [ -z "$SIGNAL" ] || [ -z "$TITLE" ]; then @@ -203,6 +235,25 @@ jobs: exit 1 fi + # The question a green pipeline and a fresh main cannot answer: not + # "is the mirror updating?" but "is anything in it no longer being + # fetched?" Aggregate freshness hides a single page that stopped + # coming back, because the other 273 keep moving. Runs every time, + # including on no-change runs, since a page going dead does not + # announce itself with a diff. + - name: Sources that stopped fetching + if: success() + run: | + out=$(python3 scripts/stale_sources.py) || true + if [ -n "$out" ]; then + echo "::warning::sources unfetched for 7+ days -- the mirror is holding fossils" + printf '%s\n' "$out" + { echo "### Sources that stopped fetching"; echo '```'; printf '%s\n' "$out"; echo '```'; } \ + >> "$GITHUB_STEP_SUMMARY" + else + echo "every source fetched within the last 7 days" + fi + # Same pattern as claude-code-docs: a failed sync run pushes to the # phone instead of rotting silently in the Actions tab. - name: Alert on failure diff --git a/scripts/fetcher.py b/scripts/fetcher.py index f9a5a9e..02bca51 100644 --- a/scripts/fetcher.py +++ b/scripts/fetcher.py @@ -460,14 +460,31 @@ def build_llms_full() -> None: def write_manifest(results: dict) -> None: + """Persist per-page fetch state, and remember how long an error has held. + + Upstream serves a fallback shell for a page now and then, so a page's + entry flips error -> title -> error across runs. That flapping is not + news, but the manifest is the only record of it, so a page that goes + dead for good looks exactly like a page having a bad afternoon. + `error_since` is what tells them apart: set on the first failing run, + carried across consecutive failures, dropped the moment the page comes + back. news250120 has carried one since 2026-08-08 and nobody noticed, + because 19 of the last 30 commits here were this file flapping. + """ manifest_path = CONTENT / ".metadata.json" manifest = {} if manifest_path.exists(): manifest = json.loads(manifest_path.read_text()) + previous = manifest.get("files", {}) any_changed = any(r.get("changed") for r in results.values()) - manifest.setdefault("files", {}).update( - {k: {kk: vv for kk, vv in v.items() if kk != "changed"} - for k, v in results.items()}) + today = date.today().isoformat() + entries = {} + for k, v in results.items(): + entry = {kk: vv for kk, vv in v.items() if kk != "changed"} + if "error" in entry: + entry["error_since"] = previous.get(k, {}).get("error_since", today) + entries[k] = entry + manifest.setdefault("files", {}).update(entries) if any_changed or "updated" not in manifest: manifest["updated"] = date.today().isoformat() manifest["site"] = SITE diff --git a/scripts/stale_sources.py b/scripts/stale_sources.py new file mode 100755 index 0000000..a7feb65 --- /dev/null +++ b/scripts/stale_sources.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Report pages this mirror still holds but no longer successfully fetches. + +A mirror can commit every six hours, keep a green pipeline and a fresh +`main`, and still be quietly wrong: the aggregate looks alive because most +pages update, while one page has been serving a fallback shell for weeks +and the copy in the repo is a fossil nobody flagged. Freshness of the whole +cannot see staleness of a part. + +Upstream flaps, so a single failing run means nothing. `error_since` in +content/.metadata.json records when a page's error first appeared and +survives only while it keeps failing; anything holding one past the +threshold has stopped being a blip and become a fact about the mirror. + +The threshold is measured, not guessed. Replaying every manifest commit +from 2026-08-02 to 09-02 for the three flappiest pages (news250120 en and +zh-cn, news1226 en) gives error spells of 0, 1, 3, 4 and 5 days -- 27 +spells, longest 5. Seven days would have fired zero times across that +month, so it stays silent on upstream having a bad week and speaks only +when a page has actually stopped coming back. + +Exit 0 and print nothing when every source is healthy. +""" +from __future__ import annotations + +import argparse +import json +import sys +from datetime import date +from pathlib import Path + +MANIFEST = Path(__file__).resolve().parent.parent / "content" / ".metadata.json" +DEFAULT_DAYS = 7 + + +def stale(manifest: dict, days: int, today: date) -> list[tuple[int, str, str]]: + out = [] + for path, meta in sorted(manifest.get("files", {}).items()): + if not isinstance(meta, dict) or "error" not in meta: + continue + since = meta.get("error_since") + if not since: + continue + try: + age = (today - date.fromisoformat(since)).days + except ValueError: + continue + if age >= days: + out.append((age, path, meta["error"])) + return sorted(out, reverse=True) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--days", type=int, default=DEFAULT_DAYS, + help=f"flag errors at least this old (default {DEFAULT_DAYS})") + ap.add_argument("--manifest", type=Path, default=MANIFEST) + ap.add_argument("--github-output", action="store_true", + help="also emit stale= for GITHUB_OUTPUT") + args = ap.parse_args() + + if not args.manifest.exists(): + print(f"no manifest at {args.manifest}", file=sys.stderr) + return 0 + manifest = json.loads(args.manifest.read_text()) + rows = stale(manifest, args.days, date.today()) + + for age, path, err in rows: + print(f"{age:>4}d {path}\n {err}") + if args.github_output: + print(f"stale={len(rows)}") + if rows: + print(f"\n{len(rows)} source(s) unfetched for {args.days}+ days", + file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main())