stats pipeline: named agents and scanner probe families

.github/workflows/branch.yml

@@ -175,6 +175,7 @@ jobs:
     env:
       GOLF_HUB_TEST_DB_URL: postgresql://moonbase_test:moonbase_test@postgres:5432/moonbase_test
       PG_TEST_DB_URL: postgresql://moonbase_test:moonbase_test@postgres:5432/moonbase_test
+      STATS_TEST_DB_URL: postgresql://moonbase_test:moonbase_test@postgres:5432/moonbase_test
 
     steps:
       - uses: actions/checkout@v7

domains/platform/apis/stats/BUILD.bazel

@@ -20,6 +20,8 @@ go_library(
 
 # store_test.go skips without STATS_TEST_DB_URL, like the repo's other
 # Postgres-gated suites — a local green run may have exercised no SQL.
+# CI supplies it from the postgres service; without env_inherit the
+# sandbox strips it and the suite skips there too.
 go_test(
     name = "stats_test",
     size = "small",
@@ -31,6 +33,11 @@ go_test(
         "store_test.go",
     ],
     embed = [":stats_lib"],
+    env_inherit = ["STATS_TEST_DB_URL"],
+    # One shared scratch database, and the version-bump test drops every
+    # aggregate table: concurrent runs would pull the tables out from under
+    # each other mid-flight.
+    tags = ["exclusive"],
 )
 
 go_binary(

domains/platform/apis/stats/README.md

@@ -16,18 +16,54 @@ conflict arm makes a duplicate application a no-op — so counts survive
 crashes without double-counting. Per-object failures are logged and
 retried next pass.
 
-Aggregates are bounded on purpose: hosts are Caddy's vhosts, methods
-collapse through the nine-verb rule the metrics rails use, user agents
-collapse to four classes (`ai_scraper`, `bot`, `browser`, `other`), and
-iili slugs are the one caller-shaped key — one path segment, max 64
-bytes, only on the redirect routes. The raw lines stay in S3, so a
-better classifier is a re-aggregation, not lost data.
+Aggregates are bounded per row, on purpose: hosts are Caddy's vhosts,
+methods collapse through the nine-verb rule the metrics rails use, and
+user agents collapse to four classes (`ai_scraper`, `bot`, `browser`,
+`other`). Each request row also carries a bounded agent name (#1458): the
+marker that classified it for AI scrapers and named bots, the UA's first
+product token (max 32 bytes) for the anonymous tail, and nothing for
+browsers, since every browser's token is `mozilla`. The token is the one
+caller-shaped key besides iili slugs, so an object may mint at most 500
+distinct ones before the rest collapse into a single `(more)` row — a
+scanner rotating its User-Agent gets one row, not one per request. The
+probe rollup counts requests whose path matched one of the scanner
+families in `classify.go` (`wordpress`, `env`, `git`, `php`, ...) and
+mints nothing for ordinary routes; there is deliberately no `admin`
+family, and backup-file shapes match only at the root, because Forgejo
+serves real archives and `.sql` files under deeper paths. iili slugs are
+one path segment, max 64 bytes, only on the two routes that reach iili.
+Row width is what's bounded; row count is what Postgres is for, which is
+the division of labor #1460 drew against the tsdb.
+
+The raw lines stay in S3, so a better classifier is a re-aggregation,
+not lost data — and re-aggregation is a mechanism, not a runbook. The
+store records a rollup version made of `RollupVersion` plus a hash of
+the schema DDL; a boot that finds a different one drops every aggregate
+table and processed marker in one transaction, recreates the tables, and
+the next pass recomputes everything from S3. Editing a table re-aggregates
+by itself; bump the constant when a classifier changes what a row means.
+The pass runs while the API serves, so for its length the counts climb
+back up from zero — minutes at this scale, and the log says when it is
+done.
+
+What stays ad hoc: IP-range clusters (a /24 key is caller-shaped and
+unbounded) and geography (nothing in the repo maps addresses to
+countries). Both are one query over the raw partitions in S3, which keep
+`request.remote_ip` and `request.client_ip` per line.
 
 ## The API
 
 - `GET /stats/v1/summary?days=7` — per day/host/agent-class request and
   error counts
 - `GET /stats/v1/iili/top?days=30&limit=20` — most-followed short links
+- `GET /stats/v1/agents?days=30&limit=500` — per day/host/class/agent
+  request and 403 counts, busiest rows first: which scrapers and bots hit
+  which host, and whether they back off after being refused
+- `GET /stats/v1/probes?days=30` — per host/scanner-family request counts
+  and how many were served (status < 400). On vhosts whose Caddy block
+  has no catch-all 404 (`api.muchq.com`, `gpt.muchq.com`), an unmatched
+  path is answered with an empty 200 and so reads as served; the 1d4 and
+  iili vhosts do have one, and there the column means what it says
 - `GET /health`
 
 Public through Caddy at `api.muchq.com/stats/v1/*`; the reasons for 500s

domains/platform/apis/stats/aggregate.go

@@ -7,16 +7,20 @@ import (
 	"io"
 )
 
-// RequestKey is one row of the per-day request rollup: everything bounded,
-// nothing caller-controlled — host is one of Caddy's configured vhosts,
-// the method collapses through the same nine-verb rule the metrics rails
-// use, and the agent class is the four-value vocabulary in classify.go.
+// RequestKey is one row of the per-day request rollup. Host is one of
+// Caddy's configured vhosts, the method collapses through the same
+// nine-verb rule the metrics rails use, the agent class is the four-value
+// vocabulary in classify.go, and the agent is the bounded name AgentOf
+// pairs with it — so a host's traffic can be opened up into which
+// scrapers, which bots, and what the unclassified tail actually sends,
+// from the same rows the class totals come from.
 type RequestKey struct {
 	Date       string
 	Host       string
 	Status     int
 	Method     string
 	AgentClass string
+	Agent      string
 }
 
 // SlugKey is one row of the iili redirect rollup. The slug is
@@ -28,17 +32,63 @@ type SlugKey struct {
 	Status int
 }
 
+// ProbeKey is one row of the scanner rollup. Rows exist only for requests
+// whose path matched a family in ProbeOf, so the key is bounded by that
+// vocabulary and ordinary traffic mints nothing here.
+type ProbeKey struct {
+	Date   string
+	Host   string
+	Probe  string
+	Status int
+}
+
+// The most distinct product-token agent names one object may mint. Marker
+// names are bounded by their lists; tokens are whatever the anonymous tail
+// sends, and a scanner rotating its User-Agent per request would otherwise
+// turn one log object into one row per request — held in memory here and
+// then written one statement at a time. Past the cap the tail collapses
+// into one row named overflowAgent, so the counts stay right and the
+// object stays applicable.
+const (
+	maxTailAgentsPerObject = 500
+	overflowAgent          = "(more)"
+)
+
 // Rollup is one processed object's aggregates, accumulated in memory and
 // applied to the store in a single transaction with the processed marker —
 // so a crash between the two reprocesses the object rather than losing or
 // double-counting it.
 type Rollup struct {
 	Requests map[RequestKey]int64
 	Slugs    map[SlugKey]int64
+	Probes   map[ProbeKey]int64
+
+	tailAgents map[string]bool
 }
 
 func NewRollup() *Rollup {
-	return &Rollup{Requests: map[RequestKey]int64{}, Slugs: map[SlugKey]int64{}}
+	return &Rollup{
+		Requests:   map[RequestKey]int64{},
+		Slugs:      map[SlugKey]int64{},
+		Probes:     map[ProbeKey]int64{},
+		tailAgents: map[string]bool{},
+	}
+}
+
+// boundedAgent applies the per-object cap to names that came from a
+// product token rather than a marker list. Browsers have no name to cap.
+func (r *Rollup) boundedAgent(class, agent string) string {
+	if class == AgentBrowser || markerNames[agent] {
+		return agent
+	}
+	if r.tailAgents[agent] {
+		return agent
+	}
+	if len(r.tailAgents) >= maxTailAgentsPerObject {
+		return overflowAgent
+	}
+	r.tailAgents[agent] = true
+	return agent
 }
 
 // caddyLine is the slice of Caddy's JSON access log this pipeline reads.
@@ -95,13 +145,18 @@ func (r *Rollup) Consume(reader io.Reader, date string) (skipped int, err error)
 			continue
 		}
 		method := boundedMethod(parsed.Request.Method)
+		agentClass, agent := AgentOf(parsed.userAgent())
 		r.Requests[RequestKey{
 			Date:       date,
 			Host:       parsed.Request.Host,
 			Status:     parsed.Status,
 			Method:     method,
-			AgentClass: AgentClassOf(parsed.userAgent()),
+			AgentClass: agentClass,
+			Agent:      r.boundedAgent(agentClass, agent),
 		}]++
+		if probe := ProbeOf(parsed.Request.URI); probe != "" {
+			r.Probes[ProbeKey{Date: date, Host: parsed.Request.Host, Probe: probe, Status: parsed.Status}]++
+		}
 		if slug := SlugOf(parsed.Request.Host, parsed.Request.Method, parsed.Request.URI); slug != "" {
 			r.Slugs[SlugKey{Date: date, Slug: slug, Status: parsed.Status}]++
 		}

domains/platform/apis/stats/aggregate_test.go

@@ -1,6 +1,7 @@
 package stats
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 )
@@ -14,6 +15,8 @@ const sampleLines = `{"status":200,"request":{"host":"api.1d4.net","method":"POS
 not json at all
 {"status":418,"request":{"method":"GET","uri":"/hostless","headers":{}}}
 {"status":200,"request":{"host":"api.muchq.com","method":"WEIRD","uri":"/x","headers":{}}}
+{"status":404,"request":{"host":"api.muchq.com","method":"GET","uri":"/wp-login.php","headers":{"User-Agent":["python-requests/2.32.0"]}}}
+{"status":404,"request":{"host":"api.muchq.com","method":"GET","uri":"/.env","headers":{"User-Agent":["Mozilla/5.0 (compatible; GPTBot/1.2)"]}}}
 `
 
 func TestConsumeAggregatesRequestsSlugsAndSkipsCorruptLines(t *testing.T) {
@@ -28,14 +31,23 @@ func TestConsumeAggregatesRequestsSlugsAndSkipsCorruptLines(t *testing.T) {
 	if skipped != 2 {
 		t.Errorf("skipped = %d, want 2", skipped)
 	}
-	if got := rollup.Requests[RequestKey{"2026-08-30", "api.1d4.net", 200, "POST", AgentBrowser}]; got != 2 {
+	// Browsers are one unnamed bucket; every other row carries its agent's
+	// bounded name, so "did meta back off after the 403" is a query over the
+	// same rows the class totals come from rather than a re-aggregation.
+	if got := rollup.Requests[RequestKey{"2026-08-30", "api.1d4.net", 200, "POST", AgentBrowser, ""}]; got != 2 {
 		t.Errorf("mcp browser POSTs = %d, want 2", got)
 	}
-	if got := rollup.Requests[RequestKey{"2026-08-30", "git.muchq.com", 403, "GET", AgentAIScraper}]; got != 1 {
+	if got := rollup.Requests[RequestKey{"2026-08-30", "git.muchq.com", 403, "GET", AgentAIScraper, "meta-externalagent"}]; got != 1 {
 		t.Errorf("blocked ai scraper = %d, want 1", got)
 	}
+	if got := rollup.Requests[RequestKey{"2026-08-30", "i.iili.uk", 302, "GET", AgentBot, "curl"}]; got != 1 {
+		t.Errorf("curl redirects = %d, want 1", got)
+	}
+	if got := rollup.Requests[RequestKey{"2026-08-30", "i.iili.uk", 404, "GET", AgentOther, "(empty)"}]; got != 1 {
+		t.Errorf("empty-UA rows = %d, want 1", got)
+	}
 	// An invented verb collapses like every metrics rail's method label.
-	if got := rollup.Requests[RequestKey{"2026-08-30", "api.muchq.com", 200, "CUSTOM", AgentOther}]; got != 1 {
+	if got := rollup.Requests[RequestKey{"2026-08-30", "api.muchq.com", 200, "CUSTOM", AgentOther, "(empty)"}]; got != 1 {
 		t.Errorf("CUSTOM-method row = %d, want 1", got)
 	}
 	// The redirect rollup counts per slug and status, across agent classes.
@@ -45,6 +57,59 @@ func TestConsumeAggregatesRequestsSlugsAndSkipsCorruptLines(t *testing.T) {
 	if got := rollup.Slugs[SlugKey{"2026-08-30", "gone", 404}]; got != 1 {
 		t.Errorf("gone-slug 404s = %d, want 1", got)
 	}
+	// Probe rows exist only for paths that match a scanner family.
+	if got := rollup.Probes[ProbeKey{"2026-08-30", "api.muchq.com", ProbeWordpress, 404}]; got != 1 {
+		t.Errorf("wordpress probes = %d, want 1", got)
+	}
+	if got := rollup.Probes[ProbeKey{"2026-08-30", "api.muchq.com", ProbeEnv, 404}]; got != 1 {
+		t.Errorf("env probes = %d, want 1", got)
+	}
+	if len(rollup.Probes) != 2 {
+		t.Errorf("probe rows = %v; ordinary routes must not mint any", rollup.Probes)
+	}
+}
+
+func TestConsumeCapsTheAnonymousAgentTailPerObject(t *testing.T) {
+	var lines strings.Builder
+	for i := 0; i < maxTailAgentsPerObject+100; i++ {
+		fmt.Fprintf(&lines, `{"status":200,"request":{"host":"h","method":"GET","uri":"/","headers":{"User-Agent":["junk-%d/1.0"]}}}`+"\n", i)
+	}
+	// Marker-named agents arriving after the cap keep their names; only
+	// product tokens are capped, and browsers were never named.
+	lines.WriteString(`{"status":200,"request":{"host":"h","method":"GET","uri":"/","headers":{"User-Agent":["Mozilla/5.0 (compatible; GPTBot/1.2)"]}}}` + "\n")
+	lines.WriteString(`{"status":200,"request":{"host":"h","method":"GET","uri":"/","headers":{"User-Agent":["curl/8.6.0"]}}}` + "\n")
+	lines.WriteString(`{"status":200,"request":{"host":"h","method":"GET","uri":"/","headers":{"User-Agent":["Mozilla/5.0 (Macintosh) Chrome/126.0"]}}}` + "\n")
+	rollup := NewRollup()
+
+	if _, err := rollup.Consume(strings.NewReader(lines.String()), "2026-08-30"); err != nil {
+		t.Fatal(err)
+	}
+
+	var total, overflow int64
+	names := map[string]bool{}
+	for key, count := range rollup.Requests {
+		total += count
+		if key.Agent == overflowAgent {
+			overflow += count
+		}
+		if key.AgentClass == AgentOther {
+			names[key.Agent] = true
+		}
+	}
+	if total != int64(maxTailAgentsPerObject+103) {
+		t.Errorf("total requests = %d; the cap must not lose a count", total)
+	}
+	if overflow != 100 {
+		t.Errorf("overflow row = %d, want the 100 past the cap", overflow)
+	}
+	if len(names) != maxTailAgentsPerObject+1 {
+		t.Errorf("distinct other-class names = %d, want the cap plus the overflow row", len(names))
+	}
+	if rollup.Requests[RequestKey{"2026-08-30", "h", 200, "GET", AgentAIScraper, "gptbot"}] != 1 ||
+		rollup.Requests[RequestKey{"2026-08-30", "h", 200, "GET", AgentBot, "curl"}] != 1 ||
+		rollup.Requests[RequestKey{"2026-08-30", "h", 200, "GET", AgentBrowser, ""}] != 1 {
+		t.Errorf("marker-named and browser rows were caught by the cap: %v", rollup.Requests)
+	}
 }
 
 func TestConsumeSurvivesOversizedLines(t *testing.T) {

domains/platform/apis/stats/api.go

@@ -13,6 +13,8 @@ import (
 type Reader interface {
 	Summary(ctx context.Context, days int) ([]SummaryRow, error)
 	TopSlugs(ctx context.Context, days, limit int) ([]SlugRow, error)
+	Agents(ctx context.Context, days, limit int) ([]AgentRow, error)
+	Probes(ctx context.Context, days int) ([]ProbeRow, error)
 }
 
 type Handlers struct {
@@ -67,6 +69,27 @@ func (h *Handlers) GetTopSlugs(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, map[string]any{"days": days, "rows": emptyIfNil(rows)})
 }
 
+func (h *Handlers) GetAgents(w http.ResponseWriter, r *http.Request) {
+	days := queryInt(r, "days", 30, 365)
+	limit := queryInt(r, "limit", 500, 2000)
+	rows, err := h.reader.Agents(r.Context(), days, limit)
+	if err != nil {
+		h.serverError(w, "agents", err)
+		return
+	}
+	writeJSON(w, map[string]any{"days": days, "rows": emptyIfNil(rows)})
+}
+
+func (h *Handlers) GetProbes(w http.ResponseWriter, r *http.Request) {
+	days := queryInt(r, "days", 30, 365)
+	rows, err := h.reader.Probes(r.Context(), days)
+	if err != nil {
+		h.serverError(w, "probes", err)
+		return
+	}
+	writeJSON(w, map[string]any{"days": days, "rows": emptyIfNil(rows)})
+}
+
 func (h *Handlers) serverError(w http.ResponseWriter, what string, err error) {
 	h.logger.Error("stats query failed", "query", what, "error", err)
 	// The reason goes to the log, not the wire: these endpoints are public.