From e47fda3e0e465f32bd1f2724fcd98efbb86f3308 Mon Sep 17 00:00:00 2001 From: Andy Aylward Date: Tue, 1 Sep 2026 18:06:10 -0400 Subject: [PATCH 1/2] stats: name agents and scanner probe families (#1458) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent rollup keys each row by class plus a bounded name — the marker for AI scrapers and named bots, the UA's first product token (max 32 bytes) for the rest — and the probe rollup counts requests whose path matches a scanner family. Served at /stats/v1/agents and /stats/v1/probes. The store records a rollup version and re-aggregates from S3 when it changes, so the new tables fill from objects already processed. The store integration test was skipping in CI: nothing set STATS_TEST_DB_URL and the target did not inherit it. --- .github/workflows/branch.yml | 1 + domains/platform/apis/stats/BUILD.bazel | 3 + domains/platform/apis/stats/README.md | 37 +++- domains/platform/apis/stats/aggregate.go | 44 ++++- domains/platform/apis/stats/aggregate_test.go | 29 ++++ domains/platform/apis/stats/api.go | 22 +++ domains/platform/apis/stats/api_test.go | 60 +++++++ domains/platform/apis/stats/classify.go | 142 +++++++++++++++- domains/platform/apis/stats/classify_test.go | 104 ++++++++++++ domains/platform/apis/stats/main/main.go | 2 + domains/platform/apis/stats/store.go | 159 +++++++++++++++++- domains/platform/apis/stats/store_test.go | 57 +++++++ 12 files changed, 643 insertions(+), 17 deletions(-) diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 8f8368a8..398d45a6 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -175,6 +175,7 @@ jobs: env: GOLF_HUB_TEST_DB_URL: postgresql://moonbase_test:moonbase_test@postgres:5432/moonbase_test PG_TEST_DB_URL: postgresql://moonbase_test:moonbase_test@postgres:5432/moonbase_test + STATS_TEST_DB_URL: postgresql://moonbase_test:moonbase_test@postgres:5432/moonbase_test steps: - uses: actions/checkout@v7 diff --git a/domains/platform/apis/stats/BUILD.bazel b/domains/platform/apis/stats/BUILD.bazel index 7cada874..70be28c2 100644 --- a/domains/platform/apis/stats/BUILD.bazel +++ b/domains/platform/apis/stats/BUILD.bazel @@ -20,6 +20,8 @@ go_library( # store_test.go skips without STATS_TEST_DB_URL, like the repo's other # Postgres-gated suites — a local green run may have exercised no SQL. +# CI supplies it from the postgres service; without env_inherit the +# sandbox strips it and the suite skips there too. go_test( name = "stats_test", size = "small", @@ -31,6 +33,7 @@ go_test( "store_test.go", ], embed = [":stats_lib"], + env_inherit = ["STATS_TEST_DB_URL"], ) go_binary( diff --git a/domains/platform/apis/stats/README.md b/domains/platform/apis/stats/README.md index 334b30e1..7fe834b5 100644 --- a/domains/platform/apis/stats/README.md +++ b/domains/platform/apis/stats/README.md @@ -16,18 +16,43 @@ conflict arm makes a duplicate application a no-op — so counts survive crashes without double-counting. Per-object failures are logged and retried next pass. -Aggregates are bounded on purpose: hosts are Caddy's vhosts, methods -collapse through the nine-verb rule the metrics rails use, user agents -collapse to four classes (`ai_scraper`, `bot`, `browser`, `other`), and -iili slugs are the one caller-shaped key — one path segment, max 64 -bytes, only on the redirect routes. The raw lines stay in S3, so a -better classifier is a re-aggregation, not lost data. +Aggregates are bounded per row, on purpose: hosts are Caddy's vhosts, +methods collapse through the nine-verb rule the metrics rails use, and +user agents collapse to four classes (`ai_scraper`, `bot`, `browser`, +`other`). Two rollups carry a name alongside those (#1458): the agent +rollup names each row by the marker that classified it (AI scrapers and +named bots) or by the UA's first product token, max 32 bytes, for the +anonymous tail — browsers are one unnamed bucket, since every browser's +token is `mozilla`; and the probe rollup counts requests whose path +matched one of the scanner families in `classify.go` (`wordpress`, `env`, +`git`, `php`, ...), minting nothing for ordinary routes. iili slugs are +the other caller-shaped key — one path segment, max 64 bytes, only on +the redirect routes. Row width is what's bounded; row count is what +Postgres is for, which is the division of labor #1460 drew against the +tsdb. + +The raw lines stay in S3, so a better classifier is a re-aggregation, +not lost data — and re-aggregation is a mechanism, not a runbook: the +store records `RollupVersion`, and a boot that finds a different one +drops every aggregate and processed marker in one transaction, so the +next pass recomputes everything from S3. Bump the constant when a rollup +gains a table or a classifier changes meaning. + +What stays ad hoc: IP-range clusters (a /24 key is caller-shaped and +unbounded) and geography (nothing in the repo maps addresses to +countries). Both are one query over the raw partitions in S3, which keep +`request.remote_ip` and `request.client_ip` per line. ## The API - `GET /stats/v1/summary?days=7` — per day/host/agent-class request and error counts - `GET /stats/v1/iili/top?days=30&limit=20` — most-followed short links +- `GET /stats/v1/agents?days=30` — per day/host/class/agent request and + 403 counts: which scrapers and bots hit which host, and whether they + back off after being refused +- `GET /stats/v1/probes?days=30` — per host/scanner-family request counts + and how many were served (status < 400): the rows worth looking at - `GET /health` Public through Caddy at `api.muchq.com/stats/v1/*`; the reasons for 500s diff --git a/domains/platform/apis/stats/aggregate.go b/domains/platform/apis/stats/aggregate.go index 36e555b5..825551a2 100644 --- a/domains/platform/apis/stats/aggregate.go +++ b/domains/platform/apis/stats/aggregate.go @@ -28,6 +28,28 @@ type SlugKey struct { Status int } +// AgentKey is one row of the per-day agent rollup: the class from the +// four-value vocabulary plus the bounded name AgentOf pairs with it, so a +// host's traffic can be opened up into which scrapers, which bots, and +// what the unclassified tail actually sends. +type AgentKey struct { + Date string + Host string + AgentClass string + Agent string + Status int +} + +// ProbeKey is one row of the scanner rollup. Rows exist only for requests +// whose path matched a family in ProbeOf, so the key is bounded by that +// vocabulary and ordinary traffic mints nothing here. +type ProbeKey struct { + Date string + Host string + Probe string + Status int +} + // Rollup is one processed object's aggregates, accumulated in memory and // applied to the store in a single transaction with the processed marker — // so a crash between the two reprocesses the object rather than losing or @@ -35,10 +57,17 @@ type SlugKey struct { type Rollup struct { Requests map[RequestKey]int64 Slugs map[SlugKey]int64 + Agents map[AgentKey]int64 + Probes map[ProbeKey]int64 } func NewRollup() *Rollup { - return &Rollup{Requests: map[RequestKey]int64{}, Slugs: map[SlugKey]int64{}} + return &Rollup{ + Requests: map[RequestKey]int64{}, + Slugs: map[SlugKey]int64{}, + Agents: map[AgentKey]int64{}, + Probes: map[ProbeKey]int64{}, + } } // caddyLine is the slice of Caddy's JSON access log this pipeline reads. @@ -95,13 +124,24 @@ func (r *Rollup) Consume(reader io.Reader, date string) (skipped int, err error) continue } method := boundedMethod(parsed.Request.Method) + agentClass, agent := AgentOf(parsed.userAgent()) r.Requests[RequestKey{ Date: date, Host: parsed.Request.Host, Status: parsed.Status, Method: method, - AgentClass: AgentClassOf(parsed.userAgent()), + AgentClass: agentClass, + }]++ + r.Agents[AgentKey{ + Date: date, + Host: parsed.Request.Host, + AgentClass: agentClass, + Agent: agent, + Status: parsed.Status, }]++ + if probe := ProbeOf(parsed.Request.URI); probe != "" { + r.Probes[ProbeKey{Date: date, Host: parsed.Request.Host, Probe: probe, Status: parsed.Status}]++ + } if slug := SlugOf(parsed.Request.Host, parsed.Request.Method, parsed.Request.URI); slug != "" { r.Slugs[SlugKey{Date: date, Slug: slug, Status: parsed.Status}]++ } diff --git a/domains/platform/apis/stats/aggregate_test.go b/domains/platform/apis/stats/aggregate_test.go index 9e489c7d..d19ee4be 100644 --- a/domains/platform/apis/stats/aggregate_test.go +++ b/domains/platform/apis/stats/aggregate_test.go @@ -14,6 +14,8 @@ const sampleLines = `{"status":200,"request":{"host":"api.1d4.net","method":"POS not json at all {"status":418,"request":{"method":"GET","uri":"/hostless","headers":{}}} {"status":200,"request":{"host":"api.muchq.com","method":"WEIRD","uri":"/x","headers":{}}} +{"status":404,"request":{"host":"api.muchq.com","method":"GET","uri":"/wp-login.php","headers":{"User-Agent":["python-requests/2.32.0"]}}} +{"status":404,"request":{"host":"api.muchq.com","method":"GET","uri":"/.env","headers":{"User-Agent":["Mozilla/5.0 (compatible; GPTBot/1.2)"]}}} ` func TestConsumeAggregatesRequestsSlugsAndSkipsCorruptLines(t *testing.T) { @@ -45,6 +47,33 @@ func TestConsumeAggregatesRequestsSlugsAndSkipsCorruptLines(t *testing.T) { if got := rollup.Slugs[SlugKey{"2026-08-30", "gone", 404}]; got != 1 { t.Errorf("gone-slug 404s = %d, want 1", got) } + // Agents are also counted by name, so "did meta back off after the 403" + // is a per-agent query rather than a re-aggregation. + if got := rollup.Agents[AgentKey{"2026-08-30", "git.muchq.com", AgentAIScraper, "meta-externalagent", 403}]; got != 1 { + t.Errorf("meta 403s = %d, want 1", got) + } + if got := rollup.Agents[AgentKey{"2026-08-30", "api.muchq.com", AgentAIScraper, "gptbot", 404}]; got != 1 { + t.Errorf("gptbot 404s = %d, want 1", got) + } + if got := rollup.Agents[AgentKey{"2026-08-30", "i.iili.uk", AgentBot, "curl", 302}]; got != 1 { + t.Errorf("curl redirects = %d, want 1", got) + } + if got := rollup.Agents[AgentKey{"2026-08-30", "api.1d4.net", AgentBrowser, "", 200}]; got != 2 { + t.Errorf("browser rows = %d, want 2 under one unnamed browser row", got) + } + if got := rollup.Agents[AgentKey{"2026-08-30", "i.iili.uk", AgentOther, "(empty)", 404}]; got != 1 { + t.Errorf("empty-UA rows = %d, want 1", got) + } + // Probe rows exist only for paths that match a scanner family. + if got := rollup.Probes[ProbeKey{"2026-08-30", "api.muchq.com", ProbeWordpress, 404}]; got != 1 { + t.Errorf("wordpress probes = %d, want 1", got) + } + if got := rollup.Probes[ProbeKey{"2026-08-30", "api.muchq.com", ProbeEnv, 404}]; got != 1 { + t.Errorf("env probes = %d, want 1", got) + } + if len(rollup.Probes) != 2 { + t.Errorf("probe rows = %v; ordinary routes must not mint any", rollup.Probes) + } } func TestConsumeSurvivesOversizedLines(t *testing.T) { diff --git a/domains/platform/apis/stats/api.go b/domains/platform/apis/stats/api.go index 43c8cb33..ef01026e 100644 --- a/domains/platform/apis/stats/api.go +++ b/domains/platform/apis/stats/api.go @@ -13,6 +13,8 @@ import ( type Reader interface { Summary(ctx context.Context, days int) ([]SummaryRow, error) TopSlugs(ctx context.Context, days, limit int) ([]SlugRow, error) + Agents(ctx context.Context, days int) ([]AgentRow, error) + Probes(ctx context.Context, days int) ([]ProbeRow, error) } type Handlers struct { @@ -67,6 +69,26 @@ func (h *Handlers) GetTopSlugs(w http.ResponseWriter, r *http.Request) { writeJSON(w, map[string]any{"days": days, "rows": emptyIfNil(rows)}) } +func (h *Handlers) GetAgents(w http.ResponseWriter, r *http.Request) { + days := queryInt(r, "days", 30, 365) + rows, err := h.reader.Agents(r.Context(), days) + if err != nil { + h.serverError(w, "agents", err) + return + } + writeJSON(w, map[string]any{"days": days, "rows": emptyIfNil(rows)}) +} + +func (h *Handlers) GetProbes(w http.ResponseWriter, r *http.Request) { + days := queryInt(r, "days", 30, 365) + rows, err := h.reader.Probes(r.Context(), days) + if err != nil { + h.serverError(w, "probes", err) + return + } + writeJSON(w, map[string]any{"days": days, "rows": emptyIfNil(rows)}) +} + func (h *Handlers) serverError(w http.ResponseWriter, what string, err error) { h.logger.Error("stats query failed", "query", what, "error", err) // The reason goes to the log, not the wire: these endpoints are public. diff --git a/domains/platform/apis/stats/api_test.go b/domains/platform/apis/stats/api_test.go index 3a588f0e..bb44f86b 100644 --- a/domains/platform/apis/stats/api_test.go +++ b/domains/platform/apis/stats/api_test.go @@ -15,6 +15,8 @@ import ( type fakeReader struct { summary []SummaryRow slugs []SlugRow + agents []AgentRow + probes []ProbeRow lastDays int lastLimit int fail bool @@ -36,6 +38,22 @@ func (f *fakeReader) TopSlugs(_ context.Context, days, limit int) ([]SlugRow, er return f.slugs, nil } +func (f *fakeReader) Agents(_ context.Context, days int) ([]AgentRow, error) { + if f.fail { + return nil, errors.New("db is having a day") + } + f.lastDays = days + return f.agents, nil +} + +func (f *fakeReader) Probes(_ context.Context, days int) ([]ProbeRow, error) { + if f.fail { + return nil, errors.New("db is having a day") + } + f.lastDays = days + return f.probes, nil +} + func handlersWith(reader *fakeReader) *Handlers { return NewHandlers(reader, slog.New(slog.NewTextHandler(io.Discard, nil))) } @@ -100,3 +118,45 @@ func TestAStoreFailureIs500WithoutTheReasonOnTheWire(t *testing.T) { t.Error("the failure reason leaked to a public endpoint") } } + +func TestAgentsAndProbesShareTheWindowRules(t *testing.T) { + reader := &fakeReader{ + agents: []AgentRow{{Date: "2026-08-30", Host: "git.muchq.com", AgentClass: AgentAIScraper, Agent: "meta-externalagent", Requests: 9, Blocked: 9}}, + probes: []ProbeRow{{Host: "api.muchq.com", Probe: ProbeWordpress, Requests: 4, Served: 0}}, + } + handlers := handlersWith(reader) + + _, body := get(t, handlers.GetAgents, "/stats/v1/agents") + if reader.lastDays != 30 { + t.Errorf("default agents window = %d, want 30", reader.lastDays) + } + row := body["rows"].([]any)[0].(map[string]any) + if row["agent"] != "meta-externalagent" || row["agent_class"] != AgentAIScraper || row["blocked"] != float64(9) { + t.Errorf("agent row = %v", row) + } + + _, body = get(t, handlers.GetProbes, "/stats/v1/probes?days=99999") + if reader.lastDays != 365 { + t.Errorf("probe window clamped to %d, want 365", reader.lastDays) + } + row = body["rows"].([]any)[0].(map[string]any) + if row["probe"] != ProbeWordpress || row["served"] != float64(0) { + t.Errorf("probe row = %v", row) + } + + // Empty is [], not null, on both — the dashboard maps them. + _, body = get(t, handlersWith(&fakeReader{}).GetAgents, "/stats/v1/agents") + if body["rows"] == nil { + t.Error("agents rows serialized as null; want []") + } + _, body = get(t, handlersWith(&fakeReader{}).GetProbes, "/stats/v1/probes") + if body["rows"] == nil { + t.Error("probes rows serialized as null; want []") + } + failing := handlersWith(&fakeReader{fail: true}) + for name, handler := range map[string]http.HandlerFunc{"agents": failing.GetAgents, "probes": failing.GetProbes} { + if recorder, _ := get(t, handler, "/x"); recorder.Code != http.StatusInternalServerError { + t.Errorf("%s on a failing store = %d, want 500", name, recorder.Code) + } + } +} diff --git a/domains/platform/apis/stats/classify.go b/domains/platform/apis/stats/classify.go index 44eea63f..7efad8a3 100644 --- a/domains/platform/apis/stats/classify.go +++ b/domains/platform/apis/stats/classify.go @@ -1,6 +1,9 @@ package stats -import "strings" +import ( + "regexp" + "strings" +) // The bounded user-agent vocabulary the stats tables key on. Four values, // not a UA string per row: the point of classification is that "how much of @@ -42,6 +45,18 @@ var aiScraperMarkers = []string{ "youbot", } +// Bots worth a row of their own. Search engines, SEO crawlers, link +// unfurlers, internet scanners, and the HTTP libraries scanners drive — +// the marker doubles as the agent name, so the vocabulary is this list. +var namedBotMarkers = []string{ + "googlebot", "bingbot", "yandexbot", "duckduckbot", "baiduspider", "applebot", + "ahrefsbot", "semrushbot", "mj12bot", "dotbot", "petalbot", "dataforseobot", + "facebookexternalhit", "twitterbot", "linkedinbot", "slackbot", "discordbot", + "telegrambot", "whatsapp", "uptimerobot", + "censysinspect", "zgrab", "nuclei", "masscan", + "python-requests", "go-http-client", "okhttp", "curl", "wget", "scrapy", +} + var botMarkers = []string{ "bot", "spider", "crawl", "curl", "wget", "python-requests", "python/", "go-http-client", "libwww", "httpclient", "okhttp", "scrapy", "java/", @@ -49,27 +64,138 @@ var botMarkers = []string{ "masscan", "nuclei", "censys", } -// AgentClassOf buckets a User-Agent header. Order matters: AI scrapers -// self-identify with names that also match the generic bot markers. +// AgentClassOf buckets a User-Agent header. func AgentClassOf(userAgent string) string { + class, _ := AgentOf(userAgent) + return class +} + +// AgentOf buckets a User-Agent header and names the agent within the +// bucket, bounded per class: AI scrapers and named bots name themselves by +// the marker that matched, so those columns' vocabularies are exactly the +// lists above; anonymous bots and the unclassified tail keep their product +// token (one run of [a-z0-9._-], max 32 bytes) so a new crawler is readable +// before it has a marker; browsers are one unnamed bucket, because every +// browser's token is "mozilla". Order matters: AI scrapers self-identify +// with names that also match the generic bot markers. +func AgentOf(userAgent string) (class, name string) { ua := strings.ToLower(userAgent) if ua == "" { - return AgentOther + return AgentOther, emptyToken } for _, marker := range aiScraperMarkers { if strings.Contains(ua, marker) { - return AgentAIScraper + return AgentAIScraper, marker + } + } + for _, marker := range namedBotMarkers { + if strings.Contains(ua, marker) { + return AgentBot, marker } } for _, marker := range botMarkers { if strings.Contains(ua, marker) { - return AgentBot + return AgentBot, productToken(ua) } } if strings.HasPrefix(ua, "mozilla/") { - return AgentBrowser + return AgentBrowser, "" + } + return AgentOther, productToken(ua) +} + +const ( + emptyToken = "(empty)" + maxTokenLength = 32 +) + +// productToken is the first run of [a-z0-9._-] in a lowercased UA — the +// product name of "product/version (comment)" — truncated to a bound so a +// scanner spraying UAs cannot mint wide rows, only many. +func productToken(lowerUA string) string { + start := -1 + for i := 0; i < len(lowerUA); i++ { + if isTokenByte(lowerUA[i]) { + if start < 0 { + start = i + } + continue + } + if start >= 0 { + return clampToken(lowerUA[start:i]) + } + } + if start < 0 { + return emptyToken + } + return clampToken(lowerUA[start:]) +} + +func isTokenByte(b byte) bool { + return b >= 'a' && b <= 'z' || b >= '0' && b <= '9' || b == '.' || b == '_' || b == '-' +} + +func clampToken(token string) string { + if len(token) > maxTokenLength { + return token[:maxTokenLength] + } + return token +} + +// The bounded scanner-path vocabulary. A family is a shape scanners probe +// for on any host — WordPress logins, dotfiles, PHP endpoints on hosts that +// serve no PHP — and a request either matches one family or is not a +// probe. There is deliberately no "admin" family: /admin/ is a real one_d4 +// route, and a family that counts real traffic is worse than none. +const ( + ProbeTraversal = "traversal" + ProbeWordpress = "wordpress" + ProbeEnv = "env" + ProbeGit = "git" + ProbeSecrets = "secrets" + ProbePhpmyadmin = "phpmyadmin" + ProbePhp = "php" + ProbeBackup = "backup" + ProbeCgi = "cgi" + ProbeJava = "java" + ProbeRouter = "router" +) + +// Ordered: the first family to match wins, so the specific ones (a +// traversal through cgi-bin, phpMyAdmin's index.php) sit above the shapes +// they also match. +var probeFamilies = []struct { + name string + match *regexp.Regexp +}{ + {ProbeTraversal, regexp.MustCompile(`\.\./|\.\.\\|%2e%2e|/etc/passwd`)}, + {ProbeWordpress, regexp.MustCompile(`wp-login|wp-admin|wp-content|wp-includes|wp-json|wp-config|xmlrpc\.php|wlwmanifest`)}, + {ProbeEnv, regexp.MustCompile(`/\.env`)}, + {ProbeGit, regexp.MustCompile(`/\.git(/|$)`)}, + {ProbeSecrets, regexp.MustCompile(`/\.aws/|/\.ssh/|id_rsa|\.htpasswd|\.htaccess|\.bash_history|/\.docker/`)}, + {ProbePhpmyadmin, regexp.MustCompile(`phpmyadmin|myadmin|/pma/|adminer`)}, + {ProbePhp, regexp.MustCompile(`\.php($|/)`)}, + {ProbeBackup, regexp.MustCompile(`\.(sql|bak|zip|tar|tar\.gz|tgz|rar|7z|old|orig|swp)$`)}, + {ProbeCgi, regexp.MustCompile(`/cgi-bin/`)}, + {ProbeJava, regexp.MustCompile(`/actuator|/solr/|/jenkins|/manager/html|jmx-console`)}, + {ProbeRouter, regexp.MustCompile(`/boaform/|/hnap1|/gponform/|/goform/|/tmui/`)}, +} + +// ProbeOf names the scanner family a request path belongs to, or "" when +// the path is not a known probe shape. Matching is on the lowercased path +// with the query string removed. +func ProbeOf(uri string) string { + path := uri + if q := strings.IndexByte(path, '?'); q >= 0 { + path = path[:q] + } + path = strings.ToLower(path) + for _, family := range probeFamilies { + if family.match.MatchString(path) { + return family.name + } } - return AgentOther + return "" } // SlugOf extracts the iili short-link slug from a request, or "" when the diff --git a/domains/platform/apis/stats/classify_test.go b/domains/platform/apis/stats/classify_test.go index 6f23ef45..cb4fd6b6 100644 --- a/domains/platform/apis/stats/classify_test.go +++ b/domains/platform/apis/stats/classify_test.go @@ -57,3 +57,107 @@ func TestSlugExtractionIsBoundedAndRouteScoped(t *testing.T) { } } } + +func TestAgentNamesAreBoundedPerClass(t *testing.T) { + cases := []struct { + ua string + wantClass string + wantName string + }{ + // AI scrapers name themselves by marker, whatever else the UA says. + {"Mozilla/5.0 AppleWebKit/537.36; compatible; GPTBot/1.2; +https://openai.com/gptbot", AgentAIScraper, "gptbot"}, + {"meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", AgentAIScraper, "meta-externalagent"}, + // Named bots by marker; anonymous tooling by its product token. + {"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", AgentBot, "googlebot"}, + {"Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)", AgentBot, "ahrefsbot"}, + {"curl/8.6.0", AgentBot, "curl"}, + {"python-requests/2.32.0", AgentBot, "python-requests"}, + {"Go-http-client/2.0", AgentBot, "go-http-client"}, + {"Mozilla/5.0 (compatible; SomeNewBot/1.0)", AgentBot, "mozilla"}, + // Browsers are one bucket: the token would be "mozilla" for all of them. + {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36", AgentBrowser, ""}, + // "other" keeps its product token so the unclassified tail is readable. + {"", AgentOther, "(empty)"}, + {"definitely-not-a-browser", AgentOther, "definitely-not-a-browser"}, + {"Weird Client 3.0", AgentOther, "weird"}, + {"", AgentOther, "script"}, + {strings.Repeat("a", 200) + "/1.0", AgentOther, strings.Repeat("a", 32)}, + {"/////", AgentOther, "(empty)"}, + } + for _, c := range cases { + class, name := AgentOf(c.ua) + if class != c.wantClass || name != c.wantName { + t.Errorf("AgentOf(%q) = (%s, %q), want (%s, %q)", c.ua, class, name, c.wantClass, c.wantName) + } + } + // Every marker names itself, so the agent column's vocabulary for the + // two marker classes is exactly the lists and cannot drift from them. + for _, marker := range aiScraperMarkers { + ua := "Mozilla/5.0 (compatible; " + strings.ToUpper(marker) + "/1.0)" + if class, name := AgentOf(ua); class != AgentAIScraper || name != marker { + t.Errorf("AgentOf(%q) = (%s, %q), want (%s, %q)", ua, class, name, AgentAIScraper, marker) + } + } + for _, marker := range namedBotMarkers { + ua := "Mozilla/5.0 (compatible; " + strings.ToUpper(marker) + "/1.0)" + if class, name := AgentOf(ua); class != AgentBot || name != marker { + t.Errorf("AgentOf(%q) = (%s, %q), want (%s, %q)", ua, class, name, AgentBot, marker) + } + } +} + +func TestProbeFamiliesAreBoundedAndRouteScoped(t *testing.T) { + cases := []struct { + uri string + want string + }{ + {"/wp-login.php", ProbeWordpress}, + {"/wp-admin/", ProbeWordpress}, + {"/xmlrpc.php", ProbeWordpress}, + {"/blog/wp-includes/wlwmanifest.xml", ProbeWordpress}, + {"/.env", ProbeEnv}, + {"/.env.production?x=1", ProbeEnv}, + {"/api/.env.bak", ProbeEnv}, + {"/.envrc", ProbeEnv}, + {"/.git/config", ProbeGit}, + {"/.git/HEAD", ProbeGit}, + {"/phpmyadmin/index.php", ProbePhpmyadmin}, + {"/PMA/", ProbePhpmyadmin}, + {"/adminer.php", ProbePhpmyadmin}, + {"/vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php", ProbePhp}, + {"/index.php?s=/Index/think/app/invokefunction", ProbePhp}, + {"/.aws/credentials", ProbeSecrets}, + {"/.ssh/id_rsa", ProbeSecrets}, + {"/.htpasswd", ProbeSecrets}, + {"/backup.sql", ProbeBackup}, + {"/site.tar.gz", ProbeBackup}, + {"/db.zip", ProbeBackup}, + {"/../../etc/passwd", ProbeTraversal}, + {"/cgi-bin/%2e%2e/%2e%2e/bin/sh", ProbeTraversal}, + {"/cgi-bin/luci", ProbeCgi}, + {"/manager/html", ProbeJava}, + {"/actuator/health", ProbeJava}, + {"/solr/admin/info/system", ProbeJava}, + {"/boaform/admin/formLogin", ProbeRouter}, + {"/HNAP1/", ProbeRouter}, + {"/GponForm/diag_Form", ProbeRouter}, + {"/WP-LOGIN.PHP", ProbeWordpress}, // case-insensitive + + // Real routes on these hosts are not probes, however they are spelled. + {"/", ""}, + {"/mcp", ""}, + {"/iili/v1/r/abc", ""}, + {"/stats/v1/summary?days=7", ""}, + {"/.well-known/acme-challenge/token", ""}, + {"/muchq/moonbase/src/branch/main/README.md", ""}, + {"/index.html", ""}, + {"/admin/reanalyze", ""}, // one_d4's real admin route; "admin" is not a family + {"/environment", ""}, + {"/gitignore", ""}, + } + for _, c := range cases { + if got := ProbeOf(c.uri); got != c.want { + t.Errorf("ProbeOf(%q) = %q, want %q", c.uri, got, c.want) + } + } +} diff --git a/domains/platform/apis/stats/main/main.go b/domains/platform/apis/stats/main/main.go index d7692c86..57522db6 100644 --- a/domains/platform/apis/stats/main/main.go +++ b/domains/platform/apis/stats/main/main.go @@ -82,6 +82,8 @@ func main() { router.HandleFunc("GET /health", handlers.Health) router.HandleFunc("GET /stats/v1/summary", handlers.GetSummary) router.HandleFunc("GET /stats/v1/iili/top", handlers.GetTopSlugs) + router.HandleFunc("GET /stats/v1/agents", handlers.GetAgents) + router.HandleFunc("GET /stats/v1/probes", handlers.GetProbes) logger.Info("stats started", "port", port, "interval", interval.String()) if err := http.ListenAndServe(":"+port, router); err != nil { diff --git a/domains/platform/apis/stats/store.go b/domains/platform/apis/stats/store.go index b2e76ec0..bf14689e 100644 --- a/domains/platform/apis/stats/store.go +++ b/domains/platform/apis/stats/store.go @@ -32,6 +32,42 @@ var schema = []string{ requests bigint NOT NULL, PRIMARY KEY (dt, slug, status) )`, + `CREATE TABLE IF NOT EXISTS agent_stats ( + dt date NOT NULL, + host text NOT NULL, + agent_class text NOT NULL, + agent text NOT NULL, + status int NOT NULL, + requests bigint NOT NULL, + PRIMARY KEY (dt, host, agent_class, agent, status) + )`, + `CREATE TABLE IF NOT EXISTS probe_stats ( + dt date NOT NULL, + host text NOT NULL, + probe text NOT NULL, + status int NOT NULL, + requests bigint NOT NULL, + PRIMARY KEY (dt, host, probe, status) + )`, + `CREATE TABLE IF NOT EXISTS stats_meta ( + key text PRIMARY KEY, + value text NOT NULL + )`, +} + +// RollupVersion is the shape of what Consume computes. Bump it when a +// rollup gains a table or a classifier changes meaning: at boot a store +// whose recorded version differs drops every aggregate and every +// processed marker in one transaction, and the next pass recomputes all +// of it from the raw lines in S3. That is what "a schema change is a +// re-aggregation" costs — one full pass — and what makes it never data +// loss. Version 2 added agent_stats and probe_stats. +const RollupVersion = "2" + +// The tables a re-aggregation rebuilds; every aggregate table belongs +// here, or a version bump leaves it double-counted. +var rollupTables = []string{ + "processed_log_objects", "request_stats", "iili_slug_stats", "agent_stats", "probe_stats", } type Store struct { @@ -49,7 +85,43 @@ func NewStore(ctx context.Context, databaseURL string) (*Store, error) { return nil, fmt.Errorf("applying schema: %w", err) } } - return &Store{pool: pool}, nil + store := &Store{pool: pool} + if err := store.reaggregateOnVersionChange(ctx); err != nil { + pool.Close() + return nil, err + } + return store, nil +} + +func (s *Store) reaggregateOnVersionChange(ctx context.Context) error { + tx, err := s.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + + var recorded string + err = tx.QueryRow(ctx, + `SELECT value FROM stats_meta WHERE key = 'rollup_version' FOR UPDATE`).Scan(&recorded) + if err != nil && err != pgx.ErrNoRows { + return fmt.Errorf("reading rollup version: %w", err) + } + if recorded == RollupVersion { + return nil + } + // A fresh database has no aggregates to drop and no version; an old + // one has both. TRUNCATE is one statement either way. + for _, table := range rollupTables { + if _, err := tx.Exec(ctx, "TRUNCATE "+table); err != nil { + return fmt.Errorf("resetting %s for re-aggregation: %w", table, err) + } + } + if _, err := tx.Exec(ctx, + `INSERT INTO stats_meta (key, value) VALUES ('rollup_version', $1) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, RollupVersion); err != nil { + return err + } + return tx.Commit(ctx) } func (s *Store) Close() { s.pool.Close() } @@ -118,6 +190,26 @@ func (s *Store) ApplyRollup(ctx context.Context, key string, rollup *Rollup) err return err } } + for k, count := range rollup.Agents { + if _, err := tx.Exec(ctx, + `INSERT INTO agent_stats (dt, host, agent_class, agent, status, requests) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (dt, host, agent_class, agent, status) + DO UPDATE SET requests = agent_stats.requests + EXCLUDED.requests`, + k.Date, k.Host, k.AgentClass, k.Agent, k.Status, count); err != nil { + return err + } + } + for k, count := range rollup.Probes { + if _, err := tx.Exec(ctx, + `INSERT INTO probe_stats (dt, host, probe, status, requests) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (dt, host, probe, status) + DO UPDATE SET requests = probe_stats.requests + EXCLUDED.requests`, + k.Date, k.Host, k.Probe, k.Status, count); err != nil { + return err + } + } return tx.Commit(ctx) } @@ -134,6 +226,26 @@ type SlugRow struct { Requests int64 `json:"requests"` } +// AgentRow is one day of one named agent on one host. Blocked is the 403 +// count — the signal for whether a scraper backs off after being refused. +type AgentRow struct { + Date string `json:"date"` + Host string `json:"host"` + AgentClass string `json:"agent_class"` + Agent string `json:"agent"` + Requests int64 `json:"requests"` + Blocked int64 `json:"blocked"` +} + +// ProbeRow is one scanner family on one host over the window. Served is +// the sub-400 count: a probe that got an answer is the row to look at. +type ProbeRow struct { + Host string `json:"host"` + Probe string `json:"probe"` + Requests int64 `json:"requests"` + Served int64 `json:"served"` +} + func (s *Store) Summary(ctx context.Context, days int) ([]SummaryRow, error) { rows, err := s.pool.Query(ctx, `SELECT dt::text, host, agent_class, @@ -176,3 +288,48 @@ func (s *Store) TopSlugs(ctx context.Context, days, limit int) ([]SlugRow, error }) return out, err } + +func (s *Store) Agents(ctx context.Context, days int) ([]AgentRow, error) { + rows, err := s.pool.Query(ctx, + `SELECT dt::text, host, agent_class, agent, + SUM(requests) AS requests, + COALESCE(SUM(requests) FILTER (WHERE status = 403), 0) AS blocked + FROM agent_stats + WHERE dt >= current_date - $1::int + GROUP BY dt, host, agent_class, agent + ORDER BY dt DESC, host, agent_class, requests DESC, agent`, days) + if err != nil { + return nil, err + } + var out []AgentRow + var row AgentRow + _, err = pgx.ForEachRow(rows, + []any{&row.Date, &row.Host, &row.AgentClass, &row.Agent, &row.Requests, &row.Blocked}, + func() error { + out = append(out, row) + return nil + }) + return out, err +} + +func (s *Store) Probes(ctx context.Context, days int) ([]ProbeRow, error) { + rows, err := s.pool.Query(ctx, + `SELECT host, probe, + SUM(requests) AS requests, + COALESCE(SUM(requests) FILTER (WHERE status < 400), 0) AS served + FROM probe_stats + WHERE dt >= current_date - $1::int + GROUP BY host, probe + ORDER BY requests DESC, host, probe`, days) + if err != nil { + return nil, err + } + var out []ProbeRow + var row ProbeRow + _, err = pgx.ForEachRow(rows, []any{&row.Host, &row.Probe, &row.Requests, &row.Served}, + func() error { + out = append(out, row) + return nil + }) + return out, err +} diff --git a/domains/platform/apis/stats/store_test.go b/domains/platform/apis/stats/store_test.go index d9d239f7..66406e80 100644 --- a/domains/platform/apis/stats/store_test.go +++ b/domains/platform/apis/stats/store_test.go @@ -89,3 +89,60 @@ func TestApplyRollupIsTransactionalIdempotentAndReadable(t *testing.T) { t.Errorf("test-slug missing from top slugs: %v", slugs) } } + +func TestAVersionBumpDropsAggregatesAndMarkersForReaggregation(t *testing.T) { + store := testStore(t) + ctx := context.Background() + date := time.Now().UTC().Format("2006-01-02") + key := fmt.Sprintf("logs/source=caddy/dt=%s/version-%d.log.gz", date, time.Now().UnixNano()) + rollup := NewRollup() + rollup.Requests[RequestKey{date, "version-host.example", 200, "GET", AgentBrowser}] = 1 + if err := store.ApplyRollup(ctx, key, rollup); err != nil { + t.Fatal(err) + } + + // Pretend those aggregates came from an older rollup shape. + if _, err := store.pool.Exec(ctx, + `UPDATE stats_meta SET value = 'stale' WHERE key = 'rollup_version'`); err != nil { + t.Fatal(err) + } + reopened, err := NewStore(ctx, os.Getenv("STATS_TEST_DB_URL")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(reopened.Close) + + // The marker is gone, so the next pass recomputes the object; the + // aggregates it produced are gone with it, so it is not double-counted. + if pending, err := reopened.Unprocessed(ctx, []string{key}); err != nil || len(pending) != 1 { + t.Errorf("Unprocessed after a version bump = (%v, %v), want the key pending again", pending, err) + } + summary, err := reopened.Summary(ctx, 2) + if err != nil { + t.Fatal(err) + } + for _, row := range summary { + if row.Host == "version-host.example" { + t.Errorf("aggregates survived the version bump: %+v", row) + } + } + var recorded string + if err := reopened.pool.QueryRow(ctx, + `SELECT value FROM stats_meta WHERE key = 'rollup_version'`).Scan(&recorded); err != nil || recorded != RollupVersion { + t.Errorf("recorded version = (%q, %v), want %q", recorded, err, RollupVersion) + } + + // Reopening at the same version is a no-op: the store must not wipe + // itself on every boot. + if err := reopened.ApplyRollup(ctx, key, rollup); err != nil { + t.Fatal(err) + } + again, err := NewStore(ctx, os.Getenv("STATS_TEST_DB_URL")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(again.Close) + if pending, err := again.Unprocessed(ctx, []string{key}); err != nil || len(pending) != 0 { + t.Errorf("Unprocessed after a same-version reopen = (%v, %v), want none", pending, err) + } +} From 6d273ec82933a2dbf630d0b2d9b74edf62b05d4f Mon Sep 17 00:00:00 2001 From: Andy Aylward Date: Tue, 1 Sep 2026 18:21:23 -0400 Subject: [PATCH 2/2] stats: review-panel fixes for the agent and probe rollups The agent name lives on request_stats rather than in a second table that duplicated its dimensions; the store drops and recreates the aggregate tables on a version change, and the recorded version hashes the DDL so a column edit re-aggregates without a constant bump. Each object caps the anonymous agent tail at 500 names, and /stats/v1/agents takes a limit, busiest rows first. Classifier fixes: telegrambot before twitterbot, browser-shaped generic bots named by the marker they tripped, backup probes only at the root (Forgejo archives), and iili slugs only on the two routes that reach iili. AgentClassOf and the empty-UA branch were dead. --- domains/platform/apis/stats/BUILD.bazel | 4 + domains/platform/apis/stats/README.md | 51 +++-- domains/platform/apis/stats/aggregate.go | 71 +++--- domains/platform/apis/stats/aggregate_test.go | 76 ++++-- domains/platform/apis/stats/api.go | 5 +- domains/platform/apis/stats/api_test.go | 35 ++- domains/platform/apis/stats/classify.go | 80 ++++--- domains/platform/apis/stats/classify_test.go | 95 +++++--- domains/platform/apis/stats/loop_test.go | 2 +- domains/platform/apis/stats/store.go | 113 ++++----- domains/platform/apis/stats/store_test.go | 216 +++++++++++++++--- 11 files changed, 527 insertions(+), 221 deletions(-) diff --git a/domains/platform/apis/stats/BUILD.bazel b/domains/platform/apis/stats/BUILD.bazel index 70be28c2..f0712801 100644 --- a/domains/platform/apis/stats/BUILD.bazel +++ b/domains/platform/apis/stats/BUILD.bazel @@ -34,6 +34,10 @@ go_test( ], embed = [":stats_lib"], env_inherit = ["STATS_TEST_DB_URL"], + # One shared scratch database, and the version-bump test drops every + # aggregate table: concurrent runs would pull the tables out from under + # each other mid-flight. + tags = ["exclusive"], ) go_binary( diff --git a/domains/platform/apis/stats/README.md b/domains/platform/apis/stats/README.md index 7fe834b5..997f1411 100644 --- a/domains/platform/apis/stats/README.md +++ b/domains/platform/apis/stats/README.md @@ -19,24 +19,32 @@ retried next pass. Aggregates are bounded per row, on purpose: hosts are Caddy's vhosts, methods collapse through the nine-verb rule the metrics rails use, and user agents collapse to four classes (`ai_scraper`, `bot`, `browser`, -`other`). Two rollups carry a name alongside those (#1458): the agent -rollup names each row by the marker that classified it (AI scrapers and -named bots) or by the UA's first product token, max 32 bytes, for the -anonymous tail — browsers are one unnamed bucket, since every browser's -token is `mozilla`; and the probe rollup counts requests whose path -matched one of the scanner families in `classify.go` (`wordpress`, `env`, -`git`, `php`, ...), minting nothing for ordinary routes. iili slugs are -the other caller-shaped key — one path segment, max 64 bytes, only on -the redirect routes. Row width is what's bounded; row count is what -Postgres is for, which is the division of labor #1460 drew against the -tsdb. +`other`). Each request row also carries a bounded agent name (#1458): the +marker that classified it for AI scrapers and named bots, the UA's first +product token (max 32 bytes) for the anonymous tail, and nothing for +browsers, since every browser's token is `mozilla`. The token is the one +caller-shaped key besides iili slugs, so an object may mint at most 500 +distinct ones before the rest collapse into a single `(more)` row — a +scanner rotating its User-Agent gets one row, not one per request. The +probe rollup counts requests whose path matched one of the scanner +families in `classify.go` (`wordpress`, `env`, `git`, `php`, ...) and +mints nothing for ordinary routes; there is deliberately no `admin` +family, and backup-file shapes match only at the root, because Forgejo +serves real archives and `.sql` files under deeper paths. iili slugs are +one path segment, max 64 bytes, only on the two routes that reach iili. +Row width is what's bounded; row count is what Postgres is for, which is +the division of labor #1460 drew against the tsdb. The raw lines stay in S3, so a better classifier is a re-aggregation, -not lost data — and re-aggregation is a mechanism, not a runbook: the -store records `RollupVersion`, and a boot that finds a different one -drops every aggregate and processed marker in one transaction, so the -next pass recomputes everything from S3. Bump the constant when a rollup -gains a table or a classifier changes meaning. +not lost data — and re-aggregation is a mechanism, not a runbook. The +store records a rollup version made of `RollupVersion` plus a hash of +the schema DDL; a boot that finds a different one drops every aggregate +table and processed marker in one transaction, recreates the tables, and +the next pass recomputes everything from S3. Editing a table re-aggregates +by itself; bump the constant when a classifier changes what a row means. +The pass runs while the API serves, so for its length the counts climb +back up from zero — minutes at this scale, and the log says when it is +done. What stays ad hoc: IP-range clusters (a /24 key is caller-shaped and unbounded) and geography (nothing in the repo maps addresses to @@ -48,11 +56,14 @@ countries). Both are one query over the raw partitions in S3, which keep - `GET /stats/v1/summary?days=7` — per day/host/agent-class request and error counts - `GET /stats/v1/iili/top?days=30&limit=20` — most-followed short links -- `GET /stats/v1/agents?days=30` — per day/host/class/agent request and - 403 counts: which scrapers and bots hit which host, and whether they - back off after being refused +- `GET /stats/v1/agents?days=30&limit=500` — per day/host/class/agent + request and 403 counts, busiest rows first: which scrapers and bots hit + which host, and whether they back off after being refused - `GET /stats/v1/probes?days=30` — per host/scanner-family request counts - and how many were served (status < 400): the rows worth looking at + and how many were served (status < 400). On vhosts whose Caddy block + has no catch-all 404 (`api.muchq.com`, `gpt.muchq.com`), an unmatched + path is answered with an empty 200 and so reads as served; the 1d4 and + iili vhosts do have one, and there the column means what it says - `GET /health` Public through Caddy at `api.muchq.com/stats/v1/*`; the reasons for 500s diff --git a/domains/platform/apis/stats/aggregate.go b/domains/platform/apis/stats/aggregate.go index 825551a2..39479a40 100644 --- a/domains/platform/apis/stats/aggregate.go +++ b/domains/platform/apis/stats/aggregate.go @@ -7,16 +7,20 @@ import ( "io" ) -// RequestKey is one row of the per-day request rollup: everything bounded, -// nothing caller-controlled — host is one of Caddy's configured vhosts, -// the method collapses through the same nine-verb rule the metrics rails -// use, and the agent class is the four-value vocabulary in classify.go. +// RequestKey is one row of the per-day request rollup. Host is one of +// Caddy's configured vhosts, the method collapses through the same +// nine-verb rule the metrics rails use, the agent class is the four-value +// vocabulary in classify.go, and the agent is the bounded name AgentOf +// pairs with it — so a host's traffic can be opened up into which +// scrapers, which bots, and what the unclassified tail actually sends, +// from the same rows the class totals come from. type RequestKey struct { Date string Host string Status int Method string AgentClass string + Agent string } // SlugKey is one row of the iili redirect rollup. The slug is @@ -28,18 +32,6 @@ type SlugKey struct { Status int } -// AgentKey is one row of the per-day agent rollup: the class from the -// four-value vocabulary plus the bounded name AgentOf pairs with it, so a -// host's traffic can be opened up into which scrapers, which bots, and -// what the unclassified tail actually sends. -type AgentKey struct { - Date string - Host string - AgentClass string - Agent string - Status int -} - // ProbeKey is one row of the scanner rollup. Rows exist only for requests // whose path matched a family in ProbeOf, so the key is bounded by that // vocabulary and ordinary traffic mints nothing here. @@ -50,6 +42,18 @@ type ProbeKey struct { Status int } +// The most distinct product-token agent names one object may mint. Marker +// names are bounded by their lists; tokens are whatever the anonymous tail +// sends, and a scanner rotating its User-Agent per request would otherwise +// turn one log object into one row per request — held in memory here and +// then written one statement at a time. Past the cap the tail collapses +// into one row named overflowAgent, so the counts stay right and the +// object stays applicable. +const ( + maxTailAgentsPerObject = 500 + overflowAgent = "(more)" +) + // Rollup is one processed object's aggregates, accumulated in memory and // applied to the store in a single transaction with the processed marker — // so a crash between the two reprocesses the object rather than losing or @@ -57,19 +61,36 @@ type ProbeKey struct { type Rollup struct { Requests map[RequestKey]int64 Slugs map[SlugKey]int64 - Agents map[AgentKey]int64 Probes map[ProbeKey]int64 + + tailAgents map[string]bool } func NewRollup() *Rollup { return &Rollup{ - Requests: map[RequestKey]int64{}, - Slugs: map[SlugKey]int64{}, - Agents: map[AgentKey]int64{}, - Probes: map[ProbeKey]int64{}, + Requests: map[RequestKey]int64{}, + Slugs: map[SlugKey]int64{}, + Probes: map[ProbeKey]int64{}, + tailAgents: map[string]bool{}, } } +// boundedAgent applies the per-object cap to names that came from a +// product token rather than a marker list. Browsers have no name to cap. +func (r *Rollup) boundedAgent(class, agent string) string { + if class == AgentBrowser || markerNames[agent] { + return agent + } + if r.tailAgents[agent] { + return agent + } + if len(r.tailAgents) >= maxTailAgentsPerObject { + return overflowAgent + } + r.tailAgents[agent] = true + return agent +} + // caddyLine is the slice of Caddy's JSON access log this pipeline reads. // Everything else in the line is ignored on decode. type caddyLine struct { @@ -131,13 +152,7 @@ func (r *Rollup) Consume(reader io.Reader, date string) (skipped int, err error) Status: parsed.Status, Method: method, AgentClass: agentClass, - }]++ - r.Agents[AgentKey{ - Date: date, - Host: parsed.Request.Host, - AgentClass: agentClass, - Agent: agent, - Status: parsed.Status, + Agent: r.boundedAgent(agentClass, agent), }]++ if probe := ProbeOf(parsed.Request.URI); probe != "" { r.Probes[ProbeKey{Date: date, Host: parsed.Request.Host, Probe: probe, Status: parsed.Status}]++ diff --git a/domains/platform/apis/stats/aggregate_test.go b/domains/platform/apis/stats/aggregate_test.go index d19ee4be..18b5562f 100644 --- a/domains/platform/apis/stats/aggregate_test.go +++ b/domains/platform/apis/stats/aggregate_test.go @@ -1,6 +1,7 @@ package stats import ( + "fmt" "strings" "testing" ) @@ -30,14 +31,23 @@ func TestConsumeAggregatesRequestsSlugsAndSkipsCorruptLines(t *testing.T) { if skipped != 2 { t.Errorf("skipped = %d, want 2", skipped) } - if got := rollup.Requests[RequestKey{"2026-08-30", "api.1d4.net", 200, "POST", AgentBrowser}]; got != 2 { + // Browsers are one unnamed bucket; every other row carries its agent's + // bounded name, so "did meta back off after the 403" is a query over the + // same rows the class totals come from rather than a re-aggregation. + if got := rollup.Requests[RequestKey{"2026-08-30", "api.1d4.net", 200, "POST", AgentBrowser, ""}]; got != 2 { t.Errorf("mcp browser POSTs = %d, want 2", got) } - if got := rollup.Requests[RequestKey{"2026-08-30", "git.muchq.com", 403, "GET", AgentAIScraper}]; got != 1 { + if got := rollup.Requests[RequestKey{"2026-08-30", "git.muchq.com", 403, "GET", AgentAIScraper, "meta-externalagent"}]; got != 1 { t.Errorf("blocked ai scraper = %d, want 1", got) } + if got := rollup.Requests[RequestKey{"2026-08-30", "i.iili.uk", 302, "GET", AgentBot, "curl"}]; got != 1 { + t.Errorf("curl redirects = %d, want 1", got) + } + if got := rollup.Requests[RequestKey{"2026-08-30", "i.iili.uk", 404, "GET", AgentOther, "(empty)"}]; got != 1 { + t.Errorf("empty-UA rows = %d, want 1", got) + } // An invented verb collapses like every metrics rail's method label. - if got := rollup.Requests[RequestKey{"2026-08-30", "api.muchq.com", 200, "CUSTOM", AgentOther}]; got != 1 { + if got := rollup.Requests[RequestKey{"2026-08-30", "api.muchq.com", 200, "CUSTOM", AgentOther, "(empty)"}]; got != 1 { t.Errorf("CUSTOM-method row = %d, want 1", got) } // The redirect rollup counts per slug and status, across agent classes. @@ -47,23 +57,6 @@ func TestConsumeAggregatesRequestsSlugsAndSkipsCorruptLines(t *testing.T) { if got := rollup.Slugs[SlugKey{"2026-08-30", "gone", 404}]; got != 1 { t.Errorf("gone-slug 404s = %d, want 1", got) } - // Agents are also counted by name, so "did meta back off after the 403" - // is a per-agent query rather than a re-aggregation. - if got := rollup.Agents[AgentKey{"2026-08-30", "git.muchq.com", AgentAIScraper, "meta-externalagent", 403}]; got != 1 { - t.Errorf("meta 403s = %d, want 1", got) - } - if got := rollup.Agents[AgentKey{"2026-08-30", "api.muchq.com", AgentAIScraper, "gptbot", 404}]; got != 1 { - t.Errorf("gptbot 404s = %d, want 1", got) - } - if got := rollup.Agents[AgentKey{"2026-08-30", "i.iili.uk", AgentBot, "curl", 302}]; got != 1 { - t.Errorf("curl redirects = %d, want 1", got) - } - if got := rollup.Agents[AgentKey{"2026-08-30", "api.1d4.net", AgentBrowser, "", 200}]; got != 2 { - t.Errorf("browser rows = %d, want 2 under one unnamed browser row", got) - } - if got := rollup.Agents[AgentKey{"2026-08-30", "i.iili.uk", AgentOther, "(empty)", 404}]; got != 1 { - t.Errorf("empty-UA rows = %d, want 1", got) - } // Probe rows exist only for paths that match a scanner family. if got := rollup.Probes[ProbeKey{"2026-08-30", "api.muchq.com", ProbeWordpress, 404}]; got != 1 { t.Errorf("wordpress probes = %d, want 1", got) @@ -76,6 +69,49 @@ func TestConsumeAggregatesRequestsSlugsAndSkipsCorruptLines(t *testing.T) { } } +func TestConsumeCapsTheAnonymousAgentTailPerObject(t *testing.T) { + var lines strings.Builder + for i := 0; i < maxTailAgentsPerObject+100; i++ { + fmt.Fprintf(&lines, `{"status":200,"request":{"host":"h","method":"GET","uri":"/","headers":{"User-Agent":["junk-%d/1.0"]}}}`+"\n", i) + } + // Marker-named agents arriving after the cap keep their names; only + // product tokens are capped, and browsers were never named. + lines.WriteString(`{"status":200,"request":{"host":"h","method":"GET","uri":"/","headers":{"User-Agent":["Mozilla/5.0 (compatible; GPTBot/1.2)"]}}}` + "\n") + lines.WriteString(`{"status":200,"request":{"host":"h","method":"GET","uri":"/","headers":{"User-Agent":["curl/8.6.0"]}}}` + "\n") + lines.WriteString(`{"status":200,"request":{"host":"h","method":"GET","uri":"/","headers":{"User-Agent":["Mozilla/5.0 (Macintosh) Chrome/126.0"]}}}` + "\n") + rollup := NewRollup() + + if _, err := rollup.Consume(strings.NewReader(lines.String()), "2026-08-30"); err != nil { + t.Fatal(err) + } + + var total, overflow int64 + names := map[string]bool{} + for key, count := range rollup.Requests { + total += count + if key.Agent == overflowAgent { + overflow += count + } + if key.AgentClass == AgentOther { + names[key.Agent] = true + } + } + if total != int64(maxTailAgentsPerObject+103) { + t.Errorf("total requests = %d; the cap must not lose a count", total) + } + if overflow != 100 { + t.Errorf("overflow row = %d, want the 100 past the cap", overflow) + } + if len(names) != maxTailAgentsPerObject+1 { + t.Errorf("distinct other-class names = %d, want the cap plus the overflow row", len(names)) + } + if rollup.Requests[RequestKey{"2026-08-30", "h", 200, "GET", AgentAIScraper, "gptbot"}] != 1 || + rollup.Requests[RequestKey{"2026-08-30", "h", 200, "GET", AgentBot, "curl"}] != 1 || + rollup.Requests[RequestKey{"2026-08-30", "h", 200, "GET", AgentBrowser, ""}] != 1 { + t.Errorf("marker-named and browser rows were caught by the cap: %v", rollup.Requests) + } +} + func TestConsumeSurvivesOversizedLines(t *testing.T) { huge := `{"status":200,"request":{"host":"x","method":"GET","uri":"/` + strings.Repeat("a", 2*1024*1024) + `","headers":{}}}` diff --git a/domains/platform/apis/stats/api.go b/domains/platform/apis/stats/api.go index ef01026e..c13c8185 100644 --- a/domains/platform/apis/stats/api.go +++ b/domains/platform/apis/stats/api.go @@ -13,7 +13,7 @@ import ( type Reader interface { Summary(ctx context.Context, days int) ([]SummaryRow, error) TopSlugs(ctx context.Context, days, limit int) ([]SlugRow, error) - Agents(ctx context.Context, days int) ([]AgentRow, error) + Agents(ctx context.Context, days, limit int) ([]AgentRow, error) Probes(ctx context.Context, days int) ([]ProbeRow, error) } @@ -71,7 +71,8 @@ func (h *Handlers) GetTopSlugs(w http.ResponseWriter, r *http.Request) { func (h *Handlers) GetAgents(w http.ResponseWriter, r *http.Request) { days := queryInt(r, "days", 30, 365) - rows, err := h.reader.Agents(r.Context(), days) + limit := queryInt(r, "limit", 500, 2000) + rows, err := h.reader.Agents(r.Context(), days, limit) if err != nil { h.serverError(w, "agents", err) return diff --git a/domains/platform/apis/stats/api_test.go b/domains/platform/apis/stats/api_test.go index bb44f86b..228ad54b 100644 --- a/domains/platform/apis/stats/api_test.go +++ b/domains/platform/apis/stats/api_test.go @@ -38,11 +38,11 @@ func (f *fakeReader) TopSlugs(_ context.Context, days, limit int) ([]SlugRow, er return f.slugs, nil } -func (f *fakeReader) Agents(_ context.Context, days int) ([]AgentRow, error) { +func (f *fakeReader) Agents(_ context.Context, days, limit int) ([]AgentRow, error) { if f.fail { return nil, errors.New("db is having a day") } - f.lastDays = days + f.lastDays, f.lastLimit = days, limit return f.agents, nil } @@ -109,6 +109,24 @@ func TestTopSlugsPassesWindowAndLimit(t *testing.T) { } } +func TestNonPositiveAndOversizedParametersClampRatherThan400(t *testing.T) { + reader := &fakeReader{} + handlers := handlersWith(reader) + + for _, raw := range []string{"0", "-5"} { + if recorder, _ := get(t, handlers.GetSummary, "/stats/v1/summary?days="+raw); recorder.Code != http.StatusOK { + t.Errorf("days=%s answered %d, want 200", raw, recorder.Code) + } + if reader.lastDays != 1 { + t.Errorf("days=%s clamped to %d, want 1", raw, reader.lastDays) + } + } + get(t, handlers.GetTopSlugs, "/stats/v1/iili/top?limit=99999") + if reader.lastLimit != 200 { + t.Errorf("slug limit clamped to %d, want 200", reader.lastLimit) + } +} + func TestAStoreFailureIs500WithoutTheReasonOnTheWire(t *testing.T) { recorder, _ := get(t, handlersWith(&fakeReader{fail: true}).GetSummary, "/stats/v1/summary") if recorder.Code != http.StatusInternalServerError { @@ -127,14 +145,23 @@ func TestAgentsAndProbesShareTheWindowRules(t *testing.T) { handlers := handlersWith(reader) _, body := get(t, handlers.GetAgents, "/stats/v1/agents") - if reader.lastDays != 30 { - t.Errorf("default agents window = %d, want 30", reader.lastDays) + if reader.lastDays != 30 || reader.lastLimit != 500 { + t.Errorf("default agents (days, limit) = (%d, %d), want (30, 500)", reader.lastDays, reader.lastLimit) } row := body["rows"].([]any)[0].(map[string]any) if row["agent"] != "meta-externalagent" || row["agent_class"] != AgentAIScraper || row["blocked"] != float64(9) { t.Errorf("agent row = %v", row) } + get(t, handlers.GetAgents, "/stats/v1/agents?days=99999&limit=99999") + if reader.lastDays != 365 || reader.lastLimit != 2000 { + t.Errorf("clamped agents (days, limit) = (%d, %d), want (365, 2000)", reader.lastDays, reader.lastLimit) + } + + get(t, handlers.GetProbes, "/stats/v1/probes") + if reader.lastDays != 30 { + t.Errorf("default probes window = %d, want 30", reader.lastDays) + } _, body = get(t, handlers.GetProbes, "/stats/v1/probes?days=99999") if reader.lastDays != 365 { t.Errorf("probe window clamped to %d, want 365", reader.lastDays) diff --git a/domains/platform/apis/stats/classify.go b/domains/platform/apis/stats/classify.go index 7efad8a3..182160ea 100644 --- a/domains/platform/apis/stats/classify.go +++ b/domains/platform/apis/stats/classify.go @@ -45,44 +45,52 @@ var aiScraperMarkers = []string{ "youbot", } -// Bots worth a row of their own. Search engines, SEO crawlers, link -// unfurlers, internet scanners, and the HTTP libraries scanners drive — -// the marker doubles as the agent name, so the vocabulary is this list. +// Bots worth a row of their own: search engines, SEO crawlers, link +// unfurlers, internet scanners, and the HTTP libraries scanners drive. The +// marker doubles as the agent name, so the vocabulary is this list. First +// match wins, so a marker another agent's UA quotes must come after it — +// Telegram's UA reads "TelegramBot (like TwitterBot)". var namedBotMarkers = []string{ "googlebot", "bingbot", "yandexbot", "duckduckbot", "baiduspider", "applebot", "ahrefsbot", "semrushbot", "mj12bot", "dotbot", "petalbot", "dataforseobot", - "facebookexternalhit", "twitterbot", "linkedinbot", "slackbot", "discordbot", - "telegrambot", "whatsapp", "uptimerobot", + "facebookexternalhit", "telegrambot", "twitterbot", "linkedinbot", "slackbot", + "discordbot", "whatsapp", "uptimerobot", "censysinspect", "zgrab", "nuclei", "masscan", "python-requests", "go-http-client", "okhttp", "curl", "wget", "scrapy", } +// Generic shapes that mark a bot without naming one. Anything also in +// namedBotMarkers is matched there first and does not belong here. var botMarkers = []string{ - "bot", "spider", "crawl", "curl", "wget", "python-requests", "python/", - "go-http-client", "libwww", "httpclient", "okhttp", "scrapy", "java/", - "apache-httpclient", "phantom", "headless", "scanner", "nmap", "zgrab", - "masscan", "nuclei", "censys", + "bot", "spider", "crawl", "python/", "libwww", "httpclient", "java/", + "apache-httpclient", "phantom", "headless", "scanner", "nmap", "censys", } -// AgentClassOf buckets a User-Agent header. -func AgentClassOf(userAgent string) string { - class, _ := AgentOf(userAgent) - return class -} +// markerNames is every name AgentOf can return from a marker list, so the +// rollup can tell a bounded name from a caller-shaped product token. +var markerNames = func() map[string]bool { + names := map[string]bool{} + for _, marker := range aiScraperMarkers { + names[marker] = true + } + for _, marker := range namedBotMarkers { + names[marker] = true + } + return names +}() // AgentOf buckets a User-Agent header and names the agent within the // bucket, bounded per class: AI scrapers and named bots name themselves by // the marker that matched, so those columns' vocabularies are exactly the // lists above; anonymous bots and the unclassified tail keep their product // token (one run of [a-z0-9._-], max 32 bytes) so a new crawler is readable -// before it has a marker; browsers are one unnamed bucket, because every -// browser's token is "mozilla". Order matters: AI scrapers self-identify -// with names that also match the generic bot markers. +// before it has a marker — except that a browser-shaped token is "mozilla" +// for every one of them, so a generic bot with that token is named by the +// marker it tripped ("headless", "bot") instead. Browsers are one unnamed +// bucket. Order matters: AI scrapers self-identify with names that also +// match the generic bot markers. func AgentOf(userAgent string) (class, name string) { ua := strings.ToLower(userAgent) - if ua == "" { - return AgentOther, emptyToken - } for _, marker := range aiScraperMarkers { if strings.Contains(ua, marker) { return AgentAIScraper, marker @@ -95,7 +103,10 @@ func AgentOf(userAgent string) (class, name string) { } for _, marker := range botMarkers { if strings.Contains(ua, marker) { - return AgentBot, productToken(ua) + if token := productToken(ua); token != "mozilla" { + return AgentBot, token + } + return AgentBot, marker } } if strings.HasPrefix(ua, "mozilla/") { @@ -111,7 +122,8 @@ const ( // productToken is the first run of [a-z0-9._-] in a lowercased UA — the // product name of "product/version (comment)" — truncated to a bound so a -// scanner spraying UAs cannot mint wide rows, only many. +// scanner spraying UAs cannot mint wide rows, only many (and Rollup caps +// how many). func productToken(lowerUA string) string { start := -1 for i := 0; i < len(lowerUA); i++ { @@ -145,8 +157,10 @@ func clampToken(token string) string { // The bounded scanner-path vocabulary. A family is a shape scanners probe // for on any host — WordPress logins, dotfiles, PHP endpoints on hosts that // serve no PHP — and a request either matches one family or is not a -// probe. There is deliberately no "admin" family: /admin/ is a real one_d4 -// route, and a family that counts real traffic is worse than none. +// probe. A family that also matches real traffic is worse than none, which +// is why there is no "admin" family (/admin/ is a one_d4 route) and why +// backup files match only at the root: Forgejo serves repository archives +// and raw files with the same extensions, several segments deep. const ( ProbeTraversal = "traversal" ProbeWordpress = "wordpress" @@ -175,7 +189,7 @@ var probeFamilies = []struct { {ProbeSecrets, regexp.MustCompile(`/\.aws/|/\.ssh/|id_rsa|\.htpasswd|\.htaccess|\.bash_history|/\.docker/`)}, {ProbePhpmyadmin, regexp.MustCompile(`phpmyadmin|myadmin|/pma/|adminer`)}, {ProbePhp, regexp.MustCompile(`\.php($|/)`)}, - {ProbeBackup, regexp.MustCompile(`\.(sql|bak|zip|tar|tar\.gz|tgz|rar|7z|old|orig|swp)$`)}, + {ProbeBackup, regexp.MustCompile(`^/[^/]+\.(sql|bak|zip|tar|tar\.gz|tgz|rar|7z|old|orig|swp)$`)}, {ProbeCgi, regexp.MustCompile(`/cgi-bin/`)}, {ProbeJava, regexp.MustCompile(`/actuator|/solr/|/jenkins|/manager/html|jmx-console`)}, {ProbeRouter, regexp.MustCompile(`/boaform/|/hnap1|/gponform/|/goform/|/tmui/`)}, @@ -199,21 +213,23 @@ func ProbeOf(uri string) string { } // SlugOf extracts the iili short-link slug from a request, or "" when the -// request is not a redirect lookup. Two shapes reach iili: the public -// i.iili.uk/r/{slug} host and the api.muchq.com/iili/v1/r/{slug} route. +// request is not a redirect lookup. Two shapes reach iili, and only two: +// the public i.iili.uk/r/{slug} host (GET and HEAD) and the GET-only +// api.muchq.com/iili/v1/r/{slug} route. The same path on another vhost, or +// a HEAD on the api one, never reaches iili — Caddy answers it itself — so +// it is not a follow. func SlugOf(host, method, uri string) string { - if method != "GET" && method != "HEAD" { - return "" - } path := uri if q := strings.IndexByte(path, '?'); q >= 0 { path = path[:q] } var rest string switch { - case strings.HasPrefix(host, "i.iili.uk") && strings.HasPrefix(path, "/r/"): + case strings.HasPrefix(host, "i.iili.uk") && (method == "GET" || method == "HEAD") && + strings.HasPrefix(path, "/r/"): rest = path[len("/r/"):] - case strings.HasPrefix(path, "/iili/v1/r/"): + case strings.HasPrefix(host, "api.muchq.com") && method == "GET" && + strings.HasPrefix(path, "/iili/v1/r/"): rest = path[len("/iili/v1/r/"):] default: return "" diff --git a/domains/platform/apis/stats/classify_test.go b/domains/platform/apis/stats/classify_test.go index cb4fd6b6..0fa3dca7 100644 --- a/domains/platform/apis/stats/classify_test.go +++ b/domains/platform/apis/stats/classify_test.go @@ -28,32 +28,8 @@ func TestAgentClassificationCoversTheVocabulary(t *testing.T) { {"definitely-not-a-browser", AgentOther}, } for _, c := range cases { - if got := AgentClassOf(c.ua); got != c.want { - t.Errorf("AgentClassOf(%q) = %s, want %s", c.ua, got, c.want) - } - } -} - -func TestSlugExtractionIsBoundedAndRouteScoped(t *testing.T) { - cases := []struct { - host, method, uri string - want string - }{ - {"i.iili.uk", "GET", "/r/abc123", "abc123"}, - {"i.iili.uk", "HEAD", "/r/abc123?utm=x", "abc123"}, - {"api.muchq.com", "GET", "/iili/v1/r/xyz", "xyz"}, - // POSTs are not redirect lookups; deep paths and oversized slugs - // are scanner shapes, not slugs. - {"i.iili.uk", "POST", "/r/abc123", ""}, - {"i.iili.uk", "GET", "/r/a/b", ""}, - {"i.iili.uk", "GET", "/r/", ""}, - {"i.iili.uk", "GET", "/r/" + strings.Repeat("a", 100), ""}, - {"api.muchq.com", "GET", "/portrait/v1/trace", ""}, - {"git.muchq.com", "GET", "/r/abc", ""}, - } - for _, c := range cases { - if got := SlugOf(c.host, c.method, c.uri); got != c.want { - t.Errorf("SlugOf(%q, %s, %q) = %q, want %q", c.host, c.method, c.uri, got, c.want) + if got, _ := AgentOf(c.ua); got != c.want { + t.Errorf("AgentOf(%q) = %s, want %s", c.ua, got, c.want) } } } @@ -73,7 +49,14 @@ func TestAgentNamesAreBoundedPerClass(t *testing.T) { {"curl/8.6.0", AgentBot, "curl"}, {"python-requests/2.32.0", AgentBot, "python-requests"}, {"Go-http-client/2.0", AgentBot, "go-http-client"}, - {"Mozilla/5.0 (compatible; SomeNewBot/1.0)", AgentBot, "mozilla"}, + {"my-crawler/0.1 (+https://example.com)", AgentBot, "my-crawler"}, + // Telegram quotes Twitter's marker in its own UA; the real one wins. + {"TelegramBot (like TwitterBot)", AgentBot, "telegrambot"}, + {"Twitterbot/1.0", AgentBot, "twitterbot"}, + // A browser-shaped generic bot would be "mozilla" like every browser, + // so the marker it tripped names it instead. + {"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36", AgentBot, "headless"}, + {"Mozilla/5.0 (compatible; SomeNewBot/1.0)", AgentBot, "bot"}, // Browsers are one bucket: the token would be "mozilla" for all of them. {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36", AgentBrowser, ""}, // "other" keeps its product token so the unclassified tail is readable. @@ -91,18 +74,41 @@ func TestAgentNamesAreBoundedPerClass(t *testing.T) { } } // Every marker names itself, so the agent column's vocabulary for the - // two marker classes is exactly the lists and cannot drift from them. + // two marker classes is exactly the lists and cannot drift from them — + // and markerNames knows every one of them. for _, marker := range aiScraperMarkers { ua := "Mozilla/5.0 (compatible; " + strings.ToUpper(marker) + "/1.0)" if class, name := AgentOf(ua); class != AgentAIScraper || name != marker { t.Errorf("AgentOf(%q) = (%s, %q), want (%s, %q)", ua, class, name, AgentAIScraper, marker) } + if !markerNames[marker] { + t.Errorf("markerNames lacks %q", marker) + } } for _, marker := range namedBotMarkers { ua := "Mozilla/5.0 (compatible; " + strings.ToUpper(marker) + "/1.0)" if class, name := AgentOf(ua); class != AgentBot || name != marker { t.Errorf("AgentOf(%q) = (%s, %q), want (%s, %q)", ua, class, name, AgentBot, marker) } + if !markerNames[marker] { + t.Errorf("markerNames lacks %q", marker) + } + } + // A generic marker that a named one already covers is unreachable; + // keeping the lists disjoint is what makes the named list the vocabulary. + for _, marker := range botMarkers { + if markerNames[marker] { + t.Errorf("botMarkers repeats %q, which namedBotMarkers matches first", marker) + } + } +} + +// The bare "bot" marker has no word boundary, so a phone brand ending in +// it reads as a bot. Known and kept: a boundary rule would also lose +// "Googlebot"-shaped names, and the AI list is consulted first regardless. +func TestBotSubstringHasNoWordBoundaryOnPurpose(t *testing.T) { + if class, name := AgentOf("Mozilla/5.0 (Linux; Android 10; CUBOT X30) AppleWebKit/537.36 Chrome/120 Mobile Safari/537.36"); class != AgentBot || name != "bot" { + t.Errorf("CUBOT = (%s, %q); if this changed on purpose, update the comment above", class, name) } } @@ -154,6 +160,11 @@ func TestProbeFamiliesAreBoundedAndRouteScoped(t *testing.T) { {"/admin/reanalyze", ""}, // one_d4's real admin route; "admin" is not a family {"/environment", ""}, {"/gitignore", ""}, + {"/muchq/MoonBase.git/info/refs", ""}, // an HTTP clone, not a dotdir probe + // Forgejo serves archives and raw files with backup-looking + // extensions, always several segments deep; backups probe the root. + {"/muchq/MoonBase/archive/main.tar.gz", ""}, + {"/muchq/MoonBase/raw/branch/main/migrations/V004__x.sql", ""}, } for _, c := range cases { if got := ProbeOf(c.uri); got != c.want { @@ -161,3 +172,31 @@ func TestProbeFamiliesAreBoundedAndRouteScoped(t *testing.T) { } } } + +func TestSlugExtractionIsBoundedAndRouteScoped(t *testing.T) { + cases := []struct { + host, method, uri string + want string + }{ + {"i.iili.uk", "GET", "/r/abc123", "abc123"}, + {"i.iili.uk", "HEAD", "/r/abc123?utm=x", "abc123"}, + {"api.muchq.com", "GET", "/iili/v1/r/xyz", "xyz"}, + // POSTs are not redirect lookups; deep paths and oversized slugs + // are scanner shapes, not slugs. + {"i.iili.uk", "POST", "/r/abc123", ""}, + {"i.iili.uk", "GET", "/r/a/b", ""}, + {"i.iili.uk", "GET", "/r/", ""}, + {"i.iili.uk", "GET", "/r/" + strings.Repeat("a", 100), ""}, + {"api.muchq.com", "GET", "/portrait/v1/trace", ""}, + {"git.muchq.com", "GET", "/r/abc", ""}, + // Only api.muchq.com routes /iili/v1/r/ to iili, and only for GET; + // anywhere else Caddy answers the path itself, so nothing was followed. + {"gpt.muchq.com", "GET", "/iili/v1/r/anything", ""}, + {"api.muchq.com", "HEAD", "/iili/v1/r/xyz", ""}, + } + for _, c := range cases { + if got := SlugOf(c.host, c.method, c.uri); got != c.want { + t.Errorf("SlugOf(%q, %s, %q) = %q, want %q", c.host, c.method, c.uri, got, c.want) + } + } +} diff --git a/domains/platform/apis/stats/loop_test.go b/domains/platform/apis/stats/loop_test.go index dbe00d2e..7b3a8bc4 100644 --- a/domains/platform/apis/stats/loop_test.go +++ b/domains/platform/apis/stats/loop_test.go @@ -98,7 +98,7 @@ func TestRunOnceAggregatesNewObjectsAndSkipsProcessedAndForeignKeys(t *testing.T t.Fatal("the new object was not applied") } // The partition date keys the rollup — the object's own dt=, not today. - if got := rollup.Requests[RequestKey{"2026-08-31", "api.1d4.net", 200, "GET", AgentOther}]; got != 1 { + if got := rollup.Requests[RequestKey{"2026-08-31", "api.1d4.net", 200, "GET", AgentOther, "(empty)"}]; got != 1 { t.Errorf("rollup rows = %v", rollup.Requests) } } diff --git a/domains/platform/apis/stats/store.go b/domains/platform/apis/stats/store.go index bf14689e..5020405c 100644 --- a/domains/platform/apis/stats/store.go +++ b/domains/platform/apis/stats/store.go @@ -2,7 +2,9 @@ package stats import ( "context" + "crypto/sha256" "fmt" + "strings" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -22,8 +24,9 @@ var schema = []string{ status int NOT NULL, http_method text NOT NULL, agent_class text NOT NULL, + agent text NOT NULL, requests bigint NOT NULL, - PRIMARY KEY (dt, host, status, http_method, agent_class) + PRIMARY KEY (dt, host, status, http_method, agent_class, agent) )`, `CREATE TABLE IF NOT EXISTS iili_slug_stats ( dt date NOT NULL, @@ -32,15 +35,6 @@ var schema = []string{ requests bigint NOT NULL, PRIMARY KEY (dt, slug, status) )`, - `CREATE TABLE IF NOT EXISTS agent_stats ( - dt date NOT NULL, - host text NOT NULL, - agent_class text NOT NULL, - agent text NOT NULL, - status int NOT NULL, - requests bigint NOT NULL, - PRIMARY KEY (dt, host, agent_class, agent, status) - )`, `CREATE TABLE IF NOT EXISTS probe_stats ( dt date NOT NULL, host text NOT NULL, @@ -49,27 +43,38 @@ var schema = []string{ requests bigint NOT NULL, PRIMARY KEY (dt, host, probe, status) )`, - `CREATE TABLE IF NOT EXISTS stats_meta ( - key text PRIMARY KEY, - value text NOT NULL - )`, } -// RollupVersion is the shape of what Consume computes. Bump it when a -// rollup gains a table or a classifier changes meaning: at boot a store -// whose recorded version differs drops every aggregate and every -// processed marker in one transaction, and the next pass recomputes all -// of it from the raw lines in S3. That is what "a schema change is a -// re-aggregation" costs — one full pass — and what makes it never data -// loss. Version 2 added agent_stats and probe_stats. +// RollupVersion is the meaning of what Consume computes. Bump it when a +// classifier changes what a row means; a change to the tables themselves +// needs no bump, because the recorded version also carries a hash of the +// schema DDL. Either way, at boot a store whose recorded version differs +// drops every aggregate table and every processed marker in one +// transaction, recreates them from the schema above, and the next pass +// recomputes all of it from the raw lines in S3. That is what "a schema +// change is a re-aggregation" costs — one full pass, during which the +// served counts climb back up from zero — and what makes it never data +// loss. Version 2 named agents and added probe_stats. const RollupVersion = "2" -// The tables a re-aggregation rebuilds; every aggregate table belongs -// here, or a version bump leaves it double-counted. +func rollupVersionFor(meaning string, ddl []string) string { + sum := sha256.Sum256([]byte(strings.Join(ddl, "\n"))) + return fmt.Sprintf("%s-%x", meaning, sum[:8]) +} + +func currentRollupVersion() string { return rollupVersionFor(RollupVersion, schema) } + +// The tables a re-aggregation rebuilds; every aggregate table in schema +// belongs here, or a version bump leaves it double-counted. var rollupTables = []string{ - "processed_log_objects", "request_stats", "iili_slug_stats", "agent_stats", "probe_stats", + "processed_log_objects", "request_stats", "iili_slug_stats", "probe_stats", } +const metaSchema = `CREATE TABLE IF NOT EXISTS stats_meta ( + key text PRIMARY KEY, + value text NOT NULL +)` + type Store struct { pool *pgxpool.Pool } @@ -79,21 +84,31 @@ func NewStore(ctx context.Context, databaseURL string) (*Store, error) { if err != nil { return nil, err } + store := &Store{pool: pool} + if _, err := pool.Exec(ctx, metaSchema); err != nil { + pool.Close() + return nil, fmt.Errorf("applying schema: %w", err) + } + if err := store.dropAggregatesOnVersionChange(ctx); err != nil { + pool.Close() + return nil, err + } for _, ddl := range schema { if _, err := pool.Exec(ctx, ddl); err != nil { pool.Close() return nil, fmt.Errorf("applying schema: %w", err) } } - store := &Store{pool: pool} - if err := store.reaggregateOnVersionChange(ctx); err != nil { - pool.Close() - return nil, err - } return store, nil } -func (s *Store) reaggregateOnVersionChange(ctx context.Context) error { +// dropAggregatesOnVersionChange runs before the schema so that a version +// bump which changed a table's columns recreates it: DROP rather than +// TRUNCATE is what makes a column change and a new table the same +// operation. The version row lands in the same transaction, so a crash +// in between leaves either the old tables at the old version or no +// tables at the new one — never new-version tables holding old rows. +func (s *Store) dropAggregatesOnVersionChange(ctx context.Context) error { tx, err := s.pool.Begin(ctx) if err != nil { return err @@ -106,19 +121,17 @@ func (s *Store) reaggregateOnVersionChange(ctx context.Context) error { if err != nil && err != pgx.ErrNoRows { return fmt.Errorf("reading rollup version: %w", err) } - if recorded == RollupVersion { + if recorded == currentRollupVersion() { return nil } - // A fresh database has no aggregates to drop and no version; an old - // one has both. TRUNCATE is one statement either way. for _, table := range rollupTables { - if _, err := tx.Exec(ctx, "TRUNCATE "+table); err != nil { - return fmt.Errorf("resetting %s for re-aggregation: %w", table, err) + if _, err := tx.Exec(ctx, "DROP TABLE IF EXISTS "+table); err != nil { + return fmt.Errorf("dropping %s for re-aggregation: %w", table, err) } } if _, err := tx.Exec(ctx, `INSERT INTO stats_meta (key, value) VALUES ('rollup_version', $1) - ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, RollupVersion); err != nil { + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, currentRollupVersion()); err != nil { return err } return tx.Commit(ctx) @@ -172,11 +185,11 @@ func (s *Store) ApplyRollup(ctx context.Context, key string, rollup *Rollup) err } for k, count := range rollup.Requests { if _, err := tx.Exec(ctx, - `INSERT INTO request_stats (dt, host, status, http_method, agent_class, requests) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (dt, host, status, http_method, agent_class) + `INSERT INTO request_stats (dt, host, status, http_method, agent_class, agent, requests) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (dt, host, status, http_method, agent_class, agent) DO UPDATE SET requests = request_stats.requests + EXCLUDED.requests`, - k.Date, k.Host, k.Status, k.Method, k.AgentClass, count); err != nil { + k.Date, k.Host, k.Status, k.Method, k.AgentClass, k.Agent, count); err != nil { return err } } @@ -190,16 +203,6 @@ func (s *Store) ApplyRollup(ctx context.Context, key string, rollup *Rollup) err return err } } - for k, count := range rollup.Agents { - if _, err := tx.Exec(ctx, - `INSERT INTO agent_stats (dt, host, agent_class, agent, status, requests) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (dt, host, agent_class, agent, status) - DO UPDATE SET requests = agent_stats.requests + EXCLUDED.requests`, - k.Date, k.Host, k.AgentClass, k.Agent, k.Status, count); err != nil { - return err - } - } for k, count := range rollup.Probes { if _, err := tx.Exec(ctx, `INSERT INTO probe_stats (dt, host, probe, status, requests) @@ -228,6 +231,9 @@ type SlugRow struct { // AgentRow is one day of one named agent on one host. Blocked is the 403 // count — the signal for whether a scraper backs off after being refused. +// The agent column is the one caller-shaped key in request_stats, so the +// query takes a limit and hands back the busiest rows first: the tail a +// UA-rotating scanner leaves is exactly the part that never makes the cut. type AgentRow struct { Date string `json:"date"` Host string `json:"host"` @@ -289,15 +295,16 @@ func (s *Store) TopSlugs(ctx context.Context, days, limit int) ([]SlugRow, error return out, err } -func (s *Store) Agents(ctx context.Context, days int) ([]AgentRow, error) { +func (s *Store) Agents(ctx context.Context, days, limit int) ([]AgentRow, error) { rows, err := s.pool.Query(ctx, `SELECT dt::text, host, agent_class, agent, SUM(requests) AS requests, COALESCE(SUM(requests) FILTER (WHERE status = 403), 0) AS blocked - FROM agent_stats + FROM request_stats WHERE dt >= current_date - $1::int GROUP BY dt, host, agent_class, agent - ORDER BY dt DESC, host, agent_class, requests DESC, agent`, days) + ORDER BY requests DESC, dt DESC, host, agent_class, agent + LIMIT $2`, days, limit) if err != nil { return nil, err } diff --git a/domains/platform/apis/stats/store_test.go b/domains/platform/apis/stats/store_test.go index 66406e80..11107459 100644 --- a/domains/platform/apis/stats/store_test.go +++ b/domains/platform/apis/stats/store_test.go @@ -4,12 +4,13 @@ import ( "context" "fmt" "os" + "strings" "testing" "time" ) // Real-database coverage for the store: schema, the processed-marker -// transaction, upsert accumulation, and both read queries. Gated the same +// transaction, upsert accumulation, and every read query. Gated the same // way the repo's other Postgres suites are: without STATS_TEST_DB_URL this // skips, and CI supplies the URL from its postgres service. func testStore(t *testing.T) *Store { @@ -26,18 +27,30 @@ func testStore(t *testing.T) *Store { return store } +// The database persists across runs, so every run works under its own +// host and object key; that is what lets the assertions below be exact. +func uniqueFixture(t *testing.T) (date, host, key string) { + t.Helper() + nanos := time.Now().UnixNano() + date = time.Now().UTC().Format("2006-01-02") + host = fmt.Sprintf("host-%d.example", nanos) + key = fmt.Sprintf("logs/source=caddy/dt=%s/%s-%d.log.gz", date, t.Name(), nanos) + return date, host, key +} + func TestApplyRollupIsTransactionalIdempotentAndReadable(t *testing.T) { store := testStore(t) ctx := context.Background() - // Unique key per run: the database persists across test runs. - key := fmt.Sprintf("logs/source=caddy/dt=%s/test-%d.log.gz", - time.Now().UTC().Format("2006-01-02"), time.Now().UnixNano()) - date := time.Now().UTC().Format("2006-01-02") + date, host, key := uniqueFixture(t) rollup := NewRollup() - rollup.Requests[RequestKey{date, "test-host.example", 200, "GET", AgentBrowser}] = 5 - rollup.Requests[RequestKey{date, "test-host.example", 403, "GET", AgentAIScraper}] = 2 - rollup.Slugs[SlugKey{date, "test-slug", 302}] = 3 + rollup.Requests[RequestKey{date, host, 200, "GET", AgentBrowser, ""}] = 5 + rollup.Requests[RequestKey{date, host, 403, "GET", AgentAIScraper, "gptbot"}] = 2 + rollup.Requests[RequestKey{date, host, 200, "GET", AgentAIScraper, "gptbot"}] = 1 + rollup.Requests[RequestKey{date, host, 404, "GET", AgentOther, "(empty)"}] = 4 + rollup.Slugs[SlugKey{date, host + "-slug", 302}] = 3 + rollup.Probes[ProbeKey{date, host, ProbeEnv, 404}] = 4 + rollup.Probes[ProbeKey{date, host, ProbeEnv, 200}] = 1 if pending, err := store.Unprocessed(ctx, []string{key}); err != nil || len(pending) != 1 { t.Fatalf("Unprocessed = (%v, %v), want the fresh key pending", pending, err) @@ -46,7 +59,8 @@ func TestApplyRollupIsTransactionalIdempotentAndReadable(t *testing.T) { t.Fatal(err) } // Applying the same object twice must not double-count: the marker's - // conflict arm turns the second application into a no-op. + // conflict arm turns the second application into a no-op. The exact + // counts below are what prove it. if err := store.ApplyRollup(ctx, key, rollup); err != nil { t.Fatal(err) } @@ -58,45 +72,108 @@ func TestApplyRollupIsTransactionalIdempotentAndReadable(t *testing.T) { if err != nil { t.Fatal(err) } - var browser, scraper *SummaryRow - for i := range summary { - row := &summary[i] - if row.Host == "test-host.example" && row.AgentClass == AgentBrowser { - browser = row - } - if row.Host == "test-host.example" && row.AgentClass == AgentAIScraper { - scraper = row + got := map[string]SummaryRow{} + for _, row := range summary { + if row.Host == host { + got[row.AgentClass] = row } } - if browser == nil || browser.Requests < 5 || browser.Errors != 0 { - t.Errorf("browser row = %+v", browser) + if row := got[AgentBrowser]; row.Requests != 5 || row.Errors != 0 { + t.Errorf("browser row = %+v, want 5 requests and no errors", row) } - if scraper == nil || scraper.Requests < 2 || scraper.Errors < 2 { - t.Errorf("scraper row = %+v; 403s must count as errors", scraper) + if row := got[AgentAIScraper]; row.Requests != 3 || row.Errors != 2 { + t.Errorf("scraper row = %+v, want 3 requests with the 403s as errors", row) + } + if row := got[AgentOther]; row.Requests != 4 || row.Errors != 4 { + t.Errorf("other row = %+v, want 4 requests, all errors", row) } - slugs, err := store.TopSlugs(ctx, 2, 100) + slugs, err := store.TopSlugs(ctx, 2, 1000) if err != nil { t.Fatal(err) } found := false for _, row := range slugs { - if row.Slug == "test-slug" && row.Requests >= 3 { + if row.Slug == host+"-slug" { found = true + if row.Requests != 3 { + t.Errorf("slug row = %+v, want 3", row) + } } } if !found { - t.Errorf("test-slug missing from top slugs: %v", slugs) + t.Errorf("%s-slug missing from top slugs: %v", host, slugs) + } + + agents, err := store.Agents(ctx, 2, 2000) + if err != nil { + t.Fatal(err) + } + named := map[string]AgentRow{} + for _, row := range agents { + if row.Host == host { + named[row.AgentClass+" "+row.Agent] = row + } + } + if row := named[AgentAIScraper+" gptbot"]; row.Requests != 3 || row.Blocked != 2 || row.Date != date { + t.Errorf("gptbot row = %+v, want 3 requests across statuses, 2 blocked, on %s", row, date) + } + if row := named[AgentOther+" (empty)"]; row.Requests != 4 || row.Blocked != 0 { + t.Errorf("empty-UA row = %+v; a 404 is not a block", row) + } + if row := named[AgentBrowser+" "]; row.Requests != 5 { + t.Errorf("browser row = %+v, want the unnamed bucket carried through", row) + } + + probes, err := store.Probes(ctx, 2) + if err != nil { + t.Fatal(err) + } + var env *ProbeRow + for i := range probes { + if probes[i].Host == host && probes[i].Probe == ProbeEnv { + env = &probes[i] + } + } + if env == nil || env.Requests != 5 || env.Served != 1 { + t.Errorf("env probe row = %+v; want 5 across statuses with the one 200 served", env) + } +} + +func TestAgentsHonoursTheLimitBusiestFirst(t *testing.T) { + store := testStore(t) + ctx := context.Background() + date, host, key := uniqueFixture(t) + + rollup := NewRollup() + rollup.Requests[RequestKey{date, host, 200, "GET", AgentBot, "curl"}] = 50 + rollup.Requests[RequestKey{date, host, 200, "GET", AgentBot, "wget"}] = 5 + if err := store.ApplyRollup(ctx, key, rollup); err != nil { + t.Fatal(err) + } + + agents, err := store.Agents(ctx, 2, 1) + if err != nil { + t.Fatal(err) + } + if len(agents) != 1 { + t.Fatalf("Agents(limit=1) returned %d rows", len(agents)) + } + // Other runs' rows share the window, so the survivor is whichever row + // is busiest overall; it must at least outrank this run's small one. + if agents[0].Requests < 50 { + t.Errorf("the one row kept was %+v; want the busiest, not the first", agents[0]) } } func TestAVersionBumpDropsAggregatesAndMarkersForReaggregation(t *testing.T) { store := testStore(t) ctx := context.Background() - date := time.Now().UTC().Format("2006-01-02") - key := fmt.Sprintf("logs/source=caddy/dt=%s/version-%d.log.gz", date, time.Now().UnixNano()) + date, host, key := uniqueFixture(t) rollup := NewRollup() - rollup.Requests[RequestKey{date, "version-host.example", 200, "GET", AgentBrowser}] = 1 + rollup.Requests[RequestKey{date, host, 200, "GET", AgentBot, "curl"}] = 1 + rollup.Slugs[SlugKey{date, host + "-slug", 302}] = 1 + rollup.Probes[ProbeKey{date, host, ProbeGit, 404}] = 1 if err := store.ApplyRollup(ctx, key, rollup); err != nil { t.Fatal(err) } @@ -112,8 +189,8 @@ func TestAVersionBumpDropsAggregatesAndMarkersForReaggregation(t *testing.T) { } t.Cleanup(reopened.Close) - // The marker is gone, so the next pass recomputes the object; the - // aggregates it produced are gone with it, so it is not double-counted. + // The marker is gone, so the next pass recomputes the object; every + // aggregate it produced is gone with it, so nothing is double-counted. if pending, err := reopened.Unprocessed(ctx, []string{key}); err != nil || len(pending) != 1 { t.Errorf("Unprocessed after a version bump = (%v, %v), want the key pending again", pending, err) } @@ -122,14 +199,41 @@ func TestAVersionBumpDropsAggregatesAndMarkersForReaggregation(t *testing.T) { t.Fatal(err) } for _, row := range summary { - if row.Host == "version-host.example" { - t.Errorf("aggregates survived the version bump: %+v", row) + if row.Host == host { + t.Errorf("request aggregates survived the version bump: %+v", row) + } + } + agents, err := reopened.Agents(ctx, 2, 2000) + if err != nil { + t.Fatal(err) + } + for _, row := range agents { + if row.Host == host { + t.Errorf("agent aggregates survived the version bump: %+v", row) + } + } + slugs, err := reopened.TopSlugs(ctx, 2, 1000) + if err != nil { + t.Fatal(err) + } + for _, row := range slugs { + if row.Slug == host+"-slug" { + t.Errorf("slug aggregates survived the version bump: %+v", row) + } + } + probes, err := reopened.Probes(ctx, 2) + if err != nil { + t.Fatal(err) + } + for _, row := range probes { + if row.Host == host { + t.Errorf("probe aggregates survived the version bump: %+v", row) } } var recorded string if err := reopened.pool.QueryRow(ctx, - `SELECT value FROM stats_meta WHERE key = 'rollup_version'`).Scan(&recorded); err != nil || recorded != RollupVersion { - t.Errorf("recorded version = (%q, %v), want %q", recorded, err, RollupVersion) + `SELECT value FROM stats_meta WHERE key = 'rollup_version'`).Scan(&recorded); err != nil || recorded != currentRollupVersion() { + t.Errorf("recorded version = (%q, %v), want %q", recorded, err, currentRollupVersion()) } // Reopening at the same version is a no-op: the store must not wipe @@ -146,3 +250,49 @@ func TestAVersionBumpDropsAggregatesAndMarkersForReaggregation(t *testing.T) { t.Errorf("Unprocessed after a same-version reopen = (%v, %v), want none", pending, err) } } + +// A column added to a table without anyone remembering to bump the +// constant must still re-aggregate: the recorded version carries the DDL. +func TestTheRecordedVersionFollowsTheSchemaText(t *testing.T) { + changed := append([]string{}, schema...) + changed[1] = strings.Replace(changed[1], "agent text NOT NULL,", "agent text NOT NULL,\n\t\textra int,", 1) + if changed[1] == schema[1] { + t.Fatal("the fixture did not change the DDL; the test proves nothing") + } + if rollupVersionFor(RollupVersion, changed) == rollupVersionFor(RollupVersion, schema) { + t.Error("a DDL change left the rollup version unchanged") + } + if rollupVersionFor("3", schema) == rollupVersionFor(RollupVersion, schema) { + t.Error("a meaning bump left the rollup version unchanged") + } +} + +// A version change recreates the tables rather than emptying them, so a +// table whose columns changed shape comes back in the new shape. +func TestAVersionChangeRecreatesTablesInTheirNewShape(t *testing.T) { + store := testStore(t) + ctx := context.Background() + date, host, key := uniqueFixture(t) + + // An older deployment's request_stats, without the agent column. + for _, ddl := range []string{ + `DROP TABLE request_stats`, + `CREATE TABLE request_stats (dt date NOT NULL, host text NOT NULL, requests bigint NOT NULL)`, + `UPDATE stats_meta SET value = 'older' WHERE key = 'rollup_version'`, + } { + if _, err := store.pool.Exec(ctx, ddl); err != nil { + t.Fatal(err) + } + } + + reopened, err := NewStore(ctx, os.Getenv("STATS_TEST_DB_URL")) + if err != nil { + t.Fatalf("reopening over an old-shaped table: %v", err) + } + t.Cleanup(reopened.Close) + rollup := NewRollup() + rollup.Requests[RequestKey{date, host, 200, "GET", AgentBot, "curl"}] = 1 + if err := reopened.ApplyRollup(ctx, key, rollup); err != nil { + t.Errorf("the new-shape insert failed after the version change: %v", err) + } +}