From 23d71b0566caf354ea463f84f724c5e9565b43c7 Mon Sep 17 00:00:00 2001 From: Andy Aylward Date: Tue, 1 Sep 2026 18:22:37 -0400 Subject: [PATCH 1/3] stats: own page under Projects, per-host breakdown of agents and probes The stats tab moves out of the metrics dashboard to /stats (the old /metrics/stats redirects). Each host row opens into its named AI scrapers, bots, and unclassified agents plus the scanner families that hit it, from the new /agents and /probes endpoints (MoonBase#1466); top-level tables cover AI scraper volume by day, the busiest agents, and probes across hosts. Pure rollup helpers carry the grouping so the component stays a renderer. --- README.md | 1 + src/App.tsx | 4 + src/__tests__/App.routes.test.tsx | 11 + src/apps/metrics-systems/api.ts | 30 -- .../components/StatsDashboard.tsx | 144 -------- .../__tests__/StatsDashboard.test.tsx | 96 ------ .../metrics-systems/pages/MetricsPage.tsx | 6 +- .../pages/__tests__/MetricsPage.test.tsx | 21 +- src/apps/stats/__tests__/rollup.test.ts | 85 +++++ src/apps/stats/api.ts | 62 ++++ .../components/StatsDashboard.module.css | 83 +++++ src/apps/stats/components/StatsDashboard.tsx | 326 ++++++++++++++++++ .../__tests__/StatsDashboard.test.tsx | 207 +++++++++++ src/apps/stats/pages/StatsPage.tsx | 48 +++ src/apps/stats/rollup.ts | 132 +++++++ src/shared/components/Navigation.tsx | 1 + .../components/__tests__/Navigation.test.tsx | 10 + 17 files changed, 988 insertions(+), 279 deletions(-) delete mode 100644 src/apps/metrics-systems/components/StatsDashboard.tsx delete mode 100644 src/apps/metrics-systems/components/__tests__/StatsDashboard.test.tsx create mode 100644 src/apps/stats/__tests__/rollup.test.ts create mode 100644 src/apps/stats/api.ts create mode 100644 src/apps/stats/components/StatsDashboard.module.css create mode 100644 src/apps/stats/components/StatsDashboard.tsx create mode 100644 src/apps/stats/components/__tests__/StatsDashboard.test.tsx create mode 100644 src/apps/stats/pages/StatsPage.tsx create mode 100644 src/apps/stats/rollup.ts diff --git a/README.md b/README.md index dcfeae6..25c653d 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ npm run deploy | `/posterize` | Image posterizer | | `/wordchains` | Word chain puzzles | | `/metrics` | Live service/host dashboards for the backend fleet | +| `/stats` | Traffic aggregates from the access logs: crawlers, scanners, short-link popularity | | `/resilience` | Distributed-systems game | | `/groups`, `/sets`, `/top` | Math learning modules (permutation groups, sets, Topology Quest) | diff --git a/src/App.tsx b/src/App.tsx index f21f523..e002922 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -12,6 +12,7 @@ import PosterizePage from './apps/posterize/pages/PosterizePage' import SystemsPage from './apps/metrics-systems/pages/SystemsPage' import ResilienceGamePage from './apps/metrics-systems/pages/ResilienceGamePage' import MetricsPage from './apps/metrics-systems/pages/MetricsPage' +import StatsPage from './apps/stats/pages/StatsPage' import WordchainsPage from './apps/wordchains/pages/WordchainsPage' import IiliPage from './apps/iili/pages/IiliPage' import NotFoundPage from './core/pages/NotFoundPage' @@ -34,7 +35,10 @@ function App() { } /> } /> } /> + {/* The stats tab moved out of metrics to its own page; old links still arrive. */} + } /> } /> + } /> } /> } /> } /> diff --git a/src/__tests__/App.routes.test.tsx b/src/__tests__/App.routes.test.tsx index ed265d8..5814ff2 100644 --- a/src/__tests__/App.routes.test.tsx +++ b/src/__tests__/App.routes.test.tsx @@ -21,4 +21,15 @@ describe('App routes', () => { at('/r3dr') expect(within(screen.getByRole('navigation')).getByText('MuchQ : iili')).toBeDefined() }) + + it('serves the traffic stats at /stats', () => { + at('/stats') + expect(within(screen.getByRole('navigation')).getByText('MuchQ : Stats')).toBeDefined() + }) + + // The stats view began life as a metrics tab; bookmarks of it still arrive. + it('redirects the old /metrics/stats tab to /stats', () => { + at('/metrics/stats') + expect(within(screen.getByRole('navigation')).getByText('MuchQ : Stats')).toBeDefined() + }) }) diff --git a/src/apps/metrics-systems/api.ts b/src/apps/metrics-systems/api.ts index 96b5a07..21a24c3 100644 --- a/src/apps/metrics-systems/api.ts +++ b/src/apps/metrics-systems/api.ts @@ -416,33 +416,3 @@ export function splitHostTimeseries(merged: TimeSeriesResponse): { } return { system, container } } - -// --- Log-derived stats (MoonBase#1460): aggregates the stats service -// computes from shipped Caddy access logs. Separate base URL because the -// backend is a different service behind the same gateway. - -export const STATS_API_URL = - import.meta.env.VITE_STATS_API_URL || 'https://api.muchq.com/stats/v1' - -export interface StatsSummaryRow { - date: string - host: string - agent_class: string - requests: number - errors: number -} - -export interface StatsSummary { - days: number - rows: StatsSummaryRow[] -} - -export interface TopSlugRow { - slug: string - requests: number -} - -export interface TopSlugs { - days: number - rows: TopSlugRow[] -} diff --git a/src/apps/metrics-systems/components/StatsDashboard.tsx b/src/apps/metrics-systems/components/StatsDashboard.tsx deleted file mode 100644 index a555b48..0000000 --- a/src/apps/metrics-systems/components/StatsDashboard.tsx +++ /dev/null @@ -1,144 +0,0 @@ -import { useEffect, useMemo, useState } from 'react' -import styles from './MetricsDashboard.module.css' -import { ConnectionState } from '@/shared/components/nav/ConnectionStatus' -import { - STATS_API_URL, - fetchJson, - type StatsSummary, - type TopSlugs, -} from '../api' - -interface Props { - onConnectionStateChange: (status: ConnectionState) => void -} - -// Traffic stats derived from shipped Caddy access logs (MoonBase#1460): -// who is crawling (the four agent classes, AI scrapers split out), per -// vhost, plus the most-followed iili short links. Counts refresh on the -// aggregator's own cadence, so this fetches once per mount rather than -// polling. -const StatsDashboard = ({ onConnectionStateChange }: Props) => { - const [summary, setSummary] = useState(null) - const [slugs, setSlugs] = useState(null) - const [loaded, setLoaded] = useState(false) - - useEffect(() => { - let cancelled = false - onConnectionStateChange('connecting') - Promise.all([ - fetchJson(`${STATS_API_URL}/summary?days=7`), - fetchJson(`${STATS_API_URL}/iili/top?days=30&limit=20`), - ]).then(([summaryResult, slugResult]) => { - if (cancelled) return - setSummary(summaryResult) - setSlugs(slugResult) - setLoaded(true) - onConnectionStateChange(summaryResult ? 'connected' : 'failed') - }) - return () => { - cancelled = true - } - }, [onConnectionStateChange]) - - // One row per host over the window, with the agent classes as columns — - // "how much of my traffic is AI scrapers" should be one glance, not a - // pivot the reader does in their head. - const byHost = useMemo(() => { - const hosts = new Map< - string, - { total: number; errors: number; classes: Record } - >() - for (const row of summary?.rows ?? []) { - const entry = hosts.get(row.host) ?? { total: 0, errors: 0, classes: {} } - entry.total += row.requests - entry.errors += row.errors - entry.classes[row.agent_class] = (entry.classes[row.agent_class] ?? 0) + row.requests - hosts.set(row.host, entry) - } - return [...hosts.entries()].sort((a, b) => b[1].total - a[1].total) - }, [summary]) - - if (!loaded) { - return
Loading stats…
- } - if (!summary) { - return ( -
- Stats API unavailable. The stats profile may not be deployed yet. -
- ) - } - - const agentClasses = ['browser', 'ai_scraper', 'bot', 'other'] - - return ( -
-
-

Traffic by host — last {summary.days} days

-
- - - - - - - - - - - - - - {byHost.map(([host, entry]) => ( - - - - - {agentClasses.map((agentClass) => ( - - ))} - - ))} - {byHost.length === 0 && ( - - - - )} - -
HostRequestsErrorsBrowserAI scrapersBotsOther
{host}{entry.total.toLocaleString()}{entry.errors.toLocaleString()}{(entry.classes[agentClass] ?? 0).toLocaleString()}
No aggregated traffic yet.
-
-
- -
-

- Top short links — last {slugs?.days ?? 30} days -

-
- - - - - - - - - {(slugs?.rows ?? []).map((row) => ( - - - - - ))} - {(slugs?.rows ?? []).length === 0 && ( - - - - )} - -
SlugFollows
{row.slug}{row.requests.toLocaleString()}
No redirects aggregated yet.
-
-
-
- ) -} - -export default StatsDashboard diff --git a/src/apps/metrics-systems/components/__tests__/StatsDashboard.test.tsx b/src/apps/metrics-systems/components/__tests__/StatsDashboard.test.tsx deleted file mode 100644 index e710fc8..0000000 --- a/src/apps/metrics-systems/components/__tests__/StatsDashboard.test.tsx +++ /dev/null @@ -1,96 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' -import { render, screen, cleanup, within } from '@testing-library/react' -import StatsDashboard from '../StatsDashboard' - -const summaryResponse = { - days: 7, - rows: [ - { date: '2026-08-30', host: 'api.1d4.net', agent_class: 'browser', requests: 100, errors: 2 }, - { date: '2026-08-31', host: 'api.1d4.net', agent_class: 'browser', requests: 50, errors: 0 }, - { date: '2026-08-30', host: 'git.muchq.com', agent_class: 'ai_scraper', requests: 900, errors: 700 }, - ], -} - -const slugsResponse = { - days: 30, - rows: [ - { slug: 'abc123', requests: 41 }, - { slug: 'xyz', requests: 7 }, - ], -} - -function mockFetch(bodies: Record) { - vi.stubGlobal( - 'fetch', - vi.fn(async (url: string) => { - for (const [fragment, body] of Object.entries(bodies)) { - if (url.includes(fragment)) { - return new Response(JSON.stringify(body), { status: 200 }) - } - } - return new Response('', { status: 500 }) - }) - ) -} - -afterEach(() => { - cleanup() - vi.unstubAllGlobals() -}) - -beforeEach(() => { - vi.restoreAllMocks() -}) - -describe('StatsDashboard', () => { - it('rolls the summary up per host with the agent classes as columns', async () => { - mockFetch({ '/summary': summaryResponse, '/iili/top': slugsResponse }) - const onState = vi.fn() - - render() - - // git.muchq.com sorts first (900 > 150), and the two api.1d4.net days - // roll up into one row. - const gitRow = (await screen.findByText('git.muchq.com')).closest('tr')! - expect(within(gitRow).getAllByRole('cell').map((c) => c.textContent)).toEqual([ - 'git.muchq.com', - '900', - '700', - '0', - '900', - '0', - '0', - ]) - const apiRow = screen.getByText('api.1d4.net').closest('tr')! - expect(within(apiRow).getAllByRole('cell').map((c) => c.textContent)).toEqual([ - 'api.1d4.net', - '150', - '2', - '150', - '0', - '0', - '0', - ]) - expect(screen.getByText('abc123')).toBeInTheDocument() - expect(onState).toHaveBeenLastCalledWith('connected') - }) - - it('reports failure without rendering a broken table when the API is down', async () => { - mockFetch({}) - const onState = vi.fn() - - render() - - expect(await screen.findByText(/Stats API unavailable/)).toBeInTheDocument() - expect(onState).toHaveBeenLastCalledWith('failed') - }) - - it('renders empty states rather than empty tables', async () => { - mockFetch({ '/summary': { days: 7, rows: [] }, '/iili/top': { days: 30, rows: [] } }) - - render() - - expect(await screen.findByText('No aggregated traffic yet.')).toBeInTheDocument() - expect(screen.getByText('No redirects aggregated yet.')).toBeInTheDocument() - }) -}) diff --git a/src/apps/metrics-systems/pages/MetricsPage.tsx b/src/apps/metrics-systems/pages/MetricsPage.tsx index 262d504..8bba54d 100644 --- a/src/apps/metrics-systems/pages/MetricsPage.tsx +++ b/src/apps/metrics-systems/pages/MetricsPage.tsx @@ -5,7 +5,6 @@ import ConnectionStatus, { ConnectionState } from '@/shared/components/nav/Conne import RotatingText from '@/shared/components/nav/RotatingText' import MetricsDashboard from '../components/MetricsDashboard' import ContainersDashboard from '../components/ContainersDashboard' -import StatsDashboard from '../components/StatsDashboard' import ServiceDashboard from '../components/ServiceDashboard' import styles from '../components/MetricsDashboard.module.css' import { METRICS_API_URL, fetchJson, serviceDisplayName, type ServiceCatalog } from '../api' @@ -33,7 +32,7 @@ const LEGACY_TABS: Record = { // Tabs that aren't services, so the catalog can't vouch for them and the // unknown-tab redirect must not bounce them. -const BUILT_IN_TABS = ['host', 'containers', 'stats'] +const BUILT_IN_TABS = ['host', 'containers'] const MetricsPage = () => { const { tab } = useParams<{ tab: string }>() @@ -73,7 +72,6 @@ const MetricsPage = () => { const tabs = [ { id: 'host', label: 'Host' }, { id: 'containers', label: 'Containers' }, - { id: 'stats', label: 'Stats' }, ...(catalog?.services ?? []).map((service) => ({ id: service.name, label: serviceDisplayName(service.name), @@ -97,8 +95,6 @@ const MetricsPage = () => { ) : activeTab === 'containers' ? ( - ) : activeTab === 'stats' ? ( - ) : ( )} diff --git a/src/apps/metrics-systems/pages/__tests__/MetricsPage.test.tsx b/src/apps/metrics-systems/pages/__tests__/MetricsPage.test.tsx index 9174d9a..0a206f7 100644 --- a/src/apps/metrics-systems/pages/__tests__/MetricsPage.test.tsx +++ b/src/apps/metrics-systems/pages/__tests__/MetricsPage.test.tsx @@ -85,11 +85,12 @@ describe('MetricsPage', () => { await new Promise((resolve) => setTimeout(resolve, 100)) }) - // Host, Containers and Stats are the tabs that aren't services: the - // catalog never lists them, so they're prepended rather than derived - // from it. + // Host and Containers are the tabs that aren't services: the catalog + // never lists them, so they're prepended rather than derived from it. + // Stats is not a tab any more; it has its own page at /stats. const tabs = screen.getAllByRole('button').map((button) => button.textContent) - expect(tabs.slice(0, 6)).toEqual(['Host', 'Containers', 'Stats', 'Golf Hub', 'MicroGPT', 'Portrait']) + expect(tabs.slice(0, 5)).toEqual(['Host', 'Containers', 'Golf Hub', 'MicroGPT', 'Portrait']) + expect(tabs).not.toContain('Stats') expect(screen.getByText('Host Metrics')).toBeTruthy() }) @@ -128,6 +129,18 @@ describe('MetricsPage', () => { expect(screen.getByTestId('location').textContent).toBe('/metrics/microgpt-serve') }) + it('bounces the retired stats tab to host like any unknown name', async () => { + // App.tsx redirects /metrics/stats to /stats before this page sees it; + // if that route ever goes, the tab must not resurrect an empty page. + renderAt('/metrics/stats') + + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 100)) + }) + + expect(screen.getByTestId('location').textContent).toBe('/metrics/host') + }) + it('bounces unknown services to host once the catalog loads', async () => { renderAt('/metrics/nonesuch') diff --git a/src/apps/stats/__tests__/rollup.test.ts b/src/apps/stats/__tests__/rollup.test.ts new file mode 100644 index 0000000..dc751d7 --- /dev/null +++ b/src/apps/stats/__tests__/rollup.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest' +import { rollupHosts, scrapersByDay, topAgents } from '../rollup' + +const summary = { + days: 30, + rows: [ + { date: '2026-08-30', host: 'a', agent_class: 'browser', requests: 5, errors: 1 }, + { date: '2026-08-31', host: 'a', agent_class: 'ai_scraper', requests: 7, errors: 7 }, + { date: '2026-08-30', host: 'b', agent_class: 'bot', requests: 20, errors: 0 }, + ], +} + +const agents = { + days: 30, + rows: [ + { date: '2026-08-30', host: 'a', agent_class: 'ai_scraper', agent: 'gptbot', requests: 3, blocked: 3 }, + { date: '2026-08-31', host: 'a', agent_class: 'ai_scraper', agent: 'gptbot', requests: 4, blocked: 4 }, + { date: '2026-08-31', host: 'a', agent_class: 'ai_scraper', agent: 'claudebot', requests: 9, blocked: 0 }, + { date: '2026-08-30', host: 'b', agent_class: 'bot', agent: 'curl', requests: 20, blocked: 0 }, + { date: '2026-08-30', host: 'b', agent_class: 'ai_scraper', agent: 'gptbot', requests: 1, blocked: 0 }, + { date: '2026-08-30', host: 'a', agent_class: 'browser', agent: '', requests: 5, blocked: 0 }, + ], +} + +const probes = { + days: 30, + rows: [ + { host: 'a', probe: 'env', requests: 2, served: 0 }, + { host: 'a', probe: 'wordpress', requests: 8, served: 1 }, + { host: 'c', probe: 'git', requests: 1, served: 0 }, + ], +} + +describe('rollupHosts', () => { + it('sums the summary per host and attaches named agents and probes, busiest first', () => { + const hosts = rollupHosts(summary, agents, probes) + + expect(hosts.map((h) => h.host)).toEqual(['b', 'a', 'c']) + const a = hosts[1] + expect(a.total).toBe(12) + expect(a.errors).toBe(8) + expect(a.classes).toEqual({ browser: 5, ai_scraper: 7 }) + // Two gptbot days merge; claudebot's 9 outranks the merged 7. + expect(a.agents.ai_scraper).toEqual([ + { agent: 'claudebot', requests: 9, blocked: 0 }, + { agent: 'gptbot', requests: 7, blocked: 7 }, + ]) + expect(a.agents.browser).toEqual([{ agent: '', requests: 5, blocked: 0 }]) + expect(a.probes.map((p) => p.probe)).toEqual(['wordpress', 'env']) + }) + + it('keeps a host that only the probes mention', () => { + const c = rollupHosts(summary, agents, probes).find((h) => h.host === 'c')! + expect(c.total).toBe(0) + expect(c.probes).toEqual([{ host: 'c', probe: 'git', requests: 1, served: 0 }]) + }) + + it('tolerates missing responses', () => { + expect(rollupHosts(null, null, null)).toEqual([]) + expect(rollupHosts(summary, null, null).map((h) => h.agents)).toEqual([{}, {}]) + }) +}) + +describe('scrapersByDay', () => { + it('sums AI scrapers only, across hosts, newest day first', () => { + expect(scrapersByDay(agents)).toEqual([ + { date: '2026-08-31', requests: 13, blocked: 4 }, + { date: '2026-08-30', requests: 4, blocked: 3 }, + ]) + }) +}) + +describe('topAgents', () => { + it('ranks named agents across hosts, counts hosts, and leaves browsers out', () => { + expect(topAgents(agents, 10)).toEqual([ + { agent: 'curl', agent_class: 'bot', requests: 20, blocked: 0, hosts: 1 }, + { agent: 'claudebot', agent_class: 'ai_scraper', requests: 9, blocked: 0, hosts: 1 }, + { agent: 'gptbot', agent_class: 'ai_scraper', requests: 8, blocked: 7, hosts: 2 }, + ]) + }) + + it('honours the limit', () => { + expect(topAgents(agents, 1).map((a) => a.agent)).toEqual(['curl']) + }) +}) diff --git a/src/apps/stats/api.ts b/src/apps/stats/api.ts new file mode 100644 index 0000000..b7330dc --- /dev/null +++ b/src/apps/stats/api.ts @@ -0,0 +1,62 @@ +// Log-derived stats (MoonBase#1460, #1458): aggregates the stats service +// computes from shipped Caddy access logs. Its own base URL because the +// backend is a different service behind the same gateway as the metrics +// API, whose fetch helper is reused. +export { fetchJson } from '@/apps/metrics-systems/api' + +export const STATS_API_URL = + import.meta.env.VITE_STATS_API_URL || 'https://api.muchq.com/stats/v1' + +export interface StatsSummaryRow { + date: string + host: string + agent_class: string + requests: number + errors: number +} + +export interface StatsSummary { + days: number + rows: StatsSummaryRow[] +} + +export interface TopSlugRow { + slug: string + requests: number +} + +export interface TopSlugs { + days: number + rows: TopSlugRow[] +} + +// One day of one named agent on one host. The name is bounded per class +// on the server: the marker for AI scrapers and named bots, the UA's +// product token for the rest, empty for browsers. Blocked is the 403 count. +export interface AgentRow { + date: string + host: string + agent_class: string + agent: string + requests: number + blocked: number +} + +export interface StatsAgents { + days: number + rows: AgentRow[] +} + +// One scanner family on one host over the window; served is how many of +// those probes got a sub-400 answer. +export interface ProbeRow { + host: string + probe: string + requests: number + served: number +} + +export interface StatsProbes { + days: number + rows: ProbeRow[] +} diff --git a/src/apps/stats/components/StatsDashboard.module.css b/src/apps/stats/components/StatsDashboard.module.css new file mode 100644 index 0000000..f5d8c9e --- /dev/null +++ b/src/apps/stats/components/StatsDashboard.module.css @@ -0,0 +1,83 @@ +/* The host row is a disclosure: the whole first cell is the button so the + hit target is the host name, not a chevron. */ +.hostToggle { + background: none; + border: none; + color: inherit; + font: inherit; + cursor: pointer; + padding: 0; + display: inline-flex; + align-items: center; + gap: 8px; +} + +.chevron { + display: inline-block; + transition: transform 0.15s ease; + color: rgba(255, 255, 255, 0.5); +} + +.chevronOpen { + transform: rotate(90deg); +} + +.hostRow td { + cursor: pointer; +} + +.hostRow:hover td { + background: rgba(255, 255, 255, 0.04); +} + +.detailCell { + padding: 0 !important; + background: rgba(255, 255, 255, 0.02); +} + +.detailGrid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); + gap: 16px 32px; + padding: 16px 24px 20px; +} + +.detailTitle { + font-size: 0.8rem; + font-weight: 500; + text-transform: uppercase; + letter-spacing: 0.06em; + color: rgba(255, 255, 255, 0.55); + margin: 0 0 8px; +} + +.detailTable { + width: 100%; + border-collapse: collapse; + font-size: 0.8rem; +} + +.detailTable td { + padding: 4px 8px 4px 0; + border-bottom: 1px solid rgba(255, 255, 255, 0.05); + vertical-align: top; +} + +.detailTable td:not(:first-child) { + text-align: right; + white-space: nowrap; +} + +.agentName { + font-family: 'JetBrains Mono', 'Fira Code', monospace; + word-break: break-all; +} + +.none { + color: rgba(255, 255, 255, 0.4); + font-size: 0.8rem; +} + +.served { + color: #ffb347; +} diff --git a/src/apps/stats/components/StatsDashboard.tsx b/src/apps/stats/components/StatsDashboard.tsx new file mode 100644 index 0000000..59eb9cc --- /dev/null +++ b/src/apps/stats/components/StatsDashboard.tsx @@ -0,0 +1,326 @@ +import { Fragment, useEffect, useMemo, useState } from 'react' +import styles from '@/apps/metrics-systems/components/MetricsDashboard.module.css' +import own from './StatsDashboard.module.css' +import { ConnectionState } from '@/shared/components/nav/ConnectionStatus' +import { + STATS_API_URL, + fetchJson, + type StatsAgents, + type StatsProbes, + type StatsSummary, + type TopSlugs, +} from '../api' +import { + AGENT_CLASSES, + CLASS_LABELS, + rollupHosts, + scrapersByDay, + topAgents, + type HostEntry, + type NamedAgent, +} from '../rollup' + +interface Props { + onConnectionStateChange: (status: ConnectionState) => void +} + +const WINDOW_DAYS = 30 +const TOP_AGENTS = 25 + +const n = (value: number) => value.toLocaleString() + +// Traffic stats derived from shipped Caddy access logs (MoonBase#1460, +// #1458): who is crawling, per vhost and by name, which scanner shapes are +// probing, plus the most-followed iili short links. Counts refresh on the +// aggregator's own cadence, so this fetches once per mount rather than +// polling. +const StatsDashboard = ({ onConnectionStateChange }: Props) => { + const [summary, setSummary] = useState(null) + const [agents, setAgents] = useState(null) + const [probes, setProbes] = useState(null) + const [slugs, setSlugs] = useState(null) + const [loaded, setLoaded] = useState(false) + const [openHost, setOpenHost] = useState(null) + + useEffect(() => { + let cancelled = false + onConnectionStateChange('connecting') + Promise.all([ + fetchJson(`${STATS_API_URL}/summary?days=${WINDOW_DAYS}`), + fetchJson(`${STATS_API_URL}/agents?days=${WINDOW_DAYS}`), + fetchJson(`${STATS_API_URL}/probes?days=${WINDOW_DAYS}`), + fetchJson(`${STATS_API_URL}/iili/top?days=${WINDOW_DAYS}&limit=20`), + ]).then(([summaryResult, agentsResult, probesResult, slugResult]) => { + if (cancelled) return + setSummary(summaryResult) + setAgents(agentsResult) + setProbes(probesResult) + setSlugs(slugResult) + setLoaded(true) + onConnectionStateChange(summaryResult ? 'connected' : 'failed') + }) + return () => { + cancelled = true + } + }, [onConnectionStateChange]) + + const hosts = useMemo(() => rollupHosts(summary, agents, probes), [summary, agents, probes]) + const byDay = useMemo(() => scrapersByDay(agents), [agents]) + const busiest = useMemo(() => topAgents(agents, TOP_AGENTS), [agents]) + + if (!loaded) { + return
Loading stats…
+ } + if (!summary) { + return ( +
+ Stats API unavailable. The stats profile may not be deployed yet. +
+ ) + } + + const days = summary.days + + return ( +
+
+

Traffic by host — last {days} days

+
+ + + + + + + {AGENT_CLASSES.map((agentClass) => ( + + ))} + + + + {hosts.map((entry) => { + const open = openHost === entry.host + const toggle = () => setOpenHost(open ? null : entry.host) + return ( + + + + + + {AGENT_CLASSES.map((agentClass) => ( + + ))} + + {open && ( + + + + )} + + ) + })} + {hosts.length === 0 && ( + + + + )} + +
HostRequestsErrors{CLASS_LABELS[agentClass]}
+ + {n(entry.total)}{n(entry.errors)}{n(entry.classes[agentClass] ?? 0)}
+ +
No aggregated traffic yet.
+
+
+ +
+
+

AI scrapers by day

+
+ + + + + + + + + + {byDay.map((day) => ( + + + + + + ))} + {byDay.length === 0 && ( + + + + )} + +
DateRequestsBlocked
{day.date}{n(day.requests)}{n(day.blocked)}
No AI scraper traffic in the window.
+
+
+ +
+

Busiest agents — last {days} days

+
+ + + + + + + + + + + + {busiest.map((agent) => ( + + + + + + + + ))} + {busiest.length === 0 && ( + + + + )} + +
AgentClassRequestsBlockedHosts
{agent.agent}{CLASS_LABELS[agent.agent_class] ?? agent.agent_class}{n(agent.requests)}{n(agent.blocked)}{n(agent.hosts)}
No named agents aggregated yet.
+
+
+
+ +
+
+

Scanner probes — last {probes?.days ?? days} days

+
+ + + + + + + + + + + {(probes?.rows ?? []).map((row) => ( + + + + + + + ))} + {(probes?.rows ?? []).length === 0 && ( + + + + )} + +
HostProbeRequestsServed
{row.host}{row.probe}{n(row.requests)} 0 ? own.served : undefined}>{n(row.served)}
No scanner probes in the window.
+
+
+ +
+

Top short links — last {slugs?.days ?? days} days

+
+ + + + + + + + + {(slugs?.rows ?? []).map((row) => ( + + + + + ))} + {(slugs?.rows ?? []).length === 0 && ( + + + + )} + +
SlugFollows
{row.slug}{n(row.requests)}
No redirects aggregated yet.
+
+
+
+
+ ) +} + +// The expanded half of a host row: the three named classes side by side +// (browsers are one bucket and have nothing to expand into) and the +// scanner families that hit this host. +const HostDetail = ({ entry }: { entry: HostEntry }) => ( +
+ {(['ai_scraper', 'bot', 'other'] as const).map((agentClass) => ( +
+

{CLASS_LABELS[agentClass]}

+ +
+ ))} +
+

Probes

+ {entry.probes.length === 0 ? ( + none + ) : ( + + + {entry.probes.map((probe) => ( + + + + + + ))} + +
{probe.probe}{n(probe.requests)} 0 ? own.served : undefined}>{n(probe.served)} served
+ )} +
+
+) + +const AgentList = ({ agents }: { agents: NamedAgent[] }) => + agents.length === 0 ? ( + none + ) : ( + + + {agents.map((agent) => ( + + + + + + ))} + +
{agent.agent}{n(agent.requests)}{n(agent.blocked)} blocked
+ ) + +export default StatsDashboard diff --git a/src/apps/stats/components/__tests__/StatsDashboard.test.tsx b/src/apps/stats/components/__tests__/StatsDashboard.test.tsx new file mode 100644 index 0000000..7ca5cf4 --- /dev/null +++ b/src/apps/stats/components/__tests__/StatsDashboard.test.tsx @@ -0,0 +1,207 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { render, screen, cleanup, within, fireEvent } from '@testing-library/react' +import StatsDashboard from '../StatsDashboard' + +const summaryResponse = { + days: 30, + rows: [ + { date: '2026-08-30', host: 'api.1d4.net', agent_class: 'browser', requests: 100, errors: 2 }, + { date: '2026-08-31', host: 'api.1d4.net', agent_class: 'browser', requests: 50, errors: 0 }, + { date: '2026-08-30', host: 'git.muchq.com', agent_class: 'ai_scraper', requests: 900, errors: 700 }, + { date: '2026-08-30', host: 'git.muchq.com', agent_class: 'bot', requests: 10, errors: 0 }, + { date: '2026-08-30', host: 'git.muchq.com', agent_class: 'other', requests: 3, errors: 3 }, + ], +} + +const agentsResponse = { + days: 30, + rows: [ + { date: '2026-08-30', host: 'git.muchq.com', agent_class: 'ai_scraper', agent: 'meta-externalagent', requests: 600, blocked: 500 }, + { date: '2026-08-31', host: 'git.muchq.com', agent_class: 'ai_scraper', agent: 'meta-externalagent', requests: 300, blocked: 200 }, + { date: '2026-08-30', host: 'git.muchq.com', agent_class: 'bot', agent: 'curl', requests: 10, blocked: 0 }, + { date: '2026-08-30', host: 'git.muchq.com', agent_class: 'other', agent: '(empty)', requests: 3, blocked: 0 }, + { date: '2026-08-30', host: 'api.1d4.net', agent_class: 'browser', agent: '', requests: 150, blocked: 0 }, + ], +} + +const probesResponse = { + days: 30, + rows: [ + { host: 'git.muchq.com', probe: 'wordpress', requests: 12, served: 0 }, + { host: 'api.1d4.net', probe: 'env', requests: 4, served: 1 }, + ], +} + +const slugsResponse = { + days: 30, + rows: [ + { slug: 'abc123', requests: 41 }, + { slug: 'xyz', requests: 7 }, + ], +} + +const everything = { + '/summary': summaryResponse, + '/agents': agentsResponse, + '/probes': probesResponse, + '/iili/top': slugsResponse, +} + +function mockFetch(bodies: Record) { + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string) => { + for (const [fragment, body] of Object.entries(bodies)) { + if (url.includes(fragment)) { + return new Response(JSON.stringify(body), { status: 200 }) + } + } + return new Response('', { status: 500 }) + }) + ) +} + +const cellsOf = (row: HTMLElement) => within(row).getAllByRole('cell').map((c) => c.textContent) + +afterEach(() => { + cleanup() + vi.unstubAllGlobals() +}) + +beforeEach(() => { + vi.restoreAllMocks() +}) + +describe('StatsDashboard', () => { + it('rolls the summary up per host with the agent classes as columns', async () => { + mockFetch(everything) + const onState = vi.fn() + + render() + + // git.muchq.com sorts first (913 > 150), and the two api.1d4.net days + // roll up into one row. The host cell is the disclosure button, so its + // text carries the chevron. + const gitRow = (await screen.findByRole('button', { name: /git\.muchq\.com/ })).closest('tr')! + expect(cellsOf(gitRow)).toEqual(['›git.muchq.com', '913', '703', '0', '900', '10', '3']) + const apiRow = screen.getByRole('button', { name: /api\.1d4\.net/ }).closest('tr')! + expect(cellsOf(apiRow)).toEqual(['›api.1d4.net', '150', '2', '150', '0', '0', '0']) + expect(screen.getByText('abc123')).toBeInTheDocument() + expect(onState).toHaveBeenLastCalledWith('connected') + }) + + it('opens a host into its named agents by class and its probes', async () => { + mockFetch(everything) + render() + + const toggle = await screen.findByRole('button', { name: /git\.muchq\.com/ }) + expect(toggle).toHaveAttribute('aria-expanded', 'false') + expect(screen.queryByTestId('host-detail-git.muchq.com')).not.toBeInTheDocument() + + fireEvent.click(toggle) + + expect(toggle).toHaveAttribute('aria-expanded', 'true') + const detail = within(screen.getByTestId('host-detail-git.muchq.com')) + // The two meta days sum; blocked comes along. + const meta = detail.getByText('meta-externalagent').closest('tr')! + expect(cellsOf(meta)).toEqual(['meta-externalagent', '900', '700 blocked']) + expect(cellsOf(detail.getByText('curl').closest('tr')!)).toEqual(['curl', '10', '0 blocked']) + // The unclassified tail is readable by product token, "(empty)" included. + expect(cellsOf(detail.getByText('(empty)').closest('tr')!)).toEqual(['(empty)', '3', '0 blocked']) + // Only this host's probes, not api.1d4.net's. + expect(cellsOf(detail.getByText('wordpress').closest('tr')!)).toEqual(['wordpress', '12', '0 served']) + expect(detail.queryByText('env')).not.toBeInTheDocument() + + // Opening another host closes the first: one open row at a time. + fireEvent.click(screen.getByRole('button', { name: /api\.1d4\.net/ })) + expect(screen.queryByTestId('host-detail-git.muchq.com')).not.toBeInTheDocument() + const api = within(screen.getByTestId('host-detail-api.1d4.net')) + expect(cellsOf(api.getByText('env').closest('tr')!)).toEqual(['env', '4', '1 served']) + // A browser-only host has nothing named to show in the three classes. + expect(api.getAllByText('none')).toHaveLength(3) + + fireEvent.click(screen.getByRole('button', { name: /api\.1d4\.net/ })) + expect(screen.queryByTestId('host-detail-api.1d4.net')).not.toBeInTheDocument() + }) + + it('shows the top-level scraper, agent, and probe views across hosts', async () => { + mockFetch(everything) + render() + + await screen.findByRole('button', { name: /git\.muchq\.com/ }) + + // AI scrapers by day, newest first, summed across hosts. + const byDay = screen.getByText('AI scrapers by day').closest('div')! + const dayRows = within(byDay).getAllByRole('row').slice(1).map(cellsOf) + expect(dayRows).toEqual([ + ['2026-08-31', '300', '200'], + ['2026-08-30', '600', '500'], + ]) + + // Busiest agents: browsers excluded, meta summed across its two days. + const busiest = screen.getByText(/Busiest agents/).closest('div')! + const agentRows = within(busiest).getAllByRole('row').slice(1).map(cellsOf) + expect(agentRows).toEqual([ + ['meta-externalagent', 'AI scrapers', '900', '700', '1'], + ['curl', 'Bots', '10', '0', '1'], + ['(empty)', 'Other', '3', '0', '1'], + ]) + + const probes = screen.getByText(/Scanner probes/).closest('div')! + const probeRows = within(probes).getAllByRole('row').slice(1).map(cellsOf) + expect(probeRows).toEqual([ + ['git.muchq.com', 'wordpress', '12', '0'], + ['api.1d4.net', 'env', '4', '1'], + ]) + }) + + it('asks for one window across all four aggregates', async () => { + mockFetch(everything) + render() + await screen.findByRole('button', { name: /git\.muchq\.com/ }) + + const urls = (fetch as unknown as ReturnType).mock.calls.map((call) => String(call[0])) + expect(urls).toHaveLength(4) + for (const url of urls) expect(url).toContain('days=30') + }) + + it('reports failure without rendering a broken table when the API is down', async () => { + mockFetch({}) + const onState = vi.fn() + + render() + + expect(await screen.findByText(/Stats API unavailable/)).toBeInTheDocument() + expect(onState).toHaveBeenLastCalledWith('failed') + }) + + it('renders empty states rather than empty tables', async () => { + mockFetch({ + '/summary': { days: 30, rows: [] }, + '/agents': { days: 30, rows: [] }, + '/probes': { days: 30, rows: [] }, + '/iili/top': { days: 30, rows: [] }, + }) + + render() + + expect(await screen.findByText('No aggregated traffic yet.')).toBeInTheDocument() + expect(screen.getByText('No AI scraper traffic in the window.')).toBeInTheDocument() + expect(screen.getByText('No named agents aggregated yet.')).toBeInTheDocument() + expect(screen.getByText('No scanner probes in the window.')).toBeInTheDocument() + expect(screen.getByText('No redirects aggregated yet.')).toBeInTheDocument() + }) + + it('still renders the host table when only the summary answers', async () => { + // The agents and probes endpoints are newer than the summary; a stats + // service from before they existed must not blank the page. + mockFetch({ '/summary': summaryResponse, '/iili/top': slugsResponse }) + const onState = vi.fn() + + render() + + const gitRow = (await screen.findByRole('button', { name: /git\.muchq\.com/ })).closest('tr')! + expect(cellsOf(gitRow)).toEqual(['›git.muchq.com', '913', '703', '0', '900', '10', '3']) + expect(onState).toHaveBeenLastCalledWith('connected') + }) +}) diff --git a/src/apps/stats/pages/StatsPage.tsx b/src/apps/stats/pages/StatsPage.tsx new file mode 100644 index 0000000..67285ec --- /dev/null +++ b/src/apps/stats/pages/StatsPage.tsx @@ -0,0 +1,48 @@ +import { useCallback, useState } from 'react' +import Navigation from '@/shared/components/Navigation' +import ConnectionStatus, { ConnectionState } from '@/shared/components/nav/ConnectionStatus' +import RotatingText from '@/shared/components/nav/RotatingText' +import styles from '@/apps/metrics-systems/components/MetricsDashboard.module.css' +import StatsDashboard from '../components/StatsDashboard' + +const statsFacts = [ + 'Every request leaves a line in the access log.', + 'Scrapers that ignore robots.txt still send a User-Agent.', + 'A 403 is a question: does the crawler come back?', + 'The probes tell you what the internet thinks you run.', +] + +// Log-derived traffic stats on their own page: the metrics dashboard +// stayed live operational series, and this is long-window aggregates +// (MoonBase#1460). Shares the dashboard chrome so the two read as kin. +const StatsPage = () => { + const [connectionStatus, setConnectionStatus] = useState('disconnected') + const handleConnectionStateChange = useCallback((status: ConnectionState) => { + setConnectionStatus(status) + }, []) + + return ( +
+ + + + + + } + /> +
+ ) +} + +export default StatsPage diff --git a/src/apps/stats/rollup.ts b/src/apps/stats/rollup.ts new file mode 100644 index 0000000..0cd0b6b --- /dev/null +++ b/src/apps/stats/rollup.ts @@ -0,0 +1,132 @@ +import type { ProbeRow, StatsAgents, StatsProbes, StatsSummary } from './api' + +export const AGENT_CLASSES = ['browser', 'ai_scraper', 'bot', 'other'] as const + +export const CLASS_LABELS: Record = { + browser: 'Browser', + ai_scraper: 'AI scrapers', + bot: 'Bots', + other: 'Other', +} + +export interface NamedAgent { + agent: string + requests: number + blocked: number +} + +export interface HostEntry { + host: string + total: number + errors: number + classes: Record + /** Per class, named agents summed over the window, busiest first. */ + agents: Record + /** Scanner families seen on this host, busiest first. */ + probes: ProbeRow[] +} + +const byRequestsDesc = (a: T, b: T) => b.requests - a.requests + +// One entry per host over the window: totals and the class columns from the +// summary, the named breakdown from the agents rollup, and the host's probe +// families. Hosts come from whichever response mentions them, so a host +// with only probe rows still gets a (zero-total) row rather than vanishing. +export function rollupHosts( + summary: StatsSummary | null, + agents: StatsAgents | null, + probes: StatsProbes | null +): HostEntry[] { + const hosts = new Map() + const entryFor = (host: string) => { + let entry = hosts.get(host) + if (!entry) { + entry = { host, total: 0, errors: 0, classes: {}, agents: {}, probes: [] } + hosts.set(host, entry) + } + return entry + } + + for (const row of summary?.rows ?? []) { + const entry = entryFor(row.host) + entry.total += row.requests + entry.errors += row.errors + entry.classes[row.agent_class] = (entry.classes[row.agent_class] ?? 0) + row.requests + } + + const named = new Map() + for (const row of agents?.rows ?? []) { + const key = `${row.host} ${row.agent_class} ${row.agent}` + let agent = named.get(key) + if (!agent) { + agent = { agent: row.agent, requests: 0, blocked: 0 } + named.set(key, agent) + const entry = entryFor(row.host) + const list = entry.agents[row.agent_class] ?? (entry.agents[row.agent_class] = []) + list.push(agent) + } + agent.requests += row.requests + agent.blocked += row.blocked + } + + for (const row of probes?.rows ?? []) { + entryFor(row.host).probes.push(row) + } + + for (const entry of hosts.values()) { + for (const list of Object.values(entry.agents)) list.sort(byRequestsDesc) + entry.probes.sort(byRequestsDesc) + } + return [...hosts.values()].sort((a, b) => b.total - a.total) +} + +export interface DayRow { + date: string + requests: number + blocked: number +} + +// AI scraper volume per day across every host, newest first: "do they back +// off after a 403" is blocked against requests over time. +export function scrapersByDay(agents: StatsAgents | null): DayRow[] { + const days = new Map() + for (const row of agents?.rows ?? []) { + if (row.agent_class !== 'ai_scraper') continue + const day = days.get(row.date) ?? { date: row.date, requests: 0, blocked: 0 } + day.requests += row.requests + day.blocked += row.blocked + days.set(row.date, day) + } + return [...days.values()].sort((a, b) => b.date.localeCompare(a.date)) +} + +export interface TopAgent extends NamedAgent { + agent_class: string + hosts: number +} + +// The busiest named agents across hosts, browsers excluded: they are one +// unnamed bucket per host and would only ever top the list. +export function topAgents(agents: StatsAgents | null, limit: number): TopAgent[] { + const totals = new Map }>() + for (const row of agents?.rows ?? []) { + if (row.agent_class === 'browser') continue + const key = `${row.agent_class} ${row.agent}` + const total = totals.get(key) ?? { + agent: row.agent, + agent_class: row.agent_class, + requests: 0, + blocked: 0, + hosts: 0, + hostSet: new Set(), + } + total.requests += row.requests + total.blocked += row.blocked + total.hostSet.add(row.host) + totals.set(key, total) + } + return [...totals.values()] + .map(({ hostSet, ...rest }) => ({ ...rest, hosts: hostSet.size })) + .sort(byRequestsDesc) + .slice(0, limit) +} diff --git a/src/shared/components/Navigation.tsx b/src/shared/components/Navigation.tsx index df9c7cc..0ac9c05 100644 --- a/src/shared/components/Navigation.tsx +++ b/src/shared/components/Navigation.tsx @@ -35,6 +35,7 @@ const MENU: MenuGroup[] = [ { label: 'Tracy', to: '/tracy' }, { label: 'Posterize', to: '/posterize' }, { label: 'Metrics', to: '/metrics' }, + { label: 'Stats', to: '/stats', description: 'Traffic from the access logs' }, { label: 'Wordchains', to: '/wordchains' }, { label: 'iili', to: '/iili', description: 'URL shortener' }, ], diff --git a/src/shared/components/__tests__/Navigation.test.tsx b/src/shared/components/__tests__/Navigation.test.tsx index 388e3f6..624ea31 100644 --- a/src/shared/components/__tests__/Navigation.test.tsx +++ b/src/shared/components/__tests__/Navigation.test.tsx @@ -55,6 +55,16 @@ describe('Navigation', () => { } }) + it('links stats as an internal Projects page beside metrics', () => { + renderWithRouter() + const groupEl = testingScreen.getByText('Projects').closest('li') + if (!groupEl) throw new Error('Projects nav group not found') + const link = within(groupEl).getByRole('link', { name: /^Stats/ }) + expect(link.getAttribute('href')).toBe('/stats') + expect(link.textContent).not.toContain('(external site)') + expect(within(link).getByText('Traffic from the access logs')).toBeDefined() + }) + it('links iili as an internal Projects page', () => { renderWithRouter() const groupEl = testingScreen.getByText('Projects').closest('li') From c309e5b6bdd73f8571df833a113fc59eedeb02d7 Mon Sep 17 00:00:00 2001 From: Andy Aylward Date: Tue, 1 Sep 2026 18:27:54 -0400 Subject: [PATCH 2/3] stats: review fixes for the stats page Ask the agents endpoint for its row ceiling so thin days of a scraper show as rows rather than gaps; let the detail cell wrap instead of inheriting the host table's nowrap; say "not available" for a table whose endpoint failed rather than claiming zero rows; drop the aria-controls that pointed at an unmounted row. Tests pin the row-click toggle, per-endpoint windows, server ordering, browser exclusion from the breakdown, and one name under two classes. --- src/apps/stats/__tests__/rollup.test.ts | 4 ++ .../components/StatsDashboard.module.css | 3 ++ src/apps/stats/components/StatsDashboard.tsx | 21 +++++--- .../__tests__/StatsDashboard.test.tsx | 52 ++++++++++++++++--- 4 files changed, 67 insertions(+), 13 deletions(-) diff --git a/src/apps/stats/__tests__/rollup.test.ts b/src/apps/stats/__tests__/rollup.test.ts index dc751d7..31aee39 100644 --- a/src/apps/stats/__tests__/rollup.test.ts +++ b/src/apps/stats/__tests__/rollup.test.ts @@ -19,6 +19,8 @@ const agents = { { date: '2026-08-30', host: 'b', agent_class: 'bot', agent: 'curl', requests: 20, blocked: 0 }, { date: '2026-08-30', host: 'b', agent_class: 'ai_scraper', agent: 'gptbot', requests: 1, blocked: 0 }, { date: '2026-08-30', host: 'a', agent_class: 'browser', agent: '', requests: 5, blocked: 0 }, + // The same name under another class is a different agent. + { date: '2026-08-30', host: 'a', agent_class: 'bot', agent: 'gptbot', requests: 2, blocked: 0 }, ], } @@ -46,6 +48,7 @@ describe('rollupHosts', () => { { agent: 'gptbot', requests: 7, blocked: 7 }, ]) expect(a.agents.browser).toEqual([{ agent: '', requests: 5, blocked: 0 }]) + expect(a.agents.bot).toEqual([{ agent: 'gptbot', requests: 2, blocked: 0 }]) expect(a.probes.map((p) => p.probe)).toEqual(['wordpress', 'env']) }) @@ -76,6 +79,7 @@ describe('topAgents', () => { { agent: 'curl', agent_class: 'bot', requests: 20, blocked: 0, hosts: 1 }, { agent: 'claudebot', agent_class: 'ai_scraper', requests: 9, blocked: 0, hosts: 1 }, { agent: 'gptbot', agent_class: 'ai_scraper', requests: 8, blocked: 7, hosts: 2 }, + { agent: 'gptbot', agent_class: 'bot', requests: 2, blocked: 0, hosts: 1 }, ]) }) diff --git a/src/apps/stats/components/StatsDashboard.module.css b/src/apps/stats/components/StatsDashboard.module.css index f5d8c9e..8d1b9ba 100644 --- a/src/apps/stats/components/StatsDashboard.module.css +++ b/src/apps/stats/components/StatsDashboard.module.css @@ -33,6 +33,9 @@ .detailCell { padding: 0 !important; background: rgba(255, 255, 255, 0.02); + /* The outer table's cells are nowrap; the detail is prose-shaped and + must wrap, or a long agent token pushes the host table sideways. */ + white-space: normal; } .detailGrid { diff --git a/src/apps/stats/components/StatsDashboard.tsx b/src/apps/stats/components/StatsDashboard.tsx index 59eb9cc..917acb8 100644 --- a/src/apps/stats/components/StatsDashboard.tsx +++ b/src/apps/stats/components/StatsDashboard.tsx @@ -26,9 +26,17 @@ interface Props { const WINDOW_DAYS = 30 const TOP_AGENTS = 25 +// The agents endpoint caps its rows (busiest first) and this is its +// ceiling; anything less and thin days of a real scraper fall off the +// by-day table as missing rows rather than zeros. +const AGENT_ROWS = 2000 const n = (value: number) => value.toLocaleString() +// A table whose endpoint failed says so; "no rows" is a claim about the +// data, and this page never got any to make it about. +const UNAVAILABLE = 'Not available from the stats service.' + // Traffic stats derived from shipped Caddy access logs (MoonBase#1460, // #1458): who is crawling, per vhost and by name, which scanner shapes are // probing, plus the most-followed iili short links. Counts refresh on the @@ -47,7 +55,7 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => { onConnectionStateChange('connecting') Promise.all([ fetchJson(`${STATS_API_URL}/summary?days=${WINDOW_DAYS}`), - fetchJson(`${STATS_API_URL}/agents?days=${WINDOW_DAYS}`), + fetchJson(`${STATS_API_URL}/agents?days=${WINDOW_DAYS}&limit=${AGENT_ROWS}`), fetchJson(`${STATS_API_URL}/probes?days=${WINDOW_DAYS}`), fetchJson(`${STATS_API_URL}/iili/top?days=${WINDOW_DAYS}&limit=20`), ]).then(([summaryResult, agentsResult, probesResult, slugResult]) => { @@ -109,7 +117,6 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => { type="button" className={own.hostToggle} aria-expanded={open} - aria-controls={`host-detail-${entry.host}`} onClick={(event) => { event.stopPropagation() toggle() @@ -128,7 +135,7 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => { ))} {open && ( - + @@ -169,7 +176,7 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => { ))} {byDay.length === 0 && ( - No AI scraper traffic in the window. + {agents ? 'No AI scraper traffic in the window.' : UNAVAILABLE} )} @@ -202,7 +209,7 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => { ))} {busiest.length === 0 && ( - No named agents aggregated yet. + {agents ? 'No named agents aggregated yet.' : UNAVAILABLE} )} @@ -235,7 +242,7 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => { ))} {(probes?.rows ?? []).length === 0 && ( - No scanner probes in the window. + {probes ? 'No scanner probes in the window.' : UNAVAILABLE} )} @@ -262,7 +269,7 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => { ))} {(slugs?.rows ?? []).length === 0 && ( - No redirects aggregated yet. + {slugs ? 'No redirects aggregated yet.' : UNAVAILABLE} )} diff --git a/src/apps/stats/components/__tests__/StatsDashboard.test.tsx b/src/apps/stats/components/__tests__/StatsDashboard.test.tsx index 7ca5cf4..31d4603 100644 --- a/src/apps/stats/components/__tests__/StatsDashboard.test.tsx +++ b/src/apps/stats/components/__tests__/StatsDashboard.test.tsx @@ -25,18 +25,20 @@ const agentsResponse = { } const probesResponse = { - days: 30, + days: 14, + // Server order is what renders; the fixture is ascending to prove it. rows: [ - { host: 'git.muchq.com', probe: 'wordpress', requests: 12, served: 0 }, { host: 'api.1d4.net', probe: 'env', requests: 4, served: 1 }, + { host: 'git.muchq.com', probe: 'wordpress', requests: 12, served: 0 }, ], } const slugsResponse = { - days: 30, + days: 90, + // Server order is what renders; the fixture is ascending to prove it. rows: [ - { slug: 'abc123', requests: 41 }, { slug: 'xyz', requests: 7 }, + { slug: 'abc123', requests: 41 }, ], } @@ -117,8 +119,16 @@ describe('StatsDashboard', () => { expect(screen.queryByTestId('host-detail-git.muchq.com')).not.toBeInTheDocument() const api = within(screen.getByTestId('host-detail-api.1d4.net')) expect(cellsOf(api.getByText('env').closest('tr')!)).toEqual(['env', '4', '1 served']) - // A browser-only host has nothing named to show in the three classes. + // A browser-only host has nothing named to show in the three classes, + // and browsers themselves are not a fourth: one bucket has no breakdown. expect(api.getAllByText('none')).toHaveLength(3) + expect(api.getAllByRole('heading', { level: 3 }).map((h) => h.textContent)).toEqual([ + 'AI scrapers', + 'Bots', + 'Other', + 'Probes', + ]) + expect(api.queryByText('150')).not.toBeInTheDocument() fireEvent.click(screen.getByRole('button', { name: /api\.1d4\.net/ })) expect(screen.queryByTestId('host-detail-api.1d4.net')).not.toBeInTheDocument() @@ -150,9 +160,32 @@ describe('StatsDashboard', () => { const probes = screen.getByText(/Scanner probes/).closest('div')! const probeRows = within(probes).getAllByRole('row').slice(1).map(cellsOf) expect(probeRows).toEqual([ - ['git.muchq.com', 'wordpress', '12', '0'], ['api.1d4.net', 'env', '4', '1'], + ['git.muchq.com', 'wordpress', '12', '0'], ]) + const links = screen.getByText(/Top short links/).closest('div')! + expect(within(links).getAllByRole('row').slice(1).map(cellsOf)).toEqual([ + ['xyz', '7'], + ['abc123', '41'], + ]) + + // Each heading carries the window its own endpoint reported. + expect(screen.getByText('Traffic by host — last 30 days')).toBeInTheDocument() + expect(screen.getByText('Busiest agents — last 30 days')).toBeInTheDocument() + expect(screen.getByText('Scanner probes — last 14 days')).toBeInTheDocument() + expect(screen.getByText('Top short links — last 90 days')).toBeInTheDocument() + }) + + it('toggles a host from anywhere on its row, not only the button', async () => { + mockFetch(everything) + render() + const row = (await screen.findByRole('button', { name: /git\.muchq\.com/ })).closest('tr')! + const requestsCell = within(row).getAllByRole('cell')[1] + + fireEvent.click(requestsCell) + expect(screen.getByTestId('host-detail-git.muchq.com')).toBeInTheDocument() + fireEvent.click(requestsCell) + expect(screen.queryByTestId('host-detail-git.muchq.com')).not.toBeInTheDocument() }) it('asks for one window across all four aggregates', async () => { @@ -163,6 +196,9 @@ describe('StatsDashboard', () => { const urls = (fetch as unknown as ReturnType).mock.calls.map((call) => String(call[0])) expect(urls).toHaveLength(4) for (const url of urls) expect(url).toContain('days=30') + // The agents endpoint truncates busiest-first; ask for its ceiling so + // a scraper's thin days are rows, not gaps. + expect(urls.find((url) => url.includes('/agents'))).toContain('limit=2000') }) it('reports failure without rendering a broken table when the API is down', async () => { @@ -203,5 +239,9 @@ describe('StatsDashboard', () => { const gitRow = (await screen.findByRole('button', { name: /git\.muchq\.com/ })).closest('tr')! expect(cellsOf(gitRow)).toEqual(['›git.muchq.com', '913', '703', '0', '900', '10', '3']) expect(onState).toHaveBeenLastCalledWith('connected') + // The tables whose endpoints failed say so rather than claiming zero. + expect(screen.getAllByText('Not available from the stats service.')).toHaveLength(3) + expect(screen.queryByText('No scanner probes in the window.')).not.toBeInTheDocument() + expect(screen.getByText('abc123')).toBeInTheDocument() }) }) From 703a791539bea168550fbad6fb1b3a96d7ce6217 Mon Sep 17 00:00:00 2001 From: Andy Aylward Date: Tue, 1 Sep 2026 19:38:48 -0400 Subject: [PATCH 3/3] stats: no redirect from the old metrics tab --- src/App.tsx | 2 -- src/__tests__/App.routes.test.tsx | 6 ------ .../metrics-systems/pages/__tests__/MetricsPage.test.tsx | 4 ++-- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/App.tsx b/src/App.tsx index e002922..169e4c0 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -35,8 +35,6 @@ function App() { } /> } /> } /> - {/* The stats tab moved out of metrics to its own page; old links still arrive. */} - } /> } /> } /> } /> diff --git a/src/__tests__/App.routes.test.tsx b/src/__tests__/App.routes.test.tsx index 5814ff2..6a305c2 100644 --- a/src/__tests__/App.routes.test.tsx +++ b/src/__tests__/App.routes.test.tsx @@ -26,10 +26,4 @@ describe('App routes', () => { at('/stats') expect(within(screen.getByRole('navigation')).getByText('MuchQ : Stats')).toBeDefined() }) - - // The stats view began life as a metrics tab; bookmarks of it still arrive. - it('redirects the old /metrics/stats tab to /stats', () => { - at('/metrics/stats') - expect(within(screen.getByRole('navigation')).getByText('MuchQ : Stats')).toBeDefined() - }) }) diff --git a/src/apps/metrics-systems/pages/__tests__/MetricsPage.test.tsx b/src/apps/metrics-systems/pages/__tests__/MetricsPage.test.tsx index 0a206f7..75905da 100644 --- a/src/apps/metrics-systems/pages/__tests__/MetricsPage.test.tsx +++ b/src/apps/metrics-systems/pages/__tests__/MetricsPage.test.tsx @@ -130,8 +130,8 @@ describe('MetricsPage', () => { }) it('bounces the retired stats tab to host like any unknown name', async () => { - // App.tsx redirects /metrics/stats to /stats before this page sees it; - // if that route ever goes, the tab must not resurrect an empty page. + // Stats has its own page now; the old tab name must not resurrect an + // empty page here. renderAt('/metrics/stats') await act(async () => {