From 98d152b18d98f242bbe7a4624878b8bee30d5460 Mon Sep 17 00:00:00 2001 From: Andy Aylward Date: Wed, 2 Sep 2026 09:46:41 -0400 Subject: [PATCH] stats: where scrapers, bots, and probes come from A countries table over the non-browser classes, busiest first, with the probe and refusal counts and the DB-IP attribution the data requires, and a Countries list in each host's accordion. Addresses no database placed show as Unknown. --- src/apps/stats/__tests__/rollup.test.ts | 33 ++++++- src/apps/stats/api.ts | 17 ++++ .../components/StatsDashboard.module.css | 10 +++ src/apps/stats/components/StatsDashboard.tsx | 86 ++++++++++++++++++- .../__tests__/StatsDashboard.test.tsx | 47 +++++++++- src/apps/stats/rollup.ts | 67 ++++++++++++++- 6 files changed, 251 insertions(+), 9 deletions(-) diff --git a/src/apps/stats/__tests__/rollup.test.ts b/src/apps/stats/__tests__/rollup.test.ts index 31aee39..42d86f1 100644 --- a/src/apps/stats/__tests__/rollup.test.ts +++ b/src/apps/stats/__tests__/rollup.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { rollupHosts, scrapersByDay, topAgents } from '../rollup' +import { rollupHosts, scrapersByDay, topAgents, topCountries } from '../rollup' const summary = { days: 30, @@ -87,3 +87,34 @@ describe('topAgents', () => { expect(topAgents(agents, 1).map((a) => a.agent)).toEqual(['curl']) }) }) + +describe('topCountries', () => { + const countries = { + days: 30, + rows: [ + { host: 'a', agent_class: 'ai_scraper', country: 'US', requests: 8, blocked: 7, probes: 0 }, + { host: 'b', agent_class: 'ai_scraper', country: 'US', requests: 1, blocked: 0, probes: 0 }, + { host: 'a', agent_class: 'bot', country: 'GB', requests: 10, blocked: 0, probes: 12 }, + { host: 'a', agent_class: 'other', country: '--', requests: 2, blocked: 0, probes: 1 }, + { host: 'a', agent_class: 'browser', country: 'DE', requests: 5000, blocked: 0, probes: 0 }, + ], + } + + it('sums the non-browser classes per country across hosts, busiest first', () => { + expect(topCountries(countries, 10)).toEqual([ + { country: 'GB', scrapers: 0, bots: 10, other: 0, probes: 12, blocked: 0, total: 10 }, + { country: 'US', scrapers: 9, bots: 0, other: 0, probes: 0, blocked: 7, total: 9 }, + { country: '--', scrapers: 0, bots: 0, other: 2, probes: 1, blocked: 0, total: 2 }, + ]) + expect(topCountries(countries, 1).map((c) => c.country)).toEqual(['GB']) + expect(topCountries(null, 10)).toEqual([]) + }) + + it('gives each host its own country list, browsers excluded', () => { + const hosts = rollupHosts(null, null, null, countries) + expect(hosts.map((h) => [h.host, h.countries.map((c) => `${c.country}:${c.total}`)])).toEqual([ + ['a', ['GB:10', 'US:8', '--:2']], + ['b', ['US:1']], + ]) + }) +}) diff --git a/src/apps/stats/api.ts b/src/apps/stats/api.ts index b7330dc..a30c078 100644 --- a/src/apps/stats/api.ts +++ b/src/apps/stats/api.ts @@ -60,3 +60,20 @@ export interface StatsProbes { days: number rows: ProbeRow[] } + +// One host's traffic of one class from one country over the window; +// blocked is the 403 count and probes how many were scanner probes. "--" +// is an address no database placed. +export interface CountryRow { + host: string + agent_class: string + country: string + requests: number + blocked: number + probes: number +} + +export interface StatsCountries { + days: number + rows: CountryRow[] +} diff --git a/src/apps/stats/components/StatsDashboard.module.css b/src/apps/stats/components/StatsDashboard.module.css index bd1aa6d..1e5cf72 100644 --- a/src/apps/stats/components/StatsDashboard.module.css +++ b/src/apps/stats/components/StatsDashboard.module.css @@ -92,3 +92,13 @@ .served { color: #ffb347; } + +.attribution { + margin: 8px 0 0; + font-size: 0.8rem; + color: rgba(255, 255, 255, 0.5); +} + +.attribution a { + color: inherit; +} diff --git a/src/apps/stats/components/StatsDashboard.tsx b/src/apps/stats/components/StatsDashboard.tsx index 917acb8..5052466 100644 --- a/src/apps/stats/components/StatsDashboard.tsx +++ b/src/apps/stats/components/StatsDashboard.tsx @@ -6,6 +6,7 @@ import { STATS_API_URL, fetchJson, type StatsAgents, + type StatsCountries, type StatsProbes, type StatsSummary, type TopSlugs, @@ -16,6 +17,9 @@ import { rollupHosts, scrapersByDay, topAgents, + topCountries, + UNKNOWN_COUNTRY, + type CountryTotal, type HostEntry, type NamedAgent, } from '../rollup' @@ -30,6 +34,8 @@ const TOP_AGENTS = 25 // ceiling; anything less and thin days of a real scraper fall off the // by-day table as missing rows rather than zeros. const AGENT_ROWS = 2000 +const TOP_COUNTRIES = 25 +const HOST_COUNTRIES = 8 const n = (value: number) => value.toLocaleString() @@ -46,6 +52,7 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => { const [summary, setSummary] = useState(null) const [agents, setAgents] = useState(null) const [probes, setProbes] = useState(null) + const [countries, setCountries] = useState(null) const [slugs, setSlugs] = useState(null) const [loaded, setLoaded] = useState(false) const [openHost, setOpenHost] = useState(null) @@ -58,12 +65,14 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => { fetchJson(`${STATS_API_URL}/agents?days=${WINDOW_DAYS}&limit=${AGENT_ROWS}`), fetchJson(`${STATS_API_URL}/probes?days=${WINDOW_DAYS}`), fetchJson(`${STATS_API_URL}/iili/top?days=${WINDOW_DAYS}&limit=20`), - ]).then(([summaryResult, agentsResult, probesResult, slugResult]) => { + fetchJson(`${STATS_API_URL}/countries?days=${WINDOW_DAYS}`), + ]).then(([summaryResult, agentsResult, probesResult, slugResult, countriesResult]) => { if (cancelled) return setSummary(summaryResult) setAgents(agentsResult) setProbes(probesResult) setSlugs(slugResult) + setCountries(countriesResult) setLoaded(true) onConnectionStateChange(summaryResult ? 'connected' : 'failed') }) @@ -72,9 +81,13 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => { } }, [onConnectionStateChange]) - const hosts = useMemo(() => rollupHosts(summary, agents, probes), [summary, agents, probes]) + const hosts = useMemo( + () => rollupHosts(summary, agents, probes, countries), + [summary, agents, probes, countries] + ) const byDay = useMemo(() => scrapersByDay(agents), [agents]) const busiest = useMemo(() => topAgents(agents, TOP_AGENTS), [agents]) + const fromWhere = useMemo(() => topCountries(countries, TOP_COUNTRIES), [countries]) if (!loaded) { return
Loading stats…
@@ -218,6 +231,52 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => { +
+

+ Where scrapers, bots, and probes come from — last {countries?.days ?? days} days +

+
+ + + + + + + + + + + + + + {fromWhere.map((row) => ( + + + + + + + + + + ))} + {fromWhere.length === 0 && ( + + + + )} + +
CountryRequestsAI scrapersBotsOtherProbesBlocked
{countryLabel(row.country)}{n(row.total)}{n(row.scrapers)}{n(row.bots)}{n(row.other)}{n(row.probes)}{n(row.blocked)}
{countries ? 'No non-browser traffic in the window.' : UNAVAILABLE}
+
+

+ Browsers are left out. IP geolocation by{' '} + + DB-IP + + ; addresses no database placed read as Unknown. +

+
+

Scanner probes — last {probes?.days ?? days} days

@@ -310,9 +369,32 @@ const HostDetail = ({ entry }: { entry: HostEntry }) => ( )}
+
+

Countries

+ +
) +const countryLabel = (country: string) => (country === UNKNOWN_COUNTRY ? 'Unknown' : country) + +const CountryList = ({ countries }: { countries: CountryTotal[] }) => + countries.length === 0 ? ( + none + ) : ( + + + {countries.map((row) => ( + + + + + + ))} + +
{countryLabel(row.country)}{n(row.total)}{n(row.blocked)} blocked
+ ) + const AgentList = ({ agents }: { agents: NamedAgent[] }) => agents.length === 0 ? ( none diff --git a/src/apps/stats/components/__tests__/StatsDashboard.test.tsx b/src/apps/stats/components/__tests__/StatsDashboard.test.tsx index 31d4603..5063300 100644 --- a/src/apps/stats/components/__tests__/StatsDashboard.test.tsx +++ b/src/apps/stats/components/__tests__/StatsDashboard.test.tsx @@ -42,11 +42,23 @@ const slugsResponse = { ], } +const countriesResponse = { + days: 30, + rows: [ + { host: 'git.muchq.com', agent_class: 'ai_scraper', country: 'US', requests: 800, blocked: 700, probes: 0 }, + { host: 'git.muchq.com', agent_class: 'bot', country: 'GB', requests: 10, blocked: 0, probes: 12 }, + { host: 'git.muchq.com', agent_class: 'browser', country: 'US', requests: 5000, blocked: 0, probes: 0 }, + { host: 'api.1d4.net', agent_class: 'other', country: '--', requests: 20, blocked: 0, probes: 4 }, + { host: 'api.1d4.net', agent_class: 'ai_scraper', country: 'US', requests: 100, blocked: 0, probes: 0 }, + ], +} + const everything = { '/summary': summaryResponse, '/agents': agentsResponse, '/probes': probesResponse, '/iili/top': slugsResponse, + '/countries': countriesResponse, } function mockFetch(bodies: Record) { @@ -113,6 +125,9 @@ describe('StatsDashboard', () => { // Only this host's probes, not api.1d4.net's. expect(cellsOf(detail.getByText('wordpress').closest('tr')!)).toEqual(['wordpress', '12', '0 served']) expect(detail.queryByText('env')).not.toBeInTheDocument() + // Browsers do not count toward a host's countries: US is the scraper's 800, not 5,800. + expect(cellsOf(detail.getByText('US').closest('tr')!)).toEqual(['US', '800', '700 blocked']) + expect(cellsOf(detail.getByText('GB').closest('tr')!)).toEqual(['GB', '10', '0 blocked']) // Opening another host closes the first: one open row at a time. fireEvent.click(screen.getByRole('button', { name: /api\.1d4\.net/ })) @@ -127,8 +142,14 @@ describe('StatsDashboard', () => { 'Bots', 'Other', 'Probes', + 'Countries', ]) expect(api.queryByText('150')).not.toBeInTheDocument() + // Its countries, non-browser only, busiest first, with the unplaced bucket named. + expect(api.getAllByRole('row').slice(-2).map(cellsOf)).toEqual([ + ['US', '100', '0 blocked'], + ['Unknown', '20', '0 blocked'], + ]) fireEvent.click(screen.getByRole('button', { name: /api\.1d4\.net/ })) expect(screen.queryByTestId('host-detail-api.1d4.net')).not.toBeInTheDocument() @@ -188,13 +209,31 @@ describe('StatsDashboard', () => { expect(screen.queryByTestId('host-detail-git.muchq.com')).not.toBeInTheDocument() }) - it('asks for one window across all four aggregates', async () => { + it('shows where non-browser traffic comes from, with the attribution the data requires', async () => { + mockFetch(everything) + render() + + const table = within(await screen.findByTestId('countries')) + // Summed across hosts and classes, browsers excluded, busiest first. + expect(table.getAllByRole('row').slice(1).map(cellsOf)).toEqual([ + ['US', '900', '900', '0', '0', '0', '700'], + ['Unknown', '20', '0', '0', '20', '4', '0'], + ['GB', '10', '0', '10', '0', '12', '0'], + ]) + expect(screen.getByText(/Where scrapers, bots, and probes come from — last 30 days/)).toBeInTheDocument() + // CC BY: the source is named, and linked, on the page that shows its data. + const credit = screen.getByRole('link', { name: 'DB-IP' }) + expect(credit.getAttribute('href')).toBe('https://db-ip.com') + expect(credit.closest('p')?.textContent).toContain('IP geolocation by DB-IP') + }) + + it('asks for one window across all five aggregates', async () => { mockFetch(everything) render() await screen.findByRole('button', { name: /git\.muchq\.com/ }) const urls = (fetch as unknown as ReturnType).mock.calls.map((call) => String(call[0])) - expect(urls).toHaveLength(4) + expect(urls).toHaveLength(5) for (const url of urls) expect(url).toContain('days=30') // The agents endpoint truncates busiest-first; ask for its ceiling so // a scraper's thin days are rows, not gaps. @@ -217,11 +256,13 @@ describe('StatsDashboard', () => { '/agents': { days: 30, rows: [] }, '/probes': { days: 30, rows: [] }, '/iili/top': { days: 30, rows: [] }, + '/countries': { days: 30, rows: [] }, }) render() expect(await screen.findByText('No aggregated traffic yet.')).toBeInTheDocument() + expect(screen.getByText('No non-browser traffic in the window.')).toBeInTheDocument() expect(screen.getByText('No AI scraper traffic in the window.')).toBeInTheDocument() expect(screen.getByText('No named agents aggregated yet.')).toBeInTheDocument() expect(screen.getByText('No scanner probes in the window.')).toBeInTheDocument() @@ -240,7 +281,7 @@ describe('StatsDashboard', () => { expect(cellsOf(gitRow)).toEqual(['›git.muchq.com', '913', '703', '0', '900', '10', '3']) expect(onState).toHaveBeenLastCalledWith('connected') // The tables whose endpoints failed say so rather than claiming zero. - expect(screen.getAllByText('Not available from the stats service.')).toHaveLength(3) + expect(screen.getAllByText('Not available from the stats service.')).toHaveLength(4) expect(screen.queryByText('No scanner probes in the window.')).not.toBeInTheDocument() expect(screen.getByText('abc123')).toBeInTheDocument() }) diff --git a/src/apps/stats/rollup.ts b/src/apps/stats/rollup.ts index 0cd0b6b..aa20877 100644 --- a/src/apps/stats/rollup.ts +++ b/src/apps/stats/rollup.ts @@ -1,4 +1,4 @@ -import type { ProbeRow, StatsAgents, StatsProbes, StatsSummary } from './api' +import type { CountryRow, ProbeRow, StatsAgents, StatsCountries, StatsProbes, StatsSummary } from './api' export const AGENT_CLASSES = ['browser', 'ai_scraper', 'bot', 'other'] as const @@ -24,6 +24,53 @@ export interface HostEntry { agents: Record /** Scanner families seen on this host, busiest first. */ probes: ProbeRow[] + /** Where this host's non-browser traffic came from, busiest first. */ + countries: CountryTotal[] +} + +export const UNKNOWN_COUNTRY = '--' + +// Junk traffic by country: everything that is not a browser, split by +// class, with the probe and refusal counts alongside. Browsers are left +// out on purpose — where the humans are is a different question, and one +// this page is not for. +export interface CountryTotal { + country: string + scrapers: number + bots: number + other: number + probes: number + blocked: number + total: number +} + +function addCountry(totals: Map, row: CountryRow) { + if (row.agent_class === 'browser') return + const total = totals.get(row.country) ?? { + country: row.country, + scrapers: 0, + bots: 0, + other: 0, + probes: 0, + blocked: 0, + total: 0, + } + if (row.agent_class === 'ai_scraper') total.scrapers += row.requests + else if (row.agent_class === 'bot') total.bots += row.requests + else total.other += row.requests + total.probes += row.probes + total.blocked += row.blocked + total.total += row.requests + totals.set(row.country, total) +} + +const byTotalDesc = (a: CountryTotal, b: CountryTotal) => + b.total - a.total || a.country.localeCompare(b.country) + +export function topCountries(countries: StatsCountries | null, limit: number): CountryTotal[] { + const totals = new Map() + for (const row of countries?.rows ?? []) addCountry(totals, row) + return [...totals.values()].sort(byTotalDesc).slice(0, limit) } const byRequestsDesc = (a: T, b: T) => b.requests - a.requests @@ -35,13 +82,14 @@ const byRequestsDesc = (a: T, b: T) => b.request export function rollupHosts( summary: StatsSummary | null, agents: StatsAgents | null, - probes: StatsProbes | null + probes: StatsProbes | null, + countries: StatsCountries | null = null ): HostEntry[] { const hosts = new Map() const entryFor = (host: string) => { let entry = hosts.get(host) if (!entry) { - entry = { host, total: 0, errors: 0, classes: {}, agents: {}, probes: [] } + entry = { host, total: 0, errors: 0, classes: {}, agents: {}, probes: [], countries: [] } hosts.set(host, entry) } return entry @@ -73,6 +121,19 @@ export function rollupHosts( entryFor(row.host).probes.push(row) } + const perHost = new Map>() + for (const row of countries?.rows ?? []) { + let totals = perHost.get(row.host) + if (!totals) { + totals = new Map() + perHost.set(row.host, totals) + } + addCountry(totals, row) + } + for (const [host, totals] of perHost) { + entryFor(host).countries = [...totals.values()].sort(byTotalDesc) + } + for (const entry of hosts.values()) { for (const list of Object.values(entry.agents)) list.sort(byRequestsDesc) entry.probes.sort(byRequestsDesc)