stats: where scrapers, bots, and probes come from

src/apps/stats/__tests__/rollup.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest'
-import { rollupHosts, scrapersByDay, topAgents } from '../rollup'
+import { rollupHosts, scrapersByDay, topAgents, topCountries } from '../rollup'
 
 const summary = {
   days: 30,
@@ -87,3 +87,34 @@ describe('topAgents', () => {
     expect(topAgents(agents, 1).map((a) => a.agent)).toEqual(['curl'])
   })
 })
+
+describe('topCountries', () => {
+  const countries = {
+    days: 30,
+    rows: [
+      { host: 'a', agent_class: 'ai_scraper', country: 'US', requests: 8, blocked: 7, probes: 0 },
+      { host: 'b', agent_class: 'ai_scraper', country: 'US', requests: 1, blocked: 0, probes: 0 },
+      { host: 'a', agent_class: 'bot', country: 'GB', requests: 10, blocked: 0, probes: 12 },
+      { host: 'a', agent_class: 'other', country: '--', requests: 2, blocked: 0, probes: 1 },
+      { host: 'a', agent_class: 'browser', country: 'DE', requests: 5000, blocked: 0, probes: 0 },
+    ],
+  }
+
+  it('sums the non-browser classes per country across hosts, busiest first', () => {
+    expect(topCountries(countries, 10)).toEqual([
+      { country: 'GB', scrapers: 0, bots: 10, other: 0, probes: 12, blocked: 0, total: 10 },
+      { country: 'US', scrapers: 9, bots: 0, other: 0, probes: 0, blocked: 7, total: 9 },
+      { country: '--', scrapers: 0, bots: 0, other: 2, probes: 1, blocked: 0, total: 2 },
+    ])
+    expect(topCountries(countries, 1).map((c) => c.country)).toEqual(['GB'])
+    expect(topCountries(null, 10)).toEqual([])
+  })
+
+  it('gives each host its own country list, browsers excluded', () => {
+    const hosts = rollupHosts(null, null, null, countries)
+    expect(hosts.map((h) => [h.host, h.countries.map((c) => `${c.country}:${c.total}`)])).toEqual([
+      ['a', ['GB:10', 'US:8', '--:2']],
+      ['b', ['US:1']],
+    ])
+  })
+})

src/apps/stats/api.ts

@@ -60,3 +60,20 @@ export interface StatsProbes {
   days: number
   rows: ProbeRow[]
 }
+
+// One host's traffic of one class from one country over the window;
+// blocked is the 403 count and probes how many were scanner probes. "--"
+// is an address no database placed.
+export interface CountryRow {
+  host: string
+  agent_class: string
+  country: string
+  requests: number
+  blocked: number
+  probes: number
+}
+
+export interface StatsCountries {
+  days: number
+  rows: CountryRow[]
+}

src/apps/stats/components/StatsDashboard.module.css

@@ -92,3 +92,13 @@
 .served {
   color: #ffb347;
 }
+
+.attribution {
+  margin: 8px 0 0;
+  font-size: 0.8rem;
+  color: rgba(255, 255, 255, 0.5);
+}
+
+.attribution a {
+  color: inherit;
+}

src/apps/stats/components/StatsDashboard.tsx

@@ -6,6 +6,7 @@ import {
   STATS_API_URL,
   fetchJson,
   type StatsAgents,
+  type StatsCountries,
   type StatsProbes,
   type StatsSummary,
   type TopSlugs,
@@ -16,6 +17,9 @@ import {
   rollupHosts,
   scrapersByDay,
   topAgents,
+  topCountries,
+  UNKNOWN_COUNTRY,
+  type CountryTotal,
   type HostEntry,
   type NamedAgent,
 } from '../rollup'
@@ -30,6 +34,8 @@ const TOP_AGENTS = 25
 // ceiling; anything less and thin days of a real scraper fall off the
 // by-day table as missing rows rather than zeros.
 const AGENT_ROWS = 2000
+const TOP_COUNTRIES = 25
+const HOST_COUNTRIES = 8
 
 const n = (value: number) => value.toLocaleString()
 
@@ -46,6 +52,7 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => {
   const [summary, setSummary] = useState<StatsSummary | null>(null)
   const [agents, setAgents] = useState<StatsAgents | null>(null)
   const [probes, setProbes] = useState<StatsProbes | null>(null)
+  const [countries, setCountries] = useState<StatsCountries | null>(null)
   const [slugs, setSlugs] = useState<TopSlugs | null>(null)
   const [loaded, setLoaded] = useState(false)
   const [openHost, setOpenHost] = useState<string | null>(null)
@@ -58,12 +65,14 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => {
       fetchJson<StatsAgents>(`${STATS_API_URL}/agents?days=${WINDOW_DAYS}&limit=${AGENT_ROWS}`),
       fetchJson<StatsProbes>(`${STATS_API_URL}/probes?days=${WINDOW_DAYS}`),
       fetchJson<TopSlugs>(`${STATS_API_URL}/iili/top?days=${WINDOW_DAYS}&limit=20`),
-    ]).then(([summaryResult, agentsResult, probesResult, slugResult]) => {
+      fetchJson<StatsCountries>(`${STATS_API_URL}/countries?days=${WINDOW_DAYS}`),
+    ]).then(([summaryResult, agentsResult, probesResult, slugResult, countriesResult]) => {
       if (cancelled) return
       setSummary(summaryResult)
       setAgents(agentsResult)
       setProbes(probesResult)
       setSlugs(slugResult)
+      setCountries(countriesResult)
       setLoaded(true)
       onConnectionStateChange(summaryResult ? 'connected' : 'failed')
     })
@@ -72,9 +81,13 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => {
     }
   }, [onConnectionStateChange])
 
-  const hosts = useMemo(() => rollupHosts(summary, agents, probes), [summary, agents, probes])
+  const hosts = useMemo(
+    () => rollupHosts(summary, agents, probes, countries),
+    [summary, agents, probes, countries]
+  )
   const byDay = useMemo(() => scrapersByDay(agents), [agents])
   const busiest = useMemo(() => topAgents(agents, TOP_AGENTS), [agents])
+  const fromWhere = useMemo(() => topCountries(countries, TOP_COUNTRIES), [countries])
 
   if (!loaded) {
     return <div className={styles.noData}>Loading stats…</div>
@@ -218,6 +231,52 @@ const StatsDashboard = ({ onConnectionStateChange }: Props) => {
         </div>
       </div>
 
+      <div className={styles.section}>
+        <h2 className={styles.sectionTitle}>
+          Where scrapers, bots, and probes come from — last {countries?.days ?? days} days
+        </h2>
+        <div className={styles.tableScroll}>
+          <table className={styles.containerTable} data-testid="countries">
+            <thead>
+              <tr>
+                <th>Country</th>
+                <th>Requests</th>
+                <th>AI scrapers</th>
+                <th>Bots</th>
+                <th>Other</th>
+                <th>Probes</th>
+                <th>Blocked</th>
+              </tr>
+            </thead>
+            <tbody>
+              {fromWhere.map((row) => (
+                <tr key={row.country}>
+                  <td>{countryLabel(row.country)}</td>
+                  <td>{n(row.total)}</td>
+                  <td>{n(row.scrapers)}</td>
+                  <td>{n(row.bots)}</td>
+                  <td>{n(row.other)}</td>
+                  <td>{n(row.probes)}</td>
+                  <td>{n(row.blocked)}</td>
+                </tr>
+              ))}
+              {fromWhere.length === 0 && (
+                <tr>
+                  <td colSpan={7}>{countries ? 'No non-browser traffic in the window.' : UNAVAILABLE}</td>
+                </tr>
+              )}
+            </tbody>
+          </table>
+        </div>
+        <p className={own.attribution}>
+          Browsers are left out. IP geolocation by{' '}
+          <a href="https://db-ip.com" rel="noreferrer">
+            DB-IP
+          </a>
+          ; addresses no database placed read as Unknown.
+        </p>
+      </div>
+
       <div className={styles.sectionGrid}>
         <div className={styles.section}>
           <h2 className={styles.sectionTitle}>Scanner probes — last {probes?.days ?? days} days</h2>
@@ -310,9 +369,32 @@ const HostDetail = ({ entry }: { entry: HostEntry }) => (
         </table>
       )}
     </div>
+    <div>
+      <h3 className={own.detailTitle}>Countries</h3>
+      <CountryList countries={entry.countries.slice(0, HOST_COUNTRIES)} />
+    </div>
   </div>
 )
 
+const countryLabel = (country: string) => (country === UNKNOWN_COUNTRY ? 'Unknown' : country)
+
+const CountryList = ({ countries }: { countries: CountryTotal[] }) =>
+  countries.length === 0 ? (
+    <span className={own.none}>none</span>
+  ) : (
+    <table className={own.detailTable}>
+      <tbody>
+        {countries.map((row) => (
+          <tr key={row.country}>
+            <td>{countryLabel(row.country)}</td>
+            <td>{n(row.total)}</td>
+            <td>{n(row.blocked)} blocked</td>
+          </tr>
+        ))}
+      </tbody>
+    </table>
+  )
+
 const AgentList = ({ agents }: { agents: NamedAgent[] }) =>
   agents.length === 0 ? (
     <span className={own.none}>none</span>

src/apps/stats/components/__tests__/StatsDashboard.test.tsx

@@ -42,11 +42,23 @@ const slugsResponse = {
   ],
 }
 
+const countriesResponse = {
+  days: 30,
+  rows: [
+    { host: 'git.muchq.com', agent_class: 'ai_scraper', country: 'US', requests: 800, blocked: 700, probes: 0 },
+    { host: 'git.muchq.com', agent_class: 'bot', country: 'GB', requests: 10, blocked: 0, probes: 12 },
+    { host: 'git.muchq.com', agent_class: 'browser', country: 'US', requests: 5000, blocked: 0, probes: 0 },
+    { host: 'api.1d4.net', agent_class: 'other', country: '--', requests: 20, blocked: 0, probes: 4 },
+    { host: 'api.1d4.net', agent_class: 'ai_scraper', country: 'US', requests: 100, blocked: 0, probes: 0 },
+  ],
+}
+
 const everything = {
   '/summary': summaryResponse,
   '/agents': agentsResponse,
   '/probes': probesResponse,
   '/iili/top': slugsResponse,
+  '/countries': countriesResponse,
 }
 
 function mockFetch(bodies: Record<string, unknown>) {
@@ -113,6 +125,9 @@ describe('StatsDashboard', () => {
     // Only this host's probes, not api.1d4.net's.
     expect(cellsOf(detail.getByText('wordpress').closest('tr')!)).toEqual(['wordpress', '12', '0 served'])
     expect(detail.queryByText('env')).not.toBeInTheDocument()
+    // Browsers do not count toward a host's countries: US is the scraper's 800, not 5,800.
+    expect(cellsOf(detail.getByText('US').closest('tr')!)).toEqual(['US', '800', '700 blocked'])
+    expect(cellsOf(detail.getByText('GB').closest('tr')!)).toEqual(['GB', '10', '0 blocked'])
 
     // Opening another host closes the first: one open row at a time.
     fireEvent.click(screen.getByRole('button', { name: /api\.1d4\.net/ }))
@@ -127,8 +142,14 @@ describe('StatsDashboard', () => {
       'Bots',
       'Other',
       'Probes',
+      'Countries',
     ])
     expect(api.queryByText('150')).not.toBeInTheDocument()
+    // Its countries, non-browser only, busiest first, with the unplaced bucket named.
+    expect(api.getAllByRole('row').slice(-2).map(cellsOf)).toEqual([
+      ['US', '100', '0 blocked'],
+      ['Unknown', '20', '0 blocked'],
+    ])
 
     fireEvent.click(screen.getByRole('button', { name: /api\.1d4\.net/ }))
     expect(screen.queryByTestId('host-detail-api.1d4.net')).not.toBeInTheDocument()
@@ -188,13 +209,31 @@ describe('StatsDashboard', () => {
     expect(screen.queryByTestId('host-detail-git.muchq.com')).not.toBeInTheDocument()
   })
 
-  it('asks for one window across all four aggregates', async () => {
+  it('shows where non-browser traffic comes from, with the attribution the data requires', async () => {
+    mockFetch(everything)
+    render(<StatsDashboard onConnectionStateChange={vi.fn()} />)
+
+    const table = within(await screen.findByTestId('countries'))
+    // Summed across hosts and classes, browsers excluded, busiest first.
+    expect(table.getAllByRole('row').slice(1).map(cellsOf)).toEqual([
+      ['US', '900', '900', '0', '0', '0', '700'],
+      ['Unknown', '20', '0', '0', '20', '4', '0'],
+      ['GB', '10', '0', '10', '0', '12', '0'],
+    ])
+    expect(screen.getByText(/Where scrapers, bots, and probes come from — last 30 days/)).toBeInTheDocument()
+    // CC BY: the source is named, and linked, on the page that shows its data.
+    const credit = screen.getByRole('link', { name: 'DB-IP' })
+    expect(credit.getAttribute('href')).toBe('https://db-ip.com')
+    expect(credit.closest('p')?.textContent).toContain('IP geolocation by DB-IP')
+  })
+
+  it('asks for one window across all five aggregates', async () => {
     mockFetch(everything)
     render(<StatsDashboard onConnectionStateChange={vi.fn()} />)
     await screen.findByRole('button', { name: /git\.muchq\.com/ })
 
     const urls = (fetch as unknown as ReturnType<typeof vi.fn>).mock.calls.map((call) => String(call[0]))
-    expect(urls).toHaveLength(4)
+    expect(urls).toHaveLength(5)
     for (const url of urls) expect(url).toContain('days=30')
     // The agents endpoint truncates busiest-first; ask for its ceiling so
     // a scraper's thin days are rows, not gaps.
@@ -217,11 +256,13 @@ describe('StatsDashboard', () => {
       '/agents': { days: 30, rows: [] },
       '/probes': { days: 30, rows: [] },
       '/iili/top': { days: 30, rows: [] },
+      '/countries': { days: 30, rows: [] },
     })
 
     render(<StatsDashboard onConnectionStateChange={vi.fn()} />)
 
     expect(await screen.findByText('No aggregated traffic yet.')).toBeInTheDocument()
+    expect(screen.getByText('No non-browser traffic in the window.')).toBeInTheDocument()
     expect(screen.getByText('No AI scraper traffic in the window.')).toBeInTheDocument()
     expect(screen.getByText('No named agents aggregated yet.')).toBeInTheDocument()
     expect(screen.getByText('No scanner probes in the window.')).toBeInTheDocument()
@@ -240,7 +281,7 @@ describe('StatsDashboard', () => {
     expect(cellsOf(gitRow)).toEqual(['›git.muchq.com', '913', '703', '0', '900', '10', '3'])
     expect(onState).toHaveBeenLastCalledWith('connected')
     // The tables whose endpoints failed say so rather than claiming zero.
-    expect(screen.getAllByText('Not available from the stats service.')).toHaveLength(3)
+    expect(screen.getAllByText('Not available from the stats service.')).toHaveLength(4)
     expect(screen.queryByText('No scanner probes in the window.')).not.toBeInTheDocument()
     expect(screen.getByText('abc123')).toBeInTheDocument()
   })