stats: bundle the IP-to-country table in the image

bazel/rules/oci.bzl

@@ -32,14 +32,15 @@ def _push_and_load(image_name, bin_name):
         repo_tags = ["ghcr.io/muchq/" + bin_name + ":latest"],
     )
 
-def _create_oci_image(bin_name, binary_target, binary_path):
+def _create_oci_image(bin_name, binary_target, binary_path, extra_tars = []):
     """
     create oci image, push, and load targets for a binary
 
     Args:
       bin_name: the binary name for the image
       binary_target: the bazel target containing the binary
       binary_path: the path to remap the binary to in the container
+      extra_tars: further layers for the image
     """
     tar_name = bin_name + "_tar"
     image_name = bin_name + "_image"
@@ -61,7 +62,7 @@ def _create_oci_image(bin_name, binary_target, binary_path):
         tars = [
             "//bazel/rules:ca_certificates_layer",
             ":" + tar_name,
-        ],
+        ] + extra_tars,
     )
 
     _push_and_load(image_name = image_name, bin_name = bin_name)
@@ -74,12 +75,13 @@ def _create_oci_image(bin_name, binary_target, binary_path):
         tags = ["manual"],
     )
 
-def linux_oci_go(bin_name):
+def linux_oci_go(bin_name, extra_tars = []):
     """
     generate linux oci container for go binaries
 
     Args:
       bin_name: the binary target name to be wrapper
+      extra_tars: further layers for the image, e.g. a data file the binary reads
     """
 
     linux_amd_target_name = bin_name + "_linux_amd64"
@@ -91,7 +93,7 @@ def linux_oci_go(bin_name):
         visibility = ["//visibility:public"],
     )
 
-    _create_oci_image(bin_name, linux_amd_target_name, "/" + bin_name)
+    _create_oci_image(bin_name, linux_amd_target_name, "/" + bin_name, extra_tars)
 
 def linux_amd64_oci_binary(bin_name):
     """

bazel/tools.MODULE.bazel

@@ -35,3 +35,14 @@ http_file(
     sha256 = "3ed0c94610d8bcf7c11bbb49c56aa49c7234d32b66824df91f554169e572da48",
     urls = ["https://raw.githubusercontent.com/dwyl/english-words/master/words_alpha.txt"],
 )
+
+# DB-IP's free IP-to-country table (CC BY 4.0; muchq.com/stats carries the
+# attribution), bundled into the stats image (#1467). DB-IP publishes a new
+# file each month under a dated URL and keeps the old ones up, so a refresh
+# is a pin bump here and nothing on the host.
+http_file(
+    name = "dbip_country_lite",
+    downloaded_file_path = "dbip-country-lite.csv.gz",
+    sha256 = "a32bb3c384bd3de60ad9024596aa5b395a6dd5beaa27a7223407cc2edc681d0b",
+    urls = ["https://download.db-ip.com/free/dbip-country-lite-2026-09.csv.gz"],
+)

deploy/consolidated/README.md

@@ -220,11 +220,7 @@ COMPOSE_PROFILES=stats
 
 deploy.sh runs compose in `~`, where compose reads that file, so every normal deploy includes
 the trio. Alongside it live `STATS_AWS_ACCESS_KEY_ID`, `STATS_AWS_SECRET_ACCESS_KEY`,
-`STATS_S3_BUCKET`, `STATS_S3_REGION`, optionally `STATS_GEO_DB_KEY` (the key of
-DB-IP's country CSV in that bucket — upload `dbip-country-lite-YYYY-MM.csv.gz`
-under `geo/` and restart `stats` for a new month; unset, the geo rows all read
-`--`, and a key that will not load is an error in the log, not a boot
-failure), and
+`STATS_S3_BUCKET`, `STATS_S3_REGION`, and
 `STATS_DB_PASSWORD` (same URL-safe rules as the
 other database passwords: it rides in a libpq URL and a single-quoted SQL literal). Without
 the `COMPOSE_PROFILES` line the containers keep running after a deploy but silently stop

deploy/consolidated/compose.yaml

@@ -662,9 +662,6 @@ services:
       - AWS_SECRET_ACCESS_KEY=${STATS_AWS_SECRET_ACCESS_KEY}
       - S3_BUCKET=${STATS_S3_BUCKET}
       - S3_REGION=${STATS_S3_REGION:-us-east-1}
-      # DB-IP's country CSV in the same bucket (#1467); empty means every
-      # geo row reads "--", and so does a key that will not load.
-      - GEO_DB_KEY=${STATS_GEO_DB_KEY:-}
       - PORT=8092
     networks:
       - app_network

deploy/consolidated/deploy_config_test.go

@@ -2116,11 +2116,6 @@ func TestTheStatsPairIsProfileGatedTogether(t *testing.T) {
 	if !strings.Contains(serviceBlock(t, "compose.yaml", "stats"), "postgresql://stats:") {
 		t.Errorf("stats names no stats database URL; the aggregates have nowhere to land")
 	}
-	// The geo database key rides the stats block (#1467); dropping it is an
-	// all-"--" table with nothing else to say so.
-	if !strings.Contains(serviceBlock(t, "compose.yaml", "stats"), "GEO_DB_KEY=") {
-		t.Errorf("the stats service does not pass GEO_DB_KEY; the geo rollup has no database to load")
-	}
 }
 
 // catchAllIsLastHandle reports whether the site block ends its handle

domains/platform/apis/stats/BUILD.bazel

@@ -1,4 +1,5 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
 load("//bazel/rules:oci.bzl", "linux_oci_go")
 
 # otel_contract pins the query-event vocabulary here against QueryEvent.java.
@@ -18,6 +19,7 @@ go_library(
     importpath = "github.com/muchq/moonbase/domains/platform/apis/stats",
     visibility = ["//visibility:public"],
     deps = [
+        "//domains/platform/libs/mucks",
         "@com_github_jackc_pgx_v5//:pgx",
         "@com_github_jackc_pgx_v5//pgxpool",
     ],
@@ -34,6 +36,7 @@ go_test(
         "aggregate_test.go",
         "api_test.go",
         "classify_test.go",
+        "fixtures_test.go",
         "geo_test.go",
         "loop_test.go",
         "queries_test.go",
@@ -53,9 +56,45 @@ go_binary(
     visibility = ["//visibility:public"],
     deps = [
         ":stats_lib",
-        "//domains/platform/libs/mucks",
         "//domains/platform/libs/s3lite",
     ],
 )
 
-linux_oci_go(bin_name = "stats")
+# The IP-to-country table rides in the image at /geo (#1467); main.go's
+# GEO_DB_PATH default points at it, and geo_vendor_test pins the two.
+pkg_tar(
+    name = "geo_layer",
+    srcs = ["@dbip_country_lite//file"],
+    package_dir = "/geo",
+)
+
+# Integration: the pinned vendor file through the real parser, the layer's
+# path against the binary's default, and the whole service end to end —
+# shipped objects in a memory bucket, the real loop, the real store on the
+# CI postgres, the real router over HTTP. Its own target: the data is a
+# few MB the unit suite has no use for, and the database is shared.
+go_test(
+    name = "stats_integration_test",
+    size = "small",
+    srcs = [
+        "e2e_test.go",
+        "fixtures_test.go",
+        "geo_vendor_test.go",
+    ],
+    data = [
+        ":geo_layer",
+        "@dbip_country_lite//file",
+    ],
+    embed = [":stats_lib"],
+    env = {
+        "DBIP_COUNTRY_LITE": "$(rootpath @dbip_country_lite//file)",
+        "GEO_LAYER_TAR": "$(rootpath :geo_layer)",
+    },
+    env_inherit = ["STATS_TEST_DB_URL"],
+    tags = ["exclusive"],
+)
+
+linux_oci_go(
+    bin_name = "stats",
+    extra_tars = [":geo_layer"],
+)

domains/platform/apis/stats/README.md

@@ -60,15 +60,17 @@ The geo rollup (#1467) places each request's `client_ip` (`remote_ip` on
 older lines) in a country and keys `geo_stats` on day, host, agent class,
 and the two-letter code, with request, 403, and probe counts — where the
 scrapers, bots, and scanners come from. The database is DB-IP's free
-country CSV (`dbip-country-lite-YYYY-MM.csv.gz`, CC BY 4.0; muchq.com's
-stats page carries the attribution), uploaded by the operator to the stats
-bucket under the key `GEO_DB_KEY` names; the service loads it at boot
-into a sorted range table and binary-searches it, no library. Overlapping
-rows lose to the range they sit in. An address outside every range, or no
-database at all, files under `--`. A key that will not load, after a few
-retries for a bucket still waking up, is an error in the log and an
-all-`--` table, never a boot failure. A new monthly file is a restart.
-Rows aggregated before the database was uploaded stay `--` until a
+country CSV (CC BY 4.0; muchq.com's stats page carries the attribution),
+pinned by URL and sha256 as `@dbip_country_lite` in
+`bazel/tools.MODULE.bazel` and bundled into the image at
+`/geo/dbip-country-lite.csv.gz`; a new month is a pin bump, nothing on
+the host. The service loads it at boot into a sorted range table and
+binary-searches it, no library. Overlapping rows lose to the range they
+sit in, and the `ZZ` rows DB-IP uses for reserved and private space are
+not placements. An address outside every range files under `--`, and so
+does everything when the file will not load — an error in the log, never a
+boot failure. `GEO_DB_PATH` points at another file, or empty switches geo
+off. Rows aggregated before a database was available stay `--` until a
 re-aggregation (bump `RollupVersion`).
 
 What stays ad hoc: IP-range clusters — a /24 key is caller-shaped and
@@ -105,11 +107,19 @@ stay in the log, not on the wire.
 ## Configuration
 
 `STATS_DB_URL` (postgres), `S3_BUCKET`, `S3_REGION`, `AWS_ACCESS_KEY_ID`,
-`AWS_SECRET_ACCESS_KEY`, and optionally `GEO_DB_KEY` — the same stats IAM user the shipper writes with,
+`AWS_SECRET_ACCESS_KEY` — the same stats IAM user the shipper writes with,
 which therefore needs `s3:GetObject` and `s3:ListBucket` on the `logs/*`
-prefix as well as `s3:PutObject`, and `s3:GetObject` on the geo key's
-prefix (`geo/*` in the deployment). `AGGREGATE_INTERVAL` and `PORT`
-(default 8092) are optional.
+prefix as well as `s3:PutObject`. `AGGREGATE_INTERVAL`, `PORT`
+(default 8092), and `GEO_DB_PATH` are optional.
 
-The store integration test needs `STATS_TEST_DB_URL` and skips without it,
-like the repo's other Postgres-gated suites.
+## Tests
+
+Unit tests per layer, and two gated suites. `stats_test` holds the store
+tests, which need `STATS_TEST_DB_URL` and skip without it, like the repo's
+other Postgres-gated suites. `stats_integration_test` is the end-to-end
+pattern: `e2e_test.go` puts shipped objects in a memory bucket, runs the
+real loop with the real vendor geo file into the real store, and reads
+every endpoint through `NewRouter` over HTTP — the same objects production
+wires together. A new endpoint or source gets a few lines there, not a new
+harness. Locally, a throwaway `postgres:18` in Docker on port 55432 and
+`STATS_TEST_DB_URL=postgresql://t:t@localhost:55432/t` run both.

domains/platform/apis/stats/api.go

@@ -6,6 +6,8 @@ import (
 	"log/slog"
 	"net/http"
 	"strconv"
+
+	"github.com/muchq/moonbase/domains/platform/libs/mucks"
 )
 
 // Reader is what the HTTP handlers need from the store — an interface so
@@ -29,6 +31,21 @@ func NewHandlers(reader Reader, logger *slog.Logger) *Handlers {
 	return &Handlers{reader: reader, logger: logger}
 }
 
+// NewRouter is the service's whole HTTP surface, the one main serves and
+// the end-to-end test drives; a route added here is on both.
+func NewRouter(h *Handlers) http.Handler {
+	router := mucks.NewJsonMucks()
+	router.HandleFunc("GET /health", h.Health)
+	router.HandleFunc("GET /stats/v1/summary", h.GetSummary)
+	router.HandleFunc("GET /stats/v1/iili/top", h.GetTopSlugs)
+	router.HandleFunc("GET /stats/v1/agents", h.GetAgents)
+	router.HandleFunc("GET /stats/v1/probes", h.GetProbes)
+	router.HandleFunc("GET /stats/v1/one_d4/queries", h.GetQueries)
+	router.HandleFunc("GET /stats/v1/one_d4/terms", h.GetQueryTerms)
+	router.HandleFunc("GET /stats/v1/countries", h.GetCountries)
+	return router
+}
+
 func (h *Handlers) Health(w http.ResponseWriter, _ *http.Request) {
 	w.Write([]byte(`{"status":"healthy"}`))
 }