From 961b33e91f0a2f9037d4c22915e6951c0efbc89f Mon Sep 17 00:00:00 2001 From: Andy Aylward Date: Wed, 2 Sep 2026 11:22:11 -0400 Subject: [PATCH 1/4] stats: bundle the IP-to-country table in the image DB-IP's country CSV is an http_file pinned by URL and sha256, laid into the stats image at /geo, and read from disk at boot; GEO_DB_PATH points elsewhere or, empty, switches geo off. A new month is a pin bump. The image rule takes extra layers for data a binary reads. --- bazel/rules/oci.bzl | 10 +-- bazel/tools.MODULE.bazel | 11 +++ deploy/consolidated/README.md | 6 +- deploy/consolidated/compose.yaml | 3 - deploy/consolidated/deploy_config_test.go | 5 -- domains/platform/apis/stats/BUILD.bazel | 14 +++- domains/platform/apis/stats/README.md | 26 +++---- domains/platform/apis/stats/geo.go | 54 ++++++--------- domains/platform/apis/stats/geo_test.go | 77 +++++++++++---------- domains/platform/apis/stats/main/main.go | 22 +++--- domains/platform/apps/log_shipper/README.md | 2 +- 11 files changed, 119 insertions(+), 111 deletions(-) diff --git a/bazel/rules/oci.bzl b/bazel/rules/oci.bzl index 989ec94b4..7082959b8 100644 --- a/bazel/rules/oci.bzl +++ b/bazel/rules/oci.bzl @@ -32,7 +32,7 @@ def _push_and_load(image_name, bin_name): repo_tags = ["ghcr.io/muchq/" + bin_name + ":latest"], ) -def _create_oci_image(bin_name, binary_target, binary_path): +def _create_oci_image(bin_name, binary_target, binary_path, extra_tars = []): """ create oci image, push, and load targets for a binary @@ -40,6 +40,7 @@ def _create_oci_image(bin_name, binary_target, binary_path): bin_name: the binary name for the image binary_target: the bazel target containing the binary binary_path: the path to remap the binary to in the container + extra_tars: further layers for the image """ tar_name = bin_name + "_tar" image_name = bin_name + "_image" @@ -61,7 +62,7 @@ def _create_oci_image(bin_name, binary_target, binary_path): tars = [ "//bazel/rules:ca_certificates_layer", ":" + tar_name, - ], + ] + extra_tars, ) _push_and_load(image_name = image_name, bin_name = bin_name) @@ -74,12 +75,13 @@ def _create_oci_image(bin_name, binary_target, binary_path): tags = ["manual"], ) -def linux_oci_go(bin_name): +def linux_oci_go(bin_name, extra_tars = []): """ generate linux oci container for go binaries Args: bin_name: the binary target name to be wrapper + extra_tars: further layers for the image, e.g. a data file the binary reads """ linux_amd_target_name = bin_name + "_linux_amd64" @@ -91,7 +93,7 @@ def linux_oci_go(bin_name): visibility = ["//visibility:public"], ) - _create_oci_image(bin_name, linux_amd_target_name, "/" + bin_name) + _create_oci_image(bin_name, linux_amd_target_name, "/" + bin_name, extra_tars) def linux_amd64_oci_binary(bin_name): """ diff --git a/bazel/tools.MODULE.bazel b/bazel/tools.MODULE.bazel index 48b61a7ea..c4b19e2c8 100644 --- a/bazel/tools.MODULE.bazel +++ b/bazel/tools.MODULE.bazel @@ -35,3 +35,14 @@ http_file( sha256 = "3ed0c94610d8bcf7c11bbb49c56aa49c7234d32b66824df91f554169e572da48", urls = ["https://raw.githubusercontent.com/dwyl/english-words/master/words_alpha.txt"], ) + +# DB-IP's free IP-to-country table (CC BY 4.0; muchq.com/stats carries the +# attribution), bundled into the stats image (#1467). DB-IP publishes a new +# file each month under a dated URL and keeps the old ones up, so a refresh +# is a pin bump here and nothing on the host. +http_file( + name = "dbip_country_lite", + downloaded_file_path = "dbip-country-lite.csv.gz", + sha256 = "a32bb3c384bd3de60ad9024596aa5b395a6dd5beaa27a7223407cc2edc681d0b", + urls = ["https://download.db-ip.com/free/dbip-country-lite-2026-09.csv.gz"], +) diff --git a/deploy/consolidated/README.md b/deploy/consolidated/README.md index f24e743cb..323b82f80 100644 --- a/deploy/consolidated/README.md +++ b/deploy/consolidated/README.md @@ -220,11 +220,7 @@ COMPOSE_PROFILES=stats deploy.sh runs compose in `~`, where compose reads that file, so every normal deploy includes the trio. Alongside it live `STATS_AWS_ACCESS_KEY_ID`, `STATS_AWS_SECRET_ACCESS_KEY`, -`STATS_S3_BUCKET`, `STATS_S3_REGION`, optionally `STATS_GEO_DB_KEY` (the key of -DB-IP's country CSV in that bucket — upload `dbip-country-lite-YYYY-MM.csv.gz` -under `geo/` and restart `stats` for a new month; unset, the geo rows all read -`--`, and a key that will not load is an error in the log, not a boot -failure), and +`STATS_S3_BUCKET`, `STATS_S3_REGION`, and `STATS_DB_PASSWORD` (same URL-safe rules as the other database passwords: it rides in a libpq URL and a single-quoted SQL literal). Without the `COMPOSE_PROFILES` line the containers keep running after a deploy but silently stop diff --git a/deploy/consolidated/compose.yaml b/deploy/consolidated/compose.yaml index b6b5a9349..d0af252be 100644 --- a/deploy/consolidated/compose.yaml +++ b/deploy/consolidated/compose.yaml @@ -662,9 +662,6 @@ services: - AWS_SECRET_ACCESS_KEY=${STATS_AWS_SECRET_ACCESS_KEY} - S3_BUCKET=${STATS_S3_BUCKET} - S3_REGION=${STATS_S3_REGION:-us-east-1} - # DB-IP's country CSV in the same bucket (#1467); empty means every - # geo row reads "--", and so does a key that will not load. - - GEO_DB_KEY=${STATS_GEO_DB_KEY:-} - PORT=8092 networks: - app_network diff --git a/deploy/consolidated/deploy_config_test.go b/deploy/consolidated/deploy_config_test.go index 8270730fc..269118e06 100644 --- a/deploy/consolidated/deploy_config_test.go +++ b/deploy/consolidated/deploy_config_test.go @@ -2116,11 +2116,6 @@ func TestTheStatsPairIsProfileGatedTogether(t *testing.T) { if !strings.Contains(serviceBlock(t, "compose.yaml", "stats"), "postgresql://stats:") { t.Errorf("stats names no stats database URL; the aggregates have nowhere to land") } - // The geo database key rides the stats block (#1467); dropping it is an - // all-"--" table with nothing else to say so. - if !strings.Contains(serviceBlock(t, "compose.yaml", "stats"), "GEO_DB_KEY=") { - t.Errorf("the stats service does not pass GEO_DB_KEY; the geo rollup has no database to load") - } } // catchAllIsLastHandle reports whether the site block ends its handle diff --git a/domains/platform/apis/stats/BUILD.bazel b/domains/platform/apis/stats/BUILD.bazel index c1d3af272..1fc2969b1 100644 --- a/domains/platform/apis/stats/BUILD.bazel +++ b/domains/platform/apis/stats/BUILD.bazel @@ -1,4 +1,5 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") +load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("//bazel/rules:oci.bzl", "linux_oci_go") # otel_contract pins the query-event vocabulary here against QueryEvent.java. @@ -58,4 +59,15 @@ go_binary( ], ) -linux_oci_go(bin_name = "stats") +# The IP-to-country table rides in the image at /geo (#1467); main.go's +# GEO_DB_PATH default points at it. +pkg_tar( + name = "geo_layer", + srcs = ["@dbip_country_lite//file"], + package_dir = "/geo", +) + +linux_oci_go( + bin_name = "stats", + extra_tars = [":geo_layer"], +) diff --git a/domains/platform/apis/stats/README.md b/domains/platform/apis/stats/README.md index a0cc28980..62e72792a 100644 --- a/domains/platform/apis/stats/README.md +++ b/domains/platform/apis/stats/README.md @@ -60,15 +60,16 @@ The geo rollup (#1467) places each request's `client_ip` (`remote_ip` on older lines) in a country and keys `geo_stats` on day, host, agent class, and the two-letter code, with request, 403, and probe counts — where the scrapers, bots, and scanners come from. The database is DB-IP's free -country CSV (`dbip-country-lite-YYYY-MM.csv.gz`, CC BY 4.0; muchq.com's -stats page carries the attribution), uploaded by the operator to the stats -bucket under the key `GEO_DB_KEY` names; the service loads it at boot -into a sorted range table and binary-searches it, no library. Overlapping -rows lose to the range they sit in. An address outside every range, or no -database at all, files under `--`. A key that will not load, after a few -retries for a bucket still waking up, is an error in the log and an -all-`--` table, never a boot failure. A new monthly file is a restart. -Rows aggregated before the database was uploaded stay `--` until a +country CSV (CC BY 4.0; muchq.com's stats page carries the attribution), +pinned by URL and sha256 as `@dbip_country_lite` in +`bazel/tools.MODULE.bazel` and bundled into the image at +`/geo/dbip-country-lite.csv.gz`; a new month is a pin bump, nothing on +the host. The service loads it at boot into a sorted range table and +binary-searches it, no library. Overlapping rows lose to the range they +sit in. An address outside every range files under `--`, and so does +everything when the file will not load — an error in the log, never a +boot failure. `GEO_DB_PATH` points at another file, or empty switches geo +off. Rows aggregated before a database was available stay `--` until a re-aggregation (bump `RollupVersion`). What stays ad hoc: IP-range clusters — a /24 key is caller-shaped and @@ -105,11 +106,10 @@ stay in the log, not on the wire. ## Configuration `STATS_DB_URL` (postgres), `S3_BUCKET`, `S3_REGION`, `AWS_ACCESS_KEY_ID`, -`AWS_SECRET_ACCESS_KEY`, and optionally `GEO_DB_KEY` — the same stats IAM user the shipper writes with, +`AWS_SECRET_ACCESS_KEY` — the same stats IAM user the shipper writes with, which therefore needs `s3:GetObject` and `s3:ListBucket` on the `logs/*` -prefix as well as `s3:PutObject`, and `s3:GetObject` on the geo key's -prefix (`geo/*` in the deployment). `AGGREGATE_INTERVAL` and `PORT` -(default 8092) are optional. +prefix as well as `s3:PutObject`. `AGGREGATE_INTERVAL`, `PORT` +(default 8092), and `GEO_DB_PATH` are optional. The store integration test needs `STATS_TEST_DB_URL` and skips without it, like the repo's other Postgres-gated suites. diff --git a/domains/platform/apis/stats/geo.go b/domains/platform/apis/stats/geo.go index 9caac6810..50f3a0945 100644 --- a/domains/platform/apis/stats/geo.go +++ b/domains/platform/apis/stats/geo.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/netip" + "os" "sort" "strings" ) @@ -126,21 +127,20 @@ func (g *Geo) Country(ip string) string { return candidate.country } -// LoadGeo reads the CSV object (gzipped when the key says so) from the -// bucket the logs live in. The key is the operator's: DB-IP publishes a -// new file monthly under CC BY 4.0, and muchq.com/stats carries the -// attribution. -func LoadGeo(objects ObjectStore, key string) (*Geo, int, error) { - body, err := objects.Get(key) +// LoadGeoFile reads the CSV from disk, gunzipping when the name says so. +// The file rides in the image: DB-IP publishes a new one monthly under +// CC BY 4.0, and muchq.com/stats carries the attribution. +func LoadGeoFile(path string) (*Geo, int, error) { + file, err := os.Open(path) if err != nil { - return nil, 0, fmt.Errorf("fetching geo database %s: %w", key, err) + return nil, 0, fmt.Errorf("opening geo database: %w", err) } - defer body.Close() - var reader io.Reader = body - if strings.HasSuffix(key, ".gz") { - gz, err := gzip.NewReader(body) + defer file.Close() + var reader io.Reader = file + if strings.HasSuffix(path, ".gz") { + gz, err := gzip.NewReader(file) if err != nil { - return nil, 0, fmt.Errorf("geo database %s is not gzip: %w", key, err) + return nil, 0, fmt.Errorf("geo database %s is not gzip: %w", path, err) } defer gz.Close() reader = gz @@ -148,29 +148,19 @@ func LoadGeo(objects ObjectStore, key string) (*Geo, int, error) { return ParseDBIP(reader) } -// Locate is the boot path: no key means no database, and a key that -// fails is retried — a bucket that is briefly unreachable while the -// container starts must not take every endpoint down with it — then -// reported for the caller to carry on without. It never fails a boot: -// an all-"--" table with an error in the log is the visible outcome, and -// a crash loop would hide every other table behind it. -func Locate(objects ObjectStore, key string, attempts int, wait func()) (Locator, int, error) { - if key == "" { +// Locate is the boot path: an empty path means no database, and a file +// that will not load is reported for the caller to run without — an +// all-"--" table with an error in the log, never a boot failure that +// hides every other table behind it. +func Locate(path string) (Locator, int, error) { + if path == "" { return NoLocator{}, 0, nil } - var err error - for attempt := 0; attempt < attempts; attempt++ { - if attempt > 0 { - wait() - } - var geo *Geo - var skipped int - geo, skipped, err = LoadGeo(objects, key) - if err == nil { - return geo, skipped, nil - } + geo, skipped, err := LoadGeoFile(path) + if err != nil { + return NoLocator{}, 0, err } - return NoLocator{}, 0, err + return geo, skipped, nil } // GeoKey is one row of the per-day geo rollup: where a host's traffic of diff --git a/domains/platform/apis/stats/geo_test.go b/domains/platform/apis/stats/geo_test.go index 6d769a4ce..45aaf8ad7 100644 --- a/domains/platform/apis/stats/geo_test.go +++ b/domains/platform/apis/stats/geo_test.go @@ -3,7 +3,8 @@ package stats import ( "bytes" "compress/gzip" - "errors" + "os" + "path/filepath" "strings" "testing" ) @@ -99,7 +100,7 @@ func TestAGeoFileWithNoUsableRowsIsAnError(t *testing.T) { } } -func TestLoadGeoReadsAGzippedObjectAndReportsAMissingOne(t *testing.T) { +func TestLoadGeoFileReadsGzippedOrPlainAndReportsAMissingOne(t *testing.T) { var buf bytes.Buffer w := gzip.NewWriter(&buf) if _, err := w.Write([]byte(dbipSample)); err != nil { @@ -108,59 +109,59 @@ func TestLoadGeoReadsAGzippedObjectAndReportsAMissingOne(t *testing.T) { if err := w.Close(); err != nil { t.Fatal(err) } - objects := &fakeObjects{objects: map[string][]byte{ - "geo/dbip-country-lite.csv.gz": buf.Bytes(), - "geo/plain.csv": []byte(dbipSample), - }, fail: map[string]error{"geo/missing.csv": errors.New("404")}} + dir := t.TempDir() + for name, contents := range map[string][]byte{ + "dbip-country-lite.csv.gz": buf.Bytes(), + "plain.csv": []byte(dbipSample), + "notgzip.csv.gz": []byte("not gzip"), + } { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o644); err != nil { + t.Fatal(err) + } + } - for _, key := range []string{"geo/dbip-country-lite.csv.gz", "geo/plain.csv"} { - geo, skipped, err := LoadGeo(objects, key) + for _, name := range []string{"dbip-country-lite.csv.gz", "plain.csv"} { + geo, skipped, err := LoadGeoFile(filepath.Join(dir, name)) if err != nil { - t.Fatalf("%s: %v", key, err) + t.Fatalf("%s: %v", name, err) } if geo.Country("57.141.3.4") != "US" || skipped != 6 { - t.Errorf("%s loaded but places nothing, or lost the skip count (%d)", key, skipped) + t.Errorf("%s loaded but places nothing, or lost the skip count (%d)", name, skipped) } } - if _, _, err := LoadGeo(objects, "geo/missing.csv"); err == nil { + if _, _, err := LoadGeoFile(filepath.Join(dir, "missing.csv")); err == nil { t.Error("a missing database loaded without complaint") } - if _, _, err := LoadGeo(&fakeObjects{objects: map[string][]byte{"geo/x.gz": []byte("not gzip")}}, "geo/x.gz"); err == nil { + if _, _, err := LoadGeoFile(filepath.Join(dir, "notgzip.csv.gz")); err == nil { t.Error("a .gz that is not gzip loaded without complaint") } } -// A quoted field with a stray quote is a bad row, not a dead database. -func TestAStrayQuoteIsOneSkippedRowNotAFailedLoad(t *testing.T) { - geo, skipped, err := ParseDBIP(strings.NewReader("1.0.0.0,1.0.0.255,AU\n\"1.0.1.0,1.0.1.255,\"C\"N\n")) - if err != nil || skipped != 1 || geo.Country("1.0.0.7") != "AU" { - t.Errorf("ParseDBIP = (%v, %d, %v); want the good row kept and the bad one counted", geo != nil, skipped, err) - } -} - -func TestLocateNeverFailsABootAndRetriesABucketThatIsNotThereYet(t *testing.T) { - geo, _, err := Locate(&fakeObjects{}, "", 3, func() {}) +func TestLocateNeverFailsABoot(t *testing.T) { + geo, _, err := Locate("") if _, none := geo.(NoLocator); !none || err != nil { - t.Errorf("Locate with no key = (%T, %v), want NoLocator and no error", geo, err) + t.Errorf("Locate with no path = (%T, %v), want NoLocator and no error", geo, err) } - objects := &fakeObjects{objects: map[string][]byte{"geo/plain.csv": []byte(dbipSample)}, - fail: map[string]error{"geo/plain.csv": errors.New("503 slow down")}} - waits := 0 - // The bucket answers on the third try. - geo, _, err = Locate(objects, "geo/plain.csv", 5, func() { - waits++ - if waits == 2 { - delete(objects.fail, "geo/plain.csv") - } - }) - if err != nil || geo.Country("57.141.3.4") != "US" || waits != 2 { - t.Errorf("Locate after a flaky bucket = (%v, %v, %d waits); want the database on the third try", geo, err, waits) + path := filepath.Join(t.TempDir(), "plain.csv") + if err := os.WriteFile(path, []byte(dbipSample), 0o644); err != nil { + t.Fatal(err) + } + geo, _, err = Locate(path) + if err != nil || geo.Country("57.141.3.4") != "US" { + t.Errorf("Locate(%s) = (%v, %v); want the database", path, geo, err) } - geo, _, err = Locate(&fakeObjects{fail: map[string]error{"geo/missing.csv": errors.New("404")}}, - "geo/missing.csv", 3, func() {}) + geo, _, err = Locate(filepath.Join(t.TempDir(), "missing.csv")) if _, none := geo.(NoLocator); !none || err == nil { - t.Errorf("Locate with a key that never loads = (%T, %v); want NoLocator and the error to log", geo, err) + t.Errorf("Locate with a file that is not there = (%T, %v); want NoLocator and the error to log", geo, err) + } +} + +// A quoted field with a stray quote is a bad row, not a dead database. +func TestAStrayQuoteIsOneSkippedRowNotAFailedLoad(t *testing.T) { + geo, skipped, err := ParseDBIP(strings.NewReader("1.0.0.0,1.0.0.255,AU\n\"1.0.1.0,1.0.1.255,\"C\"N\n")) + if err != nil || skipped != 1 || geo.Country("1.0.0.7") != "AU" { + t.Errorf("ParseDBIP = (%v, %d, %v); want the good row kept and the bad one counted", geo != nil, skipped, err) } } diff --git a/domains/platform/apis/stats/main/main.go b/domains/platform/apis/stats/main/main.go index d7d3fc49d..fcb6ed031 100644 --- a/domains/platform/apis/stats/main/main.go +++ b/domains/platform/apis/stats/main/main.go @@ -59,19 +59,23 @@ func main() { Client: &http.Client{Timeout: 5 * time.Minute}, Now: time.Now, } - // The geo database is optional. A key that will not load is logged and - // the service runs without it — every geo row reads "--" — rather than - // crash-looping the other endpoints behind a bucket hiccup or a typo. - geoKey := os.Getenv("GEO_DB_KEY") - geo, skipped, err := stats.Locate(objects, geoKey, 5, func() { time.Sleep(10 * time.Second) }) + // The geo database rides in the image; GEO_DB_PATH points elsewhere or, + // empty, switches geo off. A file that will not load is logged and the + // service runs without it — every geo row reads "--" — rather than + // failing a boot the other endpoints do not depend on. + geoPath, geoSet := os.LookupEnv("GEO_DB_PATH") + if !geoSet { + geoPath = "/geo/dbip-country-lite.csv.gz" + } + geo, skipped, err := stats.Locate(geoPath) switch { case err != nil: logger.Error("cannot load the geo database; geo rows will all read "+stats.UnknownCountry, - "key", geoKey, "error", err) - case geoKey == "": - logger.Warn("GEO_DB_KEY is not set; geo rows will all read " + stats.UnknownCountry) + "path", geoPath, "error", err) + case geoPath == "": + logger.Warn("GEO_DB_PATH is empty; geo rows will all read " + stats.UnknownCountry) default: - logger.Info("geo database loaded", "key", geoKey, "skipped_lines", skipped) + logger.Info("geo database loaded", "path", geoPath, "skipped_lines", skipped) } aggregator := &stats.Aggregator{ diff --git a/domains/platform/apps/log_shipper/README.md b/domains/platform/apps/log_shipper/README.md index 866f2fc4b..7f7655bf7 100644 --- a/domains/platform/apps/log_shipper/README.md +++ b/domains/platform/apps/log_shipper/README.md @@ -41,7 +41,7 @@ and anything that is not a rolled log are never touched. | --- | --- | | `S3_BUCKET` | Destination bucket (required) | | `S3_REGION` | Bucket's region (required) | -| `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | The stats IAM user: `s3:PutObject` for this shipper, plus `s3:GetObject`/`s3:ListBucket` for the aggregator, all scoped to the bucket's `logs/*` prefix, plus `s3:GetObject` on the geo database's prefix (required) | +| `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | The stats IAM user: `s3:PutObject` for this shipper, plus `s3:GetObject`/`s3:ListBucket` for the aggregator, all scoped to the bucket's `logs/*` prefix (required) | | `LOG_DIRS` | `label=dir` pairs, comma-separated: each directory ships under `logs/source=