stats pipeline: structured request logs across services

bazel/go.MODULE.bazel

@@ -17,6 +17,7 @@ use_repo(
     "com_github_golang_jwt_jwt_v5",
     "com_github_google_uuid",
     "com_github_gorilla_websocket",
+    "com_github_jackc_pgx_v5",
     "com_github_prometheus_client_golang",
     "com_github_prometheus_common",
     "com_github_stretchr_testify",

bazel/rules/java_rules_test.bzl

@@ -315,6 +315,7 @@ def java_rules_test_suite(name):
 
     java_test_suite(
         name = name + "_suite_fixture",
+        size = "small",
         srcs = [
             "testdata/SuiteProbeHelper.java",
             "testdata/SuiteProbeTest.java",
@@ -346,13 +347,15 @@ def java_rules_test_suite(name):
             test_name = "{}_{}_{}".format(name, fixture_label, guard_label)
             guard(
                 name = test_name,
+                size = "small",
                 target_under_test = fixture,
             )
             tests.append(test_name)
 
         micronaut_name = "{}_{}_micronaut".format(name, fixture_label)
         micronaut_test(
             name = micronaut_name,
+            size = "small",
             target_under_test = fixture,
             expected = micronaut,
         )

bazel/rules/oci.bzl

@@ -68,6 +68,7 @@ def _create_oci_image(bin_name, binary_target, binary_path):
 
     container_structure_test(
         name = bin_name + "_image_test",
+        size = "small",
         configs = ["//bazel/rules:oci_image_test.yaml"],
         image = ":" + image_name,
         tags = ["manual"],
@@ -135,6 +136,7 @@ def linux_oci_java(bin_name):
 
     container_structure_test(
         name = bin_name + "_image_test",
+        size = "small",
         configs = ["//bazel/rules:java_image_test.yaml"],
         image = ":" + image_name,
         tags = ["manual"],

deploy/consolidated/Caddyfile

@@ -59,6 +59,11 @@ api.muchq.com {
 		path /v2/analyze
 	}
 
+	@get_stats {
+		method GET
+		path /stats/v1/*
+	}
+
 	@post_iili_shorten {
 		method POST
 		path /iili/v1/shorten
@@ -157,6 +162,8 @@ api.muchq.com {
 
 	# Same pattern (#1359): iili serves /iili/v1/* unrewritten.
 	reverse_proxy @post_iili_shorten iili:8091
+	# stats (#1460): read-only aggregates, unrewritten like the rest.
+	reverse_proxy @get_stats stats:8092
 	reverse_proxy @get_iili_redirect iili:8091
 
 	log {

deploy/consolidated/README.md

@@ -208,6 +208,23 @@ a role and database holding live rows is an operation, not a rename. Keep it URL
 `@ / ? # %` or quotes): it rides in a libpq URL and a single-quoted SQL literal. Compose
 refuses to start the service if it's unset.
 
+### The stats profile
+
+`log_shipper`, `stats` and `stats_db_init` sit behind the `stats` compose profile: they fail
+fast without the `STATS_*` credentials, so a fresh host must not start them by default. This
+host opts in permanently with one more `~/.env` line:
+
+```
+COMPOSE_PROFILES=stats
+```
+
+deploy.sh runs compose in `~`, where compose reads that file, so every normal deploy includes
+the trio. Alongside it live `STATS_AWS_ACCESS_KEY_ID`, `STATS_AWS_SECRET_ACCESS_KEY`,
+`STATS_S3_BUCKET`, `STATS_S3_REGION`, and `STATS_DB_PASSWORD` (same URL-safe rules as the
+other database passwords: it rides in a libpq URL and a single-quoted SQL literal). Without
+the `COMPOSE_PROFILES` line the containers keep running after a deploy but silently stop
+being updated — compose ignores profile-gated services on an unflagged `up -d`.
+
 Keeping a URL here rather than in a host file is what makes the hostname visible to this repo:
 `deploy_config_test.go` fails if a database host is not a Postgres service this file publishes, so
 the instance can be renamed (#1225) by editing one file instead of by keeping the old name

deploy/consolidated/compose.yaml

@@ -615,6 +615,68 @@ services:
           cpus: '0.25'
           memory: 128M
 
+  stats_db_init:
+    image: postgres:18
+    restart: "no"
+    logging: *default-logging
+    profiles:
+      - stats
+    environment:
+      PGHOST: shared_postgres
+      PGUSER: one_d4
+      PGDATABASE: one_d4
+      PGPASSWORD: ${ONE_D4_DB_PASSWORD}
+      STATS_DB_PASSWORD: ${STATS_DB_PASSWORD}
+    entrypoint: ["/bin/bash", "-ec"]
+    command:
+      - |
+        psql -v ON_ERROR_STOP=1 -tAc "SELECT 1 FROM pg_roles WHERE rolname = 'stats'" | grep -q 1 \
+          || psql -v ON_ERROR_STOP=1 -c "CREATE ROLE stats LOGIN"
+        psql -v ON_ERROR_STOP=1 -c "ALTER ROLE stats WITH LOGIN PASSWORD '$${STATS_DB_PASSWORD}'"
+        psql -v ON_ERROR_STOP=1 -tAc "SELECT 1 FROM pg_database WHERE datname = 'stats'" | grep -q 1 \
+          || psql -v ON_ERROR_STOP=1 -c "CREATE DATABASE stats OWNER stats"
+    networks:
+      - app_network
+    depends_on:
+      shared_postgres:
+        condition: service_healthy
+
+  stats:
+    image: ghcr.io/muchq/stats:${STATS_SHA:-${DEPLOY_SHA:-latest}}
+    labels:
+      com.muchq.description: "Log-derived stats aggregator and API (Go)"
+    restart: always
+    logging: *default-logging
+    # Same profile as log_shipper: both halves of the pipeline need the
+    # stats S3 credentials in ~/.env, so neither starts on a default up -d.
+    profiles:
+      - stats
+    environment:
+      - STATS_DB_URL=postgresql://stats:${STATS_DB_PASSWORD}@shared_postgres:5432/stats
+      - AWS_ACCESS_KEY_ID=${STATS_AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${STATS_AWS_SECRET_ACCESS_KEY}
+      - S3_BUCKET=${STATS_S3_BUCKET}
+      - S3_REGION=${STATS_S3_REGION:-us-east-1}
+      - PORT=8092
+    networks:
+      - app_network
+    depends_on:
+      shared_postgres:
+        condition: service_healthy
+      stats_db_init:
+        condition: service_completed_successfully
+    healthcheck:
+      # Steady /health probe (#1307); see golf_hub's for the full rationale.
+      test: ["CMD", "timeout", "4", "bash", "-c", 'exec 3<>/dev/tcp/127.0.0.1/8092 && printf "GET /health HTTP/1.0\r\n\r\n" >&3 && head -1 <&3 | grep -q " 200"']
+      interval: 30s
+      timeout: 5s
+      retries: 3
+    deploy:
+      resources:
+        limits:
+          cpus: '0.25'
+          memory: 256M
+
   forgejo:
     image: codeberg.org/forgejo/forgejo:16
     restart: always

deploy/consolidated/deploy_config_test.go

@@ -1025,6 +1025,8 @@ var publicRoutes = []struct {
 	// iili (#1359): the redirect matcher is the product.
 	{"@post_iili_shorten", []string{"method POST", "path /iili/v1/shorten"}, "iili:8091"},
 	{"@get_iili_redirect", []string{"method GET", "path /iili/v1/r/*"}, "iili:8091"},
+	// stats (#1460): read-only aggregates, GET-only on purpose.
+	{"@get_stats", []string{"method GET", "path /stats/v1/*"}, "stats:8092"},
 }
 
 func TestPublicRoutesAreDeliberatelyExact(t *testing.T) {
@@ -2033,3 +2035,20 @@ func TestLogShipperReadsTheCaddyLogMountAndIsProfileGated(t *testing.T) {
 			"start it with no S3 credentials and it would crash-loop. Block was:\n%s", block)
 	}
 }
+
+// The stats pair is profile-gated together: the aggregator needs the same
+// S3 credentials the shipper does, so a default `up -d` must start
+// neither the service nor its db-init — half the pair running is a
+// crash-loop or a database nothing writes to.
+func TestTheStatsPairIsProfileGatedTogether(t *testing.T) {
+	for _, service := range []string{"stats", "stats_db_init"} {
+		block := serviceBlock(t, "compose.yaml", service)
+		if !strings.Contains(block, "profiles:") {
+			t.Errorf("%s is not profile-gated; a default `up -d` starts it without "+
+				"S3 credentials (#1460). Block was:\n%s", service, block)
+		}
+	}
+	if !strings.Contains(serviceBlock(t, "compose.yaml", "stats"), "postgresql://stats:") {
+		t.Errorf("stats names no stats database URL; the aggregates have nowhere to land")
+	}
+}

domains/ai/apis/microgpt_serve/Cargo.toml

@@ -19,7 +19,6 @@ serde_json = { workspace = true }
 server_pal = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 tracing = { workspace = true }
-tracing-subscriber = { workspace = true }
 
 [dev-dependencies]
 tower = { workspace = true }

domains/ai/apis/microgpt_serve/src/main.rs

@@ -25,7 +25,7 @@ pub struct AppState {
 
 #[tokio::main]
 async fn main() {
-    tracing_subscriber::fmt::init();
+    server_pal::init_logging();
 
     // Must be initialised before AppMetrics::new() so the global provider is
     // in place when OTel instruments are created.

domains/games/apis/mithril/Cargo.toml

@@ -11,5 +11,4 @@ serde = { workspace = true, features = ["derive"] }
 server_pal = {workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 tracing = { workspace = true }
-tracing-subscriber = { workspace = true }
 wordchains = { workspace = true }

domains/games/apis/mithril/src/main.rs

@@ -4,7 +4,6 @@ use serde::{Deserialize, Deserializer, Serialize};
 use server_pal::{listen_addr_pal, router_builder, serve};
 use std::sync::Arc;
 use tracing::{Level, event};
-use tracing_subscriber;
 use wordchains::{Graph, bfs_for_target, initialize_graph};
 
 fn validate_word<'de, D>(deserializer: D) -> Result<String, D::Error>
@@ -60,7 +59,7 @@ async fn wordchain_post(
 
 #[tokio::main]
 async fn main() {
-    tracing_subscriber::fmt::init();
+    server_pal::init_logging();
     // Keeps the exporter alive for the process lifetime; without this the
     // http_server_* instruments record into the no-op global meter.
     let _otel_provider = server_pal::init_otel();

domains/graphics/apis/posterize/Cargo.toml

@@ -13,7 +13,6 @@ serde = { workspace = true, features = ["derive"] }
 server_pal = {workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 tracing = { workspace = true }
-tracing-subscriber = { workspace = true }
 imagine = { workspace = true }
 
 [dev-dependencies]

domains/graphics/apis/posterize/src/main.rs

@@ -9,7 +9,7 @@ use tracing::{Level, event};
 
 #[tokio::main]
 async fn main() {
-    tracing_subscriber::fmt::init();
+    server_pal::init_logging();
     // Keeps the exporter alive for the process lifetime; without this the
     // http_server_* instruments record into the no-op global meter.
     let _otel_provider = server_pal::init_otel();

domains/platform/apis/stats/BUILD.bazel

@@ -0,0 +1,47 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+load("//bazel/rules:oci.bzl", "linux_oci_go")
+
+go_library(
+    name = "stats_lib",
+    srcs = [
+        "aggregate.go",
+        "api.go",
+        "classify.go",
+        "loop.go",
+        "store.go",
+    ],
+    importpath = "github.com/muchq/moonbase/domains/platform/apis/stats",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_jackc_pgx_v5//:pgx",
+        "@com_github_jackc_pgx_v5//pgxpool",
+    ],
+)
+
+# store_test.go skips without STATS_TEST_DB_URL, like the repo's other
+# Postgres-gated suites — a local green run may have exercised no SQL.
+go_test(
+    name = "stats_test",
+    size = "small",
+    srcs = [
+        "aggregate_test.go",
+        "api_test.go",
+        "classify_test.go",
+        "loop_test.go",
+        "store_test.go",
+    ],
+    embed = [":stats_lib"],
+)
+
+go_binary(
+    name = "stats",
+    srcs = ["main/main.go"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":stats_lib",
+        "//domains/platform/libs/mucks",
+        "//domains/platform/libs/s3lite",
+    ],
+)
+
+linux_oci_go(bin_name = "stats")