feat(observability): BackendT2Degraded alert for silent tier-2 offload degradation

config/observability/alerting-rules.yaml

@@ -76,12 +76,16 @@ groups:
         # BOTH. Upstream docs list the unsuffixed names at
         # https://docs.vllm.ai/en/latest/usage/metrics/; the series
         # have been present since vLLM ~0.18 (the release that added
-        # LMCache-style offload support). This operator does not
-        # currently scrape them
-        # (pkg/adapters/engine/metrics_scraper.go only reads
-        # vllm:prefix_cache_{hits,queries} for T1 plus
-        # vllm:*_cache_usage_perc), so the alert binds directly to
-        # vLLM's exposition. Operators should sanity-check the metric
+        # LMCache-style offload support). The operator's scraper
+        # (pkg/adapters/engine/metrics_scraper.go) now ALSO reads these
+        # external counters and projects them per-CacheBackend as the
+        # inferencecache_backend_t2_hit_rate gauge + the T2Degraded
+        # condition + the BackendT2Degraded alert in the
+        # inferencecache-cachebackend group below — prefer that for clean
+        # per-CacheBackend attribution. This per-pod alert still binds
+        # directly to vLLM's exposition for pod-level granularity and to
+        # stay useful where the operator's scrape path isn't wired.
+        # Operators should sanity-check the metric
         # exists on at least one engine pod before relying on the
         # alert: see the runbook's "verify the metric is exposed"
         # step.
@@ -240,6 +244,55 @@ groups:
           summary: "Synthetic CacheBackend probe failing on {{ $labels.backend }} in namespace {{ $labels.namespace }} (stage={{ $labels.stage }})."
           runbook_url: "https://github.com/cachebox-project/inference-cache/blob/main/docs/observability/alerts.md#serverprobefail"
 
+      - alert: BackendT2Degraded
+        # Per-CacheBackend tier-2 (external offload) health, read from the
+        # controller-emitted `inferencecache_backend_t2_hit_rate` gauge (the
+        # CacheIndex poller projects it from status.indexParticipation.t2HitRate,
+        # computed from the engine's vllm:external_prefix_cache_{hits,queries}
+        # counters). A value of 0 means the tier was queried but has served ZERO
+        # reloads since it was first exercised — the silent-degradation signal
+        # (offload store/connection failure, an under-sized offload server, or a
+        # scheduler/worker hash mismatch).
+        #
+        # This is the per-CacheBackend counterpart to LMCacheT2NoHits above. That
+        # alert reads vLLM's per-POD exposition directly and so needs a vLLM
+        # PodMonitor scoped to cache-bound pods (see its SCRAPE-SCOPING note);
+        # this one binds to the operator's own gauge, whose `backend` label is the
+        # canonical <namespace>/<name> — so attribution is per-CacheBackend with
+        # no scrape-scoping caveat. It is the Alertmanager-side companion of the
+        # advisory T2Degraded CR condition (CacheBackend.status, not scraped) —
+        # but BROADER: the poller projects these metrics for ANY exercised backend,
+        # so this alert also fires for External backends, whose T2Degraded
+        # condition the controller does not manage. The condition is the
+        # managed-backend status view; this alert is the Prometheus view over
+        # every exercised backend.
+        #
+        # The query-token rate is the ACTIVITY GATE: it keeps a backend that took
+        # a few cold misses and then went idle from paging — its query counter goes
+        # flat (rate ~0, below the floor), so only a backend with sustained tier-2
+        # traffic AND zero reloads alerts. The 1000 tokens/sec floor matches the
+        # per-pod LMCacheT2NoHits (the counters are TOKENS, not requests — one
+        # ~1000-token prefixed request/sec clears it; idle replicas stay under).
+        # `max by` collapses the controller pod/instance labels (and any HA-replica
+        # duplicates), leaving a clean (namespace, backend) alert; `namespace` is
+        # the install scope Prometheus injects from the controller scrape target,
+        # so installs stay isolated. (inferencecache_backend_t2_query_tokens_total
+        # is a MONOTONIC counter the controller builds by accumulating only the
+        # positive per-tick deltas of the per-backend aggregate, so a replica/tenant
+        # drain or engine restart is clamped out and rate() never sees a phantom
+        # reset.)
+        expr: |
+          (max by (namespace, backend) (rate(inferencecache_backend_t2_query_tokens_total[10m])) > 1000)
+          and on (namespace, backend)
+          (max by (namespace, backend) (inferencecache_backend_t2_hit_rate) == 0)
+        for: 5m
+        labels:
+          severity: warning
+          component: inference-cache
+        annotations:
+          summary: "Tier-2 (external offload) on CacheBackend {{ $labels.backend }} is serving queries but zero reloads — silently degraded (store/connection failure, under-sized offload server, or scheduler/worker hash mismatch)."
+          runbook_url: "https://github.com/cachebox-project/inference-cache/blob/main/docs/observability/alerts.md#backendt2degraded"
+
   # ----------------------------------------------------------------------------
   # DEFERRED — the metrics these alerts consume are not yet exposed on
   # /metrics. Enable each rule in the same change that ships its metric.

config/observability/prometheus-rules-tests.yaml

@@ -1,8 +1,9 @@
-# Unit tests for the alerting rules in alerting-rules.yaml. Each Stage 1
-# alert has both a "should fire" and a "should not fire" case; together they
-# pin the threshold and confirm the alert is not spuriously firing on healthy
-# baselines. Run with `promtool test rules prometheus-rules-tests.yaml` (or
-# via `make verify-prometheus`).
+# Unit tests for the alerting rules in alerting-rules.yaml. Each active alert
+# — the server-side Stage 1 alerts plus the controller-side ServerProbeFail and
+# BackendT2Degraded — has "should fire" and "should not fire" cases; together
+# they pin the threshold and confirm the alert is not spuriously firing on
+# healthy baselines. Run with `promtool test rules prometheus-rules-tests.yaml`
+# (or via `make verify-prometheus`).
 #
 # Synthetic time-series notation: `N+Mx20` is "start at N, add M every step,
 # produce 21 samples total". The default step is `interval:` (30s here). For
@@ -479,3 +480,116 @@ tests:
       - eval_time: 15m
         alertname: ServerProbeFail
         exp_alerts: []
+
+  # --------------------------------------------------------------------------
+  # BackendT2Degraded (for: 5m, warning)
+  #
+  # Per-CacheBackend tier-2 health from two controller series:
+  #   inferencecache_backend_t2_query_tokens_total (cumulative; the ACTIVITY GATE via
+  #     rate(); backend=<namespace>/<name>) and
+  #   inferencecache_backend_t2_hit_rate (the ratio; == 0 means zero reloads).
+  # `namespace` is the install scope Prometheus injects from the controller
+  # scrape. Fires only on rate(query_tokens) > 1000 tokens/sec AND hit_rate == 0.
+  # query_tokens 0+60000x40 = 2000 tokens/sec (above the floor); 40 samples at
+  # 30s = 20m, past the [10m] rate window plus the 5m `for:` dwell.
+  # --------------------------------------------------------------------------
+  - interval: 30s
+    name: BackendT2Degraded fires when a queried backend serves zero reloads
+    input_series:
+      - series: 'inferencecache_backend_t2_query_tokens_total{namespace="inference-cache-system",backend="team-a/sample"}'
+        values: '0+60000x40'
+      - series: 'inferencecache_backend_t2_hit_rate{namespace="inference-cache-system",backend="team-a/sample"}'
+        values: '0+0x40'
+    alert_rule_test:
+      # Before the 5m `for:` dwell elapses the alert is only pending, not firing,
+      # even though the rate gate is already satisfied — this pins the dwell.
+      - eval_time: 3m
+        alertname: BackendT2Degraded
+        exp_alerts: []
+      - eval_time: 20m
+        alertname: BackendT2Degraded
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              component: inference-cache
+              namespace: inference-cache-system
+              backend: team-a/sample
+            exp_annotations:
+              summary: "Tier-2 (external offload) on CacheBackend team-a/sample is serving queries but zero reloads — silently degraded (store/connection failure, under-sized offload server, or scheduler/worker hash mismatch)."
+              runbook_url: "https://github.com/cachebox-project/inference-cache/blob/main/docs/observability/alerts.md#backendt2degraded"
+
+  - interval: 30s
+    name: BackendT2Degraded does not fire when the backend is serving reloads (hit-rate > 0)
+    input_series:
+      - series: 'inferencecache_backend_t2_query_tokens_total{namespace="inference-cache-system",backend="team-a/sample"}'
+        values: '0+60000x40'
+      - series: 'inferencecache_backend_t2_hit_rate{namespace="inference-cache-system",backend="team-a/sample"}'
+        values: '0.75+0x40'
+    alert_rule_test:
+      - eval_time: 20m
+        alertname: BackendT2Degraded
+        exp_alerts: []
+
+  - interval: 30s
+    # THE activity gate: a backend that took a few cold misses (hit-rate 0) and
+    # then went idle (query gauge flat -> rate 0, below the 1000/sec floor) must
+    # NOT page. This is the false positive a cumulative-only `== 0` form would hit.
+    name: BackendT2Degraded does not fire on an idle backend even with hit-rate zero
+    input_series:
+      - series: 'inferencecache_backend_t2_query_tokens_total{namespace="inference-cache-system",backend="team-a/sample"}'
+        values: '500+0x40'
+      - series: 'inferencecache_backend_t2_hit_rate{namespace="inference-cache-system",backend="team-a/sample"}'
+        values: '0+0x40'
+    alert_rule_test:
+      - eval_time: 20m
+        alertname: BackendT2Degraded
+        exp_alerts: []
+
+  - interval: 30s
+    # Coverage for the 1000 tokens/sec floor: a low-traffic backend (500/sec,
+    # below the floor) with zero reloads must NOT page.
+    name: BackendT2Degraded does not fire below the query-rate floor
+    input_series:
+      - series: 'inferencecache_backend_t2_query_tokens_total{namespace="inference-cache-system",backend="team-a/sample"}'
+        values: '0+15000x40'
+      - series: 'inferencecache_backend_t2_hit_rate{namespace="inference-cache-system",backend="team-a/sample"}'
+        values: '0+0x40'
+    alert_rule_test:
+      - eval_time: 20m
+        alertname: BackendT2Degraded
+        exp_alerts: []
+
+  - interval: 30s
+    # Two installs sharing one Prometheus, each with an actively-queried zero-hit
+    # backend of the same name in a different install namespace. Both alert
+    # independently -- `max by (namespace, backend)` preserves the install scope.
+    name: BackendT2Degraded keeps installs separate when backend names collide
+    input_series:
+      - series: 'inferencecache_backend_t2_query_tokens_total{namespace="install-a",backend="team-a/cache"}'
+        values: '0+60000x40'
+      - series: 'inferencecache_backend_t2_hit_rate{namespace="install-a",backend="team-a/cache"}'
+        values: '0+0x40'
+      - series: 'inferencecache_backend_t2_query_tokens_total{namespace="install-b",backend="team-a/cache"}'
+        values: '0+60000x40'
+      - series: 'inferencecache_backend_t2_hit_rate{namespace="install-b",backend="team-a/cache"}'
+        values: '0+0x40'
+    alert_rule_test:
+      - eval_time: 20m
+        alertname: BackendT2Degraded
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              component: inference-cache
+              namespace: install-a
+              backend: team-a/cache
+            exp_annotations:
+              summary: "Tier-2 (external offload) on CacheBackend team-a/cache is serving queries but zero reloads — silently degraded (store/connection failure, under-sized offload server, or scheduler/worker hash mismatch)."
+              runbook_url: "https://github.com/cachebox-project/inference-cache/blob/main/docs/observability/alerts.md#backendt2degraded"
+          - exp_labels:
+              severity: warning
+              component: inference-cache
+              namespace: install-b
+              backend: team-a/cache
+            exp_annotations:
+              summary: "Tier-2 (external offload) on CacheBackend team-a/cache is serving queries but zero reloads — silently degraded (store/connection failure, under-sized offload server, or scheduler/worker hash mismatch)."
+              runbook_url: "https://github.com/cachebox-project/inference-cache/blob/main/docs/observability/alerts.md#backendt2degraded"

config/observability/prometheus-rules.yaml

@@ -165,6 +165,28 @@ spec:
             summary: "Synthetic CacheBackend probe failing on {{ $labels.backend }} in namespace {{ $labels.namespace }} (stage={{ $labels.stage }})."
             runbook_url: "https://github.com/cachebox-project/inference-cache/blob/main/docs/observability/alerts.md#serverprobefail"
 
+        - alert: BackendT2Degraded
+          # Per-CacheBackend tier-2 (external offload) silent-degradation signal:
+          # the controller's inferencecache_backend_t2_hit_rate gauge is 0, i.e.
+          # the tier was queried but has served zero reloads. Per-CacheBackend
+          # counterpart to the per-pod LMCacheT2NoHits (no scrape-scoping caveat);
+          # companion to the advisory T2Degraded CR condition (broader: also fires
+          # for External backends, whose condition the controller does not manage).
+          # The query-token rate is
+          # the activity gate (an idle backend's flat counter → rate ~0 → no page).
+          # See the flat alerting-rules.yaml for the full rationale.
+          expr: |
+            (max by (namespace, backend) (rate(inferencecache_backend_t2_query_tokens_total[10m])) > 1000)
+            and on (namespace, backend)
+            (max by (namespace, backend) (inferencecache_backend_t2_hit_rate) == 0)
+          for: 5m
+          labels:
+            severity: warning
+            component: inference-cache
+          annotations:
+            summary: "Tier-2 (external offload) on CacheBackend {{ $labels.backend }} is serving queries but zero reloads — silently degraded (store/connection failure, under-sized offload server, or scheduler/worker hash mismatch)."
+            runbook_url: "https://github.com/cachebox-project/inference-cache/blob/main/docs/observability/alerts.md#backendt2degraded"
+
     # ------------------------------------------------------------------------
     # DEFERRED — the metrics these alerts consume are not yet exposed on
     # /metrics. Enable each rule in the same change that ships its metric.