Feat/observability

README.md

@@ -91,11 +91,11 @@ terraform destroy
 ### Opening again (same machine, VPS still exists)
 
 1. Hetzner console → power **ON** if you powered off.
-2. Confirm your public IP still matches the firewall allow-list in `main.tf` (`ssh_source_cidrs`). If your ISP changed it, update the CIDR and `terraform apply` before SSH will work.
+2. Confirm your public IP still matches `admin_cidrs` in `terraform.tfvars`. If your ISP changed it, update and `terraform apply` before SSH/kubectl will work (`curl -4 ifconfig.me`).
 3. Point kubectl at the hobby cluster and verify:
 
 ```bash
-export KUBECONFIG=~/.kube/hobby.yaml
+./scripts/fetch-hobby-kubeconfig.sh   # or: export KUBECONFIG=~/.kube/hobby.yaml
 kubectl get nodes
 kubectl -n argocd get pods
 kubectl -n argocd port-forward svc/argocd-server 8080:443
@@ -112,13 +112,8 @@ You need: this repo, your SSH **private** key (same key Terraform registered), `
 git clone https://github.com/notsubash/cloud-native-AI-platform.git
 cd cloud-native-AI-platform
 
-# Laptop only — do not run scp while SSH'd into the VPS
-mkdir -p ~/.kube
-scp -i ~/.ssh/id_ed25519 root@<VPS_IP>:/etc/rancher/k3s/k3s.yaml ~/.kube/hobby.yaml
-# Edit hobby.yaml: replace 127.0.0.1 with <VPS_IP>
-
-export KUBECONFIG=~/.kube/hobby.yaml
-kubectl get nodes
+./scripts/fetch-hobby-kubeconfig.sh
+# needs terraform state (or set IP manually — see script / terraform output)
 
 # Argo UI
 kubectl -n argocd port-forward svc/argocd-server 8080:443
@@ -130,23 +125,26 @@ If you lack Terraform state on the new machine, manage the existing server from
 
 ```bash
 cd infrastructure/terraform/environments/hobby
-cp terraform.tfvars.example terraform.tfvars   # set ssh_public_key_path
+cp terraform.tfvars.example terraform.tfvars
+# set ssh_public_key_path + admin_cidrs = ["YOUR.IP/32"]  (curl -4 ifconfig.me)
 export HCLOUD_TOKEN=...
-# Update ssh_source_cidrs in main.tf to YOUR current public IP/32
 terraform init && terraform plan && terraform apply
 
-# Wait ~2–3 min for cloud-init/k3s, then copy kubeconfig (see A)
+# from repo root — waits for k3s, writes ~/.kube/hobby.yaml with public IP
+./scripts/fetch-hobby-kubeconfig.sh
+
 # Install Argo CD (server-side apply), create ghcr-pull secret, apply gitops/applications/api.yaml
 # Full sequence: gitops/README.md
 ```
 
+Firewall allows **SSH (22)** and **kubectl (6443)** only from `admin_cidrs`. No SSH tunnel needed when your IP matches.
 ### Cost hygiene checklist
 
 - [ ] Before leaving for the day: power off **or** destroy (know which you chose).
 - [ ] If paused > 7 days: destroy, don’t leave an idle ON server.
 - [ ] After destroy: confirm Hetzner console shows **no** `cnai-hobby` server.
 - [ ] Never commit `HCLOUD_TOKEN`, `terraform.tfvars`, `*.tfstate`, or the GHCR PAT.
-- [ ] Firewall is IP-locked — a new network/café IP blocks SSH until you update `ssh_source_cidrs`.
+- [ ] Firewall is IP-locked — a new network/café IP blocks SSH/kubectl until you update `admin_cidrs` in `terraform.tfvars` and apply.
 
 ## Local development (Compose)
 

apps/api/app/main.py

@@ -1,25 +1,88 @@
 from __future__ import annotations
 
+import uuid
 import json
 import logging
 from contextlib import asynccontextmanager
 from typing import Annotated
-
+import time
 import httpx
 import psycopg
 import redis
-from fastapi import Depends, FastAPI, HTTPException, Response
-from prometheus_client import CONTENT_TYPE_LATEST, Counter, generate_latest
+from fastapi import Depends, FastAPI, HTTPException, Response, Request
+from prometheus_client import CONTENT_TYPE_LATEST, Counter, generate_latest, Histogram
 from pydantic import BaseModel, Field
-
+from starlette.middleware.base import BaseHTTPMiddleware
 from app.config import Settings, get_settings
 from app.llm import summarize
 
-logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s request_id=%(request_id)s %(message)s")
 log = logging.getLogger("api")
 
 SUMMARIZE_REQUESTS = Counter("summarize_requests_total", "Summarize endpoint calls", ["status"])
 
+# ---- RED metrics for ALL HTTP requests -------------------------------------
+HTTP_REQUESTS = Counter(
+    "http_requests_total",
+    "HTTP requests",
+    ["method", "path", "status"],
+)
+
+# Buckets in seconds — tune for your SLOs. These are fine for a learning API.
+HTTP_DURATION = Histogram(
+    "http_request_duration_seconds",
+    "HTTP request latency",
+    ["method", "path"],
+    buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0),
+)
+
+class RequestIdFilter(logging.Filter):
+    """Ensure every log record has request_id so the format string never KeyErrors."""
+    def filter(self, record: logging.LogRecord) -> bool:
+        if not hasattr(record, "request_id"):
+            record.request_id = "-"
+        return True
+logging.getLogger().addFilter(RequestIdFilter())
+class ObservabilityMiddleware(BaseHTTPMiddleware):
+    """
+    One middleware, two teaching jobs:
+      1) Attach X-Request-ID (client-provided or generated) for log correlation
+      2) Record RED metrics for every request
+    Later (OTel): the same place is where a trace span often starts.
+    """
+    async def dispatch(self, request: Request, call_next):
+        # Prefer inbound header so a client / gateway can propagate the id
+        request_id = request.headers.get("x-request-id") or str(uuid.uuid4())
+        request.state.request_id = request_id
+        # Skip scraping /metrics itself from RED (optional; avoids noise)
+        path = request.url.path
+        track = path != "/metrics"
+        start = time.perf_counter()
+        status_code = 500
+        try:
+            response = await call_next(request)
+            status_code = response.status_code
+            response.headers["X-Request-ID"] = request_id
+            return response
+        finally:
+            if track:
+                elapsed = time.perf_counter() - start
+                # Use path as-is for this tiny API; normalize if you add path params
+                HTTP_REQUESTS.labels(
+                    method=request.method,
+                    path=path,
+                    status=str(status_code),
+                ).inc()
+                HTTP_DURATION.labels(method=request.method, path=path).observe(elapsed)
+                # Structured-ish log line — Loki can filter on request_id=
+                log.info(
+                    "request method=%s path=%s status=%s duration_ms=%.1f",
+                    request.method,
+                    path,
+                    status_code,
+                    elapsed * 1000.0,
+                    extra={"request_id": request_id},
+                )
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
@@ -30,6 +93,7 @@ async def lifespan(app: FastAPI):
 
 
 app = FastAPI(title="cloud-native-ai-api", version="0.1.0", lifespan=lifespan)
+app.add_middleware(ObservabilityMiddleware)
 
 SettingsDep = Annotated[Settings, Depends(get_settings)]
 
@@ -99,3 +163,11 @@ def summarize_endpoint(body: SummarizeIn, settings: SettingsDep):
     except Exception:
         SUMMARIZE_REQUESTS.labels(status="error").inc()
         raise
+
+# ---------------------------------------------------------------------------
+# TEACHING ONLY — remove or gate behind env before "real" use.
+# Hit this to practice: metrics spike → Loki shows request_id → you diagnose.
+# ---------------------------------------------------------------------------
+@app.get("/debug/boom")
+def boom():
+    raise HTTPException(status_code=500, detail="forced failure for observability drill")

gitops/README.md

@@ -18,14 +18,19 @@ GitHub (branch in Application) → Argo CD → Helm (helm/api + values-hobby.yam
 
 ## Prerequisites (cluster already up)
 
-- Hobby VPS with k3s (Terraform + cloud-init)
-- `KUBECONFIG` pointing at the VPS (e.g. `~/.kube/hobby.yaml` with `127.0.0.1` replaced by the public IP)
+- Hobby VPS with k3s (Terraform + cloud-init; API cert includes public IP via `--tls-san`)
+- Firewall allows your IP on **22** and **6443** (`admin_cidrs` in `terraform.tfvars`)
+- Laptop kubeconfig from `./scripts/fetch-hobby-kubeconfig.sh` → `export KUBECONFIG=~/.kube/hobby.yaml`
 - Argo CD installed in namespace `argocd`
 - Namespace `ai-platform` and docker-registry secret `ghcr-pull` (GitHub PAT with `read:packages` — not an image tag)
 
 ## Bootstrap Argo CD (once per cluster)
 
 ```bash
+# From repo root, after terraform apply:
+./scripts/fetch-hobby-kubeconfig.sh
+export KUBECONFIG=~/.kube/hobby.yaml
+
 kubectl create namespace argocd
 kubectl apply --server-side -n argocd \
   -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
@@ -83,7 +88,8 @@ Automated sync / self-heal can be enabled later in `api.yaml` (`syncPolicy.autom
 | `values-hobby.yaml: no such file` | File not on the tracked revision / not committed |
 | Sync OK but `ImagePullBackOff` | Bad `ghcr-pull` secret (need PAT with `read:packages`) |
 | Application Healthy but empty namespace | Manual sync not run yet |
-| SSH to VPS hangs | Your public IP changed; update `ssh_source_cidrs` in Terraform |
+| SSH to VPS hangs / kubectl :6443 times out | Your public IP changed; update `admin_cidrs` in `terraform.tfvars` + `terraform apply` |
+| Host key verification failed after recreate | Normal after destroy/recreate; `./scripts/fetch-hobby-kubeconfig.sh` clears the old key |
 
 ## Cost note
 

helm/api/templates/servicemonitor.yaml

@@ -0,0 +1,31 @@
+{{- /*
+  ServiceMonitor is a Prometheus Operator CRD.
+  When present, the Operator configures Prometheus to scrape this Service.
+
+  Without this (or pod annotations), Prometheus has no idea your /metrics exists.
+*/ -}}
+{{- if .Values.metrics.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "api.fullname" . }}
+  labels:
+    {{- include "api.labels" . | nindent 4 }}
+    # Optional: match whatever label your Prometheus selects on
+    release: mon
+spec:
+  # Which Services to scrape (label selector)
+  selector:
+    matchLabels:
+      {{- include "api.selectorLabels" . | nindent 6 }}
+
+  # Namespace of the Service (usually same as the chart release)
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+
+  endpoints:
+    - port: http          # must match Service port *name* in service.yaml
+      path: /metrics      # your FastAPI route
+      interval: 30s
+{{- end }}

helm/api/values.yaml

@@ -58,4 +58,9 @@ probes:
     initialDelaySeconds: 5
     periodSeconds: 5
 
+# Prometheus scrape via Operator
+metrics:
+  serviceMonitor:
+    enabled: true
+
 # Namespace is usually set by helm -n, not hardcoded int emplates

infrastructure/terraform/environments/hobby/cloud-init.yaml.tpl

@@ -1,11 +1,10 @@
 #cloud-config
 # =============================================================================
-# Minimal cloud-init — install single-node k3s
-# =============================================================================
-# After terraform apply:
-#   ssh root@<ip>
-#   cat /etc/rancher/k3s/k3s.yaml   # this is your kubeconfig
-# On your laptop: replace 127.0.0.1 with the public IP.
+# First-boot: install single-node k3s with TLS SAN = this VPS public IP
+# so kubectl from your laptop (https://<public-ip>:6443) trusts the API cert.
+#
+# Public IP comes from Hetzner link-local metadata (no Terraform cycle needed).
+# After apply: ./scripts/fetch-hobby-kubeconfig.sh
 # =============================================================================
 
 package_update: true
@@ -14,6 +13,9 @@ packages:
   - curl
 
 runcmd:
-  # Install k3s as a single-node cluster (default Traefik + servicelb).
-  # DISABLE traefik later if you prefer nginx — for Phase 6, leave defaults.
-  - curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644
+  - |
+    set -e
+    PUBLIC_IP=$(curl -fsSL http://169.254.169.254/hetzner/v1/metadata/public-ipv4)
+    curl -sfL https://get.k3s.io | sh -s - \
+      --write-kubeconfig-mode 644 \
+      --tls-san "$PUBLIC_IP"

infrastructure/terraform/environments/hobby/main.tf

@@ -1,38 +1,38 @@
 # =============================================================================
-# templatefile() reads the .tpl and substitutes ${...} variables.
-# Here we keep the template simple (no vars) — empty map {}.
+# Hobby env: one VPS + firewall. cloud-init installs k3s with --tls-san so
+# kubectl can talk to the public IP on :6443 (firewall-locked to admin_cidrs).
 #
-# MONEY MOMENT: the next `terraform apply` creates a billable Hetzner server.
-# ==========================================================================
+# MONEY: terraform apply starts Hetzner billing; destroy stops it.
+# =============================================================================
 
 locals {
-    ssh_public_key = trimspace(file(pathexpand(var.ssh_public_key_path)))
+  ssh_public_key = trimspace(file(pathexpand(var.ssh_public_key_path)))
 }
 
 module "firewall" {
-    source = "../../modules/firewall"
+  source = "../../modules/firewall"
 
-    name = "${var.server_name}-fw"
-    ssh_source_cidrs = ["103.129.135.175/32"] 
+  name             = "${var.server_name}-fw"
+  ssh_source_cidrs = var.admin_cidrs
 }
 
 module "server" {
-    source = "../../modules/server"
+  source = "../../modules/server"
 
-    name         = var.server_name
-    server_type  = var.server_type
-    location     = var.location
-    ssh_public_key = local.ssh_public_key
-    firewall_id  = module.firewall.id
+  name           = var.server_name
+  server_type    = var.server_type
+  location       = var.location
+  ssh_public_key = local.ssh_public_key
+  firewall_id    = module.firewall.id
 
-    user_data = templatefile("${path.module}/cloud-init.yaml.tpl", {})
+  user_data = templatefile("${path.module}/cloud-init.yaml.tpl", {})
 }
 
 # module "dns" {
-#  source = "../../modules/dns"
-#  count  = var.enable_dns ? 1 : 0
-#  
-#  zone_name    = var.domain
-#  record_name  = var.subdomain
-#  ipv4_address = module.server.ipv4_address
-#}
+#   source = "../../modules/dns"
+#   count  = var.enable_dns ? 1 : 0
+#
+#   zone_name    = var.domain
+#   record_name  = var.subdomain
+#   ipv4_address = module.server.ipv4_address
+# }

infrastructure/terraform/environments/hobby/outputs.tf

@@ -1,5 +1,5 @@
 output "public_ip" {
-  description = "VPS IPv4 — use for SSH and later DNS"
+  description = "VPS IPv4 — SSH and kubectl API"
   value       = module.server.ipv4_address
 }
 
@@ -8,7 +8,12 @@ output "ssh_command" {
   value       = "ssh root@${module.server.ipv4_address}"
 }
 
+output "kubeconfig_hint" {
+  description = "After apply + ~2 min for cloud-init, run this from repo root"
+  value       = "./scripts/fetch-hobby-kubeconfig.sh"
+}
+
 output "dns_name" {
   description = "FQDN if DNS enabled"
   value       = null
-}
+}

infrastructure/terraform/environments/hobby/terraform.tfvars.example

@@ -1,9 +1,17 @@
 # Copy to terraform.tfvars (gitignored) and fill in.
+#
+# Your current public IP (must match the network you kubectl/SSH from):
+#   curl -4 ifconfig.me
+# If your ISP changes IP, update admin_cidrs and terraform apply.
 
 server_name         = "cnai-hobby"
-server_type         = "cx22"
+server_type         = "cx23"
 location            = "nbg1"
 ssh_public_key_path = "~/.ssh/id_ed25519.pub"
-enable_dns          = false
-# domain            = "example.com"
-# subdomain         = "lab"
+
+# SSH + Kubernetes API (6443) — lock to your IP only
+admin_cidrs = ["203.0.113.10/32"]
+
+enable_dns = false
+# domain    = "example.com"
+# subdomain = "lab"