feat(loadgen): zainod gRPC concurrency/load generator + multi-RPC driver

.containerignore

@@ -0,0 +1,2 @@
+target/
+.git/

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["macros"]
+members = ["macros", "crates/loadgen"]
 resolver = "2"
 
 # Post-ironwood dependency line. The whole zcash_* wallet stack now resolves

crates/loadgen/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "loadgen"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+authors.workspace = true
+repository.workspace = true
+homepage.workspace = true
+
+[[bin]]
+name = "loadgen"
+path = "src/main.rs"
+
+[dependencies]
+# The whole point: reuse ztest's load-test library (LwdClient / LoadDriver /
+# ChainLinkOracle / LoadReport). default-features off drops the wallet backends
+# (librustzcash/zingo) we don't need; kube/k8s-openapi remain non-optional but
+# we never touch them — we only speak gRPC to an already-running zainod.
+ztest = { path = "../..", default-features = false }
+
+tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
+clap = { version = "4", features = ["derive"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = { workspace = true }
+tracing = { workspace = true }
+thiserror = { workspace = true }
+anyhow = "1"

crates/loadgen/Containerfile

@@ -0,0 +1,25 @@
+# Multi-stage build for the `loadgen` zainod load generator.
+#
+# Build context MUST be the ztest repo root (loadgen depends on the ztest lib
+# via `path = "../.."` and inherits the workspace `[patch.crates-io]`):
+#
+#   podman build -f crates/loadgen/Containerfile -t loadgen:dev .
+#
+# `ztest` is pulled with default-features off (see crates/loadgen/Cargo.toml),
+# so the optional zingolib/pepper-sync/test-vector git deps are NOT fetched —
+# only crates.io + the public lightwallet-protocol patch. No build creds needed.
+
+FROM docker.io/library/rust:1.95-slim-bookworm AS builder
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends protobuf-compiler pkg-config git \
+ && rm -rf /var/lib/apt/lists/*
+WORKDIR /src
+COPY . .
+RUN cargo build --release -p loadgen
+
+FROM docker.io/library/debian:bookworm-slim AS runtime
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /src/target/release/loadgen /usr/local/bin/loadgen
+ENTRYPOINT ["/usr/local/bin/loadgen"]

crates/loadgen/README.md

@@ -0,0 +1,103 @@
+# loadgen
+
+A concurrency/load generator for a **running** zainod. It attaches to a zainod's
+gRPC `CompactTxStreamer` endpoint, fans out N concurrent clients driving a chosen
+RPC, and reports real latency percentiles, throughput, error counts, and
+correctness violations.
+
+Unlike the rest of ztest it does **not** spawn a topology — it points at an
+endpoint you give it (`--target`). The same binary loads a regtest node on a
+laptop or a mainnet node in-cluster, and runs as a plain Kubernetes Job.
+
+## Architecture
+
+```mermaid
+flowchart LR
+    subgraph job["k8s Job (runs IN-cluster)"]
+        lg["loadgen<br/>N per-task gRPC connections"]
+    end
+    lg -->|"CompactTxStreamer RPC<br/>GetBlockRange · GetBlock · GetLatestBlock"| z["zainod service<br/>zaino.&lt;ns&gt;.svc:8137"]
+    z -->|"fetch backend: JSON-RPC<br/>state backend: direct DB read"| zeb["zebra"]
+    lg -.->|"hdrhistogram + chain-link oracle"| rep["LoadReport<br/>human table (stderr) + JSON (stdout)"]
+```
+
+It reuses `ztest::loadtest` (the `LwdClient` / `LoadDriver` / `ChainLinkOracle` /
+`LoadReport` stack); this crate is the runnable front-end. Three seams are kept
+independent:
+
+- **driver** — the swarm + metrics (`ztest::loadtest`), speaks only the gRPC wire
+  protocol, so it is version- and target-agnostic.
+- **target** — supplied as a URI; how the zainod got there (regtest spawn,
+  ephemeral deploy, mainnet) is not loadgen's concern.
+- **reporting** — a human table on stderr and a machine-readable JSON summary on
+  stdout (`--json`).
+
+## Running
+
+### Locally (against any reachable endpoint)
+
+```
+cargo run -p loadgen -- \
+    --target http://127.0.0.1:8137 \
+    --rpc block-range --connections 64 --duration 30 --json
+```
+
+### In-cluster (the representative path)
+
+Latency is only meaningful measured **inside** the cluster — a port-forward from a
+remote node would dominate the numbers. Run it as a Job in the target namespace;
+the target is derived from the pod's own namespace:
+
+```
+kubectl -n <zaino-namespace> apply -f crates/loadgen/k8s/job.yaml
+kubectl -n <zaino-namespace> logs -f job/loadgen
+```
+
+### Connection sweep (find the knee)
+
+`scripts/sweep.sh` runs a sequential connection sweep (one Job per level — never
+parallel, which would confound the measurement) and collects one JSON line per
+level:
+
+```
+scripts/sweep.sh <zaino-namespace> "1 4 8 16 32 64 128" 20
+```
+
+## Flags
+
+| flag | default | meaning |
+|---|---|---|
+| `--target` | (required) | zainod gRPC endpoint, e.g. `http://zaino.ns.svc:8137` |
+| `--rpc` | `block-range` | `block-range` \| `latest-block` \| `block` |
+| `--connections` | 64 | concurrent connections (each a spawned task) |
+| `--conn-mode` | `per-task` | `per-task` (socket per client) or `shared` (one multiplexed channel) |
+| `--range` | — | `START..END`; either side may be empty (`a..` = to tip, `..b` = from genesis) |
+| `--tip-window` | 50000 | when `--range` is absent, sweep the last N blocks below the discovered tip |
+| `--blocks` | 100 | blocks per `GetBlockRange` window |
+| `--dist` | `even` | how windows spread across the pool (`even` \| `scatter`) |
+| `--duration` / `--count` | 30s | run for D seconds, or N ops per connection |
+| `--no-oracle` | off | disable the chain-link correctness oracle |
+| `--json` | off | emit the JSON summary to stdout |
+
+## Output
+
+- **Human table → stderr** (via `ztest::loadtest::LoadReport::print`).
+- **JSON summary → stdout** with `--json`: per-op `p50/p90/p99/p99.9/max` (ms),
+  throughput, total ops, error count, and any correctness violations.
+- **Logs** — structured `tracing` (set `ZTEST_LOG=loadgen=info,ztest=info`).
+
+The **chain-link oracle** validates every response under load: blocks must link
+(`prev_hash == prior.hash`), heights strictly increase, genesis is well-formed.
+It adapts per RPC — it validates single blocks for `GetBlock` and is skipped for
+the block-less `GetLatestBlock`. `0 violations` means every block served under
+load was a correctly-linked chain segment, not just that the server was fast.
+
+## Reproducibility & caveats
+
+- The container image (`zingodevops/loadgen`) pins the exact binary; the committed
+  `k8s/job.yaml` pins the exact args. A run is `kubectl apply` + read the logs.
+- Results are a **measurement, not a calibrated SLO**: absolute latency is only
+  trustworthy on a CPU-pinned, I/O-calibrated node. Trust the *shapes* (saturation,
+  tail knee, collapse) and the differential (backend/version A/B), not raw absolutes.
+- Target zainod should run with a **real finalised state** (not
+  `ZAINO_EPHEMERAL_FINALISED_STATE`, which inflates read performance).

crates/loadgen/k8s/job.yaml

@@ -0,0 +1,52 @@
+# In-cluster load run: a plain Job that hits the LOCAL zaino service directly.
+#
+# Nothing here is target-specific — apply it into whichever namespace's zaino
+# you want to test; the target is derived from the pod's own namespace, and the
+# height window is auto-discovered from the chain tip:
+#
+#   kubectl --context zingo-infra -n preview-070-rc1b apply -f job.yaml
+#   kubectl --context zingo-infra -n preview-070-rc1b logs -f job/loadgen
+#
+# The Job runs in the target namespace so in-cluster DNS resolves and no
+# cross-namespace NetworkPolicy is in the way.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: loadgen
+  labels:
+    app: loadgen
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    metadata:
+      labels:
+        app: loadgen
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: loadgen
+          image: zingodevops/loadgen:dev
+          imagePullPolicy: Always
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: ZTEST_LOG
+              value: loadgen=info,ztest=info
+          args:
+            - --target=http://zaino.$(POD_NAMESPACE).svc.cluster.local:8137
+            - --connections=64
+            - --tip-window=50000
+            - --blocks=100
+            - --conn-mode=per-task
+            - --duration=30
+            - --json
+          resources:
+            requests:
+              cpu: "1"
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 1Gi

crates/loadgen/scripts/sweep.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Connection-count sweep for loadgen: run one Job per concurrency level,
+# SEQUENTIALLY (parallel levels would load the server at once and confound the
+# measurement), and collect one JSON line per level.
+#
+#   scripts/sweep.sh <namespace> [connections] [duration_s] [rpc]
+#   scripts/sweep.sh preview-070-rc1c "1 4 8 16 32 64 128" 20 block-range
+#
+# Requires: kubectl context already pointed at the target cluster, and a zaino
+# service reachable at zaino.<namespace>.svc.cluster.local:8137.
+set -euo pipefail
+
+NS="${1:?usage: sweep.sh <namespace> [connections] [duration_s] [rpc]}"
+CONNS="${2:-1 4 8 16 32 64 128}"
+DUR="${3:-20}"
+RPC="${4:-block-range}"
+IMG="${LOADGEN_IMAGE:-docker.io/zingodevops/loadgen:dev}"
+OUT="${OUT:-sweep-${NS}-${RPC}.jsonl}"
+
+: > "$OUT"
+for c in $CONNS; do
+  name="loadgen-c${c}"
+  kubectl -n "$NS" delete job "$name" --ignore-not-found >/dev/null 2>&1 || true
+  cat <<EOF | kubectl -n "$NS" apply -f - >/dev/null
+apiVersion: batch/v1
+kind: Job
+metadata: { name: $name, labels: { app: loadgen } }
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 1800
+  template:
+    metadata: { labels: { app: loadgen } }
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: loadgen
+        image: $IMG
+        imagePullPolicy: Always
+        env:
+        - { name: POD_NAMESPACE, valueFrom: { fieldRef: { fieldPath: metadata.namespace } } }
+        - { name: ZTEST_LOG, value: "loadgen=info,ztest=info" }
+        args:
+        - --target=http://zaino.$NS.svc.cluster.local:8137
+        - --rpc=$RPC
+        - --connections=$c
+        - --tip-window=50000
+        - --blocks=100
+        - --conn-mode=per-task
+        - --duration=$DUR
+        - --json
+EOF
+  kubectl -n "$NS" wait --for=condition=complete "job/$name" --timeout=300s >/dev/null 2>&1 \
+    || kubectl -n "$NS" wait --for=condition=failed "job/$name" --timeout=5s >/dev/null 2>&1 || true
+  json=$(kubectl -n "$NS" logs "job/$name" 2>/dev/null | grep '^{' | head -1 || true)
+  if [ -n "$json" ]; then
+    echo "$json" >> "$OUT"
+    echo "conns=$c OK"
+  else
+    echo "conns=$c FAILED"
+    kubectl -n "$NS" logs "job/$name" 2>/dev/null | tail -3 || true
+  fi
+  kubectl -n "$NS" delete job "$name" --ignore-not-found >/dev/null 2>&1 || true
+done
+
+echo "=== results in $OUT ==="
+cat "$OUT"