diff --git a/api/v1alpha1/cachebackend_types.go b/api/v1alpha1/cachebackend_types.go index 57bb78b8..758873f3 100644 --- a/api/v1alpha1/cachebackend_types.go +++ b/api/v1alpha1/cachebackend_types.go @@ -46,22 +46,24 @@ const ( // (spec.autoscaling) are both reconciled today. The autoscaling spec drives a // HorizontalPodAutoscaler. On a managed Deployment backend // (deploymentKind=Deployment, the default), spec.storage.pvc provisions a -// PersistentVolumeClaim owner-referenced to the CacheBackend (reclaimed only -// when the CacheBackend is deleted), mounts it into the cache-server pod at the -// runtime adapter's declared data path, and reports the bound PVC's actual size -// in status.capacity. +// single PersistentVolumeClaim owner-referenced to the CacheBackend (reclaimed +// only when the CacheBackend is deleted), mounts it into the cache-server pod at +// the runtime adapter's declared data path, and reports the bound PVC's actual +// size in status.capacity. On a managed StatefulSet backend, spec.storage.pvc +// renders a volumeClaimTemplate instead, giving each replica its own +// ReadWriteOnce PVC. // // Two scoping rules apply: // -// - Only a managed Deployment backend provisions storage. A StatefulSet -// backend is routed to the unmanaged path today (see DeploymentKind), so it -// provisions nothing, storage included; per-replica volumeClaimTemplates are -// a follow-up. An External backend provisions no workload at all, so -// spec.storage.pvc is a no-op there. -// - A persistent backend must be single-replica. A ReadWriteOnce PVC cannot be -// multi-attached, so replicas (or an autoscaling ceiling) > 1 is surfaced -// Ready=False/InvalidStorageConfiguration rather than provisioned; -// per-replica persistent storage via StatefulSet is a separate follow-up. +// - Managed Deployment backends provision one shared PVC. Managed StatefulSet +// backends provision per-replica PVCs via volumeClaimTemplates. External +// backends provision no workload at all, so spec.storage.pvc is a no-op +// there. +// - A persistent Deployment backend must be single-replica. A ReadWriteOnce +// PVC cannot be multi-attached, so replicas (or an autoscaling ceiling) > 1 +// is surfaced Ready=False/InvalidStorageConfiguration rather than +// provisioned. Use deploymentKind=StatefulSet for multi-replica persistent +// storage. // // NOTE: provisioning + mounting the PVC does not by itself make the cache server // spill KV to it — switching the LMCache server to a disk-backed storage device @@ -79,9 +81,9 @@ type CacheBackendSpec struct { Type CacheBackendType `json:"type,omitempty"` // DeploymentKind identifies whether a managed backend is reconciled as a - // Deployment or StatefulSet. Defaults to Deployment — the only kind the - // Phase-1 reconciler templates; StatefulSet is reserved for future - // per-replica-PVC topologies and is a no-op today. + // Deployment or StatefulSet. Defaults to Deployment. StatefulSet uses the + // same adapter-rendered pod and service shape, but can back + // spec.storage.pvc with per-replica volumeClaimTemplates. // +optional // +kubebuilder:default=Deployment DeploymentKind CacheBackendDeploymentKind `json:"deploymentKind,omitempty"` @@ -563,12 +565,13 @@ type CacheBackendStatus struct { Endpoint string `json:"endpoint,omitempty"` // Capacity is a human-readable summary of the backend's provisioned - // capacity: the bound PersistentVolumeClaim's actual capacity when - // spec.storage.pvc is set and the PVC has bound (the real provisioned size, - // which may exceed the request), or empty for an ephemeral backend or while - // the PVC is still pending (e.g. a WaitForFirstConsumer StorageClass that - // binds only once the pod schedules). It is informational; clients must not - // parse it. + // capacity: for Deployment-backed persistent storage, the bound shared + // PersistentVolumeClaim's actual capacity when the PVC has bound (the real + // provisioned size, which may exceed the request). Empty for an ephemeral + // backend, a StatefulSet backend (per-replica PVCs have no aggregate + // capacity projection yet), or while the Deployment PVC is still pending + // (e.g. a WaitForFirstConsumer StorageClass that binds only once the pod + // schedules). It is informational; clients must not parse it. // +optional Capacity string `json:"capacity,omitempty"` @@ -623,42 +626,41 @@ type CacheBackendStatus struct { FirstKVEventObservedAt *metav1.Time `json:"firstKVEventObservedAt,omitempty"` // FirstAvailableAt latches the first time the managed cache-backend - // workload was observed Available — the stable anchor for the - // firstEventTimeout clock. It is deliberately a latched timestamp rather - // than the live Deployment's Available condition LastTransitionTime: that - // condition resets on an availability flap, which would restart the + // workload was observed serving (Deployment Available or StatefulSet Ready) + // — the stable anchor for the firstEventTimeout clock. It is deliberately + // a latched timestamp rather than the live workload transition time: that + // transition resets on an availability flap, which would restart the // timeout window and let a backend that already breached the timeout // (Degraded / NoKVEventsObserved) bounce back to AwaitingFirstKVEvent // without any KV event — contradicting the "once Degraded, stays Degraded // until an event arrives" contract. Anchoring on this write-once value // keeps the elapsed window monotonic, so Degraded is sticky. Written - // write-once when the workload first reports Available and never cleared + // write-once when the workload first reports serving and never cleared // (inert while the backend is not managed). A genuinely recreated managed - // Deployment keeps the prior anchor; the gate re-evaluates from it, which - // is safe because the engine event source is unchanged by a cache-server + // workload keeps the prior anchor; the gate re-evaluates from it, which is + // safe because the engine event source is unchanged by a cache-server // restart. // +optional FirstAvailableAt *metav1.Time `json:"firstAvailableAt,omitempty"` - // ObservedServerInstance is the controller's cascade-decision - // baseline — a stable identifier for the Ready cache-server pod - // set the controller last anchored against. NOT a live current- - // pod-set view: the controller intentionally pins this through - // transient rolling-update midpoints and through no-Ready - // windows so the cascade does not fire on rollbacks or transient - // outages. For the live pod inventory, operators should consult + // ObservedServerInstance is the controller's cascade-decision baseline — a + // stable identifier for the Ready cache-server pod set the controller last + // anchored against. NOT a live current-pod-set view: the controller + // intentionally pins this through transient rolling-update midpoints and + // through no-Ready windows so the cascade does not fire on rollbacks or + // transient outages. For the live pod inventory, operators should consult // status.matchedEnginePods (engine side) and `kubectl get pod` // (cache-server side). // - // Shape: `:` per Ready pod, comma-joined - // and lex-sorted by pod name. restart-sum is the per-pod - // containerStatuses[].RestartCount summed across cache-server - // containers (the names from the owned Deployment's pod - // template; foreign sidecars are excluded). Inert and cleared - // for External backends, unsupported-runtime backends, and on - // the InvalidStorageConfiguration gate (a persistent - // multi-replica spec the controller refuses to provision until - // the operator scales to 1 or removes spec.storage.pvc). + // Shape: `:` per Ready pod, comma-joined and + // lex-sorted by pod name. restart-sum is the per-pod + // containerStatuses[].RestartCount summed across cache-server containers + // (the names from the owned workload's pod template; foreign sidecars are + // excluded). Inert and cleared for External backends, unsupported-runtime + // backends, and on the InvalidStorageConfiguration gate (a persistent + // multi-replica Deployment spec the controller refuses to provision until + // the operator scales to 1, removes spec.storage.pvc, or switches to + // StatefulSet). // // Operator-side recovery for the upstream LMCache // LMServerConnector EPIPE-on-restart bug. See diff --git a/config/crd/bases/inferencecache.io_cachebackends.yaml b/config/crd/bases/inferencecache.io_cachebackends.yaml index bb3bb0fa..d57eeb78 100644 --- a/config/crd/bases/inferencecache.io_cachebackends.yaml +++ b/config/crd/bases/inferencecache.io_cachebackends.yaml @@ -68,22 +68,24 @@ spec: (spec.autoscaling) are both reconciled today. The autoscaling spec drives a HorizontalPodAutoscaler. On a managed Deployment backend (deploymentKind=Deployment, the default), spec.storage.pvc provisions a - PersistentVolumeClaim owner-referenced to the CacheBackend (reclaimed only - when the CacheBackend is deleted), mounts it into the cache-server pod at the - runtime adapter's declared data path, and reports the bound PVC's actual size - in status.capacity. + single PersistentVolumeClaim owner-referenced to the CacheBackend (reclaimed + only when the CacheBackend is deleted), mounts it into the cache-server pod at + the runtime adapter's declared data path, and reports the bound PVC's actual + size in status.capacity. On a managed StatefulSet backend, spec.storage.pvc + renders a volumeClaimTemplate instead, giving each replica its own + ReadWriteOnce PVC. Two scoping rules apply: - - Only a managed Deployment backend provisions storage. A StatefulSet - backend is routed to the unmanaged path today (see DeploymentKind), so it - provisions nothing, storage included; per-replica volumeClaimTemplates are - a follow-up. An External backend provisions no workload at all, so - spec.storage.pvc is a no-op there. - - A persistent backend must be single-replica. A ReadWriteOnce PVC cannot be - multi-attached, so replicas (or an autoscaling ceiling) > 1 is surfaced - Ready=False/InvalidStorageConfiguration rather than provisioned; - per-replica persistent storage via StatefulSet is a separate follow-up. + - Managed Deployment backends provision one shared PVC. Managed StatefulSet + backends provision per-replica PVCs via volumeClaimTemplates. External + backends provision no workload at all, so spec.storage.pvc is a no-op + there. + - A persistent Deployment backend must be single-replica. A ReadWriteOnce + PVC cannot be multi-attached, so replicas (or an autoscaling ceiling) > 1 + is surfaced Ready=False/InvalidStorageConfiguration rather than + provisioned. Use deploymentKind=StatefulSet for multi-replica persistent + storage. NOTE: provisioning + mounting the PVC does not by itself make the cache server spill KV to it — switching the LMCache server to a disk-backed storage device @@ -152,9 +154,9 @@ spec: default: Deployment description: |- DeploymentKind identifies whether a managed backend is reconciled as a - Deployment or StatefulSet. Defaults to Deployment — the only kind the - Phase-1 reconciler templates; StatefulSet is reserved for future - per-replica-PVC topologies and is a no-op today. + Deployment or StatefulSet. Defaults to Deployment. StatefulSet uses the + same adapter-rendered pod and service shape, but can back + spec.storage.pvc with per-replica volumeClaimTemplates. enum: - Deployment - StatefulSet @@ -2133,12 +2135,13 @@ spec: capacity: description: |- Capacity is a human-readable summary of the backend's provisioned - capacity: the bound PersistentVolumeClaim's actual capacity when - spec.storage.pvc is set and the PVC has bound (the real provisioned size, - which may exceed the request), or empty for an ephemeral backend or while - the PVC is still pending (e.g. a WaitForFirstConsumer StorageClass that - binds only once the pod schedules). It is informational; clients must not - parse it. + capacity: for Deployment-backed persistent storage, the bound shared + PersistentVolumeClaim's actual capacity when the PVC has bound (the real + provisioned size, which may exceed the request). Empty for an ephemeral + backend, a StatefulSet backend (per-replica PVCs have no aggregate + capacity projection yet), or while the Deployment PVC is still pending + (e.g. a WaitForFirstConsumer StorageClass that binds only once the pod + schedules). It is informational; clients must not parse it. type: string conditions: description: Conditions describe the latest observations of the backend. @@ -2214,19 +2217,19 @@ spec: firstAvailableAt: description: |- FirstAvailableAt latches the first time the managed cache-backend - workload was observed Available — the stable anchor for the - firstEventTimeout clock. It is deliberately a latched timestamp rather - than the live Deployment's Available condition LastTransitionTime: that - condition resets on an availability flap, which would restart the + workload was observed serving (Deployment Available or StatefulSet Ready) + — the stable anchor for the firstEventTimeout clock. It is deliberately + a latched timestamp rather than the live workload transition time: that + transition resets on an availability flap, which would restart the timeout window and let a backend that already breached the timeout (Degraded / NoKVEventsObserved) bounce back to AwaitingFirstKVEvent without any KV event — contradicting the "once Degraded, stays Degraded until an event arrives" contract. Anchoring on this write-once value keeps the elapsed window monotonic, so Degraded is sticky. Written - write-once when the workload first reports Available and never cleared + write-once when the workload first reports serving and never cleared (inert while the backend is not managed). A genuinely recreated managed - Deployment keeps the prior anchor; the gate re-evaluates from it, which - is safe because the engine event source is unchanged by a cache-server + workload keeps the prior anchor; the gate re-evaluates from it, which is + safe because the engine event source is unchanged by a cache-server restart. format: date-time type: string @@ -2314,25 +2317,24 @@ spec: type: integer observedServerInstance: description: |- - ObservedServerInstance is the controller's cascade-decision - baseline — a stable identifier for the Ready cache-server pod - set the controller last anchored against. NOT a live current- - pod-set view: the controller intentionally pins this through - transient rolling-update midpoints and through no-Ready - windows so the cascade does not fire on rollbacks or transient - outages. For the live pod inventory, operators should consult + ObservedServerInstance is the controller's cascade-decision baseline — a + stable identifier for the Ready cache-server pod set the controller last + anchored against. NOT a live current-pod-set view: the controller + intentionally pins this through transient rolling-update midpoints and + through no-Ready windows so the cascade does not fire on rollbacks or + transient outages. For the live pod inventory, operators should consult status.matchedEnginePods (engine side) and `kubectl get pod` (cache-server side). - Shape: `:` per Ready pod, comma-joined - and lex-sorted by pod name. restart-sum is the per-pod - containerStatuses[].RestartCount summed across cache-server - containers (the names from the owned Deployment's pod - template; foreign sidecars are excluded). Inert and cleared - for External backends, unsupported-runtime backends, and on - the InvalidStorageConfiguration gate (a persistent - multi-replica spec the controller refuses to provision until - the operator scales to 1 or removes spec.storage.pvc). + Shape: `:` per Ready pod, comma-joined and + lex-sorted by pod name. restart-sum is the per-pod + containerStatuses[].RestartCount summed across cache-server containers + (the names from the owned workload's pod template; foreign sidecars are + excluded). Inert and cleared for External backends, unsupported-runtime + backends, and on the InvalidStorageConfiguration gate (a persistent + multi-replica Deployment spec the controller refuses to provision until + the operator scales to 1, removes spec.storage.pvc, or switches to + StatefulSet). Operator-side recovery for the upstream LMCache LMServerConnector EPIPE-on-restart bug. See diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 94443ead..64ff5797 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -47,6 +47,7 @@ rules: - apps resources: - deployments + - statefulsets verbs: - create - delete @@ -60,6 +61,7 @@ rules: resources: - deployments/status - replicasets + - statefulsets/status verbs: - get - apiGroups: diff --git a/config/samples/README.md b/config/samples/README.md index d7a118b2..aed3eeba 100644 --- a/config/samples/README.md +++ b/config/samples/README.md @@ -30,7 +30,7 @@ before applying. All but `recipe-gpu-production` run without a GPU. | [`recipe-external-cache.yaml`](recipe-external-cache.yaml) | `type: External` — point the operator at a cache server you manage yourself; the controller provisions nothing. | | [`recipe-multi-tenant.yaml`](recipe-multi-tenant.yaml) | Two CacheTenants + two CacheBackends across two namespaces — isolated cache identity and entry-count quotas; separate engines for per-tenant memory isolation. | | [`recipe-tuning.yaml`](recipe-tuning.yaml) | CPU-dev shape plus a meaningful `engineOverrides` block (tune `LMCACHE_CHUNK_SIZE`, add `LMCACHE_LOG_LEVEL=DEBUG`). | -| [`recipe-persistent-cache.yaml`](recipe-persistent-cache.yaml) | `spec.storage.pvc` — provisions a PVC owner-referenced to the CacheBackend and mounts it into the cache-server pod (single replica; ReadWriteOnce). Adopt-and-keep on removal. (Server-side disk-backed KV is a separate follow-up; the volume is mounted but the server still keeps KV in memory.) | +| [`recipe-persistent-cache.yaml`](recipe-persistent-cache.yaml) | `spec.storage.pvc` on `deploymentKind: Deployment` — provisions one owner-referenced ReadWriteOnce PVC and mounts it into the cache-server pod. Use `deploymentKind: StatefulSet` for per-replica PVCs. (Server-side disk-backed KV is a separate follow-up; the volume is mounted but the server still keeps KV in memory.) | **Prerequisites per recipe.** Most recipes are self-contained. One has an external dependency: `recipe-external-cache.yaml` needs a cache server already diff --git a/config/samples/cachebackend-lmcache.yaml b/config/samples/cachebackend-lmcache.yaml index 292af611..43d0abc0 100644 --- a/config/samples/cachebackend-lmcache.yaml +++ b/config/samples/cachebackend-lmcache.yaml @@ -10,15 +10,15 @@ # survive injection (the adapter merges, never clobbers). A pod can opt # out per-instance with the annotation `inferencecache.io/skip-inject`. # -# Phase 1 manages a Deployment + ClusterIP Service on port 65432 (the -# canonical LMCache lm:// port), plus an optional HorizontalPodAutoscaler -# from spec.autoscaling. spec.storage.pvc provisions a PersistentVolumeClaim -# owner-referenced to the CacheBackend and mounts it into the cache-server -# pod (see recipe-persistent-cache.yaml for a worked example). Switching the +# The controller manages a Deployment or StatefulSet plus a ClusterIP Service +# on port 65432 (the canonical LMCache lm:// port), plus an optional +# HorizontalPodAutoscaler from spec.autoscaling. spec.storage.pvc provisions a +# shared PVC for deploymentKind=Deployment or per-replica PVCs via +# volumeClaimTemplates for deploymentKind=StatefulSet (see +# recipe-persistent-cache.yaml for a worked example). Switching the # lmcache-server to actually spill KV to that volume — a disk-backed storage # device — is a separate follow-up; until then the volume is mounted but the -# server still keeps KV in memory. A persistent backend must be single-replica -# (a ReadWriteOnce PVC can't be multi-attached). +# server still keeps KV in memory. # # Override the lmcache-server image (defaults to lmcache/standalone:latest) # or the server command via backendConfig until first-class spec fields land. diff --git a/config/samples/recipe-persistent-cache.yaml b/config/samples/recipe-persistent-cache.yaml index aba641ce..c043a70c 100644 --- a/config/samples/recipe-persistent-cache.yaml +++ b/config/samples/recipe-persistent-cache.yaml @@ -15,10 +15,11 @@ # * Removing spec.storage.pvc later LEAVES the PVC in place (adopt-and-keep) — # destroying persistent storage on a spec edit would be data loss. To # reclaim it, run `kubectl delete cachebackend persistent-cache`. -# * SINGLE-REPLICA ONLY: a ReadWriteOnce PVC cannot be multi-attached, so keep +# * deploymentKind: Deployment uses one shared ReadWriteOnce PVC, so keep # replicas: 1 and do not set an autoscaling ceiling > 1, or the backend is # surfaced Ready=False / InvalidStorageConfiguration and not provisioned. -# Per-replica PVCs via a StatefulSet are a separate ticket. +# Use deploymentKind: StatefulSet for multi-replica persistent backends; +# the controller then renders volumeClaimTemplates for per-replica PVCs. # # NOT YET KV-PERSISTENT: provisioning + mounting the PVC is wired, but the # lmcache-server still runs its in-memory storage device, so KV does NOT yet @@ -45,7 +46,7 @@ metadata: spec: type: LMCache deploymentKind: Deployment - # Single replica is required for a PVC-backed (ReadWriteOnce) backend. + # Single replica is required for a Deployment-backed ReadWriteOnce PVC. replicas: 1 integration: engine: vllm diff --git a/docs/design/cachebackend-api.md b/docs/design/cachebackend-api.md index 7ea28247..40610c03 100644 --- a/docs/design/cachebackend-api.md +++ b/docs/design/cachebackend-api.md @@ -25,8 +25,8 @@ The `v1alpha1` contract is pre-launch and explicitly unstable (see the carve-out | `type` | string | Backend implementation identifier. Defaults to `LMCache`. Known constants are `LMCache`, `SGLangHiCache`, `AIBrix`, `Mooncake`, `NIXL`, and `External`; validation intentionally stays open for v1alpha1 compatibility. | | `deploymentKind` | enum | Managed workload kind: `Deployment` or `StatefulSet`. Defaults to `Deployment`. | | `replicas` | integer | Desired managed backend replicas. Defaults to `1`. Minimum `0`. See [Defaulting](#defaulting-mutating) for the interaction with `spec.autoscaling.minReplicas` (first-apply-only). | -| `storage.pvc.size` | quantity | Requested PVC capacity when persistent storage is used. Required only when `storage.pvc` is present. Provisions a PVC mounted into the cache-server pod (see [Persistent storage](#persistent-storage)). Mutable upward only — the controller patches the request up (the StorageClass expands the volume if it allows expansion); a smaller value is ignored (Kubernetes does not support shrinking). | -| `storage.pvc.storageClassName` | string | Optional StorageClass for the PVC. Omit (null) for the cluster default; set `""` to explicitly opt out of the default (static / no-provisioner binding) — the controller preserves null-vs-`""`. Immutable after creation — Kubernetes rejects StorageClass changes, so a later edit is ignored (the controller logs and keeps the existing class). See [Persistent storage](#persistent-storage). | +| `storage.pvc.size` | quantity | Requested PVC capacity when persistent storage is used. Required only when `storage.pvc` is present. Provisions storage mounted into the cache-server pod (see [Persistent storage](#persistent-storage)). For Deployment-backed shared PVCs, mutable upward only — the controller patches the request up (the StorageClass expands the volume if it allows expansion); a smaller value is ignored (Kubernetes does not support shrinking). For StatefulSet-backed per-replica PVCs, size changes alter `volumeClaimTemplates` and are surfaced as `Ready=False/ImmutableStatefulSetStorage` rather than applied in place. | +| `storage.pvc.storageClassName` | string | Optional StorageClass for the PVC. Omit (null) for the cluster default; set `""` to explicitly opt out of the default (static / no-provisioner binding) — the controller preserves null-vs-`""`. On Deployment-backed shared PVCs, immutable after creation — Kubernetes rejects StorageClass changes, so a later edit is ignored (the controller logs and keeps the existing class). On StatefulSet-backed per-replica PVCs, changing the class changes the desired `volumeClaimTemplates` shape and is surfaced as `Ready=False/ImmutableStatefulSetStorage`. See [Persistent storage](#persistent-storage). | | `autoscaling.minReplicas` | integer | Lower bound for HPA replica count. Auto-defaulted to `spec.replicas` on FIRST APPLY ONLY by the admission defaulter when `spec.autoscaling` is set and `minReplicas` is left unset (see [Defaulting](#defaulting-mutating) for the first-apply-only semantics); subsequent edits to `spec.replicas` do NOT move this floor. Minimum `1`. | | `autoscaling.maxReplicas` | integer | Upper bound for HPA replica count. Required when `autoscaling` is set. Minimum `1`. Cross-field validation: `minReplicas <= maxReplicas`. | | `autoscaling.targetCPUUtilizationPercent` | integer | Target average per-pod CPU utilization for the HPA. Defaults to `80` when unset. Range `[1, 100]`. | @@ -131,15 +131,17 @@ The auto-attach itself is opt-in: the controller's `--kvevent-subscriber-image` ### Persistent storage -Setting `spec.storage.pvc` on a **managed Deployment backend** (`deploymentKind: Deployment`, the default) makes the controller provision a `PersistentVolumeClaim` and mount it into the cache-server pod. (A `StatefulSet` backend is routed to the unmanaged path today — see [`deploymentKind`](#spec) — so it provisions nothing, storage included; per-replica PVCs via `volumeClaimTemplates` are a follow-up. An `External` backend is operator-managed and provisions no workload at all, so `spec.storage.pvc` is a no-op there — persistence is configured on the pre-existing cache directly.) The seam is the runtime adapter: `ResolveCacheServer` returns a `ResolvedCacheServer` whose `DataVolume` declares the volume name + in-container mount path where the backend keeps persistent data (the adapter is the only layer that knows where its data lives). The controller, seeing the adapter wants persistent data *and* `spec.storage.pvc` set, provisions the PVC and merges the volume + mount into the rendered pod. Future backends (SGLang HiCache, Mooncake) plug in by declaring a different `DataVolume` — the controller stays generic. +Setting `spec.storage.pvc` on a **managed Deployment backend** (`deploymentKind: Deployment`, the default) makes the controller provision a single `PersistentVolumeClaim` and mount it into the cache-server pod. Setting it on a **managed StatefulSet backend** makes the controller render a `volumeClaimTemplates` entry instead, so each replica gets its own ReadWriteOnce PVC. An `External` backend is operator-managed and provisions no workload at all, so `spec.storage.pvc` is a no-op there — persistence is configured on the pre-existing cache directly. The seam is the runtime adapter: `ResolveCacheServer` returns a `ResolvedCacheServer` whose `DataVolume` declares the volume name + in-container mount path where the backend keeps persistent data (the adapter is the only layer that knows where its data lives). The controller, seeing the adapter wants persistent data *and* `spec.storage.pvc` set, provisions the storage for the selected workload kind and merges the mount into the rendered pod. Future backends (SGLang HiCache, Mooncake) plug in by declaring a different `DataVolume` — the controller stays generic. Behavior: -- **Provisioning.** The PVC is named `-data`, `ReadWriteOnce`, sized from `spec.storage.pvc.size`, with `spec.storage.pvc.storageClassName` when set (else the cluster default StorageClass). It is **owner-referenced** to the CacheBackend, so it is garbage-collected when — and only when — the CacheBackend is deleted. The controller `Owns` PVCs, so a bind/unbind re-triggers a reconcile. -- **Mutable fields.** `size` is patchable upward (the controller raises the request; the StorageClass expands the volume if it allows expansion). A smaller `size` is ignored — Kubernetes does not support shrinking. `storageClassName` is immutable; a change request is a no-op + debug log (Kubernetes rejects StorageClass edits). -- **Adopt-and-keep on removal.** Removing `spec.storage.pvc` from the CR does **not** delete the PVC. Destroying persistent storage in response to a spec edit would be irreversible data loss, so the PVC stays (owner-referenced; reclaimed only when the CacheBackend itself is deleted) and the workload reverts to ephemeral on the next rollout. The controller emits a Warning Event (`OrphanedPVCRetained`) so the retention is visible. To reclaim the volume, delete the CacheBackend. -- **Multi-replica is gated, not provisioned.** A single ReadWriteOnce PVC cannot be multi-attached, so a persistent backend with `spec.replicas > 1` (or `spec.autoscaling.maxReplicas > 1`) is surfaced `Ready=False` with reason `InvalidStorageConfiguration` and a Warning Event; neither the PVC nor the workload is provisioned until the operator scales to 1 or drops `spec.storage.pvc`. Admission does **not** reject this (it is a reconcile-time condition) — per-replica persistent storage via a StatefulSet's `volumeClaimTemplates` is a separate follow-up. -- **`status.capacity`** reflects the bound PVC's *actual* `status.capacity[storage]`, not the requested size (see the status table). +- **Deployment provisioning.** The shared PVC is named `-data`, `ReadWriteOnce`, sized from `spec.storage.pvc.size`, with `spec.storage.pvc.storageClassName` when set (else the cluster default StorageClass). It is **owner-referenced** to the CacheBackend, so it is garbage-collected when — and only when — the CacheBackend is deleted. The controller `Owns` PVCs, so a bind/unbind re-triggers a reconcile. +- **StatefulSet provisioning.** The `volumeClaimTemplates` entry is named from the runtime adapter's `DataVolume.volumeName`, sized from `spec.storage.pvc.size`, and carries `storageClassName` when set. Kubernetes then creates one PVC per StatefulSet replica. The pod template mounts that template volume at the adapter-declared data path. The StatefulSet explicitly sets `persistentVolumeClaimRetentionPolicy` to `Retain` on deletion and scale-down so per-replica cache data is not destroyed by a CR edit, scale-down, kind switch, or delete. Operators must reclaim those PVCs deliberately. +- **Mutable fields.** For the Deployment shared PVC, `size` is patchable upward (the controller raises the request; the StorageClass expands the volume if it allows expansion). A smaller `size` is ignored — Kubernetes does not support shrinking. `storageClassName` is immutable; a change request is a no-op + debug log (Kubernetes rejects StorageClass edits). For StatefulSets, `volumeClaimTemplates` are effectively immutable after creation; a storage-shape edit is surfaced as `Ready=False` with reason `ImmutableStatefulSetStorage` and the live StatefulSet is left untouched. Change storage shape by creating a new backend or migrating data deliberately. +- **Adopt-and-keep on removal.** Removing `spec.storage.pvc` from a Deployment-backed CR does **not** delete the shared PVC. Destroying persistent storage in response to a spec edit would be irreversible data loss, so the PVC stays (owner-referenced; reclaimed only when the CacheBackend itself is deleted) and the workload reverts to ephemeral on the next rollout. The controller emits a Warning Event (`OrphanedPVCRetained`) so the retention is visible. To reclaim the volume, delete the CacheBackend. +- **Deployment multi-replica is gated.** A single ReadWriteOnce PVC cannot be multi-attached, so a persistent Deployment backend with `spec.replicas > 1` (or `spec.autoscaling.maxReplicas > 1`) is surfaced `Ready=False` with reason `InvalidStorageConfiguration` and a Warning Event; neither the PVC nor the workload is provisioned until the operator scales to 1, drops `spec.storage.pvc`, or switches to `deploymentKind: StatefulSet`. +- **StatefulSet template drift.** The controller compares desired `volumeClaimTemplates` against the live StatefulSet after normalizing apiserver defaults such as `volumeMode: Filesystem`, so ordinary defaulting does not look like immutable storage drift. Real storage-shape edits still surface as `Ready=False` / `ImmutableStatefulSetStorage`. +- **`status.capacity`** reflects the Deployment shared PVC's *actual* bound `status.capacity[storage]`, not the requested size (see the status table). StatefulSet per-replica PVCs do not have one aggregate bound-PVC capacity today, so this field remains empty for that path. **Scope note — not yet KV-persistent.** Provisioning + mounting the PVC is wired, but the lmcache-server still runs with its in-memory storage device (`defaultLMCacheServerStorage = "cpu"`), so KV does **not** yet survive a pod restart — the mounted volume is attached but not written to. Switching the server to a disk-backed storage device that spills to the mounted directory is a separate follow-up: the supported on-server-disk mechanism depends on the pinned LMCache server version (the legacy `lmcache_server ` standalone server is in-memory only; on-server disk persistence requires the multiprocess-mode server's L2 storage adapter, a different command/port surface), so it is intentionally split out from this version-agnostic provisioning work. @@ -148,30 +150,30 @@ Behavior: | Field | Type | Purpose | |---|---|---| | `endpoint` | string | Observed endpoint clients should use. For external backends this is mirrored from `spec.endpoint`; for managed backends it is populated by the controller that creates the serving resource. | -| `capacity` | string | Human-readable summary of the backend's provisioned capacity. Informational; clients must not parse it. Set to the bound PVC's actual `status.capacity[storage]` when `spec.storage.pvc` is set and the PVC has bound (the real provisioned size, which may exceed the request) — NOT the requested size, which would misreport a pending or rounded-up bind. Empty for an ephemeral backend or while the PVC is still `Pending` (e.g. a `WaitForFirstConsumer` StorageClass binds only once the pod schedules). See [Persistent storage](#persistent-storage). | +| `capacity` | string | Human-readable summary of the backend's provisioned capacity. Informational; clients must not parse it. Set to the Deployment shared PVC's actual `status.capacity[storage]` when `spec.storage.pvc` is set and the PVC has bound (the real provisioned size, which may exceed the request) — NOT the requested size, which would misreport a pending or rounded-up bind. Empty for an ephemeral backend, a StatefulSet backend, or while the Deployment PVC is still `Pending` (e.g. a `WaitForFirstConsumer` StorageClass binds only once the pod schedules). See [Persistent storage](#persistent-storage). | | `matchedEnginePods` | integer | Snapshot count, at the last reconcile, of pods in the CacheBackend's namespace whose labels satisfy `spec.engineSelector`. Pointer in Go so nil ("not yet computed") is distinguishable from an observed `0` ("computed and zero pods match"). Refreshed at reconcile cadence — not a real-time per-pod counter. The per-pod signal is the `InjectedByCacheBackend` Event the `engine-pod-events` controller emits after the mutating Pod webhook stamps a pod (see [Mutating Pod webhook](#mutating-pod-webhook-engine-wiring) for why the event is emitted by a controller and not the webhook). The field stays nil when no claim-capable selector is configured — both when `spec.engineSelector` is absent AND when `spec.engineSelector.matchLabels` is present but empty (the webhook treats an empty match map as no-claim by design, so the count is no-claim too). A CR that previously had a non-empty selector and just lost it gets its prior value cleared back to nil so the printer column does not advertise a stale match. | | `failOpen` | boolean | Observed echo of the effective `spec.integration.failOpen`. Represented as a pointer in Go so an explicit `false` is serialized and operators can read the current mode from status alone. | | `indexParticipation` | object | Per-backend slice of the cluster-wide cache index, projected from the server's `/snapshot` by grouping replicas by owning `CacheBackend`. Populated by the CacheIndex poller (status-only). Object is unset until the poller has observed a successful scrape that names the backend's replicas (see [Index Participation](#index-participation)). | | `firstKVEventObservedAt` | time | Write-once latch: the first time the [KV-event readiness gate](#kv-event-readiness-gate) observed `indexParticipation.lastEventAt` populated. This is the durable "have we EVER seen a KV event" signal — `lastEventAt` itself is a current-view projection the poller legitimately clears when a backend's replicas drain, so reading it alone would let a backend that already passed the gate regress. Set write-once by the controller and never cleared (a monotonic marker). It is inert while the backend is not managed (External / unsupported runtime) and is intentionally left in place there — clearing it would be ineffective anyway, since the preserved poller-owned `lastEventAt` would immediately re-satisfy the gate on a return to the managed path — so a return to managed stays Ready without re-gating, consistent with the "ever observed" contract. | -| `firstAvailableAt` | time | Write-once latch: the first time the managed cache-backend workload was observed `Available`. It is the **stable anchor** for the `firstEventTimeout` clock — used instead of the live Deployment `Available` condition's `LastTransitionTime` precisely because that resets on an availability flap. Anchoring on this monotonic value keeps the elapsed window growing, so once a backend breaches the timeout (`Degraded`/`NoKVEventsObserved`) a later flap cannot bounce it back to `AwaitingFirstKVEvent` — it stays Degraded until an event arrives. Written write-once and never cleared. | -| `observedServerInstance` | string | The controller's **cascade-decision baseline** — a stable identifier for the Ready cache-server pod set the controller last anchored against. NOT a live "current pod set" view: it is intentionally pinned through transient rolling-update midpoints and through no-Ready windows so the cascade does not fire on rollbacks or transient outages. For the current matched-pod inventory operators should consult `status.matchedEnginePods` (engine side) and `kubectl get pod` (cache-server side). Shape: `:` per Ready pod, comma-joined and lex-sorted by pod name; `` sums `pod.status.containerStatuses[].RestartCount` filtered to the cache-server's own containers (the container names declared on the owned Deployment's pod template). Sidecars injected by other admission webhooks (service-mesh proxies, Datadog, etc.) appear in `containerStatuses` but are absent from the template and are intentionally excluded — a sidecar crash-loop must not advance the identifier and roll the engine fleet. An in-place restart of a cache-server container (kubelet respawning a crashed container — OOM with `restartPolicy=Always` reuses pod.UID) DOES advance the identifier and is observable. On a transition that reflects an actual replacement (a prior pod is gone, or a persisting pod's restart-sum advanced — NOT a rolling-update strict-superset midpoint) the reconciler cascade-restarts every engine Deployment that owns pods carrying this backend's `inferencecache.io/injected-by` + matching `injected-by-uid`, by patching `inferencecache.io/cache-server-restart-trigger` onto each Deployment's pod template — the same mechanism `kubectl rollout restart` uses. Rate-limited to once per ~30s per backend. Empty until the first Ready pod; empty→set never cascades (there is no prior server-instance to invalidate, so any engines that connected during the empty window are connecting to the very pod now being baselined). **Strict-superset transitions are persisted as the new baseline ONLY when the owning Deployment is converged** (`spec.replicas == status.readyReplicas == status.updatedReplicas == len(live Ready pods)` AND `observedGeneration >= metadata.generation` — the live-count clause cross-checks the Deployment's reported state against the pod list this reconcile actually saw, so a stale `status.readyReplicas=1` while two pods are mid-rollout cannot fake convergence) — a converged steady-state widening is an operator-driven scale-up, so the added pods must enter the baseline or a later replacement of just an added pod would still look like a strict superset and miss the cascade. Strict-superset transitions where the Deployment is NOT converged are rolling-update midpoints; persisting them would let a rolled-back rollout (new pod briefly Ready, then killed, leaving the original pod alone) look like "the new pod was replaced" and false-cascade. **Stale-while-unavailable**: when no Ready cache-server pod exists at all (Deployment scaled to 0, mid-rollout, image-pull stuck), this field intentionally retains its prior value rather than clearing — clearing would turn the eventual recovery's `""` → `new-uid:0` transition back into a first-observation baseline and silently skip the cascade, defeating the controller's purpose. Inert and cleared on every transition out of the managed path: External backends, unsupported-runtime backends, and the **InvalidStorageConfiguration** gate (a persistent multi-replica spec — `replicas > 1` with `spec.storage.pvc` — that the controller refuses to provision until the operator scales to 1 or drops the PVC; see `reconcileInvalidStorage`). The in-process cascade shadow is wiped alongside the field on each of these paths so a later return to the managed path starts from a clean baseline rather than the prior-period UID. Operator-side recovery for the upstream LMCache `LMServerConnector` EPIPE-on-restart bug — see [LMCache/LMCache#3565](https://github.com/LMCache/LMCache/issues/3565). | +| `firstAvailableAt` | time | Write-once latch: the first time the managed cache-backend workload was observed serving (`Available` for Deployment, `Ready` for StatefulSet). It is the **stable anchor** for the `firstEventTimeout` clock — used instead of a live workload transition time that can reset on an availability flap. Anchoring on this monotonic value keeps the elapsed window growing, so once a backend breaches the timeout (`Degraded`/`NoKVEventsObserved`) a later flap cannot bounce it back to `AwaitingFirstKVEvent` — it stays Degraded until an event arrives. Written write-once and never cleared. | +| `observedServerInstance` | string | Managed-workload cascade-decision baseline. The cache-server restart cascade authenticates cache-server pods through the owned Deployment/ReplicaSet chain or direct StatefulSet ownerRef and writes a stable identifier for the Ready pod set it last anchored against. Shape: `:` per Ready pod, comma-joined and lex-sorted by pod name; `` sums `pod.status.containerStatuses[].RestartCount` filtered to the cache-server's own containers (the container names declared on the owned workload's pod template). On a real replacement, the reconciler cascade-restarts every engine Deployment injected against this backend by patching `inferencecache.io/cache-server-restart-trigger` onto each Deployment's pod template. Rate-limited to once per ~30s per backend. Empty until the first Ready pod; empty→set never cascades. Strict-superset transitions are persisted as the new baseline only when the owning workload is converged. Inert and cleared on transitions out of the managed cascade path: External backends, unsupported-runtime backends, and the `InvalidStorageConfiguration` gate. Operator-side recovery for the upstream LMCache `LMServerConnector` EPIPE-on-restart bug — see [LMCache/LMCache#3565](https://github.com/LMCache/LMCache/issues/3565). | | `observedGeneration` | integer | The `.metadata.generation` last reconciled by the controller. Lets clients tell whether the observed status reflects the current spec. | | `conditions` | array | Kubernetes conditions keyed by `type`. See [Conditions](#conditions). | ### Conditions -Four condition types are published on managed backends (`Ready`, `Degraded`, `Progressing`, `FunctionalProbeOK`); External backends publish `Ready` + `Progressing` only (there is no rollout to degrade and no probe to drive). The semantics differ for managed backends (where the controller renders a Deployment + Service) and for External (where the operator manages the cache out-of-band and the controller only mirrors the endpoint). +Four condition types are published on managed backends (`Ready`, `Degraded`, `Progressing`, `FunctionalProbeOK`); External backends publish `Ready` + `Progressing` only (there is no rollout to degrade and no probe to drive). The semantics differ for managed backends (where the controller renders a Deployment or StatefulSet + Service) and for External (where the operator manages the cache out-of-band and the controller only mirrors the endpoint). **Managed backends**: | Type | Meaning | |---|---| -| `Ready` | True once the backend Deployment has rolled out its current generation, has enough updated + available replicas to serve traffic, **and** — when the [KV-event readiness gate](#kv-event-readiness-gate) applies — at least one KV event has been observed for the backend (reason `KVEventsObserved`), **and** — when the [functional-probe gate](#functional-probe-gate) applies — the most recent probe call succeeded across every stage the backend runs. Workload Available but no event yet is `Ready=False`, reason `AwaitingFirstKVEvent`. Workload Available and KV-event observed but the probe reported a stage failure is `Ready=False` with reason `ProbeIngestFailed` / `ProbeRoutingFailed` / `ProbeT2Failed`. Deployment-level reasons: `BackendReady` (both gates disabled and Available), `RolloutInProgress`, `ScaledToZero`, `ReplicasUnavailable`. The `BackendDegraded` / `BackendRecovered` Events narrate the `ReplicasUnavailable` → `BackendReady` / `KVEventsObserved` transitions. | -| `Degraded` | True when the backend is in a terminal unhealthy state: rolled out but replicas unavailable (reason `ReplicasUnavailable`), or the managed workload is Available but no KV event observed within `firstEventTimeout` (reason `NoKVEventsObserved`). False (`NotDegraded`) otherwise. The functional-probe gate does NOT participate in `Degraded` — a probe failure is reflected only in `Ready` and `FunctionalProbeOK`, leaving `Degraded` reserved for managed-Deployment health (so an operator can tell "the probe says the cache plane is broken" apart from "the workload itself is in a terminal state"). | +| `Ready` | True once the managed workload has observed its current generation, has enough updated + serving replicas (`availableReplicas` for Deployment, `readyReplicas` for StatefulSet), **and** — when the [KV-event readiness gate](#kv-event-readiness-gate) applies — at least one KV event has been observed for the backend (reason `KVEventsObserved`), **and** — when the [functional-probe gate](#functional-probe-gate) applies — the most recent probe call succeeded across every stage the backend runs. Workload serving but no event yet is `Ready=False`, reason `AwaitingFirstKVEvent`. Workload serving and KV-event observed but the probe reported a stage failure is `Ready=False` with reason `ProbeIngestFailed` / `ProbeRoutingFailed` / `ProbeT2Failed`. Workload-level reasons: `BackendReady` (both gates disabled and serving), `RolloutInProgress`, `ScaledToZero`, `ReplicasUnavailable`. Storage-shape reasons: `InvalidStorageConfiguration` for a multi-replica Deployment with one shared ReadWriteOnce PVC, and `ImmutableStatefulSetStorage` when a StatefulSet `volumeClaimTemplates` edit cannot be applied in place. The `BackendDegraded` / `BackendRecovered` Events narrate the `ReplicasUnavailable` → `BackendReady` / `KVEventsObserved` transitions. | +| `Degraded` | True when the backend is in a terminal unhealthy state: rolled out but replicas unavailable (reason `ReplicasUnavailable`), or the managed workload is serving but no KV event observed within `firstEventTimeout` (reason `NoKVEventsObserved`). False (`NotDegraded`) otherwise. The functional-probe gate does NOT participate in `Degraded` — a probe failure is reflected only in `Ready` and `FunctionalProbeOK`, leaving `Degraded` reserved for managed-workload health (so an operator can tell "the probe says the cache plane is broken" apart from "the workload itself is in a terminal state"). | | `Progressing` | True while the controller is still driving the live state toward the desired state (rollout in flight, first apply, awaiting first KV event). False once converged (`Synced`), stuck (`Degraded`), or scaled to zero (`ScaledToZero`). The pair (`Ready=False`, `Progressing=True`) means "still converging"; (`Ready=False`, `Progressing=False`) means "stuck/degraded" (or scaled to zero). | | `FunctionalProbeOK` | The most recent functional-probe outcome. `True/ProbeOK` when every enabled stage (ingest, routing, and — for LMCache — tier-2 put/get) round-tripped; `True/ProbeBypassed` when the operator opted this CR out via the `inferencecache.io/skip-functional-probe: "true"` annotation; `False/ProbeIngestFailed`, `False/ProbeRoutingFailed`, or `False/ProbeT2Failed` when the named stage failed, with the server's diagnostic in `.message`; `Unknown/ProbeError` when the controller could not reach the server's `/probe` endpoint at all (transport error, 5xx) AND no prior stage failure existed. **Sticky-False**: an HTTP error while a `False/Probe*Failed` is already published preserves the prior failure and keeps `Ready` downgraded, so a transient server outage cannot fade a known per-stage failure back to `Unknown` and then to `Ready=True`. See [functional-probe gate](#functional-probe-gate). | -When the desired replica count is owned by an HPA (`spec.autoscaling` set) the controller compares the Ready condition against the HPA-written Deployment `spec.replicas` rather than the user-set `spec.replicas`. +When the desired replica count is owned by an HPA (`spec.autoscaling` set) the controller compares the Ready condition against the HPA-written workload `spec.replicas` rather than the user-set `spec.replicas`. **External backends**: @@ -190,11 +192,11 @@ Reachability of the external endpoint is **not** probed by the controller; trust ### KV-event readiness gate -`Ready` (for managed backends) means "the managed backend is up **and** the cache plane is actually receiving engine state" — not merely "the workload rolled out". The reconciler observes the **managed cache-backend Deployment** it owns (its rollout + `Available` condition); that proves the backend workload is up but says nothing about whether engine pods are attached and their ZMQ KV-event publishers are publishing. An engine can be serving HTTP while its publisher is silent (mis-configured `--kv-events-config`, a ZMQ bind failure, or an in-process publisher crash), or no engine pods may be wired to the backend at all — in either case `LookupRoute` keeps returning `NO_HINT` for that backend's prefixes while the CR claims everything is fine. The gate makes that silent degradation loud. +`Ready` (for managed backends) means "the managed backend is up **and** the cache plane is actually receiving engine state" — not merely "the workload rolled out". The reconciler observes the managed cache-backend workload it owns (Deployment `availableReplicas` or StatefulSet `readyReplicas`); that proves the backend workload is up but says nothing about whether engine pods are attached and their ZMQ KV-event publishers are publishing. An engine can be serving HTTP while its publisher is silent (mis-configured `--kv-events-config`, a ZMQ bind failure, or an in-process publisher crash), or no engine pods may be wired to the backend at all — in either case `LookupRoute` keeps returning `NO_HINT` for that backend's prefixes while the CR claims everything is fine. The gate makes that silent degradation loud. -The signal source is `status.indexParticipation.lastEventAt` (written by the CacheIndex poller from engine-pod reports); the reconciler only reads it. Once the managed cache-backend Deployment is `Available`: +The signal source is `status.indexParticipation.lastEventAt` (written by the CacheIndex poller from engine-pod reports); the reconciler only reads it. Once the managed cache-backend workload is serving (Deployment `Available` or StatefulSet `readyReplicas`): -- **no event yet, within `firstEventTimeout`** → `Ready=False/AwaitingFirstKVEvent`, `Degraded=False`. The timeout clock starts when the managed Deployment first reports `Available`, captured once in `status.firstAvailableAt` (a stable anchor, so a later availability flap does not restart the window). +- **no event yet, within `firstEventTimeout`** → `Ready=False/AwaitingFirstKVEvent`, `Degraded=False`. The timeout clock starts when the managed workload first reports serving, captured once in `status.firstAvailableAt` (a stable anchor, so a later availability flap does not restart the window). - **at least one event ever observed** → `Ready=True/KVEventsObserved`, `Degraded=False`. An event already present on the first reconcile counts — there is no required transition through `AwaitingFirstKVEvent`. "Ever observed" is durable: the first observation is latched into `status.firstKVEventObservedAt`, because the poller's `lastEventAt` is a current-view value it clears when the backend's replicas drain — without the latch a drained-but-healthy backend would wrongly fall back to `AwaitingFirstKVEvent`. The gate is a first-event startup probe, not an ongoing liveness check. - **no event by `firstEventTimeout`** → `Ready=False/NoKVEventsObserved`, `Degraded=True/NoKVEventsObserved`. Once Degraded it stays Degraded until an event arrives, then transitions to Ready. @@ -215,7 +217,7 @@ Composition order on a managed CacheBackend's Reconcile is `managed-readiness - If `FunctionalProbeOK` is already `False/Probe*Failed` (a prior stage failure), **preserve the existing condition AND keep `Ready=False` with the prior failure's reason** (sticky-False). Letting an HTTP error fade a known per-stage failure to `Unknown` (and therefore back to `Ready=True`) would mask a real regression every time the server happens to be transiently unreachable. The False condition is sticky until a successful probe explicitly resolves it. - **rate-limited reconcile** → no probe call. If the existing `FunctionalProbeOK` is already `False`, the Ready downgrade is re-applied so the status patch doesn't silently overwrite the prior failure with the upstream KV gate's `Ready=True`. The reconciler also schedules a requeue at the rate-limit window expiry so a quiet stuck backend re-probes even without external watch events. -The gate is **on by default** when the controller is wired with a `--server-probe-url`. The operator escape hatch is the **annotation `inferencecache.io/skip-functional-probe: "true"`** (alpha soft-rollout knob; annotation not spec field so it can be retired once the gate is trusted): when set, the probe call is skipped entirely and `FunctionalProbeOK=True/ProbeBypassed` is published — the Ready gate does not downgrade. Disabling the controller-side gate entirely is achieved by passing `--server-probe-url=""` to the controller binary; the CacheBackend reconciler then never calls the endpoint and clears any stale `FunctionalProbeOK` condition the next time it processes the CR. `spec.type: External` backends are **always exempt** — there is no managed Deployment for the gate to compose with, and the controller doesn't drive any cache-plane round-trip for an external endpoint. +The gate is **on by default** when the controller is wired with a `--server-probe-url`. The operator escape hatch is the **annotation `inferencecache.io/skip-functional-probe: "true"`** (alpha soft-rollout knob; annotation not spec field so it can be retired once the gate is trusted): when set, the probe call is skipped entirely and `FunctionalProbeOK=True/ProbeBypassed` is published — the Ready gate does not downgrade. Disabling the controller-side gate entirely is achieved by passing `--server-probe-url=""` to the controller binary; the CacheBackend reconciler then never calls the endpoint and clears any stale `FunctionalProbeOK` condition the next time it processes the CR. `spec.type: External` backends are **always exempt** — there is no managed workload for the gate to compose with, and the controller doesn't drive any cache-plane round-trip for an external endpoint. **Operator note.** If a backend is stuck at `Ready=False/Probe*Failed`, read the condition's `.message` — the server populates a stage-specific diagnostic (e.g. "synthesized event not in index — in-process index ingest path is broken"). For `ProbeIngestFailed` the cache-server's ingest path is broken; for `ProbeRoutingFailed` the index routing / key-derivation layer is broken; for `ProbeT2Failed` the configured tier-2 backend rejected the put or returned nothing on the get. `FunctionalProbeOK=Unknown/ProbeError` means the controller could not reach `/probe` at all — check that `--server-probe-url` is reachable from the controller pod (intra-cluster `inference-cache-server:8081` by default), that the projected SA token is mounted (`/var/run/secrets/inferencecache.io/controller-token/token`), and that the audience-bound TokenReview accepts the controller SA. @@ -238,7 +240,7 @@ Only ONE CacheBackend ever claims a given replica — overlapping selectors must ## Contract Notes - Lookup paths fail open by default. `spec.integration.failOpen` defaults to `true` and the engine adapter MUST fall back to local prefill on cache unreachability — the cache is an optimization, never a serving dependency. Operators may opt into fail-closed serving by setting `failOpen: false`, which is loud and visible: the controller emits a Warning `FailClosedEnabled` Event on the `CacheBackend` to make it explicit that the cache has been promoted to a serving dependency. -- The controller emits Events on the `CacheBackend` only on meaningful state changes, never on steady-state reconciles. Condition-transition-keyed Events: `BackendDegraded` (Warning) on entering `Conditions[Degraded]=True` with reason `ReplicasUnavailable` (the KV-event-gate `NoKVEventsObserved` flavor is suppressed — it carries its own event), `BackendRecovered` (Normal) on the transition back to `Ready=True` (similarly suppressed when recovering from `NoKVEventsObserved`, which carries its own `KVEventsObserved` event); the `FailClosedEnabled` / `FailOpenRestored` pair above; the KV-event readiness gate's `AwaitingFirstKVEvent` (Normal), `KVEventsObserved` (Normal), and `NoKVEventsObserved` (Warning); and `InvalidStorageConfiguration` (Warning) on entering the multi-replica-on-RWO-PVC gate (keyed on the `Ready` reason, like the others). One storage Event is NOT a condition transition: `OrphanedPVCRetained` (Warning) is emitted when `spec.storage.pvc` is removed but an owner-referenced PVC is retained (adopt-and-keep); because that runs on the steady-state path, it is fired exactly once per orphaning via an annotation guard on the retained PVC (cleared when the backend re-adopts the PVC) rather than on a condition transition. +- The controller emits Events on the `CacheBackend` only on meaningful state changes, never on steady-state reconciles. Condition-transition-keyed Events: `BackendDegraded` (Warning) on entering `Conditions[Degraded]=True` with reason `ReplicasUnavailable` (the KV-event-gate `NoKVEventsObserved` flavor is suppressed — it carries its own event), `BackendRecovered` (Normal) on the transition back to `Ready=True` (similarly suppressed when recovering from `NoKVEventsObserved`, which carries its own `KVEventsObserved` event); the `FailClosedEnabled` / `FailOpenRestored` pair above; the KV-event readiness gate's `AwaitingFirstKVEvent` (Normal), `KVEventsObserved` (Normal), and `NoKVEventsObserved` (Warning); and `InvalidStorageConfiguration` (Warning) on entering the multi-replica-on-RWO-PVC gate (keyed on the `Ready` reason, like the others). Storage-retention Events are NOT condition transitions: `OrphanedPVCRetained` (Warning) is emitted when `spec.storage.pvc` is removed but an owner-referenced PVC is retained (adopt-and-keep), and `SharedPVCRetained` (Warning) is emitted when a persistent Deployment backend switches to `deploymentKind: StatefulSet` and the prior shared Deployment PVC is retained but no longer mounted because StatefulSet storage uses per-replica `volumeClaimTemplates`. Because these run on the steady-state path, each is fired exactly once per retained PVC via an annotation guard on the PVC (cleared when the backend re-adopts the PVC) rather than on a condition transition. - A `Normal InjectedByCacheBackend` Event is emitted on engine pods the mutating webhook stamps with both `inferencecache.io/injected-by` AND `inferencecache.io/injected-by-uid`, where the UID annotation matches the live CacheBackend's `metadata.uid` at reconcile time. The controller deliberately skips emission when (a) the named CR cannot be looked up (NotFound), (b) the UID annotation is absent (failurePolicy=Ignore forgery shape), or (c) the UID does not match the live CR (forgery or CR was recreated under the same name). Non-NotFound lookup errors surface as reconcile errors so controller-runtime retries with backoff. The Event is recorded by a Pod-watching controller, not by the webhook itself: at mutating-admission time the apiserver hasn't assigned `metadata.uid` to the pod yet, so an event recorded from the webhook would carry `involvedObject.uid=""` and be invisible to describe (which filters events by UID). Routing the emission through a post-create controller is what guarantees the event reaches the user-visible surface. There is no `NoMatchingCacheBackend` Event in this PR; the no-match signal is `status.matchedEnginePods == 0` on the CR (visible in `kubectl get cachebackend`). - Optional nested specs are pointer fields in Go so omitted objects stay absent in JSON and server-side apply does not claim empty nested objects. **`spec.integration` is the deliberate exception** — the defaulting webhook materialises it on admission so the nested schema-level defaults have a parent object to apply to: once the webhook stamps an empty `spec.integration{}`, the apiserver applies the `+kubebuilder:default=` markers on `engine` (`vllm`), `role` (`ReadWrite`), `failOpen` (`true`), and `firstEventTimeout` (`5m`) before persisting the CR. So a CR that came through admission with no integration block carries all four fields populated in etcd — operators reading the persisted CR see the effective defaults explicitly, not as nil-implied semantic defaults. The `IntegrationFailOpen` reader helper still exists (nil spec or nil field ⇒ `true`) as defence-in-depth for any caller that bypasses the apiserver (raw-struct test invocation, partial deserialization). Other optional nested specs (`spec.storage`, `spec.autoscaling`, `spec.template`, `spec.engineSelector`) are NOT materialised — omitted means absent. Webhook-stamped and apiserver-stamped fields are owned by their respective field managers, not the operator's SSA apply, so SSA semantics for operator-set fields are unaffected. @@ -269,6 +271,7 @@ Rejects structurally-broken specs that the reconciler cannot do anything useful | `spec.endpoint` is only valid on `External` | A non-`External` `spec.type` with a non-empty `spec.endpoint`. The field is meaningful only for the External passthrough adapter; for managed types the controller overwrites `status.endpoint` from the live Service it provisions, so a user-supplied `spec.endpoint` would be silently ignored. Whitespace-only values are treated as empty. | | Memory-only backends cannot declare PVC storage | `spec.storage.pvc` set when `spec.type` is in the Phase-1 memory-only set (`AIBrix`, `NIXL`). These backends have no persistent tier; a PVC would never mount. | | PVC size must be positive | `spec.storage.pvc.size` is not a strictly positive quantity (e.g. `0`). The field is required, but `required` does not stop `0`/negative; since the value drives a real PVC request, a non-positive size is rejected at admission rather than failing later as a child-PVC error. | +| Generated PVC names must fit Kubernetes object-name limits | `spec.storage.pvc` is set and the child PVC name the controller/Kubernetes would create would exceed the 253-character object-name limit. Deployment storage is checked as `-data`; StatefulSet storage is checked using the selected runtime adapter's `DataVolume` name and the StatefulSet PVC expansion shape `--`, with the ordinal derived from `spec.replicas` or `spec.autoscaling.maxReplicas`. Rejecting at admission keeps an otherwise schema-valid CR from later creating a stuck StatefulSet whose per-replica PVCs cannot be materialized. | | Cross-namespace endpoint requires opt-in | `spec.endpoint` resolves to a Service in a namespace other than the CacheBackend's, and `spec.allowCrossNamespace` is `false`. Crossing the namespace is a tenancy boundary the operator should acknowledge. Bare hostnames, IPs, and unqualified names pass through (no namespace to compare against). | | `spec.replicas=0` + autoscaling requires explicit `minReplicas` | `spec.replicas=0` with `spec.autoscaling != nil` and `spec.autoscaling.minReplicas == nil`. The defaulter declines to compute `minReplicas` from a 0 replicas value (it would violate the schema's `Minimum=1`), so without this rule the apiserver accepts the CR and the reconciler's HPA fallback silently picks `1` — overriding the operator's "scale to zero" intent with no notification. The rejection tells the operator to either set `minReplicas` explicitly or remove `spec.autoscaling` to scale to zero unconditionally. | | `spec.integration.engineOverrides` cannot touch reserved args/env | An entry in `engineOverrides.args` / `engineOverrides.suppressArgs` matches a leading flag token the adapter declares as `ReservedArgs()`, or an entry in `engineOverrides.env` / `engineOverrides.suppressEnv` matches a name in `ReservedEnv()`. The rejection names both the offending flag/env and the adapter so the operator can fix the spec rather than wait for the engine to crash. The reserved set is per-adapter (the vLLM+LMCache adapter reserves `--kv-transfer-config`, `VLLM_USE_V1`, `LMCACHE_REMOTE_URL`, `INFERENCECACHE_FAIL_OPEN`). | diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index 82b1ea7d..44c88b0f 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -58,12 +58,15 @@ # reconciler's self-RequeueAfter cadence (no CR or owned-workload # event needed) within ~30s, the bound on stale-Matched the # cadence guarantees. -# 8b. PVC-backed storage wire-up (spec.storage.pvc): a single-replica LMCache -# CacheBackend with spec.storage.pvc.size set provisions a PVC -# owner-referenced to the CacheBackend, mounts it into the cache-server -# pod, binds (waiting for the WaitForFirstConsumer pod schedule first), -# and populates status.capacity from the bound size. (Server-side -# disk-backed KV is a separate follow-up; not asserted here.) +# 8b. PVC-backed storage wire-up (spec.storage.pvc): a single-replica +# Deployment LMCache CacheBackend with spec.storage.pvc.size set +# provisions a shared PVC owner-referenced to the CacheBackend, mounts +# it into the cache-server pod, binds (waiting for the +# WaitForFirstConsumer pod schedule first), and populates status.capacity +# from the bound size. A StatefulSet LMCache CacheBackend with +# spec.storage.pvc uses volumeClaimTemplates for per-replica PVCs. +# (Server-side disk-backed KV is a separate follow-up; not asserted +# here.) # 8c. CacheBackend.spec.resources defaults + threading: the CRD-schema # default stamps spec.resources.limits.memory on every admitted # CacheBackend (so the cache-server pod is bounded by the cgroup @@ -1621,6 +1624,110 @@ if [ -z "$capacity" ]; then fi log "persistent storage wired: PVC Bound, status.capacity=$capacity" +# 5. StatefulSet persistent storage uses per-replica volumeClaimTemplates +# instead of the Deployment path's single shared PVC. +cat </dev/null +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: persistent-cache-sts + namespace: $STORAGE_SMOKE_NS + annotations: + inferencecache.io/require-kv-events: "false" +spec: + type: LMCache + deploymentKind: StatefulSet + replicas: 2 + integration: + engine: vllm + backendConfig: + serverImage: $SAMPLE_CACHE_SERVER_IMAGE + storage: + pvc: + size: 1Gi +EOF + +sts_deadline=$(($(date +%s) + 60)) +sts_service="" +while [ "$(date +%s)" -lt "$sts_deadline" ]; do + sts_service="$(kubectl -n "$STORAGE_SMOKE_NS" get statefulset persistent-cache-sts \ + -o jsonpath='{.spec.serviceName}' 2>/dev/null || true)" + [ -n "$sts_service" ] && break + sleep 2 +done +if [ "$sts_service" != "persistent-cache-sts" ]; then + kubectl -n "$STORAGE_SMOKE_NS" get statefulset persistent-cache-sts -o yaml || true + fail "StatefulSet persistent-cache-sts not created with serviceName=persistent-cache-sts (serviceName=$sts_service)" +fi +if kubectl -n "$STORAGE_SMOKE_NS" get deployment persistent-cache-sts >/dev/null 2>&1; then + kubectl -n "$STORAGE_SMOKE_NS" get deployment persistent-cache-sts -o yaml || true + fail "deploymentKind=StatefulSet created an unexpected Deployment persistent-cache-sts" +fi +claim_template="$(kubectl -n "$STORAGE_SMOKE_NS" get statefulset persistent-cache-sts \ + -o jsonpath='{.spec.volumeClaimTemplates[0].metadata.name}' 2>/dev/null || true)" +claim_size="$(kubectl -n "$STORAGE_SMOKE_NS" get statefulset persistent-cache-sts \ + -o jsonpath='{.spec.volumeClaimTemplates[0].spec.resources.requests.storage}' 2>/dev/null || true)" +claim_mount="$(kubectl -n "$STORAGE_SMOKE_NS" get statefulset persistent-cache-sts \ + -o jsonpath='{.spec.template.spec.containers[0].volumeMounts[?(@.name=="cache-data")].mountPath}' 2>/dev/null || true)" +if [ "$claim_template" != "cache-data" ] || [ "$claim_size" != "1Gi" ] || [ -z "$claim_mount" ]; then + kubectl -n "$STORAGE_SMOKE_NS" get statefulset persistent-cache-sts -o yaml || true + fail "StatefulSet storage template/mount mismatch (template=$claim_template size=$claim_size mount=$claim_mount)" +fi +log "StatefulSet storage template wired: claimTemplate=$claim_template size=$claim_size mount=$claim_mount" + +log "waiting up to ${SAMPLE_GATE_TIMEOUT}s for the persistent StatefulSet to roll out" +if ! kubectl -n "$STORAGE_SMOKE_NS" rollout status statefulset/persistent-cache-sts \ + --timeout="${SAMPLE_GATE_TIMEOUT}s" >/dev/null 2>&1; then + kubectl -n "$STORAGE_SMOKE_NS" get statefulset persistent-cache-sts -o yaml || true + kubectl -n "$STORAGE_SMOKE_NS" get pods -l app.kubernetes.io/instance=persistent-cache-sts -o wide || true + kubectl -n "$STORAGE_SMOKE_NS" get pvc -o wide || true + fail "persistent StatefulSet did not roll out within ${SAMPLE_GATE_TIMEOUT}s" +fi +sts_pvc_count="$(kubectl -n "$STORAGE_SMOKE_NS" get pvc \ + -l app.kubernetes.io/instance=persistent-cache-sts \ + --no-headers 2>/dev/null | wc -l | tr -d ' ')" +if [ "$sts_pvc_count" != "2" ]; then + kubectl -n "$STORAGE_SMOKE_NS" get pvc -o wide || true + fail "StatefulSet persistent-cache-sts PVC count=$sts_pvc_count, want 2 per-replica PVCs" +fi +log "StatefulSet persistent storage wired: per-replica PVC count=$sts_pvc_count" + +# Force a follow-up reconcile after the StatefulSet exists so the controller +# re-compares the live apiserver-defaulted volumeClaimTemplates. A false +# immutable-drift detection here would clear status.endpoint and Ready. +sts_reconcile_marker="$(date +%s)" +kubectl -n "$STORAGE_SMOKE_NS" annotate cb persistent-cache-sts \ + inferencecache.io/smoke-reconcile-ts="$sts_reconcile_marker" --overwrite >/dev/null +sleep 8 +sts_expected_endpoint="persistent-cache-sts.${STORAGE_SMOKE_NS}.svc.cluster.local:65432" +sts_status_deadline=$(($(date +%s) + 60)) +sts_status_endpoint="" +sts_ready_status="" +sts_ready_reason="" +while [ "$(date +%s)" -lt "$sts_status_deadline" ]; do + sts_status_endpoint="$(kubectl -n "$STORAGE_SMOKE_NS" get cb persistent-cache-sts \ + -o jsonpath='{.status.endpoint}' 2>/dev/null || true)" + sts_ready_status="$(kubectl -n "$STORAGE_SMOKE_NS" get cb persistent-cache-sts \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" + sts_ready_reason="$(kubectl -n "$STORAGE_SMOKE_NS" get cb persistent-cache-sts \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].reason}' 2>/dev/null || true)" + if [ "$sts_status_endpoint" = "$sts_expected_endpoint" ] && + [ "$sts_ready_status" = "True" ] && + [ "$sts_ready_reason" = "BackendReady" ]; then + break + fi + sleep 2 +done +if [ "$sts_status_endpoint" != "$sts_expected_endpoint" ] || + [ "$sts_ready_status" != "True" ] || + [ "$sts_ready_reason" != "BackendReady" ]; then + kubectl -n "$STORAGE_SMOKE_NS" get cb persistent-cache-sts -o yaml || true + kubectl -n "$STORAGE_SMOKE_NS" get statefulset persistent-cache-sts -o yaml || true + kubectl -n "$STORAGE_SMOKE_NS" get pvc -o wide || true + fail "StatefulSet CacheBackend status after follow-up reconcile endpoint=$sts_status_endpoint Ready=$sts_ready_status/$sts_ready_reason, want $sts_expected_endpoint True/BackendReady" +fi +log "StatefulSet CacheBackend status held after follow-up reconcile: endpoint=$sts_status_endpoint Ready=$sts_ready_status/$sts_ready_reason" + # Sample cleanup: drop the dedicated namespace (best-effort). kubectl delete namespace "$STORAGE_SMOKE_NS" \ --wait=false --ignore-not-found=true >/dev/null 2>&1 || true diff --git a/internal/controller/cachebackend_autoscaling_test.go b/internal/controller/cachebackend_autoscaling_test.go index ed14178a..4c27b9f8 100644 --- a/internal/controller/cachebackend_autoscaling_test.go +++ b/internal/controller/cachebackend_autoscaling_test.go @@ -63,6 +63,21 @@ func TestReconcileHPACreated(t *testing.T) { } } +func TestReconcileHPAStatefulSetTarget(t *testing.T) { + scheme := newScheme(t) + cb := autoscalingBackend("cache", "ns1", 2, 5, ptrInt32(60)) + cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + r := newReconciler(scheme, cb) + + reconcile(t, r, "cache", "ns1") + + getStatefulSet(t, r, "cache", "ns1") + hpa := getHPA(t, r, "cache", "ns1") + if hpa.Spec.ScaleTargetRef.Kind != "StatefulSet" || hpa.Spec.ScaleTargetRef.Name != "cache" { + t.Fatalf("HPA target = %+v, want StatefulSet/cache", hpa.Spec.ScaleTargetRef) + } +} + func TestReconcileHPADefaults(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") diff --git a/internal/controller/cachebackend_controller.go b/internal/controller/cachebackend_controller.go index 8553169e..f70baef1 100644 --- a/internal/controller/cachebackend_controller.go +++ b/internal/controller/cachebackend_controller.go @@ -107,7 +107,7 @@ const ( eventReasonFailClosedEnabled = "FailClosedEnabled" eventReasonFailOpenRestored = "FailOpenRestored" // eventReasonInvalidStorageConfiguration is the Warning fired when a - // persistent (spec.storage.pvc) backend asks for more than one replica — + // persistent Deployment backend asks for more than one replica — // a single ReadWriteOnce PVC cannot be multi-attached. Mirrors the // Ready=False reason so a transition into the gated state is narrated once. eventReasonInvalidStorageConfiguration = conditionReasonInvalidStorageConfiguration @@ -116,6 +116,11 @@ const ( // the operator knows the storage was kept, not silently dropped. Event // aggregation collapses the per-reconcile repeats into one event. eventReasonOrphanedPVCRetained = "OrphanedPVCRetained" + // eventReasonSharedPVCRetained is the Warning fired when a persistent + // Deployment backend switches to deploymentKind=StatefulSet and the old + // shared PVC is retained but no longer mounted. StatefulSet storage uses + // per-replica volumeClaimTemplates instead. + eventReasonSharedPVCRetained = "SharedPVCRetained" ) // Condition reasons published on a CacheBackend's Ready condition. Stable @@ -238,6 +243,8 @@ func (r *CacheBackendReconciler) matchedEnginePodsRequeueInterval() time.Duratio // +kubebuilder:rbac:groups=inferencecache.io,resources=cachebackends/finalizers,verbs=update // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get // +kubebuilder:rbac:groups=apps,resources=replicasets,verbs=get // +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch @@ -249,9 +256,9 @@ func (r *CacheBackendReconciler) matchedEnginePodsRequeueInterval() time.Duratio // Reconcile drives a CacheBackend toward its desired state. External backends // only mirror their configured endpoint to status; managed backends (LMCache // in Phase 1) ask the registered runtime adapter for the cache-server pod -// spec + service spec, wrap them into a Deployment + Service the controller -// owns, optionally reconcile an HPA from spec.autoscaling, and publish the -// resolved endpoint. +// spec + service spec, wrap them into a Deployment or StatefulSet plus a +// Service the controller owns, optionally reconcile an HPA from +// spec.autoscaling, and publish the resolved endpoint. // // On every reconcile — including ones that return an apply error — transitions // in the observed Ready condition (entering/leaving Ready=False/ @@ -336,20 +343,22 @@ func (r *CacheBackendReconciler) Reconcile(ctx context.Context, req ctrl.Request } // dispatch routes a CacheBackend to the right reconcile path. External backends -// only mirror their configured endpoint to status; unsupported / deferred kinds -// shed any previously managed workload; LMCache (Phase 1) templates a -// Deployment + Service. +// only mirror their configured endpoint to status; unsupported runtime/backend +// pairs shed any previously managed workload; managed backends template a +// Deployment or StatefulSet + Service. func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logger, backend *cachev1alpha1.CacheBackend) (ctrl.Result, error) { // Adopt-and-keep visibility: if spec.storage.pvc is absent but we still own a // data PVC from a prior persistent generation, warn (once). Done here, before // the type/kind routing, so the warning fires regardless of how a now - // storage-less backend is dispatched — External, StatefulSet/unmanaged, an - // unsupported (runtime, type) pair, or managed-ephemeral all otherwise bypass - // the per-path provisioning logic. The PVC itself is retained by its owner + // storage-less backend is dispatched — External, an unsupported (runtime, + // type) pair, or managed-ephemeral all otherwise bypass the Deployment PVC + // provisioning logic. The PVC itself is retained by its owner // reference (reclaimed only on CR delete); warnRetainedPVC no-ops when no // owned PVC exists, so this is a cheap cached Get on the common path. if backend.Spec.Storage == nil || backend.Spec.Storage.PVC == nil { r.warnRetainedPVC(ctx, backend) + } else if desiredDeploymentKind(backend) == cachev1alpha1.CacheBackendDeploymentKindStatefulSet { + r.warnRetainedSharedPVCForStatefulSet(ctx, backend) } if backend.Spec.Type == cachev1alpha1.CacheBackendTypeExternal { @@ -360,14 +369,6 @@ func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logge return ctrl.Result{}, r.reconcileExternal(ctx, backend) } - // StatefulSet (per-replica PVCs via volumeClaimTemplates) is a later - // module. Phase 1 manages a Deployment only. - if backend.Spec.DeploymentKind == cachev1alpha1.CacheBackendDeploymentKindStatefulSet { - logger.V(1).Info("StatefulSet deploymentKind not yet supported; skipping", - "namespace", backend.Namespace, "name", backend.Name) - return ctrl.Result{}, r.reconcileUnmanaged(ctx, backend) - } - registry := r.Registry if registry == nil { registry = adapterruntime.DefaultRegistry() @@ -453,7 +454,7 @@ func (r *CacheBackendReconciler) reconcileExternal(ctx context.Context, backend // Clear the cache-server-instance latch — External backends // have no controller-managed cache-server pods, and // cleanupOwnedWorkload above has just deleted any prior - // managed Deployment. Leaving the latch set would expose a + // managed workload. Leaving the latch set would expose a // stale UID to operators. backend.Status.ObservedServerInstance = "" backend.Status.ObservedGeneration = backend.Generation @@ -537,7 +538,7 @@ func (r *CacheBackendReconciler) reconcileUnmanaged(ctx context.Context, backend backend.Status.Endpoint = "" backend.Status.Capacity = "" // Clear the cache-server-instance latch — cleanupOwnedWorkload - // above has just deleted any prior managed Deployment and we + // above has just deleted any prior managed workload and we // no longer provision one, so a retained UID would advertise // a stale identifier. backend.Status.ObservedServerInstance = "" @@ -550,8 +551,8 @@ func (r *CacheBackendReconciler) reconcileUnmanaged(ctx context.Context, backend } // reconcileManaged renders the cache-server PodSpec + Service via the runtime -// adapter, wraps them into a Deployment + Service owned by the CR, and -// publishes the resolved endpoint to status. +// adapter, wraps them into the selected managed workload + Service owned by the +// CR, and publishes the resolved endpoint to status. // // Apply drives desired state; status reflects observed state. The two must not // block each other: if a desired-state write fails (e.g. a transient API-server @@ -575,80 +576,154 @@ func (r *CacheBackendReconciler) reconcileManaged(ctx context.Context, logger lo podSpec := resolved.PodSpec svcSpec := resolved.Service - // Persistent storage: provision a PVC and mount it when the adapter declares - // a data volume AND the operator asked for PVC-backed storage. The adapter - // owns the volume name + mount path (it knows where its data lives); the - // controller owns PVC identity, owner-ref GC, and the mutable-field reconcile. + desiredKind := desiredDeploymentKind(backend) + + // Persistent storage: when the adapter declares a data volume AND the + // operator asked for PVC-backed storage, wire that volume according to the + // workload kind. Deployments get the legacy single shared PVC. StatefulSets + // get per-replica PVCs via volumeClaimTemplates, using the adapter-declared + // volume name + mount path. capacity := "" + var volumeClaimTemplates []corev1.PersistentVolumeClaim if resolved.DataVolume != nil && backend.Spec.Storage != nil && backend.Spec.Storage.PVC != nil { - // Multi-replica gate: a single ReadWriteOnce PVC cannot be multi-attached. - // Refuse to provision the workload until the operator scales to 1 or - // drops spec.storage.pvc. The pod is not created in this state. - if multiReplicaRequested(backend) { - return r.reconcileInvalidStorage(ctx, backend) - } - pvc, pvcErr := r.applyPVC(ctx, logger, backend) - if pvcErr != nil { - return ctrl.Result{}, pvcErr + if desiredKind == cachev1alpha1.CacheBackendDeploymentKindStatefulSet { + mountDataVolumeClaimTemplate(podSpec, resolved.DataVolume) + volumeClaimTemplates = []corev1.PersistentVolumeClaim{ + buildVolumeClaimTemplate(backend, resolved.DataVolume), + } + } else { + // Multi-replica gate: a single ReadWriteOnce PVC cannot be multi-attached. + // Refuse to provision the Deployment until the operator scales to 1 or + // drops spec.storage.pvc. StatefulSet storage uses per-replica PVCs above. + if multiReplicaRequested(backend) { + return r.reconcileInvalidStorage(ctx, backend) + } + pvc, pvcErr := r.applyPVC(ctx, logger, backend) + if pvcErr != nil { + return ctrl.Result{}, pvcErr + } + // Merge the PVC-backed volume + mount into the rendered cache-server pod + // before it is wrapped into the Deployment. The apply path already + // propagates podSpec.Volumes + container VolumeMounts to the live + // Deployment (see reconcileManagedPodSpec), so no extra apply step is + // needed. + mountDataVolume(podSpec, resolved.DataVolume, pvc.Name) + capacity = boundCapacity(pvc) } - // Merge the PVC-backed volume + mount into the rendered cache-server pod - // before it is wrapped into the Deployment. The apply path already - // propagates podSpec.Volumes + container VolumeMounts to the live - // Deployment (see reconcileManagedPodSpec), so no extra apply step is - // needed. - mountDataVolume(podSpec, resolved.DataVolume, pvc.Name) - capacity = boundCapacity(pvc) } // (When spec.storage.pvc is absent the rendered pod carries no PVC volume, so // the workload reverts to ephemeral on the next rollout. The adopt-and-keep // Warning for a removed spec.storage.pvc is emitted in dispatch — it covers // every routing outcome, not just this managed path.) - dep := r.buildDeployment(backend, podSpec) svc := r.buildService(backend, svcSpec) - // Skip Service + HPA when applyDeployment failed. The HPA targets the - // Deployment by name, so running it after a foreign-ownership failure - // could scale another controller's workload; the Service is independent - // but pointless to expose alongside a Deployment we don't own. Status - // observation still runs below (it has its own ownership guards) so the - // CR isn't held hostage to apply churn. - applyErr := r.applyDeployment(ctx, backend, dep) - if applyErr == nil { - if svcErr := r.applyService(ctx, backend, svc); svcErr != nil { - applyErr = svcErr + var ( + applyErr error + replacementUnavailable bool + workload managedWorkloadObservation + ) + switch desiredKind { + case cachev1alpha1.CacheBackendDeploymentKindStatefulSet: + if err := r.deleteOwnedDeployment(ctx, backend); err != nil { + return ctrl.Result{}, err + } + sts := r.buildStatefulSet(backend, podSpec, svc.Name, volumeClaimTemplates) + storageDrift, err := r.statefulSetVolumeClaimTemplatesDrift(ctx, backend, sts) + if err != nil { + return ctrl.Result{}, err } - if hpaErr := r.reconcileHPA(ctx, backend, dep); hpaErr != nil && applyErr == nil { - applyErr = hpaErr + if storageDrift { + result, driftErr := r.reconcileImmutableStatefulSetStorage(ctx, backend) + if hpaErr := r.reconcileHPA(ctx, backend, workloadRefFromStatefulSet(sts)); hpaErr != nil && driftErr == nil { + driftErr = hpaErr + } + cascadeWait := r.reconcileServerInstance(ctx, logger, backend) + if cascadeWait > 0 && (result.RequeueAfter == 0 || cascadeWait < result.RequeueAfter) { + result.RequeueAfter = cascadeWait + } + pollCadence := r.minServerRestartCascadeInterval() + if result.RequeueAfter == 0 || pollCadence < result.RequeueAfter { + result.RequeueAfter = pollCadence + } + return result, driftErr + } + applyErr = r.applyStatefulSet(ctx, backend, sts) + if applyErr == nil { + if svcErr := r.applyService(ctx, backend, svc); svcErr != nil { + applyErr = svcErr + } + if hpaErr := r.reconcileHPA(ctx, backend, workloadRefFromStatefulSet(sts)); hpaErr != nil && applyErr == nil { + applyErr = hpaErr + } } - } - var live appsv1.Deployment - if err := r.Get(ctx, client.ObjectKeyFromObject(dep), &live); err != nil { - if apierrors.IsNotFound(err) && applyErr != nil { - // Apply failed before creating the Deployment, so there is no - // observed state to publish — surface the apply error to requeue. - return ctrl.Result{}, applyErr + var live appsv1.StatefulSet + if err := r.Get(ctx, client.ObjectKeyFromObject(sts), &live); err != nil { + if apierrors.IsNotFound(err) && applyErr != nil { + replacementUnavailable = true + workload = observationFromStatefulSet(sts) + } else { + return ctrl.Result{}, fmt.Errorf("get statefulset %s/%s: %w", sts.Namespace, sts.Name, err) + } + } else if !metav1.IsControlledBy(&live, backend) { + if applyErr != nil { + replacementUnavailable = true + workload = observationFromStatefulSet(sts) + } else { + return ctrl.Result{}, fmt.Errorf("statefulset %s/%s lost controller reference after apply", sts.Namespace, sts.Name) + } + } else { + workload = observationFromStatefulSet(&live) } - // Either a transient Get error, or NotFound after a successful apply - // (deleted out-of-band between apply and Get). Both must requeue; - // silently reporting success here would freeze status at a stale - // snapshot. - return ctrl.Result{}, fmt.Errorf("get deployment %s/%s: %w", dep.Namespace, dep.Name, err) - } - // Never publish status derived from a foreign workload. The common case - // is an AlreadyOwned collision during apply (applyErr is set; surface - // it). The race case is that apply succeeded but the live Deployment's - // controller ref was changed out-of-band between Update and this Get — - // applyErr is nil, but we no longer own the object. Returning nil there - // would silently mark the reconcile successful AND lose the owned-object - // watch (no future event would re-trigger), so synthesize an explicit - // error to requeue. - if !metav1.IsControlledBy(&live, backend) { - if applyErr != nil { - return ctrl.Result{}, applyErr + default: + if err := r.deleteOwnedStatefulSet(ctx, backend); err != nil { + return ctrl.Result{}, err + } + dep := r.buildDeployment(backend, podSpec) + applyErr = r.applyDeployment(ctx, backend, dep) + if applyErr == nil { + if svcErr := r.applyService(ctx, backend, svc); svcErr != nil { + applyErr = svcErr + } + if hpaErr := r.reconcileHPA(ctx, backend, workloadRefFromDeployment(dep)); hpaErr != nil && applyErr == nil { + applyErr = hpaErr + } + } + + var live appsv1.Deployment + if err := r.Get(ctx, client.ObjectKeyFromObject(dep), &live); err != nil { + if apierrors.IsNotFound(err) && applyErr != nil { + // Apply failed before creating the Deployment. Continue to the + // status pass with an empty endpoint so a kind-switch failure + // does not keep advertising the workload deleted above. + replacementUnavailable = true + workload = observationFromDeployment(dep) + } else { + // Either a transient Get error, or NotFound after a successful apply + // (deleted out-of-band between apply and Get). Both must requeue; + // silently reporting success here would freeze status at a stale + // snapshot. + return ctrl.Result{}, fmt.Errorf("get deployment %s/%s: %w", dep.Namespace, dep.Name, err) + } + } else if !metav1.IsControlledBy(&live, backend) { + // Never publish status derived from a foreign workload. The common + // case is an AlreadyOwned collision during apply (applyErr is set; + // surface it). The race case is that apply succeeded but the live + // Deployment's controller ref was changed out-of-band between Update + // and this Get — applyErr is nil, but we no longer own the object. + // Returning nil there would silently mark the reconcile successful + // AND lose the owned-object watch (no future event would re-trigger), + // so synthesize an explicit error to requeue. + if applyErr != nil { + replacementUnavailable = true + workload = observationFromDeployment(dep) + } else { + return ctrl.Result{}, fmt.Errorf("deployment %s/%s lost controller reference after apply", dep.Namespace, dep.Name) + } + } else { + workload = observationFromDeployment(&live) } - return ctrl.Result{}, fmt.Errorf("deployment %s/%s lost controller reference after apply", dep.Namespace, dep.Name) } // Endpoint is derived from the *live* Service, not the desired one: if @@ -658,15 +733,19 @@ func (r *CacheBackendReconciler) reconcileManaged(ctx context.Context, logger lo // when the Service hasn't materialized or is owned by someone else. var liveSvc corev1.Service endpoint := "" - if err := r.Get(ctx, client.ObjectKeyFromObject(svc), &liveSvc); err != nil { - if !apierrors.IsNotFound(err) { - return ctrl.Result{}, fmt.Errorf("get service %s/%s: %w", svc.Namespace, svc.Name, err) + if !replacementUnavailable { + if err := r.Get(ctx, client.ObjectKeyFromObject(svc), &liveSvc); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("get service %s/%s: %w", svc.Namespace, svc.Name, err) + } + } else if metav1.IsControlledBy(&liveSvc, backend) { + endpoint = serviceEndpoint(&liveSvc) } - } else if metav1.IsControlledBy(&liveSvc, backend) { - endpoint = serviceEndpoint(&liveSvc) + } else { + capacity = "" } - requeueAfter, statusErr := r.updateManagedStatus(ctx, backend, endpoint, capacity, &live, applyErr == nil) + requeueAfter, statusErr := r.updateManagedStatus(ctx, backend, endpoint, capacity, workload, applyErr == nil) // Do NOT short-circuit on statusErr — the cascade is independent // recovery for stale engine sockets and must not be skipped just // because the unrelated managed-status patch (matchedEnginePods, @@ -696,19 +775,18 @@ func (r *CacheBackendReconciler) reconcileManaged(ctx context.Context, logger lo requeueAfter = cascadeWait } // Schedule an unconditional periodic re-poll of the cache-server - // pod set on managed backends. Reason: an in-place container + // pod set for managed backends. Reason: an in-place container // restart (kubelet respawning a crashed cache-server container - // without bumping pod.UID) does NOT change owned-Deployment status - // counts, and the controller deliberately does not watch Pods + // without bumping pod.UID) does NOT change owned workload readiness + // status, and the controller deliberately does not watch Pods // cluster-wide (see refreshMatchedEnginePods godoc). The - // matched-engine-pods cadence above does not cover this case - // either: when an operator removes spec.engineSelector after - // engines were injected, len(matchedEnginePods)→0 and that - // cadence stops firing, leaving in-place restarts unobservable - // until something unrelated triggers a reconcile. Pinning a - // floor at the rate-limit interval bounds the observation - // latency for in-place restarts at one cadence (cheap: one - // Pod List + one Deployment Get per backend per cadence). + // matched-engine-pods cadence above does not cover this case either: + // when an operator removes spec.engineSelector after engines were + // injected, len(matchedEnginePods)→0 and that cadence stops firing, + // leaving in-place restarts unobservable until something unrelated + // triggers a reconcile. Pinning a floor at the rate-limit interval + // bounds the observation latency for in-place restarts at one cadence + // (cheap: one Pod List + one workload Get per backend per cadence). pollCadence := r.minServerRestartCascadeInterval() if requeueAfter == 0 || pollCadence < requeueAfter { requeueAfter = pollCadence @@ -780,6 +858,48 @@ func (r *CacheBackendReconciler) buildDeployment(backend *cachev1alpha1.CacheBac } } +// buildStatefulSet wraps the adapter-rendered PodSpec into a StatefulSet the +// controller owns. The StatefulSet reuses the same Service/selector identity as +// the Deployment path and carries volumeClaimTemplates when persistent storage +// is requested. +func (r *CacheBackendReconciler) buildStatefulSet(backend *cachev1alpha1.CacheBackend, podSpec *corev1.PodSpec, serviceName string, volumeClaimTemplates []corev1.PersistentVolumeClaim) *appsv1.StatefulSet { + replicas := initialReplicas(backend) + selector := selectorLabels(backend.Name) + podLabels := podTemplateLabels(backend) + + pod := podSpec.DeepCopy() + applyPodOverrides(pod, backend.Spec.Template) + + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: backend.Name, + Namespace: backend.Namespace, + Labels: podLabels, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + ServiceName: serviceName, + Selector: &metav1.LabelSelector{MatchLabels: selector}, + PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: podLabels}, + Spec: *pod, + }, + VolumeClaimTemplates: volumeClaimTemplates, + }, + } +} + +func desiredDeploymentKind(backend *cachev1alpha1.CacheBackend) cachev1alpha1.CacheBackendDeploymentKind { + if backend.Spec.DeploymentKind == cachev1alpha1.CacheBackendDeploymentKindStatefulSet { + return cachev1alpha1.CacheBackendDeploymentKindStatefulSet + } + return cachev1alpha1.CacheBackendDeploymentKindDeployment +} + // buildService wraps the adapter-rendered Service spec into a Service the // controller owns: ObjectMeta + Selector come from the CacheBackend identity. // Adapter-provided fields (Spec.Type, Spec.Ports) are preserved as-is. @@ -915,6 +1035,87 @@ func (r *CacheBackendReconciler) applyDeployment(ctx context.Context, backend *c return nil } +// applyStatefulSet creates or updates the backend StatefulSet idempotently, +// owned by the CR. On update it reconciles mutable fields only: replicas, +// serviceName, PVC retention policy, and pod template. volumeClaimTemplates +// and selector are effectively immutable in Kubernetes, so they are established +// on create and left intact afterward. +func (r *CacheBackendReconciler) applyStatefulSet(ctx context.Context, backend *cachev1alpha1.CacheBackend, desired *appsv1.StatefulSet) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + sts := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: desired.Name, Namespace: desired.Namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sts, func() error { + liveReplicas := sts.Spec.Replicas + + sts.Labels = desired.Labels + if sts.ResourceVersion == "" { + sts.Spec = *desired.Spec.DeepCopy() + } else { + sts.Spec.Replicas = desired.Spec.Replicas + sts.Spec.ServiceName = desired.Spec.ServiceName + sts.Spec.PersistentVolumeClaimRetentionPolicy = desired.Spec.PersistentVolumeClaimRetentionPolicy.DeepCopy() + sts.Spec.Template.Labels = desired.Spec.Template.Labels + reconcileManagedPodSpec(&sts.Spec.Template.Spec, &desired.Spec.Template.Spec) + } + if backend.Spec.Autoscaling != nil && liveReplicas != nil { + preserved := *liveReplicas + if floor := autoscalingFloor(backend.Spec.Autoscaling); preserved < floor { + preserved = floor + } + sts.Spec.Replicas = &preserved + } + return controllerutil.SetControllerReference(backend, sts, r.Scheme) + }) + return err + }) + if err != nil { + return fmt.Errorf("apply statefulset %s/%s: %w", desired.Namespace, desired.Name, err) + } + return nil +} + +// statefulSetVolumeClaimTemplatesDrift reports whether an owned StatefulSet +// already exists with immutable volumeClaimTemplates that differ from the +// desired storage shape. It intentionally ignores metadata-only differences: +// labels can drift without changing the per-replica PVC contract, while name +// and spec differences require an operator-led replacement or migration. +func (r *CacheBackendReconciler) statefulSetVolumeClaimTemplatesDrift(ctx context.Context, backend *cachev1alpha1.CacheBackend, desired *appsv1.StatefulSet) (bool, error) { + live := &appsv1.StatefulSet{} + if err := r.Get(ctx, client.ObjectKeyFromObject(desired), live); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("get statefulset %s/%s for storage drift check: %w", desired.Namespace, desired.Name, err) + } + if !metav1.IsControlledBy(live, backend) { + return false, nil + } + return !statefulSetVolumeClaimTemplatesEqual(live.Spec.VolumeClaimTemplates, desired.Spec.VolumeClaimTemplates), nil +} + +func statefulSetVolumeClaimTemplatesEqual(live, desired []corev1.PersistentVolumeClaim) bool { + if len(live) != len(desired) { + return false + } + for i := range live { + if live[i].Name != desired[i].Name { + return false + } + liveSpec := normalizeVolumeClaimTemplateSpec(live[i].Spec) + desiredSpec := normalizeVolumeClaimTemplateSpec(desired[i].Spec) + if !equality.Semantic.DeepEqual(liveSpec, desiredSpec) { + return false + } + } + return true +} + +func normalizeVolumeClaimTemplateSpec(spec corev1.PersistentVolumeClaimSpec) corev1.PersistentVolumeClaimSpec { + if spec.VolumeMode != nil && *spec.VolumeMode == corev1.PersistentVolumeFilesystem { + spec.VolumeMode = nil + } + return spec +} + // autoscalingFloor is the effective minReplicas value for the HPA — the // user's setting, or the default floor when unset. Mirrors the resolution // buildHPA does so the reconciler and the HPA agree on the lower bound. @@ -951,10 +1152,10 @@ func (r *CacheBackendReconciler) applyService(ctx context.Context, backend *cach return nil } -// pvcName is the stable name of the data PVC a persistent backend provisions. +// pvcName is the stable name of the shared data PVC a persistent Deployment +// backend provisions. // Derived from the CacheBackend name so it is deterministic across reconciles -// and unique within the namespace (one shared PVC per backend on the Deployment -// path; per-replica PVCs via StatefulSet volumeClaimTemplates are a follow-up). +// and unique within the namespace. func pvcName(backend *cachev1alpha1.CacheBackend) string { return backend.Name + "-data" } @@ -969,8 +1170,9 @@ func pvcName(backend *cachev1alpha1.CacheBackend) string { // hazard and must not be gated. When autoscaling is unset, spec.replicas is the // ceiling. The ceiling — not the live count — is what matters: the operator's // intent to ever run more than one pod against the shared PVC is the hazard, and -// admission deliberately surfaces this as Ready=False rather than rejecting -// (per-replica PVCs via StatefulSet are a follow-up). +// admission deliberately surfaces this as Ready=False rather than rejecting. +// StatefulSet storage uses per-replica volumeClaimTemplates and bypasses this +// shared-PVC hazard. func multiReplicaRequested(backend *cachev1alpha1.CacheBackend) bool { if backend.Spec.Autoscaling != nil { return backend.Spec.Autoscaling.MaxReplicas > 1 @@ -1049,10 +1251,10 @@ func (r *CacheBackendReconciler) applyPVC(ctx context.Context, logger logr.Logge } } } - // Re-adoption: clear the "already warned" marker (if any) left by a - // prior adopt-and-keep orphaning, so a future spec.storage.pvc - // removal warns again. delete on an absent key is a no-op. - delete(pvc.Annotations, annotationStorageRetainedWarned) + // Re-adoption: clear "already warned" markers (if any) left by prior + // retention modes, so a future removal or kind switch warns again. + // delete on an absent key is a no-op. + clearRetainedPVCWarningAnnotations(pvc.Annotations) return controllerutil.SetControllerReference(backend, pvc, r.Scheme) }) return e @@ -1097,6 +1299,46 @@ func mountDataVolume(podSpec *corev1.PodSpec, dv *adapterruntime.AdapterDataVolu } } +// mountDataVolumeClaimTemplate adds the adapter-declared mount for a +// StatefulSet volumeClaimTemplate. The StatefulSet controller materializes the +// volume from VolumeClaimTemplates, so the pod template must not carry an +// explicit PersistentVolumeClaim volume that would point every replica at one +// shared claim. +func mountDataVolumeClaimTemplate(podSpec *corev1.PodSpec, dv *adapterruntime.AdapterDataVolume) { + volumes := podSpec.Volumes[:0] + for _, volume := range podSpec.Volumes { + if volume.Name != dv.VolumeName { + volumes = append(volumes, volume) + } + } + podSpec.Volumes = volumes + + mount := corev1.VolumeMount{Name: dv.VolumeName, MountPath: dv.MountPath} + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = upsertVolumeMount(podSpec.Containers[i].VolumeMounts, mount) + } +} + +func buildVolumeClaimTemplate(backend *cachev1alpha1.CacheBackend, dv *adapterruntime.AdapterDataVolume) corev1.PersistentVolumeClaim { + pvcSpec := backend.Spec.Storage.PVC + template := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: dv.VolumeName, + Labels: podTemplateLabels(backend), + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: pvcSpec.Size}, + }, + }, + } + if pvcSpec.StorageClassName != nil { + template.Spec.StorageClassName = pvcSpec.StorageClassName + } + return template +} + // upsertVolumeMount replaces a mount with the same Name (idempotent re-render) // or appends it, returning the updated slice. func upsertVolumeMount(mounts []corev1.VolumeMount, m corev1.VolumeMount) []corev1.VolumeMount { @@ -1131,10 +1373,11 @@ func boundCapacity(pvc *corev1.PersistentVolumeClaim) string { } // reconcileInvalidStorage publishes the Ready=False/InvalidStorageConfiguration -// terminal state for a persistent backend that asked for more than one replica, -// WITHOUT provisioning the PVC or the workload. A single ReadWriteOnce PVC -// cannot be mounted by multiple pods, so creating a multi-replica Deployment -// that shares it would wedge all but one replica in ContainerCreating +// terminal state for a persistent Deployment backend that asked for more than +// one replica, WITHOUT provisioning the PVC or the workload. A single +// ReadWriteOnce PVC cannot be mounted by multiple pods, so creating a +// multi-replica Deployment that shares it would wedge all but one replica in +// ContainerCreating // (multi-attach). The operator must scale to 1 or drop spec.storage.pvc; the // Warning event is emitted on the state transition (see emitTransitionEvents). // @@ -1198,6 +1441,40 @@ func (r *CacheBackendReconciler) reconcileInvalidStorage(ctx context.Context, ba return ctrl.Result{}, err } +// reconcileImmutableStatefulSetStorage publishes a terminal drift condition +// when a StatefulSet backend asks to add, remove, or reshape +// volumeClaimTemplates after the StatefulSet already exists. Kubernetes does +// not let the controller mutate those templates in place, so the safest +// reconciliation is to leave the live workload untouched and tell the operator +// to replace or migrate it explicitly. Because the live StatefulSet keeps +// running, callers still reconcile HPA ownership and server-instance cascade +// observation around this status update. +func (r *CacheBackendReconciler) reconcileImmutableStatefulSetStorage(ctx context.Context, backend *cachev1alpha1.CacheBackend) (ctrl.Result, error) { + r.probeLimiter.forget(client.ObjectKeyFromObject(backend).String()) + err := r.patchStatus(ctx, backend, func() { + backend.Status.Endpoint = "" + backend.Status.Capacity = "" + backend.Status.ObservedGeneration = backend.Generation + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: conditionTypeReady, + Status: metav1.ConditionFalse, + Reason: conditionReasonImmutableStatefulSetStorage, + Message: immutableStatefulSetStorageMessage, + ObservedGeneration: backend.Generation, + }) + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: conditionTypeProgressing, + Status: metav1.ConditionFalse, + Reason: conditionReasonImmutableStatefulSetStorage, + Message: "controller will not mutate StatefulSet volumeClaimTemplates after creation", + ObservedGeneration: backend.Generation, + }) + meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeDegraded) + meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeFunctionalProbeOK) + }) + return ctrl.Result{}, err +} + // warnRetainedPVC emits a Warning ONCE when spec.storage.pvc has been removed // but the CR still owns a data PVC (adopt-and-keep). The PVC is intentionally // NOT deleted — destroying persistent storage on a spec edit is irreversible @@ -1208,16 +1485,34 @@ func (r *CacheBackendReconciler) reconcileInvalidStorage(ctx context.Context, ba // every resync, so emitting unconditionally would re-fire the Warning forever. // We stamp the retained PVC with an annotation the first time we warn and skip // when it is already present, so the event fires once per orphaning rather than -// per reconcile. applyPVC clears the annotation when the backend re-adopts the -// PVC (spec.storage.pvc re-added), so a later removal warns again. +// per reconcile. Each retention mode gets its own annotation so a PVC that was +// already reported as an adopt-and-keep orphan can still later report that it +// is retained but no longer mounted after a StatefulSet kind switch. applyPVC +// clears these annotations when the backend re-adopts the PVC +// (spec.storage.pvc re-added on the Deployment path), so a later removal warns +// again. // // Fail-soft: a NotFound (the common case — no PVC was ever provisioned) or a // transient Get/Patch error simply emits nothing (the next reconcile retries). // A PVC we do not own is left entirely alone. func (r *CacheBackendReconciler) warnRetainedPVC(ctx context.Context, backend *cachev1alpha1.CacheBackend) { + r.warnRetainedDataPVC(ctx, backend, eventReasonOrphanedPVCRetained, + "spec.storage.pvc was removed but PVC %q is retained (owner-referenced; reclaimed only when this CacheBackend is deleted). The workload reverts to ephemeral storage; delete the CacheBackend to reclaim the volume, or re-add spec.storage.pvc to mount it again.") +} + +func (r *CacheBackendReconciler) warnRetainedSharedPVCForStatefulSet(ctx context.Context, backend *cachev1alpha1.CacheBackend) { + r.warnRetainedDataPVC(ctx, backend, eventReasonSharedPVCRetained, + "deploymentKind=StatefulSet uses per-replica volumeClaimTemplates; prior Deployment PVC %q is retained but no longer mounted (owner-referenced; reclaimed only when this CacheBackend is deleted, or reused if the backend switches back to deploymentKind=Deployment).") +} + +func (r *CacheBackendReconciler) warnRetainedDataPVC(ctx context.Context, backend *cachev1alpha1.CacheBackend, reason, message string) { if r.Recorder == nil { return } + warnedAnnotation := retainedPVCWarningAnnotation(reason) + if warnedAnnotation == "" { + return + } pvc := &corev1.PersistentVolumeClaim{} key := types.NamespacedName{Name: pvcName(backend), Namespace: backend.Namespace} if err := r.Get(ctx, key, pvc); err != nil { @@ -1226,8 +1521,8 @@ func (r *CacheBackendReconciler) warnRetainedPVC(ctx context.Context, backend *c if !metav1.IsControlledBy(pvc, backend) { return } - if pvc.Annotations[annotationStorageRetainedWarned] == "true" { - return // already warned for this orphaned PVC — stay quiet on resync. + if pvc.Annotations[warnedAnnotation] == "true" { + return // already warned for this reason on this PVC — stay quiet on resync. } // Stamp first, then emit: if the stamp write fails we skip the event and // retry on the next reconcile, so a persistent Patch failure can't spam. @@ -1235,12 +1530,11 @@ func (r *CacheBackendReconciler) warnRetainedPVC(ctx context.Context, backend *c if pvc.Annotations == nil { pvc.Annotations = map[string]string{} } - pvc.Annotations[annotationStorageRetainedWarned] = "true" + pvc.Annotations[warnedAnnotation] = "true" if err := r.Patch(ctx, pvc, patch); err != nil { return } - r.Recorder.Eventf(backend, nil, corev1.EventTypeWarning, eventReasonOrphanedPVCRetained, eventReasonOrphanedPVCRetained, - "spec.storage.pvc was removed but PVC %q is retained (owner-referenced; reclaimed only when this CacheBackend is deleted). The workload reverts to ephemeral storage; delete the CacheBackend to reclaim the volume, or re-add spec.storage.pvc to mount it again.", pvc.Name) + r.Recorder.Eventf(backend, nil, corev1.EventTypeWarning, reason, reason, message, pvc.Name) } // reconcileManagedPodSpec updates the spec-driven fields of the live pod spec in @@ -1350,18 +1644,19 @@ func reconcileManagedContainer(live *corev1.PodSpec, desired *corev1.PodSpec) { } } -// cleanupOwnedWorkload best-effort deletes the Deployment + Service + HPA this -// CR owns, used when a backend is no longer a managed Deployment (type/kind +// cleanupOwnedWorkload best-effort deletes the Deployment/StatefulSet + Service +// + HPA this CR owns, used when a backend is no longer managed (type // changed). Normal CR deletion is handled by owner-reference garbage // collection; this covers the in-place mutation case where the CR itself // still exists. func (r *CacheBackendReconciler) cleanupOwnedWorkload(ctx context.Context, backend *cachev1alpha1.CacheBackend) error { - key := types.NamespacedName{Name: backend.Name, Namespace: backend.Namespace} - - var dep appsv1.Deployment - if err := r.deleteIfOwned(ctx, key, &dep, backend); err != nil { + if err := r.deleteOwnedDeployment(ctx, backend); err != nil { + return err + } + if err := r.deleteOwnedStatefulSet(ctx, backend); err != nil { return err } + key := types.NamespacedName{Name: backend.Name, Namespace: backend.Namespace} var svc corev1.Service if err := r.deleteIfOwned(ctx, key, &svc, backend); err != nil { return err @@ -1370,6 +1665,18 @@ func (r *CacheBackendReconciler) cleanupOwnedWorkload(ctx context.Context, backe return r.deleteIfOwned(ctx, key, &hpa, backend) } +func (r *CacheBackendReconciler) deleteOwnedDeployment(ctx context.Context, backend *cachev1alpha1.CacheBackend) error { + key := types.NamespacedName{Name: backend.Name, Namespace: backend.Namespace} + var dep appsv1.Deployment + return r.deleteIfOwned(ctx, key, &dep, backend) +} + +func (r *CacheBackendReconciler) deleteOwnedStatefulSet(ctx context.Context, backend *cachev1alpha1.CacheBackend) error { + key := types.NamespacedName{Name: backend.Name, Namespace: backend.Namespace} + var sts appsv1.StatefulSet + return r.deleteIfOwned(ctx, key, &sts, backend) +} + // deleteIfOwned deletes obj only if it exists and is controller-owned by backend. func (r *CacheBackendReconciler) deleteIfOwned(ctx context.Context, key types.NamespacedName, obj client.Object, backend *cachev1alpha1.CacheBackend) error { if err := r.Get(ctx, key, obj); err != nil { @@ -1384,10 +1691,11 @@ func (r *CacheBackendReconciler) deleteIfOwned(ctx context.Context, key types.Na return nil } -// updateManagedStatus derives the Ready + Progressing conditions from the Deployment and patches status only when it changes. +// updateManagedStatus derives the Ready + Progressing conditions from the +// managed workload and patches status only when it changes. // // applyOK is the convergence signal from reconcileManaged: when apply failed, -// the live Deployment we read may still reflect a *prior* CR generation, so +// the live workload we read may still reflect a *prior* CR generation, so // advancing Status.ObservedGeneration to the current CR generation would tell // clients the controller has caught up when it hasn't. The published // observedGeneration therefore stays at its prior value until apply succeeds @@ -1401,9 +1709,9 @@ func (r *CacheBackendReconciler) deleteIfOwned(ctx context.Context, key types.Na // over-provisioned bind would otherwise be misreported. The Owns(PVC) watch // re-triggers this reconcile when the PVC binds so the empty→size transition is // observed promptly. -func (r *CacheBackendReconciler) updateManagedStatus(ctx context.Context, backend *cachev1alpha1.CacheBackend, endpoint, capacity string, dep *appsv1.Deployment, applyOK bool) (time.Duration, error) { +func (r *CacheBackendReconciler) updateManagedStatus(ctx context.Context, backend *cachev1alpha1.CacheBackend, endpoint, capacity string, workload managedWorkloadObservation, applyOK bool) (time.Duration, error) { now := time.Now() - readyStatus, reason, message := managedReadiness(backend, dep) + readyStatus, reason, message := managedReadinessForWorkload(backend, workload) // Resolve the stable timeout anchor: the latched FirstAvailableAt, or — the // first time the workload is Available — now. Using a latched value (not the // live Deployment Available condition, which resets on a flap) keeps the @@ -1414,8 +1722,8 @@ func (r *CacheBackendReconciler) updateManagedStatus(ctx context.Context, backen } else if readyStatus == metav1.ConditionTrue { anchor = now } - // Layer the KV-event readiness gate on top of the Deployment-level readiness. - // Only the workload-Available state is gated; every other Deployment state + // Layer the KV-event readiness gate on top of workload-level readiness. + // Only the workload-Ready state is gated; every other workload state // passes through unchanged. gate := evaluateKVEventReadiness(backend, readyStatus, reason, message, anchor, now) // Layer the functional-probe gate on top of the @@ -1519,7 +1827,7 @@ func minNonZero(a, b time.Duration) time.Duration { } // kvReadiness is the resolved readiness verdict after layering the KV-event -// gate on top of the Deployment-level readiness. readyStatus/readyReason/ +// gate on top of workload-level readiness. readyStatus/readyReason/ // readyMessage drive Conditions[Ready]; degraded* drive Conditions[Degraded]; // requeueAfter is non-zero only inside the AwaitingFirstKVEvent window, where // it schedules the automatic Degraded flip once firstEventTimeout elapses @@ -1535,7 +1843,7 @@ type kvReadiness struct { } // evaluateKVEventReadiness layers the KV-event readiness gate on top of the -// Deployment-derived health. The signal it adds is whether at least one KV +// managed workload's health. The signal it adds is whether at least one KV // event has been observed for this backend (status.indexParticipation. // lastEventAt, written by the CacheIndex poller from engine-pod reports). // @@ -1543,15 +1851,15 @@ type kvReadiness struct { // useless to the cache plane — the inference engine may be serving HTTP while // its ZMQ KV-event publisher is mis-configured or crashed, so nothing flows // into the index and LookupRoute keeps returning NO_HINT. The managed -// Deployment's own readiness cannot see that; the first-KV-event signal can. +// workload's own readiness cannot see that; the first-KV-event signal can. // // State machine — the gate only refines the state once the managed -// cache-backend Deployment is Available (managedReadiness Ready=True). Every -// other Deployment state (rollout in progress, scaled to zero, replicas +// cache-backend workload is serving (managedReadiness Ready=True). Every +// other workload state (rollout in progress, scaled to zero, replicas // unavailable) passes through unchanged: // // Workload Available? | event seen? | within timeout? | Ready | Degraded | reason -// No | - | - | (passthrough deployment readiness) +// No | - | - | (passthrough workload readiness) // Yes | No | Yes | False | False | AwaitingFirstKVEvent // Yes | Yes | - | True | False | KVEventsObserved // Yes | No | No | False | True | NoKVEventsObserved @@ -1565,16 +1873,16 @@ type kvReadiness struct { // // Timeout anchor: `anchor` is the caller-resolved start of the firstEventTimeout // clock — the write-once status.firstAvailableAt latch (the time the workload -// was first observed Available). It is deliberately NOT the live Deployment's -// Available condition LastTransitionTime, which resets on an availability flap: +// was first observed serving). It is deliberately NOT the live workload's +// availability transition time, which can reset on an availability flap: // a latched anchor keeps the elapsed window monotonic, so once a backend // breaches the timeout (Degraded / NoKVEventsObserved) a later flap cannot move // it back to AwaitingFirstKVEvent — it stays Degraded until an event arrives. // (Once firstKVEventObservedAt is latched the gate is satisfied regardless of // the anchor, since lastKVEventSeen short-circuits.) func evaluateKVEventReadiness(backend *cachev1alpha1.CacheBackend, readyStatus metav1.ConditionStatus, reason, message string, anchor, now time.Time) kvReadiness { - // Base verdict mirrors the Deployment-level readiness; the Degraded - // condition tracks the deployment-level Ready=False/ReplicasUnavailable + // Base verdict mirrors workload-level readiness; the Degraded + // condition tracks the workload-level Ready=False/ReplicasUnavailable // shape so it is consistent on every path (including the opt-out and // not-yet-Available paths below). base := kvReadiness{ @@ -1592,7 +1900,7 @@ func evaluateKVEventReadiness(backend *cachev1alpha1.CacheBackend, readyStatus m base.degradedMessage = "backend is not in a degraded state" } - // Opt-out, or a Deployment that is not yet Available: nothing to gate. + // Opt-out, or a managed workload that is not yet serving: nothing to gate. if !kvEventGateEnabled(backend) || readyStatus != metav1.ConditionTrue { return base } @@ -1719,31 +2027,92 @@ const ( conditionReasonRolloutInProgress = "RolloutInProgress" conditionReasonReplicasUnavailable = "ReplicasUnavailable" // conditionReasonInvalidStorageConfiguration is set Ready=False when a - // persistent backend (spec.storage.pvc) requests more than one replica + // persistent Deployment backend requests more than one replica // (spec.replicas > 1 or spec.autoscaling.maxReplicas > 1). A single // ReadWriteOnce PVC cannot be mounted by multiple pods, so the controller // refuses to provision the workload until the operator scales to 1 or drops - // spec.storage.pvc. Per-replica PVCs via StatefulSet volumeClaimTemplates - // are a separate follow-up. + // spec.storage.pvc. StatefulSet uses per-replica PVCs via + // volumeClaimTemplates instead. conditionReasonInvalidStorageConfiguration = "InvalidStorageConfiguration" + // conditionReasonImmutableStatefulSetStorage is set Ready=False when a + // StatefulSet backend's desired volumeClaimTemplates no longer match the + // templates established at StatefulSet creation. Kubernetes treats those + // templates as immutable, so the controller surfaces the drift instead of + // applying a pod template that references uncreated or stale claims. + conditionReasonImmutableStatefulSetStorage = "ImmutableStatefulSetStorage" ) // invalidStorageMessage is the Ready=False/InvalidStorageConfiguration message, // shared by the status condition and the Warning event so kubectl describe and // the event stream say the same thing. -const invalidStorageMessage = "spec.storage.pvc requires a single replica: a ReadWriteOnce PVC cannot be multi-attached across replicas. Scale to spec.replicas=1 and spec.autoscaling.maxReplicas<=1, or remove spec.storage.pvc. Per-replica persistent storage via StatefulSet is a separate follow-up." - -// annotationStorageRetainedWarned marks a retained (adopt-and-keep) PVC for -// which the OrphanedPVCRetained Warning has already been emitted, so the -// steady-state reconcile path warns once per orphaning rather than on every -// resync. Cleared by applyPVC when the backend re-adopts the PVC. -const annotationStorageRetainedWarned = "inferencecache.io/storage-retained-warned" - -// managedReadiness maps the Deployment's rollout state to the Ready -// condition (status + reason + message). Ready=True requires the Deployment -// to have observed its current generation and to have enough updated + -// available replicas, so a stale rollout (e.g. mid image change) is never -// reported Ready. +const invalidStorageMessage = "spec.storage.pvc on deploymentKind=Deployment requires a single replica: a ReadWriteOnce PVC cannot be multi-attached across replicas. Scale to spec.replicas=1 and spec.autoscaling.maxReplicas<=1, remove spec.storage.pvc, or use deploymentKind=StatefulSet for per-replica PVCs." + +const immutableStatefulSetStorageMessage = "StatefulSet volumeClaimTemplates are immutable after creation; create a replacement CacheBackend or migrate data deliberately to change spec.storage.pvc" + +// Retained-PVC warning annotations mark the storage-retention Warnings already +// emitted for a retained shared PVC, so the steady-state reconcile path warns +// once per reason rather than on every resync. Stored on the PVC rather than +// the CacheBackend so the guard follows the retained storage object. Cleared by +// applyPVC when the backend re-adopts the PVC on the Deployment path. +const ( + annotationStorageRetainedWarnedLegacy = "inferencecache.io/storage-retained-warned" + annotationOrphanedPVCRetainedWarned = "inferencecache.io/orphaned-pvc-retained-warned" + annotationSharedPVCRetainedWarned = "inferencecache.io/shared-pvc-retained-warned" +) + +func retainedPVCWarningAnnotation(reason string) string { + switch reason { + case eventReasonOrphanedPVCRetained: + return annotationOrphanedPVCRetainedWarned + case eventReasonSharedPVCRetained: + return annotationSharedPVCRetainedWarned + default: + return "" + } +} + +func clearRetainedPVCWarningAnnotations(annotations map[string]string) { + delete(annotations, annotationStorageRetainedWarnedLegacy) + delete(annotations, annotationOrphanedPVCRetainedWarned) + delete(annotations, annotationSharedPVCRetainedWarned) +} + +type managedWorkloadObservation struct { + kind string + generation int64 + observedGeneration int64 + specReplicas *int32 + updatedReplicas int32 + servingReplicas int32 + servingNoun string +} + +func observationFromDeployment(dep *appsv1.Deployment) managedWorkloadObservation { + return managedWorkloadObservation{ + kind: "Deployment", + generation: dep.Generation, + observedGeneration: dep.Status.ObservedGeneration, + specReplicas: dep.Spec.Replicas, + updatedReplicas: dep.Status.UpdatedReplicas, + servingReplicas: dep.Status.AvailableReplicas, + servingNoun: "available", + } +} + +func observationFromStatefulSet(sts *appsv1.StatefulSet) managedWorkloadObservation { + return managedWorkloadObservation{ + kind: "StatefulSet", + generation: sts.Generation, + observedGeneration: sts.Status.ObservedGeneration, + specReplicas: sts.Spec.Replicas, + updatedReplicas: sts.Status.UpdatedReplicas, + servingReplicas: sts.Status.ReadyReplicas, + servingNoun: "ready", + } +} + +// managedReadiness maps a Deployment's rollout state to the Ready condition. +// It is the Deployment-specific wrapper around managedReadinessForWorkload. // // When the CacheBackend is autoscaled the HPA owns the desired replica count, // so the comparison target is the live Deployment's spec.replicas (which the @@ -1752,24 +2121,37 @@ const annotationStorageRetainedWarned = "inferencecache.io/storage-retained-warn // pods than spec.replicas, and avoids a false ScaledToZero when spec.replicas // happens to be 0 with autoscaling configured. func managedReadiness(backend *cachev1alpha1.CacheBackend, dep *appsv1.Deployment) (metav1.ConditionStatus, string, string) { - want := desiredReplicas(backend, dep) + return managedReadinessForWorkload(backend, observationFromDeployment(dep)) +} + +// managedReadinessForWorkload maps a managed workload rollout observation to +// the Ready condition (status + reason + message). Ready=True requires the +// workload controller to have observed its current generation and to have +// enough updated + serving replicas (available for Deployments, ready for +// StatefulSets), so a stale rollout is never reported Ready. +func managedReadinessForWorkload(backend *cachev1alpha1.CacheBackend, workload managedWorkloadObservation) (metav1.ConditionStatus, string, string) { + want := desiredReplicasFromSpec(backend, workload.specReplicas) // A backend scaled to zero is not serving; never report it Ready. if want == 0 { return metav1.ConditionFalse, conditionReasonScaledToZero, "backend scaled to zero replicas" } - rolledOut := dep.Status.ObservedGeneration >= dep.Generation + noun := workload.servingNoun + if noun == "" { + noun = "available" + } + rolledOut := workload.observedGeneration >= workload.generation switch { - case rolledOut && dep.Status.UpdatedReplicas >= want && dep.Status.AvailableReplicas >= want: + case rolledOut && workload.updatedReplicas >= want && workload.servingReplicas >= want: return metav1.ConditionTrue, conditionReasonBackendReady, - fmt.Sprintf("%d/%d replicas available", dep.Status.AvailableReplicas, want) - case !rolledOut || dep.Status.UpdatedReplicas < want: + fmt.Sprintf("%d/%d replicas %s", workload.servingReplicas, want, noun) + case !rolledOut || workload.updatedReplicas < want: return metav1.ConditionFalse, conditionReasonRolloutInProgress, - fmt.Sprintf("%d/%d replicas updated", dep.Status.UpdatedReplicas, want) + fmt.Sprintf("%d/%d replicas updated", workload.updatedReplicas, want) default: return metav1.ConditionFalse, conditionReasonReplicasUnavailable, - fmt.Sprintf("%d/%d replicas available", dep.Status.AvailableReplicas, want) + fmt.Sprintf("%d/%d replicas %s", workload.servingReplicas, want, noun) } } @@ -1795,17 +2177,21 @@ func progressingFromReady(readyStatus metav1.ConditionStatus, reason, message st } } -// desiredReplicas is the per-reconcile source of truth for "how many replicas -// should this backend be running". With autoscaling enabled the HPA writes -// spec.replicas on the Deployment, so the live value is authoritative; without -// it, the user's spec.replicas (default 1) wins. +// desiredReplicas is the Deployment-specific compatibility wrapper for tests +// and older helpers. With autoscaling enabled the HPA writes spec.replicas on +// the workload, so the live value is authoritative; without it, the user's +// spec.replicas (default 1) wins. func desiredReplicas(backend *cachev1alpha1.CacheBackend, dep *appsv1.Deployment) int32 { + return desiredReplicasFromSpec(backend, dep.Spec.Replicas) +} + +func desiredReplicasFromSpec(backend *cachev1alpha1.CacheBackend, specReplicas *int32) int32 { if backend.Spec.Autoscaling != nil { // First reconcile after an HPA spec is added may briefly see - // dep.Spec.Replicas still set by the controller; the HPA will overwrite + // specReplicas still set by the controller; the HPA will overwrite // it within one cycle. Until then, fall back to the controller value. - if dep.Spec.Replicas != nil { - return *dep.Spec.Replicas + if specReplicas != nil { + return *specReplicas } // Fall through to the floor. } @@ -1815,7 +2201,7 @@ func desiredReplicas(backend *cachev1alpha1.CacheBackend, dep *appsv1.Deployment return 1 } -// initialReplicas picks the Deployment's initial replica count. With +// initialReplicas picks the managed workload's initial replica count. With // autoscaling configured, spec.autoscaling.minReplicas is the source of truth // (defaulting to 1 when unset), so the workload comes up at or above the HPA // floor on first apply instead of starting at 1 and waiting for the HPA to @@ -1833,16 +2219,41 @@ func initialReplicas(backend *cachev1alpha1.CacheBackend) int32 { return 1 } +type managedWorkloadRef struct { + name string + namespace string + kind string + labels map[string]string +} + +func workloadRefFromDeployment(dep *appsv1.Deployment) managedWorkloadRef { + return managedWorkloadRef{ + name: dep.Name, + namespace: dep.Namespace, + kind: "Deployment", + labels: dep.Labels, + } +} + +func workloadRefFromStatefulSet(sts *appsv1.StatefulSet) managedWorkloadRef { + return managedWorkloadRef{ + name: sts.Name, + namespace: sts.Namespace, + kind: "StatefulSet", + labels: sts.Labels, + } +} + // reconcileHPA creates, updates, or deletes the HorizontalPodAutoscaler that -// drives the backend Deployment's replica count. The HPA exists iff +// drives the managed workload's replica count. The HPA exists iff // spec.autoscaling is set; otherwise any controller-owned HPA is removed. -func (r *CacheBackendReconciler) reconcileHPA(ctx context.Context, backend *cachev1alpha1.CacheBackend, deployment *appsv1.Deployment) error { +func (r *CacheBackendReconciler) reconcileHPA(ctx context.Context, backend *cachev1alpha1.CacheBackend, workload managedWorkloadRef) error { if backend.Spec.Autoscaling == nil { // Autoscaling disabled — clean up any HPA we previously owned. - return r.deleteOwnedHPA(ctx, backend, deployment.Name) + return r.deleteOwnedHPA(ctx, backend, workload.name) } - desired := buildHPA(backend, deployment) + desired := buildHPA(backend, workload) hpa := &autoscalingv2.HorizontalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: desired.Name, Namespace: desired.Namespace}} _, err := controllerutil.CreateOrUpdate(ctx, r.Client, hpa, func() error { hpa.Labels = desired.Labels @@ -1856,9 +2267,9 @@ func (r *CacheBackendReconciler) reconcileHPA(ctx context.Context, backend *cach } // buildHPA renders the desired HorizontalPodAutoscaler for a CacheBackend whose -// spec.autoscaling is set. Targets the managed Deployment by name. Phase 1 ships +// spec.autoscaling is set. Targets the managed workload by name. Phase 1 ships // a CPU-utilization target; cache-aware (custom-metric) HPAs come later. -func buildHPA(backend *cachev1alpha1.CacheBackend, deployment *appsv1.Deployment) *autoscalingv2.HorizontalPodAutoscaler { +func buildHPA(backend *cachev1alpha1.CacheBackend, workload managedWorkloadRef) *autoscalingv2.HorizontalPodAutoscaler { spec := backend.Spec.Autoscaling minReplicas := defaultHPAMinReplicas if spec.MinReplicas != nil { @@ -1870,15 +2281,15 @@ func buildHPA(backend *cachev1alpha1.CacheBackend, deployment *appsv1.Deployment } return &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: deployment.Name, - Namespace: deployment.Namespace, - Labels: deployment.Labels, + Name: workload.name, + Namespace: workload.namespace, + Labels: workload.labels, }, Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ APIVersion: "apps/v1", - Kind: "Deployment", - Name: deployment.Name, + Kind: workload.kind, + Name: workload.name, }, MinReplicas: &minReplicas, MaxReplicas: spec.MaxReplicas, @@ -2233,6 +2644,7 @@ func (r *CacheBackendReconciler) SetupWithManager(mgr ctrl.Manager) error { // AwaitingFirstKVEvent -> Ready transition. For(&cachev1alpha1.CacheBackend{}). Owns(&appsv1.Deployment{}). + Owns(&appsv1.StatefulSet{}). Owns(&corev1.Service{}). Owns(&autoscalingv2.HorizontalPodAutoscaler{}). // Owns(PVC) so a PVC binding (status.capacity goes from empty to the diff --git a/internal/controller/cachebackend_controller_test.go b/internal/controller/cachebackend_controller_test.go index 11f2f5d6..a5e6f3d4 100644 --- a/internal/controller/cachebackend_controller_test.go +++ b/internal/controller/cachebackend_controller_test.go @@ -3,6 +3,7 @@ package controller import ( "context" "errors" + "strings" "sync/atomic" "testing" "time" @@ -42,7 +43,7 @@ func newScheme(t *testing.T) *runtime.Scheme { func newReconciler(scheme *runtime.Scheme, objs ...client.Object) *CacheBackendReconciler { c := fake.NewClientBuilder(). WithScheme(scheme). - WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &appsv1.Deployment{}, &corev1.PersistentVolumeClaim{}). + WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &appsv1.Deployment{}, &appsv1.StatefulSet{}, &corev1.PersistentVolumeClaim{}). WithObjects(objs...). Build() return &CacheBackendReconciler{ @@ -103,6 +104,34 @@ func getOptionalDeployment(t *testing.T, r *CacheBackendReconciler, name, namesp return &dep, err } +func getStatefulSet(t *testing.T, r *CacheBackendReconciler, name, namespace string) *appsv1.StatefulSet { + t.Helper() + var sts appsv1.StatefulSet + if err := r.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, &sts); err != nil { + t.Fatalf("get statefulset %s/%s: %v", namespace, name, err) + } + return &sts +} + +func getOptionalStatefulSet(t *testing.T, r *CacheBackendReconciler, name, namespace string) (*appsv1.StatefulSet, error) { + t.Helper() + var sts appsv1.StatefulSet + err := r.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, &sts) + return &sts, err +} + +func markStatefulSetReady(t *testing.T, r *CacheBackendReconciler, name, namespace string, want int32) { + t.Helper() + sts := getStatefulSet(t, r, name, namespace) + sts.Status.ObservedGeneration = sts.Generation + sts.Status.Replicas = want + sts.Status.UpdatedReplicas = want + sts.Status.ReadyReplicas = want + if err := r.Status().Update(context.Background(), sts); err != nil { + t.Fatalf("update statefulset status to ready: %v", err) + } +} + func getBackend(t *testing.T, r *CacheBackendReconciler, name, namespace string) *cachev1alpha1.CacheBackend { t.Helper() var cb cachev1alpha1.CacheBackend @@ -377,6 +406,57 @@ func TestManagedReadinessZeroReplicasNotReady(t *testing.T) { } } +func TestManagedReadinessForStatefulSetUsesReadyReplicas(t *testing.T) { + cb := lmcacheBackend("cache", "ns1") + cb.Spec.Replicas = ptrInt32(2) + replicas := int32(2) + + cases := []struct { + name string + status appsv1.StatefulSetStatus + wantStatus metav1.ConditionStatus + wantReason string + }{ + { + name: "rolled out and ready", + status: appsv1.StatefulSetStatus{ + ObservedGeneration: 1, + UpdatedReplicas: 2, + ReadyReplicas: 2, + }, + wantStatus: metav1.ConditionTrue, + wantReason: conditionReasonBackendReady, + }, + { + name: "rolled out but not ready", + status: appsv1.StatefulSetStatus{ + ObservedGeneration: 1, + UpdatedReplicas: 2, + ReadyReplicas: 1, + }, + wantStatus: metav1.ConditionFalse, + wantReason: conditionReasonReplicasUnavailable, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Generation: 1}, + Spec: appsv1.StatefulSetSpec{Replicas: &replicas}, + Status: tc.status, + } + status, reason, message := managedReadinessForWorkload(cb, observationFromStatefulSet(sts)) + if status != tc.wantStatus || reason != tc.wantReason { + t.Fatalf("managedReadinessForWorkload = %v/%q (%q), want %v/%q", status, reason, message, tc.wantStatus, tc.wantReason) + } + if !strings.Contains(message, "replicas ready") { + t.Fatalf("message = %q, want StatefulSet readiness wording", message) + } + }) + } +} + func TestReconcileServicePortDriftCorrected(t *testing.T) { scheme := newScheme(t) r := newReconciler(scheme, lmcacheBackend("cache", "ns1")) @@ -529,11 +609,10 @@ func TestReconcileTypeSwitchToExternalClearsObservedServerInstance(t *testing.T) } } -// TestReconcileSwitchToStatefulSetClearsObservedServerInstance asserts +// TestReconcileSwitchToUnsupportedTypeClearsObservedServerInstance asserts // the same clearing for the managed→unsupported-runtime transition -// (reconcileUnmanaged path). The StatefulSet deployment-kind is -// currently the canonical unmanaged trigger. -func TestReconcileSwitchToStatefulSetClearsObservedServerInstance(t *testing.T) { +// (reconcileUnmanaged path). +func TestReconcileSwitchToUnsupportedTypeClearsObservedServerInstance(t *testing.T) { scheme := newScheme(t) r := newReconciler(scheme, lmcacheBackend("cache", "ns1")) @@ -550,9 +629,9 @@ func TestReconcileSwitchToStatefulSetClearsObservedServerInstance(t *testing.T) } switching := getBackend(t, r, "cache", "ns1") - switching.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + switching.Spec.Type = cachev1alpha1.CacheBackendTypeMooncake if err := r.Update(context.Background(), switching); err != nil { - t.Fatalf("switch to StatefulSet: %v", err) + t.Fatalf("switch to unsupported type: %v", err) } reconcile(t, r, "cache", "ns1") @@ -639,9 +718,9 @@ func TestReconcileLifecycleExitsClearProbeRateLimiter(t *testing.T) { }, }, { - name: "managed → Unmanaged (StatefulSet kind)", + name: "managed → Unmanaged (unsupported type)", mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + cb.Spec.Type = cachev1alpha1.CacheBackendTypeMooncake }, }, { @@ -690,24 +769,161 @@ func TestReconcileLifecycleExitsClearProbeRateLimiter(t *testing.T) { } } -func TestReconcileStatefulSetKindDeferred(t *testing.T) { +func TestReconcileStatefulSetKindCreatesStatefulSet(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + cb.Spec.Replicas = ptrInt32(2) r := newReconciler(scheme, cb) reconcile(t, r, "cache", "ns1") - var deps appsv1.DeploymentList - if err := r.List(context.Background(), &deps, client.InNamespace("ns1")); err != nil { - t.Fatalf("list deployments: %v", err) + if _, err := getOptionalDeployment(t, r, "cache", "ns1"); !apierrors.IsNotFound(err) { + t.Fatalf("StatefulSet deploymentKind must not provision a Deployment; get err=%v", err) } - if len(deps.Items) != 0 { - t.Fatalf("deployments = %d, want 0 (StatefulSet kind deferred — managed Deployments only for now)", len(deps.Items)) + sts := getStatefulSet(t, r, "cache", "ns1") + if sts.Spec.Replicas == nil || *sts.Spec.Replicas != 2 { + t.Fatalf("statefulset replicas = %v, want 2", sts.Spec.Replicas) + } + if sts.Spec.ServiceName != "cache" { + t.Fatalf("statefulset serviceName = %q, want cache", sts.Spec.ServiceName) + } + owner := metav1.GetControllerOf(sts) + if owner == nil || owner.Kind != "CacheBackend" || owner.Name != "cache" || owner.Controller == nil || !*owner.Controller { + t.Fatalf("statefulset controller owner = %+v, want controller ref to CacheBackend/cache", owner) + } + if ep := getBackend(t, r, "cache", "ns1").Status.Endpoint; ep != "cache.ns1.svc.cluster.local:65432" { + t.Fatalf("status.endpoint = %q, want service endpoint", ep) + } +} + +func TestReconcileDeploymentToStatefulSetApplyFailureClearsStatus(t *testing.T) { + scheme := newScheme(t) + cb := lmcacheBackend("cache", "ns1") + cb.Spec.Replicas = ptrInt32(1) + + gr := schema.GroupResource{Group: "apps", Resource: "statefulsets"} + funcs := interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*appsv1.StatefulSet); ok { + return apierrors.NewForbidden(gr, obj.GetName(), errors.New("denied by admission webhook")) + } + return c.Create(ctx, obj, opts...) + }, + } + r := newReconcilerWithInterceptor(scheme, funcs, cb) + + reconcile(t, r, "cache", "ns1") + markDeploymentReady(t, r, "cache", "ns1", 1) + reconcile(t, r, "cache", "ns1") + before := getBackend(t, r, "cache", "ns1") + if before.Status.Endpoint == "" { + t.Fatalf("test setup failed: status.endpoint empty before kind switch") + } + if ready := findCondition(before.Status.Conditions, conditionTypeReady); ready == nil || ready.Status != metav1.ConditionTrue { + t.Fatalf("test setup failed: Ready = %+v, want True before kind switch", ready) + } + + switching := getBackend(t, r, "cache", "ns1") + switching.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + switching.Generation = 2 + if err := r.Update(context.Background(), switching); err != nil { + t.Fatalf("switch backend to StatefulSet: %v", err) + } + + if _, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "cache", Namespace: "ns1"}, + }); err == nil { + t.Fatalf("reconcile returned nil, want error (StatefulSet create was blocked)") + } + if _, err := getOptionalDeployment(t, r, "cache", "ns1"); !apierrors.IsNotFound(err) { + t.Fatalf("old Deployment should be deleted during kind switch; get err=%v", err) + } + updated := getBackend(t, r, "cache", "ns1") + if updated.Status.Endpoint != "" { + t.Fatalf("status.endpoint = %q, want cleared after replacement apply failure", updated.Status.Endpoint) + } + if ready := findCondition(updated.Status.Conditions, conditionTypeReady); ready == nil || ready.Status == metav1.ConditionTrue { + t.Fatalf("Ready condition = %+v, want non-True after replacement apply failure", ready) + } + if got := updated.Status.ObservedGeneration; got != 1 { + t.Fatalf("status.observedGeneration = %d, want 1 (replacement apply failed)", got) } } -func TestReconcileSwitchToStatefulSetClearsStaleStatus(t *testing.T) { +func TestReconcileStatefulSetToDeploymentApplyFailureClearsStatus(t *testing.T) { + scheme := newScheme(t) + cb := lmcacheBackend("cache", "ns1") + cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + cb.Spec.Replicas = ptrInt32(1) + + gr := schema.GroupResource{Group: "apps", Resource: "deployments"} + funcs := interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*appsv1.Deployment); ok { + return apierrors.NewForbidden(gr, obj.GetName(), errors.New("denied by admission webhook")) + } + return c.Create(ctx, obj, opts...) + }, + } + r := newReconcilerWithInterceptor(scheme, funcs, cb) + + reconcile(t, r, "cache", "ns1") + markStatefulSetReady(t, r, "cache", "ns1", 1) + reconcile(t, r, "cache", "ns1") + before := getBackend(t, r, "cache", "ns1") + if before.Status.Endpoint == "" { + t.Fatalf("test setup failed: status.endpoint empty before kind switch") + } + if ready := findCondition(before.Status.Conditions, conditionTypeReady); ready == nil || ready.Status != metav1.ConditionTrue { + t.Fatalf("test setup failed: Ready = %+v, want True before kind switch", ready) + } + + switching := getBackend(t, r, "cache", "ns1") + switching.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindDeployment + switching.Generation = 2 + if err := r.Update(context.Background(), switching); err != nil { + t.Fatalf("switch backend to Deployment: %v", err) + } + + if _, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "cache", Namespace: "ns1"}, + }); err == nil { + t.Fatalf("reconcile returned nil, want error (Deployment create was blocked)") + } + if _, err := getOptionalStatefulSet(t, r, "cache", "ns1"); !apierrors.IsNotFound(err) { + t.Fatalf("old StatefulSet should be deleted during kind switch; get err=%v", err) + } + updated := getBackend(t, r, "cache", "ns1") + if updated.Status.Endpoint != "" { + t.Fatalf("status.endpoint = %q, want cleared after replacement apply failure", updated.Status.Endpoint) + } + if ready := findCondition(updated.Status.Conditions, conditionTypeReady); ready == nil || ready.Status == metav1.ConditionTrue { + t.Fatalf("Ready condition = %+v, want non-True after replacement apply failure", ready) + } + if got := updated.Status.ObservedGeneration; got != 1 { + t.Fatalf("status.observedGeneration = %d, want 1 (replacement apply failed)", got) + } +} + +func TestReconcileStatefulSetSchedulesServerInstancePoll(t *testing.T) { + scheme := newScheme(t) + cb := lmcacheBackend("cache", "ns1") + cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + r := newReconciler(scheme, cb) + + result, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "cache", Namespace: "ns1"}, + }) + if err != nil { + t.Fatalf("reconcile StatefulSet backend: %v", err) + } + if result.RequeueAfter != DefaultMinServerRestartCascadeInterval { + t.Fatalf("StatefulSet reconcile RequeueAfter = %s, want %s server-instance poll cadence", result.RequeueAfter, DefaultMinServerRestartCascadeInterval) + } +} + +func TestReconcileSwitchToStatefulSetShedsDeployment(t *testing.T) { scheme := newScheme(t) r := newReconciler(scheme, lmcacheBackend("cache", "ns1")) @@ -715,6 +931,13 @@ func TestReconcileSwitchToStatefulSetClearsStaleStatus(t *testing.T) { if ep := getBackend(t, r, "cache", "ns1").Status.Endpoint; ep == "" { t.Fatalf("expected a published endpoint after managed reconcile") } + baseline := getBackend(t, r, "cache", "ns1") + baseline.Status.ObservedServerInstance = "deployment-pod-uid:0" + if err := r.Status().Update(context.Background(), baseline); err != nil { + t.Fatalf("plant observedServerInstance: %v", err) + } + plantedKey := cascadeKey{namespace: baseline.Namespace, name: baseline.Name, uid: string(baseline.UID)} + r.serverInstanceCascade.recordAttempt(plantedKey, "deployment-pod-uid:0") live := getBackend(t, r, "cache", "ns1") live.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet @@ -724,19 +947,44 @@ func TestReconcileSwitchToStatefulSetClearsStaleStatus(t *testing.T) { reconcile(t, r, "cache", "ns1") updated := getBackend(t, r, "cache", "ns1") - if updated.Status.Endpoint != "" { - t.Fatalf("status.endpoint = %q, want cleared after no longer managed", updated.Status.Endpoint) + if updated.Status.Endpoint != "cache.ns1.svc.cluster.local:65432" { + t.Fatalf("status.endpoint = %q, want service endpoint after switching to StatefulSet", updated.Status.Endpoint) } - if cond := findCondition(updated.Status.Conditions, conditionTypeReady); cond != nil { - t.Fatalf("Ready condition = %+v, want removed", cond) + if cond := findCondition(updated.Status.Conditions, conditionTypeReady); cond == nil || cond.Reason != conditionReasonRolloutInProgress { + t.Fatalf("Ready condition = %+v, want rollout status from StatefulSet", cond) } - var deps appsv1.DeploymentList - if err := r.List(context.Background(), &deps, client.InNamespace("ns1")); err != nil { - t.Fatalf("list deployments: %v", err) + if updated.Status.ObservedServerInstance != "deployment-pod-uid:0" { + t.Fatalf("status.observedServerInstance = %q, want prior latch retained until the StatefulSet has a Ready pod", updated.Status.ObservedServerInstance) } - if len(deps.Items) != 0 { - t.Fatalf("deployments = %d, want 0 after switch to StatefulSet kind", len(deps.Items)) + if shadow := r.serverInstanceCascade.lastAttempt(plantedKey); shadow != "deployment-pod-uid:0" { + t.Fatalf("cascade shadow = %q after switch to StatefulSet; want prior latch retained until replacement observation", shadow) + } + if _, err := getOptionalDeployment(t, r, "cache", "ns1"); !apierrors.IsNotFound(err) { + t.Fatalf("deployment should be deleted after switch to StatefulSet kind; get err=%v", err) + } + getStatefulSet(t, r, "cache", "ns1") +} + +func TestReconcileSwitchFromStatefulSetToDeploymentShedsStatefulSet(t *testing.T) { + scheme := newScheme(t) + cb := lmcacheBackend("cache", "ns1") + cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + r := newReconciler(scheme, cb) + + reconcile(t, r, "cache", "ns1") + getStatefulSet(t, r, "cache", "ns1") + + live := getBackend(t, r, "cache", "ns1") + live.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindDeployment + if err := r.Update(context.Background(), live); err != nil { + t.Fatalf("switch to Deployment kind: %v", err) + } + reconcile(t, r, "cache", "ns1") + + if _, err := getOptionalStatefulSet(t, r, "cache", "ns1"); !apierrors.IsNotFound(err) { + t.Fatalf("statefulset should be deleted after switch to Deployment kind; get err=%v", err) } + getDeployment(t, r, "cache", "ns1") } func TestReconcileExternalAdvancesObservedGeneration(t *testing.T) { @@ -1224,7 +1472,7 @@ func TestReconcileIgnoresMissingObject(t *testing.T) { func newReconcilerWithInterceptor(scheme *runtime.Scheme, funcs interceptor.Funcs, objs ...client.Object) *CacheBackendReconciler { c := fake.NewClientBuilder(). WithScheme(scheme). - WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &appsv1.Deployment{}). + WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &appsv1.Deployment{}, &appsv1.StatefulSet{}). WithObjects(objs...). WithInterceptorFuncs(funcs). Build() diff --git a/internal/controller/cachebackend_probe.go b/internal/controller/cachebackend_probe.go index 79b0da55..c042f3d6 100644 --- a/internal/controller/cachebackend_probe.go +++ b/internal/controller/cachebackend_probe.go @@ -25,7 +25,7 @@ import ( // // Composition order on a managed CacheBackend's Reconcile: // -// managedReadiness → Deployment-level ready/progressing +// managedReadiness → workload-level ready/progressing // ↓ // evaluateKVEventReadiness → first-event readiness gate layered on top // ↓ diff --git a/internal/controller/cachebackend_server_restart.go b/internal/controller/cachebackend_server_restart.go index 38b9c9a1..1387cef1 100644 --- a/internal/controller/cachebackend_server_restart.go +++ b/internal/controller/cachebackend_server_restart.go @@ -392,11 +392,11 @@ func (r *CacheBackendReconciler) minServerRestartCascadeInterval() time.Duration // to invalidate, so by definition no engine sockets are stale — // any engines that connected during the "" window connected to the // very pod we are now baselining) -// - prior set strictly grows AND the owning Deployment is still -// rolling (a maxSurge midpoint: the old pod is still Ready while -// the new one comes up; the subsequent transition that drops the -// old pod IS a cascade). When the Deployment has converged at the -// wider count instead — operator-driven scale-up — the widened +// - prior set strictly grows AND the owning workload is still +// rolling (for example, a Deployment maxSurge midpoint: the old pod +// is still Ready while the new one comes up; the subsequent +// transition that drops the old pod IS a cascade). When the workload +// has converged at the wider count instead — operator-driven scale-up — the widened // set IS persisted as the new baseline, so a later replacement of // any of the added pods cascades correctly. // @@ -421,7 +421,7 @@ func (r *CacheBackendReconciler) minServerRestartCascadeInterval() time.Duration // container restart) // // Fail-soft: every error path (server-instance observation — -// owned Deployment Get, ReplicaSet owner-chain Get, Pod list; +// owned workload Get, ReplicaSet owner-chain Get, Pod list; // engine-cascade observation/annotate; status patch) logs at V(1) // and returns a positive requeue duration (typically // minServerRestartCascadeInterval) rather than escalating to the @@ -433,8 +433,8 @@ func (r *CacheBackendReconciler) minServerRestartCascadeInterval() time.Duration func (r *CacheBackendReconciler) reconcileServerInstance(ctx context.Context, logger logr.Logger, backend *cachev1alpha1.CacheBackend) time.Duration { currentID, converged, err := r.currentServerInstanceID(ctx, backend) if err != nil { - // A transient observation failure (owned Deployment Get, - // ReplicaSet owner-chain Get, or Pod list — see + // A transient observation failure (owned workload Get, + // ReplicaSet owner-chain Get for Deployment pods, or Pod list — see // currentServerInstanceID for the chain) leaves us unable // to decide whether a cascade is needed. Return the rate- // limit interval as the requeue hint so the reconcile @@ -544,7 +544,7 @@ func (r *CacheBackendReconciler) reconcileServerInstance(ctx context.Context, lo // rolling-update widening (old + new pod both Ready briefly). // // Strict-superset transitions split into two cases by the owning - // Deployment's convergence flag (see currentServerInstanceID): + // workload's convergence flag (see currentServerInstanceID): // // - NOT converged (rolling-update midpoint): do NOT persist the // widened set. If we did, a failed rollout that gets rolled @@ -575,7 +575,7 @@ func (r *CacheBackendReconciler) reconcileServerInstance(ctx context.Context, lo "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) return r.minServerRestartCascadeInterval() } - logger.V(1).Info("server-restart cascade skipped: superset at Deployment steady state (scale-up); latch advanced", + logger.V(1).Info("server-restart cascade skipped: superset at workload steady state (scale-up); latch advanced", "namespace", backend.Namespace, "name", backend.Name, "prior", prior, "current", currentID) return 0 @@ -718,17 +718,19 @@ func parseInstanceMap(s string) map[string]int32 { // currentServerInstanceID returns a stable identifier representing // the current set of Ready cache-server pods for the backend, the -// owning Deployment's convergence flag (true when the Deployment -// controller has reached steady state — `spec.replicas` == +// owning workload's convergence flag (true when the Deployment or +// StatefulSet controller has reached steady state — `spec.replicas` == // `status.readyReplicas` == `status.updatedReplicas` and // `status.observedGeneration` >= `metadata.generation`), or "" when no -// Ready pod exists yet. The candidate set is the owned Deployment's -// pods — pods whose controller-owning ReplicaSet is controller-owned -// by the backend-owned Deployment, identified by both name AND UID so -// a foreign Ready pod that happens to carry the same controller- -// managed labels (or a stale ownerRef name that resolves to a -// different live object) cannot advance observedServerInstance and -// spuriously trigger an engine rollout. +// Ready pod exists yet. The candidate set is the owned workload's +// pods — Deployment pods whose controller-owning ReplicaSet is +// controller-owned by the backend-owned Deployment, or StatefulSet pods +// directly controller-owned by the backend-owned StatefulSet. Every +// ownership link is identified by both name AND UID so a foreign Ready +// pod that happens to carry the same controller-managed labels (or a +// stale ownerRef name that resolves to a different live object) cannot +// advance observedServerInstance and spuriously trigger an engine +// rollout. // // The identifier shape is `:` per Ready pod, // comma-joined and lex-sorted by pod name; for a single-replica @@ -742,9 +744,10 @@ func parseInstanceMap(s string) map[string]int32 { // LMCache server process inside the pod is fresh. // // The convergence flag lets the caller distinguish a rolling-update -// midpoint (NOT converged: maxSurge has briefly widened the Ready set -// above target) from a steady-state scale-up (converged: the operator -// raised replicas and the new pods are part of the steady state). +// midpoint (NOT converged: the controller has not reached its target +// Ready/updated replica count yet) from a steady-state scale-up +// (converged: the operator raised replicas and the new pods are part of +// the steady state). // reconcileServerInstance persists a strict-superset baseline only // when converged is true; otherwise it could pin a transient midpoint // that a rollback would later misread as a real replacement. @@ -759,24 +762,38 @@ func (r *CacheBackendReconciler) currentServerInstanceID(ctx context.Context, ba reader = r.Client } - // Fetch the owned Deployment so we can authenticate candidate pods - // against its UID. Verify the live Deployment is still controlled by - // THIS CacheBackend before using it as the ownership anchor — name - // reuse / race conditions could otherwise let a foreign Deployment - // re-created under the same name be treated as ours. NotFound is the - // cold-start case (CR exists, the reconciler hasn't created the - // Deployment yet, or it was deleted out-of-band): no pods can be - // authoritatively attributed so report "no instance". var ownedDep appsv1.Deployment + depOwned := false if err := reader.Get(ctx, types.NamespacedName{Namespace: backend.Namespace, Name: backend.Name}, &ownedDep); err != nil { - if apierrors.IsNotFound(err) { - return "", false, nil + if !apierrors.IsNotFound(err) { + return "", false, fmt.Errorf("get owned cache-server deployment: %w", err) + } + } else { + depOwned = metav1.IsControlledBy(&ownedDep, backend) + } + + var ownedSts appsv1.StatefulSet + stsOwned := false + if err := reader.Get(ctx, types.NamespacedName{Namespace: backend.Namespace, Name: backend.Name}, &ownedSts); err != nil { + if !apierrors.IsNotFound(err) { + return "", false, fmt.Errorf("get owned cache-server statefulset: %w", err) } - return "", false, fmt.Errorf("get owned cache-server deployment: %w", err) + } else { + stsOwned = metav1.IsControlledBy(&ownedSts, backend) } - if !metav1.IsControlledBy(&ownedDep, backend) { - // Foreign Deployment sharing the backend's name. Refuse to - // attribute its pods to this CacheBackend. + + var workload serverInstanceWorkloadObserver + switch { + case backend.Spec.DeploymentKind == cachev1alpha1.CacheBackendDeploymentKindStatefulSet && stsOwned: + workload = statefulSetServerInstanceObserver(&ownedSts) + case depOwned: + workload = deploymentServerInstanceObserver(ctx, r, reader, &ownedDep) + case stsOwned: + workload = statefulSetServerInstanceObserver(&ownedSts) + default: + // No backend-owned managed workload is present yet, or a foreign + // object with the same name is present. Refuse to attribute pods + // to this CacheBackend without a UID-authenticated workload anchor. return "", false, nil } @@ -788,31 +805,28 @@ func (r *CacheBackendReconciler) currentServerInstanceID(ctx context.Context, ba ); err != nil { return "", false, fmt.Errorf("list cache-server pods: %w", err) } - // Build the set of container names the owned Deployment renders. + // Build the set of container names the owned workload renders. // containerRunSum sums restart counts ONLY for these — sidecars // injected by other admission webhooks (Istio's istio-proxy, // linkerd's linkerd-proxy, Datadog agents, etc.) are added to the - // pod's containerStatuses but are absent from the Deployment - // template, so including them would cascade-restart every engine - // any time a service-mesh sidecar crash-looped — completely - // unrelated to the LMCache server's actual state. - cacheServerContainers := make(map[string]struct{}, len(ownedDep.Spec.Template.Spec.Containers)) - for _, c := range ownedDep.Spec.Template.Spec.Containers { - cacheServerContainers[c.Name] = struct{}{} - } + // pod's containerStatuses but are absent from the owned workload + // template, so including them would cascade-restart every engine any + // time a service-mesh sidecar crash-looped — completely unrelated to + // the LMCache server's actual state. + cacheServerContainers := workload.containerNames // Collect every Ready, attributable pod's identifier. A pod that // is mid-rollout (Pending / Terminating) does not represent a // serving instance — including it would let a rollout's transient // state trigger a cascade even though the prior instance is still // serving. A pod that is Ready but not transitively controller- - // owned by THIS backend's Deployment is rejected — see the godoc + // owned by THIS backend's workload is rejected — see the godoc // above for why the ownership check is required. // // The per-pod identifier is : where // containerRunSum is the sum of restart counts across the // cache-server's own containers (filtered by the owned - // Deployment's template names). pod.UID alone is invariant across + // workload's template names). pod.UID alone is invariant across // in-place container restarts (kubelet restarting a crashed // container reuses the pod object), but the LMCache server // process inside that pod is fresh and every engine still holds @@ -831,7 +845,7 @@ func (r *CacheBackendReconciler) currentServerInstanceID(ctx context.Context, ba if !podIsReady(p) { continue } - owned, err := r.podOwnedByDeployment(ctx, reader, p, &ownedDep) + owned, err := workload.ownsPod(p) if err != nil { return "", false, err } @@ -844,10 +858,10 @@ func (r *CacheBackendReconciler) currentServerInstanceID(ctx context.Context, ba }) } - // Deployment convergence: spec.replicas == status.readyReplicas + // Workload convergence: spec.replicas == status.readyReplicas // == status.updatedReplicas == len(live Ready pods), AND the // controller has observed the current generation. When all four - // hold, the Deployment is in steady state and a strict-superset + // hold, the workload is in steady state and a strict-superset // Ready set against the prior baseline reflects a legitimate // scale-up (not a maxSurge midpoint). All four are required: // - readyReplicas == spec.replicas → right number of Ready pods @@ -858,29 +872,18 @@ func (r *CacheBackendReconciler) currentServerInstanceID(ctx context.Context, ba // - observedGeneration >= metadata.generation → the controller // has seen the current spec (rules out a just-changed spec // whose effects haven't propagated yet) - // - len(live Ready pods) == spec.replicas → the live pod list - // we just took for the identifier MATCHES the Deployment - // status's claim. Without this clause, a stale Deployment - // status (the apps controller hasn't yet observed the - // maxSurge new pod) could report readyReplicas==1 while we - // observed 2 Ready pods in the same reconcile — the - // status counters would lie convergence even though the - // midpoint is genuinely transient, and we would persist the - // widened latch as a "scale-up" baseline. A later rollback - // dropping the new pod would then look like a real - // replacement (a UID disappeared from the latch) and - // false-cascade. Cross-checking against the live count is - // cheap and closes that race. - // nil spec.Replicas defaults to 1 per the Deployment defaulter - // (kubebuilder/apiserver default), so we collapse nil to 1. - wantReplicas := int32(1) - if ownedDep.Spec.Replicas != nil { - wantReplicas = *ownedDep.Spec.Replicas - } - converged := ownedDep.Status.ObservedGeneration >= ownedDep.Generation && - ownedDep.Status.ReadyReplicas == wantReplicas && - ownedDep.Status.UpdatedReplicas == wantReplicas && - int32(len(ready)) == wantReplicas + // - len(live Ready pods) == spec.replicas → the live pod list we + // just took for the identifier MATCHES the workload status's + // claim. Without this clause, stale apps-controller status + // could report readyReplicas==1 while we observed 2 Ready pods + // in the same reconcile — the status counters would lie + // convergence even though the midpoint is genuinely transient, + // and we would persist the widened latch as a "scale-up" + // baseline. A later rollback dropping the new pod would then + // look like a real replacement (a UID disappeared from the + // latch) and false-cascade. Cross-checking against the live + // count is cheap and closes that race. + converged := workload.converged(len(ready)) if len(ready) == 0 { return "", converged, nil @@ -893,9 +896,61 @@ func (r *CacheBackendReconciler) currentServerInstanceID(ctx context.Context, ba return strings.Join(ids, ","), converged, nil } +type serverInstanceWorkloadObserver struct { + containerNames map[string]struct{} + converged func(readyCount int) bool + ownsPod func(*corev1.Pod) (bool, error) +} + +func deploymentServerInstanceObserver(ctx context.Context, r *CacheBackendReconciler, reader client.Reader, dep *appsv1.Deployment) serverInstanceWorkloadObserver { + return serverInstanceWorkloadObserver{ + containerNames: workloadContainerNames(dep.Spec.Template.Spec.Containers), + converged: func(readyCount int) bool { + wantReplicas := int32(1) + if dep.Spec.Replicas != nil { + wantReplicas = *dep.Spec.Replicas + } + return dep.Status.ObservedGeneration >= dep.Generation && + dep.Status.ReadyReplicas == wantReplicas && + dep.Status.UpdatedReplicas == wantReplicas && + int32(readyCount) == wantReplicas + }, + ownsPod: func(p *corev1.Pod) (bool, error) { + return r.podOwnedByDeployment(ctx, reader, p, dep) + }, + } +} + +func statefulSetServerInstanceObserver(sts *appsv1.StatefulSet) serverInstanceWorkloadObserver { + return serverInstanceWorkloadObserver{ + containerNames: workloadContainerNames(sts.Spec.Template.Spec.Containers), + converged: func(readyCount int) bool { + wantReplicas := int32(1) + if sts.Spec.Replicas != nil { + wantReplicas = *sts.Spec.Replicas + } + return sts.Status.ObservedGeneration >= sts.Generation && + sts.Status.ReadyReplicas == wantReplicas && + sts.Status.UpdatedReplicas == wantReplicas && + int32(readyCount) == wantReplicas + }, + ownsPod: func(p *corev1.Pod) (bool, error) { + return podOwnedByStatefulSet(p, sts), nil + }, + } +} + +func workloadContainerNames(containers []corev1.Container) map[string]struct{} { + names := make(map[string]struct{}, len(containers)) + for _, c := range containers { + names[c.Name] = struct{}{} + } + return names +} + // containerRunSum returns the sum of restart counts across the cache- // server's own containers, scoped to the set of container names from -// the owned Deployment's pod template. Used to detect in-place +// the owned workload's pod template. Used to detect in-place // restarts of the cache-server container (kubelet respawning a crashed // container reuses the pod UID but increments restartCount). Sidecars // outside the owned set are excluded — see currentServerInstanceID's @@ -948,6 +1003,18 @@ func (r *CacheBackendReconciler) podOwnedByDeployment(ctx context.Context, reade return depRef.Name == dep.Name && depRef.UID == dep.UID, nil } +// podOwnedByStatefulSet reports whether pod is directly controller- +// owned by the given StatefulSet, matched on both name AND UID. Unlike +// Deployments, StatefulSets own their pods directly rather than through +// ReplicaSets, so the authoritative chain is one hop. +func podOwnedByStatefulSet(pod *corev1.Pod, sts *appsv1.StatefulSet) bool { + stsRef := metav1.GetControllerOf(pod) + if stsRef == nil || stsRef.Kind != "StatefulSet" || !ownerRefIsAppsV1(stsRef) { + return false + } + return stsRef.Name == sts.Name && stsRef.UID == sts.UID +} + // ownerRefIsAppsV1 reports whether the owner reference points at // apps/v1. OwnerReference.apiVersion is a required field, so a strict // equality match is the right shape — an empty value is invalid input diff --git a/internal/controller/cachebackend_server_restart_test.go b/internal/controller/cachebackend_server_restart_test.go index 29124dc7..99ff88bd 100644 --- a/internal/controller/cachebackend_server_restart_test.go +++ b/internal/controller/cachebackend_server_restart_test.go @@ -11,6 +11,7 @@ import ( dto "github.com/prometheus/client_model/go" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -199,7 +200,7 @@ func newCascadeRestartFixture(t *testing.T, opts ...func(*cascadeRestartFixture) c := fake.NewClientBuilder(). WithScheme(scheme). - WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &appsv1.Deployment{}, &corev1.PersistentVolumeClaim{}). + WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &appsv1.Deployment{}, &appsv1.StatefulSet{}, &corev1.PersistentVolumeClaim{}). WithObjects(f.backend, f.serverDep, f.serverRS, f.serverPod, f.engineDep, f.engineRS, f.enginePod). Build() f.r = &CacheBackendReconciler{ @@ -315,6 +316,110 @@ func TestReconcileServerInstance_UIDChangeCascadesEngineDeployment(t *testing.T) } } +func TestReconcileServerInstance_StatefulSetPodReplacementCascadesEngineDeployment(t *testing.T) { + f := newCascadeRestartFixture(t) + switchFixtureBackendToStatefulSet(t, f, 1) + + if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { + t.Fatalf("baseline wait = %v, want 0", wait) + } + f.reload(t) + baseline := serverInstanceID(f.serverPod) + if got := f.backend.Status.ObservedServerInstance; got != baseline { + t.Fatalf("baseline ObservedServerInstance = %q, want %q", got, baseline) + } + f.reloadEngineDep(t) + if _, ok := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { + t.Fatalf("engine deployment got cascade annotation on StatefulSet first observation") + } + + if err := f.r.Delete(context.Background(), f.serverPod); err != nil { + t.Fatalf("delete StatefulSet server pod: %v", err) + } + replacement := f.serverPod.DeepCopy() + replacement.ResourceVersion = "" + replacement.Name = "cache-0-replacement" + replacement.UID = "cache-sts-pod-uid-2" + if err := f.r.Create(context.Background(), replacement); err != nil { + t.Fatalf("create StatefulSet replacement pod: %v", err) + } + if err := f.r.Status().Update(context.Background(), replacement); err != nil { + t.Fatalf("ready StatefulSet replacement pod: %v", err) + } + f.serverPod = replacement + if err := setOwnedStatefulSetConverged(f, 1); err != nil { + t.Fatalf("set StatefulSet converged after pod replacement: %v", err) + } + + time.Sleep(60 * time.Millisecond) + if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { + t.Fatalf("replacement wait = %v, want 0", wait) + } + f.reload(t) + want := serverInstanceID(replacement) + if got := f.backend.Status.ObservedServerInstance; got != want { + t.Fatalf("ObservedServerInstance after StatefulSet replacement = %q, want %q", got, want) + } + f.reloadEngineDep(t) + if got := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != want { + t.Fatalf("cascade annotation after StatefulSet replacement = %q, want %q", got, want) + } + if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 1 { + t.Fatalf("cascade counter = %v, want 1", got) + } +} + +func TestReconcileServerInstance_ImmutableStatefulSetStorageDriftStillCascades(t *testing.T) { + f := newCascadeRestartFixture(t) + switchFixtureBackendToStatefulSet(t, f, 1) + + if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { + t.Fatalf("baseline wait = %v, want 0", wait) + } + f.reload(t) + baseline := serverInstanceID(f.serverPod) + if got := f.backend.Status.ObservedServerInstance; got != baseline { + t.Fatalf("baseline ObservedServerInstance = %q, want %q", got, baseline) + } + + liveBackend := f.backend.DeepCopy() + liveBackend.Generation = 2 + withPVC(liveBackend, "20Gi", nil) + if err := f.r.Update(context.Background(), liveBackend); err != nil { + t.Fatalf("add immutable StatefulSet storage drift: %v", err) + } + + livePod := &corev1.Pod{} + if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.serverPod.Name, Namespace: f.cacheNS}, livePod); err != nil { + t.Fatalf("get StatefulSet server pod: %v", err) + } + livePod.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: "lmcache-server", + Ready: true, + RestartCount: 1, + }} + if err := f.r.Status().Update(context.Background(), livePod); err != nil { + t.Fatalf("bump StatefulSet server restart count: %v", err) + } + + time.Sleep(60 * time.Millisecond) + reconcile(t, f.r, f.cacheName, f.cacheNS) + + want := fmt.Sprintf("%s:1", f.serverPod.UID) + f.reloadEngineDep(t) + if got := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != want { + t.Fatalf("cascade annotation during immutable StatefulSet storage drift = %q, want %q", got, want) + } + f.reload(t) + if got := f.backend.Status.ObservedServerInstance; got != want { + t.Fatalf("ObservedServerInstance during immutable StatefulSet storage drift = %q, want %q", got, want) + } + cond := findCondition(f.backend.Status.Conditions, conditionTypeReady) + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != conditionReasonImmutableStatefulSetStorage { + t.Fatalf("Ready condition = %+v, want False/%s", cond, conditionReasonImmutableStatefulSetStorage) + } +} + func TestReconcileServerInstance_RateLimitedSecondCascadeIsDeferred(t *testing.T) { f := newCascadeRestartFixture(t) f.r.MinServerRestartCascadeInterval = 1 * time.Hour // make the window effectively block any second cascade @@ -1239,6 +1344,95 @@ func setOwnedDeploymentConverged(f *cascadeRestartFixture, replicas int32) error return nil } +func switchFixtureBackendToStatefulSet(t *testing.T, f *cascadeRestartFixture, replicas int32) { + t.Helper() + liveBackend := f.backend.DeepCopy() + liveBackend.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + if err := f.r.Update(context.Background(), liveBackend); err != nil { + t.Fatalf("switch backend fixture to StatefulSet: %v", err) + } + f.reload(t) + + if err := f.r.Delete(context.Background(), f.serverDep); err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("delete Deployment fixture: %v", err) + } + if err := f.r.Delete(context.Background(), f.serverRS); err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("delete ReplicaSet fixture: %v", err) + } + + tru := true + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.cacheName, + Namespace: f.cacheNS, + UID: "cache-sts-uid", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: cachev1alpha1.GroupVersion.String(), + Kind: "CacheBackend", + Name: f.backend.Name, + UID: f.backend.UID, + Controller: &tru, + }}, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptrInt32(replicas), + ServiceName: f.cacheName, + Selector: &metav1.LabelSelector{MatchLabels: selectorLabels(f.cacheName)}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: selectorLabels(f.cacheName)}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "lmcache-server", Image: "lmcache:test"}}, + }, + }, + }, + } + if err := f.r.Create(context.Background(), sts); err != nil { + t.Fatalf("create StatefulSet fixture: %v", err) + } + + pod := f.serverPod.DeepCopy() + pod.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: sts.Name, + UID: sts.UID, + Controller: &tru, + }} + if err := f.r.Update(context.Background(), pod); err != nil { + t.Fatalf("move server pod ownerRef to StatefulSet: %v", err) + } + if err := f.r.Status().Update(context.Background(), pod); err != nil { + t.Fatalf("preserve server pod status after StatefulSet ownerRef switch: %v", err) + } + f.serverPod = pod + + if err := setOwnedStatefulSetConverged(f, replicas); err != nil { + t.Fatalf("set StatefulSet fixture converged: %v", err) + } +} + +func setOwnedStatefulSetConverged(f *cascadeRestartFixture, replicas int32) error { + live := &appsv1.StatefulSet{} + if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.cacheName, Namespace: f.cacheNS}, live); err != nil { + return fmt.Errorf("get live StatefulSet: %w", err) + } + live.Spec.Replicas = ptrInt32(replicas) + if err := f.r.Update(context.Background(), live); err != nil { + return fmt.Errorf("update StatefulSet.Spec.Replicas: %w", err) + } + if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.cacheName, Namespace: f.cacheNS}, live); err != nil { + return fmt.Errorf("reload live StatefulSet: %w", err) + } + live.Status.ReadyReplicas = replicas + live.Status.UpdatedReplicas = replicas + live.Status.Replicas = replicas + live.Status.ObservedGeneration = live.Generation + if err := f.r.Status().Update(context.Background(), live); err != nil { + return fmt.Errorf("update StatefulSet.Status: %w", err) + } + return nil +} + // TestReconcileServerInstance_InPlaceContainerRestartCascades asserts // that an in-place container restart inside the cache-server pod // (kubelet respawning a crashed container, e.g. on OOM with diff --git a/internal/controller/cachebackend_storage_test.go b/internal/controller/cachebackend_storage_test.go index 784bb548..ee7eaf4e 100644 --- a/internal/controller/cachebackend_storage_test.go +++ b/internal/controller/cachebackend_storage_test.go @@ -4,7 +4,10 @@ import ( "context" "strings" "testing" + "time" + appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -341,6 +344,188 @@ func TestReconcileMultiReplicaViaAutoscalingGated(t *testing.T) { expectEvent(t, drainEvents(rec), conditionReasonInvalidStorageConfiguration) } +func TestReconcileStatefulSetPersistentStorageUsesVolumeClaimTemplates(t *testing.T) { + scheme := newScheme(t) + fast := "fast" + cb := lmcacheBackend("cache", "ns1") + cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + cb.Spec.Replicas = ptrInt32(3) + withPVC(cb, "10Gi", &fast) + r := newReconciler(scheme, cb) + + reconcile(t, r, "cache", "ns1") + + if _, err := getOptionalDeployment(t, r, "cache", "ns1"); !apierrors.IsNotFound(err) { + t.Fatalf("StatefulSet persistent backend must not provision a Deployment; get err=%v", err) + } + if _, err := getOptionalPVC(r, "cache-data", "ns1"); !apierrors.IsNotFound(err) { + t.Fatalf("StatefulSet persistent backend must not provision a shared PVC; get err=%v", err) + } + sts := getStatefulSet(t, r, "cache", "ns1") + if sts.Spec.Replicas == nil || *sts.Spec.Replicas != 3 { + t.Fatalf("statefulset replicas = %v, want 3", sts.Spec.Replicas) + } + retention := sts.Spec.PersistentVolumeClaimRetentionPolicy + if retention == nil || + retention.WhenDeleted != appsv1.RetainPersistentVolumeClaimRetentionPolicyType || + retention.WhenScaled != appsv1.RetainPersistentVolumeClaimRetentionPolicyType { + t.Fatalf("StatefulSet persistentVolumeClaimRetentionPolicy = %+v, want Retain/Retain", retention) + } + if len(sts.Spec.VolumeClaimTemplates) != 1 { + t.Fatalf("volumeClaimTemplates = %d, want 1: %#v", len(sts.Spec.VolumeClaimTemplates), sts.Spec.VolumeClaimTemplates) + } + tpl := sts.Spec.VolumeClaimTemplates[0] + if tpl.Name == "" { + t.Fatalf("volumeClaimTemplate name is empty") + } + if got := tpl.Spec.Resources.Requests[corev1.ResourceStorage]; got.Cmp(resource.MustParse("10Gi")) != 0 { + t.Fatalf("volumeClaimTemplate size = %q, want 10Gi", got.String()) + } + if tpl.Spec.StorageClassName == nil || *tpl.Spec.StorageClassName != "fast" { + t.Fatalf("volumeClaimTemplate storageClassName = %v, want fast", tpl.Spec.StorageClassName) + } + mounts := sts.Spec.Template.Spec.Containers[0].VolumeMounts + found := false + for _, mount := range mounts { + if mount.Name == tpl.Name && strings.HasPrefix(mount.MountPath, "/") { + found = true + break + } + } + if !found { + t.Fatalf("lmcache-server mounts = %v, want absolute mount for volumeClaimTemplate %q", mounts, tpl.Name) + } + cond := findCondition(getBackend(t, r, "cache", "ns1").Status.Conditions, conditionTypeReady) + if cond != nil && cond.Reason == conditionReasonInvalidStorageConfiguration { + t.Fatalf("StatefulSet per-replica PVC backend must not be gated as InvalidStorageConfiguration") + } +} + +func TestReconcileStatefulSetRepairsPVCRetentionPolicy(t *testing.T) { + cb := lmcacheBackend("cache", "ns1") + cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + cb.Spec.Replicas = ptrInt32(2) + withPVC(cb, "10Gi", nil) + r := newReconciler(newScheme(t), cb) + + reconcile(t, r, "cache", "ns1") + + sts := getStatefulSet(t, r, "cache", "ns1") + sts.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + } + if err := r.Update(context.Background(), sts); err != nil { + t.Fatalf("drift statefulset retention policy: %v", err) + } + + reconcile(t, r, "cache", "ns1") + + repaired := getStatefulSet(t, r, "cache", "ns1") + retention := repaired.Spec.PersistentVolumeClaimRetentionPolicy + if retention == nil || + retention.WhenDeleted != appsv1.RetainPersistentVolumeClaimRetentionPolicyType || + retention.WhenScaled != appsv1.RetainPersistentVolumeClaimRetentionPolicyType { + t.Fatalf("repaired persistentVolumeClaimRetentionPolicy = %+v, want Retain/Retain", retention) + } +} + +func TestStatefulSetVolumeClaimTemplateComparisonIgnoresAPIServerDefaults(t *testing.T) { + volumeMode := corev1.PersistentVolumeFilesystem + live := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "cache-data"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("10Gi")}, + }, + VolumeMode: &volumeMode, + }, + } + desired := live.DeepCopy() + desired.Spec.VolumeMode = nil + + if !statefulSetVolumeClaimTemplatesEqual([]corev1.PersistentVolumeClaim{live}, []corev1.PersistentVolumeClaim{*desired}) { + t.Fatalf("defaulted volumeMode=Filesystem must not count as immutable StatefulSet storage drift") + } + + changed := desired.DeepCopy() + changed.Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("20Gi") + if statefulSetVolumeClaimTemplatesEqual([]corev1.PersistentVolumeClaim{live}, []corev1.PersistentVolumeClaim{*changed}) { + t.Fatalf("changed storage request must still count as immutable StatefulSet storage drift") + } +} + +func TestReconcileStatefulSetStorageEditSurfacesImmutableDrift(t *testing.T) { + scheme := newScheme(t) + cb := lmcacheBackend("cache", "ns1") + cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + r := newReconciler(scheme, cb) + + reconcile(t, r, "cache", "ns1") + sts := getStatefulSet(t, r, "cache", "ns1") + if len(sts.Spec.VolumeClaimTemplates) != 0 { + t.Fatalf("initial volumeClaimTemplates = %d, want none", len(sts.Spec.VolumeClaimTemplates)) + } + + fresh := getBackend(t, r, "cache", "ns1") + fresh.Generation = 2 + withPVC(fresh, "20Gi", nil) + if err := r.Update(context.Background(), fresh); err != nil { + t.Fatalf("update backend storage: %v", err) + } + + reconcile(t, r, "cache", "ns1") + + sts = getStatefulSet(t, r, "cache", "ns1") + if len(sts.Spec.VolumeClaimTemplates) != 0 { + t.Fatalf("StatefulSet volumeClaimTemplates were mutated after creation: %#v", sts.Spec.VolumeClaimTemplates) + } + for _, mount := range sts.Spec.Template.Spec.Containers[0].VolumeMounts { + if mount.Name == "cache-data" { + t.Fatalf("pod template mounted new immutable storage claim after creation: mounts=%v", sts.Spec.Template.Spec.Containers[0].VolumeMounts) + } + } + updated := getBackend(t, r, "cache", "ns1") + cond := findCondition(updated.Status.Conditions, conditionTypeReady) + wantReason := conditionReasonImmutableStatefulSetStorage + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != wantReason { + t.Fatalf("Ready condition = %+v, want False/%s", cond, wantReason) + } + if updated.Status.ObservedGeneration != 2 { + t.Fatalf("status.observedGeneration = %d, want 2 so the immutable-storage condition is tied to the edited spec", updated.Status.ObservedGeneration) + } +} + +func TestReconcileStatefulSetImmutableStorageDriftStillCleansUpHPA(t *testing.T) { + scheme := newScheme(t) + cb := autoscalingBackend("cache", "ns1", 1, 3, nil) + cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + withPVC(cb, "10Gi", nil) + r := newReconciler(scheme, cb) + + reconcile(t, r, "cache", "ns1") + _ = getHPA(t, r, "cache", "ns1") + + fresh := getBackend(t, r, "cache", "ns1") + fresh.Generation = 2 + fresh.Spec.Autoscaling = nil + withPVC(fresh, "20Gi", nil) + if err := r.Update(context.Background(), fresh); err != nil { + t.Fatalf("update backend autoscaling/storage: %v", err) + } + + reconcile(t, r, "cache", "ns1") + + var hpas autoscalingv2.HorizontalPodAutoscalerList + if err := r.List(context.Background(), &hpas); err != nil { + t.Fatalf("list HPAs: %v", err) + } + if len(hpas.Items) != 0 { + t.Fatalf("HPAs = %d, want 0 after autoscaling cleared during immutable StatefulSet storage drift", len(hpas.Items)) + } +} + func TestReconcileStorageRemovedViaKindSwitchStillWarnsAndKeepsPVC(t *testing.T) { cb := lmcacheBackend("cache", "ns1") cb.Spec.Replicas = ptrInt32(1) @@ -351,9 +536,8 @@ func TestReconcileStorageRemovedViaKindSwitchStillWarnsAndKeepsPVC(t *testing.T) getPVC(t, r, "cache-data", "ns1") drainEvents(rec) - // Operator removes storage AND switches to StatefulSet, which dispatch routes - // to the unmanaged path — bypassing reconcileManaged entirely. The adopt-and- - // keep warning must still fire (it lives in dispatch), and the PVC must stay. + // Operator removes storage AND switches to StatefulSet. The adopt-and-keep + // warning must still fire (it lives in dispatch), and the PVC must stay. fresh := getBackend(t, r, "cache", "ns1") fresh.Spec.Storage = nil fresh.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet @@ -370,6 +554,96 @@ func TestReconcileStorageRemovedViaKindSwitchStillWarnsAndKeepsPVC(t *testing.T) } } +func TestReconcileStorageKeptViaStatefulSetSwitchWarnsAndKeepsDeploymentPVC(t *testing.T) { + cb := lmcacheBackend("cache", "ns1") + cb.Spec.Replicas = ptrInt32(1) + withPVC(cb, "10Gi", nil) + r, rec := newReconcilerWithRecorder(t, cb) + + reconcile(t, r, "cache", "ns1") // provisions the shared Deployment PVC + getPVC(t, r, "cache-data", "ns1") + drainEvents(rec) + + fresh := getBackend(t, r, "cache", "ns1") + fresh.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + fresh.Spec.Replicas = ptrInt32(2) + if err := r.Update(context.Background(), fresh); err != nil { + t.Fatalf("switch to StatefulSet kind: %v", err) + } + reconcile(t, r, "cache", "ns1") + + if _, err := getOptionalPVC(r, "cache-data", "ns1"); err != nil { + t.Fatalf("prior shared Deployment PVC must be retained across StatefulSet switch; get err=%v", err) + } + wantReason := eventReasonSharedPVCRetained + if n := countEvents(drainEvents(rec), wantReason); n != 1 { + t.Fatalf("SharedPVCRetained events after StatefulSet switch = %d, want exactly 1", n) + } + + reconcile(t, r, "cache", "ns1") + if n := countEvents(drainEvents(rec), wantReason); n != 0 { + t.Fatalf("SharedPVCRetained re-fired on resync = %d times, want 0", n) + } +} + +func TestReconcileRetainedPVCWarningsAreReasonScoped(t *testing.T) { + cb := lmcacheBackend("cache", "ns1") + cb.Spec.Replicas = ptrInt32(1) + withPVC(cb, "10Gi", nil) + r, rec := newReconcilerWithRecorder(t, cb) + + reconcile(t, r, "cache", "ns1") + getPVC(t, r, "cache-data", "ns1") + drainEvents(rec) + + removed := getBackend(t, r, "cache", "ns1") + removed.Spec.Storage = nil + removed.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + if err := r.Update(context.Background(), removed); err != nil { + t.Fatalf("remove storage and switch to StatefulSet: %v", err) + } + reconcile(t, r, "cache", "ns1") + if n := countEvents(drainEvents(rec), eventReasonOrphanedPVCRetained); n != 1 { + t.Fatalf("OrphanedPVCRetained events after storage removal = %d, want 1", n) + } + + readded := getBackend(t, r, "cache", "ns1") + withPVC(readded, "10Gi", nil) + readded.Spec.Replicas = ptrInt32(2) + if err := r.Update(context.Background(), readded); err != nil { + t.Fatalf("re-add storage while staying StatefulSet: %v", err) + } + reconcile(t, r, "cache", "ns1") + if n := countEvents(drainEvents(rec), eventReasonSharedPVCRetained); n != 1 { + t.Fatalf("SharedPVCRetained events after storage re-add on StatefulSet path = %d, want 1", n) + } +} + +func TestReconcileImmutableStatefulSetStorageClearsProbeRateLimiter(t *testing.T) { + cb := lmcacheBackend("cache", "ns1") + cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + r := newReconciler(newScheme(t), cb) + + reconcile(t, r, "cache", "ns1") + key := "ns1/cache" + now := time.Unix(2_000_000, 0) + r.probeLimiter.markCalled(key, now) + if got := r.probeLimiter.lastCalled(key); got != now { + t.Fatalf("planted rate-limit precondition failed: lastCalled = %v, want %v", got, now) + } + + fresh := getBackend(t, r, "cache", "ns1") + withPVC(fresh, "20Gi", nil) + if err := r.Update(context.Background(), fresh); err != nil { + t.Fatalf("update backend storage: %v", err) + } + reconcile(t, r, "cache", "ns1") + + if got := r.probeLimiter.lastCalled(key); !got.IsZero() { + t.Fatalf("probeLimiter.lastCalled(%q) = %v after immutable StatefulSet storage drift; want zero", key, got) + } +} + func TestReconcilePersistentSingleToMultiReplicaShedsWorkloadAndEndpoint(t *testing.T) { cb := lmcacheBackend("cache", "ns1") cb.Spec.Replicas = ptrInt32(1) diff --git a/internal/controller/integration_test.go b/internal/controller/integration_test.go index f139d6bf..b3aaa572 100644 --- a/internal/controller/integration_test.go +++ b/internal/controller/integration_test.go @@ -647,7 +647,7 @@ func TestIntegrationCacheBackendReconcile(t *testing.T) { } }) - t.Run("SwitchToStatefulSetKindCleansUpAndClearsStatus", func(t *testing.T) { + t.Run("SwitchToStatefulSetKindCreatesStatefulSetAndShedsDeployment", func(t *testing.T) { ns := freshNS(t, k8s) if err := k8s.Create(ctx, lmcacheBackend("cache", ns)); err != nil { t.Fatalf("create: %v", err) @@ -667,12 +667,16 @@ func TestIntegrationCacheBackendReconcile(t *testing.T) { if _, err := getOptionalDeployment(t, r, "cache", ns); err == nil { t.Fatalf("deployment should be deleted after switch to StatefulSet kind") } + sts := getStatefulSet(t, r, "cache", ns) + if sts.Spec.ServiceName != "cache" { + t.Fatalf("statefulset serviceName = %q, want cache", sts.Spec.ServiceName) + } cb := getBackend(t, r, "cache", ns) - if cb.Status.Endpoint != "" { - t.Fatalf("status.endpoint = %q, want cleared", cb.Status.Endpoint) + if cb.Status.Endpoint != "cache."+ns+".svc.cluster.local:65432" { + t.Fatalf("status.endpoint = %q, want service endpoint", cb.Status.Endpoint) } - if cond := findCondition(cb.Status.Conditions, conditionTypeReady); cond != nil { - t.Fatalf("Ready condition = %+v, want removed", cond) + if cond := findCondition(cb.Status.Conditions, conditionTypeReady); cond == nil || cond.Reason != conditionReasonRolloutInProgress { + t.Fatalf("Ready condition = %+v, want rollout status from StatefulSet", cond) } }) diff --git a/internal/webhook/v1alpha1/cachebackend_webhook.go b/internal/webhook/v1alpha1/cachebackend_webhook.go index 34c15812..ffacbff6 100644 --- a/internal/webhook/v1alpha1/cachebackend_webhook.go +++ b/internal/webhook/v1alpha1/cachebackend_webhook.go @@ -334,6 +334,7 @@ func (v *CacheBackendValidator) collectErrors(cb *cachev1alpha1.CacheBackend) fi errs = append(errs, rule(cb)...) } errs = append(errs, v.checkRuntimeAdapter(cb)...) + errs = append(errs, v.checkStatefulSetVolumeClaimTemplateNames(cb)...) errs = append(errs, v.checkEngineOverrides(cb)...) return errs } @@ -900,6 +901,75 @@ func rejectStorageWithOverlongName(cb *cachev1alpha1.CacheBackend) field.ErrorLi } } +// checkStatefulSetVolumeClaimTemplateNames rejects persistent StatefulSet +// backends whose generated per-replica PVC name would exceed Kubernetes' +// object-name limit. The controller uses the adapter-declared DataVolume name +// as the volumeClaimTemplate name, and the StatefulSet controller expands that +// into "--". The legacy +// rejectStorageWithOverlongName rule only guards the Deployment shared-PVC +// shape ("-data"), so StatefulSet storage needs this adapter-aware +// check at admission rather than discovering the invalid name when the +// StatefulSet later fails to materialize PVCs. +func (v *CacheBackendValidator) checkStatefulSetVolumeClaimTemplateNames(cb *cachev1alpha1.CacheBackend) field.ErrorList { + if cb.Spec.Storage == nil || cb.Spec.Storage.PVC == nil { + return nil + } + if cb.Spec.DeploymentKind != cachev1alpha1.CacheBackendDeploymentKindStatefulSet { + return nil + } + if cb.Spec.Type == "" { + return nil + } + + registry := v.Registry + if registry == nil { + registry = defaultShippingRegistry() + } + adapter, err := registry.Select(adapterruntime.ResolveRuntimeID(cb), cb) + if err != nil { + // checkRuntimeAdapter already reports unsupported runtime/backend pairs. + return nil + } + resolved, err := adapter.ResolveCacheServer(cb) + if err != nil { + return field.ErrorList{ + field.InternalError( + field.NewPath("spec", "storage", "pvc"), + fmt.Errorf("resolve cache server for StatefulSet PVC name validation: %w", err), + ), + } + } + if resolved == nil || resolved.DataVolume == nil || resolved.DataVolume.VolumeName == "" { + return nil + } + + claimName := fmt.Sprintf("%s-%s-%s", + resolved.DataVolume.VolumeName, + cb.Name, + statefulSetHighestOrdinal(cb), + ) + if len(claimName) <= maxK8sNameLen { + return nil + } + return field.ErrorList{ + field.Invalid( + field.NewPath("metadata", "name"), + cb.Name, + fmt.Sprintf("too long for a persistent StatefulSet backend: the derived per-replica PVC name %q would exceed the %d-character limit; shorten the CacheBackend name, lower the replica ceiling, or omit spec.storage.pvc", claimName, maxK8sNameLen), + ), + } +} + +func statefulSetHighestOrdinal(cb *cachev1alpha1.CacheBackend) string { + replicas := int32(1) + if cb.Spec.Autoscaling != nil && cb.Spec.Autoscaling.MaxReplicas > 0 { + replicas = cb.Spec.Autoscaling.MaxReplicas + } else if cb.Spec.Replicas != nil && *cb.Spec.Replicas > 0 { + replicas = *cb.Spec.Replicas + } + return fmt.Sprintf("%d", replicas-1) +} + // rejectCrossNamespaceEndpointWithoutOptIn rejects an Endpoint that // resolves into a Service in a namespace other than the CacheBackend's // own, unless spec.allowCrossNamespace is true. Crossing a namespace is diff --git a/internal/webhook/v1alpha1/cachebackend_webhook_test.go b/internal/webhook/v1alpha1/cachebackend_webhook_test.go index a13c3aa9..cec60495 100644 --- a/internal/webhook/v1alpha1/cachebackend_webhook_test.go +++ b/internal/webhook/v1alpha1/cachebackend_webhook_test.go @@ -448,6 +448,19 @@ func TestValidator_StorageWithOverlongNameRejected(t *testing.T) { if _, err := v.ValidateCreate(context.Background(), ok); err != nil { t.Fatalf("normal-length persistent backend rejected: %v", err) } + + // StatefulSet storage uses volumeClaimTemplates, and Kubernetes expands + // each per-replica PVC name as --. + // This name is shorter than the Deployment PVC limit above (241+5=246), + // but cache-data + name + ordinal would be 254 chars. + stateful := newBackend() + stateful.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache + stateful.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + stateful.Name = strings.Repeat("s", 241) + stateful.Spec.Storage = &cachev1alpha1.CacheBackendStorageSpec{ + PVC: &cachev1alpha1.CacheBackendPVCSpec{Size: resource.MustParse("10Gi")}, + } + requireInvalidWithCause(t, v, stateful, "metadata.name", "per-replica PVC name") } func TestValidator_ResourcesLimitsBelowRequestsRejected(t *testing.T) {