Replace Redis with Garage S3 and migrate workflow scaling to metrics API

AGENTS.md

@@ -0,0 +1,68 @@
+# Dev
+
+This repository manages the local Kubernetes development environment for Virtool using Tilt and Minikube.
+
+## Stack
+
+- **Tilt** — orchestrates the dev cluster; the `Tiltfile` is the main entry point
+- **Minikube** — local Kubernetes cluster
+- **KEDA** — scales workflow jobs via a `metrics-api` trigger pointing at `http://virtool-api-web-service.../jobs/counts`
+- **MongoDB + PostgreSQL** — persisted across `tilt up`/`tilt down` runs via PVCs
+- **Azurite** — Azure Blob Storage emulator; uses the well-known dev account `devstoreaccount1` with key `Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==` and persists blobs at `/data` via a PVC
+
+## Layout
+
+```
+manifests/       Kustomize manifests for all cluster resources
+  db/            MongoDB, PostgreSQL, Azurite
+  ingress.yaml
+  migration.yaml
+  storage.yaml
+  ui/
+  virtool/
+  workflows/     ScaledJob manifests for each workflow
+scripts/
+  init.sh        Create/reset the Minikube cluster
+  hosts.sh       Write cluster IP to /etc/hosts
+  pull.sh        Update image tags in kustomization + migration manifests
+  wipe.sh        Nuke PVCs, PVs, and host data
+Tiltfile         Tilt configuration (resources, buttons, live-edit flags)
+```
+
+## Common tasks
+
+**Start the cluster**
+```shell
+bash scripts/init.sh   # first time or full reset
+tilt up
+```
+
+**Live-edit a service** (repo must be a sibling of this directory)
+```shell
+tilt up -- --to-edit backend   # virtool/virtool
+tilt up -- --to-edit ui        # virtool/virtool-ui
+tilt up -- --to-edit <workflow-name>
+```
+
+Workflow names: `build-index`, `create-sample`, `create-subtraction`, `iimi`, `pathoscope`, `nuvs`
+
+**Update images**
+```shell
+bash scripts/pull.sh
+```
+Or click the **Pull** button in the Tilt UI.
+
+**Wipe persistent data**
+```shell
+bash scripts/wipe.sh
+```
+Or click the **Wipe** button in the Tilt UI (requires confirmation).
+
+**Update `/etc/hosts`** (needed once per cluster recreation)
+```shell
+bash scripts/hosts.sh
+```
+
+## PR guidelines
+
+- No test plan section in PR descriptions.

CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

README.md

@@ -62,7 +62,7 @@
    ```
 
    Tilt manages the Kubernetes development environment. It starts all necessary services
-   (KEDA, MongoDB, PostgreSQL, Redis) and the Virtool workloads and services.
+   (KEDA, MongoDB, PostgreSQL) and the Virtool workloads and services.
 
 ## Tilt
 

Tiltfile

@@ -16,6 +16,14 @@ cmd_button('pull',
     text='Pull',
 )
 
+cmd_button('wipe',
+    argv=['bash', 'scripts/wipe.sh'],
+    icon_name="delete_forever",
+    location=location.NAV,
+    text='Wipe',
+    requires_confirmation=True,
+)
+
 helm_repo('kedacore', 'https://kedacore.github.io/charts', labels=['helm'])
 
 helm_resource(
@@ -25,9 +33,9 @@ helm_resource(
     resource_deps=['kedacore']
 )
 
+k8s_yaml('manifests/db/azurite.yaml')
 k8s_yaml('manifests/db/mongo.yaml')
 k8s_yaml('manifests/db/postgres.yaml')
-k8s_yaml('manifests/db/redis.yaml')
 k8s_yaml('manifests/storage.yaml')
 
 k8s_resource(
@@ -38,9 +46,9 @@ k8s_resource(
     labels=['data']
 )
 
+k8s_resource("azurite", labels=['data'])
 k8s_resource("mongo", labels=['data'])
 k8s_resource("postgres", labels=['data'])
-k8s_resource('redis', labels=['data'])
 
 if 'migration' in to_edit:
     docker_build('ghcr.io/virtool/migration', '../virtool-migration/')
@@ -102,7 +110,7 @@ k8s_resource(
     'virtool-migration',
     labels=['virtool'],
     new_name="migration",
-    resource_deps=["mongo", "postgres", "redis", "storage"],
+    resource_deps=["azurite", "mongo", "postgres", "storage"],
     trigger_mode=TRIGGER_MODE_MANUAL
 )
 
@@ -173,7 +181,7 @@ k8s_kind(
 
 k8s_yaml(kustomize('manifests/workflows'))
 
-scaled_job_deps = ['keda', 'redis']
+scaled_job_deps = ['keda', 'api-web']
 
 k8s_resource(
     "virtool-workflow-build-index",

manifests/db/azurite.yaml

@@ -0,0 +1,114 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: azurite
+  labels:
+    app: azurite
+spec:
+  serviceName: azurite
+  replicas: 1
+  selector:
+    matchLabels:
+      app: azurite
+  template:
+    metadata:
+      labels:
+        app: azurite
+    spec:
+      initContainers:
+        - name: azurite-init
+          image: mcr.microsoft.com/azure-cli:latest
+          restartPolicy: Always
+          env:
+            - name: AZURE_STORAGE_CONNECTION_STRING
+              value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
+          command:
+            - /bin/sh
+            - -c
+            - |
+              set -eu
+              until az storage container create --name virtool --output none 2>/dev/null; do sleep 2; done
+              touch /tmp/ready
+              exec sleep infinity
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 200m
+              memory: 256Mi
+          readinessProbe:
+            exec:
+              command: ["test", "-f", "/tmp/ready"]
+            periodSeconds: 2
+      containers:
+        - name: azurite
+          image: mcr.microsoft.com/azure-storage/azurite
+          args:
+            - azurite
+            - --blobHost
+            - 0.0.0.0
+            - --queueHost
+            - 0.0.0.0
+            - --tableHost
+            - 0.0.0.0
+            - --location
+            - /data
+            - --skipApiVersionCheck
+            - --disableProductStyleUrl
+          ports:
+            - containerPort: 10000
+              name: blob
+            - containerPort: 10001
+              name: queue
+            - containerPort: 10002
+              name: table
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 250m
+              memory: 256Mi
+          readinessProbe:
+            tcpSocket:
+              port: 10000
+            initialDelaySeconds: 2
+            periodSeconds: 5
+          livenessProbe:
+            tcpSocket:
+              port: 10000
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          volumeMounts:
+            - name: data
+              mountPath: /data
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 5Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: azurite
+spec:
+  selector:
+    app: azurite
+  ports:
+    - name: blob
+      protocol: TCP
+      port: 10000
+      targetPort: 10000
+    - name: queue
+      protocol: TCP
+      port: 10001
+      targetPort: 10001
+    - name: table
+      protocol: TCP
+      port: 10002
+      targetPort: 10002

manifests/migration.yaml

@@ -13,7 +13,7 @@ spec:
     spec:
       restartPolicy: Never
       containers:
-        - image: ghcr.io/virtool/virtool:36.0.0
+        - image: ghcr.io/virtool/virtool:36.25.3
           name: virtool-migration
           command: ["virtool", "migration", "apply"]
           env:
@@ -27,8 +27,6 @@ spec:
               value: "mongodb://virtool:virtool@mongo.default.svc.cluster.local/virtool"
             - name: VT_POSTGRES_CONNECTION_STRING
               value: "postgresql+asyncpg://virtool@postgres.default.svc.cluster.local?password=virtool"
-            - name: VT_REDIS_CONNECTION_STRING
-              value: "redis://:virtool@redis.default.svc.cluster.local"
           imagePullPolicy: Always
           resources:
             limits:

manifests/ui/kustomization.yaml

@@ -1,6 +1,6 @@
 images:
   - name: ghcr.io/virtool/ui
-    newTag: 7.5.2
+    newTag: 7.13.3
 
 labels:
   - pairs:

manifests/virtool/base/deployment.yaml

@@ -21,10 +21,18 @@ spec:
               value: "0.0.0.0"
             - name: VT_MONGODB_CONNECTION_STRING
               value: "mongodb://virtool:virtool@mongo.default.svc.cluster.local/virtool"
-            - name: VT_REDIS_CONNECTION_STRING
-              value: "redis://:virtool@redis.default.svc.cluster.local"
             - name: VT_POSTGRES_CONNECTION_STRING
               value: "postgresql+asyncpg://virtool@postgres.default.svc.cluster.local?password=virtool"
+            - name: VT_STORAGE_BACKEND
+              value: "azure"
+            - name: VT_STORAGE_AZURE_ACCOUNT
+              value: "devstoreaccount1"
+            - name: VT_STORAGE_AZURE_CONTAINER
+              value: "virtool"
+            - name: VT_STORAGE_AZURE_ACCESS_KEY
+              value: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+            - name: VT_STORAGE_AZURE_ENDPOINT
+              value: "http://azurite.default.svc.cluster.local:10000/devstoreaccount1"
           ports:
             - containerPort: 9950
               protocol: TCP

manifests/virtool/base/kustomization.yaml

@@ -1,6 +1,6 @@
 images:
   - name: ghcr.io/virtool/virtool
-    newTag: 36.0.0
+    newTag: 36.25.3
 
 labels:
   - pairs: