diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..1ee28f0 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,68 @@ +# Dev + +This repository manages the local Kubernetes development environment for Virtool using Tilt and Minikube. + +## Stack + +- **Tilt** — orchestrates the dev cluster; the `Tiltfile` is the main entry point +- **Minikube** — local Kubernetes cluster +- **KEDA** — scales workflow jobs via a `metrics-api` trigger pointing at `http://virtool-api-web-service.../jobs/counts` +- **MongoDB + PostgreSQL** — persisted across `tilt up`/`tilt down` runs via PVCs +- **Azurite** — Azure Blob Storage emulator; uses the well-known dev account `devstoreaccount1` with key `Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==` and persists blobs at `/data` via a PVC + +## Layout + +``` +manifests/ Kustomize manifests for all cluster resources + db/ MongoDB, PostgreSQL, Azurite + ingress.yaml + migration.yaml + storage.yaml + ui/ + virtool/ + workflows/ ScaledJob manifests for each workflow +scripts/ + init.sh Create/reset the Minikube cluster + hosts.sh Write cluster IP to /etc/hosts + pull.sh Update image tags in kustomization + migration manifests + wipe.sh Nuke PVCs, PVs, and host data +Tiltfile Tilt configuration (resources, buttons, live-edit flags) +``` + +## Common tasks + +**Start the cluster** +```shell +bash scripts/init.sh # first time or full reset +tilt up +``` + +**Live-edit a service** (repo must be a sibling of this directory) +```shell +tilt up -- --to-edit backend # virtool/virtool +tilt up -- --to-edit ui # virtool/virtool-ui +tilt up -- --to-edit +``` + +Workflow names: `build-index`, `create-sample`, `create-subtraction`, `iimi`, `pathoscope`, `nuvs` + +**Update images** +```shell +bash scripts/pull.sh +``` +Or click the **Pull** button in the Tilt UI. + +**Wipe persistent data** +```shell +bash scripts/wipe.sh +``` +Or click the **Wipe** button in the Tilt UI (requires confirmation). + +**Update `/etc/hosts`** (needed once per cluster recreation) +```shell +bash scripts/hosts.sh +``` + +## PR guidelines + +- No test plan section in PR descriptions. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/README.md b/README.md index 05a6385..e048f1e 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ ``` Tilt manages the Kubernetes development environment. It starts all necessary services - (KEDA, MongoDB, PostgreSQL, Redis) and the Virtool workloads and services. + (KEDA, MongoDB, PostgreSQL) and the Virtool workloads and services. ## Tilt diff --git a/Tiltfile b/Tiltfile index 9eb04ce..eaa7691 100644 --- a/Tiltfile +++ b/Tiltfile @@ -16,6 +16,14 @@ cmd_button('pull', text='Pull', ) +cmd_button('wipe', + argv=['bash', 'scripts/wipe.sh'], + icon_name="delete_forever", + location=location.NAV, + text='Wipe', + requires_confirmation=True, +) + helm_repo('kedacore', 'https://kedacore.github.io/charts', labels=['helm']) helm_resource( @@ -25,9 +33,9 @@ helm_resource( resource_deps=['kedacore'] ) +k8s_yaml('manifests/db/azurite.yaml') k8s_yaml('manifests/db/mongo.yaml') k8s_yaml('manifests/db/postgres.yaml') -k8s_yaml('manifests/db/redis.yaml') k8s_yaml('manifests/storage.yaml') k8s_resource( @@ -38,9 +46,9 @@ k8s_resource( labels=['data'] ) +k8s_resource("azurite", labels=['data']) k8s_resource("mongo", labels=['data']) k8s_resource("postgres", labels=['data']) -k8s_resource('redis', labels=['data']) if 'migration' in to_edit: docker_build('ghcr.io/virtool/migration', '../virtool-migration/') @@ -102,7 +110,7 @@ k8s_resource( 'virtool-migration', labels=['virtool'], new_name="migration", - resource_deps=["mongo", "postgres", "redis", "storage"], + resource_deps=["azurite", "mongo", "postgres", "storage"], trigger_mode=TRIGGER_MODE_MANUAL ) @@ -173,7 +181,7 @@ k8s_kind( k8s_yaml(kustomize('manifests/workflows')) -scaled_job_deps = ['keda', 'redis'] +scaled_job_deps = ['keda', 'api-web'] k8s_resource( "virtool-workflow-build-index", diff --git a/manifests/db/azurite.yaml b/manifests/db/azurite.yaml new file mode 100644 index 0000000..70bb02c --- /dev/null +++ b/manifests/db/azurite.yaml @@ -0,0 +1,114 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: azurite + labels: + app: azurite +spec: + serviceName: azurite + replicas: 1 + selector: + matchLabels: + app: azurite + template: + metadata: + labels: + app: azurite + spec: + initContainers: + - name: azurite-init + image: mcr.microsoft.com/azure-cli:latest + restartPolicy: Always + env: + - name: AZURE_STORAGE_CONNECTION_STRING + value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" + command: + - /bin/sh + - -c + - | + set -eu + until az storage container create --name virtool --output none 2>/dev/null; do sleep 2; done + touch /tmp/ready + exec sleep infinity + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + readinessProbe: + exec: + command: ["test", "-f", "/tmp/ready"] + periodSeconds: 2 + containers: + - name: azurite + image: mcr.microsoft.com/azure-storage/azurite + args: + - azurite + - --blobHost + - 0.0.0.0 + - --queueHost + - 0.0.0.0 + - --tableHost + - 0.0.0.0 + - --location + - /data + - --skipApiVersionCheck + - --disableProductStyleUrl + ports: + - containerPort: 10000 + name: blob + - containerPort: 10001 + name: queue + - containerPort: 10002 + name: table + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi + readinessProbe: + tcpSocket: + port: 10000 + initialDelaySeconds: 2 + periodSeconds: 5 + livenessProbe: + tcpSocket: + port: 10000 + initialDelaySeconds: 10 + periodSeconds: 10 + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: azurite +spec: + selector: + app: azurite + ports: + - name: blob + protocol: TCP + port: 10000 + targetPort: 10000 + - name: queue + protocol: TCP + port: 10001 + targetPort: 10001 + - name: table + protocol: TCP + port: 10002 + targetPort: 10002 diff --git a/manifests/db/redis.yaml b/manifests/db/redis.yaml deleted file mode 100644 index 3972f30..0000000 --- a/manifests/db/redis.yaml +++ /dev/null @@ -1,72 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: redis - labels: - app: redis -spec: - serviceName: redis - replicas: 1 - selector: - matchLabels: - app: redis - template: - metadata: - labels: - app: redis - spec: - containers: - - image: redis:8 - name: redis - imagePullPolicy: Always - args: - - --requirepass - - virtool - resources: - requests: - cpu: 250m - memory: 256Mi - limits: - cpu: 375m - memory: 384Mi - readinessProbe: - exec: - command: - - redis-cli - - -a - - virtool - - ping - initialDelaySeconds: 5 - periodSeconds: 10 - livenessProbe: - exec: - command: - - redis-cli - - -a - - virtool - - ping - initialDelaySeconds: 10 - periodSeconds: 10 - volumeMounts: - - name: data - mountPath: /data - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 5Gi ---- -apiVersion: v1 -kind: Service -metadata: - name: redis -spec: - selector: - app: redis - ports: - - protocol: TCP - port: 6379 - targetPort: 6379 diff --git a/manifests/migration.yaml b/manifests/migration.yaml index 74c978e..67c8044 100644 --- a/manifests/migration.yaml +++ b/manifests/migration.yaml @@ -13,7 +13,7 @@ spec: spec: restartPolicy: Never containers: - - image: ghcr.io/virtool/virtool:36.0.0 + - image: ghcr.io/virtool/virtool:36.25.3 name: virtool-migration command: ["virtool", "migration", "apply"] env: @@ -27,8 +27,6 @@ spec: value: "mongodb://virtool:virtool@mongo.default.svc.cluster.local/virtool" - name: VT_POSTGRES_CONNECTION_STRING value: "postgresql+asyncpg://virtool@postgres.default.svc.cluster.local?password=virtool" - - name: VT_REDIS_CONNECTION_STRING - value: "redis://:virtool@redis.default.svc.cluster.local" imagePullPolicy: Always resources: limits: diff --git a/manifests/ui/kustomization.yaml b/manifests/ui/kustomization.yaml index f993652..884d34a 100644 --- a/manifests/ui/kustomization.yaml +++ b/manifests/ui/kustomization.yaml @@ -1,6 +1,6 @@ images: - name: ghcr.io/virtool/ui - newTag: 7.5.2 + newTag: 7.13.3 labels: - pairs: diff --git a/manifests/virtool/base/deployment.yaml b/manifests/virtool/base/deployment.yaml index da836f8..03e288c 100644 --- a/manifests/virtool/base/deployment.yaml +++ b/manifests/virtool/base/deployment.yaml @@ -21,10 +21,18 @@ spec: value: "0.0.0.0" - name: VT_MONGODB_CONNECTION_STRING value: "mongodb://virtool:virtool@mongo.default.svc.cluster.local/virtool" - - name: VT_REDIS_CONNECTION_STRING - value: "redis://:virtool@redis.default.svc.cluster.local" - name: VT_POSTGRES_CONNECTION_STRING value: "postgresql+asyncpg://virtool@postgres.default.svc.cluster.local?password=virtool" + - name: VT_STORAGE_BACKEND + value: "azure" + - name: VT_STORAGE_AZURE_ACCOUNT + value: "devstoreaccount1" + - name: VT_STORAGE_AZURE_CONTAINER + value: "virtool" + - name: VT_STORAGE_AZURE_ACCESS_KEY + value: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + - name: VT_STORAGE_AZURE_ENDPOINT + value: "http://azurite.default.svc.cluster.local:10000/devstoreaccount1" ports: - containerPort: 9950 protocol: TCP diff --git a/manifests/virtool/base/kustomization.yaml b/manifests/virtool/base/kustomization.yaml index 1c3b5d3..55ef2df 100644 --- a/manifests/virtool/base/kustomization.yaml +++ b/manifests/virtool/base/kustomization.yaml @@ -1,6 +1,6 @@ images: - name: ghcr.io/virtool/virtool - newTag: 36.0.0 + newTag: 36.25.3 labels: - pairs: diff --git a/manifests/workflows/build-index.yaml b/manifests/workflows/build-index.yaml index d2f2a7f..69a6a7e 100644 --- a/manifests/workflows/build-index.yaml +++ b/manifests/workflows/build-index.yaml @@ -14,14 +14,14 @@ spec: spec: containers: - name: workflow-build-index - image: ghcr.io/virtool/build-index:5.4.8 + image: ghcr.io/virtool/build-index:5.7.0 env: - name: VT_PROC value: "2" - name: VT_MEM value: "2" - - name: VT_REDIS_LIST_NAME - value: "jobs_build_index" + - name: VT_WORKFLOW + value: build_index resources: limits: cpu: 3 @@ -30,5 +30,9 @@ spec: cpu: 3 memory: 5Gi triggers: - - metadata: - listName: "jobs_build_index" + - type: metrics-api + metadata: + url: "http://virtool-api-web-service.default.svc.cluster.local/jobs/counts" + valueLocation: "pending.build_index" + targetValue: "1" + activationTargetValue: "0" diff --git a/manifests/workflows/create-sample.yaml b/manifests/workflows/create-sample.yaml index c28516b..cb84e16 100644 --- a/manifests/workflows/create-sample.yaml +++ b/manifests/workflows/create-sample.yaml @@ -14,14 +14,14 @@ spec: spec: containers: - name: workflow-create-sample - image: ghcr.io/virtool/create-sample:4.3.8 + image: ghcr.io/virtool/create-sample:4.6.0 env: - name: VT_PROC value: "2" - name: VT_MEM value: "4" - - name: VT_REDIS_LIST_NAME - value: "jobs_create_sample" + - name: VT_WORKFLOW + value: create_sample resources: limits: cpu: 2700m @@ -30,5 +30,9 @@ spec: cpu: 2700m memory: 5Gi triggers: - - metadata: - listName: "jobs_create_sample" + - type: metrics-api + metadata: + url: "http://virtool-api-web-service.default.svc.cluster.local/jobs/counts" + valueLocation: "pending.create_sample" + targetValue: "1" + activationTargetValue: "0" diff --git a/manifests/workflows/create-subtraction.yaml b/manifests/workflows/create-subtraction.yaml index b4ba2f3..30287dc 100644 --- a/manifests/workflows/create-subtraction.yaml +++ b/manifests/workflows/create-subtraction.yaml @@ -14,14 +14,14 @@ spec: spec: containers: - name: workflow-create-subtraction - image: ghcr.io/virtool/create-subtraction:5.3.8 + image: ghcr.io/virtool/create-subtraction:5.5.0 env: - name: VT_PROC value: "2" - name: VT_MEM value: "2" - - name: VT_REDIS_LIST_NAME - value: "jobs_create_subtraction" + - name: VT_WORKFLOW + value: create_subtraction resources: limits: cpu: "3" @@ -30,5 +30,9 @@ spec: cpu: "3" memory: 5Gi triggers: - - metadata: - listName: "jobs_create_subtraction" + - type: metrics-api + metadata: + url: "http://virtool-api-web-service.default.svc.cluster.local/jobs/counts" + valueLocation: "pending.create_subtraction" + targetValue: "1" + activationTargetValue: "0" diff --git a/manifests/workflows/iimi.yaml b/manifests/workflows/iimi.yaml index 54e2d48..c698e52 100644 --- a/manifests/workflows/iimi.yaml +++ b/manifests/workflows/iimi.yaml @@ -14,14 +14,14 @@ spec: spec: containers: - name: virtool-workflow - image: ghcr.io/virtool/iimi:0.4.5 + image: ghcr.io/virtool/iimi:0.5.0 env: - name: VT_MEM value: "4" - name: VT_PROC value: "10" - - name: VT_REDIS_LIST_NAME - value: "jobs_iimi" + - name: VT_WORKFLOW + value: iimi resources: limits: cpu: "5" @@ -30,5 +30,9 @@ spec: cpu: "5" memory: 11Gi triggers: - - metadata: - listName: "jobs_iimi" + - type: metrics-api + metadata: + url: "http://virtool-api-web-service.default.svc.cluster.local/jobs/counts" + valueLocation: "pending.iimi" + targetValue: "1" + activationTargetValue: "0" diff --git a/manifests/workflows/kustomization.yaml b/manifests/workflows/kustomization.yaml index d948a9e..cf828cc 100644 --- a/manifests/workflows/kustomization.yaml +++ b/manifests/workflows/kustomization.yaml @@ -14,9 +14,6 @@ patches: - path: patches/patch-json.yaml target: kind: ScaledJob - - path: patches/patch-trigger.yaml - target: - kind: ScaledJob resources: - build-index.yaml diff --git a/manifests/workflows/nuvs.yaml b/manifests/workflows/nuvs.yaml index 912df5b..a93e15e 100644 --- a/manifests/workflows/nuvs.yaml +++ b/manifests/workflows/nuvs.yaml @@ -11,14 +11,14 @@ spec: spec: containers: - name: workflow-nuvs - image: ghcr.io/virtool/nuvs:5.4.5 + image: ghcr.io/virtool/nuvs:5.7.0 env: - name: VT_PROC value: "4" - name: VT_MEM value: "10" - - name: VT_REDIS_LIST_NAME - value: "jobs_nuvs" + - name: VT_WORKFLOW + value: nuvs resources: limits: cpu: "5" @@ -27,5 +27,9 @@ spec: cpu: "4" memory: 10Gi triggers: - - metadata: - listName: "jobs_nuvs" \ No newline at end of file + - type: metrics-api + metadata: + url: "http://virtool-api-web-service.default.svc.cluster.local/jobs/counts" + valueLocation: "pending.nuvs" + targetValue: "1" + activationTargetValue: "0" diff --git a/manifests/workflows/patches/patch-json.yaml b/manifests/workflows/patches/patch-json.yaml index 8427a5e..107de52 100644 --- a/manifests/workflows/patches/patch-json.yaml +++ b/manifests/workflows/patches/patch-json.yaml @@ -8,24 +8,6 @@ name: VT_JOBS_API_CONNECTION_STRING value: http://virtool-api-jobs-service.default.svc.cluster.local -- op: add - path: /spec/jobTargetRef/template/spec/containers/0/env/-1 - value: - name: VT_REDIS_CONNECTION_STRING - value: redis://:virtool@redis.default.svc.cluster.local - -- op: add - path: /spec/jobTargetRef/template/spec/containers/0/env/-1 - value: - name: VT_REDIS_HOST - value: redis.default.svc.cluster.local:6379 - -- op: add - path: /spec/jobTargetRef/template/spec/containers/0/env/-1 - value: - name: VT_REDIS_PASSWORD - value: virtool - - op: add path: /spec/jobTargetRef/template/spec/containers/0/env/-1 value: diff --git a/manifests/workflows/patches/patch-trigger.yaml b/manifests/workflows/patches/patch-trigger.yaml deleted file mode 100644 index fee4b13..0000000 --- a/manifests/workflows/patches/patch-trigger.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- op: add - path: /spec/triggers/0/type - value: redis - -- op: add - path: /spec/triggers/0/metadata/address - value: redis.default.svc.cluster.local:6379 - -- op: add - path: /spec/triggers/0/metadata/passwordFromEnv - value: VT_REDIS_PASSWORD - -- op: add - path: /spec/triggers/0/metadata/listLength - value: "1" diff --git a/manifests/workflows/pathoscope.yaml b/manifests/workflows/pathoscope.yaml index 7ad7bcf..d40dc49 100644 --- a/manifests/workflows/pathoscope.yaml +++ b/manifests/workflows/pathoscope.yaml @@ -11,14 +11,14 @@ spec: spec: containers: - name: workflow-pathoscope - image: ghcr.io/virtool/pathoscope:5.9.4 + image: ghcr.io/virtool/pathoscope:5.13.0 env: - name: VT_MEM value: "8" - name: VT_PROC value: "4" - - name: VT_REDIS_LIST_NAME - value: "jobs_pathoscope_bowtie" + - name: VT_WORKFLOW + value: pathoscope resources: limits: cpu: "5" @@ -27,5 +27,9 @@ spec: cpu: "5" memory: 10Gi triggers: - - metadata: - listName: "jobs_pathoscope_bowtie" + - type: metrics-api + metadata: + url: "http://virtool-api-web-service.default.svc.cluster.local/jobs/counts" + valueLocation: "pending.pathoscope" + targetValue: "1" + activationTargetValue: "0" diff --git a/scripts/pull.sh b/scripts/pull.sh index ee13fca..4f483c5 100644 --- a/scripts/pull.sh +++ b/scripts/pull.sh @@ -6,18 +6,19 @@ if ! command -v jq &> /dev/null; then exit 1 fi -fetch_and_update() { +fetch_latest_tag() { local repo=$1 - local file=$2 - local prefix=$3 - - # Fetch the latest release tag local url="https://api.github.com/repos/${repo}/releases/latest" - local tag=$(curl -s "$url" | jq -r '.tag_name') + curl -s "$url" | jq -r '.tag_name' | sed 's/^v//' +} + +update_tag() { + local file=$1 + local prefix=$2 + local tag=$3 - # Update the file if it exists if [[ -f "$file" ]]; then - sed -i "s/${prefix}[0-9.]\+/${prefix}${tag}/" "$file" + sed -i.bak "s|${prefix}[0-9.]\+|${prefix}${tag}|" "$file" && rm "${file}.bak" echo "Using tag '$tag' for $file" else echo "Error: File $file not found." @@ -25,10 +26,22 @@ fetch_and_update() { fi } +fetch_and_update() { + local repo=$1 + local file=$2 + local prefix=$3 + local tag + tag=$(fetch_latest_tag "$repo") + update_tag "$file" "$prefix" "$tag" +} + echo "Server" echo "" -fetch_and_update "virtool/virtool" "manifests/virtool/base/kustomization.yaml" "newTag: " +virtool_tag=$(fetch_latest_tag "virtool/virtool") +update_tag "manifests/virtool/base/kustomization.yaml" "newTag: " "$virtool_tag" +update_tag "manifests/migration.yaml" "ghcr.io/virtool/virtool:" "$virtool_tag" + fetch_and_update "virtool/virtool-ui" "manifests/ui/kustomization.yaml" "newTag: " echo "" diff --git a/scripts/wipe.sh b/scripts/wipe.sh new file mode 100644 index 0000000..0f67014 --- /dev/null +++ b/scripts/wipe.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -e + +DATA_DIR="/virtool/data" + +current_context=$(kubectl config current-context 2>/dev/null || true) +if [[ "$current_context" != "minikube" ]]; then + echo "Refusing to wipe: kubectl current-context is '${current_context:-}', expected 'minikube'." + echo "Switch contexts with: kubectl config use-context minikube" + exit 1 +fi + +if ! minikube status >/dev/null 2>&1; then + echo "Refusing to wipe: Minikube is not running." + exit 1 +fi + +clear_data_dir() { + minikube ssh -- \ + "test -d $DATA_DIR && sudo find $DATA_DIR -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +" +} + +echo "Deleting StatefulSets..." +kubectl delete statefulset azurite mongo postgres --ignore-not-found + +echo "Deleting PVCs..." +kubectl delete pvc \ + data-azurite-0 \ + data-mongo-0 \ + data-postgres-0 \ + pvc-virtool \ + --ignore-not-found + +echo "Deleting PV..." +kubectl delete pv pv-virtool --ignore-not-found + +echo "Clearing contents of $DATA_DIR..." +clear_data_dir + +echo "Done. Tilt will recreate resources on next trigger."