test: benchmark memory optimizations

.github/workflows/base-image.yaml

@@ -16,6 +16,7 @@ jobs:
           - name: 202311
           - name: 202411
           - name: 202505
+          - name: 202511-vpp
 
     steps:
     - name: Log in to the container registry

.github/workflows/integration.yaml

@@ -7,6 +7,16 @@ on:
   push:
     branches:
       - master
+  workflow_dispatch:
+    inputs:
+      flavors:
+        description: 'Comma separated flavors to test'
+        default: 'sonic_vs,sonic_vpp,gardener,dell_sonic'
+      memory_profiles:
+        description: >-
+          Comma separated memory profiles. Each profile is one integration run
+          per flavor, so the full list multiplies the runtime accordingly.
+        default: 'baseline,low-memory,ksm,thp-madvise,all'
 
 env:
   REGISTRY: ghcr.io
@@ -53,8 +63,8 @@ jobs:
         cache-from: type=registry,ref=${{ env.MINI_LAB_VM_IMAGE }}
         cache-to: type=inline
 
-  build-mini-lab-sonic-image:
-    name: Build mini-lab-sonic image
+  build-mini-lab-sonic-vs-image:
+    name: Build mini-lab-sonic-vs image
     runs-on: ubuntu-latest
 
     steps:
@@ -73,8 +83,8 @@ jobs:
         IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest")
         SHA_TAG=${COMMIT_SHA::8}
 
-        echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic:${IMAGE_TAG}" >> $GITHUB_ENV
-        echo "MINI_LAB_SONIC_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic:${SHA_TAG}" >> $GITHUB_ENV
+        echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic-vs:${IMAGE_TAG}" >> $GITHUB_ENV
+        echo "MINI_LAB_SONIC_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic-vs:${SHA_TAG}" >> $GITHUB_ENV
       env:
         COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
@@ -88,26 +98,96 @@ jobs:
         pull: true
         push: true
         sbom: true
+        file: images/sonic/Dockerfile.vs
         tags: |
           ${{ env.MINI_LAB_SONIC_IMAGE }}
           ${{ env.MINI_LAB_SONIC_IMAGE_SHA }}
         cache-from: type=registry,ref=${{ env.MINI_LAB_SONIC_IMAGE }}
         cache-to: type=inline
 
+  build-mini-lab-sonic-vpp-image:
+    name: Build mini-lab-sonic-vpp image
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Log in to the container registry
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ secrets.DOCKER_REGISTRY_USER }}
+        password: ${{ secrets.DOCKER_REGISTRY_TOKEN }}
+
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Make tag
+      run: |
+        IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest")
+        SHA_TAG=${COMMIT_SHA::8}
+
+        echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic-vpp:${IMAGE_TAG}" >> $GITHUB_ENV
+        echo "MINI_LAB_SONIC_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic-vpp:${SHA_TAG}" >> $GITHUB_ENV
+      env:
+        COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Build and push mini-lab-sonic container
+      uses: docker/build-push-action@v6
+      with:
+        context: ./images/sonic
+        pull: true
+        push: true
+        sbom: true
+        file: images/sonic/Dockerfile.vpp
+        tags: |
+          ${{ env.MINI_LAB_SONIC_IMAGE }}
+          ${{ env.MINI_LAB_SONIC_IMAGE_SHA }}
+        cache-from: type=registry,ref=${{ env.MINI_LAB_SONIC_IMAGE }}
+        cache-to: type=inline
+  prepare-matrix:
+    name: Prepare test matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.build.outputs.matrix }}
+    steps:
+    - name: Build flavor x memory profile matrix
+      id: build
+      shell: bash
+      env:
+        # every event runs the full flavor x profile matrix, so each pull
+        # request produces on/off data for all three memory knobs. Runs are
+        # serialised on the self-hosted runner, so this is a long pipeline -
+        # narrow it down via the workflow_dispatch inputs when iterating.
+        FLAVORS: ${{ inputs.flavors || 'sonic_vs,sonic_vpp,gardener,dell_sonic' }}
+        PROFILES: ${{ inputs.memory_profiles || 'baseline,low-memory,ksm,thp-madvise,all' }}
+      run: |
+        matrix=$(python3 -c '
+        import json, os
+        flavors = [f.strip() for f in os.environ["FLAVORS"].split(",") if f.strip()]
+        profiles = [p.strip() for p in os.environ["PROFILES"].split(",") if p.strip()]
+        include = [{"flavor": f, "profile": p} for f in flavors for p in profiles]
+        print(json.dumps({"include": include}))
+        ')
+        echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
+        echo "${matrix}" | python3 -m json.tool
+
   test:
-    name: Run tests
+    name: Run tests (${{ matrix.flavor }}, ${{ matrix.profile }})
     runs-on: self-hosted
     needs:
       - build-mini-lab-vms-image
-      - build-mini-lab-sonic-image
+      - build-mini-lab-sonic-vs-image
+      - build-mini-lab-sonic-vpp-image
+      - prepare-matrix
 
     continue-on-error: true
     strategy:
-      matrix:
-        flavors:
-          - name: sonic
-          - name: gardener
-          - name: dell_sonic
+      fail-fast: false
+      # the memory numbers are host global, so runs must not overlap
+      max-parallel: 1
+      matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }}
 
     steps:
     - name: Gain back workspace permissions # https://github.com/actions/checkout/issues/211
@@ -140,7 +220,11 @@ jobs:
         IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest")
 
         echo "MINI_LAB_VM_IMAGE=ghcr.io/metal-stack/mini-lab-vms:${IMAGE_TAG}" >> $GITHUB_ENV
-        echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic:${IMAGE_TAG}" >> $GITHUB_ENV
+        echo "MINI_LAB_SONIC_IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_ENV
+
+    - name: Select memory profile
+      shell: bash
+      run: ./scripts/memory-profile.sh "${{ matrix.profile }}" >> "$GITHUB_ENV"
 
     - name: Run integration tests
       shell: bash
@@ -149,7 +233,72 @@ jobs:
         ./test/ci-cleanup.sh
         ./test/integration.sh
       env:
-        MINI_LAB_FLAVOR: ${{ matrix.flavors.name }}
+        MINI_LAB_FLAVOR: ${{ matrix.flavor }}
         DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }}
         DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }}
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Restore host memory tuning
+      if: always()
+      shell: bash
+      run: ./scripts/memory-tuning.sh restore || true
+
+    - name: Publish memory summary
+      if: always()
+      shell: bash
+      run: |
+        if [ -f memory-traces/summary.md ]; then
+          cat memory-traces/summary.md >> "$GITHUB_STEP_SUMMARY"
+        else
+          echo "no memory summary produced for ${{ matrix.flavor }} / ${{ matrix.profile }}" \
+            >> "$GITHUB_STEP_SUMMARY"
+        fi
+
+    - name: Upload memory trace
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: memory-trace-${{ matrix.flavor }}-${{ matrix.profile }}
+        path: memory-traces/
+        if-no-files-found: warn
+        retention-days: 30
+
+  memory-comparison:
+    name: Compare memory profiles
+    runs-on: ubuntu-latest
+    needs: test
+    if: always()
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Download memory traces
+      uses: actions/download-artifact@v4
+      with:
+        pattern: memory-trace-*
+        path: memory-traces
+
+    - name: Compare profiles
+      shell: bash
+      run: |
+        mkdir -p memory-traces
+        # every test job runs with continue-on-error, so it is possible that no
+        # run got far enough to produce a summary
+        if ! find memory-traces -name summary.json | grep -q .; then
+          echo "No memory summaries were produced by this run." \
+            | tee memory-traces/comparison.md >> "$GITHUB_STEP_SUMMARY"
+          exit 0
+        fi
+        ./scripts/memory-report.py compare \
+          --input-dir memory-traces \
+          --out-md memory-traces/comparison.md
+        cat memory-traces/comparison.md >> "$GITHUB_STEP_SUMMARY"
+
+    - name: Upload comparison
+      uses: actions/upload-artifact@v4
+      with:
+        name: memory-comparison
+        path: memory-traces/
+        if-no-files-found: warn
+        retention-days: 30

.gitignore

@@ -17,4 +17,6 @@ files/certs/*.pem
 files/certs/**/*.pem
 files/certs/**/*.crt
 .vscode
-vrnetlab
+vrnetlab
+memory-traces
+.memory-tuning.state

Makefile

@@ -26,10 +26,21 @@ ANSIBLE_EXTRA_VARS_FILE := $(or $(ANSIBLE_EXTRA_VARS_FILE),)
 # do not show skipped ansible tasks
 ANSIBLE_DISPLAY_SKIPPED_HOSTS=false
 
-MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic)
+MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic_vs)
 MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:latest)
-MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest)
 MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1)
+MINI_LAB_SONIC_IMAGE_TAG := $(or $(MINI_LAB_SONIC_IMAGE_TAG),latest)
+
+# Memory tuning. All of these are opt-in: the defaults reproduce the behaviour
+# the lab had before the knobs existed. See docs/memory-tuning.md.
+# QEMU RAM handed to the guests, substituted into the topology files.
+MINI_LAB_LEAF_MEMORY := $(or $(MINI_LAB_LEAF_MEMORY),4096)
+MINI_LAB_MACHINE_MEMORY := $(or $(MINI_LAB_MACHINE_MEMORY),2048)
+# host-global knobs, empty means "do not touch"
+MINI_LAB_KSM := $(or $(MINI_LAB_KSM),)
+MINI_LAB_THP := $(or $(MINI_LAB_THP),)
+# where the memory tracer writes its samples
+MEMORY_TRACE_DIR := $(or $(MEMORY_TRACE_DIR),memory-traces)
 
 MINI_LAB_INTERNAL_NETWORK=mini_lab_internal
 # define this here as well so that kind picks up the network on a clean checkout,
@@ -40,9 +51,14 @@ MACHINE_OS=debian-13.0
 MAX_RETRIES := 30
 
 # Machine flavors
-ifeq ($(MINI_LAB_FLAVOR),sonic)
+ifeq ($(MINI_LAB_FLAVOR),sonic_vs)
 LAB_TOPOLOGY=mini-lab.sonic.yaml
 MONITORING_ENABLED := $(or $(MONITORING_ENABLED),true)
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vs:$(MINI_LAB_SONIC_IMAGE_TAG))
+else ifeq ($(MINI_LAB_FLAVOR),sonic_vpp)
+LAB_TOPOLOGY=mini-lab.sonic.yaml
+MONITORING_ENABLED := $(or $(MONITORING_ENABLED),true)
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vpp:$(MINI_LAB_SONIC_IMAGE_TAG))
 else ifeq ($(MINI_LAB_FLAVOR),dell_sonic)
 LAB_TOPOLOGY=mini-lab.dell_sonic.yaml
 MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION)
@@ -52,11 +68,13 @@ MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_
 else ifeq ($(MINI_LAB_FLAVOR),kamaji)
 LAB_TOPOLOGY=mini-lab.kamaji.yaml
 KAMAJI_ENABLED=true
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vs:$(MINI_LAB_SONIC_IMAGE_TAG))
 else ifeq ($(MINI_LAB_FLAVOR),gardener)
 GARDENER_ENABLED=true
 # usually gardener restricts the maximum version for k8s:
 K8S_VERSION=1.35.5
 LAB_TOPOLOGY=mini-lab.sonic.yaml
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vs:$(MINI_LAB_SONIC_IMAGE_TAG))
 else
 $(error Unknown flavor $(MINI_LAB_FLAVOR))
 endif
@@ -132,7 +150,7 @@ partition: partition-bake
 	docker compose $(COMPOSE_ARGS) up --remove-orphans --force-recreate partition
 
 .PHONY: partition-bake
-partition-bake: external_network
+partition-bake: external_network memory-tuning-apply
 	docker pull $(MINI_LAB_VM_IMAGE)
 ifeq ($(CI),true)
 	docker pull $(MINI_LAB_SONIC_IMAGE)
@@ -141,8 +159,7 @@ ifneq ($(filter $(MINI_LAB_FLAVOR),dell_sonic capms_dell_sonic),$(MINI_LAB_FLAVO
 	docker pull $(MINI_LAB_SONIC_IMAGE)
 endif
 	@if ! sudo $(CONTAINERLAB) --topo $(LAB_TOPOLOGY) inspect | grep -i leaf01 > /dev/null; then \
-		sudo --preserve-env=MINI_LAB_SONIC_IMAGE --preserve-env=MINI_LAB_DELL_SONIC_VERSION --preserve-env=MINI_LAB_VM_IMAGE $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure && \
-		./scripts/deactivate_offloading.sh; fi
+		sudo --preserve-env=MINI_LAB_SONIC_IMAGE --preserve-env=MINI_LAB_DELL_SONIC_VERSION --preserve-env=MINI_LAB_VM_IMAGE --preserve-env=MINI_LAB_LEAF_MEMORY --preserve-env=MINI_LAB_MACHINE_MEMORY $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure; fi
 
 .PHONY: verify-deployment-image
 verify-deployment-image:
@@ -180,6 +197,69 @@ files/certs/ca.pem:
 .PHONY: gen-certs                 # keep as a convenience alias
 gen-certs: files/certs/ca.pem
 
+## MEMORY TUNING & TRACING ##
+
+# print the environment of a named profile, e.g.
+#   eval $(make memory-profile PROFILE=low-memory)
+.PHONY: memory-profile
+memory-profile:
+	@./scripts/memory-profile.sh $(or $(PROFILE),$(MINI_LAB_MEMORY_PROFILE),baseline) --export
+
+.PHONY: memory-tuning-apply
+memory-tuning-apply:
+	@./scripts/memory-tuning.sh apply
+
+.PHONY: memory-tuning-restore
+memory-tuning-restore:
+	@./scripts/memory-tuning.sh restore
+
+.PHONY: memory-tuning-show
+memory-tuning-show:
+	@./scripts/memory-tuning.sh show
+
+.PHONY: memory-trace-start
+memory-trace-start:
+	@mkdir -p $(MEMORY_TRACE_DIR)
+	@if [ -f $(MEMORY_TRACE_DIR)/tracer.pid ] && kill -0 $$(cat $(MEMORY_TRACE_DIR)/tracer.pid) 2> /dev/null; then \
+		echo "memory tracer already running"; \
+	else \
+		nohup ./scripts/memory-trace.py sample \
+			--out $(MEMORY_TRACE_DIR)/trace.csv \
+			--meta $(MEMORY_TRACE_DIR)/meta.json \
+			--interval $(or $(MEMORY_TRACE_INTERVAL),5) \
+			--flavor $(MINI_LAB_FLAVOR) \
+			--profile $(or $(MINI_LAB_MEMORY_PROFILE),unset) \
+			> $(MEMORY_TRACE_DIR)/tracer.log 2>&1 & echo $$! > $(MEMORY_TRACE_DIR)/tracer.pid; \
+		echo "memory tracer started, writing to $(MEMORY_TRACE_DIR)/trace.csv"; \
+	fi
+
+.PHONY: memory-trace-stop
+memory-trace-stop:
+	@if [ -f $(MEMORY_TRACE_DIR)/tracer.pid ]; then \
+		kill $$(cat $(MEMORY_TRACE_DIR)/tracer.pid) 2> /dev/null || true; \
+		sleep 1; \
+		rm -f $(MEMORY_TRACE_DIR)/tracer.pid; \
+		echo "memory tracer stopped"; \
+	else \
+		echo "no memory tracer running"; \
+	fi
+
+.PHONY: memory-report
+memory-report:
+	@./scripts/memory-report.py summarize \
+		--trace $(MEMORY_TRACE_DIR)/trace.csv \
+		--meta $(MEMORY_TRACE_DIR)/meta.json \
+		--out-json $(MEMORY_TRACE_DIR)/summary.json \
+		--out-md $(MEMORY_TRACE_DIR)/summary.md \
+		--flavor $(MINI_LAB_FLAVOR) \
+		--profile $(or $(MINI_LAB_MEMORY_PROFILE),unset)
+	@cat $(MEMORY_TRACE_DIR)/summary.md
+
+# compare all summaries below MEMORY_TRACE_DIR (or DIR=...)
+.PHONY: memory-compare
+memory-compare:
+	@./scripts/memory-report.py compare --input-dir $(or $(DIR),$(MEMORY_TRACE_DIR))
+
 .PHONY: cleanup
 cleanup: cleanup-control-plane cleanup-partition
 	docker network rm --force mini_lab_internal
@@ -191,7 +271,7 @@ cleanup-control-plane:
 	rm -f $(KUBECONFIG)
 
 .PHONY: cleanup-partition
-cleanup-partition:
+cleanup-partition: memory-tuning-restore
 	mkdir -p clab-mini-lab
 	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.dell_sonic.yaml
 	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml