Add SONiC VPP platform support

.github/workflows/base-image.yaml

@@ -16,6 +16,7 @@ jobs:
           - name: 202311
           - name: 202411
           - name: 202505
+          - name: 202511-vpp
 
     steps:
     - name: Log in to the container registry

.github/workflows/integration.yaml

@@ -53,8 +53,8 @@ jobs:
         cache-from: type=registry,ref=${{ env.MINI_LAB_VM_IMAGE }}
         cache-to: type=inline
 
-  build-mini-lab-sonic-image:
-    name: Build mini-lab-sonic image
+  build-mini-lab-sonic-vs-image:
+    name: Build mini-lab-sonic-vs image
     runs-on: ubuntu-latest
 
     steps:
@@ -73,8 +73,8 @@ jobs:
         IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest")
         SHA_TAG=${COMMIT_SHA::8}
 
-        echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic:${IMAGE_TAG}" >> $GITHUB_ENV
-        echo "MINI_LAB_SONIC_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic:${SHA_TAG}" >> $GITHUB_ENV
+        echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic-vs:${IMAGE_TAG}" >> $GITHUB_ENV
+        echo "MINI_LAB_SONIC_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic-vs:${SHA_TAG}" >> $GITHUB_ENV
       env:
         COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
@@ -88,24 +88,68 @@ jobs:
         pull: true
         push: true
         sbom: true
+        file: images/sonic/Dockerfile.vs
         tags: |
           ${{ env.MINI_LAB_SONIC_IMAGE }}
           ${{ env.MINI_LAB_SONIC_IMAGE_SHA }}
         cache-from: type=registry,ref=${{ env.MINI_LAB_SONIC_IMAGE }}
         cache-to: type=inline
 
+  build-mini-lab-sonic-vpp-image:
+    name: Build mini-lab-sonic-vpp image
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Log in to the container registry
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ secrets.DOCKER_REGISTRY_USER }}
+        password: ${{ secrets.DOCKER_REGISTRY_TOKEN }}
+
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Make tag
+      run: |
+        IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest")
+        SHA_TAG=${COMMIT_SHA::8}
+
+        echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic-vpp:${IMAGE_TAG}" >> $GITHUB_ENV
+        echo "MINI_LAB_SONIC_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic-vpp:${SHA_TAG}" >> $GITHUB_ENV
+      env:
+        COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Build and push mini-lab-sonic container
+      uses: docker/build-push-action@v6
+      with:
+        context: ./images/sonic
+        pull: true
+        push: true
+        sbom: true
+        file: images/sonic/Dockerfile.vpp
+        tags: |
+          ${{ env.MINI_LAB_SONIC_IMAGE }}
+          ${{ env.MINI_LAB_SONIC_IMAGE_SHA }}
+        cache-from: type=registry,ref=${{ env.MINI_LAB_SONIC_IMAGE }}
+        cache-to: type=inline
   test:
     name: Run tests
     runs-on: self-hosted
     needs:
       - build-mini-lab-vms-image
-      - build-mini-lab-sonic-image
+      - build-mini-lab-sonic-vs-image
+      - build-mini-lab-sonic-vpp-image
 
     continue-on-error: true
     strategy:
       matrix:
         flavors:
-          - name: sonic
+          - name: sonic_vs
+          - name: sonic_vpp
           - name: gardener
           - name: dell_sonic
 
@@ -140,7 +184,7 @@ jobs:
         IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest")
 
         echo "MINI_LAB_VM_IMAGE=ghcr.io/metal-stack/mini-lab-vms:${IMAGE_TAG}" >> $GITHUB_ENV
-        echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic:${IMAGE_TAG}" >> $GITHUB_ENV
+        echo "MINI_LAB_SONIC_IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_ENV
 
     - name: Run integration tests
       shell: bash

Makefile

@@ -26,10 +26,10 @@ ANSIBLE_EXTRA_VARS_FILE := $(or $(ANSIBLE_EXTRA_VARS_FILE),)
 # do not show skipped ansible tasks
 ANSIBLE_DISPLAY_SKIPPED_HOSTS=false
 
-MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic)
+MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic_vs)
 MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:latest)
-MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest)
 MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1)
+MINI_LAB_SONIC_IMAGE_TAG := $(or $(MINI_LAB_SONIC_IMAGE_TAG),latest)
 
 MINI_LAB_INTERNAL_NETWORK=mini_lab_internal
 # define this here as well so that kind picks up the network on a clean checkout,
@@ -40,9 +40,14 @@ MACHINE_OS=debian-13.0
 MAX_RETRIES := 30
 
 # Machine flavors
-ifeq ($(MINI_LAB_FLAVOR),sonic)
+ifeq ($(MINI_LAB_FLAVOR),sonic_vs)
 LAB_TOPOLOGY=mini-lab.sonic.yaml
 MONITORING_ENABLED := $(or $(MONITORING_ENABLED),true)
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vs:$(MINI_LAB_SONIC_IMAGE_TAG))
+else ifeq ($(MINI_LAB_FLAVOR),sonic_vpp)
+LAB_TOPOLOGY=mini-lab.sonic.yaml
+MONITORING_ENABLED := $(or $(MONITORING_ENABLED),true)
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vpp:$(MINI_LAB_SONIC_IMAGE_TAG))
 else ifeq ($(MINI_LAB_FLAVOR),dell_sonic)
 LAB_TOPOLOGY=mini-lab.dell_sonic.yaml
 MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION)
@@ -52,11 +57,13 @@ MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_
 else ifeq ($(MINI_LAB_FLAVOR),kamaji)
 LAB_TOPOLOGY=mini-lab.kamaji.yaml
 KAMAJI_ENABLED=true
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vs:$(MINI_LAB_SONIC_IMAGE_TAG))
 else ifeq ($(MINI_LAB_FLAVOR),gardener)
 GARDENER_ENABLED=true
 # usually gardener restricts the maximum version for k8s:
 K8S_VERSION=1.35.5
 LAB_TOPOLOGY=mini-lab.sonic.yaml
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vs:$(MINI_LAB_SONIC_IMAGE_TAG))
 else
 $(error Unknown flavor $(MINI_LAB_FLAVOR))
 endif
@@ -141,8 +148,7 @@ ifneq ($(filter $(MINI_LAB_FLAVOR),dell_sonic capms_dell_sonic),$(MINI_LAB_FLAVO
 	docker pull $(MINI_LAB_SONIC_IMAGE)
 endif
 	@if ! sudo $(CONTAINERLAB) --topo $(LAB_TOPOLOGY) inspect | grep -i leaf01 > /dev/null; then \
-		sudo --preserve-env=MINI_LAB_SONIC_IMAGE --preserve-env=MINI_LAB_DELL_SONIC_VERSION --preserve-env=MINI_LAB_VM_IMAGE $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure && \
-		./scripts/deactivate_offloading.sh; fi
+		sudo --preserve-env=MINI_LAB_SONIC_IMAGE --preserve-env=MINI_LAB_DELL_SONIC_VERSION --preserve-env=MINI_LAB_VM_IMAGE $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure; fi
 
 .PHONY: verify-deployment-image
 verify-deployment-image:

README.md

@@ -238,16 +238,17 @@ make up
 
 All available mini-lab flavors are listed below:
 
-- `sonic`: runs two Community SONiC switches
+- `sonic_vs`: runs two Community SONiC switches with the kernel based virtual switch dataplane
+- `sonic_vpp`: runs two Community SONiC switches with the [VPP](https://fd.io) dataplane
 - `dell_sonic`: runs two Enterprise SONiC switches with a [locally built vrnetlab image](https://github.com/srl-labs/vrnetlab/tree/master/dell/dell_sonic)
 - `capms_dell_sonic`: runs the `dell_sonic` flavor but with four instead of two machines (this is used for [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack) in order to have dedicated hosts for control plane / worker / firewall)
-- `kamaji`: runs a variation of the `sonic` flavor. The working example is available at the [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack)'s `capi-lab`.
-- `gardener`: runs the `sonic` flavor and installs the [Gardener](https://gardener.cloud) in the mini-lab
+- `kamaji`: runs a variation of the `sonic_vs` flavor. The working example is available at the [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack)'s `capi-lab`.
+- `gardener`: runs the `sonic_vs` flavor and installs the [Gardener](https://gardener.cloud) in the mini-lab
 
 In order to start specific flavor, you can define the flavor as follows:
 
 ```bash
-export MINI_LAB_FLAVOR=sonic
+export MINI_LAB_FLAVOR=sonic_vs
 make
 ```
 

deploy_partition.yaml

@@ -7,7 +7,7 @@
     - name: Wait for system to become reachable
       ansible.builtin.wait_for_connection:
         delay: 10
-        timeout: 50
+        timeout: 120
   roles:
     - name: ansible-common
       tags: always
@@ -161,10 +161,6 @@
   hosts: dell_sonic
   any_errors_fatal: true
   become: true
-  pre_tasks:
-    - name: Wait some time
-      pause:
-        seconds: 120
   roles:
     - name: ansible-common
       tags: always
@@ -203,6 +199,54 @@
     - name: Wait until no route entries have "queued"
       include_tasks: tasks/check_queued.yaml
 
+- name: Work around isc-dhcp/Vlan4000 startup race (Community SONiC)
+  hosts: leaves:!dell_sonic
+  any_errors_fatal: true
+  gather_facts: false
+  tasks:
+    # dhcpd binds Vlan4000 at boot, before the SVI has carrier. Its LPF receive socket then dies
+    # ("receive_packet failed on Vlan4000: Network is down") while dhcpd stays up holding
+    # 0.0.0.0:67 -- so it answers nothing and PXE clients loop DISCOVER forever, never getting a
+    # lease. Restarting it once the underlay is up rebinds the socket, hence the placement after
+    # the underlay wait above.
+    #
+    # The probe solicits a real broadcast DISCOVER rather than poking 10.0.1.1:67, because the UDP
+    # socket stays healthy even when the LPF path is dead -- a unicast check would report a false
+    # pass. Restart and probe are deliberately one task: a retry must re-do the restart, not just
+    # re-probe, and `until` only retries a single task.
+    - name: Restart isc-dhcp-server until it serves an offer on Vlan4000
+      # passed as `cmd` and not as a free-form string, because Ansible runs free-form module
+      # arguments through its own quote-aware splitter before the shell ever sees them, and
+      # that splitter has no concept of comments -- a lone apostrophe in one aborts the play
+      ansible.builtin.shell:
+        executable: /bin/bash
+        cmd: |
+          systemctl restart isc-dhcp-server
+          sleep 2
+
+          # dhcp-server-detector is a pyroute2 >= 0.9.3 console script, and SONiC 202505 pins
+          # pyroute2 0.7.12, so fall back to dhclient there. dhclient has no timeout flag and
+          # ignores `timeout` from a config file -- it always waits ~60s -- hence timeout(1).
+          # -sf /bin/true keeps it from configuring the SVI, and on success it forks a renewing
+          # daemon with an empty pid file, so the leftover is killed by its exact command line.
+          # `pkill -x -f` matches the whole cmdline, so it cannot match the shell running this
+          # script the way a plain `pkill -f` would.
+          if command -v dhcp-server-detector >/dev/null; then
+            dhcp-server-detector Vlan4000 --exit-on-first-offer --duration 10
+          else
+            probe="dhclient -4 -1 -sf /bin/true -lf /run/dhcp-probe.leases -pf /run/dhcp-probe.pid Vlan4000"
+            timeout 15 $probe
+            rc=$?
+            pkill -x -f "$probe" || true
+            exit $rc
+          fi
+      when: dhcp_subnets is defined
+      register: dhcp_offer_check
+      until: dhcp_offer_check.rc == 0
+      retries: 10
+      delay: 5
+      changed_when: true
+
 - name: Configure IPv6 and LLDP ports (Enterprise SONiC)
   hosts: dell_sonic
   any_errors_fatal: true

files/exit/frr.conf

@@ -35,6 +35,11 @@ router bgp 4200000021
  !
  address-family ipv4 unicast
   redistribute connected route-map LOOPBACKS
+  ! PXE egress: hand the leaves a default so machine traffic routes over the fabric to us, and we
+  ! NAT it out eth0 (mgmt). NB this default is also installed into each leaf's KERNEL, which
+  ! redirects the leaf's OWN egress (pixiecore -> github/DNS) through us sourced from its loopback
+  ! -- that only works because network.sh MASQUERADEs everything leaving eth0. Don't drop that NAT.
+  neighbor FABRIC default-originate
  exit-address-family
  !
  address-family ipv6 unicast

files/exit/network.sh

@@ -27,3 +27,19 @@ bridge vlan add vid 1000 untagged pvid dev vniInternet
 ip link set up dev vniInternet
 
 sysctl -w net.ipv6.conf.all.forwarding=1
+
+# PXE egress return-path NAT. Traffic reaching us over the fabric keeps its original source --
+# machines are 10.0.1.0/24, and the leaves' own traffic is sourced from their 10.0.0.0/24
+# loopbacks. The mgmt docker bridge only NATs 172.42.0.0/16, so anything we forward out eth0 must
+# be SNAT'd or the replies have no path back. Masquerade the whole interface rather than a single
+# prefix, so both the machine and leaf-originated ranges are covered.
+# The frr image ships no iptables; bounded + non-fatal on purpose, since this script runs under
+# `set -o errexit` and must never block exit bring-up if the package mirror is unreachable.
+for _ in 1 2 3 4 5; do
+	if apk add --no-cache iptables; then
+		iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
+		break
+	fi
+	echo "apk could not reach the mirror (iptables); retrying ..."
+	sleep 2
+done || true

files/sonic_frr_with_fpm.tpl

@@ -3,6 +3,12 @@ frr defaults datacenter
 # This is the only line changed from upstream: https://github.com/metal-stack/metal-core/blob/master/cmd/internal/switcher/templates/tpl/sonic_frr.tpl
 # Follow-up issue: https://github.com/metal-stack/metal-core/issues/199
 fpm address 127.0.0.1
+# SONiC default (upstream ships this in its zebra config; the metal-core template dropped it).
+# With next-hop-groups enabled (FRR's default under FPM), zebra hands fpmsyncd an NHG id instead
+# of inline nexthops, and the EVPN encap attrs (vni_label/router_mac, carried as RTA_ENCAP) are
+# lost -> ROUTE_TABLE loses vni_label/router_mac -> RouteOrch builds a plain IP nexthop instead of
+# TUNNEL_ENCAP.
+no fpm use-next-hop-groups
 hostname {{ .Name }}
 password zebra
 enable password zebra

images/sonic/Dockerfile.vpp

@@ -0,0 +1,30 @@
+FROM docker.io/library/debian:bookworm-backports
+
+ENV LIBGUESTFS_BACKEND=direct
+
+RUN apt-get update && \
+    apt-get --no-install-recommends install --yes \
+        curl \
+        libpcap0.8 \
+        iproute2 \
+        linux-image-cloud-amd64 \
+        python3 \
+        python3-pip \
+        python3-guestfs \
+        python3-scapy \
+        qemu-system-x86 \
+        telnet
+
+COPY requirements.txt /
+RUN pip install --break-system-packages -r requirements.txt
+
+COPY --from=docker.io/l0wl3vel/mini-lab-sonic-base:vpp-integration /sonic-vs.img /sonic-vs.img
+COPY --from=docker.io/l0wl3vel/mini-lab-sonic-base:vpp-integration /frr-pythontools.deb /frr-pythontools.deb
+
+ENTRYPOINT ["/launch.py"]
+
+COPY mirror_tap_to_eth.sh mirror_tap_to_front_panel.sh port_config.ini launch.py /
+
+# Readiness now waits for the dataplane (front-panel LLDP), which converges later than the
+# mgmt plane, so allow more time before the container is considered unhealthy.
+HEALTHCHECK --start-period=30s --interval=5s --retries=40 CMD test -f /healthy

images/sonic/Dockerfile → images/sonic/Dockerfile.vs

@@ -9,11 +9,15 @@ RUN apt-get update && \
         iproute2 \
         linux-image-cloud-amd64 \
         python3 \
+        python3-pip \
         python3-guestfs \
         python3-scapy \
         qemu-system-x86 \
         telnet
 
+COPY requirements.txt /
+RUN pip install --break-system-packages -r requirements.txt
+
 COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202505 /sonic-vs.img /sonic-vs.img
 COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202505 /frr-pythontools.deb /frr-pythontools.deb
 

images/sonic/README.md

@@ -0,0 +1,12 @@
+# Virtual Sonic Images
+
+We use sonic-vpp to emulate SONiC switches. It is running in kvm inside a containerlab container. To provide better emulation accuracy we use sonic-vpp, which used the Vector Package Processor to emulate somthing like a switch ASIC, like the Broadcom Tomahawk 3 used in our Edgecore Accton AS7726-X32 workhorse we use in production. We migrated to sonic-vpp because the sonic-vs image used mostly netlink primitives, which behaved differently than an ASIC driven through SONiCs SAI layer.  It's slower but still sane.
+
+
+# Configuration knobs
+
+You can edit the port_config.ini to add more ports.
+
+
+# Boot process
+The switch will boot with a default first-boot configuration. This is required since first boot will generate some required configuration for VPP. After a short while the configuration that is generated in launch.py is injected and the sonic is reloaded. After the new configuration is loaded the container will be marked ready. Check the docker logs for errors if bootup takes more than a minute. 

images/sonic/base-vpp/Dockerfile

@@ -0,0 +1,32 @@
+# Check: https://sonic-build.azurewebsites.net/ui/sonic/pipelines
+ARG SONIC_BRANCH=master
+ARG SONIC_BASE_URL=https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=${SONIC_BRANCH}&definitionId=2818&artifactName=sonic-buildimage.vpp
+ARG SONIC_IMG_URL=${SONIC_BASE_URL}&target=target%2Fsonic-vpp.img.gz
+ARG FRR_RELOAD_URL=${SONIC_BASE_URL}&target=target%2Fdebs%2Fbookworm%2Ffrr-pythontools_10.5.4-sonic-0_all.deb
+
+FROM docker.io/library/busybox:stable AS download
+
+ARG SONIC_IMG_URL
+ARG FRR_RELOAD_URL
+
+# TODO: There is no pipeline for
+# https://github.com/l0wl3vel/sonic-buildimage/tree/vpp-integration. Build it. Place the
+# sonic-vpp.img.gz and frr-pythontools_10.5.4-sonic-0_all.deb artifacts here.
+# If you need sonic-vpp source code changes yell at @l0wl3vel to push a fresh image: 
+# docker build -t docker.io/l0wl3vel/mini-lab-sonic-base:vpp-integration images/sonic/base-vpp
+# docker push docker.io/l0wl3vel/mini-lab-sonic-base:vpp-integration
+ADD ./sonic-vpp.img.gz /sonic-vs.img.gz
+ADD ./frr-pythontools_10.5.4-sonic-0_all.deb /frr-pythontools.deb
+
+RUN gunzip /sonic-vs.img.gz
+
+FROM scratch
+
+ARG SONIC_IMG_URL
+ARG FRR_RELOAD_URL
+
+LABEL sonic-img-url=${SONIC_IMG_URL} \
+      frr-reload-url=${FRR_RELOAD_URL}
+
+COPY --from=download /frr-pythontools.deb /frr-pythontools.deb
+COPY --from=download /sonic-vs.img /sonic-vs.img