From 886fd9aa8e55481eace35801a0ade8e4bb2508b1 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Wed, 20 May 2026 21:25:33 +0200 Subject: [PATCH 01/23] feat: add sonic-vpp image --- .github/workflows/base-image.yaml | 1 + Makefile | 1 + images/sonic/Dockerfile | 4 ++-- images/sonic/base-202511-vpp/Dockerfile | 25 +++++++++++++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 images/sonic/base-202511-vpp/Dockerfile diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 039e6f01..83ca6040 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -16,6 +16,7 @@ jobs: - name: 202311 - name: 202411 - name: 202505 + - name: 202511-vpp steps: - name: Log in to the container registry diff --git a/Makefile b/Makefile index bd059e06..c6b4e1b6 100644 --- a/Makefile +++ b/Makefile @@ -429,6 +429,7 @@ build-sonic-base: docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202311 images/sonic/base-202311 docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202411 images/sonic/base-202411 docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202505 images/sonic/base-202505 + docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202511 images/sonic/base-202511-vpp ## DEV TARGETS ## diff --git a/images/sonic/Dockerfile b/images/sonic/Dockerfile index 9565c738..44ad5a43 100644 --- a/images/sonic/Dockerfile +++ b/images/sonic/Dockerfile @@ -14,8 +14,8 @@ RUN apt-get update && \ qemu-system-x86 \ telnet -COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202505 /sonic-vs.img /sonic-vs.img -COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202505 /frr-pythontools.deb /frr-pythontools.deb +COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202511-vpp /sonic-vs.img /sonic-vs.img +COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202511-vpp /frr-pythontools.deb /frr-pythontools.deb ENTRYPOINT ["/launch.py"] diff --git a/images/sonic/base-202511-vpp/Dockerfile b/images/sonic/base-202511-vpp/Dockerfile new file mode 100644 index 00000000..32daf97e --- /dev/null +++ b/images/sonic/base-202511-vpp/Dockerfile @@ -0,0 +1,25 @@ +# Check: https://sonic-build.azurewebsites.net/ui/sonic/pipelines +ARG SONIC_BASE_URL=https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=202511&definitionId=2818&artifactName=sonic-buildimage.vpp +ARG SONIC_IMG_URL=${SONIC_BASE_URL}&target=target%2Fsonic-vpp.img.gz +ARG FRR_RELOAD_URL=${SONIC_BASE_URL}&target=target%2Fdebs%2Fbookworm%2Ffrr-pythontools_10.4.1-sonic-0_all.deb + +FROM docker.io/library/busybox:stable AS download + +ARG SONIC_IMG_URL +ARG FRR_RELOAD_URL + +ADD "${SONIC_IMG_URL}" /sonic-vs.img.gz +ADD "${FRR_RELOAD_URL}" /frr-pythontools.deb + +RUN gunzip /sonic-vs.img.gz + +FROM scratch + +ARG SONIC_IMG_URL +ARG FRR_RELOAD_URL + +LABEL sonic-img-url=${SONIC_IMG_URL} \ + frr-reload-url=${FRR_RELOAD_URL} + +COPY --from=download /frr-pythontools.deb /frr-pythontools.deb +COPY --from=download /sonic-vs.img /sonic-vs.img From 4284272ec32086bee4c189fc472f2e8518575df6 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Sat, 23 May 2026 10:59:08 +0200 Subject: [PATCH 02/23] feat: booting sonic-vpp Signed-off-by: Benjamin Ritter --- Makefile | 4 +- deploy_partition.yaml | 6 +- images/sonic/README.md | 8 ++ images/sonic/launch.py | 145 +++++++++++++++++------- images/sonic/port_config.ini | 125 +------------------- inventories/group_vars/leaves/main.yaml | 2 +- mini-lab.sonic.yaml | 8 +- roles/sonic/tasks/main.yaml | 22 ++-- 8 files changed, 134 insertions(+), 186 deletions(-) create mode 100644 images/sonic/README.md diff --git a/Makefile b/Makefile index c6b4e1b6..68409109 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ ANSIBLE_DISPLAY_SKIPPED_HOSTS=false MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic) MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:latest) -MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest) +MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:202511-vpp) MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1) MINI_LAB_INTERNAL_NETWORK=mini_lab_internal @@ -429,7 +429,7 @@ build-sonic-base: docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202311 images/sonic/base-202311 docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202411 images/sonic/base-202411 docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202505 images/sonic/base-202505 - docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202511 images/sonic/base-202511-vpp + docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202511-vpp images/sonic/base-202511-vpp ## DEV TARGETS ## diff --git a/deploy_partition.yaml b/deploy_partition.yaml index 2ad4de4d..295355e8 100644 --- a/deploy_partition.yaml +++ b/deploy_partition.yaml @@ -7,7 +7,7 @@ - name: Wait for system to become reachable ansible.builtin.wait_for_connection: delay: 10 - timeout: 50 + timeout: 120 roles: - name: ansible-common tags: always @@ -161,10 +161,6 @@ hosts: dell_sonic any_errors_fatal: true become: true - pre_tasks: - - name: Wait some time - pause: - seconds: 120 roles: - name: ansible-common tags: always diff --git a/images/sonic/README.md b/images/sonic/README.md new file mode 100644 index 00000000..bdec0df1 --- /dev/null +++ b/images/sonic/README.md @@ -0,0 +1,8 @@ +# Virtual Sonic Images + +We use sonic-vpp to emulate SONiC switches. It is running in kvm inside a containerlab container. To provide better emulation accuracy we use sonic-vpp, which used the Vector Package Processor to emulate somthing like a switch ASIC, like the Broadcom Tomahawk 3 used in our Edgecore Accton AS7726-X32 workhorse we use in production. We migrated to sonic-vpp because the sonic-vs image used mostly netlink primitives, which behaved differently than an ASIC driven through SONiCs SAI layer. It's slower but still sane. + + +# Configuration knobs + +You can edit the port_config.ini to add more ports. Keep the number as low as possible. It will put less strain on your system because it will spawn fewer VPP worker threads. You will have to set up the switch from scratch afterwards, since VPP will generate some configuration on first startup. \ No newline at end of file diff --git a/images/sonic/launch.py b/images/sonic/launch.py index 41926f3c..9682214b 100755 --- a/images/sonic/launch.py +++ b/images/sonic/launch.py @@ -10,6 +10,7 @@ import struct import subprocess import sys +import telnetlib import time from typing import Callable @@ -48,7 +49,15 @@ def guestfs(self) -> GuestFS: g = guestfs.GuestFS(python_return_dict=True) g.add_drive_opts(filename=self._disk, format="qcow2", readonly=False) g.launch() - g.mount('/dev/sda3', '/') + # SONiC stores its rootfs as a read-only squashfs at /image-*/fs.squashfs; + # the sibling rw/ tree only holds overlay overrides. Use mkmountpoint so + # we can expose both: the writable partition under /disk and the base + # rootfs (loop-mounted from fs.squashfs) under /rootfs. + g.mkmountpoint('/disk') + g.mkmountpoint('/rootfs') + g.mount('/dev/sda3', '/disk') + image = g.glob_expand('/disk/image-*')[0] + g.mount_loop(image + 'fs.squashfs', '/rootfs') return g def start(self) -> None: @@ -78,7 +87,7 @@ def start(self) -> None: with open(f'/sys/class/net/{iface}/address', 'r') as f: mac = f.read().strip() cmd.append('-device') - cmd.append(f'virtio-net-pci,netdev=hn{i},mac={mac}') + cmd.append(f'virtio-net-pci,netdev=hn{i},mac={mac},mq=off,host_mtu=9216') cmd.append(f'-netdev') cmd.append(f'tap,id=hn{i},ifname=tap{i},script=/mirror_tap_to_front_panel.sh,downscript=no') @@ -89,9 +98,9 @@ def wait(self) -> None: def initial_configuration(g: GuestFS, hwsku: str) -> None: - image = g.glob_expand('/image-*')[0] + image = g.glob_expand('/disk/image-*')[0] - g.rm(image + 'platform/firsttime') + # g.rm(image + 'platform/firsttime') systemd_system = image + 'rw/etc/systemd/system/' sonic_target_wants = systemd_system + 'sonic.target.wants/' @@ -101,38 +110,36 @@ def initial_configuration(g: GuestFS, hwsku: str) -> None: g.copy_in(localpath='/frr-pythontools.deb', remotedir=image + 'rw/') # Workaround: Speed up lldp startup by remove hardcoded wait of 90 seconds - g.ln_s(linkname=systemd_system + 'aaastatsd.timer', target='/dev/null') # Radius - # NOTE: featured.timer and hostcfgd.timer are intentionally NOT masked. They run - # SONiC's feature-manager and host-config reconciliation and provide the startup - # ordering that EVPN/FRR bring-up relies on. Masking them raced the remote L3VNI - # VTEP RMAC programming on the leaves, which then required a manual - # `systemctl restart bgp` to recover. lldp/pmon are still hand-wired below so they - # start immediately instead of waiting for featured's delayed timer. - g.ln_s(linkname=systemd_system + 'rasdaemon.timer', target='/dev/null') # After boot Host configuration - g.ln_s(linkname=systemd_system + 'tacacs-config.timer', target='/dev/null') # After boot Host configuration + # g.ln_s(linkname=systemd_system + 'aaastatsd.timer', target='/dev/null') # Radius + # g.ln_s(linkname=systemd_system + 'featured.timer', target='/dev/null') # Feature handling not necessary + # g.ln_s(linkname=systemd_system + 'hostcfgd.timer', target='/dev/null') # After boot Host configuration + # g.ln_s(linkname=systemd_system + 'rasdaemon.timer', target='/dev/null') # After boot Host configuration + # g.ln_s(linkname=systemd_system + 'tacacs-config.timer', target='/dev/null') # After boot Host configuration # Started by featured - g.ln_s(linkname=sonic_target_wants + 'lldp.service', target='/lib/systemd/system/lldp.service') - g.ln_s(linkname=systemd_system + 'pmon.service', target='/lib/systemd/system/pmon.service') - g.ln_s(linkname=sonic_target_wants + 'pmon.service', target='/lib/systemd/system/pmon.service') + # g.ln_s(linkname=sonic_target_wants + 'lldp.service', target='/lib/systemd/system/lldp.service') + # g.ln_s(linkname=systemd_system + 'pmon.service', target='/lib/systemd/system/pmon.service') + # g.ln_s(linkname=sonic_target_wants + 'pmon.service', target='/lib/systemd/system/pmon.service') # Workaround: Only useful for BackEndToRRouter - g.ln_s(linkname=systemd_system + 'backend-acl.service', target='/dev/null') + # g.ln_s(linkname=systemd_system + 'backend-acl.service', target='/dev/null') # Workaround: We don't need LACP - g.ln_s(linkname=systemd_system + 'teamd.service', target='/dev/null') + # g.ln_s(linkname=systemd_system + 'teamd.service', target='/dev/null') # Workaround: Python module sonic_platform not present on vs images g.ln_s(linkname=systemd_system + 'system-health.service', target='/dev/null') g.ln_s(linkname=systemd_system + 'watchdog-control.service', target='/dev/null') sonic_share = image + 'rw/usr/share/sonic/' - hwsku_dir = image + 'rw' + VS_DEVICES_PATH + hwsku - g.mkdir_p(hwsku_dir) - - g.write(path=image + 'rw' + VS_DEVICES_PATH + 'default_sku', content=f'{hwsku} empty'.encode('utf-8')) - g.ln_s(linkname=sonic_share + 'hwsku', target=VS_DEVICES_PATH + hwsku) - g.ln_s(linkname=sonic_share + 'platform', target=VS_DEVICES_PATH) + platform_dir = image + 'rw' + VS_DEVICES_PATH + hwsku_dir_rw = image + 'rw' + VS_DEVICES_PATH + hwsku + g.mkdir_p(platform_dir) + g.write(path=platform_dir + '/default_sku', content=f'{hwsku} empty'.encode('utf-8')) + # The lanemap.ini file is used by the virtual switch image to assign front panels to the Linux interfaces ethX. + # This assignment will later also be used by the script mirror_tap_to_front_panel.sh. + # g.download(remotefilename=hwsku_dir + '/port_config.ini', filename='/port_config.ini') + # g.download(remotefilename=hwsku_dir + '/lanemap.ini', filename='/lanemap.ini') ifaces = get_ethernet_interfaces() # The port_config.ini file contains the assignment of front panels to lanes. port_config = parse_port_config() @@ -142,20 +149,21 @@ def initial_configuration(g: GuestFS, hwsku: str) -> None: with open('/lanemap.ini', 'w') as f: f.write('\n'.join(lanemap)) - g.copy_in(localpath='/lanemap.ini', remotedir=hwsku_dir) - g.copy_in(localpath='/port_config.ini', remotedir=hwsku_dir) + g.mkdir_p(hwsku_dir_rw) + g.copy_in(localpath='/lanemap.ini', remotedir=hwsku_dir_rw) + g.copy_in(localpath='/port_config.ini', remotedir=hwsku_dir_rw) etc_sonic = image + 'rw/etc/sonic/' g.mkdir_p(etc_sonic) - sonic_version = image.removeprefix('/image-').removesuffix('/') - sonic_environment = f''' - SONIC_VERSION=${sonic_version} - PLATFORM=x86_64-kvm_x86_64-r0 - HWSKU={hwsku} - DEVICE_TYPE=LeafRouter - ASIC_TYPE=vs - '''.encode('utf-8') - g.write(path=etc_sonic + 'sonic-environment', content=sonic_environment) + # sonic_version = image.removeprefix('/image-').removesuffix('/') + # sonic_environment = f''' + # SONIC_VERSION=${sonic_version} + # PLATFORM=x86_64-kvm_x86_64-r0 + # HWSKU={hwsku} + # DEVICE_TYPE=LeafRouter + # ASIC_TYPE=vpp + # '''.encode('utf-8') + # g.write(path=etc_sonic + 'sonic-environment', content=sonic_environment) config_db = create_config_db(hwsku) ports = {} @@ -168,7 +176,7 @@ def initial_configuration(g: GuestFS, hwsku: str) -> None: config_db['PORT'] = ports config_db_json = json.dumps(config_db, indent=4, sort_keys=True) - g.write(path=etc_sonic + 'config_db.json', content=config_db_json.encode('utf-8')) + g.write(path=image + 'rw/golden_config_db.json', content=config_db_json.encode('utf-8')) if os.path.exists('/authorized_keys'): g.mkdir_p(image + 'rw/root/.ssh') @@ -185,8 +193,8 @@ def main(): logger = logging.getLogger() name = os.getenv('CLAB_LABEL_CLAB_NODE_NAME', default='switch') - smp = os.getenv('QEMU_SMP', default='2') - memory = os.getenv('QEMU_MEMORY', default='2048') + smp = os.getenv('QEMU_SMP', default='4') + memory = os.getenv('QEMU_MEMORY', default='4096') interfaces = int(os.getenv('CLAB_INTFS', 0)) + 1 hwsku = os.getenv('HWSKU', default='Accton-AS7726-32X') @@ -207,6 +215,8 @@ def main(): logger.info('Start QEMU') vm.start() + apply_golden_config_via_serial(logger) + # Readiness: wait until SONiC forwards its own LLDP out a *front-panel* port, not just # mgmt eth0. LLDP egress on a front-panel port requires PortConfigDone AND the port to # be programmed/oper-up in the (v)ASIC dataplane, so this signals dataplane readiness @@ -229,6 +239,57 @@ def handle_exit(signal, frame): sys.exit(0) +def apply_golden_config_via_serial(logger) -> None: + logger.info('Connecting to SONiC serial console on 127.0.0.1:5000') + while True: + try: + tn = telnetlib.Telnet('127.0.0.1', 5000, timeout=600) + break + except ConnectionRefusedError: + time.sleep(1) + + def send(data: bytes, *, redact: bool = False) -> None: + display = '***' if redact else data.rstrip(b'\n').decode('utf-8', errors='replace') + logger.info(f'serial> {display}') + tn.write(data) + + def read_until(marker: bytes, timeout: int) -> str: + text = tn.read_until(marker, timeout=timeout).decode('utf-8', errors='replace') + for line in text.splitlines(): + stripped = line.rstrip() + if stripped: + logger.info(f'serial< {stripped}') + return text + + logger.info('Waiting for login prompt') + read_until(b'login: ', timeout=600) + send(b'admin\n') + + read_until(b'Password: ', timeout=60) + send(b'YourPaSsWoRd\n', redact=True) + + read_until(b'$ ', timeout=60) + + # hacked together system readiness check since show system-health does not work in virtual sonic + # stolen from https://github.com/sonic-net/sonic-utilities/blob/master/config/main.py + logger.info('Waiting for systemctl is-system-running to return running') + while True: + send(b'sudo systemctl is-system-running\n') + text = read_until(b'$ ', timeout=30) + if any(line.strip() == 'running' for line in text.splitlines()): + break + time.sleep(5) + + logger.info('Installing golden config_db.json') + send(b'sudo config reload -f -y /golden_config_db.json \n') + read_until(b'$ ', timeout=60) + + #logger.info('Rebooting SONiC to apply golden config') + #send(b'sudo reboot\n') + + tn.close() + + def wait_until_all_interfaces_are_connected(interfaces: int) -> None: while True: i = 0 @@ -367,11 +428,13 @@ def create_config_db(hwsku: str) -> dict: 'admin_status': 'up' } }, - 'VERSIONS': { - 'DATABASE': { - 'VERSION': 'version_202311_03' + 'LLDP': { + 'GLOBAL': { + 'enabled': 'true', + 'hello_time': '10' } } + } diff --git a/images/sonic/port_config.ini b/images/sonic/port_config.ini index acc1f3d2..2dfa3d87 100644 --- a/images/sonic/port_config.ini +++ b/images/sonic/port_config.ini @@ -1,123 +1,4 @@ # name lanes alias index speed -Ethernet0 1 Eth1/1 1 25000 -Ethernet1 2 Eth1/2 1 25000 -Ethernet2 3 Eth1/3 1 25000 -Ethernet3 4 Eth1/4 1 25000 -Ethernet4 5 Eth2/1 2 25000 -Ethernet5 6 Eth2/2 2 25000 -Ethernet6 7 Eth2/3 2 25000 -Ethernet7 8 Eth2/4 2 25000 -Ethernet8 9 Eth3/1 3 25000 -Ethernet9 10 Eth3/2 3 25000 -Ethernet10 11 Eth3/3 3 25000 -Ethernet11 12 Eth3/4 3 25000 -Ethernet12 13 Eth4/1 4 25000 -Ethernet13 14 Eth4/2 4 25000 -Ethernet14 15 Eth4/3 4 25000 -Ethernet15 16 Eth4/4 4 25000 -Ethernet16 17 Eth5/1 5 25000 -Ethernet17 18 Eth5/2 5 25000 -Ethernet18 19 Eth5/3 5 25000 -Ethernet19 20 Eth5/4 5 25000 -Ethernet20 21 Eth6/1 6 25000 -Ethernet21 22 Eth6/2 6 25000 -Ethernet22 23 Eth6/3 6 25000 -Ethernet23 24 Eth6/4 6 25000 -Ethernet24 25 Eth7/1 7 25000 -Ethernet25 26 Eth7/2 7 25000 -Ethernet26 27 Eth7/3 7 25000 -Ethernet27 28 Eth7/4 7 25000 -Ethernet28 29 Eth8/1 8 25000 -Ethernet29 30 Eth8/2 8 25000 -Ethernet30 31 Eth8/3 8 25000 -Ethernet31 32 Eth8/4 8 25000 -Ethernet32 33 Eth9/1 9 25000 -Ethernet33 34 Eth9/2 9 25000 -Ethernet34 35 Eth9/3 9 25000 -Ethernet35 36 Eth9/4 9 25000 -Ethernet36 37 Eth10/1 10 25000 -Ethernet37 38 Eth10/2 10 25000 -Ethernet38 39 Eth10/3 10 25000 -Ethernet39 40 Eth10/4 10 25000 -Ethernet40 41 Eth11/1 11 25000 -Ethernet41 42 Eth11/2 11 25000 -Ethernet42 43 Eth11/3 11 25000 -Ethernet43 44 Eth11/4 11 25000 -Ethernet44 45 Eth12/1 12 25000 -Ethernet45 46 Eth12/2 12 25000 -Ethernet46 47 Eth12/3 12 25000 -Ethernet47 48 Eth12/4 12 25000 -Ethernet48 49 Eth13/1 13 25000 -Ethernet49 50 Eth13/2 13 25000 -Ethernet50 51 Eth13/3 13 25000 -Ethernet51 52 Eth13/4 13 25000 -Ethernet52 53 Eth14/1 14 25000 -Ethernet53 54 Eth14/2 14 25000 -Ethernet54 55 Eth14/3 14 25000 -Ethernet55 56 Eth14/4 14 25000 -Ethernet56 57 Eth15/1 15 25000 -Ethernet57 58 Eth15/2 15 25000 -Ethernet58 59 Eth15/3 15 25000 -Ethernet59 60 Eth15/4 15 25000 -Ethernet60 61 Eth16/1 16 25000 -Ethernet61 62 Eth16/2 16 25000 -Ethernet62 63 Eth16/3 16 25000 -Ethernet63 64 Eth16/4 16 25000 -Ethernet64 65 Eth17/1 17 25000 -Ethernet65 66 Eth17/2 17 25000 -Ethernet66 67 Eth17/3 17 25000 -Ethernet67 68 Eth17/4 17 25000 -Ethernet68 69 Eth18/1 18 25000 -Ethernet69 70 Eth18/2 18 25000 -Ethernet70 71 Eth18/3 18 25000 -Ethernet71 72 Eth18/4 18 25000 -Ethernet72 73 Eth19/1 19 25000 -Ethernet73 74 Eth19/2 19 25000 -Ethernet74 75 Eth19/3 19 25000 -Ethernet75 76 Eth19/4 19 25000 -Ethernet76 77 Eth20/1 20 25000 -Ethernet77 78 Eth20/2 20 25000 -Ethernet78 79 Eth20/3 20 25000 -Ethernet79 80 Eth20/4 20 25000 -Ethernet80 81 Eth21/1 21 25000 -Ethernet81 82 Eth21/2 21 25000 -Ethernet82 83 Eth21/3 21 25000 -Ethernet83 84 Eth21/4 21 25000 -Ethernet84 85 Eth22/1 22 25000 -Ethernet85 86 Eth22/2 22 25000 -Ethernet86 87 Eth22/3 22 25000 -Ethernet87 88 Eth22/4 22 25000 -Ethernet88 89 Eth23/1 23 25000 -Ethernet89 90 Eth23/2 23 25000 -Ethernet90 91 Eth23/3 23 25000 -Ethernet91 92 Eth23/4 23 25000 -Ethernet92 93 Eth24/1 24 25000 -Ethernet93 94 Eth24/2 24 25000 -Ethernet94 95 Eth24/3 24 25000 -Ethernet95 96 Eth24/4 24 25000 -Ethernet96 97 Eth25/1 25 25000 -Ethernet97 98 Eth25/2 25 25000 -Ethernet98 99 Eth25/3 25 25000 -Ethernet99 100 Eth25/4 25 25000 -Ethernet100 101 Eth26/1 26 25000 -Ethernet101 102 Eth26/2 26 25000 -Ethernet102 103 Eth26/3 26 25000 -Ethernet103 104 Eth26/4 26 25000 -Ethernet104 105 Eth27/1 27 25000 -Ethernet105 106 Eth27/2 27 25000 -Ethernet106 107 Eth27/3 27 25000 -Ethernet107 108 Eth27/4 27 25000 -Ethernet108 109 Eth28/1 28 25000 -Ethernet109 110 Eth28/2 28 25000 -Ethernet110 111 Eth28/3 28 25000 -Ethernet111 112 Eth28/4 28 25000 -Ethernet112 113 Eth29/1 29 25000 -Ethernet113 114 Eth29/2 29 25000 -Ethernet114 115 Eth29/3 29 25000 -Ethernet115 116 Eth29/4 29 25000 -Ethernet116 117 Eth30/1 30 25000 -Ethernet117 118 Eth30/2 30 25000 -Ethernet118 119 Eth30/3 30 25000 -Ethernet119 120 Eth30/4 30 25000 -Ethernet120 121,122,123,124 Eth31 31 100000 -Ethernet124 125,126,127,128 Eth32 32 100000 +Ethernet0 1,2,3,4 Eth1 1 100000 +Ethernet4 5,6,7,8 Eth2 2 100000 +Ethernet8 121,122,123,124 Eth3 3 100000 \ No newline at end of file diff --git a/inventories/group_vars/leaves/main.yaml b/inventories/group_vars/leaves/main.yaml index e1d1e596..765c3bec 100644 --- a/inventories/group_vars/leaves/main.yaml +++ b/inventories/group_vars/leaves/main.yaml @@ -4,7 +4,7 @@ dhcp_listening_interfaces: metal_core_cidr_mask: 25 metal_core_spine_uplinks: - - Ethernet120 + - Ethernet8 sonic_config_docker_routing_config_mode: split-unified sonic_config_frr_render: false diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml index a14ad7e0..c29fd11a 100644 --- a/mini-lab.sonic.yaml +++ b/mini-lab.sonic.yaml @@ -58,7 +58,7 @@ topology: mtu: 9000 - endpoints: ["leaf01:Ethernet0", "machine01:lan0"] - endpoints: ["leaf02:Ethernet0", "machine01:lan1"] - - endpoints: ["leaf01:Ethernet1", "machine02:lan0"] - - endpoints: ["leaf02:Ethernet1", "machine02:lan1"] - - endpoints: ["leaf01:Ethernet120", "exit:eth1"] - - endpoints: ["leaf02:Ethernet120", "exit:eth2"] + - endpoints: ["leaf01:Ethernet4", "machine02:lan0"] + - endpoints: ["leaf02:Ethernet4", "machine02:lan1"] + - endpoints: ["leaf01:Ethernet8", "exit:eth1"] + - endpoints: ["leaf02:Ethernet8", "exit:eth2"] diff --git a/roles/sonic/tasks/main.yaml b/roles/sonic/tasks/main.yaml index c8ee8460..444c7367 100644 --- a/roles/sonic/tasks/main.yaml +++ b/roles/sonic/tasks/main.yaml @@ -2,15 +2,15 @@ - name: Install frr-pythontools ansible.builtin.import_tasks: frr-reload.yaml -- name: Fix Network Performance - ansible.builtin.import_tasks: fix-network-performance.yaml +# - name: Fix Network Performance +# ansible.builtin.import_tasks: fix-network-performance.yaml -- name: Set lldp tx-interval to 10 - ansible.builtin.command: lldpcli configure lldp tx-interval 10 - retries: 10 - delay: 3 - register: result - until: result.rc == 0 +# - name: Set lldp tx-interval to 10 +# ansible.builtin.command: lldpcli configure lldp tx-interval 10 +# retries: 10 +# delay: 3 +# register: result +# until: result.rc == 0 - name: Activate IP MASQUERADE on eth0 ansible.builtin.iptables: @@ -26,9 +26,9 @@ sysctl_set: yes value: "1" -# We need to fill some values for the sonic-exporter (uses the STATE_DB) -- name: Mock sonic platform for kvm - ansible.builtin.import_tasks: mock-platform.yaml +# # We need to fill some values for the sonic-exporter (uses the STATE_DB) +# - name: Mock sonic platform for kvm +# ansible.builtin.import_tasks: mock-platform.yaml # ntp restarting for monitoring -> otherwise some NodeTimeOutOfSync error - name: restart chrony From f524136caa04c31e9372bd3dcb18de6b0c4b4217 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 26 May 2026 07:42:58 +0200 Subject: [PATCH 03/23] feat: wire up SONiC DHCP relay Signed-off-by: Benjamin Ritter From a8574f7ce6a12144b3be66e43ca2dc18e6ca7a36 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 28 May 2026 11:24:24 +0200 Subject: [PATCH 04/23] feat: use sonic-vpp master branch build includes https://github.com/sonic-net/sonic-platform-vpp/pull/212 and https://github.com/sonic-net/sonic-platform-vpp/pull/220 for troubleshooting reasons Signed-off-by: Benjamin Ritter --- images/sonic/Dockerfile | 4 ++-- images/sonic/{base-202511-vpp => base-vpp}/Dockerfile | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) rename images/sonic/{base-202511-vpp => base-vpp}/Dockerfile (81%) diff --git a/images/sonic/Dockerfile b/images/sonic/Dockerfile index 44ad5a43..09b4ebc0 100644 --- a/images/sonic/Dockerfile +++ b/images/sonic/Dockerfile @@ -14,8 +14,8 @@ RUN apt-get update && \ qemu-system-x86 \ telnet -COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202511-vpp /sonic-vs.img /sonic-vs.img -COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202511-vpp /frr-pythontools.deb /frr-pythontools.deb +COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:master-vpp /sonic-vs.img /sonic-vs.img +COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:master-vpp /frr-pythontools.deb /frr-pythontools.deb ENTRYPOINT ["/launch.py"] diff --git a/images/sonic/base-202511-vpp/Dockerfile b/images/sonic/base-vpp/Dockerfile similarity index 81% rename from images/sonic/base-202511-vpp/Dockerfile rename to images/sonic/base-vpp/Dockerfile index 32daf97e..277cda85 100644 --- a/images/sonic/base-202511-vpp/Dockerfile +++ b/images/sonic/base-vpp/Dockerfile @@ -1,7 +1,8 @@ # Check: https://sonic-build.azurewebsites.net/ui/sonic/pipelines -ARG SONIC_BASE_URL=https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=202511&definitionId=2818&artifactName=sonic-buildimage.vpp +ARG SONIC_BRANCH=master +ARG SONIC_BASE_URL=https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=${SONIC_BRANCH}&definitionId=2818&artifactName=sonic-buildimage.vpp ARG SONIC_IMG_URL=${SONIC_BASE_URL}&target=target%2Fsonic-vpp.img.gz -ARG FRR_RELOAD_URL=${SONIC_BASE_URL}&target=target%2Fdebs%2Fbookworm%2Ffrr-pythontools_10.4.1-sonic-0_all.deb +ARG FRR_RELOAD_URL=${SONIC_BASE_URL}&target=target%2Fdebs%2Fbookworm%2Ffrr-pythontools_10.5.4-sonic-0_all.deb FROM docker.io/library/busybox:stable AS download From 0df9bce17a83c176793fc98c749471160a468768 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 28 May 2026 13:09:39 +0200 Subject: [PATCH 05/23] feat: clean up sonic launch.py Removed hacks, that just about worked in sonic-vs but broke in sonic-vpp - sonic-vpp requires first time init to generate VPP config files from hwsku lanemap and port config. If skipped the syncd container, in which vpp runs, will crash immediately - /etc/sonic/sonic-environment is generated on first boot from /usr/share/sonic/device/x86_64-kvm_x86_64-r0/default_sku, so writing it serves no purpose as we reenabled firstboot - switch to telnetlib3, due to telnetlib being deprecated Signed-off-by: Benjamin Ritter --- images/sonic/Dockerfile | 4 +++ images/sonic/launch.py | 64 +++++++---------------------------- images/sonic/requirements.txt | 1 + 3 files changed, 17 insertions(+), 52 deletions(-) create mode 100644 images/sonic/requirements.txt diff --git a/images/sonic/Dockerfile b/images/sonic/Dockerfile index 09b4ebc0..c4339242 100644 --- a/images/sonic/Dockerfile +++ b/images/sonic/Dockerfile @@ -9,11 +9,15 @@ RUN apt-get update && \ iproute2 \ linux-image-cloud-amd64 \ python3 \ + python3-pip \ python3-guestfs \ python3-scapy \ qemu-system-x86 \ telnet +COPY requirements.txt / +RUN pip install --break-system-packages -r requirements.txt + COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:master-vpp /sonic-vs.img /sonic-vs.img COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:master-vpp /frr-pythontools.deb /frr-pythontools.deb diff --git a/images/sonic/launch.py b/images/sonic/launch.py index 9682214b..0f704634 100755 --- a/images/sonic/launch.py +++ b/images/sonic/launch.py @@ -10,7 +10,7 @@ import struct import subprocess import sys -import telnetlib +import telnetlib3 import time from typing import Callable @@ -99,72 +99,35 @@ def wait(self) -> None: def initial_configuration(g: GuestFS, hwsku: str) -> None: image = g.glob_expand('/disk/image-*')[0] - - # g.rm(image + 'platform/firsttime') - - systemd_system = image + 'rw/etc/systemd/system/' - sonic_target_wants = systemd_system + 'sonic.target.wants/' - g.mkdir_p(sonic_target_wants) - # Copy frr-pythontools into the image + g.mkdir_p(image + 'rw/') g.copy_in(localpath='/frr-pythontools.deb', remotedir=image + 'rw/') - # Workaround: Speed up lldp startup by remove hardcoded wait of 90 seconds - # g.ln_s(linkname=systemd_system + 'aaastatsd.timer', target='/dev/null') # Radius - # g.ln_s(linkname=systemd_system + 'featured.timer', target='/dev/null') # Feature handling not necessary - # g.ln_s(linkname=systemd_system + 'hostcfgd.timer', target='/dev/null') # After boot Host configuration - # g.ln_s(linkname=systemd_system + 'rasdaemon.timer', target='/dev/null') # After boot Host configuration - # g.ln_s(linkname=systemd_system + 'tacacs-config.timer', target='/dev/null') # After boot Host configuration - # Started by featured - # g.ln_s(linkname=sonic_target_wants + 'lldp.service', target='/lib/systemd/system/lldp.service') - # g.ln_s(linkname=systemd_system + 'pmon.service', target='/lib/systemd/system/pmon.service') - # g.ln_s(linkname=sonic_target_wants + 'pmon.service', target='/lib/systemd/system/pmon.service') - - # Workaround: Only useful for BackEndToRRouter - # g.ln_s(linkname=systemd_system + 'backend-acl.service', target='/dev/null') - - # Workaround: We don't need LACP - # g.ln_s(linkname=systemd_system + 'teamd.service', target='/dev/null') - # Workaround: Python module sonic_platform not present on vs images + systemd_system = image + 'rw/etc/systemd/system/' + g.mkdir_p(systemd_system) g.ln_s(linkname=systemd_system + 'system-health.service', target='/dev/null') g.ln_s(linkname=systemd_system + 'watchdog-control.service', target='/dev/null') sonic_share = image + 'rw/usr/share/sonic/' platform_dir = image + 'rw' + VS_DEVICES_PATH - hwsku_dir_rw = image + 'rw' + VS_DEVICES_PATH + hwsku g.mkdir_p(platform_dir) g.write(path=platform_dir + '/default_sku', content=f'{hwsku} empty'.encode('utf-8')) # The lanemap.ini file is used by the virtual switch image to assign front panels to the Linux interfaces ethX. # This assignment will later also be used by the script mirror_tap_to_front_panel.sh. - # g.download(remotefilename=hwsku_dir + '/port_config.ini', filename='/port_config.ini') - # g.download(remotefilename=hwsku_dir + '/lanemap.ini', filename='/lanemap.ini') + # Dynamic breakouts are not implemented in sonic-vs/sonic-vpp ifaces = get_ethernet_interfaces() - # The port_config.ini file contains the assignment of front panels to lanes. port_config = parse_port_config() - # The lanemap.ini file is used by the virtual switch image to assign front panels to the Linux interfaces ethX. - # This assignment will later also be used by the script mirror_tap_to_front_panel.sh. lanemap = create_lanemap(port_config, ifaces) with open('/lanemap.ini', 'w') as f: f.write('\n'.join(lanemap)) + hwsku_dir_rw = image + 'rw' + VS_DEVICES_PATH + hwsku g.mkdir_p(hwsku_dir_rw) g.copy_in(localpath='/lanemap.ini', remotedir=hwsku_dir_rw) g.copy_in(localpath='/port_config.ini', remotedir=hwsku_dir_rw) - etc_sonic = image + 'rw/etc/sonic/' - g.mkdir_p(etc_sonic) - # sonic_version = image.removeprefix('/image-').removesuffix('/') - # sonic_environment = f''' - # SONIC_VERSION=${sonic_version} - # PLATFORM=x86_64-kvm_x86_64-r0 - # HWSKU={hwsku} - # DEVICE_TYPE=LeafRouter - # ASIC_TYPE=vpp - # '''.encode('utf-8') - # g.write(path=etc_sonic + 'sonic-environment', content=sonic_environment) - config_db = create_config_db(hwsku) ports = {} for iface in ifaces: @@ -176,7 +139,7 @@ def initial_configuration(g: GuestFS, hwsku: str) -> None: config_db['PORT'] = ports config_db_json = json.dumps(config_db, indent=4, sort_keys=True) - g.write(path=image + 'rw/golden_config_db.json', content=config_db_json.encode('utf-8')) + g.write(path=image + 'rw/init_config_db.json', content=config_db_json.encode('utf-8')) if os.path.exists('/authorized_keys'): g.mkdir_p(image + 'rw/root/.ssh') @@ -215,7 +178,7 @@ def main(): logger.info('Start QEMU') vm.start() - apply_golden_config_via_serial(logger) + apply_init_config_via_serial(logger) # Readiness: wait until SONiC forwards its own LLDP out a *front-panel* port, not just # mgmt eth0. LLDP egress on a front-panel port requires PortConfigDone AND the port to @@ -239,11 +202,11 @@ def handle_exit(signal, frame): sys.exit(0) -def apply_golden_config_via_serial(logger) -> None: +def apply_init_config_via_serial(logger) -> None: logger.info('Connecting to SONiC serial console on 127.0.0.1:5000') while True: try: - tn = telnetlib.Telnet('127.0.0.1', 5000, timeout=600) + tn = telnetlib3.Telnet('127.0.0.1', 5000, timeout=600) break except ConnectionRefusedError: time.sleep(1) @@ -280,13 +243,10 @@ def read_until(marker: bytes, timeout: int) -> str: break time.sleep(5) - logger.info('Installing golden config_db.json') - send(b'sudo config reload -f -y /golden_config_db.json \n') + logger.info('Installing intial config_db.json') + send(b'sudo config reload -f -y /init_config_db.json \n') read_until(b'$ ', timeout=60) - #logger.info('Rebooting SONiC to apply golden config') - #send(b'sudo reboot\n') - tn.close() diff --git a/images/sonic/requirements.txt b/images/sonic/requirements.txt new file mode 100644 index 00000000..4973e51b --- /dev/null +++ b/images/sonic/requirements.txt @@ -0,0 +1 @@ +telnetlib3~=4.0.4 \ No newline at end of file From 7d7b6969b72bf81958dd78903e96c8cd311432ec Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 29 May 2026 15:30:54 +0200 Subject: [PATCH 06/23] fix: add more documentation Signed-off-by: Benjamin Ritter --- images/sonic/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/images/sonic/README.md b/images/sonic/README.md index bdec0df1..ecc5d770 100644 --- a/images/sonic/README.md +++ b/images/sonic/README.md @@ -5,4 +5,8 @@ We use sonic-vpp to emulate SONiC switches. It is running in kvm inside a contai # Configuration knobs -You can edit the port_config.ini to add more ports. Keep the number as low as possible. It will put less strain on your system because it will spawn fewer VPP worker threads. You will have to set up the switch from scratch afterwards, since VPP will generate some configuration on first startup. \ No newline at end of file +You can edit the port_config.ini to add more ports. + + +# Boot process +The switch will boot with a default first-boot configuration. This is required since first boot will generate some required configuration for VPP. After a short while the configuration that is generated in launch.py is injected and the sonic is reloaded. After the new configuration is loaded the container will be marked ready. Check the docker logs for errors if bootup takes more than a minute. \ No newline at end of file From 17fd94511df922c8cf6bf7049e5deefacd29580a Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Sun, 19 Jul 2026 16:15:36 +0200 Subject: [PATCH 07/23] feat: implement internet egress via exit node --- files/exit/frr.conf | 5 +++++ files/exit/network.sh | 16 ++++++++++++++++ mini-lab.sonic.yaml | 1 - 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/files/exit/frr.conf b/files/exit/frr.conf index 5a94dc6b..bf16e1dd 100644 --- a/files/exit/frr.conf +++ b/files/exit/frr.conf @@ -35,6 +35,11 @@ router bgp 4200000021 ! address-family ipv4 unicast redistribute connected route-map LOOPBACKS + ! PXE egress: hand the leaves a default so machine traffic routes over the fabric to us, and we + ! NAT it out eth0 (mgmt). NB this default is also installed into each leaf's KERNEL, which + ! redirects the leaf's OWN egress (pixiecore -> github/DNS) through us sourced from its loopback + ! -- that only works because network.sh MASQUERADEs everything leaving eth0. Don't drop that NAT. + neighbor FABRIC default-originate exit-address-family ! address-family ipv6 unicast diff --git a/files/exit/network.sh b/files/exit/network.sh index 6ad64477..ca5800db 100644 --- a/files/exit/network.sh +++ b/files/exit/network.sh @@ -27,3 +27,19 @@ bridge vlan add vid 1000 untagged pvid dev vniInternet ip link set up dev vniInternet sysctl -w net.ipv6.conf.all.forwarding=1 + +# PXE egress return-path NAT. Traffic reaching us over the fabric keeps its original source -- +# machines are 10.0.1.0/24, and the leaves' own traffic is sourced from their 10.0.0.0/24 +# loopbacks. The mgmt docker bridge only NATs 172.42.0.0/16, so anything we forward out eth0 must +# be SNAT'd or the replies have no path back. Masquerade the whole interface rather than a single +# prefix, so both the machine and leaf-originated ranges are covered. +# The frr image ships no iptables; bounded + non-fatal on purpose, since this script runs under +# `set -o errexit` and must never block exit bring-up if the package mirror is unreachable. +for _ in 1 2 3 4 5; do + if apk add --no-cache iptables; then + iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE + break + fi + echo "apk could not reach the mirror (iptables); retrying ..." + sleep 2 +done || true diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml index c29fd11a..44d7e8fd 100644 --- a/mini-lab.sonic.yaml +++ b/mini-lab.sonic.yaml @@ -10,7 +10,6 @@ topology: nodes: exit: image: quay.io/frrouting/frr:10.3.0 - network-mode: none binds: - files/exit/daemons:/etc/frr/daemons - files/exit/frr.conf:/etc/frr/frr.conf From 80755a3b68c417b92609bf964ede32626a8e9b4a Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Sun, 19 Jul 2026 16:15:44 +0200 Subject: [PATCH 08/23] fix: add workaround for isc not binding to Vlan4000 Signed-off-by: Benjamin Ritter --- deploy_partition.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/deploy_partition.yaml b/deploy_partition.yaml index 295355e8..5ae172d7 100644 --- a/deploy_partition.yaml +++ b/deploy_partition.yaml @@ -199,6 +199,35 @@ - name: Wait until no route entries have "queued" include_tasks: tasks/check_queued.yaml +- name: Work around isc-dhcp/Vlan4000 startup race (Community SONiC) + hosts: leaves:!dell_sonic + any_errors_fatal: true + gather_facts: false + tasks: + # dhcpd binds Vlan4000 at boot, before the SVI has carrier. Its LPF receive socket then dies + # ("receive_packet failed on Vlan4000: Network is down") while dhcpd stays up holding + # 0.0.0.0:67 -- so it answers nothing and PXE clients loop DISCOVER forever, never getting a + # lease. Restarting it once the underlay is up rebinds the socket, hence the placement after + # the underlay wait above. + # + # The probe solicits a real broadcast DISCOVER rather than poking 10.0.1.1:67, because the UDP + # socket stays healthy even when the LPF path is dead -- a unicast check would report a false + # pass. Restart and probe are deliberately one task: a retry must re-do the restart, not just + # re-probe, and `until` only retries a single task. + - name: Restart isc-dhcp-server until it serves an offer on Vlan4000 + ansible.builtin.shell: | + systemctl restart isc-dhcp-server + sleep 2 + dhcp-server-detector Vlan4000 --exit-on-first-offer --duration 10 + args: + executable: /bin/bash + when: dhcp_subnets is defined + register: dhcp_offer_check + until: dhcp_offer_check.rc == 0 + retries: 10 + delay: 5 + changed_when: true + - name: Configure IPv6 and LLDP ports (Enterprise SONiC) hosts: dell_sonic any_errors_fatal: true From 659d067b695237d299020409cb98a28adeadb2ed Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Sun, 19 Jul 2026 16:17:25 +0200 Subject: [PATCH 09/23] fix: fix zebra handing over invalid next-hop groups to fpmsyncd Signed-off-by: Benjamin Ritter --- files/sonic_frr_with_fpm.tpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/files/sonic_frr_with_fpm.tpl b/files/sonic_frr_with_fpm.tpl index 7019b34d..cab4f8a6 100644 --- a/files/sonic_frr_with_fpm.tpl +++ b/files/sonic_frr_with_fpm.tpl @@ -3,6 +3,12 @@ frr defaults datacenter # This is the only line changed from upstream: https://github.com/metal-stack/metal-core/blob/master/cmd/internal/switcher/templates/tpl/sonic_frr.tpl # Follow-up issue: https://github.com/metal-stack/metal-core/issues/199 fpm address 127.0.0.1 +# SONiC default (upstream ships this in its zebra config; the metal-core template dropped it). +# With next-hop-groups enabled (FRR's default under FPM), zebra hands fpmsyncd an NHG id instead +# of inline nexthops, and the EVPN encap attrs (vni_label/router_mac, carried as RTA_ENCAP) are +# lost -> ROUTE_TABLE loses vni_label/router_mac -> RouteOrch builds a plain IP nexthop instead of +# TUNNEL_ENCAP. +no fpm use-next-hop-groups hostname {{ .Name }} password zebra enable password zebra From 430e80341059d02806f22d0d1d1aa86f80784899 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Mon, 27 Jul 2026 09:47:24 +0200 Subject: [PATCH 10/23] fix: improve reliability of front panel port mapping Signed-off-by: Benjamin Ritter --- images/sonic/mirror_tap_to_front_panel.sh | 32 +++++++++++++---------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/images/sonic/mirror_tap_to_front_panel.sh b/images/sonic/mirror_tap_to_front_panel.sh index 4f8c6aa9..4b4a2f02 100755 --- a/images/sonic/mirror_tap_to_front_panel.sh +++ b/images/sonic/mirror_tap_to_front_panel.sh @@ -4,23 +4,27 @@ # Read it for better understanding TAP_IF=$1 -# get interface index number up to 3 digits (everything after first three chars) -# tap0 -> 0 -# tap123 -> 123 +# tap0 -> 0 ... tap123 -> 123 ; tap$INDEX is guest interface eth$INDEX INDEX=${TAP_IF:3:3} -# tap$INDEX corresponds to eth$INDEX in the virtual machine -# The virtual switch assigns lanes to the Linux interface ethX. The assignment is specified in the lanemap.ini file in the following format: ethX:. -LANES=$(grep ^eth$INDEX: /lanemap.ini | cut -d':' -f2) -# Identify the front panel using the lanes. -FRONT_PANEL=$(grep -E "^Ethernet[0-9]+\s+$LANES\s+Eth" /port_config.ini | cut -d' ' -f1) +# sonic-vpp assigns guest NICs to front panels strictly by their order in +# port_config.ini: guest eth$INDEX is the $INDEX-th front panel. Mirror this tap +# to the clab link (named after that front panel) using the SAME order. This is +# independent of how interface names happen to sort (Ethernet4 vs Ethernet12/16, +# breakout sub-ports, ...), which is what the lanemap-based lookup got wrong. +FRONT_PANEL=$(awk '$1 ~ /^Ethernet/ {print $1}' /port_config.ini | sed -n "${INDEX}p") -ip link set $TAP_IF up -ip link set $TAP_IF mtu 65000 +if [ -z "$FRONT_PANEL" ]; then + echo "mirror_tap_to_front_panel: no port_config.ini entry #${INDEX} for ${TAP_IF}" >&2 + exit 1 +fi + +ip link set "$TAP_IF" up +ip link set "$TAP_IF" mtu 65000 # create tc Ethernet<->tap redirect rules -tc qdisc add dev $FRONT_PANEL ingress -tc filter add dev $FRONT_PANEL parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev $TAP_IF +tc qdisc add dev "$FRONT_PANEL" ingress +tc filter add dev "$FRONT_PANEL" parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev "$TAP_IF" -tc qdisc add dev $TAP_IF ingress -tc filter add dev $TAP_IF parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev $FRONT_PANEL +tc qdisc add dev "$TAP_IF" ingress +tc filter add dev "$TAP_IF" parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev "$FRONT_PANEL" From 5973e7981341d8751036be3498c22da9a38f6450 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Mon, 27 Jul 2026 12:21:33 +0200 Subject: [PATCH 11/23] feat: split `sonic` flavor into `sonic_vs` and `sonic_vpp` - Split pipelines to build sonic_vs and sonic_vpp images - Rename sonic flavor to sonic_vs - Add sonic-vpp as sonic_vpp flavor - Run integration tests for sonic_vpp Signed-off-by: Benjamin Ritter --- .github/workflows/integration.yaml | 58 +++++++++++++++++++--- Makefile | 14 ++++-- README.md | 9 ++-- images/sonic/Dockerfile.vpp | 30 +++++++++++ images/sonic/{Dockerfile => Dockerfile.vs} | 4 +- images/sonic/base-vpp/Dockerfile | 10 +++- images/sonic/port_config.ini | 5 +- 7 files changed, 110 insertions(+), 20 deletions(-) create mode 100644 images/sonic/Dockerfile.vpp rename images/sonic/{Dockerfile => Dockerfile.vs} (80%) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 448a52da..45fae115 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -53,8 +53,8 @@ jobs: cache-from: type=registry,ref=${{ env.MINI_LAB_VM_IMAGE }} cache-to: type=inline - build-mini-lab-sonic-image: - name: Build mini-lab-sonic image + build-mini-lab-sonic-vs-image: + name: Build mini-lab-sonic-vs image runs-on: ubuntu-latest steps: @@ -73,8 +73,8 @@ jobs: IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest") SHA_TAG=${COMMIT_SHA::8} - echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic:${IMAGE_TAG}" >> $GITHUB_ENV - echo "MINI_LAB_SONIC_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic:${SHA_TAG}" >> $GITHUB_ENV + echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic-vs:${IMAGE_TAG}" >> $GITHUB_ENV + echo "MINI_LAB_SONIC_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic-vs:${SHA_TAG}" >> $GITHUB_ENV env: COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} @@ -88,24 +88,68 @@ jobs: pull: true push: true sbom: true + file: images/sonic/Dockerfile.vs tags: | ${{ env.MINI_LAB_SONIC_IMAGE }} ${{ env.MINI_LAB_SONIC_IMAGE_SHA }} cache-from: type=registry,ref=${{ env.MINI_LAB_SONIC_IMAGE }} cache-to: type=inline + build-mini-lab-sonic-vpp-image: + name: Build mini-lab-sonic-vpp image + runs-on: ubuntu-latest + + steps: + - name: Log in to the container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKER_REGISTRY_USER }} + password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + + - name: Checkout + uses: actions/checkout@v4 + + - name: Make tag + run: | + IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest") + SHA_TAG=${COMMIT_SHA::8} + + echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic-vpp:${IMAGE_TAG}" >> $GITHUB_ENV + echo "MINI_LAB_SONIC_IMAGE_SHA=ghcr.io/metal-stack/mini-lab-sonic-vpp:${SHA_TAG}" >> $GITHUB_ENV + env: + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push mini-lab-sonic container + uses: docker/build-push-action@v6 + with: + context: ./images/sonic + pull: true + push: true + sbom: true + file: images/sonic/Dockerfile.vpp + tags: | + ${{ env.MINI_LAB_SONIC_IMAGE }} + ${{ env.MINI_LAB_SONIC_IMAGE_SHA }} + cache-from: type=registry,ref=${{ env.MINI_LAB_SONIC_IMAGE }} + cache-to: type=inline test: name: Run tests runs-on: self-hosted needs: - build-mini-lab-vms-image - - build-mini-lab-sonic-image + - build-mini-lab-sonic-vs-image + - build-mini-lab-sonic-vpp-image continue-on-error: true strategy: matrix: flavors: - - name: sonic + - name: sonic_vs + - name: sonic_vpp - name: gardener - name: dell_sonic @@ -140,7 +184,7 @@ jobs: IMAGE_TAG=$([ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo ${GITHUB_HEAD_REF##*/} || echo "latest") echo "MINI_LAB_VM_IMAGE=ghcr.io/metal-stack/mini-lab-vms:${IMAGE_TAG}" >> $GITHUB_ENV - echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic:${IMAGE_TAG}" >> $GITHUB_ENV + echo "MINI_LAB_SONIC_IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_ENV - name: Run integration tests shell: bash diff --git a/Makefile b/Makefile index 68409109..005b2c08 100644 --- a/Makefile +++ b/Makefile @@ -26,10 +26,10 @@ ANSIBLE_EXTRA_VARS_FILE := $(or $(ANSIBLE_EXTRA_VARS_FILE),) # do not show skipped ansible tasks ANSIBLE_DISPLAY_SKIPPED_HOSTS=false -MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic) +MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic_vs) MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:latest) -MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:202511-vpp) MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1) +MINI_LAB_SONIC_IMAGE_TAG := $(or $(MINI_LAB_SONIC_IMAGE_TAG),latest) MINI_LAB_INTERNAL_NETWORK=mini_lab_internal # define this here as well so that kind picks up the network on a clean checkout, @@ -40,9 +40,14 @@ MACHINE_OS=debian-13.0 MAX_RETRIES := 30 # Machine flavors -ifeq ($(MINI_LAB_FLAVOR),sonic) +ifeq ($(MINI_LAB_FLAVOR),sonic_vs) LAB_TOPOLOGY=mini-lab.sonic.yaml MONITORING_ENABLED := $(or $(MONITORING_ENABLED),true) +MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vs:$(MINI_LAB_SONIC_IMAGE_TAG)) +else ifeq ($(MINI_LAB_FLAVOR),sonic_vpp) +LAB_TOPOLOGY=mini-lab.sonic.yaml +MONITORING_ENABLED := $(or $(MONITORING_ENABLED),true) +MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vpp:$(MINI_LAB_SONIC_IMAGE_TAG)) else ifeq ($(MINI_LAB_FLAVOR),dell_sonic) LAB_TOPOLOGY=mini-lab.dell_sonic.yaml MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION) @@ -52,11 +57,13 @@ MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_ else ifeq ($(MINI_LAB_FLAVOR),kamaji) LAB_TOPOLOGY=mini-lab.kamaji.yaml KAMAJI_ENABLED=true +MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vs:$(MINI_LAB_SONIC_IMAGE_TAG)) else ifeq ($(MINI_LAB_FLAVOR),gardener) GARDENER_ENABLED=true # usually gardener restricts the maximum version for k8s: K8S_VERSION=1.35.5 LAB_TOPOLOGY=mini-lab.sonic.yaml +MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic-vs:$(MINI_LAB_SONIC_IMAGE_TAG)) else $(error Unknown flavor $(MINI_LAB_FLAVOR)) endif @@ -429,7 +436,6 @@ build-sonic-base: docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202311 images/sonic/base-202311 docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202411 images/sonic/base-202411 docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202505 images/sonic/base-202505 - docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202511-vpp images/sonic/base-202511-vpp ## DEV TARGETS ## diff --git a/README.md b/README.md index c376a0db..3d273f3f 100644 --- a/README.md +++ b/README.md @@ -238,16 +238,17 @@ make up All available mini-lab flavors are listed below: -- `sonic`: runs two Community SONiC switches +- `sonic_vs`: runs two Community SONiC switches with the kernel based virtual switch dataplane +- `sonic_vpp`: runs two Community SONiC switches with the [VPP](https://fd.io) dataplane - `dell_sonic`: runs two Enterprise SONiC switches with a [locally built vrnetlab image](https://github.com/srl-labs/vrnetlab/tree/master/dell/dell_sonic) - `capms_dell_sonic`: runs the `dell_sonic` flavor but with four instead of two machines (this is used for [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack) in order to have dedicated hosts for control plane / worker / firewall) -- `kamaji`: runs a variation of the `sonic` flavor. The working example is available at the [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack)'s `capi-lab`. -- `gardener`: runs the `sonic` flavor and installs the [Gardener](https://gardener.cloud) in the mini-lab +- `kamaji`: runs a variation of the `sonic_vs` flavor. The working example is available at the [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack)'s `capi-lab`. +- `gardener`: runs the `sonic_vs` flavor and installs the [Gardener](https://gardener.cloud) in the mini-lab In order to start specific flavor, you can define the flavor as follows: ```bash -export MINI_LAB_FLAVOR=sonic +export MINI_LAB_FLAVOR=sonic_vs make ``` diff --git a/images/sonic/Dockerfile.vpp b/images/sonic/Dockerfile.vpp new file mode 100644 index 00000000..b0659514 --- /dev/null +++ b/images/sonic/Dockerfile.vpp @@ -0,0 +1,30 @@ +FROM docker.io/library/debian:bookworm-backports + +ENV LIBGUESTFS_BACKEND=direct + +RUN apt-get update && \ + apt-get --no-install-recommends install --yes \ + curl \ + libpcap0.8 \ + iproute2 \ + linux-image-cloud-amd64 \ + python3 \ + python3-pip \ + python3-guestfs \ + python3-scapy \ + qemu-system-x86 \ + telnet + +COPY requirements.txt / +RUN pip install --break-system-packages -r requirements.txt + +COPY --from=docker.io/l0wl3vel/mini-lab-sonic-base:vpp-integration /sonic-vs.img /sonic-vs.img +COPY --from=docker.io/l0wl3vel/mini-lab-sonic-base:vpp-integration /frr-pythontools.deb /frr-pythontools.deb + +ENTRYPOINT ["/launch.py"] + +COPY mirror_tap_to_eth.sh mirror_tap_to_front_panel.sh port_config.ini launch.py / + +# Readiness now waits for the dataplane (front-panel LLDP), which converges later than the +# mgmt plane, so allow more time before the container is considered unhealthy. +HEALTHCHECK --start-period=30s --interval=5s --retries=40 CMD test -f /healthy diff --git a/images/sonic/Dockerfile b/images/sonic/Dockerfile.vs similarity index 80% rename from images/sonic/Dockerfile rename to images/sonic/Dockerfile.vs index c4339242..0831bc40 100644 --- a/images/sonic/Dockerfile +++ b/images/sonic/Dockerfile.vs @@ -18,8 +18,8 @@ RUN apt-get update && \ COPY requirements.txt / RUN pip install --break-system-packages -r requirements.txt -COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:master-vpp /sonic-vs.img /sonic-vs.img -COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:master-vpp /frr-pythontools.deb /frr-pythontools.deb +COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202505 /sonic-vs.img /sonic-vs.img +COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202505 /frr-pythontools.deb /frr-pythontools.deb ENTRYPOINT ["/launch.py"] diff --git a/images/sonic/base-vpp/Dockerfile b/images/sonic/base-vpp/Dockerfile index 277cda85..fd0de804 100644 --- a/images/sonic/base-vpp/Dockerfile +++ b/images/sonic/base-vpp/Dockerfile @@ -9,8 +9,14 @@ FROM docker.io/library/busybox:stable AS download ARG SONIC_IMG_URL ARG FRR_RELOAD_URL -ADD "${SONIC_IMG_URL}" /sonic-vs.img.gz -ADD "${FRR_RELOAD_URL}" /frr-pythontools.deb +# TODO: There is no pipeline for +# https://github.com/l0wl3vel/sonic-buildimage/tree/vpp-integration. Build it. Place the +# sonic-vpp.img.gz and frr-pythontools_10.5.4-sonic-0_all.deb artifacts here. +# If you need sonic-vpp source code changes yell at @l0wl3vel to push a fresh image: +# docker build -t docker.io/l0wl3vel/mini-lab-sonic-base:vpp-integration images/sonic/base-vpp +# docker push docker.io/l0wl3vel/mini-lab-sonic-base:vpp-integration +ADD ./sonic-vpp.img.gz /sonic-vs.img.gz +ADD ./frr-pythontools_10.5.4-sonic-0_all.deb /frr-pythontools.deb RUN gunzip /sonic-vs.img.gz diff --git a/images/sonic/port_config.ini b/images/sonic/port_config.ini index 2dfa3d87..0aeca974 100644 --- a/images/sonic/port_config.ini +++ b/images/sonic/port_config.ini @@ -1,4 +1,7 @@ +# sonic-vpp uses a simple script to generate vpp interfaces from the port_config.init on first boot +# No breakouts are supported and it always expects Ethernet$(4*n) as port names +# Just continue the pattern if you want vpp to not crash # name lanes alias index speed Ethernet0 1,2,3,4 Eth1 1 100000 Ethernet4 5,6,7,8 Eth2 2 100000 -Ethernet8 121,122,123,124 Eth3 3 100000 \ No newline at end of file +Ethernet8 9,10,11,12 Eth3 3 100000 \ No newline at end of file From 0c33c50358d28dac2fe68b59962556f2d993a567 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Mon, 27 Jul 2026 16:13:19 +0200 Subject: [PATCH 12/23] feat: add memory balloon Signed-off-by: Benjamin Ritter --- images/sonic/launch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/images/sonic/launch.py b/images/sonic/launch.py index 0f704634..3c385feb 100755 --- a/images/sonic/launch.py +++ b/images/sonic/launch.py @@ -73,6 +73,7 @@ def start(self) -> None: '-m', self._memory, '-drive', f'if=virtio,format=qcow2,file={self._disk}', '-serial', 'telnet:127.0.0.1:5000,server,nowait', + '-device', 'virtio-balloon,free-page-reporting=on', ] with open(f'/sys/class/net/eth0/address', 'r') as f: From 9c45780d06f45acb4387c232a79dcc1ff82dbe49 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 30 Jul 2026 14:21:40 +0200 Subject: [PATCH 13/23] fix: add dhcp-server-detector fallback using dhclient fixes sonic_vs with older (202505) image Signed-off-by: Benjamin Ritter --- deploy_partition.yaml | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/deploy_partition.yaml b/deploy_partition.yaml index 5ae172d7..de635061 100644 --- a/deploy_partition.yaml +++ b/deploy_partition.yaml @@ -215,12 +215,31 @@ # pass. Restart and probe are deliberately one task: a retry must re-do the restart, not just # re-probe, and `until` only retries a single task. - name: Restart isc-dhcp-server until it serves an offer on Vlan4000 - ansible.builtin.shell: | - systemctl restart isc-dhcp-server - sleep 2 - dhcp-server-detector Vlan4000 --exit-on-first-offer --duration 10 - args: + # passed as `cmd` and not as a free-form string, because Ansible runs free-form module + # arguments through its own quote-aware splitter before the shell ever sees them, and + # that splitter has no concept of comments -- a lone apostrophe in one aborts the play + ansible.builtin.shell: executable: /bin/bash + cmd: | + systemctl restart isc-dhcp-server + sleep 2 + + # dhcp-server-detector is a pyroute2 >= 0.9.3 console script, and SONiC 202505 pins + # pyroute2 0.7.12, so fall back to dhclient there. dhclient has no timeout flag and + # ignores `timeout` from a config file -- it always waits ~60s -- hence timeout(1). + # -sf /bin/true keeps it from configuring the SVI, and on success it forks a renewing + # daemon with an empty pid file, so the leftover is killed by its exact command line. + # `pkill -x -f` matches the whole cmdline, so it cannot match the shell running this + # script the way a plain `pkill -f` would. + if command -v dhcp-server-detector >/dev/null; then + dhcp-server-detector Vlan4000 --exit-on-first-offer --duration 10 + else + probe="dhclient -4 -1 -sf /bin/true -lf /run/dhcp-probe.leases -pf /run/dhcp-probe.pid Vlan4000" + timeout 15 $probe + rc=$? + pkill -x -f "$probe" || true + exit $rc + fi when: dhcp_subnets is defined register: dhcp_offer_check until: dhcp_offer_check.rc == 0 From 42761a41456841f2c3b744d8170bd417516e9af7 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 30 Jul 2026 15:47:30 +0200 Subject: [PATCH 14/23] fix: enable teamd to fix sonic-vs Signed-off-by: Benjamin Ritter --- images/sonic/launch.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/images/sonic/launch.py b/images/sonic/launch.py index 3c385feb..f69d51f6 100755 --- a/images/sonic/launch.py +++ b/images/sonic/launch.py @@ -373,10 +373,15 @@ def create_config_db(hwsku: str) -> dict: }, 'snmp': { 'state': 'disabled' - }, - 'teamd': { - 'state': 'disabled' } + # teamd must stay enabled: hostcfgd masks the unit of a disabled + # feature, but swss.sh still picks teamd up as a dependency because + # check_service_exists finds masked units as well. docker-wait-any + # then returns immediately since the teamd container never runs, + # which makes systemd restart swss every ~110s. Each restart + # recreates Vlan4000, and isc-dhcp-server dies deaf on it + # ("receive_packet failed on Vlan4000: Network is down"), so + # machines never get a PXE lease. }, 'MGMT_INTERFACE': { f'eth0|{mgmt_interface_cidr}': { From 50e94f9648e36790bb789a1a8fbebe27ea57400b Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 30 Jul 2026 15:56:56 +0200 Subject: [PATCH 15/23] feat: enable gnmi service Signed-off-by: Benjamin Ritter --- images/sonic/launch.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/images/sonic/launch.py b/images/sonic/launch.py index f69d51f6..d6df89d3 100755 --- a/images/sonic/launch.py +++ b/images/sonic/launch.py @@ -365,9 +365,6 @@ def create_config_db(hwsku: str) -> dict: } }, 'FEATURE': { - 'gnmi': { - 'state': 'disabled' - }, 'mgmt-framework': { 'state': 'disabled' }, From dbe0bcbf290138f491d0cb3d415cad302b1732dd Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 30 Jul 2026 16:36:19 +0200 Subject: [PATCH 16/23] fix: remove host_mtu from front panel ports Signed-off-by: Benjamin Ritter --- images/sonic/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/sonic/launch.py b/images/sonic/launch.py index d6df89d3..f7c076af 100755 --- a/images/sonic/launch.py +++ b/images/sonic/launch.py @@ -88,7 +88,7 @@ def start(self) -> None: with open(f'/sys/class/net/{iface}/address', 'r') as f: mac = f.read().strip() cmd.append('-device') - cmd.append(f'virtio-net-pci,netdev=hn{i},mac={mac},mq=off,host_mtu=9216') + cmd.append(f'virtio-net-pci,netdev=hn{i},mac={mac},mq=off') cmd.append(f'-netdev') cmd.append(f'tap,id=hn{i},ifname=tap{i},script=/mirror_tap_to_front_panel.sh,downscript=no') From a4f7fddfe51253204dccefc48a33a40a4a952971 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 30 Jul 2026 16:39:02 +0200 Subject: [PATCH 17/23] fix: re-enable multiqueue on front panel ports Signed-off-by: Benjamin Ritter --- images/sonic/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/sonic/launch.py b/images/sonic/launch.py index f7c076af..b057b065 100755 --- a/images/sonic/launch.py +++ b/images/sonic/launch.py @@ -88,7 +88,7 @@ def start(self) -> None: with open(f'/sys/class/net/{iface}/address', 'r') as f: mac = f.read().strip() cmd.append('-device') - cmd.append(f'virtio-net-pci,netdev=hn{i},mac={mac},mq=off') + cmd.append(f'virtio-net-pci,netdev=hn{i},mac={mac}') cmd.append(f'-netdev') cmd.append(f'tap,id=hn{i},ifname=tap{i},script=/mirror_tap_to_front_panel.sh,downscript=no') From 1d19e825afeacb726a6e5ad0cb13e04ee128d1fb Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 30 Jul 2026 17:04:55 +0200 Subject: [PATCH 18/23] fix: remove unused deactiveat_offloading.sh does nothing anymore Signed-off-by: Benjamin Ritter --- Makefile | 3 +-- scripts/deactivate_offloading.sh | 7 ------- 2 files changed, 1 insertion(+), 9 deletions(-) delete mode 100755 scripts/deactivate_offloading.sh diff --git a/Makefile b/Makefile index 005b2c08..6bc39614 100644 --- a/Makefile +++ b/Makefile @@ -148,8 +148,7 @@ ifneq ($(filter $(MINI_LAB_FLAVOR),dell_sonic capms_dell_sonic),$(MINI_LAB_FLAVO docker pull $(MINI_LAB_SONIC_IMAGE) endif @if ! sudo $(CONTAINERLAB) --topo $(LAB_TOPOLOGY) inspect | grep -i leaf01 > /dev/null; then \ - sudo --preserve-env=MINI_LAB_SONIC_IMAGE --preserve-env=MINI_LAB_DELL_SONIC_VERSION --preserve-env=MINI_LAB_VM_IMAGE $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure && \ - ./scripts/deactivate_offloading.sh; fi + sudo --preserve-env=MINI_LAB_SONIC_IMAGE --preserve-env=MINI_LAB_DELL_SONIC_VERSION --preserve-env=MINI_LAB_VM_IMAGE $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure; fi .PHONY: verify-deployment-image verify-deployment-image: diff --git a/scripts/deactivate_offloading.sh b/scripts/deactivate_offloading.sh deleted file mode 100755 index 987a87df..00000000 --- a/scripts/deactivate_offloading.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -for docker_container_id in $(docker ps | grep ignite | awk '{ print $1 }'); -do - echo "deactivate offloading at veth of leaf switch in docker container ${docker_container_id}" - docker exec "${docker_container_id}" ethtool --offload vm_eth0 tx off -done; From 22ad7b6ea2db80a3284172ac2bbc89f9b6f4ba86 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 30 Jul 2026 17:27:33 +0200 Subject: [PATCH 19/23] fix: re-disable GRO offloading and add documentation Signed-off-by: Benjamin Ritter --- .../sonic/tasks/fix-network-performance.yaml | 28 +++++++++++++++++++ roles/sonic/tasks/main.yaml | 7 +++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/roles/sonic/tasks/fix-network-performance.yaml b/roles/sonic/tasks/fix-network-performance.yaml index c1ccac0a..c8ec3cbc 100644 --- a/roles/sonic/tasks/fix-network-performance.yaml +++ b/roles/sonic/tasks/fix-network-performance.yaml @@ -1,4 +1,32 @@ --- +# GRO must stay off on the front panel NICs. This is not a cosmetic tuning knob, +# it is load bearing: with GRO on, the underlay collapses to ~0.9 MB/s and a +# machine can never finish downloading the metal-hammer initrd, so it never +# leaves "PXE Booting". +# +# Why: sonic-vs forwards frames in userspace. syncd reads them from an AF_PACKET +# socket on ethX and copies the raw bytes into the EthernetX tap. GRO merges +# several TCP segments into one skb but does not rewrite the on-wire TCP +# checksum, because validity is tracked in skb metadata that a raw byte copy +# discards. The receiving kernel re-validates the merged frame, the checksum no +# longer matches, and the segment is dropped silently - visible only as +# TcpInCsumErrors. TCP sees real loss (no DSACK), cwnd collapses to 2 and stays +# there. +# +# Measured on the leaf01 <-> leaf02 underlay: +# GRO on: 0.89 MB/s, cwnd 2, 26% of bytes retransmitted +# GRO off: 7.18 MB/s, cwnd 42, 0 retransmissions +# Pulling the initrd through the fabric: 0.125 MB/s -> 11.4 MB/s. +# +# Note there is no tc-based alternative. containerlab/vrnetlab hit the same +# class of bug on their management datapath and fixed it with +# "tc ... action csum ... pipe action mirred" (srl-labs/vrnetlab#492), but that +# only repairs a copy that tc itself performs. Our corruption happens later, in +# syncd's userspace memcpy inside the guest, where there is no tc hook to attach +# a csum action to. Disabling GRO is the only lever. +# +# Added in mini-lab#200, silently lost again in 4284272 - please do not comment +# this out without re-reading the above. - name: Collect facts about interfaces ansible.builtin.setup: gather_subset: diff --git a/roles/sonic/tasks/main.yaml b/roles/sonic/tasks/main.yaml index 444c7367..c650c242 100644 --- a/roles/sonic/tasks/main.yaml +++ b/roles/sonic/tasks/main.yaml @@ -2,8 +2,11 @@ - name: Install frr-pythontools ansible.builtin.import_tasks: frr-reload.yaml -# - name: Fix Network Performance -# ansible.builtin.import_tasks: fix-network-performance.yaml +# Disables GRO on the front panel NICs. Required for the sonic-vs dataplane to +# forward without corrupting TCP checksums - without it machines never finish +# PXE booting. See the comment in the imported file before touching this. +- name: Fix Network Performance + ansible.builtin.import_tasks: fix-network-performance.yaml # - name: Set lldp tx-interval to 10 # ansible.builtin.command: lldpcli configure lldp tx-interval 10 From 804a9ab92eb5b05439b36764b9d7da9c7bff67fd Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 30 Jul 2026 18:00:10 +0200 Subject: [PATCH 20/23] fix: remove unused lldp tasks it is configured using the sonic startup config, we run lldp.service, so we do not need it anymore Signed-off-by: Benjamin Ritter --- roles/sonic/tasks/main.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/roles/sonic/tasks/main.yaml b/roles/sonic/tasks/main.yaml index c650c242..a5e52127 100644 --- a/roles/sonic/tasks/main.yaml +++ b/roles/sonic/tasks/main.yaml @@ -8,13 +8,6 @@ - name: Fix Network Performance ansible.builtin.import_tasks: fix-network-performance.yaml -# - name: Set lldp tx-interval to 10 -# ansible.builtin.command: lldpcli configure lldp tx-interval 10 -# retries: 10 -# delay: 3 -# register: result -# until: result.rc == 0 - - name: Activate IP MASQUERADE on eth0 ansible.builtin.iptables: chain: POSTROUTING From c9e183750034f322c244eaf175183a926c8c6322 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Thu, 30 Jul 2026 19:09:03 +0200 Subject: [PATCH 21/23] feat: enable management vrf Signed-off-by: Benjamin Ritter --- images/sonic/launch.py | 5 +++++ inventories/group_vars/leaves/main.yaml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/images/sonic/launch.py b/images/sonic/launch.py index b057b065..62b0334b 100755 --- a/images/sonic/launch.py +++ b/images/sonic/launch.py @@ -385,6 +385,11 @@ def create_config_db(hwsku: str) -> dict: 'gwaddr': get_default_gateway(), } }, + 'MGMT_VRF_CONFIG': { + 'vrf_global': { + 'mgmtVrfEnabled': 'true' + } + }, 'MGMT_PORT': { 'eth0': { 'alias': 'eth0', diff --git a/inventories/group_vars/leaves/main.yaml b/inventories/group_vars/leaves/main.yaml index 765c3bec..5af831b4 100644 --- a/inventories/group_vars/leaves/main.yaml +++ b/inventories/group_vars/leaves/main.yaml @@ -14,7 +14,7 @@ sonic_config_mgmt_interface: ip: "{{ ansible_host }}/16`" gateway_address: "172.42.0.1" -sonic_config_mgmt_vrf: false +sonic_config_mgmt_vrf: true sonic_config_nameservers: "{{ router_nameservers }}" sonic_config_vlans: From bb3badd20cbd482acae5f5ba6bd4cb8a62fa10d0 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 31 Jul 2026 10:05:50 +0200 Subject: [PATCH 22/23] Revert "feat: enable management vrf" This reverts commit c9e183750034f322c244eaf175183a926c8c6322. Management VRF requires some more work around bootstrapping the switch --- images/sonic/launch.py | 5 ----- inventories/group_vars/leaves/main.yaml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/images/sonic/launch.py b/images/sonic/launch.py index 62b0334b..b057b065 100755 --- a/images/sonic/launch.py +++ b/images/sonic/launch.py @@ -385,11 +385,6 @@ def create_config_db(hwsku: str) -> dict: 'gwaddr': get_default_gateway(), } }, - 'MGMT_VRF_CONFIG': { - 'vrf_global': { - 'mgmtVrfEnabled': 'true' - } - }, 'MGMT_PORT': { 'eth0': { 'alias': 'eth0', diff --git a/inventories/group_vars/leaves/main.yaml b/inventories/group_vars/leaves/main.yaml index 5af831b4..765c3bec 100644 --- a/inventories/group_vars/leaves/main.yaml +++ b/inventories/group_vars/leaves/main.yaml @@ -14,7 +14,7 @@ sonic_config_mgmt_interface: ip: "{{ ansible_host }}/16`" gateway_address: "172.42.0.1" -sonic_config_mgmt_vrf: true +sonic_config_mgmt_vrf: false sonic_config_nameservers: "{{ router_nameservers }}" sonic_config_vlans: From dec52d9e5af169bfbaca64249d006263d265e7fa Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 31 Jul 2026 10:44:33 +0200 Subject: [PATCH 23/23] feat: memory tuning knobs and memory tracing in integration tests Every leaf and machine is a QEMU VM, and QEMU RSS behaves as a high-water mark: the guest touches all of its RAM through the page cache eventually, so a guest started with -m 4096 stays resident at ~4 GiB no matter how little it actually needs. virtio-balloon with free page reporting barely helps because the page cache keeps almost nothing on the free lists. Add three opt-in knobs, all defaulting to the previous behaviour: MINI_LAB_LEAF_MEMORY / MINI_LAB_MACHINE_MEMORY guest RAM, substituted into the topologies. All three launchers read QEMU_MEMORY (SONiC launch.py, machine launch.py and vrnetlab for the dell flavors). MINI_LAB_KSM host kernel samepage merging, dedupes the identical guest RAM of leaf01/leaf02 and of the machine VMs MINI_LAB_THP transparent hugepage policy KSM and THP are host global and are applied/reverted by scripts/memory-tuning.sh, which records the pristine values so the host is left as it was found. scripts/memory-profile.sh bundles the knobs into one-factor-at-a-time profiles (baseline, low-memory, ksm, thp-madvise, all) so each knob can be measured on its own. baseline pins KSM and THP explicitly instead of leaving them untouched, so a comparison is not skewed by the host state. scripts/memory-trace.py samples host and per-container memory plus QEMU RSS into a CSV; scripts/memory-report.py renders per-run summaries and a cross-run comparison. The integration test starts the tracer and stops it in an EXIT trap, so a profile that is too tight for a flavor still produces data instead of nothing. The integration workflow builds its matrix from flavors x profiles. Pull requests and pushes run baseline only so regular CI cost is unchanged; workflow_dispatch defaults to the full matrix. Runs are serialised because the host level numbers would otherwise be contaminated by concurrent labs. Each run uploads its trace, and a final job merges them into a per-flavor comparison table. Co-Authored-By: Claude Opus 5 --- .github/workflows/integration.yaml | 121 ++++++++++- .gitignore | 4 +- Makefile | 80 ++++++- README.md | 17 ++ docs/memory-tuning.md | 135 ++++++++++++ mini-lab.capms.dell_sonic.yaml | 6 + mini-lab.dell_sonic.yaml | 4 + mini-lab.kamaji.yaml | 8 +- mini-lab.sonic.yaml | 6 +- scripts/memory-profile.sh | 65 ++++++ scripts/memory-report.py | 330 +++++++++++++++++++++++++++++ scripts/memory-trace.py | 280 ++++++++++++++++++++++++ scripts/memory-tuning.sh | 132 ++++++++++++ test/integration.sh | 15 ++ 14 files changed, 1187 insertions(+), 16 deletions(-) create mode 100644 docs/memory-tuning.md create mode 100755 scripts/memory-profile.sh create mode 100755 scripts/memory-report.py create mode 100755 scripts/memory-trace.py create mode 100755 scripts/memory-tuning.sh diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 45fae115..ee423e41 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -7,6 +7,16 @@ on: push: branches: - master + workflow_dispatch: + inputs: + flavors: + description: 'Comma separated flavors to test' + default: 'sonic_vs,sonic_vpp,gardener,dell_sonic' + memory_profiles: + description: >- + Comma separated memory profiles. Each profile is one integration run + per flavor, so the full list multiplies the runtime accordingly. + default: 'baseline,low-memory,ksm,thp-madvise,all' env: REGISTRY: ghcr.io @@ -136,22 +146,48 @@ jobs: ${{ env.MINI_LAB_SONIC_IMAGE_SHA }} cache-from: type=registry,ref=${{ env.MINI_LAB_SONIC_IMAGE }} cache-to: type=inline + prepare-matrix: + name: Prepare test matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.build.outputs.matrix }} + steps: + - name: Build flavor x memory profile matrix + id: build + shell: bash + env: + # every event runs the full flavor x profile matrix, so each pull + # request produces on/off data for all three memory knobs. Runs are + # serialised on the self-hosted runner, so this is a long pipeline - + # narrow it down via the workflow_dispatch inputs when iterating. + FLAVORS: ${{ inputs.flavors || 'sonic_vs,sonic_vpp,gardener,dell_sonic' }} + PROFILES: ${{ inputs.memory_profiles || 'baseline,low-memory,ksm,thp-madvise,all' }} + run: | + matrix=$(python3 -c ' + import json, os + flavors = [f.strip() for f in os.environ["FLAVORS"].split(",") if f.strip()] + profiles = [p.strip() for p in os.environ["PROFILES"].split(",") if p.strip()] + include = [{"flavor": f, "profile": p} for f in flavors for p in profiles] + print(json.dumps({"include": include})) + ') + echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" + echo "${matrix}" | python3 -m json.tool + test: - name: Run tests + name: Run tests (${{ matrix.flavor }}, ${{ matrix.profile }}) runs-on: self-hosted needs: - build-mini-lab-vms-image - build-mini-lab-sonic-vs-image - build-mini-lab-sonic-vpp-image + - prepare-matrix continue-on-error: true strategy: - matrix: - flavors: - - name: sonic_vs - - name: sonic_vpp - - name: gardener - - name: dell_sonic + fail-fast: false + # the memory numbers are host global, so runs must not overlap + max-parallel: 1 + matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} steps: - name: Gain back workspace permissions # https://github.com/actions/checkout/issues/211 @@ -186,6 +222,10 @@ jobs: echo "MINI_LAB_VM_IMAGE=ghcr.io/metal-stack/mini-lab-vms:${IMAGE_TAG}" >> $GITHUB_ENV echo "MINI_LAB_SONIC_IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_ENV + - name: Select memory profile + shell: bash + run: ./scripts/memory-profile.sh "${{ matrix.profile }}" >> "$GITHUB_ENV" + - name: Run integration tests shell: bash run: | @@ -193,7 +233,72 @@ jobs: ./test/ci-cleanup.sh ./test/integration.sh env: - MINI_LAB_FLAVOR: ${{ matrix.flavors.name }} + MINI_LAB_FLAVOR: ${{ matrix.flavor }} DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }} DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Restore host memory tuning + if: always() + shell: bash + run: ./scripts/memory-tuning.sh restore || true + + - name: Publish memory summary + if: always() + shell: bash + run: | + if [ -f memory-traces/summary.md ]; then + cat memory-traces/summary.md >> "$GITHUB_STEP_SUMMARY" + else + echo "no memory summary produced for ${{ matrix.flavor }} / ${{ matrix.profile }}" \ + >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload memory trace + if: always() + uses: actions/upload-artifact@v4 + with: + name: memory-trace-${{ matrix.flavor }}-${{ matrix.profile }} + path: memory-traces/ + if-no-files-found: warn + retention-days: 30 + + memory-comparison: + name: Compare memory profiles + runs-on: ubuntu-latest + needs: test + if: always() + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download memory traces + uses: actions/download-artifact@v4 + with: + pattern: memory-trace-* + path: memory-traces + + - name: Compare profiles + shell: bash + run: | + mkdir -p memory-traces + # every test job runs with continue-on-error, so it is possible that no + # run got far enough to produce a summary + if ! find memory-traces -name summary.json | grep -q .; then + echo "No memory summaries were produced by this run." \ + | tee memory-traces/comparison.md >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + ./scripts/memory-report.py compare \ + --input-dir memory-traces \ + --out-md memory-traces/comparison.md + cat memory-traces/comparison.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload comparison + uses: actions/upload-artifact@v4 + with: + name: memory-comparison + path: memory-traces/ + if-no-files-found: warn + retention-days: 30 diff --git a/.gitignore b/.gitignore index cf409683..43a4267e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,6 @@ files/certs/*.pem files/certs/**/*.pem files/certs/**/*.crt .vscode -vrnetlab \ No newline at end of file +vrnetlab +memory-traces +.memory-tuning.state diff --git a/Makefile b/Makefile index 6bc39614..0d3ab724 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,17 @@ MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms: MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1) MINI_LAB_SONIC_IMAGE_TAG := $(or $(MINI_LAB_SONIC_IMAGE_TAG),latest) +# Memory tuning. All of these are opt-in: the defaults reproduce the behaviour +# the lab had before the knobs existed. See docs/memory-tuning.md. +# QEMU RAM handed to the guests, substituted into the topology files. +MINI_LAB_LEAF_MEMORY := $(or $(MINI_LAB_LEAF_MEMORY),4096) +MINI_LAB_MACHINE_MEMORY := $(or $(MINI_LAB_MACHINE_MEMORY),2048) +# host-global knobs, empty means "do not touch" +MINI_LAB_KSM := $(or $(MINI_LAB_KSM),) +MINI_LAB_THP := $(or $(MINI_LAB_THP),) +# where the memory tracer writes its samples +MEMORY_TRACE_DIR := $(or $(MEMORY_TRACE_DIR),memory-traces) + MINI_LAB_INTERNAL_NETWORK=mini_lab_internal # define this here as well so that kind picks up the network on a clean checkout, # where .env does not exist yet at make parse time (-include .env above) @@ -139,7 +150,7 @@ partition: partition-bake docker compose $(COMPOSE_ARGS) up --remove-orphans --force-recreate partition .PHONY: partition-bake -partition-bake: external_network +partition-bake: external_network memory-tuning-apply docker pull $(MINI_LAB_VM_IMAGE) ifeq ($(CI),true) docker pull $(MINI_LAB_SONIC_IMAGE) @@ -148,7 +159,7 @@ ifneq ($(filter $(MINI_LAB_FLAVOR),dell_sonic capms_dell_sonic),$(MINI_LAB_FLAVO docker pull $(MINI_LAB_SONIC_IMAGE) endif @if ! sudo $(CONTAINERLAB) --topo $(LAB_TOPOLOGY) inspect | grep -i leaf01 > /dev/null; then \ - sudo --preserve-env=MINI_LAB_SONIC_IMAGE --preserve-env=MINI_LAB_DELL_SONIC_VERSION --preserve-env=MINI_LAB_VM_IMAGE $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure; fi + sudo --preserve-env=MINI_LAB_SONIC_IMAGE --preserve-env=MINI_LAB_DELL_SONIC_VERSION --preserve-env=MINI_LAB_VM_IMAGE --preserve-env=MINI_LAB_LEAF_MEMORY --preserve-env=MINI_LAB_MACHINE_MEMORY $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure; fi .PHONY: verify-deployment-image verify-deployment-image: @@ -186,6 +197,69 @@ files/certs/ca.pem: .PHONY: gen-certs # keep as a convenience alias gen-certs: files/certs/ca.pem +## MEMORY TUNING & TRACING ## + +# print the environment of a named profile, e.g. +# eval $(make memory-profile PROFILE=low-memory) +.PHONY: memory-profile +memory-profile: + @./scripts/memory-profile.sh $(or $(PROFILE),$(MINI_LAB_MEMORY_PROFILE),baseline) --export + +.PHONY: memory-tuning-apply +memory-tuning-apply: + @./scripts/memory-tuning.sh apply + +.PHONY: memory-tuning-restore +memory-tuning-restore: + @./scripts/memory-tuning.sh restore + +.PHONY: memory-tuning-show +memory-tuning-show: + @./scripts/memory-tuning.sh show + +.PHONY: memory-trace-start +memory-trace-start: + @mkdir -p $(MEMORY_TRACE_DIR) + @if [ -f $(MEMORY_TRACE_DIR)/tracer.pid ] && kill -0 $$(cat $(MEMORY_TRACE_DIR)/tracer.pid) 2> /dev/null; then \ + echo "memory tracer already running"; \ + else \ + nohup ./scripts/memory-trace.py sample \ + --out $(MEMORY_TRACE_DIR)/trace.csv \ + --meta $(MEMORY_TRACE_DIR)/meta.json \ + --interval $(or $(MEMORY_TRACE_INTERVAL),5) \ + --flavor $(MINI_LAB_FLAVOR) \ + --profile $(or $(MINI_LAB_MEMORY_PROFILE),unset) \ + > $(MEMORY_TRACE_DIR)/tracer.log 2>&1 & echo $$! > $(MEMORY_TRACE_DIR)/tracer.pid; \ + echo "memory tracer started, writing to $(MEMORY_TRACE_DIR)/trace.csv"; \ + fi + +.PHONY: memory-trace-stop +memory-trace-stop: + @if [ -f $(MEMORY_TRACE_DIR)/tracer.pid ]; then \ + kill $$(cat $(MEMORY_TRACE_DIR)/tracer.pid) 2> /dev/null || true; \ + sleep 1; \ + rm -f $(MEMORY_TRACE_DIR)/tracer.pid; \ + echo "memory tracer stopped"; \ + else \ + echo "no memory tracer running"; \ + fi + +.PHONY: memory-report +memory-report: + @./scripts/memory-report.py summarize \ + --trace $(MEMORY_TRACE_DIR)/trace.csv \ + --meta $(MEMORY_TRACE_DIR)/meta.json \ + --out-json $(MEMORY_TRACE_DIR)/summary.json \ + --out-md $(MEMORY_TRACE_DIR)/summary.md \ + --flavor $(MINI_LAB_FLAVOR) \ + --profile $(or $(MINI_LAB_MEMORY_PROFILE),unset) + @cat $(MEMORY_TRACE_DIR)/summary.md + +# compare all summaries below MEMORY_TRACE_DIR (or DIR=...) +.PHONY: memory-compare +memory-compare: + @./scripts/memory-report.py compare --input-dir $(or $(DIR),$(MEMORY_TRACE_DIR)) + .PHONY: cleanup cleanup: cleanup-control-plane cleanup-partition docker network rm --force mini_lab_internal @@ -197,7 +271,7 @@ cleanup-control-plane: rm -f $(KUBECONFIG) .PHONY: cleanup-partition -cleanup-partition: +cleanup-partition: memory-tuning-restore mkdir -p clab-mini-lab sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.dell_sonic.yaml sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml diff --git a/README.md b/README.md index 3d273f3f..a72ec1e2 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,23 @@ export MINI_LAB_FLAVOR=sonic_vs make ``` +## Memory tuning + +Every leaf and every machine is a QEMU VM, so a lab run is memory hungry. Three +opt-in knobs are available to shrink the footprint — the guest RAM size, host +side KSM and the transparent hugepage policy — together with a tracer that +records the memory usage of a run: + +```bash +# start the lab with reduced guest RAM, KSM and THP=madvise +eval "$(make memory-profile PROFILE=all)" +make up +``` + +The defaults are unchanged when none of the knobs is set. See +[docs/memory-tuning.md](docs/memory-tuning.md) for the available profiles, +the tradeoffs of each knob and how the CI matrix compares them. + ## Network topology An Nginx is running inside of the www container to allow automatic testing of outgoing connections. diff --git a/docs/memory-tuning.md b/docs/memory-tuning.md new file mode 100644 index 00000000..e149d2c0 --- /dev/null +++ b/docs/memory-tuning.md @@ -0,0 +1,135 @@ +# Memory tuning + +The mini-lab runs every leaf and every machine as a QEMU VM inside a +containerlab container. Those VMs dominate the memory footprint of a lab run, +and QEMU RSS behaves as a high-water mark: the guest eventually touches all of +its RAM through the page cache, so a guest started with `-m 4096` ends up +resident at roughly 4 GiB regardless of how much memory it actually needs. + +Three knobs are available to reduce that footprint. All of them are **opt-in** — +without the environment variables below, the lab behaves exactly as it did +before they existed. + +## Knobs + +| variable | default | effect | +| --- | --- | --- | +| `MINI_LAB_LEAF_MEMORY` | `4096` | `QEMU_MEMORY` handed to leaf01/leaf02 (MB) | +| `MINI_LAB_MACHINE_MEMORY` | `2048` | `QEMU_MEMORY` handed to the machine VMs (MB) | +| `MINI_LAB_KSM` | unset | `true`/`false` — kernel samepage merging on the host | +| `MINI_LAB_THP` | unset | `always`/`madvise`/`never` — transparent hugepage policy | + +`MINI_LAB_LEAF_MEMORY` and `MINI_LAB_MACHINE_MEMORY` are substituted into the +containerlab topologies and are picked up by all three VM launchers — the SONiC +`launch.py`, the machine `launch.py` and vrnetlab (used by the `dell_sonic` +flavors) all read `QEMU_MEMORY`. + +`MINI_LAB_KSM` and `MINI_LAB_THP` are **host global** and need root. They are +applied by `scripts/memory-tuning.sh`, which is run automatically as part of +`make partition-bake` and reverted by `make cleanup`. The previous values are +saved in `.memory-tuning.state` so the host is left as it was found. + +### Why these three + +* **Guest RAM sizing** is the dominant lever. A SONiC leaf idles at roughly + 2.1 GB of genuinely used guest memory; the remaining GBs of a 4096 MB guest + become page cache, which pins host RSS without doing useful work. +* **KSM** deduplicates identical guest pages. leaf01/leaf02 boot the same image, + as do the machine VMs, so there is a lot to merge. QEMU already marks guest + RAM `MADV_MERGEABLE`, so only the host-side switch is needed. The cost is + `ksmd` CPU time and some latency jitter from merge/CoW faults. +* **THP=madvise** stops the kernel from backing sparsely touched guest RAM with + 2 MiB pages. The cost is losing hugepage benefits for the dataplane, which + matters for the VPP flavor. + +Note that `virtio-balloon` with `free-page-reporting=on` is already enabled on +the SONiC VMs, but it reclaims very little in practice: the guest page cache +keeps almost nothing on the free lists, and what is free is usually fragmented +below the default reporting order of 9 (2 MiB). + +## Profiles + +`scripts/memory-profile.sh` bundles the knobs into named profiles. They are +one-factor-at-a-time variations of `baseline` so that each knob can be measured +on its own, plus `all` which combines them. + +| profile | leaf MB | machine MB | KSM | THP | +| --- | --- | --- | --- | --- | +| `baseline` | 4096 | 2048 | off | always | +| `low-memory` | 2560 | 1536 | off | always | +| `ksm` | 4096 | 2048 | **on** | always | +| `thp-madvise` | 4096 | 2048 | off | **madvise** | +| `all` | **2560** | **1536** | **on** | **madvise** | + +`baseline` pins KSM and THP explicitly rather than leaving them untouched, so +that a comparison is not skewed by whatever the host happened to be set to. + +Local use: + +```bash +eval "$(make memory-profile PROFILE=low-memory)" +make up +``` + +or directly: + +```bash +eval "$(./scripts/memory-profile.sh all --export)" +make up +``` + +## Tracing + +`scripts/memory-trace.py` samples host and per-container memory into a CSV. +Everything it reads (`/proc`, `/sys/fs/cgroup`) is world readable, so it needs +no privileges even though the QEMU processes belong to root. + +```bash +make memory-trace-start # background sampler, 5s interval +# ... run the lab ... +make memory-trace-stop +make memory-report # summary.json + summary.md next to the trace +``` + +Artifacts land in `memory-traces/`: + +| file | content | +| --- | --- | +| `trace.csv` | long format samples: `ts,elapsed_s,scope,name,metric,value` | +| `meta.json` | flavor, profile and the effective host settings of the run | +| `summary.json` | machine readable peaks and means | +| `summary.md` | the same as a markdown table with sparklines | + +To compare several runs: + +```bash +./scripts/memory-report.py compare --input-dir memory-traces +``` + +## CI + +`test/integration.sh` starts the tracer before `make up` and stops it in an +`EXIT` trap, so a summary is produced even when a test fails — a profile that is +too tight for a flavor is a result worth recording. + +The integration workflow builds its matrix from flavors × memory profiles and +runs the full 4 × 5 combination on every event, so every pull request produces +on/off data for each knob on every flavor. + +`workflow_dispatch` takes `flavors` and `memory_profiles` inputs to narrow that +down, which is what you want while iterating on the lab itself: + +```bash +gh workflow run integration.yaml \ + -f flavors=sonic_vpp \ + -f memory_profiles=baseline,all +``` + +Be aware of the cost: 20 full integration runs that cannot overlap add up to +many hours of self-hosted runner time per pull request. + +Runs are serialised (`max-parallel: 1`) because the host level numbers would +otherwise be contaminated by concurrent labs. Every run uploads a +`memory-trace--` artifact, and a final `memory-comparison` job +downloads them all, renders a per-flavor comparison table into the job summary +and uploads it as the `memory-comparison` artifact. diff --git a/mini-lab.capms.dell_sonic.yaml b/mini-lab.capms.dell_sonic.yaml index 19500361..e8ffea03 100644 --- a/mini-lab.capms.dell_sonic.yaml +++ b/mini-lab.capms.dell_sonic.yaml @@ -13,6 +13,8 @@ topology: image: ${MINI_LAB_SONIC_IMAGE} group: leaves enforce-startup-config: true + env: + QEMU_MEMORY: ${MINI_LAB_LEAF_MEMORY:=4096} stages: healthy: exec: @@ -54,6 +56,7 @@ topology: QEMU_CPU_CORES: 2 QEMU_DISK_SIZE: 20G UUID: 00000000-0000-0000-0000-000000000001 + QEMU_MEMORY: ${MINI_LAB_MACHINE_MEMORY:=2048} machine02: group: machines image: ${MINI_LAB_VM_IMAGE} @@ -61,6 +64,7 @@ topology: QEMU_CPU_CORES: 2 QEMU_DISK_SIZE: 20G UUID: 00000000-0000-0000-0000-000000000002 + QEMU_MEMORY: ${MINI_LAB_MACHINE_MEMORY:=2048} machine03: group: machines image: ${MINI_LAB_VM_IMAGE} @@ -68,6 +72,7 @@ topology: QEMU_CPU_CORES: 2 QEMU_DISK_SIZE: 20G UUID: 00000000-0000-0000-0000-000000000003 + QEMU_MEMORY: ${MINI_LAB_MACHINE_MEMORY:=2048} machine04: group: machines image: ${MINI_LAB_VM_IMAGE} @@ -75,6 +80,7 @@ topology: QEMU_CPU_CORES: 2 QEMU_DISK_SIZE: 20G UUID: 00000000-0000-0000-0000-000000000004 + QEMU_MEMORY: ${MINI_LAB_MACHINE_MEMORY:=2048} links: - endpoints: ["exit:mini_lab_ext", "mini_lab_ext:exit"] mtu: 9000 diff --git a/mini-lab.dell_sonic.yaml b/mini-lab.dell_sonic.yaml index d1342437..80cdb3c8 100644 --- a/mini-lab.dell_sonic.yaml +++ b/mini-lab.dell_sonic.yaml @@ -12,6 +12,8 @@ topology: image: ${MINI_LAB_SONIC_IMAGE} group: leaves enforce-startup-config: true + env: + QEMU_MEMORY: ${MINI_LAB_LEAF_MEMORY:=4096} stages: healthy: exec: @@ -54,11 +56,13 @@ topology: image: ${MINI_LAB_VM_IMAGE} env: UUID: 00000000-0000-0000-0000-000000000001 + QEMU_MEMORY: ${MINI_LAB_MACHINE_MEMORY:=2048} machine02: group: machines image: ${MINI_LAB_VM_IMAGE} env: UUID: 00000000-0000-0000-0000-000000000002 + QEMU_MEMORY: ${MINI_LAB_MACHINE_MEMORY:=2048} links: - endpoints: ["exit:mini_lab_ext", "mini_lab_ext:exit"] mtu: 9000 diff --git a/mini-lab.kamaji.yaml b/mini-lab.kamaji.yaml index d0b3c7f7..990a699b 100644 --- a/mini-lab.kamaji.yaml +++ b/mini-lab.kamaji.yaml @@ -39,19 +39,20 @@ topology: binds: - files/ssh/id_ed25519.pub:/authorized_keys env: - QEMU_MEMORY: 4096 + QEMU_MEMORY: ${MINI_LAB_LEAF_MEMORY:=4096} leaf02: group: leaves image: ${MINI_LAB_SONIC_IMAGE} binds: - files/ssh/id_ed25519.pub:/authorized_keys env: - QEMU_MEMORY: 4096 + QEMU_MEMORY: ${MINI_LAB_LEAF_MEMORY:=4096} machine01: group: machines image: ${MINI_LAB_VM_IMAGE} env: UUID: 00000000-0000-0000-0000-000000000001 + QEMU_MEMORY: ${MINI_LAB_MACHINE_MEMORY:=2048} QEMU_CPU_CORES: 2 QEMU_DISK_SIZE: 20G machine02: @@ -59,6 +60,7 @@ topology: image: ${MINI_LAB_VM_IMAGE} env: UUID: 00000000-0000-0000-0000-000000000002 + QEMU_MEMORY: ${MINI_LAB_MACHINE_MEMORY:=2048} QEMU_CPU_CORES: 2 QEMU_DISK_SIZE: 20G machine03: @@ -66,6 +68,7 @@ topology: image: ${MINI_LAB_VM_IMAGE} env: UUID: 00000000-0000-0000-0000-000000000003 + QEMU_MEMORY: ${MINI_LAB_MACHINE_MEMORY:=2048} QEMU_CPU_CORES: 2 QEMU_DISK_SIZE: 20G machine04: @@ -73,6 +76,7 @@ topology: image: ${MINI_LAB_VM_IMAGE} env: UUID: 00000000-0000-0000-0000-000000000004 + QEMU_MEMORY: ${MINI_LAB_MACHINE_MEMORY:=2048} QEMU_CPU_CORES: 2 QEMU_DISK_SIZE: 20G links: diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml index 44d7e8fd..1091d717 100644 --- a/mini-lab.sonic.yaml +++ b/mini-lab.sonic.yaml @@ -32,24 +32,26 @@ topology: binds: - files/ssh/id_ed25519.pub:/authorized_keys env: - QEMU_MEMORY: 4096 + QEMU_MEMORY: ${MINI_LAB_LEAF_MEMORY:=4096} leaf02: group: leaves image: ${MINI_LAB_SONIC_IMAGE} binds: - files/ssh/id_ed25519.pub:/authorized_keys env: - QEMU_MEMORY: 4096 + QEMU_MEMORY: ${MINI_LAB_LEAF_MEMORY:=4096} machine01: group: machines image: ${MINI_LAB_VM_IMAGE} env: UUID: 00000000-0000-0000-0000-000000000001 + QEMU_MEMORY: ${MINI_LAB_MACHINE_MEMORY:=2048} machine02: group: machines image: ${MINI_LAB_VM_IMAGE} env: UUID: 00000000-0000-0000-0000-000000000002 + QEMU_MEMORY: ${MINI_LAB_MACHINE_MEMORY:=2048} links: - endpoints: ["exit:mini_lab_ext", "mini_lab_ext:exit"] mtu: 9000 diff --git a/scripts/memory-profile.sh b/scripts/memory-profile.sh new file mode 100755 index 00000000..d0e4c6a8 --- /dev/null +++ b/scripts/memory-profile.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Print the environment for a named memory profile as KEY=value lines. +# +# The profiles are one-factor-at-a-time variations of "baseline" so that each +# knob can be compared on its own, plus "all" which combines them: +# +# baseline current defaults (4096 MB leaves, 2048 MB machines, no KSM, +# THP=always) - the reference every other profile is diffed against +# low-memory reduced QEMU_MEMORY for leaves and machines +# ksm kernel samepage merging enabled on the host +# thp-madvise transparent hugepages restricted to madvise +# all everything above at once +# +# Usage: +# scripts/memory-profile.sh low-memory # KEY=value lines +# eval "$(scripts/memory-profile.sh ksm --export)" +# scripts/memory-profile.sh all >> "$GITHUB_ENV" + +set -euo pipefail + +profile="${1:-baseline}" +mode="${2:-}" + +# defaults matching the committed topology defaults +leaf_memory=4096 +machine_memory=2048 +ksm=false +thp=always + +case "$profile" in + baseline) + ;; + low-memory) + leaf_memory=2560 + machine_memory=1536 + ;; + ksm) + ksm=true + ;; + thp-madvise) + thp=madvise + ;; + all) + leaf_memory=2560 + machine_memory=1536 + ksm=true + thp=madvise + ;; + *) + echo "unknown memory profile: $profile" >&2 + echo "valid profiles: baseline low-memory ksm thp-madvise all" >&2 + exit 1 + ;; +esac + +prefix="" +[ "$mode" = "--export" ] && prefix="export " + +cat < summary.json + summary.md + compare many runs -> comparison.md (profiles side by side per flavor) +""" + +import argparse +import csv +import json +import statistics +import sys +from pathlib import Path + +# containers that hold a QEMU VM; used for the "lab footprint" aggregate +VM_PREFIXES = ("leaf", "machine") + +SPARK = "▁▂▃▄▅▆▇█" + + +def mib(kb: float | None) -> float | None: + return None if kb is None else round(kb / 1024, 1) + + +def gib(kb: float | None) -> float | None: + return None if kb is None else round(kb / 1024 / 1024, 2) + + +def fmt(value, unit="GiB") -> str: + if value is None: + return "n/a" + return f"{value:.2f} {unit}" if unit else f"{value:.2f}" + + +def sparkline(values: list[float]) -> str: + if not values: + return "" + lo, hi = min(values), max(values) + if hi - lo < 1e-9: + return SPARK[0] * min(len(values), 60) + # downsample to at most 60 buckets + width = min(len(values), 60) + step = len(values) / width + out = [] + for i in range(width): + chunk = values[int(i * step):max(int((i + 1) * step), int(i * step) + 1)] + avg = sum(chunk) / len(chunk) + idx = int((avg - lo) / (hi - lo) * (len(SPARK) - 1)) + out.append(SPARK[idx]) + return "".join(out) + + +def load_trace(path: Path) -> dict: + """Return {(scope, name, metric): [(elapsed, value), ...]}.""" + series: dict = {} + with path.open(newline="") as fh: + reader = csv.DictReader(fh) + for row in reader: + try: + elapsed = float(row["elapsed_s"]) + value = float(row["value"]) + except (TypeError, ValueError): + continue + key = (row["scope"], row["name"], row["metric"]) + series.setdefault(key, []).append((elapsed, value)) + return series + + +def container_sum_over_time(series: dict, metric: str) -> list[tuple[float, float]]: + """Sum a per-container metric across containers at each sample time.""" + by_time: dict = {} + for (scope, _name, m), points in series.items(): + if scope != "container" or m != metric: + continue + for elapsed, value in points: + by_time[elapsed] = by_time.get(elapsed, 0.0) + value + return sorted(by_time.items()) + + +def vm_sum_over_time(series: dict, metric: str) -> list[tuple[float, float]]: + by_time: dict = {} + for (scope, name, m), points in series.items(): + if scope != "container" or m != metric: + continue + if not name.startswith(VM_PREFIXES): + continue + for elapsed, value in points: + by_time[elapsed] = by_time.get(elapsed, 0.0) + value + return sorted(by_time.items()) + + +def peak(points: list[tuple[float, float]]) -> float | None: + return max((v for _, v in points), default=None) + + +def mean(points: list[tuple[float, float]]) -> float | None: + values = [v for _, v in points] + return statistics.fmean(values) if values else None + + +def cmd_summarize(args: argparse.Namespace) -> int: + trace = Path(args.trace) + if not trace.exists(): + print(f"trace not found: {trace}", file=sys.stderr) + return 1 + + series = load_trace(trace) + if not series: + print(f"trace is empty: {trace}", file=sys.stderr) + return 1 + + meta = {} + if args.meta and Path(args.meta).exists(): + meta = json.loads(Path(args.meta).read_text()) + + host_used = next((p for (s, _n, m), p in series.items() + if s == "host" and m == "mem_used_kb"), []) + host_swap = next((p for (s, _n, m), p in series.items() + if s == "host" and m == "swap_used_kb"), []) + ksm_saved = next((p for (s, _n, m), p in series.items() + if s == "host" and m == "ksm_saved_kb"), []) + host_thp = next((p for (s, _n, m), p in series.items() + if s == "host" and m == "anon_hugepages_kb"), []) + + lab_total = container_sum_over_time(series, "mem_current_kb") + vm_total = vm_sum_over_time(series, "mem_current_kb") + qemu_total = vm_sum_over_time(series, "qemu_rss_kb") + + duration = max((e for e, _ in host_used), default=0.0) + + per_container = {} + for (scope, name, metric), points in sorted(series.items()): + if scope != "container" or metric not in ("mem_current_kb", "qemu_rss_kb"): + continue + entry = per_container.setdefault(name, {}) + entry[f"{metric}_peak"] = peak(points) + entry[f"{metric}_mean"] = mean(points) + + summary = { + "flavor": meta.get("flavor") or args.flavor, + "profile": meta.get("profile") or args.profile, + "duration_s": round(duration, 1), + "samples": len(host_used), + "meta": meta, + "host": { + "mem_used_peak_kb": peak(host_used), + "mem_used_mean_kb": mean(host_used), + "swap_used_peak_kb": peak(host_swap), + "ksm_saved_peak_kb": peak(ksm_saved), + "anon_hugepages_peak_kb": peak(host_thp), + }, + "lab": { + "containers_peak_kb": peak(lab_total), + "vm_containers_peak_kb": peak(vm_total), + "vm_containers_mean_kb": mean(vm_total), + "qemu_rss_peak_kb": peak(qemu_total), + }, + "containers": per_container, + } + + if args.out_json: + out = Path(args.out_json) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(summary, indent=2) + "\n") + + md = render_summary(summary, host_used, vm_total) + if args.out_md: + out = Path(args.out_md) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(md) + else: + print(md) + return 0 + + +def render_summary(summary: dict, host_used, vm_total) -> str: + host = summary["host"] + lab = summary["lab"] + meta = summary.get("meta", {}) + + lines = [ + f"## Memory trace — flavor `{summary['flavor'] or 'unknown'}`, " + f"profile `{summary['profile'] or 'unknown'}`", + "", + f"- duration: {summary['duration_s']:.0f}s ({summary['samples']} samples)", + f"- leaf memory: {meta.get('leaf_memory_mb') or 'default'} MB, " + f"machine memory: {meta.get('machine_memory_mb') or 'default'} MB", + f"- KSM: {'on' if meta.get('ksm_run') else 'off'}, " + f"THP: {meta.get('thp_setting', 'unknown')}", + "", + "| metric | peak | mean |", + "| --- | --- | --- |", + f"| host memory used | {fmt(gib(host['mem_used_peak_kb']))} | " + f"{fmt(gib(host['mem_used_mean_kb']))} |", + f"| host swap used | {fmt(gib(host['swap_used_peak_kb']))} | — |", + f"| all containers | {fmt(gib(lab['containers_peak_kb']))} | — |", + f"| VM containers | {fmt(gib(lab['vm_containers_peak_kb']))} | " + f"{fmt(gib(lab['vm_containers_mean_kb']))} |", + f"| QEMU RSS (all VMs) | {fmt(gib(lab['qemu_rss_peak_kb']))} | — |", + f"| KSM saved | {fmt(gib(host['ksm_saved_peak_kb']))} | — |", + "", + ] + + if host_used: + lines += [ + "```", + f"host used {sparkline([v for _, v in host_used])} " + f"{fmt(gib(host['mem_used_peak_kb']))} peak", + f"VM totals {sparkline([v for _, v in vm_total])} " + f"{fmt(gib(lab['vm_containers_peak_kb']))} peak", + "```", + "", + ] + + lines += ["| container | peak | mean | QEMU RSS peak |", "| --- | --- | --- | --- |"] + for name, entry in sorted(summary["containers"].items(), + key=lambda kv: -(kv[1].get("mem_current_kb_peak") or 0)): + lines.append( + f"| {name} | {fmt(gib(entry.get('mem_current_kb_peak')))} | " + f"{fmt(gib(entry.get('mem_current_kb_mean')))} | " + f"{fmt(gib(entry.get('qemu_rss_kb_peak')))} |" + ) + lines.append("") + return "\n".join(lines) + + +def cmd_compare(args: argparse.Namespace) -> int: + summaries = [] + for path in sorted(Path(args.input_dir).rglob("summary.json")): + try: + summaries.append(json.loads(path.read_text())) + except (OSError, json.JSONDecodeError) as exc: + print(f"skipping {path}: {exc}", file=sys.stderr) + if not summaries: + print(f"no summary.json found under {args.input_dir}", file=sys.stderr) + return 1 + + by_flavor: dict = {} + for summary in summaries: + by_flavor.setdefault(summary.get("flavor") or "unknown", []).append(summary) + + lines = ["# Memory profile comparison", ""] + lines += [ + "Peak values across the whole integration run. `Δ` columns compare " + "against the `baseline` profile of the same flavor.", + "", + ] + + for flavor in sorted(by_flavor): + runs = by_flavor[flavor] + baseline = next((r for r in runs if r.get("profile") == args.baseline), None) + base_host = (baseline or {}).get("host", {}).get("mem_used_peak_kb") + base_vm = (baseline or {}).get("lab", {}).get("vm_containers_peak_kb") + + lines += [ + f"## `{flavor}`", + "", + "| profile | host used (peak) | Δ host | VM containers (peak) | " + "Δ VMs | QEMU RSS (peak) | swap (peak) | KSM saved | duration |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- |", + ] + + def sort_key(run): + profile = run.get("profile") or "" + return (profile != args.baseline, profile) + + for run in sorted(runs, key=sort_key): + host = run.get("host", {}) + lab = run.get("lab", {}) + + is_baseline = run.get("profile") == args.baseline + + def delta(value, base, is_baseline=is_baseline): + if is_baseline or value is None or base is None: + return "—" + diff = gib(value - base) + pct = (value - base) / base * 100 if base else 0 + sign = "+" if diff >= 0 else "" + return f"{sign}{diff:.2f} GiB ({sign}{pct:.1f}%)" + + lines.append( + f"| `{run.get('profile') or 'unknown'}` " + f"| {fmt(gib(host.get('mem_used_peak_kb')))} " + f"| {delta(host.get('mem_used_peak_kb'), base_host)} " + f"| {fmt(gib(lab.get('vm_containers_peak_kb')))} " + f"| {delta(lab.get('vm_containers_peak_kb'), base_vm)} " + f"| {fmt(gib(lab.get('qemu_rss_peak_kb')))} " + f"| {fmt(gib(host.get('swap_used_peak_kb')))} " + f"| {fmt(gib(host.get('ksm_saved_peak_kb')))} " + f"| {run.get('duration_s', 0):.0f}s |" + ) + lines.append("") + + md = "\n".join(lines) + if args.out_md: + out = Path(args.out_md) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(md) + else: + print(md) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + s = sub.add_parser("summarize", help="summarize a single trace") + s.add_argument("--trace", required=True) + s.add_argument("--meta") + s.add_argument("--out-json") + s.add_argument("--out-md") + s.add_argument("--flavor", default="") + s.add_argument("--profile", default="") + s.set_defaults(func=cmd_summarize) + + c = sub.add_parser("compare", help="compare summaries of several runs") + c.add_argument("--input-dir", required=True) + c.add_argument("--out-md") + c.add_argument("--baseline", default="baseline") + c.set_defaults(func=cmd_compare) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/memory-trace.py b/scripts/memory-trace.py new file mode 100755 index 00000000..326ce422 --- /dev/null +++ b/scripts/memory-trace.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""Sample host and per-container memory usage into a CSV trace. + +Runs until it receives SIGTERM/SIGINT, appending one row per subject per +sampling interval. Everything it reads (/proc, /sys/fs/cgroup) is world +readable, so no root privileges are required even though the QEMU processes +themselves belong to root. + +The trace is written in long format so that new metrics can be added without +breaking existing consumers: + + ts,elapsed_s,scope,name,metric,value + +Metric names carry their unit as a suffix (``_kb``), except for plain flags +and counters. +""" + +import argparse +import csv +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +CGROUP_ROOT = Path("/sys/fs/cgroup") +KSM_ROOT = Path("/sys/kernel/mm/ksm") +PAGE_SIZE_KB = os.sysconf("SC_PAGE_SIZE") // 1024 + +# /proc/meminfo keys we care about, mapped to the metric name we emit +MEMINFO_KEYS = { + "MemTotal": "mem_total_kb", + "MemFree": "mem_free_kb", + "MemAvailable": "mem_available_kb", + "Cached": "cached_kb", + "AnonPages": "anon_kb", + "AnonHugePages": "anon_hugepages_kb", + "SwapTotal": "swap_total_kb", + "SwapFree": "swap_free_kb", +} + +_running = True + + +def _stop(signum, frame): + global _running + _running = False + + +def read_meminfo() -> dict: + values = {} + for line in Path("/proc/meminfo").read_text().splitlines(): + key, _, rest = line.partition(":") + if key in MEMINFO_KEYS: + values[MEMINFO_KEYS[key]] = int(rest.split()[0]) + total = values.get("mem_total_kb", 0) + available = values.get("mem_available_kb", 0) + values["mem_used_kb"] = total - available + values["swap_used_kb"] = values.get("swap_total_kb", 0) - values.get("swap_free_kb", 0) + return values + + +def read_ksm() -> dict: + def value(name: str) -> int: + try: + return int((KSM_ROOT / name).read_text().strip()) + except (OSError, ValueError): + return 0 + + sharing = value("pages_sharing") + shared = value("pages_shared") + return { + "ksm_run": value("run"), + # pages_sharing counts the pages that were merged away, i.e. the saving + "ksm_saved_kb": sharing * PAGE_SIZE_KB, + "ksm_shared_kb": shared * PAGE_SIZE_KB, + "ksm_unshared_kb": value("pages_unshared") * PAGE_SIZE_KB, + } + + +def read_thp_setting() -> str: + try: + raw = (Path("/sys/kernel/mm/transparent_hugepage/enabled")).read_text() + except OSError: + return "unknown" + for token in raw.split(): + if token.startswith("[") and token.endswith("]"): + return token[1:-1] + return "unknown" + + +def docker_containers() -> dict: + """Return {container_id: name} for all running containers.""" + try: + proc = subprocess.run( + ["docker", "ps", "--no-trunc", "--format", "{{.ID}}\t{{.Names}}"], + capture_output=True, + text=True, + timeout=30, + ) + except (OSError, subprocess.TimeoutExpired): + return {} + if proc.returncode != 0: + return {} + containers = {} + for line in proc.stdout.splitlines(): + cid, _, name = line.partition("\t") + if cid and name: + containers[cid.strip()] = name.strip() + return containers + + +def cgroup_dir(cid: str, cache: dict) -> Path | None: + """Locate the cgroup directory of a container, caching the result.""" + if cid in cache: + return cache[cid] + candidates = [ + CGROUP_ROOT / "system.slice" / f"docker-{cid}.scope", + CGROUP_ROOT / "docker" / cid, + CGROUP_ROOT / "memory" / "docker" / cid, # cgroup v1 + ] + found = next((c for c in candidates if (c / "memory.current").exists() + or (c / "memory.usage_in_bytes").exists()), None) + cache[cid] = found + return found + + +def read_container_memory(path: Path) -> dict: + """Read memory accounting for one cgroup (v2 preferred, v1 fallback).""" + metrics = {} + current = path / "memory.current" + if current.exists(): + try: + metrics["mem_current_kb"] = int(current.read_text().strip()) // 1024 + except (OSError, ValueError): + pass + try: + for line in (path / "memory.stat").read_text().splitlines(): + key, _, value = line.partition(" ") + if key in ("anon", "file", "slab", "shmem"): + metrics[f"{key}_kb"] = int(value) // 1024 + except (OSError, ValueError): + pass + return metrics + + legacy = path / "memory.usage_in_bytes" + if legacy.exists(): + try: + metrics["mem_current_kb"] = int(legacy.read_text().strip()) // 1024 + except (OSError, ValueError): + pass + try: + for line in (path / "memory.stat").read_text().splitlines(): + key, _, value = line.partition(" ") + if key == "rss": + metrics["anon_kb"] = int(value) // 1024 + elif key == "cache": + metrics["file_kb"] = int(value) // 1024 + except (OSError, ValueError): + pass + return metrics + + +def qemu_rss_by_container() -> dict: + """Sum the RSS of every qemu process, keyed by its container id.""" + totals = {} + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + comm = (entry / "comm").read_text().strip() + if not comm.startswith("qemu-system"): + continue + cgroup = (entry / "cgroup").read_text() + status = (entry / "status").read_text() + except OSError: + # process exited between listing and reading + continue + + cid = None + for token in cgroup.replace("/", " ").replace(".scope", " ").split(): + if token.startswith("docker-") and len(token) > 20: + cid = token[len("docker-"):] + break + if cid is None: + continue + + for line in status.splitlines(): + if line.startswith("VmRSS:"): + totals[cid] = totals.get(cid, 0) + int(line.split()[1]) + break + return totals + + +def sample(writer, started_at: float, cgroup_cache: dict, hostname: str) -> None: + now = time.time() + elapsed = round(now - started_at, 1) + ts = round(now, 1) + + def emit(scope, name, metric, value): + writer.writerow([ts, elapsed, scope, name, metric, value]) + + for metric, value in read_meminfo().items(): + emit("host", hostname, metric, value) + for metric, value in read_ksm().items(): + emit("host", hostname, metric, value) + + containers = docker_containers() + qemu = qemu_rss_by_container() + for cid, name in containers.items(): + path = cgroup_dir(cid, cgroup_cache) + if path is not None: + for metric, value in read_container_memory(path).items(): + emit("container", name, metric, value) + if cid in qemu: + emit("container", name, "qemu_rss_kb", qemu[cid]) + + +def cmd_sample(args: argparse.Namespace) -> int: + signal.signal(signal.SIGTERM, _stop) + signal.signal(signal.SIGINT, _stop) + + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + started_at = time.time() + + if args.meta: + meta = { + "flavor": args.flavor, + "profile": args.profile, + "started_at": started_at, + "interval_s": args.interval, + "hostname": os.uname().nodename, + "thp_setting": read_thp_setting(), + "ksm_run": read_ksm()["ksm_run"], + "leaf_memory_mb": os.environ.get("MINI_LAB_LEAF_MEMORY", ""), + "machine_memory_mb": os.environ.get("MINI_LAB_MACHINE_MEMORY", ""), + } + meta_path = Path(args.meta) + meta_path.parent.mkdir(parents=True, exist_ok=True) + meta_path.write_text(json.dumps(meta, indent=2) + "\n") + + hostname = os.uname().nodename + cgroup_cache: dict = {} + with out.open("w", newline="") as fh: + writer = csv.writer(fh) + writer.writerow(["ts", "elapsed_s", "scope", "name", "metric", "value"]) + while _running: + try: + sample(writer, started_at, cgroup_cache, hostname) + except Exception as exc: # never let a transient read kill the trace + print(f"memory-trace: sample failed: {exc}", file=sys.stderr) + fh.flush() + # sleep in small slices so SIGTERM is honoured promptly + deadline = time.time() + args.interval + while _running and time.time() < deadline: + time.sleep(0.2) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + s = sub.add_parser("sample", help="sample until terminated") + s.add_argument("--out", required=True, help="CSV trace output path") + s.add_argument("--meta", help="write run metadata JSON to this path") + s.add_argument("--interval", type=float, default=5.0, help="seconds between samples") + s.add_argument("--flavor", default=os.environ.get("MINI_LAB_FLAVOR", "")) + s.add_argument("--profile", default=os.environ.get("MINI_LAB_MEMORY_PROFILE", "")) + s.set_defaults(func=cmd_sample) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/memory-tuning.sh b/scripts/memory-tuning.sh new file mode 100755 index 00000000..d2a33a7d --- /dev/null +++ b/scripts/memory-tuning.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Apply (and restore) host level memory tuning for the mini-lab. +# +# Both knobs are host-global, so the previous values are saved on "apply" and +# put back on "restore". Without the environment flags set this script is a +# no-op, which keeps the default behaviour of the lab unchanged. +# +# MINI_LAB_KSM=true|false enable kernel samepage merging (dedupes the +# identical guest RAM of leaf01/leaf02 and the +# machine VMs) +# MINI_LAB_THP=madvise|always|never +# transparent hugepage policy; "madvise" stops the +# kernel from backing sparsely touched guest RAM +# with 2 MiB pages +# +# Usage: scripts/memory-tuning.sh apply|restore|show + +set -euo pipefail + +KSM_RUN=/sys/kernel/mm/ksm/run +KSM_SLEEP=/sys/kernel/mm/ksm/sleep_millisecs +KSM_PAGES=/sys/kernel/mm/ksm/pages_to_scan +THP_ENABLED=/sys/kernel/mm/transparent_hugepage/enabled +STATE_FILE="${MINI_LAB_MEMORY_TUNING_STATE:-.memory-tuning.state}" + +SUDO="" +[ "$(id -u)" -eq 0 ] || SUDO="sudo" + +write_sysfs() { + local value=$1 path=$2 + if [ ! -w "$path" ] && [ -z "$SUDO" ]; then + echo "memory-tuning: cannot write $path" >&2 + return 1 + fi + if ! echo "$value" | $SUDO tee "$path" > /dev/null; then + echo "memory-tuning: failed to write '$value' to $path." >&2 + echo " KSM and THP are host global settings and need root. Either run with" >&2 + echo " passwordless sudo, or unset MINI_LAB_KSM/MINI_LAB_THP to skip host tuning." >&2 + return 1 + fi +} + +current_thp() { + # "always [madvise] never" -> "madvise" + sed -n 's/.*\[\(.*\)\].*/\1/p' "$THP_ENABLED" 2>/dev/null || echo "" +} + +show() { + echo "KSM run: $(cat "$KSM_RUN" 2>/dev/null || echo 'n/a')" + echo "KSM pages/scan: $(cat "$KSM_PAGES" 2>/dev/null || echo 'n/a')" + echo "KSM saved: $(( $(cat /sys/kernel/mm/ksm/pages_sharing 2>/dev/null || echo 0) * 4 / 1024 )) MiB" + echo "THP enabled: $(current_thp)" +} + +apply() { + local ksm="${MINI_LAB_KSM:-}" thp="${MINI_LAB_THP:-}" + + if [ -z "$ksm" ] && [ -z "$thp" ]; then + echo "memory-tuning: no flags set, nothing to do" + return 0 + fi + + # only capture the pristine state the first time around, so repeated applies + # (e.g. "make up" after "make partition") do not record our own values + if [ ! -f "$STATE_FILE" ]; then + { + echo "KSM_RUN_ORIG=$(cat "$KSM_RUN" 2>/dev/null || echo '')" + echo "THP_ORIG=$(current_thp)" + } > "$STATE_FILE" + fi + + if [ -n "$ksm" ]; then + if [ ! -f "$KSM_RUN" ]; then + echo "memory-tuning: KSM not available on this host (CONFIG_KSM missing)" >&2 + elif [ "$ksm" = "true" ]; then + # a slightly more aggressive scan than the default so the lab converges + # within the runtime of an integration test instead of hours + write_sysfs "${MINI_LAB_KSM_PAGES_TO_SCAN:-1000}" "$KSM_PAGES" || true + write_sysfs "${MINI_LAB_KSM_SLEEP_MS:-20}" "$KSM_SLEEP" || true + write_sysfs 1 "$KSM_RUN" + echo "memory-tuning: KSM enabled" + else + write_sysfs 0 "$KSM_RUN" + echo "memory-tuning: KSM disabled" + fi + fi + + if [ -n "$thp" ]; then + if [ ! -f "$THP_ENABLED" ]; then + echo "memory-tuning: THP not available on this host" >&2 + else + write_sysfs "$thp" "$THP_ENABLED" + echo "memory-tuning: THP set to $thp" + fi + fi + + show +} + +restore() { + [ -f "$STATE_FILE" ] || { echo "memory-tuning: nothing to restore"; return 0; } + + # shellcheck disable=SC1090 + . "$STATE_FILE" + + if [ -n "${KSM_RUN_ORIG:-}" ] && [ -f "$KSM_RUN" ]; then + if [ "$KSM_RUN_ORIG" = "0" ] && [ "$(cat "$KSM_RUN")" != "0" ]; then + # 2 unmerges everything KSM merged and then stops scanning + write_sysfs 2 "$KSM_RUN" + else + write_sysfs "$KSM_RUN_ORIG" "$KSM_RUN" + fi + echo "memory-tuning: KSM restored to $KSM_RUN_ORIG" + fi + + if [ -n "${THP_ORIG:-}" ] && [ -f "$THP_ENABLED" ]; then + write_sysfs "$THP_ORIG" "$THP_ENABLED" + echo "memory-tuning: THP restored to $THP_ORIG" + fi + + rm -f "$STATE_FILE" +} + +case "${1:-}" in + apply) apply ;; + restore) restore ;; + show) show ;; + *) + echo "usage: $0 apply|restore|show" >&2 + exit 1 + ;; +esac diff --git a/test/integration.sh b/test/integration.sh index fb51fc0f..91b6782f 100755 --- a/test/integration.sh +++ b/test/integration.sh @@ -1,6 +1,21 @@ #!/usr/bin/env bash set -e +# Sample host and per-container memory for the whole run. The trap makes sure +# the tracer is stopped and a summary is written even when a test below fails, +# because a failed run is a data point too (e.g. a memory profile too tight for +# a flavor). +finish() { + local rc=$? + make --no-print-directory memory-trace-stop || true + make --no-print-directory memory-report || true + exit $rc +} +trap finish EXIT + +echo "Memory profile: ${MINI_LAB_MEMORY_PROFILE:-unset}" +make --no-print-directory memory-trace-start + echo "Starting mini-lab" make up