feat: first draft of image cache sync

compose.yaml

@@ -35,6 +35,8 @@ services:
     environment:
       - ANSIBLE_DISPLAY_SKIPPED_HOSTS=${ANSIBLE_DISPLAY_SKIPPED_HOSTS:-false}
       - ANSIBLE_INVENTORY=inventories/partition.yaml,clab-mini-lab/ansible-inventory.yml
+      - KUBECONFIG=/mini-lab/.kubeconfig
+      - K8S_AUTH_KUBECONFIG=/mini-lab/.kubeconfig
       - CI=${CI}
       - DOCKER_HUB_USER=${DOCKER_HUB_USER}
       - DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN}

deploy_partition.yaml

@@ -10,133 +10,78 @@
         timeout: 50
   roles:
     - name: ansible-common
-      tags: always
     - name: metal-roles/partition/roles/sonic-config
-      tags: sonic
     - name: sonic
-      tags: sonic
+  post_tasks:
+    - name: Temporary workaround for EOL debian bullseye backports repository (using archive.debian.org)
+      lineinfile:
+        path: /etc/apt/sources.list
+        search_string: deb [arch=amd64] http://deb.debian.org/debian/ bullseye-backports main contrib non-free
+        line: deb [arch=amd64] http://archive.debian.org/debian/ bullseye-backports main contrib non-free
 
 - name: Configure leaves (Enterprise SONiC)
   hosts: dell_sonic
   any_errors_fatal: true
-  become: true
   handlers:
     - name: reload systemd daemon
       systemd:
         daemon_reload: true
-  tasks:
-    - name: Check for bgpd.conf presence
-      ansible.builtin.stat:
-        path: /etc/sonic/frr/bgpd.conf
-      register: bgpd_conf_stat
-
-    - name: Restart bgp service when non-split docker_routing_config_mode is still active
-      ansible.builtin.service:
-        name: bgp
-        state: restarted
-      when: bgpd_conf_stat.stat.exists
-
-    - name: Ensure root has authorized_key
-      ansible.builtin.authorized_key:
-        user: root
-        state: present
-        key: "{{ lookup('file', 'ssh/id_ed25519.pub') }}"
-
-    - name: Install frr-reload.service
-      copy:
-        src: /root/.ansible/roles/metal-roles/partition/roles/sonic-config/files/frr-reload.service
-        dest: /etc/systemd/system/frr-reload.service
-        remote_src: false
-      notify:
-        - reload systemd daemon
-      changed_when: true
-
-    - name: Activate IP MASQUERADE on eth0
-      ansible.builtin.iptables:
-        chain: POSTROUTING
-        jump: MASQUERADE
-        out_interface: eth0
-        table: nat
-
-    - name: Activate IPv4 forwarding on eth0
-      ansible.posix.sysctl:
-        name: net.ipv4.conf.eth0.forwarding
-        reload: no
-        sysctl_set: yes
-        value: "1"
+  roles:
+    - name: dell-sonic
 
 - name: Deploy dhcp server on leaf01 (Community SONiC)
   hosts: leaf01:!dell_sonic
-  pre_tasks:
-    - name: Temporary workaround for EOL debian bullseye backports repository (using archive.debian.org)
-      lineinfile:
-        path: /etc/apt/sources.list
-        search_string: deb [arch=amd64] http://deb.debian.org/debian/ bullseye-backports main contrib non-free
-        line: deb [arch=amd64] http://archive.debian.org/debian/ bullseye-backports main contrib non-free
   roles:
     - name: ansible-common
       tags: always
     - name: metal-roles/partition/roles/dhcp
       tags: dhcp
 
-# FIXME: For some reason, the first docker pull always fails on dell_sonic but succeeds on second attempt.
-# Investigate the cause and remove this play
-- name: Intentionally fail on first docker pull
-  hosts: dell_sonic
-  tasks:
-    - community.docker.docker_container:
-        name: hello-world
-        image: library/hello-world:latest
-        cleanup: true
-        pull: true
-      failed_when: false
+- name: Install python client and generate deployment token
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  pre_tasks:
+    - name: Wait until the metal-apiserver is running
+      kubernetes.core.k8s_info:
+        kind: Deployment
+        name: metal-apiserver
+        namespace: "{{ metal_control_plane_namespace }}"
+        wait: true
+        wait_sleep: 1
+        wait_timeout: 600
+
+    - name: Wait until the admin_editor_token secret exists
+      kubernetes.core.k8s_info:
+        kind: Secret
+        name: "{{ metal_deployment_token_secret_name }}"
+        namespace: "{{ metal_deployment_token_secret_namespace }}"
+        wait: true
+        wait_sleep: 1
+        wait_timeout: 600
 
-- name: Deploy pixiecore on leaf01
-  hosts: leaf01
-  become: true
   roles:
     - name: ansible-common
-      tags: always
-    - name: metal-roles/partition/roles/pixiecore
-      tags: pixiecore
+    - name: metal-roles/common/roles/metal-v2-client
+    - name: metal-roles/common/roles/metal-deployment-token
 
-- name: Deploy monitoring on leaf01
+- name: Deploy mgmt-server components on leaf01
   hosts: leaf01
-  become: true
   roles:
     - name: ansible-common
-      tags: always
-    - name: metal-roles/partition/roles/monitoring/prometheus
-      when: monitoring_enabled
-      tags: prometheus
-    - name: metal-roles/partition/roles/monitoring/ipmi-exporter
-      when: monitoring_enabled
-      tags: ipmi-exporter
+    - name: metal-roles/partition/roles/pixiecore
+    - name: metal-roles/common/roles/metal-deployment-token
+    - name: metal-roles/partition/roles/image-cache
+      when: deploy_partition_image_cache
 
-- name: Deploy monitoring on leaves
-  hosts: leaves
-  become: true
-  roles:
-    - name: ansible-common
-      tags: always
-    - name: metal-roles/partition/roles/alloy
-      when: monitoring_enabled
-      tags: alloy
-    - name: metal-roles/partition/roles/monitoring/node-exporter
-      when: monitoring_enabled
-      tags: node-exporter
-    - name: metal-roles/partition/roles/monitoring/blackbox-exporter
-      when: monitoring_enabled
-      tags: blackbox-exporter
-    - name: metal-roles/partition/roles/monitoring/sonic-exporter
-      when: monitoring_enabled
-      tags: sonic-exporter
+- name: Deploy partition monitoring
+  import_playbook: deploy_partition_monitoring.yaml
 
 # So, route propagation was broken in community sonic for a while now. Turns out
 # during a change in upstream sonic some time in the end of 2024 sonic switched
 # the module used for FRR->fpmsyncd communication, which does not provide a
 # default connection endpoint. We just needed to add `fpm address 127.0.0.1`
-# to the frr.conf template and could remove all route-propagation related 
+# to the frr.conf template and could remove all route-propagation related
 # workarounds
 - name: Deploy metal-core (SONiC with dplane_fpm_sonic)
   hosts: leaves:!dell_sonic

deploy_partition_monitoring.yaml

@@ -0,0 +1,32 @@
+---
+- name: Deploy monitoring on leaf01
+  hosts: leaf01
+  become: true
+  roles:
+    - name: ansible-common
+      tags: always
+    - name: metal-roles/partition/roles/monitoring/prometheus
+      when: monitoring_enabled
+      tags: prometheus
+    - name: metal-roles/partition/roles/monitoring/ipmi-exporter
+      when: monitoring_enabled
+      tags: ipmi-exporter
+
+- name: Deploy monitoring on leaves
+  hosts: leaves
+  become: true
+  roles:
+    - name: ansible-common
+      tags: always
+    - name: metal-roles/partition/roles/alloy
+      when: monitoring_enabled
+      tags: alloy
+    - name: metal-roles/partition/roles/monitoring/node-exporter
+      when: monitoring_enabled
+      tags: node-exporter
+    - name: metal-roles/partition/roles/monitoring/blackbox-exporter
+      when: monitoring_enabled
+      tags: blackbox-exporter
+    - name: metal-roles/partition/roles/monitoring/sonic-exporter
+      when: monitoring_enabled
+      tags: sonic-exporter

inventories/group_vars/all/metal_apiserver.yaml

@@ -0,0 +1,2 @@
+---
+metal_apiserver_url: http://v2.{{ metal_control_plane_gateway_dns }}

inventories/group_vars/all/release_vector.yaml

@@ -1,3 +1,4 @@
+---
 metal_stack_release_version: develop
 metal_stack_release_vectors:
   - url: oci://ghcr.io/metal-stack/releases:{{ metal_stack_release_version }}
@@ -38,7 +39,7 @@ metal_stack_release_vectors:
 ##
 
 # ansible_common_version:
-# metal_roles_version: pr-<pr-number-and-title>
+# metal_roles_version:
 # metal_ansible_modules_version:
 
 ##

inventories/group_vars/control_plane/metal.yml

@@ -30,13 +30,13 @@ metal_api_nsq_tcp_address: nsqd:4150
 metal_apiserver_pdb_min_available: 1
 
 metal_apiserver_enabled: true
-metal_apiserver_url: http://v2.{{ metal_control_plane_gateway_dns }}
 
 metal_apiserver_oidc_discovery_url: https://auth.{{ metal_control_plane_gateway_dns }}/.well-known/openid-configuration
 metal_apiserver_oidc_end_session_url: "https://auth.{{ metal_control_plane_gateway_dns }}/oidc/v1/end_session"
 metal_apiserver_tls_skip_verify: true
 
 metal_apiserver_secure_cookie: false
+metal_apiserver_session_secret: alongsecret
 
 metal_apiserver_httproute_enabled: true
 metal_apiserver_httproute_parent_refs:

inventories/group_vars/partition/ansible.yaml

@@ -0,0 +1,2 @@
+---
+ansible_become: true

inventories/group_vars/partition/image_cache.yaml

@@ -0,0 +1,9 @@
+---
+# the cache is not properly wired up yet
+deploy_partition_image_cache: false
+
+image_cache_sync_token_expiration: 20m
+image_cache_sync_excludes:
+  - "/pull_requests/"
+  - "/stable/"
+image_cache_sync_expiration_grace_period: 100

inventories/partition.yaml

@@ -6,6 +6,7 @@ partition:
   hosts:
     localhost:
       ansible_python_interpreter: "{{ ansible_playbook_python }}"
+      ansible_connection: local
   children:
     dell_sonic:
     leaves:

roles/dell-sonic/tasks/main.yaml

@@ -0,0 +1,50 @@
+---
+- name: Check for bgpd.conf presence
+  ansible.builtin.stat:
+    path: /etc/sonic/frr/bgpd.conf
+  register: bgpd_conf_stat
+
+- name: Restart bgp service when non-split docker_routing_config_mode is still active
+  ansible.builtin.service:
+    name: bgp
+    state: restarted
+  when: bgpd_conf_stat.stat.exists
+
+- name: Ensure root has authorized_key
+  ansible.builtin.authorized_key:
+    user: root
+    state: present
+    key: "{{ lookup('file', 'ssh/id_ed25519.pub') }}"
+
+- name: Install frr-reload.service
+  copy:
+    src: /root/.ansible/roles/metal-roles/partition/roles/sonic-config/files/frr-reload.service
+    dest: /etc/systemd/system/frr-reload.service
+    remote_src: false
+  notify:
+    - reload systemd daemon
+  changed_when: true
+
+- name: Activate IP MASQUERADE on eth0
+  ansible.builtin.iptables:
+    chain: POSTROUTING
+    jump: MASQUERADE
+    out_interface: eth0
+    table: nat
+
+- name: Activate IPv4 forwarding on eth0
+  ansible.posix.sysctl:
+    name: net.ipv4.conf.eth0.forwarding
+    reload: no
+    sysctl_set: yes
+    value: "1"
+
+# FIXME: For some reason, the first docker pull always fails on dell_sonic but succeeds on second attempt.
+# Investigate the cause and remove this play
+- name: Intentionally fail on first docker pull
+  community.docker.docker_container:
+    name: hello-world
+    image: library/hello-world:latest
+    cleanup: true
+    pull: true
+  failed_when: false