Do not require ingress-controller for Gardener setup.

.github/workflows/integration.yaml

@@ -142,6 +142,14 @@ jobs:
         echo "MINI_LAB_VM_IMAGE=ghcr.io/metal-stack/mini-lab-vms:${IMAGE_TAG}" >> $GITHUB_ENV
         echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic:${IMAGE_TAG}" >> $GITHUB_ENV
 
+    - name: Setup kind
+      run: |
+        sudo curl -Lo /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
+        sudo chmod +x /usr/local/bin/kind
+        kind version
+      env:
+        KIND_VERSION: v0.32.0
+
     - name: Run integration tests
       shell: bash
       run: |

.gitignore

@@ -17,4 +17,5 @@ files/certs/*.pem
 files/certs/**/*.pem
 files/certs/**/*.crt
 .vscode
-vrnetlab
+vrnetlab
+.codegraph

Makefile

@@ -126,6 +126,16 @@ control-plane-bake:
 			--kubeconfig $(KUBECONFIG); fi
 	$(MAKE) create-proxy-registries
 	docker compose up -d --force-recreate cloud-provider-kind
+	$(MAKE) webhook-image
+
+.PHONY: webhook-image
+webhook-image:
+	docker build -t mini-lab-webhook:dev webhook/
+	@if kind get clusters | grep metal-control-plane > /dev/null; then \
+		kind load docker-image mini-lab-webhook:dev --name metal-control-plane; \
+	else \
+		echo "kind cluster metal-control-plane not found, skipping image load"; \
+	fi
 
 .PHONY: partition
 partition: partition-bake
@@ -459,7 +469,7 @@ build-dell-sonic:
 fetch-virtual-kubeconfig:
 	# TODO: it's hard to get the latest issued generic kubeconfig secret... just take the first result for now
 	kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden $(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden -l managed-by=secrets-manager,manager-identity=gardener-operator,name=generic-token-kubeconfig --no-headers | awk '{ print $$1 }') -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig
-	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.42.0.1.nip.io:4443
+	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.$(shell kubectl --kubeconfig=$(KUBECONFIG) get svc -n virtual-garden-istio-ingress istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip }').nip.io
 	@kubectl --kubeconfig=.virtual-kubeconfig config set-credentials garden --token=$(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden shoot-access-virtual-garden -o jsonpath='{.data.token}' | base64 -d)
 	@kubectl --kubeconfig=$(KUBECONFIG) config unset users.garden
 	@kubectl --kubeconfig=$(KUBECONFIG) config unset contexts.garden

compose.yaml

@@ -127,20 +127,21 @@ services:
       - REGISTRY_PROXY_TTL=168h
       - REGISTRY_STORAGE_DELETE_ENABLED=true
       - OTEL_TRACES_EXPORTER=none
+
   cloud-provider-kind:
     image: registry.k8s.io/cloud-provider-kind/cloud-controller-manager:v0.10.0
     restart: always
-    networks:
-      - kind
+    network_mode: host
     environment:
-      - KIND_EXPERIMENTAL_DOCKER_NETWORK=${KIND_EXPERIMENTAL_DOCKER_NETWORK:-kind}
+      - KIND_EXPERIMENTAL_DOCKER_NETWORK=${KIND_EXPERIMENTAL_DOCKER_NETWORK:-mini_lab_internal}
     command:
     # v0.10.0 of cloud controller does not support tcproutes, since it does not support the experimental gateway api channel
     # using envoy-gateway deployed via roles/gateway instead
       - --gateway-channel
       - disabled
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+
 volumes:
   proxy-docker:
   proxy-gcr:

deploy_control_plane.yaml

@@ -1,17 +1,21 @@
 ---
-- name: deploy control plane
+- name: deploy gateway controller
   hosts: control_plane
   connection: local
   gather_facts: false
   roles:
-    - name: ansible-common
-      tags: always
     - name: metal-roles/control-plane/roles/prepare
       tags: prepare
     - name: gateway
       tags: gateway
-    - name: ingress-controller
-      tags: ingress-controller
+
+- name: deploy control plane
+  hosts: control_plane
+  connection: local
+  gather_facts: false
+  roles:
+    - name: ansible-common
+      tags: always
     - name: metal-roles/control-plane/roles/nsq
       tags: nsq
     - name: metal-roles/control-plane/roles/metal-db
@@ -33,17 +37,15 @@
       tags: auth
     - name: metal-roles/control-plane/roles/metal
       tags: metal
-    - name: metal-roles/control-plane/roles/logging
-      when: monitoring_enabled
-      tags: logging
-    - name: metal-roles/control-plane/roles/monitoring
-      when: monitoring_enabled
-      tags: monitoring
+
+- name: deploy monitoring
+  import_playbook: deploy_monitoring.yaml
+  when: monitoring_enabled
 
 - name: deploy gardener
   import_playbook: deploy_gardener.yaml
   when: gardener_enabled and not kamaji_enabled
-    
+
 - name: deploy kamaji
   import_playbook: deploy_kamaji.yaml
   when: kamaji_enabled and not gardener_enabled

deploy_gardener.yaml

@@ -3,24 +3,13 @@
   hosts: control_plane
   connection: local
   gather_facts: false
-  pre_tasks:
-    - name: Fake Gardener metal shoot
-      k8s:
-        definition:
-          apiVersion: v1
-          kind: ConfigMap
-          metadata:
-            name: shoot-info
-            namespace: kube-system
-          data:
-            nodeNetwork: 172.42.0.0/16
-            podNetwork: 10.244.0.0/24
-            serviceNetwork: 10.96.0.0/16
-      tags: gardener
-
   roles:
     - name: ansible-common
       tags: always
+    - name: gardener-hacks
+      tags: gardener
+    - name: webhook
+      tags: gardener
     - name: minio
       tags: minio
     - name: powerdns
@@ -29,8 +18,6 @@
       tags: gardener
     - name: metal-roles/control-plane/roles/gardener-extensions
       tags: gardener
-    - name: gardener-istio-patch
-      tags: gardener
     - name: metal-roles/control-plane/roles/gardener-virtual-garden-access
       tags: gardener
     - name: metal-roles/control-plane/roles/gardener-cloud-profile
@@ -61,57 +48,6 @@
           wait_timeout: 900
         tags: gardener
 
-      - name: Wait for istio ingress gateway service
-        kubernetes.core.k8s_info:
-          api_version: v1
-          kind: Service
-          name: istio-ingressgateway
-          namespace: istio-ingress
-        register: result
-        until: result.resources
-        retries: 30
-        delay: 10
-        tags: gardener
-
-      - name: Patch istio ingress gateway service status
-        patch_service_status_k8s:
-          name: istio-ingressgateway
-          namespace: istio-ingress
-          body:
-            status:
-              loadBalancer:
-                ingress:
-                  - ip: "172.42.0.1"
-        tags: gardener
-
-      - name: Expose istio gateway through ingress-nginx (for local environments)
-        k8s:
-          definition:
-            apiVersion: networking.k8s.io/v1
-            kind: Ingress
-            metadata:
-              annotations:
-                nginx.ingress.kubernetes.io/ssl-passthrough: "true"
-              name: apiserver-ingress
-              namespace: istio-ingress
-            spec:
-              ingressClassName: nginx
-              rules:
-              - host: "{{ metal_control_plane_stage_name }}.{{ gardener_gardenlet_default_dns_domain }}"
-                http:
-                  paths:
-                  - path: /
-                    pathType: Prefix
-                    backend:
-                      service:
-                        name: istio-ingressgateway
-                        port:
-                          number: 443
-              tls:
-              - hosts:
-                - "{{ metal_control_plane_stage_name }}.{{ gardener_gardenlet_default_dns_domain }}"
-        tags: gardener
-
       - name: Wait until Garden is ready
         kubernetes.core.k8s_info:
           api_version: "operator.gardener.cloud/v1alpha1"

deploy_monitoring.yaml

@@ -0,0 +1,12 @@
+---
+- name: deploy monitoring
+  hosts: control_plane
+  connection: local
+  gather_facts: false
+  roles:
+    - name: ingress-controller
+      tags: ingress-controller
+    - name: metal-roles/control-plane/roles/logging
+      tags: logging
+    - name: metal-roles/control-plane/roles/monitoring
+      tags: monitoring

files/certs/webhook/server.json

@@ -0,0 +1,22 @@
+{
+  "CN": "mini-lab-webhook",
+  "hosts": [
+    "localhost",
+    "mini-lab-webhook",
+    "mini-lab-webhook.kube-system.svc",
+    "mini-lab-webhook.kube-system.svc.cluster.local"
+  ],
+  "key": {
+    "algo": "ecdsa",
+    "size": 256
+  },
+  "names": [
+    {
+      "C":  "DE",
+      "L":  "Munich",
+      "O":  "metal-stack",
+      "OU": "DevOps",
+      "ST": "Bavaria"
+    }
+  ]
+}

inventories/group_vars/all/release_vector.yaml

@@ -4,6 +4,10 @@ metal_stack_release_vectors:
     variable_mapping_path: metal_stack_release.mapping
     include_role_defaults: metal-roles/common/roles/defaults
     oci_cosign_verify_key: "{{ lookup('file', 'cosign.pub') }}"
+
+gardener_extension_backup_s3_helm_chart_tag: v0.0.0-pull-request.31
+gardener_extension_backup_s3_additional_network_policies: true
+
 ##
 ## for development purposes, you can override releases from our image vector here
 ##
@@ -37,8 +41,8 @@ metal_stack_release_vectors:
 ## for ansible roles
 ##
 
-# ansible_common_version:
-# metal_roles_version: pr-<pr-number-and-title>
+ansible_common_version: pr-51-remove-mini-lab-workaround
+metal_roles_version: pr-721-backup-s3-netpols
 # metal_ansible_modules_version:
 
 ##

inventories/group_vars/control_plane/gardener/gardenlet.yaml

@@ -15,7 +15,7 @@ gardener_gardenlets:
       name: backup-secret
       namespace: garden
   backup_infrastructure_secret:
-    endpoint: "{{ ('http://' + minio_dns_name + ':8080' ) | b64encode }}"
+    endpoint: "{{ ('http://' + minio_dns_name ) | b64encode }}"
     accessKeyID: "{{ minio_root_user | b64encode }}"
     secretAccessKey: "{{ minio_root_password | b64encode }}"
     region: "{{ 'us-east-1' | b64encode }}"

inventories/group_vars/control_plane/gardener/operator.yaml

@@ -1,5 +1,6 @@
 ---
-gardener_operator_ingress_dns_domain: "gardener.{{ metal_control_plane_ingress_dns }}"
+gardener_operator_ingress_dns_domain: "gardener.172.42.0.43.nip.io"
+gardener_operator_virtual_garden_public_dns: "gardener-kube-apiserver.172.42.0.43.nip.io"
 
 gardener_operator_backup_infrastructure:
   provider: S3
@@ -10,16 +11,12 @@ gardener_operator_backup_infrastructure:
    namespace: garden
 
 gardener_operator_backup_infrastructure_secret:
-  endpoint: "{{ ('http://' + minio_dns_name + ':8080' ) | b64encode }}"
+  endpoint: "{{ ('http://' + minio_dns_name ) | b64encode }}"
   accessKeyID: "{{ minio_root_user | b64encode }}"
   secretAccessKey: "{{ minio_root_password | b64encode }}"
   region: "{{ 'us-east-1' | b64encode }}"
   s3ForcePathStyle: "{{ 'true' | b64encode }}"
 
-# enable mini-lab patches
-gardener_operator_patch_istio_ingress_gateway_service_ip: 172.42.0.1
-gardener_operator_expose_virtual_garden_through_ingress_nginx: true
-
 # for local setups this should be sufficient
 gardener_operator_high_availability_control_plane: false
 

inventories/group_vars/control_plane/minio.yaml

@@ -2,4 +2,4 @@
 minio_root_user: mini-lab
 minio_root_password: change-me
 
-minio_dns_name: minio.172.42.0.1.nip.io
+minio_dns_name: minio.172.42.0.42.nip.io

inventories/group_vars/control_plane/powerdns.yaml

@@ -5,16 +5,16 @@ powerdns_webserver_password: change-me
 powerdns_mariadb_username: root
 powerdns_mariadb_password: change-me
 
-powerdns_load_balancer_dns_name: "ns.{{ metal_control_plane_ingress_dns }}"
+powerdns_load_balancer_dns_name: "ns.{{ metal_control_plane_gateway_dns }}"
 
-powerdns_api_dns_name: "powerdns-api.{{ metal_control_plane_ingress_dns }}"
+powerdns_api_dns_name: "powerdns-api.{{ metal_control_plane_gateway_dns }}"
 
 powerdns_zones:
-  - name: "gardener.172.42.0.1.nip.io."
+  - name: "gardener.172.42.0.43.nip.io."
     kind: Master
     nameservers:
       - "{{ powerdns_load_balancer_dns_name }}."
-  - name: "gardener-kube-apiserver.172.42.0.1.nip.io."
+  - name: "gardener-kube-apiserver.172.42.0.43.nip.io."
     kind: Master
     nameservers:
       - "{{ powerdns_load_balancer_dns_name }}."

roles/gardener-hacks/tasks/main.yaml

@@ -0,0 +1,14 @@
+---
+# this allows us to use the metal-roles gardener deployment simulating to run on a metalstack.cloud cluster
+- name: Fake Gardener metal shoot
+  k8s:
+    definition:
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: shoot-info
+        namespace: kube-system
+      data:
+        nodeNetwork: 172.42.0.0/16
+        podNetwork: 10.244.0.0/24
+        serviceNetwork: 10.96.0.0/16