test: Remove gardener ingress improvements

.github/workflows/integration.yaml

@@ -142,6 +142,14 @@ jobs:
         echo "MINI_LAB_VM_IMAGE=ghcr.io/metal-stack/mini-lab-vms:${IMAGE_TAG}" >> $GITHUB_ENV
         echo "MINI_LAB_SONIC_IMAGE=ghcr.io/metal-stack/mini-lab-sonic:${IMAGE_TAG}" >> $GITHUB_ENV
 
+    - name: Setup kind
+      run: |
+        sudo curl -Lo /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
+        sudo chmod +x /usr/local/bin/kind
+        kind version
+      env:
+        KIND_VERSION: v0.32.0
+
     - name: Run integration tests
       shell: bash
       run: |

.gitignore

@@ -17,4 +17,5 @@ files/certs/*.pem
 files/certs/**/*.pem
 files/certs/**/*.crt
 .vscode
-vrnetlab
+vrnetlab
+.codegraph

Makefile

@@ -123,7 +123,9 @@ control-plane-bake:
 		kind create cluster $(KIND_ARGS) \
 			--name metal-control-plane \
 			--config $(KINDCONFIG) \
-			--kubeconfig $(KUBECONFIG); fi
+			--kubeconfig $(KUBECONFIG); \
+		kubectl --kubeconfig $(KUBECONFIG) config set-cluster kind-metal-control-plane \
+			--server=https://172.42.0.1:6443; fi
 	$(MAKE) create-proxy-registries
 	docker compose up -d --force-recreate cloud-provider-kind
 
@@ -459,7 +461,7 @@ build-dell-sonic:
 fetch-virtual-kubeconfig:
 	# TODO: it's hard to get the latest issued generic kubeconfig secret... just take the first result for now
 	kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden $(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden -l managed-by=secrets-manager,manager-identity=gardener-operator,name=generic-token-kubeconfig --no-headers | awk '{ print $$1 }') -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig
-	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.42.0.1.nip.io:4443
+	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.$(shell kubectl --kubeconfig=$(KUBECONFIG) get svc -n virtual-garden-istio-ingress istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip }').nip.io
 	@kubectl --kubeconfig=.virtual-kubeconfig config set-credentials garden --token=$(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden shoot-access-virtual-garden -o jsonpath='{.data.token}' | base64 -d)
 	@kubectl --kubeconfig=$(KUBECONFIG) config unset users.garden
 	@kubectl --kubeconfig=$(KUBECONFIG) config unset contexts.garden

compose.yaml

@@ -129,20 +129,25 @@ services:
       - REGISTRY_PROXY_TTL=168h
       - REGISTRY_STORAGE_DELETE_ENABLED=true
       - OTEL_TRACES_EXPORTER=none
+
   cloud-provider-kind:
     image: registry.k8s.io/cloud-provider-kind/cloud-controller-manager:v0.10.0
     restart: always
+    # stays on the docker network with the kind nodes and the load balancer containers;
+    # reaching them does not need host networking as long as kind's external apiserver
+    # endpoint is pinned to loopback, see control-plane/kind.yaml
     networks:
       - kind
     environment:
-      - KIND_EXPERIMENTAL_DOCKER_NETWORK=${KIND_EXPERIMENTAL_DOCKER_NETWORK:-kind}
+      - KIND_EXPERIMENTAL_DOCKER_NETWORK=${KIND_EXPERIMENTAL_DOCKER_NETWORK:-mini_lab_internal}
     command:
     # v0.10.0 of cloud controller does not support tcproutes, since it does not support the experimental gateway api channel
     # using envoy-gateway deployed via roles/gateway instead
       - --gateway-channel
       - disabled
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+
 volumes:
   proxy-docker:
   proxy-gcr:
@@ -152,5 +157,5 @@ volumes:
 
 networks:
   kind:
-    name: ${KIND_EXPERIMENTAL_DOCKER_NETWORK:-kind}
+    name: ${KIND_EXPERIMENTAL_DOCKER_NETWORK:-mini_lab_internal}
     external: true

control-plane/kind.yaml

@@ -1,5 +1,14 @@
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
+# MutatingAdmissionPolicy (used by roles/loadbalancer-ip-policy to pin the istio ingress
+# gateway load balancer IPs) is beta on 1.35: the gate is off by default and the beta API
+# group is not served unless requested. Both become unnecessary on 1.36, where it is GA.
+# kind applies featureGates to the apiserver, controller-manager and scheduler but not the
+# kubelet, and the gate is registered in the shared apiserver feature set, so this is safe.
+featureGates:
+  MutatingAdmissionPolicy: true
+runtimeConfig:
+  "admissionregistration.k8s.io/v1beta1": "true"
 networking:
   apiServerPort: 6443
   apiServerAddress: 172.42.0.1
@@ -9,12 +18,36 @@ nodes:
       - hostPath: ./control-plane/config-patches
         containerPath: /etc/containerd/certs.d
     extraPortMappings:
+      # This mapping MUST stay first among the 6443 mappings.
+      # cloud-provider-kind decides how it talks to the load balancer containers by
+      # probing kind's internal and external apiserver endpoints in parallel, taking
+      # whichever answers first. kind derives the external endpoint from index 0 of the
+      # node's 6443/tcp docker port bindings and appends its own apiServerAddress
+      # mapping *after* the ones listed here. Binding loopback first therefore pins the
+      # external endpoint to 127.0.0.1:6443, which is unreachable from inside the
+      # cloud-provider-kind container, so the internal endpoint always wins and
+      # connectivity is detected as "Direct". Otherwise it can land on "Portmap", where
+      # the envoy readiness probe dials a host port that is not published on Linux, the
+      # probe fails, and the LoadBalancer service status is never patched.
+      - containerPort: 6443
+        hostPort: 6443
+        listenAddress: "127.0.0.1"
       - containerPort: 4443
         hostPort: 4443
       - containerPort: 8080
         hostPort: 8080
-    # if you want to run gardener operator + metal-stack, you need more pods
     kubeadmConfigPatches:
+      # kind generates [localhost, <apiServerAddress>] only, and the patch replaces the
+      # list instead of merging it, so every SAN has to be repeated here.
+      - |
+        kind: ClusterConfiguration
+        apiServer:
+          certSANs:
+            - localhost
+            - 127.0.0.1
+            - 172.42.0.1
+      # if you want to run gardener operator + metal-stack on a single node, 
+      # you need more pods
       - |
         kind: InitConfiguration
         nodeRegistration:

deploy_control_plane.yaml

@@ -1,17 +1,21 @@
 ---
-- name: deploy control plane
+- name: deploy gateway controller
   hosts: control_plane
   connection: local
   gather_facts: false
   roles:
-    - name: ansible-common
-      tags: always
     - name: metal-roles/control-plane/roles/prepare
       tags: prepare
     - name: gateway
       tags: gateway
-    - name: ingress-controller
-      tags: ingress-controller
+
+- name: deploy control plane
+  hosts: control_plane
+  connection: local
+  gather_facts: false
+  roles:
+    - name: ansible-common
+      tags: always
     - name: metal-roles/control-plane/roles/nsq
       tags: nsq
     - name: metal-roles/control-plane/roles/metal-db
@@ -33,17 +37,15 @@
       tags: auth
     - name: metal-roles/control-plane/roles/metal
       tags: metal
-    - name: metal-roles/control-plane/roles/logging
-      when: monitoring_enabled
-      tags: logging
-    - name: metal-roles/control-plane/roles/monitoring
-      when: monitoring_enabled
-      tags: monitoring
+
+- name: deploy monitoring
+  import_playbook: deploy_monitoring.yaml
+  when: monitoring_enabled
 
 - name: deploy gardener
   import_playbook: deploy_gardener.yaml
   when: gardener_enabled and not kamaji_enabled
-    
+
 - name: deploy kamaji
   import_playbook: deploy_kamaji.yaml
   when: kamaji_enabled and not gardener_enabled

deploy_gardener.yaml

@@ -3,24 +3,13 @@
   hosts: control_plane
   connection: local
   gather_facts: false
-  pre_tasks:
-    - name: Fake Gardener metal shoot
-      k8s:
-        definition:
-          apiVersion: v1
-          kind: ConfigMap
-          metadata:
-            name: shoot-info
-            namespace: kube-system
-          data:
-            nodeNetwork: 172.42.0.0/16
-            podNetwork: 10.244.0.0/24
-            serviceNetwork: 10.96.0.0/16
-      tags: gardener
-
   roles:
     - name: ansible-common
       tags: always
+    - name: gardener-hacks
+      tags: gardener
+    - name: loadbalancer-ip-policy
+      tags: gardener
     - name: minio
       tags: minio
     - name: powerdns
@@ -29,8 +18,6 @@
       tags: gardener
     - name: metal-roles/control-plane/roles/gardener-extensions
       tags: gardener
-    - name: gardener-istio-patch
-      tags: gardener
     - name: metal-roles/control-plane/roles/gardener-virtual-garden-access
       tags: gardener
     - name: metal-roles/control-plane/roles/gardener-cloud-profile
@@ -61,57 +48,6 @@
           wait_timeout: 900
         tags: gardener
 
-      - name: Wait for istio ingress gateway service
-        kubernetes.core.k8s_info:
-          api_version: v1
-          kind: Service
-          name: istio-ingressgateway
-          namespace: istio-ingress
-        register: result
-        until: result.resources
-        retries: 30
-        delay: 10
-        tags: gardener
-
-      - name: Patch istio ingress gateway service status
-        patch_service_status_k8s:
-          name: istio-ingressgateway
-          namespace: istio-ingress
-          body:
-            status:
-              loadBalancer:
-                ingress:
-                  - ip: "172.42.0.1"
-        tags: gardener
-
-      - name: Expose istio gateway through ingress-nginx (for local environments)
-        k8s:
-          definition:
-            apiVersion: networking.k8s.io/v1
-            kind: Ingress
-            metadata:
-              annotations:
-                nginx.ingress.kubernetes.io/ssl-passthrough: "true"
-              name: apiserver-ingress
-              namespace: istio-ingress
-            spec:
-              ingressClassName: nginx
-              rules:
-              - host: "{{ metal_control_plane_stage_name }}.{{ gardener_gardenlet_default_dns_domain }}"
-                http:
-                  paths:
-                  - path: /
-                    pathType: Prefix
-                    backend:
-                      service:
-                        name: istio-ingressgateway
-                        port:
-                          number: 443
-              tls:
-              - hosts:
-                - "{{ metal_control_plane_stage_name }}.{{ gardener_gardenlet_default_dns_domain }}"
-        tags: gardener
-
       - name: Wait until Garden is ready
         kubernetes.core.k8s_info:
           api_version: "operator.gardener.cloud/v1alpha1"

deploy_monitoring.yaml

@@ -0,0 +1,12 @@
+---
+- name: deploy monitoring
+  hosts: control_plane
+  connection: local
+  gather_facts: false
+  roles:
+    - name: ingress-controller
+      tags: ingress-controller
+    - name: metal-roles/control-plane/roles/logging
+      tags: logging
+    - name: metal-roles/control-plane/roles/monitoring
+      tags: monitoring

inventories/group_vars/all/release_vector.yaml

@@ -5,6 +5,10 @@ metal_stack_release_vectors:
     variable_mapping_path: metal_stack_release.mapping
     include_role_defaults: metal-roles/common/roles/defaults
     oci_cosign_verify_key: "{{ lookup('file', 'cosign.pub') }}"
+
+gardener_extension_backup_s3_helm_chart_tag: v0.0.0-pull-request.31
+gardener_extension_backup_s3_additional_network_policies: true
+
 ##
 ## for development purposes, you can override releases from our image vector here
 ##
@@ -38,8 +42,8 @@ metal_stack_release_vectors:
 ## for ansible roles
 ##
 
-# ansible_common_version:
-# metal_roles_version:
+ansible_common_version: pr-51-remove-mini-lab-workaround
+metal_roles_version: pr-721-backup-s3-netpols
 # metal_ansible_modules_version:
 
 ##

inventories/group_vars/control_plane/gardener/gardenlet.yaml

@@ -15,7 +15,7 @@ gardener_gardenlets:
       name: backup-secret
       namespace: garden
   backup_infrastructure_secret:
-    endpoint: "{{ ('http://' + minio_dns_name + ':8080' ) | b64encode }}"
+    endpoint: "{{ ('http://' + minio_dns_name ) | b64encode }}"
     accessKeyID: "{{ minio_root_user | b64encode }}"
     secretAccessKey: "{{ minio_root_password | b64encode }}"
     region: "{{ 'us-east-1' | b64encode }}"

inventories/group_vars/control_plane/gardener/operator.yaml

@@ -1,5 +1,6 @@
 ---
-gardener_operator_ingress_dns_domain: "gardener.{{ metal_control_plane_ingress_dns }}"
+gardener_operator_ingress_dns_domain: "gardener.172.42.0.43.nip.io"
+gardener_operator_virtual_garden_public_dns: "gardener-kube-apiserver.172.42.0.43.nip.io"
 
 gardener_operator_backup_infrastructure:
   provider: S3
@@ -10,16 +11,12 @@ gardener_operator_backup_infrastructure:
    namespace: garden
 
 gardener_operator_backup_infrastructure_secret:
-  endpoint: "{{ ('http://' + minio_dns_name + ':8080' ) | b64encode }}"
+  endpoint: "{{ ('http://' + minio_dns_name ) | b64encode }}"
   accessKeyID: "{{ minio_root_user | b64encode }}"
   secretAccessKey: "{{ minio_root_password | b64encode }}"
   region: "{{ 'us-east-1' | b64encode }}"
   s3ForcePathStyle: "{{ 'true' | b64encode }}"
 
-# enable mini-lab patches
-gardener_operator_patch_istio_ingress_gateway_service_ip: 172.42.0.1
-gardener_operator_expose_virtual_garden_through_ingress_nginx: true
-
 # for local setups this should be sufficient
 gardener_operator_high_availability_control_plane: false
 

inventories/group_vars/control_plane/minio.yaml

@@ -2,4 +2,4 @@
 minio_root_user: mini-lab
 minio_root_password: change-me
 
-minio_dns_name: minio.172.42.0.1.nip.io
+minio_dns_name: minio.172.42.0.42.nip.io

inventories/group_vars/control_plane/powerdns.yaml

@@ -5,16 +5,16 @@ powerdns_webserver_password: change-me
 powerdns_mariadb_username: root
 powerdns_mariadb_password: change-me
 
-powerdns_load_balancer_dns_name: "ns.{{ metal_control_plane_ingress_dns }}"
+powerdns_load_balancer_dns_name: "ns.{{ metal_control_plane_gateway_dns }}"
 
-powerdns_api_dns_name: "powerdns-api.{{ metal_control_plane_ingress_dns }}"
+powerdns_api_dns_name: "powerdns-api.{{ metal_control_plane_gateway_dns }}"
 
 powerdns_zones:
-  - name: "gardener.172.42.0.1.nip.io."
+  - name: "gardener.172.42.0.43.nip.io."
     kind: Master
     nameservers:
       - "{{ powerdns_load_balancer_dns_name }}."
-  - name: "gardener-kube-apiserver.172.42.0.1.nip.io."
+  - name: "gardener-kube-apiserver.172.42.0.43.nip.io."
     kind: Master
     nameservers:
       - "{{ powerdns_load_balancer_dns_name }}."

roles/gardener-hacks/tasks/main.yaml

@@ -0,0 +1,14 @@
+---
+# this allows us to use the metal-roles gardener deployment simulating to run on a metalstack.cloud cluster
+- name: Fake Gardener metal shoot
+  k8s:
+    definition:
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: shoot-info
+        namespace: kube-system
+      data:
+        nodeNetwork: 172.42.0.0/16
+        podNetwork: 10.244.0.0/24
+        serviceNetwork: 10.96.0.0/16