OPNET-782: Render BGP VIP static pods instead of keepalived

cmd/machine-config-operator/bootstrap.go

@@ -31,6 +31,8 @@ var (
 		infraImage                     string
 		releaseImage                   string
 		keepalivedImage                string
+		frrK8sImage                    string
+		kubeVipImage                   string
 		mcoImage                       string
 		oauthProxyImage                string
 		kubeRbacProxyImage             string
@@ -68,6 +70,8 @@ func init() {
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.DNS, "dns-config-file", "/assets/manifests/cluster-dns-02-config.yml", "File containing dns.config.openshift.io manifest.")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.AdditionalTrustBundle, "additional-trust-bundle-config-file", "/assets/manifests/user-ca-bundle-config.yaml", "File containing the additional user provided CA bundle manifest.")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.keepalivedImage, "keepalived-image", "", "Image for Keepalived.")
+	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.frrK8sImage, "frr-k8s-image", "", "Image for frr-k8s.")
+	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.kubeVipImage, "kube-vip-image", "", "Image for kube-vip.")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.corednsImage, "coredns-image", "", "Image for CoreDNS.")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.haproxyImage, "haproxy-image", "", "Image for haproxy.")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.baremetalRuntimeCfgImage, "baremetal-runtimecfg-image", "", "Image for baremetal-runtimecfg.")
@@ -78,6 +82,7 @@ func init() {
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dockerRegistryImage, "docker-registry-image", "", "Image for docker-registry.")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.imageReferences, "image-references", "", "File containing imagestreams (from cluster-version-operator)")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.CloudProviderCA, "cloud-provider-ca-file", "", "path to cloud provider CA certificate")
+	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.BGPVIPConfig, "bgp-vip-config-file", "/assets/manifests/bgp-vip-config.yaml", "File containing the bgp-vip-config ConfigMap manifest (optional).")
 
 }
 
@@ -142,6 +147,18 @@ func runBootstrapCmd(_ *cobra.Command, _ []string) {
 		if err != nil {
 			klog.Warningf("Base OS extensions container not found: %s", err)
 		}
+		// The frr-k8s and kube-vip images are only present in payloads that
+		// support BGP-based VIP management, so their absence is not fatal.
+		if img, err := findImage(imgstream, "metallb-frr"); err == nil {
+			bootstrapOpts.frrK8sImage = img
+		} else {
+			klog.Warningf("metallb-frr image not found in image references: %v", err)
+		}
+		if img, err := findImage(imgstream, "kube-vip"); err == nil {
+			bootstrapOpts.kubeVipImage = img
+		} else {
+			klog.Warningf("kube-vip image not found in image references: %v", err)
+		}
 	}
 
 	imgs := ctrlcommon.Images{
@@ -155,6 +172,8 @@ func runBootstrapCmd(_ *cobra.Command, _ []string) {
 			BaseOSContainerImage:           bootstrapOpts.baseOSContainerImage,
 			BaseOSExtensionsContainerImage: bootstrapOpts.baseOSExtensionsContainerImage,
 			DockerRegistryBootstrap:        bootstrapOpts.dockerRegistryImage,
+			FRRK8sBootstrap:                bootstrapOpts.frrK8sImage,
+			KubeVIPBootstrap:               bootstrapOpts.kubeVipImage,
 		},
 		ControllerConfigImages: ctrlcommon.ControllerConfigImages{
 			InfraImage:          bootstrapOpts.infraImage,
@@ -163,6 +182,8 @@ func runBootstrapCmd(_ *cobra.Command, _ []string) {
 			Haproxy:             bootstrapOpts.haproxyImage,
 			BaremetalRuntimeCfg: bootstrapOpts.baremetalRuntimeCfgImage,
 			DockerRegistry:      bootstrapOpts.dockerRegistryImage,
+			FRRK8s:              bootstrapOpts.frrK8sImage,
+			KubeVip:             bootstrapOpts.kubeVipImage,
 		},
 	}
 

manifests/on-prem/0000-frr-k8s.yaml

@@ -0,0 +1,165 @@
+---
+kind: Pod
+apiVersion: v1
+metadata:
+  name: frr-k8s
+  namespace: openshift-frr-k8s
+  labels:
+    app: frr-k8s
+spec:
+  # hostNetwork: the BGP session must originate from the node IP and the
+  # advertised VIP routes live in the host routing table (keepalived parity).
+  hostNetwork: true
+  shareProcessNamespace: true
+  priorityClassName: system-node-critical
+  # 10s (not the upstream DaemonSet's 0): a SIGTERM'd bgpd sends a Cease
+  # NOTIFICATION, which hard-resets the session; after a SIGKILL a peer in
+  # graceful-restart helper mode may retain stale VIP routes on bare TCP
+  # loss until the restart timer expires.
+  terminationGracePeriodSeconds: 10
+  tolerations:
+  - operator: Exists
+  initContainers:
+  - name: render-config-frr
+    image: {{ .Images.BaremetalRuntimeCfgBootstrap }}
+    command:
+    - /usr/bin/runtimecfg
+    - render
+    - /etc/kubernetes/kubeconfig
+    - --api-vips
+    - {{ onPremPlatformAPIServerInternalIP .ControllerConfig }}
+    - --ingress-vips
+    - {{ onPremPlatformIngressIP .ControllerConfig }}
+    - "--cluster-config"
+    - "/opt/openshift/manifests/cluster-config.yaml"
+    - /config/frr.conf.tmpl
+    - --out-dir
+    - /etc/frr
+    - --peer-file
+    - /config/frr-peers.json
+    env:
+    - name: IS_BOOTSTRAP
+      value: "yes"
+    securityContext:
+      allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop:
+        - ALL
+    resources:
+      requests:
+        cpu: 10m
+        memory: 50Mi
+      limits:
+        cpu: 100m
+        memory: 128Mi
+    terminationMessagePolicy: FallbackToLogsOnError
+    volumeMounts:
+    - name: resource-dir
+      mountPath: /config
+      readOnly: true
+    - name: frr-conf
+      mountPath: /etc/frr
+    - name: kubeconfig
+      mountPath: /etc/kubernetes
+      readOnly: true
+    - name: manifests
+      mountPath: "/opt/openshift/manifests"
+      mountPropagation: HostToContainer
+      readOnly: true
+  - name: cp-frr-files
+    image: {{ .Images.FRRK8sBootstrap }}
+    command:
+    - /bin/sh
+    - -c
+    - |
+      set -eu
+      cp -f /tmp/frr/daemons /etc/frr/daemons
+      cp -f /tmp/frr/vtysh.conf /etc/frr/vtysh.conf
+    securityContext:
+      runAsUser: 100
+      runAsGroup: 101
+      allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop:
+        - ALL
+    resources:
+      requests:
+        cpu: 10m
+        memory: 20Mi
+      limits:
+        cpu: 100m
+        memory: 128Mi
+    terminationMessagePolicy: FallbackToLogsOnError
+    volumeMounts:
+    - name: frr-startup
+      mountPath: /tmp/frr
+      readOnly: true
+    - name: frr-conf
+      mountPath: /etc/frr
+  containers:
+  - name: frr
+    image: {{ .Images.FRRK8sBootstrap }}
+    command:
+    - /bin/sh
+    - -c
+    - /sbin/tini -- /usr/lib/frr/docker-start
+    env:
+    - name: TINI_SUBREAPER
+      value: "true"
+    resources:
+      requests:
+        cpu: 100m
+        memory: 200Mi
+      limits:
+        cpu: 500m
+        memory: 512Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      capabilities:
+        # Mirrors the frr container of the frr-k8s DaemonSet shipped by CNO
+        # (upstream metallb/frr-k8s parity). No drop ALL: FRR's
+        # privilege-separated startup (watchfrr as root spawning daemons as
+        # the frr user) additionally relies on root's implicit capabilities
+        # (SETUID/SETGID/DAC_OVERRIDE/CHOWN); dropping ALL and re-adding
+        # only the network capabilities was tested and prevents the FRR
+        # daemons from starting. SYS_ADMIN is required by docker-start/
+        # watchfrr process supervision.
+        add:
+        - NET_ADMIN
+        - NET_RAW
+        - SYS_ADMIN
+        - NET_BIND_SERVICE
+    terminationMessagePolicy: FallbackToLogsOnError
+    volumeMounts:
+    - name: frr-sockets
+      mountPath: /var/run/frr
+    - name: frr-conf
+      mountPath: /etc/frr
+    - name: frr-lib
+      mountPath: /var/lib/frr
+    - name: frr-tmp
+      mountPath: /var/tmp/frr
+  volumes:
+  - name: resource-dir
+    hostPath:
+      path: /etc/kubernetes/static-pod-resources/frr-k8s
+  - name: frr-conf
+    emptyDir: {}
+  - name: frr-sockets
+    emptyDir: {}
+  - name: frr-lib
+    emptyDir: {}
+  - name: frr-tmp
+    emptyDir: {}
+  - name: frr-startup
+    hostPath:
+      path: /etc/kubernetes/static-pod-resources/frr-k8s/startup
+  - name: kubeconfig
+    hostPath:
+      path: /etc/kubernetes
+  - name: manifests
+    hostPath:
+      path: "/opt/openshift/manifests"

manifests/on-prem/0010-kube-vip-api.yaml

@@ -0,0 +1,66 @@
+---
+kind: Pod
+apiVersion: v1
+metadata:
+  name: kube-vip-api
+  namespace: openshift-kube-vip
+  labels:
+    app: kube-vip
+    component: api
+spec:
+  hostNetwork: true
+  priorityClassName: system-node-critical
+  tolerations:
+  - operator: Exists
+  containers:
+  - name: kube-vip
+    image: {{ .Images.KubeVIPBootstrap }}
+    args:
+    - manager
+    env:
+    - name: address
+      value: "{{ onPremPlatformAPIServerInternalIP .ControllerConfig }}"
+    - name: k8s_config_file
+      value: "/etc/kubernetes/kubeconfig"
+    - name: kubernetes_addr
+      value: "https://localhost:6443"
+    - name: port
+      value: "6443"
+    - name: cp_enable
+      value: "true"
+    - name: vip_leaderelection
+      value: "true"
+    - name: vip_routingtable
+      value: "true"
+    - name: vip_cleanroutingtable
+      value: "true"
+    - name: vip_routingtableid
+      value: "198"
+    - name: vip_routingtableprotocol
+      value: "248"
+    - name: backend_health_check_interval
+      value: "5"
+    resources:
+      requests:
+        cpu: 50m
+        memory: 64Mi
+      limits:
+        cpu: 200m
+        memory: 256Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop:
+        - ALL
+        add:
+        - NET_ADMIN
+        - NET_RAW
+    volumeMounts:
+    - name: kubeconfig
+      mountPath: /etc/kubernetes
+      readOnly: true
+  volumes:
+  - name: kubeconfig
+    hostPath:
+      path: /etc/kubernetes

manifests/on-prem/frr-peers.json.tmpl

@@ -0,0 +1 @@
+{{- .ControllerConfig.BGPVIPPeersJSON -}}

manifests/on-prem/frr-startup-daemons

@@ -0,0 +1,23 @@
+zebra=yes
+bgpd=yes
+ospfd=no
+ospf6d=no
+ripd=no
+ripngd=no
+isisd=no
+pimd=no
+ldpd=no
+nhrpd=no
+eigrpd=no
+babeld=no
+sharpd=no
+pbrd=no
+bfdd=yes
+fabricd=no
+vrrpd=no
+pathd=no
+
+vtysh_enable=yes
+zebra_options="  -A 127.0.0.1 -s 90000000"
+bgpd_options="   -A 127.0.0.1"
+bfdd_options="   -A 127.0.0.1"

manifests/on-prem/frr-startup-vtysh.conf

@@ -0,0 +1 @@
+service integrated-vtysh-config

manifests/on-prem/frr.conf.tmpl

@@ -0,0 +1,58 @@
+frr version 9.1
+frr defaults traditional
+hostname {{`{{.Hostname}}`}}
+!
+router bgp {{`{{.LocalASN}}`}}
+ bgp router-id {{`{{.RouterID}}`}}
+ no bgp ebgp-requires-policy
+ no bgp default ipv4-unicast
+ !
+{{`{{- range .Peers}}`}}
+ neighbor {{`{{.PeerAddress}}`}} remote-as {{`{{.PeerASN}}`}}
+{{`{{- if .Password}}`}}
+ neighbor {{`{{.PeerAddress}}`}} password {{`{{.Password}}`}}
+{{`{{- end}}`}}
+{{`{{- if eq .BFDEnabled "true"}}`}}
+ neighbor {{`{{.PeerAddress}}`}} bfd
+{{`{{- end}}`}}
+{{`{{- if eq .EBGPMultiHop "true"}}`}}
+ neighbor {{`{{.PeerAddress}}`}} ebgp-multihop
+{{`{{- end}}`}}
+{{`{{- if .HoldTime}}`}}
+ neighbor {{`{{.PeerAddress}}`}} timers {{`{{.KeepaliveTime}}`}} {{`{{.HoldTime}}`}}
+{{`{{- end}}`}}
+{{`{{- end}}`}}
+ !
+ address-family ipv4 unicast
+  redistribute table-direct 198
+{{`{{- range .Peers}}`}}
+{{`{{- if isIPv4 .PeerAddress}}`}}
+  neighbor {{`{{.PeerAddress}}`}} activate
+{{`{{- if $.Communities}}`}}
+  neighbor {{`{{.PeerAddress}}`}} route-map VIP-COMMUNITY out
+{{`{{- end}}`}}
+{{`{{- end}}`}}
+{{`{{- end}}`}}
+ exit-address-family
+ !
+ address-family ipv6 unicast
+  redistribute table-direct 198
+{{`{{- range .Peers}}`}}
+{{`{{- if isIPv6 .PeerAddress}}`}}
+  neighbor {{`{{.PeerAddress}}`}} activate
+{{`{{- if $.Communities}}`}}
+  neighbor {{`{{.PeerAddress}}`}} route-map VIP-COMMUNITY out
+{{`{{- end}}`}}
+{{`{{- end}}`}}
+{{`{{- end}}`}}
+ exit-address-family
+!
+{{`{{- if .Communities}}`}}
+route-map VIP-COMMUNITY permit 10
+{{`{{- range .Communities}}`}}
+ set community {{`{{.}}`}} additive
+{{`{{- end}}`}}
+!
+{{`{{- end}}`}}
+line vty
+!