diff --git a/cmd/machine-config-operator/bootstrap.go b/cmd/machine-config-operator/bootstrap.go index 5a37c1883a..d1ae50d15a 100644 --- a/cmd/machine-config-operator/bootstrap.go +++ b/cmd/machine-config-operator/bootstrap.go @@ -31,6 +31,8 @@ var ( infraImage string releaseImage string keepalivedImage string + frrK8sImage string + kubeVipImage string mcoImage string oauthProxyImage string kubeRbacProxyImage string @@ -68,6 +70,8 @@ func init() { bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.DNS, "dns-config-file", "/assets/manifests/cluster-dns-02-config.yml", "File containing dns.config.openshift.io manifest.") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.AdditionalTrustBundle, "additional-trust-bundle-config-file", "/assets/manifests/user-ca-bundle-config.yaml", "File containing the additional user provided CA bundle manifest.") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.keepalivedImage, "keepalived-image", "", "Image for Keepalived.") + bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.frrK8sImage, "frr-k8s-image", "", "Image for frr-k8s.") + bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.kubeVipImage, "kube-vip-image", "", "Image for kube-vip.") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.corednsImage, "coredns-image", "", "Image for CoreDNS.") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.haproxyImage, "haproxy-image", "", "Image for haproxy.") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.baremetalRuntimeCfgImage, "baremetal-runtimecfg-image", "", "Image for baremetal-runtimecfg.") @@ -78,6 +82,7 @@ func init() { bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dockerRegistryImage, "docker-registry-image", "", "Image for docker-registry.") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.imageReferences, "image-references", "", "File containing imagestreams (from cluster-version-operator)") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.CloudProviderCA, "cloud-provider-ca-file", "", "path to cloud provider CA certificate") + bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.dependencyFiles.BGPVIPConfig, "bgp-vip-config-file", "/assets/manifests/bgp-vip-config.yaml", "File containing the bgp-vip-config ConfigMap manifest (optional).") } @@ -142,6 +147,18 @@ func runBootstrapCmd(_ *cobra.Command, _ []string) { if err != nil { klog.Warningf("Base OS extensions container not found: %s", err) } + // The frr-k8s and kube-vip images are only present in payloads that + // support BGP-based VIP management, so their absence is not fatal. + if img, err := findImage(imgstream, "metallb-frr"); err == nil { + bootstrapOpts.frrK8sImage = img + } else { + klog.Warningf("metallb-frr image not found in image references: %v", err) + } + if img, err := findImage(imgstream, "kube-vip"); err == nil { + bootstrapOpts.kubeVipImage = img + } else { + klog.Warningf("kube-vip image not found in image references: %v", err) + } } imgs := ctrlcommon.Images{ @@ -155,6 +172,8 @@ func runBootstrapCmd(_ *cobra.Command, _ []string) { BaseOSContainerImage: bootstrapOpts.baseOSContainerImage, BaseOSExtensionsContainerImage: bootstrapOpts.baseOSExtensionsContainerImage, DockerRegistryBootstrap: bootstrapOpts.dockerRegistryImage, + FRRK8sBootstrap: bootstrapOpts.frrK8sImage, + KubeVIPBootstrap: bootstrapOpts.kubeVipImage, }, ControllerConfigImages: ctrlcommon.ControllerConfigImages{ InfraImage: bootstrapOpts.infraImage, @@ -163,6 +182,8 @@ func runBootstrapCmd(_ *cobra.Command, _ []string) { Haproxy: bootstrapOpts.haproxyImage, BaremetalRuntimeCfg: bootstrapOpts.baremetalRuntimeCfgImage, DockerRegistry: bootstrapOpts.dockerRegistryImage, + FRRK8s: bootstrapOpts.frrK8sImage, + KubeVip: bootstrapOpts.kubeVipImage, }, } diff --git a/manifests/on-prem/0000-frr-k8s.yaml b/manifests/on-prem/0000-frr-k8s.yaml new file mode 100644 index 0000000000..05b0c5e49b --- /dev/null +++ b/manifests/on-prem/0000-frr-k8s.yaml @@ -0,0 +1,165 @@ +--- +kind: Pod +apiVersion: v1 +metadata: + name: frr-k8s + namespace: openshift-frr-k8s + labels: + app: frr-k8s +spec: + # hostNetwork: the BGP session must originate from the node IP and the + # advertised VIP routes live in the host routing table (keepalived parity). + hostNetwork: true + shareProcessNamespace: true + priorityClassName: system-node-critical + # 10s (not the upstream DaemonSet's 0): a SIGTERM'd bgpd sends a Cease + # NOTIFICATION, which hard-resets the session; after a SIGKILL a peer in + # graceful-restart helper mode may retain stale VIP routes on bare TCP + # loss until the restart timer expires. + terminationGracePeriodSeconds: 10 + tolerations: + - operator: Exists + initContainers: + - name: render-config-frr + image: {{ .Images.BaremetalRuntimeCfgBootstrap }} + command: + - /usr/bin/runtimecfg + - render + - /etc/kubernetes/kubeconfig + - --api-vips + - {{ onPremPlatformAPIServerInternalIP .ControllerConfig }} + - --ingress-vips + - {{ onPremPlatformIngressIP .ControllerConfig }} + - "--cluster-config" + - "/opt/openshift/manifests/cluster-config.yaml" + - /config/frr.conf.tmpl + - --out-dir + - /etc/frr + - --peer-file + - /config/frr-peers.json + env: + - name: IS_BOOTSTRAP + value: "yes" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 10m + memory: 50Mi + limits: + cpu: 100m + memory: 128Mi + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: resource-dir + mountPath: /config + readOnly: true + - name: frr-conf + mountPath: /etc/frr + - name: kubeconfig + mountPath: /etc/kubernetes + readOnly: true + - name: manifests + mountPath: "/opt/openshift/manifests" + mountPropagation: HostToContainer + readOnly: true + - name: cp-frr-files + image: {{ .Images.FRRK8sBootstrap }} + command: + - /bin/sh + - -c + - | + set -eu + cp -f /tmp/frr/daemons /etc/frr/daemons + cp -f /tmp/frr/vtysh.conf /etc/frr/vtysh.conf + securityContext: + runAsUser: 100 + runAsGroup: 101 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 10m + memory: 20Mi + limits: + cpu: 100m + memory: 128Mi + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: frr-startup + mountPath: /tmp/frr + readOnly: true + - name: frr-conf + mountPath: /etc/frr + containers: + - name: frr + image: {{ .Images.FRRK8sBootstrap }} + command: + - /bin/sh + - -c + - /sbin/tini -- /usr/lib/frr/docker-start + env: + - name: TINI_SUBREAPER + value: "true" + resources: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + # Mirrors the frr container of the frr-k8s DaemonSet shipped by CNO + # (upstream metallb/frr-k8s parity). No drop ALL: FRR's + # privilege-separated startup (watchfrr as root spawning daemons as + # the frr user) additionally relies on root's implicit capabilities + # (SETUID/SETGID/DAC_OVERRIDE/CHOWN); dropping ALL and re-adding + # only the network capabilities was tested and prevents the FRR + # daemons from starting. SYS_ADMIN is required by docker-start/ + # watchfrr process supervision. + add: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + - NET_BIND_SERVICE + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: frr-sockets + mountPath: /var/run/frr + - name: frr-conf + mountPath: /etc/frr + - name: frr-lib + mountPath: /var/lib/frr + - name: frr-tmp + mountPath: /var/tmp/frr + volumes: + - name: resource-dir + hostPath: + path: /etc/kubernetes/static-pod-resources/frr-k8s + - name: frr-conf + emptyDir: {} + - name: frr-sockets + emptyDir: {} + - name: frr-lib + emptyDir: {} + - name: frr-tmp + emptyDir: {} + - name: frr-startup + hostPath: + path: /etc/kubernetes/static-pod-resources/frr-k8s/startup + - name: kubeconfig + hostPath: + path: /etc/kubernetes + - name: manifests + hostPath: + path: "/opt/openshift/manifests" diff --git a/manifests/on-prem/0010-kube-vip-api.yaml b/manifests/on-prem/0010-kube-vip-api.yaml new file mode 100644 index 0000000000..55f6eda4fc --- /dev/null +++ b/manifests/on-prem/0010-kube-vip-api.yaml @@ -0,0 +1,66 @@ +--- +kind: Pod +apiVersion: v1 +metadata: + name: kube-vip-api + namespace: openshift-kube-vip + labels: + app: kube-vip + component: api +spec: + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - operator: Exists + containers: + - name: kube-vip + image: {{ .Images.KubeVIPBootstrap }} + args: + - manager + env: + - name: address + value: "{{ onPremPlatformAPIServerInternalIP .ControllerConfig }}" + - name: k8s_config_file + value: "/etc/kubernetes/kubeconfig" + - name: kubernetes_addr + value: "https://localhost:6443" + - name: port + value: "6443" + - name: cp_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_routingtable + value: "true" + - name: vip_cleanroutingtable + value: "true" + - name: vip_routingtableid + value: "198" + - name: vip_routingtableprotocol + value: "248" + - name: backend_health_check_interval + value: "5" + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - name: kubeconfig + mountPath: /etc/kubernetes + readOnly: true + volumes: + - name: kubeconfig + hostPath: + path: /etc/kubernetes diff --git a/manifests/on-prem/frr-peers.json.tmpl b/manifests/on-prem/frr-peers.json.tmpl new file mode 100644 index 0000000000..b91e5f25b4 --- /dev/null +++ b/manifests/on-prem/frr-peers.json.tmpl @@ -0,0 +1 @@ +{{- .ControllerConfig.BGPVIPPeersJSON -}} diff --git a/manifests/on-prem/frr-startup-daemons b/manifests/on-prem/frr-startup-daemons new file mode 100644 index 0000000000..f5696cb62d --- /dev/null +++ b/manifests/on-prem/frr-startup-daemons @@ -0,0 +1,23 @@ +zebra=yes +bgpd=yes +ospfd=no +ospf6d=no +ripd=no +ripngd=no +isisd=no +pimd=no +ldpd=no +nhrpd=no +eigrpd=no +babeld=no +sharpd=no +pbrd=no +bfdd=yes +fabricd=no +vrrpd=no +pathd=no + +vtysh_enable=yes +zebra_options=" -A 127.0.0.1 -s 90000000" +bgpd_options=" -A 127.0.0.1" +bfdd_options=" -A 127.0.0.1" diff --git a/manifests/on-prem/frr-startup-vtysh.conf b/manifests/on-prem/frr-startup-vtysh.conf new file mode 100644 index 0000000000..e0ab9cb6f3 --- /dev/null +++ b/manifests/on-prem/frr-startup-vtysh.conf @@ -0,0 +1 @@ +service integrated-vtysh-config diff --git a/manifests/on-prem/frr.conf.tmpl b/manifests/on-prem/frr.conf.tmpl new file mode 100644 index 0000000000..687ec9c50b --- /dev/null +++ b/manifests/on-prem/frr.conf.tmpl @@ -0,0 +1,58 @@ +frr version 9.1 +frr defaults traditional +hostname {{`{{.Hostname}}`}} +! +router bgp {{`{{.LocalASN}}`}} + bgp router-id {{`{{.RouterID}}`}} + no bgp ebgp-requires-policy + no bgp default ipv4-unicast + ! +{{`{{- range .Peers}}`}} + neighbor {{`{{.PeerAddress}}`}} remote-as {{`{{.PeerASN}}`}} +{{`{{- if .Password}}`}} + neighbor {{`{{.PeerAddress}}`}} password {{`{{.Password}}`}} +{{`{{- end}}`}} +{{`{{- if eq .BFDEnabled "true"}}`}} + neighbor {{`{{.PeerAddress}}`}} bfd +{{`{{- end}}`}} +{{`{{- if eq .EBGPMultiHop "true"}}`}} + neighbor {{`{{.PeerAddress}}`}} ebgp-multihop +{{`{{- end}}`}} +{{`{{- if .HoldTime}}`}} + neighbor {{`{{.PeerAddress}}`}} timers {{`{{.KeepaliveTime}}`}} {{`{{.HoldTime}}`}} +{{`{{- end}}`}} +{{`{{- end}}`}} + ! + address-family ipv4 unicast + redistribute table-direct 198 +{{`{{- range .Peers}}`}} +{{`{{- if isIPv4 .PeerAddress}}`}} + neighbor {{`{{.PeerAddress}}`}} activate +{{`{{- if $.Communities}}`}} + neighbor {{`{{.PeerAddress}}`}} route-map VIP-COMMUNITY out +{{`{{- end}}`}} +{{`{{- end}}`}} +{{`{{- end}}`}} + exit-address-family + ! + address-family ipv6 unicast + redistribute table-direct 198 +{{`{{- range .Peers}}`}} +{{`{{- if isIPv6 .PeerAddress}}`}} + neighbor {{`{{.PeerAddress}}`}} activate +{{`{{- if $.Communities}}`}} + neighbor {{`{{.PeerAddress}}`}} route-map VIP-COMMUNITY out +{{`{{- end}}`}} +{{`{{- end}}`}} +{{`{{- end}}`}} + exit-address-family +! +{{`{{- if .Communities}}`}} +route-map VIP-COMMUNITY permit 10 +{{`{{- range .Communities}}`}} + set community {{`{{.}}`}} additive +{{`{{- end}}`}} +! +{{`{{- end}}`}} +line vty +! diff --git a/pkg/controller/common/helpers.go b/pkg/controller/common/helpers.go index a0fe7b1bce..5d543c7f1c 100644 --- a/pkg/controller/common/helpers.go +++ b/pkg/controller/common/helpers.go @@ -1605,3 +1605,23 @@ func DetectRuncInMachineConfig(mc *mcfgv1.MachineConfig) (string, error) { } return "", nil } + +// IsBGPVIPManagement reports whether BGP-based VIP management is active: +// BareMetal platform, vipManagement "BGP", and VIPs present. Without VIPs +// (user-managed load balancer) there is nothing to advertise and the +// VIP-consuming templates would render empty values, so the keepalived +// default applies. Shared by the template renderer, the bootstrap manifest +// selection, and the operator's peers ingestion so they cannot disagree. +func IsBGPVIPManagement(infra *configv1.Infrastructure) bool { + if infra == nil || infra.Status.PlatformStatus == nil { + return false + } + ps := infra.Status.PlatformStatus + if ps.Type != configv1.BareMetalPlatformType || ps.BareMetal == nil { + return false + } + if len(ps.BareMetal.APIServerInternalIPs) == 0 || len(ps.BareMetal.IngressIPs) == 0 { + return false + } + return ps.BareMetal.VIPManagement == configv1.VIPManagementTypeBGP +} diff --git a/pkg/controller/common/images.go b/pkg/controller/common/images.go index edb7fb0399..9762254066 100644 --- a/pkg/controller/common/images.go +++ b/pkg/controller/common/images.go @@ -41,6 +41,8 @@ type RenderConfigImages struct { OauthProxy string `json:"oauthProxy"` KubeRbacProxy string `json:"kubeRbacProxy"` DockerRegistryBootstrap string `json:"dockerRegistry"` + FRRK8sBootstrap string `json:"frrK8s"` + KubeVIPBootstrap string `json:"kubeVip"` } // ControllerConfigImages are image names used to render templates under ./templates/ @@ -51,6 +53,8 @@ type ControllerConfigImages struct { Haproxy string `json:"haproxyImage"` BaremetalRuntimeCfg string `json:"baremetalRuntimeCfgImage"` DockerRegistry string `json:"dockerRegistryImage"` + FRRK8s string `json:"frrK8sImage"` + KubeVip string `json:"kubeVipImage"` } // Parses the JSON blob containing the images information into an Images struct. diff --git a/pkg/controller/template/constants.go b/pkg/controller/template/constants.go index 91baec288e..d033553bff 100644 --- a/pkg/controller/template/constants.go +++ b/pkg/controller/template/constants.go @@ -22,6 +22,16 @@ const ( // BaremetalRuntimeCfgKey is the key that references the baremetal-runtimecfg image in the controller BaremetalRuntimeCfgKey string = "baremetalRuntimeCfgImage" + // FRRK8sKey is the key for the frr-k8s image used by the frr-k8s static pod. + // A single image is used for all frr-k8s containers (controller, FRR daemon, + // reloader, metrics, status) in OpenShift. + FRRK8sKey string = "frrK8sImage" + + // KubeVIPKey is the key for the kube-vip image used by the kube-vip static pods. + // kube-vip manages API and Ingress VIPs in Routing Table Mode for BGP-based + // VIP management. + KubeVIPKey string = "kubeVipImage" + // KubeRbacProxyKey the key that references the kubeRbacProxy image KubeRbacProxyKey string = "kubeRbacProxyImage" diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index 39492770f9..ead241f145 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -386,6 +386,7 @@ func renderTemplate(config RenderConfig, path string, b []byte) ([]byte, error) funcs["urlHost"] = urlHost funcs["urlPort"] = urlPort funcs["isOpenShiftManagedDefaultLB"] = isOpenShiftManagedDefaultLB + funcs["isBGPVIPManagement"] = isBGPVIPManagement funcs["dnsRecordsType"] = dnsRecordsType funcs["cloudPlatformAPIIntLoadBalancerIPs"] = cloudPlatformAPIIntLoadBalancerIPs funcs["cloudPlatformAPILoadBalancerIPs"] = cloudPlatformAPILoadBalancerIPs @@ -497,10 +498,19 @@ func onPremPlatformIngressIP(cfg RenderConfig) (interface{}, error) { if cfg.Infra.Status.PlatformStatus != nil { switch cfg.Infra.Status.PlatformStatus.Type { case configv1.BareMetalPlatformType: + if len(cfg.Infra.Status.PlatformStatus.BareMetal.IngressIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.BareMetal.IngressIPs[0], nil case configv1.OvirtPlatformType: + if len(cfg.Infra.Status.PlatformStatus.Ovirt.IngressIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.Ovirt.IngressIPs[0], nil case configv1.OpenStackPlatformType: + if len(cfg.Infra.Status.PlatformStatus.OpenStack.IngressIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.OpenStack.IngressIPs[0], nil case configv1.VSpherePlatformType: if cfg.Infra.Status.PlatformStatus.VSphere != nil { @@ -513,6 +523,9 @@ func onPremPlatformIngressIP(cfg RenderConfig) (interface{}, error) { // and there is also no data return nil, nil case configv1.NutanixPlatformType: + if len(cfg.Infra.Status.PlatformStatus.Nutanix.IngressIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.Nutanix.IngressIPs[0], nil default: return nil, fmt.Errorf("invalid platform for Ingress IP") @@ -557,10 +570,19 @@ func onPremPlatformAPIServerInternalIP(cfg RenderConfig) (interface{}, error) { if cfg.Infra.Status.PlatformStatus != nil { switch cfg.Infra.Status.PlatformStatus.Type { case configv1.BareMetalPlatformType: + if len(cfg.Infra.Status.PlatformStatus.BareMetal.APIServerInternalIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.BareMetal.APIServerInternalIPs[0], nil case configv1.OvirtPlatformType: + if len(cfg.Infra.Status.PlatformStatus.Ovirt.APIServerInternalIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.Ovirt.APIServerInternalIPs[0], nil case configv1.OpenStackPlatformType: + if len(cfg.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIPs[0], nil case configv1.VSpherePlatformType: if cfg.Infra.Status.PlatformStatus.VSphere != nil { @@ -573,6 +595,9 @@ func onPremPlatformAPIServerInternalIP(cfg RenderConfig) (interface{}, error) { // and there is also no data return nil, nil case configv1.NutanixPlatformType: + if len(cfg.Infra.Status.PlatformStatus.Nutanix.APIServerInternalIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.Nutanix.APIServerInternalIPs[0], nil default: return nil, fmt.Errorf("invalid platform for API Server Internal IP") @@ -723,6 +748,14 @@ func isOpenShiftManagedDefaultLB(cfg RenderConfig) bool { return false } +// isBGPVIPManagement returns true when the cluster is configured to use +// BGP-based VIP management instead of keepalived/VRRP. This controls +// whether MCO renders frr-k8s static pod manifests (BGP) or keepalived +// static pod manifests (default). +func isBGPVIPManagement(cfg RenderConfig) bool { + return ctrlcommon.IsBGPVIPManagement(cfg.Infra) +} + func dnsRecordsType(cfg RenderConfig) configv1.DNSRecordsType { if cfg.Infra.Status.PlatformStatus != nil { switch cfg.Infra.Status.PlatformStatus.Type { diff --git a/pkg/controller/template/render_test.go b/pkg/controller/template/render_test.go index 10e6ab739b..af89d9dcc3 100644 --- a/pkg/controller/template/render_test.go +++ b/pkg/controller/template/render_test.go @@ -596,3 +596,148 @@ func verifyIgn(actual [][]byte, dir string, t *testing.T) { t.Errorf("can't find expected file:\n%v", key) } } + +func TestIsBGPVIPManagement(t *testing.T) { + dummyTemplate := []byte(`{{isBGPVIPManagement .}}`) + + cases := []struct { + name string + infra *configv1.Infrastructure + result string + }{ + { + name: "nil infra", + infra: nil, + result: "false", + }, + { + name: "nil platform status", + infra: &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{}, + }, + result: "false", + }, + { + name: "baremetal without VIPManagement", + infra: &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{ + PlatformStatus: &configv1.PlatformStatus{ + Type: configv1.BareMetalPlatformType, + BareMetal: &configv1.BareMetalPlatformStatus{}, + }, + }, + }, + result: "false", + }, + { + name: "baremetal with VIPManagement BGP", + infra: &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{ + PlatformStatus: &configv1.PlatformStatus{ + Type: configv1.BareMetalPlatformType, + BareMetal: &configv1.BareMetalPlatformStatus{ + VIPManagement: "BGP", + APIServerInternalIPs: []string{"192.168.111.5"}, + IngressIPs: []string{"192.168.111.4"}, + }, + }, + }, + }, + result: "true", + }, + { + name: "baremetal with VIPManagement BGP but no VIPs (user-managed LB)", + infra: &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{ + PlatformStatus: &configv1.PlatformStatus{ + Type: configv1.BareMetalPlatformType, + BareMetal: &configv1.BareMetalPlatformStatus{ + VIPManagement: "BGP", + }, + }, + }, + }, + result: "false", + }, + { + name: "baremetal with VIPManagement BGP but no ingress VIPs", + infra: &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{ + PlatformStatus: &configv1.PlatformStatus{ + Type: configv1.BareMetalPlatformType, + BareMetal: &configv1.BareMetalPlatformStatus{ + VIPManagement: "BGP", + APIServerInternalIPs: []string{"192.168.111.5"}, + }, + }, + }, + }, + result: "false", + }, + { + name: "baremetal with VIPManagement Keepalived", + infra: &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{ + PlatformStatus: &configv1.PlatformStatus{ + Type: configv1.BareMetalPlatformType, + BareMetal: &configv1.BareMetalPlatformStatus{ + VIPManagement: "Keepalived", + }, + }, + }, + }, + result: "false", + }, + { + name: "nil baremetal status", + infra: &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{ + PlatformStatus: &configv1.PlatformStatus{ + Type: configv1.BareMetalPlatformType, + }, + }, + }, + result: "false", + }, + { + name: "vsphere platform", + infra: &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{ + PlatformStatus: &configv1.PlatformStatus{ + Type: configv1.VSpherePlatformType, + }, + }, + }, + result: "false", + }, + { + name: "aws platform", + infra: &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{ + PlatformStatus: &configv1.PlatformStatus{ + Type: configv1.AWSPlatformType, + }, + }, + }, + result: "false", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + config := &mcfgv1.ControllerConfig{ + Spec: mcfgv1.ControllerConfigSpec{ + Infra: c.infra, + }, + } + + got, err := renderTemplate(RenderConfig{&config.Spec, `{"dummy":"dummy"}`, "dummy", nil, nil}, c.name, dummyTemplate) + if err != nil { + t.Fatalf("expected nil error, got: %v", err) + } + if string(got) != c.result { + t.Fatalf("mismatch: got %q, want %q", string(got), c.result) + } + }) + } +} diff --git a/pkg/operator/bootstrap.go b/pkg/operator/bootstrap.go index 459143e431..b0c968ca42 100644 --- a/pkg/operator/bootstrap.go +++ b/pkg/operator/bootstrap.go @@ -112,6 +112,15 @@ func buildSpec(dependencies *BootstrapDependencies, imgs *ctrlcommon.Images, rel return nil, err } + // BGP-based VIP management renders frr-k8s and kube-vip static pods at + // bootstrap; without their images the manifests would carry empty image + // fields and the install dies much later with no useful signal. + if ctrlcommon.IsBGPVIPManagement(dependencies.Infrastructure) { + if imgs.FRRK8s == "" || imgs.KubeVip == "" { + return nil, fmt.Errorf("BGP-based VIP management is enabled but the frr-k8s (%q) or kube-vip (%q) image is missing from the release payload", imgs.FRRK8s, imgs.KubeVip) + } + } + if dependencies.AdditionalTrustBundle != "" { spec.AdditionalTrustBundle = []byte(dependencies.AdditionalTrustBundle) } @@ -130,6 +139,7 @@ func buildSpec(dependencies *BootstrapDependencies, imgs *ctrlcommon.Images, rel } spec.RootCAData = []byte(dependencies.MCSCA) + spec.BGPVIPPeersJSON = dependencies.BGPVIPPeersJSON spec.PullSecret = nil spec.BaseOSContainerImage = imgs.BaseOSContainerImage spec.BaseOSExtensionsContainerImage = imgs.BaseOSExtensionsContainerImage @@ -145,6 +155,8 @@ func buildSpec(dependencies *BootstrapDependencies, imgs *ctrlcommon.Images, rel templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg, templatectrl.KubeRbacProxyKey: imgs.KubeRbacProxy, templatectrl.DockerRegistryKey: imgs.DockerRegistry, + templatectrl.FRRK8sKey: imgs.FRRK8s, + templatectrl.KubeVIPKey: imgs.KubeVip, } config := getRenderConfig("", dependencies.KubeAPIServerServingCA, spec, @@ -154,25 +166,29 @@ func buildSpec(dependencies *BootstrapDependencies, imgs *ctrlcommon.Images, rel func appendManifestsByPlatform(manifests []manifest, infra *configv1.Infrastructure) []manifest { lbType := configv1.LoadBalancerTypeOpenShiftManagedDefault + var vipManagement configv1.VIPManagementType if infra.Status.PlatformStatus.BareMetal != nil { if infra.Status.PlatformStatus.BareMetal.LoadBalancer != nil { lbType = infra.Status.PlatformStatus.BareMetal.LoadBalancer.Type } - manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.BareMetalPlatformType)), lbType) + if ctrlcommon.IsBGPVIPManagement(infra) { + vipManagement = configv1.VIPManagementTypeBGP + } + manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.BareMetalPlatformType)), lbType, vipManagement) } if infra.Status.PlatformStatus.OpenStack != nil { if infra.Status.PlatformStatus.OpenStack.LoadBalancer != nil { lbType = infra.Status.PlatformStatus.OpenStack.LoadBalancer.Type } - manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.OpenStackPlatformType)), lbType) + manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.OpenStackPlatformType)), lbType, "") } if infra.Status.PlatformStatus.Ovirt != nil { if infra.Status.PlatformStatus.Ovirt.LoadBalancer != nil { lbType = infra.Status.PlatformStatus.Ovirt.LoadBalancer.Type } - manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.OvirtPlatformType)), lbType) + manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.OvirtPlatformType)), lbType, "") } if infra.Status.PlatformStatus.VSphere != nil { @@ -189,14 +205,14 @@ func appendManifestsByPlatform(manifests []manifest, infra *configv1.Infrastruct return manifests } } - manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.VSpherePlatformType)), lbType) + manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.VSpherePlatformType)), lbType, "") } if infra.Status.PlatformStatus.Nutanix != nil { if infra.Status.PlatformStatus.Nutanix.LoadBalancer != nil { lbType = infra.Status.PlatformStatus.Nutanix.LoadBalancer.Type } - manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.NutanixPlatformType)), lbType) + manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.NutanixPlatformType)), lbType, "") } if infra.Status.PlatformStatus.GCP != nil { @@ -205,7 +221,7 @@ func appendManifestsByPlatform(manifests []manifest, infra *configv1.Infrastruct // We do not need the keepalived manifests to be generated because the cloud default Load Balancers are in use. // So, setting the lbType to `UserManaged` although the default cloud LBs are not user managed. lbType = configv1.LoadBalancerTypeUserManaged - manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.GCPPlatformType)), lbType) + manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.GCPPlatformType)), lbType, "") } } if infra.Status.PlatformStatus.AWS != nil { @@ -214,7 +230,7 @@ func appendManifestsByPlatform(manifests []manifest, infra *configv1.Infrastruct // We do not need the keepalived manifests to be generated because the cloud default Load Balancers are in use. // So, setting the lbType to `UserManaged` although the default cloud LBs are not user managed. lbType = configv1.LoadBalancerTypeUserManaged - manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.AWSPlatformType)), lbType) + manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.AWSPlatformType)), lbType, "") } } if infra.Status.PlatformStatus.Azure != nil { @@ -223,14 +239,14 @@ func appendManifestsByPlatform(manifests []manifest, infra *configv1.Infrastruct // We do not need the keepalived manifests to be generated because the cloud default Load Balancers are in use. // So, setting the lbType to `UserManaged` although the default cloud LBs are not user managed. lbType = configv1.LoadBalancerTypeUserManaged - manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.AzurePlatformType)), lbType) + manifests = getPlatformManifests(manifests, strings.ToLower(string(configv1.AzurePlatformType)), lbType, "") } } return manifests } -func getPlatformManifests(manifests []manifest, platformName string, lbType configv1.PlatformLoadBalancerType) []manifest { +func getPlatformManifests(manifests []manifest, platformName string, lbType configv1.PlatformLoadBalancerType, vipManagement configv1.VIPManagementType) []manifest { var corednsName string var corefileName string switch platformName { @@ -254,16 +270,27 @@ func getPlatformManifests(manifests []manifest, platformName string, lbType conf ) if lbType == configv1.LoadBalancerTypeOpenShiftManagedDefault || lbType == "" { - platformManifests = append(platformManifests, - manifest{ - name: "manifests/on-prem/keepalived.yaml", - filename: platformName + "/manifests/keepalived.yaml", - }, - manifest{ - name: "manifests/on-prem/keepalived.conf.tmpl", - filename: platformName + "/static-pod-resources/keepalived/keepalived.conf.tmpl", - }, - ) + if vipManagement == "BGP" { + platformManifests = append(platformManifests, + manifest{name: "manifests/on-prem/0000-frr-k8s.yaml", filename: platformName + "/manifests/0000-frr-k8s.yaml"}, + manifest{name: "manifests/on-prem/frr.conf.tmpl", filename: platformName + "/static-pod-resources/frr-k8s/frr.conf.tmpl"}, + manifest{name: "manifests/on-prem/frr-peers.json.tmpl", filename: platformName + "/static-pod-resources/frr-k8s/frr-peers.json"}, + manifest{name: "manifests/on-prem/frr-startup-daemons", filename: platformName + "/static-pod-resources/frr-k8s/startup/daemons"}, + manifest{name: "manifests/on-prem/frr-startup-vtysh.conf", filename: platformName + "/static-pod-resources/frr-k8s/startup/vtysh.conf"}, + manifest{name: "manifests/on-prem/0010-kube-vip-api.yaml", filename: platformName + "/manifests/0010-kube-vip-api.yaml"}, + ) + } else { + platformManifests = append(platformManifests, + manifest{ + name: "manifests/on-prem/keepalived.yaml", + filename: platformName + "/manifests/keepalived.yaml", + }, + manifest{ + name: "manifests/on-prem/keepalived.conf.tmpl", + filename: platformName + "/static-pod-resources/keepalived/keepalived.conf.tmpl", + }, + ) + } } return append(manifests, platformManifests...) diff --git a/pkg/operator/bootstrap_dependencies.go b/pkg/operator/bootstrap_dependencies.go index a9ae876214..e6a86de13c 100644 --- a/pkg/operator/bootstrap_dependencies.go +++ b/pkg/operator/bootstrap_dependencies.go @@ -25,6 +25,7 @@ type BootstrapDependenciesFiles struct { MCSCAFile string KubeAPIServerServingCA string PullSecret string + BGPVIPConfig string } // Validate ensures all required files are set and all provided file paths exist. @@ -45,6 +46,7 @@ func (b BootstrapDependenciesFiles) Validate() error { {"CloudProviderCA", b.CloudProviderCA, false}, {"AdditionalTrustBundle", b.AdditionalTrustBundle, false}, {"CloudConfig", b.CloudConfig, false}, + {"BGPVIPConfig", b.BGPVIPConfig, false}, } for _, file := range files { if err := b.validateFile(file.name, file.path, file.required); err != nil { @@ -87,6 +89,7 @@ type BootstrapDependencies struct { KubeAPIServerServingCA string MCSCA string AdditionalTrustBundle string + BGPVIPPeersJSON string } // NewBootstrapDependencies creates a new BootstrapDependencies instance by validating and loading @@ -131,6 +134,9 @@ func (b *BootstrapDependencies) fillDependencies() error { if err := b.fillClusterConfig(); err != nil { return err } + if err := b.fillBGPVIPConfig(b.Files.BGPVIPConfig); err != nil { + return err + } if err := b.fillCloudConfigData(); err != nil { return err } @@ -276,3 +282,30 @@ func (b *BootstrapDependencies) fillClusterConfig() error { } return nil } + +// fillBGPVIPConfig loads the optional bgp-vip-config ConfigMap manifest and +// extracts its config.json payload for frr-peers.json rendering. +func (b *BootstrapDependencies) fillBGPVIPConfig(path string) error { + if path == "" { + return nil + } + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil // optional: only present when BGP VIP management is enabled + } + cm, err := readCoreCR[*corev1.ConfigMap](path) + if err != nil { + return fmt.Errorf("failed to read bgp-vip-config ConfigMap: %w", err) + } + // Mirror the day-2 sync: a present ConfigMap without a config.json + // payload is a hard error, not an empty peers file. + raw := cm.Data["config.json"] + if raw == "" { + return fmt.Errorf("bgp-vip-config ConfigMap has no config.json payload") + } + peersJSON, err := compactBGPVIPPeersJSON(raw) + if err != nil { + return err + } + b.BGPVIPPeersJSON = peersJSON + return nil +} diff --git a/pkg/operator/bootstrap_test.go b/pkg/operator/bootstrap_test.go new file mode 100644 index 0000000000..5f00bc9831 --- /dev/null +++ b/pkg/operator/bootstrap_test.go @@ -0,0 +1,238 @@ +package operator + +import ( + "os" + "path/filepath" + "testing" + + configv1 "github.com/openshift/api/config/v1" + ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetPlatformManifests(t *testing.T) { + cases := []struct { + name string + platformName string + lbType configv1.PlatformLoadBalancerType + vipManagement configv1.VIPManagementType + expectKeepalived bool + expectFRRK8s bool + expectCoredns bool + expectKubeVIPAPI bool + }{ + { + name: "baremetal default LB, no BGP", + platformName: "baremetal", + lbType: configv1.LoadBalancerTypeOpenShiftManagedDefault, + vipManagement: "", + expectKeepalived: true, + expectFRRK8s: false, + expectCoredns: true, + expectKubeVIPAPI: false, + }, + { + name: "baremetal default LB, BGP enabled", + platformName: "baremetal", + lbType: configv1.LoadBalancerTypeOpenShiftManagedDefault, + vipManagement: "BGP", + expectKeepalived: false, + expectFRRK8s: true, + expectCoredns: true, + expectKubeVIPAPI: true, + }, + { + name: "baremetal user-managed LB, no BGP", + platformName: "baremetal", + lbType: configv1.LoadBalancerTypeUserManaged, + vipManagement: "", + expectKeepalived: false, + expectFRRK8s: false, + expectCoredns: true, + expectKubeVIPAPI: false, + }, + { + name: "baremetal user-managed LB, BGP enabled", + platformName: "baremetal", + lbType: configv1.LoadBalancerTypeUserManaged, + vipManagement: "BGP", + expectKeepalived: false, + expectFRRK8s: false, + expectCoredns: true, + expectKubeVIPAPI: false, + }, + { + name: "baremetal empty LB type, no BGP", + platformName: "baremetal", + lbType: "", + vipManagement: "", + expectKeepalived: true, + expectFRRK8s: false, + expectCoredns: true, + expectKubeVIPAPI: false, + }, + { + name: "baremetal empty LB type, BGP enabled", + platformName: "baremetal", + lbType: "", + vipManagement: "BGP", + expectKeepalived: false, + expectFRRK8s: true, + expectCoredns: true, + expectKubeVIPAPI: true, + }, + { + name: "openstack default LB, no BGP", + platformName: "openstack", + lbType: configv1.LoadBalancerTypeOpenShiftManagedDefault, + vipManagement: "", + expectKeepalived: true, + expectFRRK8s: false, + expectCoredns: true, + expectKubeVIPAPI: false, + }, + { + name: "gcp cloud platform", + platformName: "gcp", + lbType: configv1.LoadBalancerTypeUserManaged, + vipManagement: "", + expectKeepalived: false, + expectFRRK8s: false, + expectCoredns: true, + expectKubeVIPAPI: false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + result := getPlatformManifests(nil, c.platformName, c.lbType, c.vipManagement) + + hasKeepalived := false + hasFRRK8s := false + hasCoredns := false + hasKubeVIPAPI := false + for _, m := range result { + if m.name == "manifests/on-prem/keepalived.yaml" { + hasKeepalived = true + } + if m.name == "manifests/on-prem/0000-frr-k8s.yaml" { + hasFRRK8s = true + } + if m.name == "manifests/on-prem/coredns.yaml" || m.name == "manifests/cloud-platform-alt-dns/coredns.yaml" { + hasCoredns = true + } + if m.name == "manifests/on-prem/0010-kube-vip-api.yaml" { + hasKubeVIPAPI = true + } + } + + assert.Equal(t, c.expectKeepalived, hasKeepalived, "keepalived manifest presence") + assert.Equal(t, c.expectFRRK8s, hasFRRK8s, "frr-k8s manifest presence") + assert.Equal(t, c.expectCoredns, hasCoredns, "coredns manifest presence") + assert.Equal(t, c.expectKubeVIPAPI, hasKubeVIPAPI, "kube-vip-api manifest presence") + }) + } +} + +func TestFillBGPVIPConfig(t *testing.T) { + dir := t.TempDir() + peersJSON := `{"localASN":64512,"defaultPeers":[{"peerAddress":"192.168.111.1","peerASN":64513}],"apiVIPs":["192.168.111.5"],"ingressVIPs":["192.168.111.4"]}` + cmYAML := `apiVersion: v1 +kind: ConfigMap +metadata: + name: bgp-vip-config + namespace: openshift-network-operator +data: + config.json: '` + peersJSON + `' +` + path := filepath.Join(dir, "bgp-vip-config.yaml") + require.NoError(t, os.WriteFile(path, []byte(cmYAML), 0o644)) + + deps := &BootstrapDependencies{} + require.NoError(t, deps.fillBGPVIPConfig(path), "fillBGPVIPConfig") + assert.Equal(t, peersJSON, deps.BGPVIPPeersJSON, "BGPVIPPeersJSON must be the compacted config.json payload") + + // Missing file is tolerated (optional dependency). + deps2 := &BootstrapDependencies{} + require.NoError(t, deps2.fillBGPVIPConfig(filepath.Join(dir, "nonexistent.yaml")), "missing file must not error") + assert.Empty(t, deps2.BGPVIPPeersJSON, "expected empty for missing file") + + // Malformed config.json payload must be rejected. + badCMYAML := `apiVersion: v1 +kind: ConfigMap +metadata: + name: bgp-vip-config + namespace: openshift-network-operator +data: + config.json: '{not json' +` + badPath := filepath.Join(dir, "bgp-vip-config-bad.yaml") + require.NoError(t, os.WriteFile(badPath, []byte(badCMYAML), 0o644)) + + deps3 := &BootstrapDependencies{} + assert.Error(t, deps3.fillBGPVIPConfig(badPath), "malformed config.json must error") + assert.Empty(t, deps3.BGPVIPPeersJSON, "expected empty for malformed config.json") + + // A present ConfigMap without a config.json payload must be rejected + // (mirrors the day-2 sync behavior). + emptyCMYAML := `apiVersion: v1 +kind: ConfigMap +metadata: + name: bgp-vip-config + namespace: openshift-network-operator +data: {} +` + emptyPath := filepath.Join(dir, "bgp-vip-config-empty.yaml") + require.NoError(t, os.WriteFile(emptyPath, []byte(emptyCMYAML), 0o644)) + + deps4 := &BootstrapDependencies{} + assert.Error(t, deps4.fillBGPVIPConfig(emptyPath), "empty config.json payload must error") + assert.Empty(t, deps4.BGPVIPPeersJSON, "expected empty for empty config.json") +} + +// TestBuildSpecBGPImageValidation locks the bootstrap hard-fail: rendering a +// BGP VIP management cluster without the frr-k8s/kube-vip images must error +// at render time rather than emit static pod manifests with empty images. +func TestBuildSpecBGPImageValidation(t *testing.T) { + bgpInfra := &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{ + PlatformStatus: &configv1.PlatformStatus{ + Type: configv1.BareMetalPlatformType, + BareMetal: &configv1.BareMetalPlatformStatus{ + VIPManagement: configv1.VIPManagementTypeBGP, + APIServerInternalIPs: []string{"192.168.111.5"}, + IngressIPs: []string{"192.168.111.4"}, + }, + }, + EtcdDiscoveryDomain: "tt.testing", + }, + } + deps := func(infra *configv1.Infrastructure) *BootstrapDependencies { + return &BootstrapDependencies{ + Infrastructure: infra, + Network: &configv1.Network{ + Spec: configv1.NetworkSpec{ServiceNetwork: []string{"192.168.1.0/24"}}}, + DNS: &configv1.DNS{ + Spec: configv1.DNSSpec{BaseDomain: "tt.testing"}}, + } + } + imgs := &ctrlcommon.Images{} + + if _, err := buildSpec(deps(bgpInfra), imgs, ""); err == nil { + t.Fatal("expected an error for BGP VIP management without frr-k8s/kube-vip images") + } + + imgs.FRRK8s = "frr-k8s-image" + imgs.KubeVip = "kube-vip-image" + if _, err := buildSpec(deps(bgpInfra), imgs, ""); err != nil { + t.Fatalf("unexpected error with images present: %v", err) + } + + // VIP-less BGP falls back to keepalived; missing images must not fail. + noVIPs := bgpInfra.DeepCopy() + noVIPs.Status.PlatformStatus.BareMetal.APIServerInternalIPs = nil + if _, err := buildSpec(deps(noVIPs), &ctrlcommon.Images{}, ""); err != nil { + t.Fatalf("unexpected error for VIP-less BGP without images: %v", err) + } +} diff --git a/pkg/operator/render.go b/pkg/operator/render.go index 5fb2c3aa85..43b72ed13c 100644 --- a/pkg/operator/render.go +++ b/pkg/operator/render.go @@ -281,10 +281,19 @@ func onPremPlatformIngressIP(cfg mcfgv1.ControllerConfigSpec) (interface{}, erro if cfg.Infra.Status.PlatformStatus != nil { switch cfg.Infra.Status.PlatformStatus.Type { case configv1.BareMetalPlatformType: + if len(cfg.Infra.Status.PlatformStatus.BareMetal.IngressIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.BareMetal.IngressIPs[0], nil case configv1.OvirtPlatformType: + if len(cfg.Infra.Status.PlatformStatus.Ovirt.IngressIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.Ovirt.IngressIPs[0], nil case configv1.OpenStackPlatformType: + if len(cfg.Infra.Status.PlatformStatus.OpenStack.IngressIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.OpenStack.IngressIPs[0], nil case configv1.VSpherePlatformType: if len(cfg.Infra.Status.PlatformStatus.VSphere.IngressIPs) > 0 { @@ -292,6 +301,9 @@ func onPremPlatformIngressIP(cfg mcfgv1.ControllerConfigSpec) (interface{}, erro } return nil, nil case configv1.NutanixPlatformType: + if len(cfg.Infra.Status.PlatformStatus.Nutanix.IngressIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.Nutanix.IngressIPs[0], nil default: return nil, fmt.Errorf("invalid platform for Ingress IP") @@ -332,10 +344,19 @@ func onPremPlatformAPIServerInternalIP(cfg mcfgv1.ControllerConfigSpec) (interfa if cfg.Infra.Status.PlatformStatus != nil { switch cfg.Infra.Status.PlatformStatus.Type { case configv1.BareMetalPlatformType: + if len(cfg.Infra.Status.PlatformStatus.BareMetal.APIServerInternalIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.BareMetal.APIServerInternalIPs[0], nil case configv1.OvirtPlatformType: + if len(cfg.Infra.Status.PlatformStatus.Ovirt.APIServerInternalIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.Ovirt.APIServerInternalIPs[0], nil case configv1.OpenStackPlatformType: + if len(cfg.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIPs[0], nil case configv1.VSpherePlatformType: if len(cfg.Infra.Status.PlatformStatus.VSphere.APIServerInternalIPs) > 0 { @@ -343,6 +364,9 @@ func onPremPlatformAPIServerInternalIP(cfg mcfgv1.ControllerConfigSpec) (interfa } return nil, nil case configv1.NutanixPlatformType: + if len(cfg.Infra.Status.PlatformStatus.Nutanix.APIServerInternalIPs) == 0 { + return nil, nil + } return cfg.Infra.Status.PlatformStatus.Nutanix.APIServerInternalIPs[0], nil default: return nil, fmt.Errorf("invalid platform for API Server Internal IP") diff --git a/pkg/operator/render_test.go b/pkg/operator/render_test.go index 4602089634..b5244a3b7c 100644 --- a/pkg/operator/render_test.go +++ b/pkg/operator/render_test.go @@ -120,6 +120,7 @@ func TestRenderAllManifests(t *testing.T) { HTTPSProxy: "https://i.am.a.proxy.server", NoProxy: "*", }, + BGPVIPPeersJSON: `{"localASN":64512,"defaultPeers":[{"peerAddress":"192.168.111.1","peerASN":64513}]}`, Infra: &configv1.Infrastructure{ Status: configv1.InfrastructureStatus{ PlatformStatus: &configv1.PlatformStatus{ diff --git a/pkg/operator/sync.go b/pkg/operator/sync.go index dd8cbe5b18..90898a8a1e 100644 --- a/pkg/operator/sync.go +++ b/pkg/operator/sync.go @@ -374,6 +374,48 @@ func (optr *Operator) syncCloudConfig(spec *mcfgv1.ControllerConfigSpec, infra * return nil } +// compactBGPVIPPeersJSON validates and compacts the bgp-vip-config payload so +// it is safe to embed in single-line template contexts. +func compactBGPVIPPeersJSON(raw string) (string, error) { + if raw == "" { + return "", nil + } + var buf bytes.Buffer + if err := json.Compact(&buf, []byte(raw)); err != nil { + return "", fmt.Errorf("bgp-vip-config config.json is not valid JSON: %w", err) + } + return buf.String(), nil +} + +// syncBGPVIPPeersJSON populates spec.BGPVIPPeersJSON from the +// openshift-network-operator/bgp-vip-config ConfigMap when BGP-based VIP +// management is enabled on a BareMetal platform. +func (optr *Operator) syncBGPVIPPeersJSON(spec *mcfgv1.ControllerConfigSpec, infra *configv1.Infrastructure) error { + if !ctrlcommon.IsBGPVIPManagement(infra) { + return nil + } + cm, err := optr.clusterCmLister.ConfigMaps("openshift-network-operator").Get("bgp-vip-config") + if err != nil { + if apierrors.IsNotFound(err) { + // The spec is rebuilt from scratch on every sync, so silently + // tolerating absence would blank the peers file fleet-wide. + // Degrade instead, holding the last good ControllerConfig. + return fmt.Errorf("BGP VIP management is enabled but the openshift-network-operator/bgp-vip-config ConfigMap is missing") + } + return fmt.Errorf("failed to read bgp-vip-config ConfigMap: %w", err) + } + raw := cm.Data["config.json"] + if raw == "" { + return fmt.Errorf("BGP VIP management is enabled but the bgp-vip-config ConfigMap has no config.json payload") + } + peersJSON, err := compactBGPVIPPeersJSON(raw) + if err != nil { + return err + } + spec.BGPVIPPeersJSON = peersJSON + return nil +} + //nolint:gocyclo func (optr *Operator) syncRenderConfig(_ *renderConfig, _ *configv1.ClusterOperator) error { if optr.inClusterBringup { @@ -598,6 +640,10 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig, _ *configv1.ClusterOpera return err } + if err := optr.syncBGPVIPPeersJSON(spec, infra); err != nil { + return err + } + if !osimagestream.IsFeatureEnabled(optr.fgHandler) { oscontainer, osextensionscontainer, err := optr.getOSImageURLsFromConfigMap() if err != nil { @@ -625,6 +671,8 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig, _ *configv1.ClusterOpera templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg, templatectrl.KubeRbacProxyKey: imgs.KubeRbacProxy, templatectrl.DockerRegistryKey: imgs.DockerRegistry, + templatectrl.FRRK8sKey: imgs.FRRK8s, + templatectrl.KubeVIPKey: imgs.KubeVip, } ignitionHost, err := getIgnitionHost(&infra.Status) diff --git a/pkg/operator/sync_test.go b/pkg/operator/sync_test.go index 7c5ca80260..5dfeaed461 100644 --- a/pkg/operator/sync_test.go +++ b/pkg/operator/sync_test.go @@ -10,6 +10,7 @@ import ( ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" "github.com/openshift/machine-config-operator/test/helpers" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" @@ -179,6 +180,121 @@ func withCABundle(caBundle string) kubeCloudConfigOption { } } +func withBareMetalVIPManagement(vipManagement configv1.VIPManagementType) infraOption { + return func(infra *configv1.Infrastructure) { + if infra.Status.PlatformStatus == nil { + infra.Status.PlatformStatus = &configv1.PlatformStatus{} + } + infra.Status.PlatformStatus.Type = configv1.BareMetalPlatformType + infra.Status.PlatformStatus.BareMetal = &configv1.BareMetalPlatformStatus{ + VIPManagement: vipManagement, + APIServerInternalIPs: []string{"192.168.111.5"}, + IngressIPs: []string{"192.168.111.4"}, + } + } +} + +func buildBGPVIPConfigMap(configJSON string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "openshift-network-operator", + Name: "bgp-vip-config", + }, + Data: map[string]string{ + "config.json": configJSON, + }, + } +} + +func TestSyncBGPVIPPeersJSON(t *testing.T) { + compactJSON := `{"localASN":64512,"defaultPeers":[{"peerAddress":"192.168.111.1","peerASN":64513}]}` + prettyJSON := `{ + "localASN": 64512, + "defaultPeers": [ + { + "peerAddress": "192.168.111.1", + "peerASN": 64513 + } + ] +}` + cases := []struct { + name string + infra *configv1.Infrastructure + configMap *corev1.ConfigMap + expectError bool + expectedBGPVIPPeersJSON string + }{ + { + name: "non-BareMetal platform is a no-op", + infra: buildInfra(withPlatformType(configv1.AWSPlatformType)), + }, + { + name: "BareMetal without BGP VIP management is a no-op", + infra: buildInfra(withPlatformType(configv1.BareMetalPlatformType), withBareMetalVIPManagement("")), + }, + { + // User-managed load balancer: BGP mode without VIPs renders + // keepalived-neither-BGP, and the peers ingestion must not + // degrade the operator over the (rightfully) absent ConfigMap. + name: "BGP without VIPs is a no-op", + infra: buildInfra(withPlatformType(configv1.BareMetalPlatformType), withBareMetalVIPManagement("BGP"), + func(infra *configv1.Infrastructure) { + infra.Status.PlatformStatus.BareMetal.APIServerInternalIPs = nil + }), + }, + { + name: "BGP enabled with ConfigMap present", + infra: buildInfra(withPlatformType(configv1.BareMetalPlatformType), withBareMetalVIPManagement("BGP")), + configMap: buildBGPVIPConfigMap(compactJSON), + expectedBGPVIPPeersJSON: compactJSON, + }, + { + name: "BGP enabled with pretty-printed ConfigMap payload is compacted", + infra: buildInfra(withPlatformType(configv1.BareMetalPlatformType), withBareMetalVIPManagement("BGP")), + configMap: buildBGPVIPConfigMap(prettyJSON), + expectedBGPVIPPeersJSON: compactJSON, + }, + { + name: "BGP enabled with missing ConfigMap degrades", + infra: buildInfra(withPlatformType(configv1.BareMetalPlatformType), withBareMetalVIPManagement("BGP")), + expectError: true, + }, + { + name: "BGP enabled with malformed payload degrades", + infra: buildInfra(withPlatformType(configv1.BareMetalPlatformType), withBareMetalVIPManagement("BGP")), + configMap: buildBGPVIPConfigMap("{not json"), + expectError: true, + }, + { + name: "BGP enabled with empty config.json payload degrades", + infra: buildInfra(withPlatformType(configv1.BareMetalPlatformType), withBareMetalVIPManagement("BGP")), + configMap: buildBGPVIPConfigMap(""), + expectError: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + sharedInformer := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 0) + cmInformer := sharedInformer.Core().V1().ConfigMaps() + if tc.configMap != nil { + require.NoError(t, cmInformer.Informer().GetIndexer().Add(tc.configMap)) + } + optr := &Operator{ + clusterCmLister: cmInformer.Lister(), + } + spec := &mcfgv1.ControllerConfigSpec{} + err := optr.syncBGPVIPPeersJSON(spec, tc.infra) + if tc.expectError { + assert.Error(t, err) + assert.Empty(t, spec.BGPVIPPeersJSON) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expectedBGPVIPPeersJSON, spec.BGPVIPPeersJSON) + }) + } +} + func TestMachineOSBuilderSecretReconciliation(t *testing.T) { masterPool := helpers.NewMachineConfigPool("master", nil, helpers.MasterSelector, "v0") workerPool := helpers.NewMachineConfigPool("worker", nil, helpers.MasterSelector, "v0") diff --git a/templates/common/on-prem/files/0020-kube-vip-ingress.yaml b/templates/common/on-prem/files/0020-kube-vip-ingress.yaml new file mode 100644 index 0000000000..99abe8a755 --- /dev/null +++ b/templates/common/on-prem/files/0020-kube-vip-ingress.yaml @@ -0,0 +1,74 @@ +mode: 0644 +path: {{ if isBGPVIPManagement . }}"/etc/kubernetes/manifests/0020-kube-vip-ingress.yaml"{{ else }}"/etc/kubernetes/disabled-manifests/0020-kube-vip-ingress.yaml"{{ end }} +contents: + inline: | + --- + kind: Pod + apiVersion: v1 + metadata: + name: kube-vip-ingress + namespace: openshift-kube-vip + labels: + app: kube-vip + component: ingress + spec: + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - operator: Exists + containers: + - name: kube-vip + image: {{.Images.kubeVipImage}} + args: + - manager + env: + - name: address + value: "{{ onPremPlatformIngressIP . }}" + - name: k8s_config_file + value: "/etc/kubernetes/kubeconfig" + - name: kubernetes_addr + value: "https://localhost:6443" + - name: port + value: "443" + - name: cp_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_routingtable + value: "true" + - name: vip_cleanroutingtable + value: "true" + - name: vip_routingtableid + value: "198" + - name: vip_routingtableprotocol + value: "248" + - name: backend_health_check_interval + value: "5" + - name: control_plane_health_check_address + value: "http://localhost:1936/healthz" + - name: control_plane_health_check_timeout_seconds + value: "3" + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - name: kubeconfig + mountPath: /etc/kubernetes + readOnly: true + volumes: + - name: kubeconfig + hostPath: + path: /etc/kubernetes diff --git a/templates/common/on-prem/files/keepalived.yaml b/templates/common/on-prem/files/keepalived.yaml index 0605b2eb30..94981e1d82 100644 --- a/templates/common/on-prem/files/keepalived.yaml +++ b/templates/common/on-prem/files/keepalived.yaml @@ -1,5 +1,5 @@ mode: 0644 -path: {{ if isOpenShiftManagedDefaultLB . }} "/etc/kubernetes/manifests/keepalived.yaml" {{ else }} "/etc/kubernetes/disabled-manifests/keepalived.yaml" {{ end }} +path: {{ if and (isOpenShiftManagedDefaultLB .) (not (isBGPVIPManagement .)) }} "/etc/kubernetes/manifests/keepalived.yaml" {{ else }} "/etc/kubernetes/disabled-manifests/keepalived.yaml" {{ end }} contents: inline: | kind: Pod diff --git a/templates/master/00-master/on-prem/files/0000-frr-k8s.yaml b/templates/master/00-master/on-prem/files/0000-frr-k8s.yaml new file mode 100644 index 0000000000..1204af78b6 --- /dev/null +++ b/templates/master/00-master/on-prem/files/0000-frr-k8s.yaml @@ -0,0 +1,340 @@ +mode: 0644 +path: {{ if isBGPVIPManagement . }}"/etc/kubernetes/manifests/0000-frr-k8s.yaml"{{ else }}"/etc/kubernetes/disabled-manifests/0000-frr-k8s.yaml"{{ end }} +contents: + inline: | + --- + kind: Pod + apiVersion: v1 + metadata: + name: frr-k8s + namespace: openshift-frr-k8s + labels: + app: frr-k8s + spec: + # hostNetwork: the BGP session must originate from the node IP and the + # advertised VIP routes live in the host routing table (keepalived parity). + hostNetwork: true + shareProcessNamespace: true + priorityClassName: system-node-critical + # 10s (not the upstream DaemonSet's 0): a SIGTERM'd bgpd sends a Cease + # NOTIFICATION, which hard-resets the session; after a SIGKILL a peer + # in graceful-restart helper mode may retain stale VIP routes on bare + # TCP loss until the restart timer expires. + terminationGracePeriodSeconds: 10 + tolerations: + - operator: Exists + initContainers: + - name: render-config-frr + image: {{.Images.baremetalRuntimeCfgImage}} + command: + - /usr/bin/runtimecfg + - render + - /etc/kubernetes/kubeconfig + - --api-vips + - {{ onPremPlatformAPIServerInternalIP . }} + - --ingress-vips + - {{ onPremPlatformIngressIP . }} + - /config/frr.conf.tmpl + - --out-dir + - /etc/frr + - --peer-file + - /config/frr-peers.json + env: + - name: IS_BOOTSTRAP + value: "no" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 10m + memory: 50Mi + limits: + cpu: 100m + memory: 128Mi + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: resource-dir + mountPath: /config + readOnly: true + - name: frr-conf + mountPath: /etc/frr + - name: kubeconfig + mountPath: /etc/kubernetes + readOnly: true + - name: cp-frr-files + image: {{.Images.frrK8sImage}} + command: + - /bin/sh + - -c + - | + set -eu + cp -f /tmp/frr/daemons /etc/frr/daemons + cp -f /tmp/frr/vtysh.conf /etc/frr/vtysh.conf + securityContext: + runAsUser: 100 + runAsGroup: 101 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 10m + memory: 20Mi + limits: + cpu: 100m + memory: 128Mi + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: frr-startup + mountPath: /tmp/frr + readOnly: true + - name: frr-conf + mountPath: /etc/frr + - name: cp-reloader + image: {{.Images.frrK8sImage}} + command: + - /bin/sh + - -c + - cp -f /frr-reloader.sh /etc/frr_reloader/ + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 10m + memory: 20Mi + limits: + cpu: 100m + memory: 128Mi + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: reloader + mountPath: /etc/frr_reloader + - name: cp-frr-status + image: {{.Images.frrK8sImage}} + command: + - /bin/sh + - -c + - cp -f /frr-status /etc/frr_status/ + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 10m + memory: 20Mi + limits: + cpu: 100m + memory: 128Mi + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: frr-status + mountPath: /etc/frr_status + containers: + - name: controller + image: {{.Images.frrK8sImage}} + command: + - /frr-k8s + args: + - --node-name=$(NODE_NAME) + - --namespace=openshift-frr-k8s + env: + - name: FRR_CONFIG_FILE + value: /etc/frr_reloader/frr.conf + - name: FRR_RELOADER_PID_FILE + value: /etc/frr_reloader/reloader.pid + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KUBECONFIG + value: /etc/kubernetes/kubeconfig + resources: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + add: + - NET_RAW + livenessProbe: + httpGet: + path: /healthz + port: 7572 + host: 127.0.0.1 + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /readyz + port: 7572 + host: 127.0.0.1 + initialDelaySeconds: 10 + periodSeconds: 10 + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: reloader + mountPath: /etc/frr_reloader + - name: kubeconfig + mountPath: /etc/kubernetes + readOnly: true + - name: frr + image: {{.Images.frrK8sImage}} + command: + - /bin/sh + - -c + - /sbin/tini -- /usr/lib/frr/docker-start + env: + - name: TINI_SUBREAPER + value: "true" + resources: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + # Mirrors the frr container of the frr-k8s DaemonSet shipped by + # CNO (upstream metallb/frr-k8s parity). No drop ALL: FRR's + # privilege-separated startup (watchfrr as root spawning daemons + # as the frr user) additionally relies on root's implicit + # capabilities (SETUID/SETGID/DAC_OVERRIDE/CHOWN); dropping ALL + # and re-adding only the network capabilities was tested and + # prevents the FRR daemons from starting. SYS_ADMIN is required + # by docker-start/watchfrr process supervision. + add: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + - NET_BIND_SERVICE + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: frr-sockets + mountPath: /var/run/frr + - name: frr-conf + mountPath: /etc/frr + - name: frr-lib + mountPath: /var/lib/frr + - name: frr-tmp + mountPath: /var/tmp/frr + - name: reloader + image: {{.Images.frrK8sImage}} + command: + - /etc/frr_reloader/frr-reloader.sh + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + # vtysh as root against the frr-user-owned vty sockets. + add: + - DAC_OVERRIDE + resources: + requests: + cpu: 10m + memory: 20Mi + limits: + cpu: 500m + memory: 128Mi + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: frr-sockets + mountPath: /var/run/frr + - name: frr-conf + mountPath: /etc/frr + - name: reloader + mountPath: /etc/frr_reloader + - name: frr-status + image: {{.Images.frrK8sImage}} + command: + - /etc/frr_status/frr-status + args: + - --node-name=$(NODE_NAME) + - --namespace=openshift-frr-k8s + - --pod-name=frr-k8s-$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KUBECONFIG + value: /etc/kubernetes/kubeconfig + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + # vtysh as root against the frr-user-owned vty sockets. + add: + - DAC_OVERRIDE + resources: + requests: + cpu: 10m + memory: 20Mi + limits: + cpu: 500m + memory: 128Mi + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: frr-sockets + mountPath: /var/run/frr + - name: frr-conf + mountPath: /etc/frr + - name: frr-status + mountPath: /etc/frr_status + - name: kubeconfig + mountPath: /etc/kubernetes + readOnly: true + volumes: + - name: resource-dir + hostPath: + path: /etc/kubernetes/static-pod-resources/frr-k8s + # hostPath (not emptyDir) so the CNO-managed metrics companion + # DaemonSet can reach the FRR sockets; perms via tmpfiles.d. + - name: frr-conf + hostPath: + path: /run/frr-k8s/conf + type: DirectoryOrCreate + - name: frr-sockets + hostPath: + path: /run/frr-k8s/sockets + type: DirectoryOrCreate + - name: reloader + emptyDir: {} + - name: frr-status + emptyDir: {} + - name: frr-lib + emptyDir: {} + - name: frr-tmp + emptyDir: {} + - name: frr-startup + hostPath: + path: /etc/kubernetes/static-pod-resources/frr-k8s/startup + - name: kubeconfig + hostPath: + path: /etc/kubernetes diff --git a/templates/master/00-master/on-prem/files/0010-kube-vip-api.yaml b/templates/master/00-master/on-prem/files/0010-kube-vip-api.yaml new file mode 100644 index 0000000000..a1bbf9454a --- /dev/null +++ b/templates/master/00-master/on-prem/files/0010-kube-vip-api.yaml @@ -0,0 +1,70 @@ +mode: 0644 +path: {{ if isBGPVIPManagement . }}"/etc/kubernetes/manifests/0010-kube-vip-api.yaml"{{ else }}"/etc/kubernetes/disabled-manifests/0010-kube-vip-api.yaml"{{ end }} +contents: + inline: | + --- + kind: Pod + apiVersion: v1 + metadata: + name: kube-vip-api + namespace: openshift-kube-vip + labels: + app: kube-vip + component: api + spec: + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - operator: Exists + containers: + - name: kube-vip + image: {{.Images.kubeVipImage}} + args: + - manager + env: + - name: address + value: "{{ onPremPlatformAPIServerInternalIP . }}" + - name: k8s_config_file + value: "/etc/kubernetes/kubeconfig" + - name: kubernetes_addr + value: "https://localhost:6443" + - name: port + value: "6443" + - name: cp_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_routingtable + value: "true" + - name: vip_cleanroutingtable + value: "true" + - name: vip_routingtableid + value: "198" + - name: vip_routingtableprotocol + value: "248" + - name: backend_health_check_interval + value: "5" + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - name: kubeconfig + mountPath: /etc/kubernetes + readOnly: true + volumes: + - name: kubeconfig + hostPath: + path: /etc/kubernetes diff --git a/templates/master/00-master/on-prem/files/frr-k8s-conf.yaml b/templates/master/00-master/on-prem/files/frr-k8s-conf.yaml new file mode 100644 index 0000000000..e6181426dd --- /dev/null +++ b/templates/master/00-master/on-prem/files/frr-k8s-conf.yaml @@ -0,0 +1,62 @@ +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/frr-k8s/frr.conf.tmpl" +contents: + inline: | + frr version 9.1 + frr defaults traditional + hostname {{`{{.Hostname}}`}} + ! + router bgp {{`{{.LocalASN}}`}} + bgp router-id {{`{{.RouterID}}`}} + no bgp ebgp-requires-policy + no bgp default ipv4-unicast + ! + {{`{{- range .Peers}}`}} + neighbor {{`{{.PeerAddress}}`}} remote-as {{`{{.PeerASN}}`}} + {{`{{- if .Password}}`}} + neighbor {{`{{.PeerAddress}}`}} password {{`{{.Password}}`}} + {{`{{- end}}`}} + {{`{{- if eq .BFDEnabled "true"}}`}} + neighbor {{`{{.PeerAddress}}`}} bfd + {{`{{- end}}`}} + {{`{{- if eq .EBGPMultiHop "true"}}`}} + neighbor {{`{{.PeerAddress}}`}} ebgp-multihop + {{`{{- end}}`}} + {{`{{- if .HoldTime}}`}} + neighbor {{`{{.PeerAddress}}`}} timers {{`{{.KeepaliveTime}}`}} {{`{{.HoldTime}}`}} + {{`{{- end}}`}} + {{`{{- end}}`}} + ! + address-family ipv4 unicast + redistribute table-direct 198 + {{`{{- range .Peers}}`}} + {{`{{- if isIPv4 .PeerAddress}}`}} + neighbor {{`{{.PeerAddress}}`}} activate + {{`{{- if $.Communities}}`}} + neighbor {{`{{.PeerAddress}}`}} route-map VIP-COMMUNITY out + {{`{{- end}}`}} + {{`{{- end}}`}} + {{`{{- end}}`}} + exit-address-family + ! + address-family ipv6 unicast + redistribute table-direct 198 + {{`{{- range .Peers}}`}} + {{`{{- if isIPv6 .PeerAddress}}`}} + neighbor {{`{{.PeerAddress}}`}} activate + {{`{{- if $.Communities}}`}} + neighbor {{`{{.PeerAddress}}`}} route-map VIP-COMMUNITY out + {{`{{- end}}`}} + {{`{{- end}}`}} + {{`{{- end}}`}} + exit-address-family + ! + {{`{{- if .Communities}}`}} + route-map VIP-COMMUNITY permit 10 + {{`{{- range .Communities}}`}} + set community {{`{{.}}`}} additive + {{`{{- end}}`}} + ! + {{`{{- end}}`}} + line vty + ! diff --git a/templates/master/00-master/on-prem/files/frr-k8s-peers.yaml b/templates/master/00-master/on-prem/files/frr-k8s-peers.yaml new file mode 100644 index 0000000000..e4284d9f57 --- /dev/null +++ b/templates/master/00-master/on-prem/files/frr-k8s-peers.yaml @@ -0,0 +1,5 @@ +mode: 0600 +path: "/etc/kubernetes/static-pod-resources/frr-k8s/frr-peers.json" +contents: + inline: |- + {{.BGPVIPPeersJSON}} diff --git a/templates/master/00-master/on-prem/files/frr-k8s-startup-daemons.yaml b/templates/master/00-master/on-prem/files/frr-k8s-startup-daemons.yaml new file mode 100644 index 0000000000..633516007d --- /dev/null +++ b/templates/master/00-master/on-prem/files/frr-k8s-startup-daemons.yaml @@ -0,0 +1,27 @@ +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/frr-k8s/startup/daemons" +contents: + inline: | + zebra=yes + bgpd=yes + ospfd=no + ospf6d=no + ripd=no + ripngd=no + isisd=no + pimd=no + ldpd=no + nhrpd=no + eigrpd=no + babeld=no + sharpd=no + pbrd=no + bfdd=yes + fabricd=no + vrrpd=no + pathd=no + + vtysh_enable=yes + zebra_options=" -A 127.0.0.1 -s 90000000" + bgpd_options=" -A 127.0.0.1" + bfdd_options=" -A 127.0.0.1" diff --git a/templates/master/00-master/on-prem/files/frr-k8s-startup-vtysh.yaml b/templates/master/00-master/on-prem/files/frr-k8s-startup-vtysh.yaml new file mode 100644 index 0000000000..7cc769bd2b --- /dev/null +++ b/templates/master/00-master/on-prem/files/frr-k8s-startup-vtysh.yaml @@ -0,0 +1,5 @@ +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/frr-k8s/startup/vtysh.conf" +contents: + inline: | + service integrated-vtysh-config diff --git a/templates/master/00-master/on-prem/units/frr-k8s-hostpath.service.yaml b/templates/master/00-master/on-prem/units/frr-k8s-hostpath.service.yaml new file mode 100644 index 0000000000..3ee97217ae --- /dev/null +++ b/templates/master/00-master/on-prem/units/frr-k8s-hostpath.service.yaml @@ -0,0 +1,21 @@ +name: frr-k8s-hostpath.service +enabled: {{if isBGPVIPManagement .}}true{{else}}false{{end}} +contents: | + [Unit] + Description=Prepare the frr-k8s static pod hostPath directories + # /run/frr-k8s is shared between the frr-k8s static pod and the + # CNO-managed metrics companion DaemonSet. 0777 mirrors emptyDir + # semantics: FRR daemons run as uid 100 while watchfrr runs as root + # without CAP_DAC_OVERRIDE. container_file_t because the companion runs + # as plain container_t (MCS-confined): the static pod itself is spc_t + # (host network) and unconstrained, but its files inherit the directory + # type and must stay accessible to the companion. + Before=kubelet-dependencies.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/bin/bash -c 'mkdir -p /run/frr-k8s/sockets /run/frr-k8s/conf && chmod 0777 /run/frr-k8s/sockets /run/frr-k8s/conf && chcon -R -t container_file_t /run/frr-k8s' + + [Install] + WantedBy=kubelet-dependencies.target