Implement Unicast VRRP for CPLB

Signed-off-by: Juan-Luis de Sousa-Valadas Castaño <jvaladas@mirantis.com>

docs/configuration.md

@@ -334,6 +334,8 @@ Configuration options required for using VRRP to configure VIPs in control plane
 | `virtualRouterID`       | The VRRP router ID. If not specified, k0s will automatically number the IDs for each VRRP instance, starting with 51. It must be in the range of 1-255, all the control plane nodes must use the same `virtualRouterID`. Other clusters in the same network must not use the same `virtualRouterID`. |
 | `advertIntervalSeconds` | Advertisement interval in seconds. Defaults to 1 second.                                                                                                                                                                                                                                             |
 | `authPass`              | The password for accessing VRRPD. This is not a security feature but a way to prevent accidental misconfigurations. It must be in the range of 1-8 characters                                                                                                                                        |
+| `unicastPeers`          | A list of IP addresses to connect using unicast. If this field is specified, `unicastSourceIP` is mandatory, this list must not contain the IP address specified in `unicastSourceIP`.                                                                                                               |
+| `unicastSourceIP`       | The source IP address when using unicast. If `unicastPeers` isn't defined this field is ignored.                                                                                                                                                                                                     |
 
 ##### `spec.network.controlPlaneLoadBalancing.keepalived.virtualServers`
 

docs/cplb.md

@@ -80,6 +80,28 @@ spec:
           authPass: "<my password>"
 ```
 
+By default, VRRP Intances use multicast as per [RFC 3768]. It's possible to configure VRRP
+instances to use unicast:
+
+```yaml
+spec:
+  network:
+    controlPlaneLoadBalancing:
+      enabled: true
+      type: Keepalived
+      keepalived:
+        vrrpInstances:
+        - virtualIPs: ["<VIP address>/<netmask>"] # for instance ["172.16.0.100/16"]
+          authPass: "<my password>"
+          unicastSourceIP: <ip address of this controller>
+          unicastPeers: [<ip address of other controllers>, ...]
+```
+
+When using unicast, k0st does not attempt to detect `unicastSourceIP` and it must be defined explicitly and
+`unicastPeers` must include the IP address of the other controllers' `unicastSourceIP`.
+
+[RFC 3768]: https://datatracker.ietf.org/doc/html/rfc3768#section-5.2.2
+
 ## Load Balancing
 
 Currently k0s allows to chose one of two load balancing mechanism:

docs/networking.md

@@ -57,7 +57,7 @@ One goal of k0s is to allow for the deployment of an isolated control plane, whi
 | TCP      | 10250 | kubelet        | controller, worker => host `*` | Authenticated kubelet API for the controller node `kube-apiserver` (and `heapster`/`metrics-server` addons) using TLS client certs
 | TCP      | 9443  | k0s-api        | controller <-> controller      | k0s controller join API, TLS with token auth
 | TCP      | 8132  | konnectivity   | worker <-> controller          | Konnectivity is used as "reverse" tunnel between kube-apiserver and worker kubelets
-| TCP      | 112   | keepalived     | controller <-> controller      | Only required for control plane load balancing vrrpInstances for ip address 224.0.0.18. 224.0.0.18 is a multicast IP address defined in [RFC 3768].
+| TCP      | 112   | keepalived     | controller <-> controller      | Only required for control plane load balancing VRRPInstances. Unless unicast is explicitly enabled, port 122 works on the ip address 224.0.0.18. 224.0.0.18 is a multicast IP address defined in [RFC 3768].
 
 You also need enable all traffic to and from the [podCIDR and serviceCIDR] subnets on nodes with a worker role.
 

inttest/cplb-ipvs/cplbipvs_test.go

@@ -41,6 +41,8 @@ spec:
         vrrpInstances:
         - virtualIPs: ["%s/16"]
           authPass: "123456"
+          unicastSourceIP: %s
+          unicastPeers: [%s, %s]
         virtualServers:
         - ipAddress: %s
     nodeLocalLoadBalancing:
@@ -57,7 +59,11 @@ func (s *cplbIPVSSuite) TestK0sGetsUp() {
 
 	for idx := range s.BootlooseSuite.ControllerCount {
 		s.Require().NoError(s.WaitForSSH(s.ControllerNode(idx), 2*time.Minute, 1*time.Second))
-		s.PutFile(s.ControllerNode(idx), "/tmp/k0s.yaml", fmt.Sprintf(haControllerConfig, lb, lb))
+		addr := s.getUnicastAddresses(idx)
+		s.T().Log("AAAAA")
+		s.T().Log(addr)
+		s.PutFile(s.ControllerNode(idx), "/tmp/k0s.yaml",
+			fmt.Sprintf(haControllerConfig, lb, addr[0], addr[1], addr[2], lb))
 
 		// Note that the token is intentionally empty for the first controller
 		s.Require().NoError(s.InitController(idx, "--config=/tmp/k0s.yaml", "--disable-components=metrics-server", joinToken))
@@ -128,6 +134,16 @@ func (s *cplbIPVSSuite) getLBAddress() string {
 	return fmt.Sprintf("%s.%d", strings.Join(parts[:3], "."), lastOctet)
 }
 
+// getUnicastAddreses returns the IP addresses of the controllers. The first IP
+// is the address of the controller with the ID provided.
+func (s *cplbIPVSSuite) getUnicastAddresses(i int) []string {
+	return []string{
+		s.GetIPAddress(s.ControllerNode(i % s.BootlooseSuite.ControllerCount)),
+		s.GetIPAddress(s.ControllerNode((i + 1) % s.BootlooseSuite.ControllerCount)),
+		s.GetIPAddress(s.ControllerNode((i + 2) % s.BootlooseSuite.ControllerCount)),
+	}
+}
+
 // validateRealServers checks that the real servers are present in the
 // ipvsadm output.
 func (s *cplbIPVSSuite) validateRealServers(ctx context.Context, node string, vip string) {

pkg/apis/k0s/v1beta1/cplb.go

@@ -114,6 +114,15 @@ type VRRPInstance struct {
 	// +kubebuilder:validation:MinLength=1
 	// +kubebuilder:validation:MaxLength=8
 	AuthPass string `json:"authPass"`
+
+	// UnicastPeers is a list of unicast peers. If not specified, k0s will use multicast.
+	// If specified, UnicastSourceIP must be specified as well.
+	// +listType=set
+	UnicastPeers []string `json:"unicastPeers,omitempty"`
+
+	// UnicastSourceIP is the source address for unicast peers.
+	// If not specified, k0s will use the first address of the interface.
+	UnicastSourceIP string `json:"unicastSourceIP,omitempty"`
 }
 
 // validateVRRPInstances validates existing configuration and sets the default
@@ -161,6 +170,20 @@ func (k *KeepalivedSpec) validateVRRPInstances(getDefaultNICFn func() (string, e
 				errs = append(errs, fmt.Errorf("VirtualIPs must be a CIDR. Got: %s", vip))
 			}
 		}
+
+		if len(k.VRRPInstances[i].UnicastPeers) > 0 {
+			if net.ParseIP(k.VRRPInstances[i].UnicastSourceIP) == nil {
+				errs = append(errs, fmt.Errorf("UnicastPeers require a valid UnicastSourceIP. Got: %s", k.VRRPInstances[i].UnicastSourceIP))
+			}
+			for _, peer := range k.VRRPInstances[i].UnicastPeers {
+				if net.ParseIP(peer) == nil {
+					errs = append(errs, fmt.Errorf("UnicastPeers require valid IP addresses. Got: %s", peer))
+				}
+				if peer == k.VRRPInstances[i].UnicastSourceIP {
+					errs = append(errs, fmt.Errorf("UnicastPeers must not contain the UnicastSourceIP. Got: %s", peer))
+				}
+			}
+		}
 	}
 	return errs
 }

pkg/apis/k0s/v1beta1/cplb_test.go

@@ -72,9 +72,11 @@ func (s *CPLBSuite) TestValidateVRRPInstances() {
 				{
 					VirtualRouterID:       1,
 					Interface:             "eth0",
-					VirtualIPs:            []string{"192.168.1.1/24"},
+					VirtualIPs:            []string{"192.168.1.100/24"},
 					AdvertIntervalSeconds: 1,
 					AuthPass:              "123456",
+					UnicastSourceIP:       "192.168.1.1",
+					UnicastPeers:          []string{"192.168.1.2", "192.168.1.3"},
 				},
 			},
 			expectedVRRPs: []VRRPInstance{
@@ -84,6 +86,8 @@ func (s *CPLBSuite) TestValidateVRRPInstances() {
 					VirtualIPs:            []string{"192.168.1.1/24"},
 					AdvertIntervalSeconds: 1,
 					AuthPass:              "123456",
+					UnicastSourceIP:       "192.168.1.1",
+					UnicastPeers:          []string{"192.168.1.2", "192.168.1.3"},
 				},
 			},
 			wantErr: false,
@@ -116,6 +120,60 @@ func (s *CPLBSuite) TestValidateVRRPInstances() {
 				},
 			},
 			wantErr: true,
+		}, {
+			name: "Unicast Peers without unicast source",
+			vrrps: []VRRPInstance{
+				{
+					VirtualRouterID:       1,
+					Interface:             "eth0",
+					VirtualIPs:            []string{"192.168.1.100/24"},
+					AdvertIntervalSeconds: 1,
+					AuthPass:              "123456",
+					UnicastPeers:          []string{"192.168.1.2", "192.168.1.3"},
+				},
+			},
+			wantErr: true,
+		}, {
+			name: "Invalid unicast peers",
+			vrrps: []VRRPInstance{
+				{
+					VirtualRouterID:       1,
+					Interface:             "eth0",
+					VirtualIPs:            []string{"192.168.1.100/24"},
+					AdvertIntervalSeconds: 1,
+					AuthPass:              "123456",
+					UnicastPeers:          []string{"example.com", "192.168.1.3"},
+				},
+			},
+			wantErr: true,
+		}, {
+			name: "Invalid unicast source",
+			vrrps: []VRRPInstance{
+				{
+					VirtualRouterID:       1,
+					Interface:             "eth0",
+					VirtualIPs:            []string{"192.168.1.100/24"},
+					AdvertIntervalSeconds: 1,
+					AuthPass:              "123456",
+					UnicastSourceIP:       "example.com",
+					UnicastPeers:          []string{"192.168.1.2", "192.168.1.3"},
+				},
+			},
+			wantErr: true,
+		}, {
+			name: "Unicast peers includes unicast source",
+			vrrps: []VRRPInstance{
+				{
+					VirtualRouterID:       1,
+					Interface:             "eth0",
+					VirtualIPs:            []string{"192.168.1.100/24"},
+					AdvertIntervalSeconds: 1,
+					AuthPass:              "123456",
+					UnicastSourceIP:       "192.168.1.1",
+					UnicastPeers:          []string{"192.168.1.1", "192.168.1.2", "192.168.1.3"},
+				},
+			},
+			wantErr: true,
 		},
 	}
 

pkg/component/controller/cplb/cplb_linux.go

@@ -485,6 +485,16 @@ vrrp_instance k0s-vip-{{$i}} {
         {{ . }}
         {{ end }}
     }
+    {{ if .UnicastPeers }}
+    unicast_src_ip {{ .UnicastSourceIP }}
+    unicast_peer {
+        {{ range .UnicastPeers }}
+        {{ . }}
+        {{ end }}
+    }
+    {{ else}}
+	 #F
+    {{ end }}
 }
 {{ end }}
 

static/_crds/k0s/k0s.k0sproject.io_clusterconfigs.yaml

@@ -609,6 +609,19 @@ spec:
                                     Interface specifies the NIC used by the virtual router. If not specified,
                                     k0s will use the interface that owns the default route.
                                   type: string
+                                unicastPeers:
+                                  description: |-
+                                    UnicastPeers is a list of unicast peers. If not specified, k0s will use multicast.
+                                    If specified, UnicastSourceIP must be specified as well.
+                                  items:
+                                    type: string
+                                  type: array
+                                  x-kubernetes-list-type: set
+                                unicastSourceIP:
+                                  description: |-
+                                    UnicastSourceIP is the source address for unicast peers.
+                                    If not specified, k0s will use the first address of the interface.
+                                  type: string
                                 virtualIPs:
                                   description: |-
                                     VirtualIPs is the list of virtual IP address used by the VRRP instance.