diff --git a/docs/configuration.md b/docs/configuration.md index 39b7e9247c5b..9438d14a9ae6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -334,6 +334,8 @@ Configuration options required for using VRRP to configure VIPs in control plane | `virtualRouterID` | The VRRP router ID. If not specified, k0s will automatically number the IDs for each VRRP instance, starting with 51. It must be in the range of 1-255, all the control plane nodes must use the same `virtualRouterID`. Other clusters in the same network must not use the same `virtualRouterID`. | | `advertIntervalSeconds` | Advertisement interval in seconds. Defaults to 1 second. | | `authPass` | The password for accessing VRRPD. This is not a security feature but a way to prevent accidental misconfigurations. It must be in the range of 1-8 characters | +| `unicastPeers` | A list of IP addresses to connect using unicast. If this field is specified, `unicastSourceIP` is mandatory, this list must not contain the IP address specified in `unicastSourceIP`. | +| `unicastSourceIP` | The source IP address when using unicast. If `unicastPeers` isn't defined this field is ignored. | ##### `spec.network.controlPlaneLoadBalancing.keepalived.virtualServers` diff --git a/docs/cplb.md b/docs/cplb.md index f510f5c2d986..0ab891453338 100644 --- a/docs/cplb.md +++ b/docs/cplb.md @@ -80,6 +80,28 @@ spec: authPass: "" ``` +By default, VRRP Intances use multicast as per [RFC 3768]. It's possible to configure VRRP +instances to use unicast: + +```yaml +spec: + network: + controlPlaneLoadBalancing: + enabled: true + type: Keepalived + keepalived: + vrrpInstances: + - virtualIPs: ["/"] # for instance ["172.16.0.100/16"] + authPass: "" + unicastSourceIP: + unicastPeers: [, ...] +``` + +When using unicast, k0st does not attempt to detect `unicastSourceIP` and it must be defined explicitly and +`unicastPeers` must include the IP address of the other controllers' `unicastSourceIP`. + +[RFC 3768]: https://datatracker.ietf.org/doc/html/rfc3768#section-5.2.2 + ## Load Balancing Currently k0s allows to chose one of two load balancing mechanism: diff --git a/docs/networking.md b/docs/networking.md index 98a9804b0cce..fe53b1726ed8 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -57,7 +57,7 @@ One goal of k0s is to allow for the deployment of an isolated control plane, whi | TCP | 10250 | kubelet | controller, worker => host `*` | Authenticated kubelet API for the controller node `kube-apiserver` (and `heapster`/`metrics-server` addons) using TLS client certs | TCP | 9443 | k0s-api | controller <-> controller | k0s controller join API, TLS with token auth | TCP | 8132 | konnectivity | worker <-> controller | Konnectivity is used as "reverse" tunnel between kube-apiserver and worker kubelets -| TCP | 112 | keepalived | controller <-> controller | Only required for control plane load balancing vrrpInstances for ip address 224.0.0.18. 224.0.0.18 is a multicast IP address defined in [RFC 3768]. +| TCP | 112 | keepalived | controller <-> controller | Only required for control plane load balancing VRRPInstances. Unless unicast is explicitly enabled, port 122 works on the ip address 224.0.0.18. 224.0.0.18 is a multicast IP address defined in [RFC 3768]. You also need enable all traffic to and from the [podCIDR and serviceCIDR] subnets on nodes with a worker role. diff --git a/inttest/cplb-ipvs/cplbipvs_test.go b/inttest/cplb-ipvs/cplbipvs_test.go index 8a3e18bf140f..1943bacd48f3 100644 --- a/inttest/cplb-ipvs/cplbipvs_test.go +++ b/inttest/cplb-ipvs/cplbipvs_test.go @@ -41,6 +41,8 @@ spec: vrrpInstances: - virtualIPs: ["%s/16"] authPass: "123456" + unicastSourceIP: %s + unicastPeers: [%s, %s] virtualServers: - ipAddress: %s nodeLocalLoadBalancing: @@ -57,7 +59,11 @@ func (s *cplbIPVSSuite) TestK0sGetsUp() { for idx := range s.BootlooseSuite.ControllerCount { s.Require().NoError(s.WaitForSSH(s.ControllerNode(idx), 2*time.Minute, 1*time.Second)) - s.PutFile(s.ControllerNode(idx), "/tmp/k0s.yaml", fmt.Sprintf(haControllerConfig, lb, lb)) + addr := s.getUnicastAddresses(idx) + s.T().Log("AAAAA") + s.T().Log(addr) + s.PutFile(s.ControllerNode(idx), "/tmp/k0s.yaml", + fmt.Sprintf(haControllerConfig, lb, addr[0], addr[1], addr[2], lb)) // Note that the token is intentionally empty for the first controller s.Require().NoError(s.InitController(idx, "--config=/tmp/k0s.yaml", "--disable-components=metrics-server", joinToken)) @@ -128,6 +134,16 @@ func (s *cplbIPVSSuite) getLBAddress() string { return fmt.Sprintf("%s.%d", strings.Join(parts[:3], "."), lastOctet) } +// getUnicastAddreses returns the IP addresses of the controllers. The first IP +// is the address of the controller with the ID provided. +func (s *cplbIPVSSuite) getUnicastAddresses(i int) []string { + return []string{ + s.GetIPAddress(s.ControllerNode(i % s.BootlooseSuite.ControllerCount)), + s.GetIPAddress(s.ControllerNode((i + 1) % s.BootlooseSuite.ControllerCount)), + s.GetIPAddress(s.ControllerNode((i + 2) % s.BootlooseSuite.ControllerCount)), + } +} + // validateRealServers checks that the real servers are present in the // ipvsadm output. func (s *cplbIPVSSuite) validateRealServers(ctx context.Context, node string, vip string) { diff --git a/pkg/apis/k0s/v1beta1/cplb.go b/pkg/apis/k0s/v1beta1/cplb.go index f38bdd504a3e..4cbe55bf6a61 100644 --- a/pkg/apis/k0s/v1beta1/cplb.go +++ b/pkg/apis/k0s/v1beta1/cplb.go @@ -114,6 +114,15 @@ type VRRPInstance struct { // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=8 AuthPass string `json:"authPass"` + + // UnicastPeers is a list of unicast peers. If not specified, k0s will use multicast. + // If specified, UnicastSourceIP must be specified as well. + // +listType=set + UnicastPeers []string `json:"unicastPeers,omitempty"` + + // UnicastSourceIP is the source address for unicast peers. + // If not specified, k0s will use the first address of the interface. + UnicastSourceIP string `json:"unicastSourceIP,omitempty"` } // validateVRRPInstances validates existing configuration and sets the default @@ -161,6 +170,20 @@ func (k *KeepalivedSpec) validateVRRPInstances(getDefaultNICFn func() (string, e errs = append(errs, fmt.Errorf("VirtualIPs must be a CIDR. Got: %s", vip)) } } + + if len(k.VRRPInstances[i].UnicastPeers) > 0 { + if net.ParseIP(k.VRRPInstances[i].UnicastSourceIP) == nil { + errs = append(errs, fmt.Errorf("UnicastPeers require a valid UnicastSourceIP. Got: %s", k.VRRPInstances[i].UnicastSourceIP)) + } + for _, peer := range k.VRRPInstances[i].UnicastPeers { + if net.ParseIP(peer) == nil { + errs = append(errs, fmt.Errorf("UnicastPeers require valid IP addresses. Got: %s", peer)) + } + if peer == k.VRRPInstances[i].UnicastSourceIP { + errs = append(errs, fmt.Errorf("UnicastPeers must not contain the UnicastSourceIP. Got: %s", peer)) + } + } + } } return errs } diff --git a/pkg/apis/k0s/v1beta1/cplb_test.go b/pkg/apis/k0s/v1beta1/cplb_test.go index 5be26b37bc3d..2a997bfa1280 100644 --- a/pkg/apis/k0s/v1beta1/cplb_test.go +++ b/pkg/apis/k0s/v1beta1/cplb_test.go @@ -72,9 +72,11 @@ func (s *CPLBSuite) TestValidateVRRPInstances() { { VirtualRouterID: 1, Interface: "eth0", - VirtualIPs: []string{"192.168.1.1/24"}, + VirtualIPs: []string{"192.168.1.100/24"}, AdvertIntervalSeconds: 1, AuthPass: "123456", + UnicastSourceIP: "192.168.1.1", + UnicastPeers: []string{"192.168.1.2", "192.168.1.3"}, }, }, expectedVRRPs: []VRRPInstance{ @@ -84,6 +86,8 @@ func (s *CPLBSuite) TestValidateVRRPInstances() { VirtualIPs: []string{"192.168.1.1/24"}, AdvertIntervalSeconds: 1, AuthPass: "123456", + UnicastSourceIP: "192.168.1.1", + UnicastPeers: []string{"192.168.1.2", "192.168.1.3"}, }, }, wantErr: false, @@ -116,6 +120,60 @@ func (s *CPLBSuite) TestValidateVRRPInstances() { }, }, wantErr: true, + }, { + name: "Unicast Peers without unicast source", + vrrps: []VRRPInstance{ + { + VirtualRouterID: 1, + Interface: "eth0", + VirtualIPs: []string{"192.168.1.100/24"}, + AdvertIntervalSeconds: 1, + AuthPass: "123456", + UnicastPeers: []string{"192.168.1.2", "192.168.1.3"}, + }, + }, + wantErr: true, + }, { + name: "Invalid unicast peers", + vrrps: []VRRPInstance{ + { + VirtualRouterID: 1, + Interface: "eth0", + VirtualIPs: []string{"192.168.1.100/24"}, + AdvertIntervalSeconds: 1, + AuthPass: "123456", + UnicastPeers: []string{"example.com", "192.168.1.3"}, + }, + }, + wantErr: true, + }, { + name: "Invalid unicast source", + vrrps: []VRRPInstance{ + { + VirtualRouterID: 1, + Interface: "eth0", + VirtualIPs: []string{"192.168.1.100/24"}, + AdvertIntervalSeconds: 1, + AuthPass: "123456", + UnicastSourceIP: "example.com", + UnicastPeers: []string{"192.168.1.2", "192.168.1.3"}, + }, + }, + wantErr: true, + }, { + name: "Unicast peers includes unicast source", + vrrps: []VRRPInstance{ + { + VirtualRouterID: 1, + Interface: "eth0", + VirtualIPs: []string{"192.168.1.100/24"}, + AdvertIntervalSeconds: 1, + AuthPass: "123456", + UnicastSourceIP: "192.168.1.1", + UnicastPeers: []string{"192.168.1.1", "192.168.1.2", "192.168.1.3"}, + }, + }, + wantErr: true, }, } diff --git a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go index 0c45c14c44fc..c5d277ff46d7 100644 --- a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go @@ -1126,6 +1126,11 @@ func (in *VRRPInstance) DeepCopyInto(out *VRRPInstance) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.UnicastPeers != nil { + in, out := &in.UnicastPeers, &out.UnicastPeers + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VRRPInstance. diff --git a/pkg/component/controller/cplb/cplb_linux.go b/pkg/component/controller/cplb/cplb_linux.go index 0496ffa08c48..fd88df64ef58 100644 --- a/pkg/component/controller/cplb/cplb_linux.go +++ b/pkg/component/controller/cplb/cplb_linux.go @@ -485,6 +485,16 @@ vrrp_instance k0s-vip-{{$i}} { {{ . }} {{ end }} } + {{ if .UnicastPeers }} + unicast_src_ip {{ .UnicastSourceIP }} + unicast_peer { + {{ range .UnicastPeers }} + {{ . }} + {{ end }} + } + {{ else}} + #F + {{ end }} } {{ end }} diff --git a/static/_crds/k0s/k0s.k0sproject.io_clusterconfigs.yaml b/static/_crds/k0s/k0s.k0sproject.io_clusterconfigs.yaml index a9a3da692bb7..d1d5f2ffd58f 100644 --- a/static/_crds/k0s/k0s.k0sproject.io_clusterconfigs.yaml +++ b/static/_crds/k0s/k0s.k0sproject.io_clusterconfigs.yaml @@ -609,6 +609,19 @@ spec: Interface specifies the NIC used by the virtual router. If not specified, k0s will use the interface that owns the default route. type: string + unicastPeers: + description: |- + UnicastPeers is a list of unicast peers. If not specified, k0s will use multicast. + If specified, UnicastSourceIP must be specified as well. + items: + type: string + type: array + x-kubernetes-list-type: set + unicastSourceIP: + description: |- + UnicastSourceIP is the source address for unicast peers. + If not specified, k0s will use the first address of the interface. + type: string virtualIPs: description: |- VirtualIPs is the list of virtual IP address used by the VRRP instance.