From f0fa2c613f37afa3c482bac806cbbf14dfa8c94b Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Thu, 5 Dec 2024 11:39:50 +0100 Subject: [PATCH] cmd/microcloud: refactoring to respectively use `OVNGeneveNetwork`, `MicroCephPublicNetwork` and `MicroCephInternalNetwork` instead of `OVNGeneveAddr`, `MicroCephPublicNetworkSubnet` and `MicroCephInternalNetworkSubnet` Signed-off-by: Gabriel Mougard --- cmd/microcloud/ask.go | 70 ++++++++++++++++++++++++++++--------- cmd/microcloud/main_init.go | 40 +++++++++++---------- cmd/microcloud/preseed.go | 48 +++++++++++++++++++++++-- 3 files changed, 120 insertions(+), 38 deletions(-) diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index 3ce8c5a42..aee9b438b 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -920,11 +920,11 @@ type Network struct { Interface string // IP is the IP address of the network. An example of why this is useful in MicroCloud is when we want to store // a member OVN underlay network IP address. - IP net.IP + IP net.IP // Subnet is the subnet of the network. An example of why this is useful in MicroCloud is when we want to check // that we don't have subnet collisions between different network types (OVN, Ceph, etc) in a cluster. // For example, we don't want a member using 'subnet A' for OVN and an other member using 'subnet A' for Ceph. - Subnet *net.IPNet + Subnet *net.IPNet } func (c *initConfig) askOVNNetwork(sh *service.Handler) error { @@ -1149,7 +1149,7 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { } } - var ovnUnderlaySelectedIPs map[string]string + var ovnUnderlaySelectedNets map[string]*Network ovnUnderlayData := [][]string{} for peer, system := range c.systems { // skip any systems that have already been clustered, but are available for other configuration. @@ -1175,7 +1175,6 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { if wantsDedicatedUnderlay { header = []string{"LOCATION", "IFACE", "TYPE", "IP ADDRESS (CIDR)"} - ovnUnderlaySelectedIPs = map[string]string{} err = c.askRetry("Retry selecting underlay network interfaces?", func() error { table := tui.NewSelectableTable(header, ovnUnderlayData) answers, err := table.Render(context.Background(), c.asker, "Select exactly one network interface from each cluster member:") @@ -1183,20 +1182,22 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { return err } - ovnUnderlaySelectedIPs = map[string]string{} + ovnUnderlaySelectedNets = make(map[string]*Network) for _, answer := range answers { target := answer["LOCATION"] ipAddr := answer["IP ADDRESS (CIDR)"] - if ovnUnderlaySelectedIPs[target] != "" { + ifaceName := answer["IFACE"] + + if ovnUnderlaySelectedNets[target] != nil { return fmt.Errorf("Failed to configure OVN underlay traffic: Selected more than one interface for target %q", target) } - ip, _, err := net.ParseCIDR(ipAddr) + ip, ipNet, err := net.ParseCIDR(ipAddr) if err != nil { return err } - ovnUnderlaySelectedIPs[target] = ip.String() + ovnUnderlaySelectedNets[target] = &Network{Interface: ifaceName, IP: ip, Subnet: ipNet} } return nil @@ -1207,11 +1208,11 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { } } - if len(ovnUnderlaySelectedIPs) > 0 { + if len(ovnUnderlaySelectedNets) > 0 { for peer := range askSystems { - underlayIP, ok := ovnUnderlaySelectedIPs[peer] + underlayNetwork, ok := ovnUnderlaySelectedNets[peer] if ok { - fmt.Printf(" Using %q for OVN underlay traffic on %q\n", underlayIP, peer) + fmt.Printf(" Using %q for OVN underlay traffic on %q\n", underlayNetwork.IP.String(), peer) } } @@ -1248,10 +1249,10 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { system.Networks = append(system.Networks, finalConfigs...) } - if ovnUnderlaySelectedIPs != nil { - ovnUnderlayIpAddr, ok := ovnUnderlaySelectedIPs[peer] + if ovnUnderlaySelectedNets != nil { + ovnUnderlayNet, ok := ovnUnderlaySelectedNets[peer] if ok { - system.OVNGeneveAddr = ovnUnderlayIpAddr + system.OVNGeneveNetwork = ovnUnderlayNet } } @@ -1409,14 +1410,34 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error { return err } + // Also register the MicroCloud internal network in the bootstrap system information. + microcloudNetworkInterface, err := lxd.FindInterfaceForSubnet(microCloudInternalNetworkAddrCIDR) + if err != nil { + return fmt.Errorf("Failed to find interface for subnet %q: %w", microCloudInternalNetworkAddrCIDR, err) + } + + bootstrapSystem := c.systems[sh.Name] + bootstrapSystem.MicroCloudInternalNetwork = &Network{Interface: microcloudNetworkInterface.Name, Subnet: c.lookupSubnet, IP: microCloudInternalNetworkAddr} + c.systems[sh.Name] = bootstrapSystem + + internalCephNetworkInterface, err := lxd.FindInterfaceForSubnet(internalCephSubnet) + if err != nil { + return fmt.Errorf("Failed to find interface for subnet %q: %w", internalCephSubnet, err) + } + if internalCephSubnet != microCloudInternalNetworkAddrCIDR { err = c.validateCephInterfacesForSubnet(lxd, availableCephNetworkInterfaces, internalCephSubnet) if err != nil { return err } + internalCephIP, internalCephNet, err := net.ParseCIDR(internalCephSubnet) + if err != nil { + return fmt.Errorf("Failed to parse the internal Ceph network: %w", err) + } + bootstrapSystem := c.systems[sh.Name] - bootstrapSystem.MicroCephInternalNetworkSubnet = internalCephSubnet + bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: internalCephNetworkInterface.Name, Subnet: internalCephNet, IP: internalCephIP} c.systems[sh.Name] = bootstrapSystem } @@ -1425,6 +1446,11 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error { return err } + publicCephNetworkInterface, err := lxd.FindInterfaceForSubnet(publicCephSubnet) + if err != nil { + return fmt.Errorf("Failed to find interface for subnet %q: %w", publicCephSubnet, err) + } + if publicCephSubnet != internalCephSubnet { err = c.validateCephInterfacesForSubnet(lxd, availableCephNetworkInterfaces, publicCephSubnet) if err != nil { @@ -1433,15 +1459,25 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error { } if publicCephSubnet != microCloudInternalNetworkAddrCIDR { + publicCephIP, publicCephNet, err := net.ParseCIDR(publicCephSubnet) + if err != nil { + return fmt.Errorf("Failed to parse the public Ceph network: %w", err) + } + bootstrapSystem := c.systems[sh.Name] - bootstrapSystem.MicroCephPublicNetworkSubnet = publicCephSubnet + bootstrapSystem.MicroCephPublicNetwork = &Network{Interface: publicCephNetworkInterface.Name, Subnet: publicCephNet, IP: publicCephIP} c.systems[sh.Name] = bootstrapSystem // This is to avoid the situation where the internal network for Ceph has been skipped, but the public network has been set. // Ceph will automatically set the internal network to the public Ceph network if the internal network is not set, which is not what we want. // Instead, we still want to keep the internal Ceph network to use the MicroCloud internal network as a default. if internalCephSubnet == microCloudInternalNetworkAddrCIDR { - bootstrapSystem.MicroCephInternalNetworkSubnet = microCloudInternalNetworkAddrCIDR + microcloudInternalIP, microcloudNet, err := net.ParseCIDR(microCloudInternalNetworkAddrCIDR) + if err != nil { + return fmt.Errorf("Failed to parse the internal MicroCloud network: %w", err) + } + + bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: microcloudNetworkInterface.Name, Subnet: microcloudNet, IP: microcloudInternalIP} c.systems[sh.Name] = bootstrapSystem } } diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go index 2e5e56c07..e137675bb 100644 --- a/cmd/microcloud/main_init.go +++ b/cmd/microcloud/main_init.go @@ -53,19 +53,24 @@ type InitSystem struct { AvailableDisks []lxdAPI.ResourcesStorageDisk // MicroCephDisks contains the disks intended to be passed to MicroCeph. MicroCephDisks []cephTypes.DisksPost - // MicroCephPublicNetworkSubnet is an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph public network. - MicroCephPublicNetworkSubnet string - // MicroCephClusterNetworkSubnet is an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph cluster network. - MicroCephInternalNetworkSubnet string + // MicroCephPublicNetwork contains an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph public network and + // the network interface name to use for the Ceph public network and its IP address within the subnet. + MicroCephPublicNetwork *Network + // MicroCephInternalNetwork contains an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph cluster network and + // the network interface name to use for the Ceph cluster network and its IP address within the subnet. + MicroCephInternalNetwork *Network + // MicroCloudInternalNetwork contains the network configuration for the MicroCloud internal network. + MicroCloudInternalNetwork *Network // TargetNetworks contains the network configuration for the target system. TargetNetworks []lxdAPI.NetworksPost // TargetStoragePools contains the storage pool configuration for the target system. TargetStoragePools []lxdAPI.StoragePoolsPost // Networks is the cluster-wide network configuration. Networks []lxdAPI.NetworksPost - // OVNGeneveAddr represents an IP address to use for the OVN (if OVN is supported) Geneve tunnel on this system. + // OVNGeneveNetwork contains an IP address to use for the OVN (if OVN is supported) Geneve tunnel on this system. // If left empty, the system will choose to route the Geneve traffic through the management network. - OVNGeneveAddr string + // It also contains the network interface name to use for the OVN Geneve tunnel and the network subnet. + OVNGeneveNetwork *Network // StoragePools is the cluster-wide storage pool configuration. StoragePools []lxdAPI.StoragePoolsPost // StorageVolumes is the cluster-wide storage volume configuration. @@ -354,9 +359,9 @@ func (c *initConfig) addPeers(sh *service.Handler) (revert.Hook, error) { CephConfig: info.MicroCephDisks, } - if info.OVNGeneveAddr != "" { + if info.OVNGeneveNetwork != nil { p := joinConfig[peer] - p.OVNConfig = map[string]string{"ovn-encap-ip": info.OVNGeneveAddr} + p.OVNConfig = map[string]string{"ovn-encap-ip": info.OVNGeneveNetwork.IP.String()} joinConfig[peer] = p } } @@ -513,7 +518,7 @@ func validateGatewayNet(config map[string]string, ipPrefix string, cidrValidator func (c *initConfig) validateSystems(s *service.Handler) (err error) { for _, sys := range c.systems { - if sys.MicroCephInternalNetworkSubnet == "" || sys.OVNGeneveAddr == "" { + if sys.MicroCephInternalNetwork == nil || sys.OVNGeneveNetwork == nil { continue } @@ -527,9 +532,8 @@ func (c *initConfig) validateSystems(s *service.Handler) (err error) { return fmt.Errorf("OVN underlay IP %q is invalid", sys.OVNGeneveAddr) } - if subnet.Contains(underlayIP) { - tui.PrintWarning(fmt.Sprintf("OVN underlay IP (%s) is shared with the Ceph cluster network (%s)\n", underlayIP.String(), subnet.String())) - + if sys.MicroCephInternalNetwork.Subnet.Contains(sys.OVNGeneveNetwork.IP) { + tui.PrintWarning(fmt.Sprintf("OVN underlay IP (%s) is shared with the Ceph cluster network (%s)\n", sys.OVNGeneveNetwork.IP.String(), sys.MicroCephInternalNetwork.Subnet.String())) break } } @@ -660,12 +664,12 @@ func (c *initConfig) setupCluster(s *service.Handler) error { if s.Type() == types.MicroCeph { microCephBootstrapConf := make(map[string]string) - if bootstrapSystem.MicroCephInternalNetworkSubnet != "" { - microCephBootstrapConf["ClusterNet"] = bootstrapSystem.MicroCephInternalNetworkSubnet + if bootstrapSystem.MicroCephInternalNetwork != nil { + microCephBootstrapConf["ClusterNet"] = bootstrapSystem.MicroCephInternalNetwork.Subnet.String() } - if bootstrapSystem.MicroCephPublicNetworkSubnet != "" { - microCephBootstrapConf["PublicNet"] = bootstrapSystem.MicroCephPublicNetworkSubnet + if bootstrapSystem.MicroCephPublicNetwork != nil { + microCephBootstrapConf["PublicNet"] = bootstrapSystem.MicroCephPublicNetwork.Subnet.String() } if len(microCephBootstrapConf) > 0 { @@ -675,8 +679,8 @@ func (c *initConfig) setupCluster(s *service.Handler) error { if s.Type() == types.MicroOVN { microOvnBootstrapConf := make(map[string]string) - if bootstrapSystem.OVNGeneveAddr != "" { - microOvnBootstrapConf["ovn-encap-ip"] = bootstrapSystem.OVNGeneveAddr + if bootstrapSystem.OVNGeneveNetwork != nil { + microOvnBootstrapConf["ovn-encap-ip"] = bootstrapSystem.OVNGeneveNetwork.IP.String() } if len(microOvnBootstrapConf) > 0 { diff --git a/cmd/microcloud/preseed.go b/cmd/microcloud/preseed.go index 2cee2af08..a140124db 100644 --- a/cmd/microcloud/preseed.go +++ b/cmd/microcloud/preseed.go @@ -794,6 +794,8 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map return nil, fmt.Errorf("Failed to parse supplied underlay IP %q", sys.UnderlayIP) } + ovnUnderlayIfaceByPeer := make(map[string]string) + ovnUnderlaySubnetByPeer := make(map[string]*net.IPNet) for _, iface := range addressedInterfaces[sys.Name] { for _, cidr := range iface.Addresses { _, subnet, err := net.ParseCIDR(cidr) @@ -803,6 +805,8 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map if subnet.Contains(underlayIP) { assignedSystems[sys.Name] = true + ovnUnderlayIfaceByPeer[sys.Name] = iface.Network.Name + ovnUnderlaySubnetByPeer[sys.Name] = subnet break } } @@ -812,8 +816,13 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map return nil, fmt.Errorf("No available interface found for OVN underlay IP %q", sys.UnderlayIP) } + ifaceName, ok := ovnUnderlayIfaceByPeer[sys.Name] + if !ok { + return nil, fmt.Errorf("Failed to find OVN underlay interface name for system %q", sys.Name) + } + system := c.systems[sys.Name] - system.OVNGeneveAddr = sys.UnderlayIP + system.OVNGeneveNetwork = &Network{Interface: ifaceName, Subnet: ovnUnderlaySubnetByPeer[sys.Name], IP: underlayIP} c.systems[sys.Name] = system } } @@ -1095,14 +1104,37 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map internalCephNetwork = customTargetCephInternalNetwork } + // Also register the MicroCloud internal network in the bootstrap system information. + microCloudInternalNetworkAddr := c.lookupSubnet.IP.Mask(c.lookupSubnet.Mask) + ones, _ := c.lookupSubnet.Mask.Size() + microCloudInternalNetworkAddrCIDR := fmt.Sprintf("%s/%d", microCloudInternalNetworkAddr.String(), ones) + microcloudNetworkInterface, err := lxd.FindInterfaceForSubnet(microCloudInternalNetworkAddrCIDR) + if err != nil { + return nil, fmt.Errorf("Failed to find interface for MicroCloud internal subnet %q: %w", microCloudInternalNetworkAddrCIDR, err) + } + + bootstrapSystem := c.systems[s.Name] + bootstrapSystem.MicroCloudInternalNetwork = &Network{Interface: microcloudNetworkInterface.Name, Subnet: c.lookupSubnet, IP: microCloudInternalNetworkAddr} + c.systems[s.Name] = bootstrapSystem + if internalCephNetwork != "" { err = c.validateCephInterfacesForSubnet(lxd, addressedInterfaces, internalCephNetwork) if err != nil { return nil, err } + iface, err := lxd.FindInterfaceForSubnet(internalCephNetwork) + if err != nil { + return nil, err + } + + _, internalCephSubnet, err := net.ParseCIDR(internalCephNetwork) + if err != nil { + return nil, fmt.Errorf("Invalid CIDR for internal Ceph subnet: %v", err) + } + bootstrapSystem := c.systems[s.Name] - bootstrapSystem.MicroCephInternalNetworkSubnet = internalCephNetwork + bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: iface.Name, Subnet: internalCephSubnet, IP: nil} c.systems[s.Name] = bootstrapSystem } @@ -1119,8 +1151,18 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map return nil, err } + iface, err := lxd.FindInterfaceForSubnet(publicCephNetwork) + if err != nil { + return nil, err + } + + _, publicCephSubnet, err := net.ParseCIDR(publicCephNetwork) + if err != nil { + return nil, fmt.Errorf("Invalid CIDR for public Ceph subnet: %w", err) + } + bootstrapSystem := c.systems[s.Name] - bootstrapSystem.MicroCephPublicNetworkSubnet = publicCephNetwork + bootstrapSystem.MicroCephPublicNetwork = &Network{Interface: iface.Name, Subnet: publicCephSubnet, IP: nil} c.systems[s.Name] = bootstrapSystem } } else {