diff --git a/.gitignore b/.gitignore index 1eecf4af1..10bf494e3 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ bin/ /github.com/ + +.idea/ diff --git a/cmd/whereabouts.go b/cmd/whereabouts.go index c64854b2d..3c2f63f68 100644 --- a/cmd/whereabouts.go +++ b/cmd/whereabouts.go @@ -24,12 +24,14 @@ func cmdAddFunc(args *skel.CmdArgs) error { return err } logging.Debugf("ADD - IPAM configuration successfully read: %+v", *ipamConf) - ipam, err := kubernetes.NewKubernetesIPAM(args.ContainerID, *ipamConf) + ipam, err := kubernetes.NewKubernetesIPAM(args.ContainerID, args.IfName, *ipamConf) if err != nil { return logging.Errorf("failed to create Kubernetes IPAM manager: %v", err) } defer func() { safeCloseKubernetesBackendConnection(ipam) }() - return cmdAdd(args, ipam, confVersion) + + logging.Debugf("Beginning IPAM for ContainerID: %q - podRef: %q - ifName: %q", args.ContainerID, ipamConf.GetPodRef(), args.IfName) + return cmdAdd(ipam, confVersion) } func cmdDelFunc(args *skel.CmdArgs) error { @@ -40,12 +42,14 @@ func cmdDelFunc(args *skel.CmdArgs) error { } logging.Debugf("DEL - IPAM configuration successfully read: %+v", *ipamConf) - ipam, err := kubernetes.NewKubernetesIPAM(args.ContainerID, *ipamConf) + ipam, err := kubernetes.NewKubernetesIPAM(args.ContainerID, args.IfName, *ipamConf) if err != nil { return logging.Errorf("IPAM client initialization error: %v", err) } defer func() { safeCloseKubernetesBackendConnection(ipam) }() - return cmdDel(args, ipam) + + logging.Debugf("Beginning delete for ContainerID: %q - podRef: %q - ifName: %q", args.ContainerID, ipamConf.GetPodRef(), args.IfName) + return cmdDel(ipam) } func main() { @@ -69,13 +73,12 @@ func cmdCheck(args *skel.CmdArgs) error { return fmt.Errorf("CNI CHECK method is not implemented") } -func cmdAdd(args *skel.CmdArgs, client *kubernetes.KubernetesIPAM, cniVersion string) error { +func cmdAdd(client *kubernetes.KubernetesIPAM, cniVersion string) error { // Initialize our result, and assign DNS & routing. result := ¤t.Result{} result.DNS = client.Config.DNS result.Routes = client.Config.Routes - logging.Debugf("Beginning IPAM for ContainerID: %v", args.ContainerID) var newips []net.IPNet ctx, cancel := context.WithTimeout(context.Background(), types.AddTimeLimit) @@ -103,9 +106,7 @@ func cmdAdd(args *skel.CmdArgs, client *kubernetes.KubernetesIPAM, cniVersion st return cnitypes.PrintResult(result, cniVersion) } -func cmdDel(args *skel.CmdArgs, client *kubernetes.KubernetesIPAM) error { - logging.Debugf("Beginning delete for ContainerID: %v", args.ContainerID) - +func cmdDel(client *kubernetes.KubernetesIPAM) error { ctx, cancel := context.WithTimeout(context.Background(), types.DelTimeLimit) defer cancel() diff --git a/cmd/whereabouts_test.go b/cmd/whereabouts_test.go index e80f8a170..895f181ac 100644 --- a/cmd/whereabouts_test.go +++ b/cmd/whereabouts_test.go @@ -42,38 +42,45 @@ func TestAPIs(t *testing.T) { []Reporter{}) } -func AllocateAndReleaseAddressesTest(ipVersion string, ipamConf *whereaboutstypes.IPAMConfig, expectedAddresses []string) { +func AllocateAndReleaseAddressesTest(ipRange string, gw string, kubeconfigPath string, expectedAddresses []string) { const ( - ifname string = "eth0" - nspath string = "/some/where" - cniVersion = "0.3.1" - podName = "dummyPOD" - podNamespace = "dummyNS" + ifname = "eth0" + nspath = "/some/where" + cniVersion = "0.3.1" + podName = "dummyPOD" + podNamespace = "dummyNS" + ipamNetworkName = "" ) - addressArgs := []*skel.CmdArgs{} - - cniConf, err := newCNINetConf(cniVersion, ipamConf) - Expect(err).NotTo(HaveOccurred()) - Expect(ipamConf.IPRanges).NotTo(BeEmpty()) + // Only used to get the parsed IP range. + conf := ipamConfig(podName, podNamespace, ipamNetworkName, ipRange, gw, kubeconfigPath) wbClient := *kubernetes.NewKubernetesClient( fake.NewSimpleClientset( - ipPool(ipamConf.IPRanges[0].Range, podNamespace, ipamConf.NetworkName)), + ipPool(conf.IPRanges[0].Range, podNamespace, ipamNetworkName)), fakek8sclient.NewSimpleClientset(), 0) + for i := 0; i < len(expectedAddresses); i++ { + name := fmt.Sprintf("%s-%d", podName, i) + + ipamConf := ipamConfig(name, podNamespace, ipamNetworkName, ipRange, gw, kubeconfigPath) + Expect(ipamConf.IPRanges).NotTo(BeEmpty()) + + cniConf, err := newCNINetConf(cniVersion, ipamConf) + Expect(err).NotTo(HaveOccurred()) + args := &skel.CmdArgs{ ContainerID: fmt.Sprintf("dummy-%d", i), Netns: nspath, IfName: ifname, StdinData: cniConf, - Args: cniArgs(podNamespace, podName), + Args: cniArgs(podNamespace, name), } - client := mutateK8sIPAM(args.ContainerID, ipamConf, wbClient) + client := mutateK8sIPAM(args.ContainerID, ifname, ipamConf, wbClient) // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, client, cniVersion) + return cmdAdd(client, cniVersion) }) Expect(err).NotTo(HaveOccurred()) Expect(strings.Index(string(raw), "\"version\":")).Should(BeNumerically(">", 0)) @@ -90,14 +97,14 @@ func AllocateAndReleaseAddressesTest(ipVersion string, ipamConf *whereaboutstype // Release the IP err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, client) + return cmdDel(client) }) ExpectWithOffset(1, err).NotTo(HaveOccurred()) // Now, create the same thing again, and expect the same IP // That way we know it dealloced the IP and assigned it again. r, _, err = testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, client, cniVersion) + return cmdAdd(client, cniVersion) }) Expect(err).NotTo(HaveOccurred()) @@ -109,23 +116,6 @@ func AllocateAndReleaseAddressesTest(ipVersion string, ipamConf *whereaboutstype Address: mustCIDR(expectedAddresses[i]), Gateway: ipamConf.Gateway, })) - - addressArgs = append(addressArgs, args) - } - - for _, args := range addressArgs { - Expect(ipamConf.IPRanges).NotTo(BeEmpty()) - // And we'll release the IP again. - err := testutils.CmdDelWithArgs(args, func() error { - client := newK8sIPAM( - args.ContainerID, - ipamConf, - fakek8sclient.NewSimpleClientset(), - fake.NewSimpleClientset( - ipPool(ipamConf.IPRanges[0].Range, podNamespace, ipamConf.NetworkName))) - return cmdDel(args, client) - }) - Expect(err).NotTo(HaveOccurred()) } } @@ -133,6 +123,8 @@ var _ = Describe("Whereabouts operations", func() { const ( podName = "dummyPOD" podNamespace = "dummyNS" + ifname = "eth0" + nspath = "/some/where" ) var ( @@ -157,31 +149,68 @@ var _ = Describe("Whereabouts operations", func() { }() }) + It("returns a previously allocated IP", func() { + ipamNetworkName := "" + cniVersion := "0.3.1" + + ipRange := "192.168.1.0/24" + ipGateway := "192.168.10.1" + expectedAddress := "192.168.1.1/24" + + ipamConf := ipamConfig(podName, podNamespace, ipamNetworkName, ipRange, ipGateway, kubeConfigPath) + Expect(ipamConf.IPRanges).NotTo(BeEmpty()) + + wbClient := *kubernetes.NewKubernetesClient( + fake.NewSimpleClientset( + ipPool(ipamConf.IPRanges[0].Range, podNamespace, ipamNetworkName, []whereaboutstypes.IPReservation{ + {PodRef: ipamConf.GetPodRef(), IfName: ifname, IP: net.ParseIP(expectedAddress)}, {PodRef: "test"}}...)), + fakek8sclient.NewSimpleClientset(), + 0) + + cniConf, err := newCNINetConf(cniVersion, ipamConf) + Expect(err).NotTo(HaveOccurred()) + + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: nspath, + IfName: ifname, + StdinData: cniConf, + Args: cniArgs(podNamespace, podName), + } + client := mutateK8sIPAM(args.ContainerID, ifname, ipamConf, wbClient) + + // Allocate the IP + r, raw, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(client, cniVersion) + }) + Expect(err).NotTo(HaveOccurred()) + Expect(strings.Index(string(raw), "\"version\":")).Should(BeNumerically(">", 0)) + + result, err := current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + ExpectWithOffset(1, *result.IPs[0]).To(Equal( + current.IPConfig{ + Address: mustCIDR(expectedAddress), + Gateway: ipamConf.Gateway, + })) + }) + It("allocates and releases addresses on ADD/DEL", func() { - ipVersion := "4" ipRange := "192.168.1.0/24" ipGateway := "192.168.10.1" expectedAddress := "192.168.1.1/24" - AllocateAndReleaseAddressesTest( - ipVersion, - ipamConfig(podName, podNamespace, ipRange, ipGateway, kubeConfigPath), - []string{expectedAddress}) + AllocateAndReleaseAddressesTest(ipRange, ipGateway, kubeConfigPath, []string{expectedAddress}) - ipVersion = "6" ipRange = "2001::1/116" ipGateway = "2001::f:1" expectedAddress = "2001::1/116" - AllocateAndReleaseAddressesTest( - ipVersion, - ipamConfig(podName, podNamespace, ipRange, ipGateway, kubeConfigPath), - []string{expectedAddress}, - ) + AllocateAndReleaseAddressesTest(ipRange, ipGateway, kubeConfigPath, []string{expectedAddress}) }) It("allocates and releases addresses on ADD/DEL with a Kubernetes backend", func() { - ipVersion := "4" ipRange := "192.168.1.11-192.168.1.23/24" ipGateway := "192.168.10.1" @@ -200,81 +229,46 @@ var _ = Describe("Whereabouts operations", func() { "192.168.1.22/24", } - AllocateAndReleaseAddressesTest( - ipVersion, - ipamConfig(podName, podNamespace, ipRange, ipGateway, kubeConfigPath), - expectedAddresses, - ) + AllocateAndReleaseAddressesTest(ipRange, ipGateway, kubeConfigPath, expectedAddresses) - ipVersion = "6" ipRange = "2001::1/116" ipGateway = "2001::f:1" expectedAddress := "2001::1/116" - AllocateAndReleaseAddressesTest( - ipVersion, - ipamConfig(podName, podNamespace, ipRange, ipGateway, kubeConfigPath), - []string{expectedAddress}, - ) + + AllocateAndReleaseAddressesTest(ipRange, ipGateway, kubeConfigPath, []string{expectedAddress}) }) It("allocates and releases an IPv6 address with left-hand zeroes on ADD/DEL with a Kubernetes backend", func() { - - ipVersion := "6" ipRange := "fd::1/116" ipGateway := "fd::f:1" expectedAddress := "fd::1/116" - AllocateAndReleaseAddressesTest( - ipVersion, - ipamConfig(podName, podNamespace, ipRange, ipGateway, kubeConfigPath), - []string{expectedAddress}, - ) + AllocateAndReleaseAddressesTest(ipRange, ipGateway, kubeConfigPath, []string{expectedAddress}) }) It("allocates and releases an IPv6 range that ends with zeroes with a Kubernetes backend", func() { - - ipVersion := "6" ipRange := "2001:db8:480:603d:0304:0403:000:0000-2001:db8:480:603d:0304:0403:0000:0004/64" ipGateway := "2001:db8:480:603d::1" expectedAddress := "2001:db8:480:603d:0304:0403:000:0000/64" - AllocateAndReleaseAddressesTest( - ipVersion, - ipamConfig(podName, podNamespace, ipRange, ipGateway, kubeConfigPath), - []string{expectedAddress}, - ) + AllocateAndReleaseAddressesTest(ipRange, ipGateway, kubeConfigPath, []string{expectedAddress}) ipRange = "2001:db8:5422:0005::-2001:db8:5422:0005:7fff:ffff:ffff:ffff/64" ipGateway = "2001:db8:5422:0005::1" expectedAddress = "2001:db8:5422:0005::1/64" - AllocateAndReleaseAddressesTest( - ipVersion, - ipamConfig(podName, podNamespace, ipRange, ipGateway, kubeConfigPath), - []string{expectedAddress}, - ) + AllocateAndReleaseAddressesTest(ipRange, ipGateway, kubeConfigPath, []string{expectedAddress}) }) It("allocates IPv6 addresses with DNS-1123 conformant naming with a Kubernetes backend", func() { - - ipVersion := "6" ipRange := "fd00:0:0:10:0:0:3:1-fd00:0:0:10:0:0:3:6/64" ipGateway := "2001::f:1" expectedAddress := "fd00:0:0:10:0:0:3:1/64" - AllocateAndReleaseAddressesTest( - ipVersion, - ipamConfig(podName, podNamespace, ipRange, ipGateway, kubeConfigPath), - []string{expectedAddress}, - ) + AllocateAndReleaseAddressesTest(ipRange, ipGateway, kubeConfigPath, []string{expectedAddress}) }) It("excludes a range of addresses", func() { - const ( - ifname string = "eth0" - nspath string = "/some/where" - ) - conf := fmt.Sprintf(`{ "cniVersion": "0.3.1", "name": "mynet", @@ -312,6 +306,7 @@ var _ = Describe("Whereabouts operations", func() { Expect(ipamConf.IPRanges).NotTo(BeEmpty()) k8sClient = newK8sIPAM( args.ContainerID, + ifname, ipamConf, fakek8sclient.NewSimpleClientset(), fake.NewSimpleClientset( @@ -319,7 +314,7 @@ var _ = Describe("Whereabouts operations", func() { // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, k8sClient, cniVersion) + return cmdAdd(k8sClient, cniVersion) }) Expect(err).NotTo(HaveOccurred()) // fmt.Printf("!bang raw: %s\n", raw) @@ -337,15 +332,12 @@ var _ = Describe("Whereabouts operations", func() { // Release the IP err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, k8sClient) + return cmdDel(k8sClient) }) Expect(err).NotTo(HaveOccurred()) }) It("excludes a range of IPv6 addresses", func() { - const ifname string = "eth0" - const nspath string = "/some/where" - backend := fmt.Sprintf(`"kubernetes": {"kubeconfig": "%s"}`, kubeConfigPath) conf := fmt.Sprintf(`{ "cniVersion": "0.3.1", @@ -385,6 +377,7 @@ var _ = Describe("Whereabouts operations", func() { Expect(ipamConf.IPRanges).NotTo(BeEmpty()) k8sClient = newK8sIPAM( args.ContainerID, + ifname, ipamConf, fakek8sclient.NewSimpleClientset(), fake.NewSimpleClientset( @@ -392,7 +385,7 @@ var _ = Describe("Whereabouts operations", func() { // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, k8sClient, cniVersion) + return cmdAdd(k8sClient, cniVersion) }) Expect(err).NotTo(HaveOccurred()) // fmt.Printf("!bang raw: %s\n", raw) @@ -410,16 +403,13 @@ var _ = Describe("Whereabouts operations", func() { // Release the IP err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, k8sClient) + return cmdDel(k8sClient) }) Expect(err).NotTo(HaveOccurred()) }) It("excludes a range of IPv6 addresses, omitting broadcast", func() { - const ifname string = "eth0" - const nspath string = "/some/where" - backend := fmt.Sprintf(`"kubernetes": {"kubeconfig": "%s"}`, kubeConfigPath) conf := fmt.Sprintf(`{ "cniVersion": "0.3.1", @@ -455,6 +445,7 @@ var _ = Describe("Whereabouts operations", func() { Expect(ipamConf.IPRanges).NotTo(BeEmpty()) k8sClient = newK8sIPAM( args.ContainerID, + ifname, ipamConf, fakek8sclient.NewSimpleClientset(), fake.NewSimpleClientset( @@ -462,7 +453,7 @@ var _ = Describe("Whereabouts operations", func() { // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, k8sClient, cniVersion) + return cmdAdd(k8sClient, cniVersion) }) Expect(err).NotTo(HaveOccurred()) // fmt.Printf("!bang raw: %s\n", raw) @@ -480,16 +471,13 @@ var _ = Describe("Whereabouts operations", func() { // Release the IP err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, k8sClient) + return cmdDel(k8sClient) }) Expect(err).NotTo(HaveOccurred()) }) It("can still assign static parameters", func() { - const ifname string = "eth0" - const nspath string = "/some/where" - backend := fmt.Sprintf(`"kubernetes": {"kubeconfig": "%s"}`, kubeConfigPath) conf := fmt.Sprintf(`{ "cniVersion": "0.3.1", @@ -536,6 +524,7 @@ var _ = Describe("Whereabouts operations", func() { Expect(ipamConf.IPRanges).NotTo(BeEmpty()) k8sClient = newK8sIPAM( args.ContainerID, + ifname, ipamConf, fakek8sclient.NewSimpleClientset(), fake.NewSimpleClientset( @@ -543,7 +532,7 @@ var _ = Describe("Whereabouts operations", func() { // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, k8sClient, cniVersion) + return cmdAdd(k8sClient, cniVersion) }) // fmt.Printf("!bang raw: %s\n", raw) Expect(err).NotTo(HaveOccurred()) @@ -582,16 +571,12 @@ var _ = Describe("Whereabouts operations", func() { // Release the IP err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, k8sClient) + return cmdDel(k8sClient) }) Expect(err).NotTo(HaveOccurred()) - }) It("allocates an address using IPRanges notation", func() { - const ifname string = "eth0" - const nspath string = "/some/where" - backend := fmt.Sprintf(`"kubernetes": {"kubeconfig": "%s"}`, kubeConfigPath) conf := fmt.Sprintf(`{ "cniVersion": "0.3.1", @@ -624,6 +609,7 @@ var _ = Describe("Whereabouts operations", func() { Expect(ipamConf.IPRanges).NotTo(BeEmpty()) k8sClient = newK8sIPAM( args.ContainerID, + ifname, ipamConf, fakek8sclient.NewSimpleClientset(), fake.NewSimpleClientset( @@ -631,7 +617,7 @@ var _ = Describe("Whereabouts operations", func() { // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, k8sClient, cniVersion) + return cmdAdd(k8sClient, cniVersion) }) Expect(err).NotTo(HaveOccurred()) @@ -647,15 +633,12 @@ var _ = Describe("Whereabouts operations", func() { // Release the IP err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, k8sClient) + return cmdDel(k8sClient) }) Expect(err).NotTo(HaveOccurred()) }) It("allocates DualStack address using IPRanges notation", func() { - const ifname string = "eth0" - const nspath string = "/some/where" - backend := fmt.Sprintf(`"kubernetes": {"kubeconfig": "%s"}`, kubeConfigPath) conf := fmt.Sprintf(`{ "cniVersion": "0.3.1", @@ -690,6 +673,7 @@ var _ = Describe("Whereabouts operations", func() { Expect(ipamConf.IPRanges).To(HaveLen(2)) k8sClient = newK8sIPAM( args.ContainerID, + ifname, ipamConf, fakek8sclient.NewSimpleClientset(), fake.NewSimpleClientset( @@ -698,7 +682,7 @@ var _ = Describe("Whereabouts operations", func() { // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, k8sClient, cniVersion) + return cmdAdd(k8sClient, cniVersion) }) Expect(err).NotTo(HaveOccurred()) @@ -715,15 +699,12 @@ var _ = Describe("Whereabouts operations", func() { // Release the IP err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, k8sClient) + return cmdDel(k8sClient) }) Expect(err).NotTo(HaveOccurred()) }) It("allocates addresses using both IPRanges and range notations", func() { - const ifname string = "eth0" - const nspath string = "/some/where" - backend := fmt.Sprintf(`"kubernetes": {"kubeconfig": "%s"}`, kubeConfigPath) conf := fmt.Sprintf(`{ "cniVersion": "0.3.1", @@ -757,6 +738,7 @@ var _ = Describe("Whereabouts operations", func() { Expect(ipamConf.IPRanges).To(HaveLen(2)) k8sClient = newK8sIPAM( args.ContainerID, + ifname, ipamConf, fakek8sclient.NewSimpleClientset(), fake.NewSimpleClientset( @@ -765,7 +747,7 @@ var _ = Describe("Whereabouts operations", func() { // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, k8sClient, cniVersion) + return cmdAdd(k8sClient, cniVersion) }) Expect(err).NotTo(HaveOccurred()) @@ -782,15 +764,12 @@ var _ = Describe("Whereabouts operations", func() { // Release the IP err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, k8sClient) + return cmdDel(k8sClient) }) Expect(err).NotTo(HaveOccurred()) }) It("allocates an address using start/end cidr notation", func() { - const ifname string = "eth0" - const nspath string = "/some/where" - backend := fmt.Sprintf(`"kubernetes": {"kubeconfig": "%s"}`, kubeConfigPath) conf := fmt.Sprintf(`{ "cniVersion": "0.3.1", @@ -822,6 +801,7 @@ var _ = Describe("Whereabouts operations", func() { Expect(ipamConf.IPRanges).NotTo(BeEmpty()) k8sClient = newK8sIPAM( args.ContainerID, + ifname, ipamConf, fakek8sclient.NewSimpleClientset(), fake.NewSimpleClientset( @@ -829,7 +809,7 @@ var _ = Describe("Whereabouts operations", func() { // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, k8sClient, cniVersion) + return cmdAdd(k8sClient, cniVersion) }) Expect(err).NotTo(HaveOccurred()) @@ -848,15 +828,12 @@ var _ = Describe("Whereabouts operations", func() { // Release the IP err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, k8sClient) + return cmdDel(k8sClient) }) Expect(err).NotTo(HaveOccurred()) }) It("allocates an address using the range_start parameter", func() { - const ifname string = "eth0" - const nspath string = "/some/where" - backend := fmt.Sprintf(`"kubernetes": {"kubeconfig": "%s"}`, kubeConfigPath) conf := fmt.Sprintf(`{ "cniVersion": "0.3.1", @@ -889,6 +866,7 @@ var _ = Describe("Whereabouts operations", func() { Expect(ipamConf.IPRanges).NotTo(BeEmpty()) k8sClient = newK8sIPAM( args.ContainerID, + ifname, ipamConf, fakek8sclient.NewSimpleClientset(), fake.NewSimpleClientset( @@ -896,7 +874,7 @@ var _ = Describe("Whereabouts operations", func() { // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, k8sClient, cniVersion) + return cmdAdd(k8sClient, cniVersion) }) Expect(err).NotTo(HaveOccurred()) @@ -915,15 +893,12 @@ var _ = Describe("Whereabouts operations", func() { // Release the IP err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, k8sClient) + return cmdDel(k8sClient) }) Expect(err).NotTo(HaveOccurred()) }) It("allocates addresses using range_end as an upper limit", func() { - const ifname string = "eth0" - const nspath string = "/some/where" - backend := fmt.Sprintf(`"kubernetes": {"kubeconfig": "%s"}`, kubeConfigPath) conf := fmt.Sprintf(`{ "cniVersion": "0.3.1", @@ -944,29 +919,35 @@ var _ = Describe("Whereabouts operations", func() { confPath := filepath.Join(tmpDir, "whereabouts.conf") Expect(os.WriteFile(confPath, []byte(conf), 0755)).To(Succeed()) - ipamConf, cniVersion, err := config.LoadIPAMConfig([]byte(conf), cniArgs(podNamespace, podName), confPath) + + // Only used to get the parsed IP range. + ipamConf, _, err := config.LoadIPAMConfig([]byte(conf), cniArgs(podNamespace, podName), confPath) Expect(err).NotTo(HaveOccurred()) Expect(ipamConf.IPRanges).NotTo(BeEmpty()) + wbClient := *kubernetes.NewKubernetesClient( fake.NewSimpleClientset( ipPool(ipamConf.IPRanges[0].Range, podNamespace, ipamConf.NetworkName)), fakek8sclient.NewSimpleClientset(), 0) - var ipArgs []*skel.CmdArgs // allocate 8 IPs (192.168.1.5 - 192.168.1.12); the entirety of the pool defined above for i := 0; i < 8; i++ { + name := fmt.Sprintf("%s-%d", podName, i) args := &skel.CmdArgs{ ContainerID: fmt.Sprintf("dummy-%d", i), Netns: nspath, IfName: ifname, StdinData: []byte(conf), - Args: cniArgs(podNamespace, podName), + Args: cniArgs(podNamespace, name), } - k8sClient = mutateK8sIPAM(args.ContainerID, ipamConf, wbClient) + ipamConf, cniVersion, err := config.LoadIPAMConfig([]byte(conf), cniArgs(podNamespace, name), confPath) + Expect(err).NotTo(HaveOccurred()) + + k8sClient = mutateK8sIPAM(args.ContainerID, ifname, ipamConf, wbClient) r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, k8sClient, cniVersion) + return cmdAdd(k8sClient, cniVersion) }) Expect(err).NotTo(HaveOccurred()) @@ -980,40 +961,47 @@ var _ = Describe("Whereabouts operations", func() { Address: mustCIDR(fmt.Sprintf("192.168.1.%d/24", 5+i)), Gateway: net.ParseIP("192.168.10.1"), })) - ipArgs = append(ipArgs, args) } // assigning more IPs should result in error due to the defined range_start - range_end + name := fmt.Sprintf("%s-dummy-failure", podName) args := &skel.CmdArgs{ ContainerID: "dummy-failure", Netns: nspath, IfName: ifname, StdinData: []byte(conf), - Args: cniArgs(podNamespace, podName), + Args: cniArgs(podNamespace, name), } + + ipamConf, cniVersion, err := config.LoadIPAMConfig([]byte(conf), cniArgs(podNamespace, name), confPath) + Expect(err).NotTo(HaveOccurred()) + + k8sClient = mutateK8sIPAM(args.ContainerID, ifname, ipamConf, wbClient) _, _, err = testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, mutateK8sIPAM(args.ContainerID, ipamConf, wbClient), "0.3.1") + return cmdAdd(k8sClient, cniVersion) }) Expect(err).To(HaveOccurred()) + // ensure the error is of the correct type switch e := errors.Unwrap(err); e.(type) { case allocate.AssignmentError: default: Fail(fmt.Sprintf("expected AssignmentError, got: %s", e)) } - - // Release assigned IPs - for _, args := range ipArgs { - err := testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, mutateK8sIPAM(args.ContainerID, ipamConf, wbClient)) - }) - Expect(err).NotTo(HaveOccurred()) - } }) It("detects IPv4 addresses used in other ranges, to allow for overlapping IP address ranges", func() { - const ifname string = "eth0" - const nspath string = "/some/where" + firstPodName := "dummyfirstrange" + secondPodName := "dummysecondrange" + + firstRange := "192.168.22.0/24" + secondRange := "192.168.22.0/28" + + wbClient := *kubernetes.NewKubernetesClient( + fake.NewSimpleClientset( + ipPool(firstRange, podNamespace, ""), ipPool(secondRange, podNamespace, "")), + fakek8sclient.NewSimpleClientset(), + 0) // ----------------------------- range 1 @@ -1028,32 +1016,27 @@ var _ = Describe("Whereabouts operations", func() { "log_file" : "/tmp/whereabouts.log", "log_level" : "debug", "kubernetes": {"kubeconfig": "%s"}, - "range": "192.168.22.0/24" + "range": %q } - }`, kubeConfigPath) + }`, kubeConfigPath, firstRange) args := &skel.CmdArgs{ ContainerID: "dummyfirstrange", Netns: nspath, IfName: ifname, StdinData: []byte(conf), - Args: cniArgs(podNamespace, podName), + Args: cniArgs(podNamespace, firstPodName), } confPath := filepath.Join(tmpDir, "whereabouts.conf") Expect(os.WriteFile(confPath, []byte(conf), 0755)).To(Succeed()) - ipamConf, cniVersion, err := config.LoadIPAMConfig([]byte(conf), cniArgs(podNamespace, podName), confPath) + ipamConf, cniVersion, err := config.LoadIPAMConfig([]byte(conf), cniArgs(podNamespace, firstPodName), confPath) Expect(err).NotTo(HaveOccurred()) Expect(ipamConf.IPRanges).NotTo(BeEmpty()) - wbClient := *kubernetes.NewKubernetesClient( - fake.NewSimpleClientset( - ipPool(ipamConf.IPRanges[0].Range, podNamespace, ipamConf.NetworkName)), - fakek8sclient.NewSimpleClientset(), - 0) // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, mutateK8sIPAM(args.ContainerID, ipamConf, wbClient), cniVersion) + return cmdAdd(mutateK8sIPAM(args.ContainerID, ifname, ipamConf, wbClient), cniVersion) }) Expect(err).NotTo(HaveOccurred()) // fmt.Printf("!bang raw: %s\n", raw) @@ -1081,26 +1064,26 @@ var _ = Describe("Whereabouts operations", func() { "log_file" : "/tmp/whereabouts.log", "log_level" : "debug", "kubernetes": {"kubeconfig": "%s"}, - "range": "192.168.22.0/28" + "range": %q } - }`, kubeConfigPath) + }`, kubeConfigPath, secondRange) argssecond := &skel.CmdArgs{ ContainerID: "dummysecondrange", Netns: nspath, IfName: ifname, StdinData: []byte(confsecond), - Args: cniArgs(podNamespace, podName), + Args: cniArgs(podNamespace, secondPodName), } secondConfPath := filepath.Join(tmpDir, "whereabouts.conf") Expect(os.WriteFile(confPath, []byte(confsecond), 0755)).To(Succeed()) - secondIPAMConf, secondCNIVersion, err := config.LoadIPAMConfig([]byte(confsecond), cniArgs(podNamespace, podName), secondConfPath) + secondIPAMConf, secondCNIVersion, err := config.LoadIPAMConfig([]byte(confsecond), cniArgs(podNamespace, secondPodName), secondConfPath) Expect(err).NotTo(HaveOccurred()) // Allocate the IP r, raw, err = testutils.CmdAddWithArgs(argssecond, func() error { - return cmdAdd(argssecond, mutateK8sIPAM(argssecond.ContainerID, secondIPAMConf, wbClient), secondCNIVersion) + return cmdAdd(mutateK8sIPAM(args.ContainerID, ifname, secondIPAMConf, wbClient), secondCNIVersion) }) Expect(err).NotTo(HaveOccurred()) // fmt.Printf("!bang raw: %s\n", raw) @@ -1119,21 +1102,30 @@ var _ = Describe("Whereabouts operations", func() { // Release the IP, first range err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, mutateK8sIPAM(args.ContainerID, ipamConf, wbClient)) + return cmdDel(mutateK8sIPAM(args.ContainerID, ifname, ipamConf, wbClient)) }) Expect(err).NotTo(HaveOccurred()) // Release the IP, second range err = testutils.CmdDelWithArgs(argssecond, func() error { - return cmdDel(argssecond, mutateK8sIPAM(argssecond.ContainerID, secondIPAMConf, wbClient)) + return cmdDel(mutateK8sIPAM(args.ContainerID, ifname, secondIPAMConf, wbClient)) }) Expect(err).NotTo(HaveOccurred()) }) It("detects IPv6 addresses used in other ranges, to allow for overlapping IP address ranges", func() { - const ifname string = "eth0" - const nspath string = "/some/where" + firstPodName := "dummyfirstrange" + secondPodName := "dummysecondrange" + + firstRange := "2001::2:3:0/124" + secondRange := "2001::2:3:0/126" + + wbClient := *kubernetes.NewKubernetesClient( + fake.NewSimpleClientset( + ipPool(firstRange, podNamespace, ""), ipPool(secondRange, podNamespace, "")), + fakek8sclient.NewSimpleClientset(), + 0) // ----------------------------- range 1 @@ -1148,32 +1140,27 @@ var _ = Describe("Whereabouts operations", func() { "log_file" : "/tmp/whereabouts.log", "log_level" : "debug", "kubernetes": {"kubeconfig": "%s"}, - "range": "2001::2:3:0/124" + "range": %q } - }`, kubeConfigPath) + }`, kubeConfigPath, firstRange) args := &skel.CmdArgs{ ContainerID: "dummyfirstrange", Netns: nspath, IfName: ifname, StdinData: []byte(conf), - Args: cniArgs(podNamespace, podName), + Args: cniArgs(podNamespace, firstPodName), } confPath := filepath.Join(tmpDir, "whereabouts.conf") Expect(os.WriteFile(confPath, []byte(conf), 0755)).To(Succeed()) - ipamConf, cniVersion, err := config.LoadIPAMConfig([]byte(conf), cniArgs(podNamespace, podName), confPath) + ipamConf, cniVersion, err := config.LoadIPAMConfig([]byte(conf), cniArgs(podNamespace, firstPodName), confPath) Expect(err).NotTo(HaveOccurred()) Expect(ipamConf.IPRanges).NotTo(BeEmpty()) - wbClient := *kubernetes.NewKubernetesClient( - fake.NewSimpleClientset( - ipPool(ipamConf.IPRanges[0].Range, podNamespace, ipamConf.NetworkName)), - fakek8sclient.NewSimpleClientset(), - 0) // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, mutateK8sIPAM(args.ContainerID, ipamConf, wbClient), cniVersion) + return cmdAdd(mutateK8sIPAM(args.ContainerID, ifname, ipamConf, wbClient), cniVersion) }) Expect(err).NotTo(HaveOccurred()) // fmt.Printf("!bang raw: %s\n", raw) @@ -1201,26 +1188,26 @@ var _ = Describe("Whereabouts operations", func() { "log_file" : "/tmp/whereabouts.log", "log_level" : "debug", "kubernetes": {"kubeconfig": "%s"}, - "range": "2001::2:3:0/126" + "range": %q } - }`, kubeConfigPath) + }`, kubeConfigPath, secondRange) argssecond := &skel.CmdArgs{ ContainerID: "dummysecondrange", Netns: nspath, IfName: ifname, StdinData: []byte(confsecond), - Args: cniArgs(podNamespace, podName), + Args: cniArgs(podNamespace, secondPodName), } secondConfPath := filepath.Join(tmpDir, "whereabouts.conf") Expect(os.WriteFile(confPath, []byte(confsecond), 0755)).To(Succeed()) - secondIPAMConf, secondCNIVersion, err := config.LoadIPAMConfig([]byte(confsecond), cniArgs(podNamespace, podName), secondConfPath) + secondIPAMConf, secondCNIVersion, err := config.LoadIPAMConfig([]byte(confsecond), cniArgs(podNamespace, secondPodName), secondConfPath) Expect(err).NotTo(HaveOccurred()) // Allocate the IP r, raw, err = testutils.CmdAddWithArgs(argssecond, func() error { - return cmdAdd(argssecond, mutateK8sIPAM(argssecond.ContainerID, secondIPAMConf, wbClient), secondCNIVersion) + return cmdAdd(mutateK8sIPAM(args.ContainerID, ifname, secondIPAMConf, wbClient), secondCNIVersion) }) Expect(err).NotTo(HaveOccurred()) // fmt.Printf("!bang raw: %s\n", raw) @@ -1239,21 +1226,30 @@ var _ = Describe("Whereabouts operations", func() { // Release the IP, first range err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, mutateK8sIPAM(args.ContainerID, ipamConf, wbClient)) + return cmdDel(mutateK8sIPAM(args.ContainerID, ifname, ipamConf, wbClient)) }) Expect(err).NotTo(HaveOccurred()) // Release the IP, second range err = testutils.CmdDelWithArgs(argssecond, func() error { - return cmdDel(argssecond, mutateK8sIPAM(argssecond.ContainerID, secondIPAMConf, wbClient)) + return cmdDel(mutateK8sIPAM(args.ContainerID, ifname, secondIPAMConf, wbClient)) }) Expect(err).NotTo(HaveOccurred()) }) It("allows IP collisions across ranges when enable_overlapping_ranges is set to false", func() { - const ifname string = "eth0" - const nspath string = "/some/where" + firstPodName := "dummyfirstrange" + secondPodName := "dummysecondrange" + + firstRange := "192.168.33.0/24" + secondRange := "192.168.33.0/28" + + wbClient := *kubernetes.NewKubernetesClient( + fake.NewSimpleClientset( + ipPool(firstRange, podNamespace, ""), ipPool(secondRange, podNamespace, "")), + fakek8sclient.NewSimpleClientset(), + 0) // ----------------------------- range 1 @@ -1269,32 +1265,27 @@ var _ = Describe("Whereabouts operations", func() { "log_level" : "debug", "kubernetes": {"kubeconfig": "%s"}, "enable_overlapping_ranges": false, - "range": "192.168.33.0/24" + "range": %q } - }`, kubeConfigPath) + }`, kubeConfigPath, firstRange) args := &skel.CmdArgs{ ContainerID: "dummyfirstrange", Netns: nspath, IfName: ifname, StdinData: []byte(conf), - Args: cniArgs(podNamespace, podName), + Args: cniArgs(podNamespace, firstPodName), } confPath := filepath.Join(tmpDir, "whereabouts.conf") Expect(os.WriteFile(confPath, []byte(conf), 0755)).To(Succeed()) - ipamConf, cniVersion, err := config.LoadIPAMConfig([]byte(conf), cniArgs(podNamespace, podName), confPath) + ipamConf, cniVersion, err := config.LoadIPAMConfig([]byte(conf), cniArgs(podNamespace, firstPodName), confPath) Expect(err).NotTo(HaveOccurred()) Expect(ipamConf.IPRanges).NotTo(BeEmpty()) - wbClient := *kubernetes.NewKubernetesClient( - fake.NewSimpleClientset( - ipPool(ipamConf.IPRanges[0].Range, podNamespace, ipamConf.NetworkName)), - fakek8sclient.NewSimpleClientset(), - 0) // Allocate the IP r, raw, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args, mutateK8sIPAM(args.ContainerID, ipamConf, wbClient), cniVersion) + return cmdAdd(mutateK8sIPAM(args.ContainerID, ifname, ipamConf, wbClient), cniVersion) }) Expect(err).NotTo(HaveOccurred()) // fmt.Printf("!bang raw: %s\n", raw) @@ -1322,26 +1313,26 @@ var _ = Describe("Whereabouts operations", func() { "log_file" : "/tmp/whereabouts.log", "log_level" : "debug", "kubernetes": {"kubeconfig": "%s"}, - "range": "192.168.33.0/28" + "range": %q } - }`, kubeConfigPath) + }`, kubeConfigPath, secondRange) argssecond := &skel.CmdArgs{ ContainerID: "dummysecondrange", Netns: nspath, IfName: ifname, StdinData: []byte(confsecond), - Args: cniArgs(podNamespace, podName), + Args: cniArgs(podNamespace, secondPodName), } secondConfPath := filepath.Join(tmpDir, "whereabouts.conf") Expect(os.WriteFile(secondConfPath, []byte(confsecond), 0755)).To(Succeed()) - secondIPAMConf, secondCNIVersion, err := config.LoadIPAMConfig([]byte(confsecond), cniArgs(podNamespace, podName), secondConfPath) + secondIPAMConf, secondCNIVersion, err := config.LoadIPAMConfig([]byte(confsecond), cniArgs(podNamespace, secondPodName), secondConfPath) Expect(err).NotTo(HaveOccurred()) // Allocate the IP r, raw, err = testutils.CmdAddWithArgs(argssecond, func() error { - return cmdAdd(argssecond, mutateK8sIPAM(argssecond.ContainerID, secondIPAMConf, wbClient), secondCNIVersion) + return cmdAdd(mutateK8sIPAM(args.ContainerID, ifname, secondIPAMConf, wbClient), secondCNIVersion) }) Expect(err).NotTo(HaveOccurred()) // fmt.Printf("!bang raw: %s\n", raw) @@ -1360,24 +1351,25 @@ var _ = Describe("Whereabouts operations", func() { // Release the IP, first range err = testutils.CmdDelWithArgs(args, func() error { - return cmdDel(args, mutateK8sIPAM(args.ContainerID, ipamConf, wbClient)) + return cmdDel(mutateK8sIPAM(args.ContainerID, ifname, ipamConf, wbClient)) }) Expect(err).NotTo(HaveOccurred()) // Release the IP, second range err = testutils.CmdDelWithArgs(argssecond, func() error { - return cmdDel(argssecond, mutateK8sIPAM(argssecond.ContainerID, secondIPAMConf, wbClient)) + return cmdDel(mutateK8sIPAM(args.ContainerID, ifname, secondIPAMConf, wbClient)) }) Expect(err).NotTo(HaveOccurred()) }) + }) func cniArgs(podNamespace string, podName string) string { return fmt.Sprintf("IgnoreUnknown=1;K8S_POD_NAMESPACE=%s;K8S_POD_NAME=%s", podNamespace, podName) } -func newK8sIPAM(containerID string, ipamConf *whereaboutstypes.IPAMConfig, k8sCoreClient k8sclient.Interface, wbClient wbclientset.Interface) *kubernetes.KubernetesIPAM { - k8sIPAM, err := kubernetes.NewKubernetesIPAMWithNamespace(containerID, *ipamConf, ipamConf.PodNamespace) +func newK8sIPAM(containerID, ifName string, ipamConf *whereaboutstypes.IPAMConfig, k8sCoreClient k8sclient.Interface, wbClient wbclientset.Interface) *kubernetes.KubernetesIPAM { + k8sIPAM, err := kubernetes.NewKubernetesIPAMWithNamespace(containerID, ifName, *ipamConf, ipamConf.PodNamespace) if err != nil { return nil } @@ -1385,8 +1377,8 @@ func newK8sIPAM(containerID string, ipamConf *whereaboutstypes.IPAMConfig, k8sCo return k8sIPAM } -func mutateK8sIPAM(containerID string, ipamConf *whereaboutstypes.IPAMConfig, client kubernetes.Client) *kubernetes.KubernetesIPAM { - k8sIPAM, err := kubernetes.NewKubernetesIPAMWithNamespace(containerID, *ipamConf, ipamConf.PodNamespace) +func mutateK8sIPAM(containerID, ifName string, ipamConf *whereaboutstypes.IPAMConfig, client kubernetes.Client) *kubernetes.KubernetesIPAM { + k8sIPAM, err := kubernetes.NewKubernetesIPAMWithNamespace(containerID, ifName, *ipamConf, ipamConf.PodNamespace) if err != nil { return nil } @@ -1403,7 +1395,7 @@ func mustCIDR(s string) net.IPNet { return *n } -func ipamConfig(podName string, namespace string, ipRange string, gw string, kubeconfigPath string) *whereaboutstypes.IPAMConfig { +func ipamConfig(podName, namespace, networkName, ipRange, gw, kubeconfigPath string) *whereaboutstypes.IPAMConfig { const ( cniVersion = "0.3.1" netName = "net1" @@ -1420,6 +1412,7 @@ func ipamConfig(podName string, namespace string, ipRange string, gw string, kub Kubernetes: whereaboutstypes.KubernetesConfig{ KubeConfigPath: kubeconfigPath, }, + NetworkName: networkName, } bytes, err := json.Marshal(&whereaboutstypes.Net{ Name: netName, @@ -1483,7 +1476,7 @@ users: `) } -func ipPool(ipRange string, namespace string, networkName string, podReferences ...string) *v1alpha1.IPPool { +func ipPool(ipRange string, namespace string, networkName string, podReferences ...whereaboutstypes.IPReservation) *v1alpha1.IPPool { return &v1alpha1.IPPool{ ObjectMeta: metav1.ObjectMeta{ Name: kubernetes.IPPoolName(kubernetes.PoolIdentifier{IpRange: ipRange, NetworkName: networkName}), @@ -1497,12 +1490,13 @@ func ipPool(ipRange string, namespace string, networkName string, podReferences } } -func allocations(podReferences ...string) map[string]v1alpha1.IPAllocation { +func allocations(podReferences ...whereaboutstypes.IPReservation) map[string]v1alpha1.IPAllocation { poolAllocations := map[string]v1alpha1.IPAllocation{} - for i, podRef := range podReferences { - poolAllocations[fmt.Sprintf("%d", i)] = v1alpha1.IPAllocation{ + for i, r := range podReferences { + poolAllocations[fmt.Sprintf("%d", i+1)] = v1alpha1.IPAllocation{ ContainerID: "", - PodRef: podRef, + PodRef: r.PodRef, + IfName: r.IfName, } } return poolAllocations diff --git a/doc/crds/whereabouts.cni.cncf.io_ippools.yaml b/doc/crds/whereabouts.cni.cncf.io_ippools.yaml index d2350c483..8e0ef3628 100644 --- a/doc/crds/whereabouts.cni.cncf.io_ippools.yaml +++ b/doc/crds/whereabouts.cni.cncf.io_ippools.yaml @@ -1,11 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: ippools.whereabouts.cni.cncf.io spec: group: whereabouts.cni.cncf.io @@ -22,14 +20,19 @@ spec: description: IPPool is the Schema for the ippools API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -43,14 +46,17 @@ spec: properties: id: type: string + ifname: + type: string podref: type: string required: - id + - podref type: object - description: Allocations is the set of allocated IPs for the given - range. Its` indices are a direct mapping to the IP with the same - index/offset for the pool's range. + description: |- + Allocations is the set of allocated IPs for the given range. Its` indices are a direct mapping to the + IP with the same index/offset for the pool's range. type: object range: description: Range is a RFC 4632/4291-style string that represents @@ -63,9 +69,3 @@ spec: type: object served: true storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml b/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml index b21db8c93..f95cdfb8b 100644 --- a/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml +++ b/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml @@ -1,11 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: overlappingrangeipreservations.whereabouts.cni.cncf.io spec: group: whereabouts.cni.cncf.io @@ -23,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -40,19 +43,15 @@ spec: properties: containerid: type: string + ifname: + type: string podref: type: string required: - - containerid + - podref type: object required: - spec type: object served: true storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/e2e/client/replicaset.go b/e2e/client/replicaset.go index 25e4b01c6..9ee256529 100644 --- a/e2e/client/replicaset.go +++ b/e2e/client/replicaset.go @@ -46,9 +46,9 @@ func isReplicaSetSteady(ctx context.Context, cs *kubernetes.Clientset, replicaSe } // check two things: -// 1. number of pods that are ready should equal that of spec -// 2. number of pods matching replicaSet's selector should equal that of spec -// (in 0 replicas case, replicas should finish terminating before this comes true) +// 1. number of pods that are ready should equal that of spec +// 2. number of pods matching replicaSet's selector should equal that of spec +// (in 0 replicas case, replicas should finish terminating before this comes true) func isReplicaSetSynchronized(replicaSet *appsv1.ReplicaSet, podList *corev1.PodList) bool { return replicaSet.Status.ReadyReplicas == (*replicaSet.Spec.Replicas) && int32(len(podList.Items)) == (*replicaSet.Spec.Replicas) } diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 9a0b2274d..7180958d8 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -15,6 +15,7 @@ import ( v1 "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -27,12 +28,14 @@ import ( "github.com/k8snetworkplumbingwg/whereabouts/e2e/retrievers" testenv "github.com/k8snetworkplumbingwg/whereabouts/e2e/testenvironment" "github.com/k8snetworkplumbingwg/whereabouts/pkg/api/whereabouts.cni.cncf.io/v1alpha1" + "github.com/k8snetworkplumbingwg/whereabouts/pkg/iphelpers" wbstorage "github.com/k8snetworkplumbingwg/whereabouts/pkg/storage/kubernetes" "github.com/k8snetworkplumbingwg/whereabouts/pkg/types" ) const ( createPodTimeout = 10 * time.Second + ipPoolNamespace = "kube-system" ) func TestWhereaboutsE2E(t *testing.T) { @@ -86,10 +89,15 @@ var _ = Describe("Whereabouts functionality", func() { }) Context("Single pod tests", func() { - BeforeEach(func() { - const singlePodName = "whereabouts-basic-test" - var err error + const singlePodName = "whereabouts-basic-test" + var err error + + AfterEach(func() { + By("deleting pod with whereabouts net-attach-def") + _ = clientInfo.DeletePod(pod) + }) + It("allocates a single pod with a single interface", func() { By("creating a pod with whereabouts net-attach-def") pod, err = clientInfo.ProvisionPod( singlePodName, @@ -98,19 +106,55 @@ var _ = Describe("Whereabouts functionality", func() { entities.PodNetworkSelectionElements(testNetworkName), ) Expect(err).NotTo(HaveOccurred()) - }) - AfterEach(func() { - By("deleting pod with whereabouts net-attach-def") - Expect(clientInfo.DeletePod(pod)).To(Succeed()) - }) - - It("allocates a single pod within the correct IP range", func() { By("checking pod IP is within whereabouts IPAM range") - secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod) + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, "net1") Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs).NotTo(BeEmpty()) Expect(inRange(ipv4TestRange, secondaryIfaceIPs[0])).To(Succeed()) + + By("verifying allocation") + verifyAllocations(clientInfo, ipv4TestRange, secondaryIfaceIPs[0], testNamespace, pod.Name, "net1") + + By("deleting pod") + err = clientInfo.DeletePod(pod) + Expect(err).NotTo(HaveOccurred()) + + By("checking that the IP allocation is removed") + verifyNoAllocationsForPodRef(clientInfo, ipv4TestRange, testNamespace, pod.Name, secondaryIfaceIPs) + }) + It("allocates a single pod with multiple interfaces", func() { + By("creating a pod with whereabouts net-attach-def") + pod, err = clientInfo.ProvisionPod( + singlePodName, + testNamespace, + podTierLabel(singlePodName), + entities.PodNetworkSelectionElements(testNetworkName, testNetworkName, testNetworkName), + ) + Expect(err).NotTo(HaveOccurred()) + + By("checking pod IP is within whereabouts IPAM range") + var secondaryIPs []string + + for _, ifName := range []string{"net1", "net2", "net3"} { + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, ifName) + Expect(err).NotTo(HaveOccurred()) + Expect(secondaryIfaceIPs).NotTo(BeEmpty()) + for _, ip := range secondaryIfaceIPs { + Expect(inRange(ipv4TestRange, ip)).To(Succeed()) + + By("verifying allocation") + verifyAllocations(clientInfo, ipv4TestRange, ip, testNamespace, pod.Name, ifName) + } + secondaryIPs = append(secondaryIPs, secondaryIfaceIPs...) + } + + By("deleting pod") + err = clientInfo.DeletePod(pod) + Expect(err).NotTo(HaveOccurred()) + + By("checking that the IP allocation is removed") + verifyNoAllocationsForPodRef(clientInfo, ipv4TestRange, testNamespace, pod.Name, secondaryIPs) }) }) @@ -160,7 +204,7 @@ var _ = Describe("Whereabouts functionality", func() { It("allocates a single pod within the correct IP ranges", func() { By("checking pod IP is within whereabouts IPAM ranges") - secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod) + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, "net1") Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs).To(HaveLen(2)) Expect(inRange(dualStackIPv4Range, secondaryIfaceIPs[0])).To(Succeed()) @@ -202,7 +246,7 @@ var _ = Describe("Whereabouts functionality", func() { It("allocates a single pod within the correct IP ranges", func() { By("checking pod IP is within whereabouts IPAM ranges") - secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod) + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, "net1") Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs).To(HaveLen(3)) Expect(inRange(ipv4TestRange, secondaryIfaceIPs[0])).To(Succeed()) @@ -225,8 +269,7 @@ var _ = Describe("Whereabouts functionality", func() { By("creating a replicaset with whereabouts net-attach-def") var err error - const ipPoolNamespace = "kube-system" - k8sIPAM, err = wbstorage.NewKubernetesIPAMWithNamespace("", types.IPAMConfig{ + k8sIPAM, err = wbstorage.NewKubernetesIPAMWithNamespace("", "", types.IPAMConfig{ Kubernetes: types.KubernetesConfig{ KubeConfigPath: testConfig.KubeconfigPath, }, @@ -296,7 +339,6 @@ var _ = Describe("Whereabouts functionality", func() { Context("stateful set tests", func() { const ( initialReplicaNumber = 20 - ipPoolNamespace = "kube-system" namespace = "default" serviceName = "web" selector = "app=" + serviceName @@ -484,10 +526,137 @@ var _ = Describe("Whereabouts functionality", func() { Expect(err).NotTo(HaveOccurred()) Expect(ipPool.Spec.Allocations).NotTo(BeEmpty()) - Expect(allocationForPodRef(podRef, *ipPool).ContainerID).NotTo(Equal(containerID)) + Expect(allocationForPodRef(podRef, *ipPool)[0].ContainerID).NotTo(Equal(containerID)) + Expect(allocationForPodRef(podRef, *ipPool)[0].PodRef).To(Equal(podRef)) }) }) }) + + Context("reclaim previously allocated IP", func() { + const ( + namespace = "default" + networkName = "recovernet" + rangeWithTwoIPs = "10.10.0.0/30" + replicaNumber = 1 + ) + + var podName string + var secondaryIPs []string + var ifNames = []string{"net1", "net2"} + + var tinyNetwork *nettypes.NetworkAttachmentDefinition + var originalAllocations []v1alpha1.IPAllocation + var originalClusterWideAllocations []*v1alpha1.OverlappingRangeIPReservation + + BeforeEach(func() { + var err error + + podName = fmt.Sprintf("%s-0", serviceName) + + tinyNetwork, err = clientInfo.AddNetAttachDef( + macvlanNetworkWithWhereaboutsIPAMNetwork(networkName, namespace, rangeWithTwoIPs, []string{}, wbstorage.UnnamedNetwork, true)) + Expect(err).NotTo(HaveOccurred()) + + // Request 2 interfaces. + _, err = clientInfo.ProvisionStatefulSet(statefulSetName, namespace, serviceName, replicaNumber, networkName, networkName) + Expect(err).NotTo(HaveOccurred()) + + By("getting pod info") + pod, err := clientInfo.Client.CoreV1().Pods(namespace).Get(context.Background(), podName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying allocation") + for _, ifName := range ifNames { + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, ifName) + Expect(err).NotTo(HaveOccurred()) + + for _, ip := range secondaryIfaceIPs { + verifyAllocations(clientInfo, rangeWithTwoIPs, ip, namespace, podName, ifName) + } + secondaryIPs = append(secondaryIPs, secondaryIfaceIPs...) + } + + By("saving initial allocations") + ipPool, err := clientInfo.WbClient.WhereaboutsV1alpha1().IPPools(ipPoolNamespace).Get(context.Background(), wbstorage.IPPoolName(wbstorage.PoolIdentifier{IpRange: rangeWithTwoIPs, NetworkName: wbstorage.UnnamedNetwork}), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + originalAllocations = allocationForPodRef(getPodRef(namespace, podName), *ipPool) + Expect(originalAllocations).To(HaveLen(2)) + + for _, ip := range secondaryIPs { + overlapping, err := clientInfo.WbClient.WhereaboutsV1alpha1().OverlappingRangeIPReservations(ipPoolNamespace).Get(context.Background(), wbstorage.NormalizeIP(net.ParseIP(ip), wbstorage.UnnamedNetwork), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + originalClusterWideAllocations = append(originalClusterWideAllocations, overlapping) + } + }) + + AfterEach(func() { + Expect(clientInfo.DelNetAttachDef(tinyNetwork)).To(Succeed()) + Expect(clientInfo.DeleteStatefulSet(namespace, serviceName, selector)).To(Succeed()) + }) + + It("can reclaim the previously allocated IPs", func() { + By("checking that the IP allocation is removed when the pod is deleted") + Expect(clientInfo.ScaleStatefulSet(serviceName, namespace, -1)).To(Succeed()) + verifyNoAllocationsForPodRef(clientInfo, rangeWithTwoIPs, namespace, podName, secondaryIPs) + + By("adding previous allocations") + ipPool, err := clientInfo.WbClient.WhereaboutsV1alpha1().IPPools(ipPoolNamespace).Get(context.Background(), wbstorage.IPPoolName(wbstorage.PoolIdentifier{IpRange: rangeWithTwoIPs, NetworkName: wbstorage.UnnamedNetwork}), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + updatedPool := ipPool.DeepCopy() + for i, ip := range secondaryIPs { + firstIP, _, err := net.ParseCIDR(ipv4TestRange) + Expect(err).NotTo(HaveOccurred()) + offset, err := iphelpers.IPGetOffset(net.ParseIP(ip), firstIP) + Expect(err).NotTo(HaveOccurred()) + + updatedPool.Spec.Allocations[fmt.Sprintf("%d", offset)] = originalAllocations[i] + } + + _, err = clientInfo.WbClient.WhereaboutsV1alpha1().IPPools(ipPoolNamespace).Update(context.Background(), updatedPool, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + for _, allocation := range originalClusterWideAllocations { + allocation.ResourceVersion = "" + _, err := clientInfo.WbClient.WhereaboutsV1alpha1().OverlappingRangeIPReservations(ipPoolNamespace).Create(context.Background(), allocation, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + } + + By("increasing replica count") + Expect(clientInfo.ScaleStatefulSet(serviceName, namespace, 1)).To(Succeed()) + err = wbtestclient.WaitForStatefulSetCondition(context.Background(), clientInfo.Client, namespace, serviceName, replicaNumber, 1*time.Minute, wbtestclient.IsStatefulSetReadyPredicate) + Expect(err).NotTo(HaveOccurred()) + + By("getting pod info") + pod, err := clientInfo.Client.CoreV1().Pods(namespace).Get(context.Background(), podName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying allocation") + for _, ifName := range ifNames { + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, ifName) + Expect(err).NotTo(HaveOccurred()) + + for _, ip := range secondaryIfaceIPs { + verifyAllocations(clientInfo, rangeWithTwoIPs, ip, namespace, podName, ifName) + } + secondaryIPs = append(secondaryIPs, secondaryIfaceIPs...) + } + + By("comparing with previous allocations") + ipPool, err = clientInfo.WbClient.WhereaboutsV1alpha1().IPPools(ipPoolNamespace).Get(context.Background(), wbstorage.IPPoolName(wbstorage.PoolIdentifier{IpRange: rangeWithTwoIPs, NetworkName: wbstorage.UnnamedNetwork}), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + currentAllocation := allocationForPodRef(getPodRef(namespace, podName), *ipPool) + Expect(currentAllocation).To(HaveLen(2)) + + for i, allocation := range currentAllocation { + Expect(allocation.ContainerID).ToNot(Equal(originalAllocations[i].ContainerID)) + Expect(allocation.IfName).To(Equal(originalAllocations[i].IfName)) + Expect(allocation.PodRef).To(Equal(originalAllocations[i].PodRef)) + } + }) + }) }) Context("OverlappingRangeIPReservation", func() { @@ -550,13 +719,14 @@ var _ = Describe("Whereabouts functionality", func() { }) It("allocates the correct IP address to the second pod", func() { + ifName := "net1" By("checking pod IP is within whereabouts IPAM range") - secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod) + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, ifName) Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs).NotTo(BeEmpty()) By("checking pod 2 IP is within whereabouts IPAM range") - secondaryIfaceIPs2, err := retrievers.SecondaryIfaceIPValue(pod2) + secondaryIfaceIPs2, err := retrievers.SecondaryIfaceIPValue(pod2, ifName) Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs2).NotTo(BeEmpty()) @@ -655,19 +825,20 @@ var _ = Describe("Whereabouts functionality", func() { }) It("allocates the same IP to the Pods as they are in different address collision domains", func() { + ifName := "net1" By("checking pod IP is within whereabouts IPAM range") - secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod) + secondaryIfaceIPs, err := retrievers.SecondaryIfaceIPValue(pod, ifName) Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs).NotTo(BeEmpty()) By("checking pod 2 IP is within whereabouts IPAM range and has the same IP as pod 1") - secondaryIfaceIPs2, err := retrievers.SecondaryIfaceIPValue(pod2) + secondaryIfaceIPs2, err := retrievers.SecondaryIfaceIPValue(pod2, ifName) Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs2).NotTo(BeEmpty()) Expect(secondaryIfaceIPs[0]).To(Equal(secondaryIfaceIPs2[0])) By("checking pod 3 IP is within whereabouts IPAM range and has a different IP from pod 2") - secondaryIfaceIPs3, err := retrievers.SecondaryIfaceIPValue(pod3) + secondaryIfaceIPs3, err := retrievers.SecondaryIfaceIPValue(pod3, ifName) Expect(err).NotTo(HaveOccurred()) Expect(secondaryIfaceIPs3).NotTo(BeEmpty()) Expect(secondaryIfaceIPs2[0]).NotTo(Equal(secondaryIfaceIPs3[0])) @@ -676,13 +847,55 @@ var _ = Describe("Whereabouts functionality", func() { }) }) -func allocationForPodRef(podRef string, ipPool v1alpha1.IPPool) *v1alpha1.IPAllocation { +func verifyNoAllocationsForPodRef(clientInfo *wbtestclient.ClientInfo, ipv4TestRange, testNamespace, podName string, secondaryIfaceIPs []string) { + Eventually(func() bool { + ipPool, err := clientInfo.WbClient.WhereaboutsV1alpha1().IPPools(ipPoolNamespace).Get(context.Background(), wbstorage.IPPoolName(wbstorage.PoolIdentifier{IpRange: ipv4TestRange, NetworkName: wbstorage.UnnamedNetwork}), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + allocation := allocationForPodRef(getPodRef(testNamespace, podName), *ipPool) + return len(allocation) == 0 + }, 3*time.Second, 500*time.Millisecond).Should(BeTrue()) + + for _, ip := range secondaryIfaceIPs { + Eventually(func() bool { + _, err := clientInfo.WbClient.WhereaboutsV1alpha1().OverlappingRangeIPReservations(ipPoolNamespace).Get(context.Background(), wbstorage.NormalizeIP(net.ParseIP(ip), wbstorage.UnnamedNetwork), metav1.GetOptions{}) + if err != nil && errors.IsNotFound(err) { + return true + } + return false + }, 3*time.Second, 500*time.Millisecond).Should(BeTrue()) + } +} + +func verifyAllocations(clientInfo *wbtestclient.ClientInfo, ipv4TestRange, ip, testNamespace, podName, ifName string) { + ipPool, err := clientInfo.WbClient.WhereaboutsV1alpha1().IPPools(ipPoolNamespace).Get(context.Background(), wbstorage.IPPoolName(wbstorage.PoolIdentifier{IpRange: ipv4TestRange, NetworkName: wbstorage.UnnamedNetwork}), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + firstIP, _, err := net.ParseCIDR(ipv4TestRange) + Expect(err).NotTo(HaveOccurred()) + offset, err := iphelpers.IPGetOffset(net.ParseIP(ip), firstIP) + Expect(err).NotTo(HaveOccurred()) + + allocation, ok := ipPool.Spec.Allocations[fmt.Sprintf("%d", offset)] + Expect(ok).To(BeTrue()) + Expect(allocation.PodRef).To(Equal(getPodRef(testNamespace, podName))) + Expect(allocation.IfName).To(Equal(ifName)) + + overlapping, err := clientInfo.WbClient.WhereaboutsV1alpha1().OverlappingRangeIPReservations(ipPoolNamespace).Get(context.Background(), wbstorage.NormalizeIP(net.ParseIP(ip), wbstorage.UnnamedNetwork), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(overlapping.Spec.IfName).To(Equal(ifName)) + Expect(overlapping.Spec.PodRef).To(Equal(getPodRef(testNamespace, podName))) +} + +func allocationForPodRef(podRef string, ipPool v1alpha1.IPPool) []v1alpha1.IPAllocation { + var allocations []v1alpha1.IPAllocation for _, allocation := range ipPool.Spec.Allocations { if allocation.PodRef == podRef { - return &allocation + allocations = append(allocations, allocation) } } - return nil + return allocations } func clusterConfig() (*rest.Config, error) { @@ -759,25 +972,21 @@ func macvlanNetworkWithWhereaboutsIPAMNetwork(networkName string, namespaceName macvlanConfig := fmt.Sprintf(`{ "cniVersion": "0.3.0", "disableCheck": true, - "plugins": [ - { - "type": "macvlan", - "master": "eth0", - "mode": "bridge", - "ipam": { - "type": "whereabouts", - "leader_lease_duration": 1500, - "leader_renew_deadline": 1000, - "leader_retry_period": 500, - "range": "%s", - "ipRanges": %s, - "log_level": "debug", - "log_file": "/tmp/wb", - "network_name": "%s", - "enable_overlapping_ranges": %v - } - } - ] + "type": "macvlan", + "master": "eth0", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "leader_lease_duration": 1500, + "leader_renew_deadline": 1000, + "leader_retry_period": 500, + "range": "%s", + "ipRanges": %s, + "log_level": "debug", + "log_file": "/tmp/wb", + "network_name": "%s", + "enable_overlapping_ranges": %v + } }`, ipRange, createIPRanges(ipRanges), poolName, enableOverlappingRanges) return generateNetAttachDefSpec(networkName, namespaceName, macvlanConfig) } @@ -804,3 +1013,7 @@ func createIPRanges(ranges []string) string { ipRanges := "[" + strings.Join(formattedRanges[:], ",") + "]" return ipRanges } + +func getPodRef(namespace, name string) string { + return fmt.Sprintf("%s/%s", namespace, name) +} diff --git a/e2e/poolconsistency/checker.go b/e2e/poolconsistency/checker.go index fbc202b3a..fca9dca1c 100644 --- a/e2e/poolconsistency/checker.go +++ b/e2e/poolconsistency/checker.go @@ -22,7 +22,7 @@ func NewPoolConsistencyCheck(ipPool storage.IPPool, podList []corev1.Pod) *Check func (pc *Checker) MissingIPs() []string { var mismatchedIPs []string for _, pod := range pc.podList { - podIPs, err := retrievers.SecondaryIfaceIPValue(&pod) + podIPs, err := retrievers.SecondaryIfaceIPValue(&pod, "net1") podIP := podIPs[len(podIPs)-1] if err != nil { return []string{} @@ -51,7 +51,7 @@ func (pc *Checker) StaleIPs() []string { reservedIP := allocation.IP.String() found := false for _, pod := range pc.podList { - podIPs, err := retrievers.SecondaryIfaceIPValue(&pod) + podIPs, err := retrievers.SecondaryIfaceIPValue(&pod, "net1") podIP := podIPs[len(podIPs)-1] if err != nil { continue diff --git a/e2e/poolconsistency/poolconsistency_test.go b/e2e/poolconsistency/poolconsistency_test.go index 352813909..de1c3f47b 100644 --- a/e2e/poolconsistency/poolconsistency_test.go +++ b/e2e/poolconsistency/poolconsistency_test.go @@ -53,6 +53,7 @@ var _ = Describe("IP Pool consistency checker", func() { pool = NewMockedPool(types.IPReservation{ IP: net.ParseIP(ip), ContainerID: "abc", + IfName: "eth0", PodRef: "cba", IsAllocated: true, }) diff --git a/e2e/retrievers/pod.go b/e2e/retrievers/pod.go index 6a266ca20..d8a2307a3 100644 --- a/e2e/retrievers/pod.go +++ b/e2e/retrievers/pod.go @@ -19,7 +19,7 @@ func filterNetworkStatus( return nil } -func SecondaryIfaceIPValue(pod *core.Pod) ([]string, error) { +func SecondaryIfaceIPValue(pod *core.Pod, ifName string) ([]string, error) { podNetStatus, found := pod.Annotations[nettypes.NetworkStatusAnnot] if !found { return nil, fmt.Errorf("the pod must feature the `networks-status` annotation") @@ -31,7 +31,7 @@ func SecondaryIfaceIPValue(pod *core.Pod) ([]string, error) { } secondaryInterfaceNetworkStatus := filterNetworkStatus(netStatus, func(status nettypes.NetworkStatus) bool { - return status.Interface == "net1" + return status.Interface == ifName }) if secondaryInterfaceNetworkStatus == nil { diff --git a/pkg/allocate/allocate.go b/pkg/allocate/allocate.go index c7d596343..abd913465 100644 --- a/pkg/allocate/allocate.go +++ b/pkg/allocate/allocate.go @@ -23,12 +23,25 @@ func (a AssignmentError) Error() string { } // AssignIP assigns an IP using a range and a reserve list. -func AssignIP(ipamConf types.RangeConfiguration, reservelist []types.IPReservation, containerID string, podRef string) (net.IPNet, []types.IPReservation, error) { +func AssignIP(ipamConf types.RangeConfiguration, reservelist []types.IPReservation, containerID, podRef, ifName string) (net.IPNet, []types.IPReservation, error) { // Setup the basics here. _, ipnet, _ := net.ParseCIDR(ipamConf.Range) - newip, updatedreservelist, err := IterateForAssignment(*ipnet, ipamConf.RangeStart, ipamConf.RangeEnd, reservelist, ipamConf.OmitRanges, containerID, podRef) + // Verify if podRef and ifName have already an allocation. + for i, r := range reservelist { + if r.PodRef == podRef && r.IfName == ifName { + logging.Debugf("IP already allocated for podRef: %q - ifName:%q - IP: %s", podRef, ifName, r.IP.String()) + if r.ContainerID != containerID { + logging.Debugf("updating container ID: %q", containerID) + reservelist[i].ContainerID = containerID + } + + return net.IPNet{IP: r.IP, Mask: ipnet.Mask}, reservelist, nil + } + } + + newip, updatedreservelist, err := IterateForAssignment(*ipnet, ipamConf.RangeStart, ipamConf.RangeEnd, reservelist, ipamConf.OmitRanges, containerID, podRef, ifName) if err != nil { return net.IPNet{}, nil, err } @@ -70,7 +83,7 @@ func removeIdxFromSlice(s []types.IPReservation, i int) []types.IPReservation { // If rangeEnd is specified, it is respected if it lies within the ipnet and if it is >= rangeStart. // reserveList holds a list of reserved IPs. // excludeRanges holds a list of subnets to be excluded (meaning the full subnet, including the network and broadcast IP). -func IterateForAssignment(ipnet net.IPNet, rangeStart net.IP, rangeEnd net.IP, reserveList []types.IPReservation, excludeRanges []string, containerID string, podRef string) (net.IP, []types.IPReservation, error) { +func IterateForAssignment(ipnet net.IPNet, rangeStart net.IP, rangeEnd net.IP, reserveList []types.IPReservation, excludeRanges []string, containerID, podRef, ifName string) (net.IP, []types.IPReservation, error) { // Get the valid range, delimited by the ipnet's first and last usable IP as well as the rangeStart and rangeEnd. firstIP, lastIP, err := iphelpers.GetIPRange(ipnet, rangeStart, rangeEnd) if err != nil { @@ -110,8 +123,8 @@ func IterateForAssignment(ipnet net.IPNet, rangeStart net.IP, rangeEnd net.IP, r continue } // Assign and reserve the IP and return. - logging.Debugf("Reserving IP: |%v|", ip.String()+" "+containerID) - reserveList = append(reserveList, types.IPReservation{IP: ip, ContainerID: containerID, PodRef: podRef}) + logging.Debugf("Reserving IP: %q - container ID %q - podRef: %q - ifName: %q", ip.String(), containerID, podRef, ifName) + reserveList = append(reserveList, types.IPReservation{IP: ip, ContainerID: containerID, PodRef: podRef, IfName: ifName}) return ip, reserveList, nil } diff --git a/pkg/allocate/allocate_test.go b/pkg/allocate/allocate_test.go index d88ed9696..d59c59e60 100644 --- a/pkg/allocate/allocate_test.go +++ b/pkg/allocate/allocate_test.go @@ -27,7 +27,7 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation var exrange []string - newip, _, err := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + newip, _, err := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(err).NotTo(HaveOccurred()) Expect(fmt.Sprint(newip)).To(Equal("192.168.1.1")) @@ -43,7 +43,7 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation var exrange []string - newip, _, err := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + newip, _, err := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(err).NotTo(HaveOccurred()) Expect(fmt.Sprint(newip)).To(Equal("caa5::1")) @@ -59,7 +59,7 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation var exrange []string - newip, _, err := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + newip, _, err := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(err).NotTo(HaveOccurred()) Expect(fmt.Sprint(newip)).To(Equal("::1")) @@ -77,7 +77,7 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation var exrange []string - newip, _, err := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + newip, _, err := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(err).NotTo(HaveOccurred()) Expect(fmt.Sprint(newip)).To(Equal("fd::1")) @@ -93,7 +93,7 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation var exrange []string - newip, _, err := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + newip, _, err := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(err).NotTo(HaveOccurred()) Expect(fmt.Sprint(newip)).To(Equal("100::2:1")) }) @@ -108,7 +108,7 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation exrange := []string{"192.168.0.0/30"} - newip, _, _ := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + newip, _, _ := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(fmt.Sprint(newip)).To(Equal("192.168.0.4")) }) @@ -122,7 +122,7 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation exrange := []string{"192.168.0.1"} - newip, _, err := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + newip, _, err := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(err).NotTo(HaveOccurred()) Expect(fmt.Sprint(newip)).To(Equal("192.168.0.2")) }) @@ -136,7 +136,7 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation exrange := []string{"192.168.0.1/123"} - _, _, err = IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + _, _, err = IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(err).To(MatchError(HavePrefix("could not parse exclude range"))) }) @@ -150,7 +150,7 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation exrange := []string{"100::2:1/126"} - newip, _, _ := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + newip, _, _ := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(fmt.Sprint(newip)).To(Equal("100::2:4")) }) @@ -164,7 +164,7 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation exrange := []string{"100::2:1"} - newip, _, _ := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + newip, _, _ := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(fmt.Sprint(newip)).To(Equal("100::2:2")) }) @@ -177,7 +177,7 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation exrange := []string{"100::2::1"} - _, _, err = IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + _, _, err = IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(err).To(MatchError(HavePrefix("could not parse exclude range"))) }) @@ -191,7 +191,7 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation exrange := []string{"2001:db8::0/32"} - newip, _, _ := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + newip, _, _ := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(fmt.Sprint(newip)).To(Equal("2001:db9::")) }) @@ -206,11 +206,11 @@ var _ = Describe("Allocation operations", func() { var ipres []types.IPReservation exrange := []string{"192.168.0.0/30", "192.168.0.6/31", "192.168.0.8/31", "192.168.0.4/30"} - newip, _, _ := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + newip, _, _ := IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(fmt.Sprint(newip)).To(Equal("192.168.0.10")) exrange = []string{"192.168.0.0/30", "192.168.0.14/31", "192.168.0.4/30", "192.168.0.6/31", "192.168.0.8/31"} - newip, _, _ = IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "") + newip, _, _ = IterateForAssignment(*ipnet, calculatedrangestart, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(fmt.Sprint(newip)).To(Equal("192.168.0.10")) }) @@ -234,7 +234,7 @@ var _ = Describe("Allocation operations", func() { }, } exrange := []string{"192.168.0.0/30"} - _, _, err = IterateForAssignment(*ipnet, firstip, nil, ipres, exrange, "0xdeadbeef", "") + _, _, err = IterateForAssignment(*ipnet, firstip, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(err).To(MatchError(HavePrefix("Could not allocate IP in range"))) }) @@ -258,7 +258,7 @@ var _ = Describe("Allocation operations", func() { }, } exrange := []string{"192.168.0.4/30"} - _, _, err = IterateForAssignment(*ipnet, firstip, nil, ipres, exrange, "0xdeadbeef", "") + _, _, err = IterateForAssignment(*ipnet, firstip, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(err).To(MatchError(HavePrefix("Could not allocate IP in range"))) }) @@ -284,7 +284,7 @@ var _ = Describe("Allocation operations", func() { } exrange := []string{"100::2:4/126"} - _, _, err = IterateForAssignment(*ipnet, firstip, nil, ipres, exrange, "0xdeadbeef", "") + _, _, err = IterateForAssignment(*ipnet, firstip, nil, ipres, exrange, "0xdeadbeef", "", "") Expect(err).To(MatchError(HavePrefix("Could not allocate IP in range"))) }) @@ -297,7 +297,7 @@ var _ = Describe("Allocation operations", func() { _, ipnet, err := net.ParseCIDR("192.168.0.0/29") Expect(err).NotTo(HaveOccurred()) rangeStart := net.ParseIP("192.168.0.0") // Network address, out of bounds. - newip, _, err := IterateForAssignment(*ipnet, rangeStart, nil, nil, nil, "0xdeadbeef", "") + newip, _, err := IterateForAssignment(*ipnet, rangeStart, nil, nil, nil, "0xdeadbeef", "", "") Expect(err).NotTo(HaveOccurred()) Expect(fmt.Sprint(newip)).To(Equal("192.168.0.1")) }) @@ -309,7 +309,7 @@ var _ = Describe("Allocation operations", func() { Expect(err).NotTo(HaveOccurred()) rangeStart := net.ParseIP("192.168.0.0") // Network address, out of bounds. rangeEnd := net.ParseIP("192.168.0.8") // Broadcast address, out of bounds. - newip, _, err := IterateForAssignment(*ipnet, rangeStart, rangeEnd, nil, nil, "0xdeadbeef", "") + newip, _, err := IterateForAssignment(*ipnet, rangeStart, rangeEnd, nil, nil, "0xdeadbeef", "", "") Expect(err).NotTo(HaveOccurred()) Expect(fmt.Sprint(newip)).To(Equal("192.168.0.1")) }) @@ -337,7 +337,7 @@ var _ = Describe("Allocation operations", func() { }, } exrange := []string{"192.168.0.4/30"} - _, _, err = IterateForAssignment(*ipnet, startip, lastip, ipres, exrange, "0xdeadbeef", "") + _, _, err = IterateForAssignment(*ipnet, startip, lastip, ipres, exrange, "0xdeadbeef", "", "") Expect(err).To(MatchError(HavePrefix("Could not allocate IP in range"))) }) @@ -350,7 +350,7 @@ var _ = Describe("Allocation operations", func() { lastip := net.ParseIP("192.168.0.6") ipres := []types.IPReservation{} - _, ipres, err = IterateForAssignment(*ipnet, startip, lastip, ipres, nil, "0xdeadbeef", "dummy-0") + _, ipres, err = IterateForAssignment(*ipnet, startip, lastip, ipres, nil, "0xdeadbeef", "dummy-0", "") Expect(err).NotTo(HaveOccurred()) Expect(len(ipres)).To(Equal(1)) Expect(fmt.Sprint(ipres[0].IP)).To(Equal("192.168.0.1")) @@ -379,7 +379,7 @@ var _ = Describe("Allocation operations", func() { }, } - _, ipres, err = IterateForAssignment(*ipnet, startip, lastip, ipres, nil, "0xdeadbeef", "dummy-0") + _, ipres, err = IterateForAssignment(*ipnet, startip, lastip, ipres, nil, "0xdeadbeef", "dummy-0", "") Expect(err).NotTo(HaveOccurred()) Expect(len(ipres)).To(Equal(4)) Expect(fmt.Sprint(ipres[3].IP)).To(Equal("192.168.0.4")) @@ -408,7 +408,7 @@ var _ = Describe("Allocation operations", func() { }, } - _, ipres, err = IterateForAssignment(*ipnet, startip, lastip, ipres, nil, "0xdeadbeef", "dummy-0") + _, ipres, err = IterateForAssignment(*ipnet, startip, lastip, ipres, nil, "0xdeadbeef", "dummy-0", "") Expect(err).NotTo(HaveOccurred()) Expect(len(ipres)).To(Equal(4)) Expect(fmt.Sprint(ipres[3].IP)).To(Equal("192.168.0.3")) diff --git a/pkg/api/whereabouts.cni.cncf.io/v1alpha1/ippool_types.go b/pkg/api/whereabouts.cni.cncf.io/v1alpha1/ippool_types.go index 7bc391e60..cc86dad30 100644 --- a/pkg/api/whereabouts.cni.cncf.io/v1alpha1/ippool_types.go +++ b/pkg/api/whereabouts.cni.cncf.io/v1alpha1/ippool_types.go @@ -23,7 +23,8 @@ func (i IPPool) ParseCIDR() (net.IP, *net.IPNet, error) { // IPAllocation represents metadata about the pod/container owner of a specific IP type IPAllocation struct { ContainerID string `json:"id"` - PodRef string `json:"podref,omitempty"` + PodRef string `json:"podref"` + IfName string `json:"ifname,omitempty"` } // +genclient diff --git a/pkg/api/whereabouts.cni.cncf.io/v1alpha1/overlappingrangeipreservation_types.go b/pkg/api/whereabouts.cni.cncf.io/v1alpha1/overlappingrangeipreservation_types.go index cab76cce1..438677041 100644 --- a/pkg/api/whereabouts.cni.cncf.io/v1alpha1/overlappingrangeipreservation_types.go +++ b/pkg/api/whereabouts.cni.cncf.io/v1alpha1/overlappingrangeipreservation_types.go @@ -4,8 +4,9 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // OverlappingRangeIPReservationSpec defines the desired state of OverlappingRangeIPReservation type OverlappingRangeIPReservationSpec struct { - ContainerID string `json:"containerid"` - PodRef string `json:"podref,omitempty"` + ContainerID string `json:"containerid,omitempty"` + PodRef string `json:"podref"` + IfName string `json:"ifname,omitempty"` } // +genclient diff --git a/pkg/api/whereabouts.cni.cncf.io/v1alpha1/zz_generated.deepcopy.go b/pkg/api/whereabouts.cni.cncf.io/v1alpha1/zz_generated.deepcopy.go index 18d1e4b1f..9e7da987b 100644 --- a/pkg/api/whereabouts.cni.cncf.io/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/whereabouts.cni.cncf.io/v1alpha1/zz_generated.deepcopy.go @@ -1,5 +1,4 @@ //go:build !ignore_autogenerated -// +build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT. diff --git a/pkg/storage/kubernetes/ipam.go b/pkg/storage/kubernetes/ipam.go index 32f043dcf..02a9342c5 100644 --- a/pkg/storage/kubernetes/ipam.go +++ b/pkg/storage/kubernetes/ipam.go @@ -29,8 +29,29 @@ import ( "gomodules.xyz/jsonpatch/v2" ) +const UnnamedNetwork string = "" + +// KubernetesIPAM manages ip blocks in an kubernetes CRD backend +type KubernetesIPAM struct { + Client + Config whereaboutstypes.IPAMConfig + namespace string + containerID string + IfName string +} + +func newKubernetesIPAM(containerID, ifName string, ipamConf whereaboutstypes.IPAMConfig, namespace string, kubernetesClient Client) *KubernetesIPAM { + return &KubernetesIPAM{ + Config: ipamConf, + containerID: containerID, + IfName: ifName, + namespace: namespace, + Client: kubernetesClient, + } +} + // NewKubernetesIPAM returns a new KubernetesIPAM Client configured to a kubernetes CRD backend -func NewKubernetesIPAM(containerID string, ipamConf whereaboutstypes.IPAMConfig) (*KubernetesIPAM, error) { +func NewKubernetesIPAM(containerID, ifName string, ipamConf whereaboutstypes.IPAMConfig) (*KubernetesIPAM, error) { var namespace string if cfg, err := clientcmd.LoadFromFile(ipamConf.Kubernetes.KubeConfigPath); err != nil { return nil, err @@ -44,13 +65,13 @@ func NewKubernetesIPAM(containerID string, ipamConf whereaboutstypes.IPAMConfig) if err != nil { return nil, fmt.Errorf("failed instantiating kubernetes client: %v", err) } - k8sIPAM := newKubernetesIPAM(containerID, ipamConf, namespace, *kubernetesClient) + k8sIPAM := newKubernetesIPAM(containerID, ifName, ipamConf, namespace, *kubernetesClient) return k8sIPAM, nil } // NewKubernetesIPAMWithNamespace returns a new KubernetesIPAM Client configured to a kubernetes CRD backend -func NewKubernetesIPAMWithNamespace(containerID string, ipamConf whereaboutstypes.IPAMConfig, namespace string) (*KubernetesIPAM, error) { - k8sIPAM, err := NewKubernetesIPAM(containerID, ipamConf) +func NewKubernetesIPAMWithNamespace(containerID, ifName string, ipamConf whereaboutstypes.IPAMConfig, namespace string) (*KubernetesIPAM, error) { + k8sIPAM, err := NewKubernetesIPAM(containerID, ifName, ipamConf) if err != nil { return nil, err } @@ -58,58 +79,11 @@ func NewKubernetesIPAMWithNamespace(containerID string, ipamConf whereaboutstype return k8sIPAM, nil } -func newKubernetesIPAM(containerID string, ipamConf whereaboutstypes.IPAMConfig, namespace string, kubernetesClient Client) *KubernetesIPAM { - return &KubernetesIPAM{ - Config: ipamConf, - containerID: containerID, - namespace: namespace, - Client: kubernetesClient, - } -} - -// KubernetesIPAM manages ip blocks in an kubernetes CRD backend -type KubernetesIPAM struct { - Client - Config whereaboutstypes.IPAMConfig - containerID string - namespace string -} - -const UnnamedNetwork string = "" - type PoolIdentifier struct { IpRange string NetworkName string } -func toIPReservationList(allocations map[string]whereaboutsv1alpha1.IPAllocation, firstip net.IP) []whereaboutstypes.IPReservation { - reservelist := []whereaboutstypes.IPReservation{} - for offset, a := range allocations { - numOffset, err := strconv.ParseInt(offset, 10, 64) - if err != nil { - // allocations that are invalid int64s should be ignored - // toAllocationMap should be the only writer of offsets, via `fmt.Sprintf("%d", ...)`` - logging.Errorf("Error decoding ip offset (backend: kubernetes): %v", err) - continue - } - ip := iphelpers.IPAddOffset(firstip, uint64(numOffset)) - reservelist = append(reservelist, whereaboutstypes.IPReservation{IP: ip, ContainerID: a.ContainerID, PodRef: a.PodRef}) - } - return reservelist -} - -func toAllocationMap(reservelist []whereaboutstypes.IPReservation, firstip net.IP) (map[string]whereaboutsv1alpha1.IPAllocation, error) { - allocations := make(map[string]whereaboutsv1alpha1.IPAllocation) - for _, r := range reservelist { - index, err := iphelpers.IPGetOffset(r.IP, firstip) - if err != nil { - return nil, err - } - allocations[fmt.Sprintf("%d", index)] = whereaboutsv1alpha1.IPAllocation{ContainerID: r.ContainerID, PodRef: r.PodRef} - } - return allocations, nil -} - // GetIPPool returns a storage.IPPool for the given range func (i *KubernetesIPAM) GetIPPool(ctx context.Context, poolIdentifier PoolIdentifier) (storage.IPPool, error) { name := IPPoolName(poolIdentifier) @@ -124,27 +98,7 @@ func (i *KubernetesIPAM) GetIPPool(ctx context.Context, poolIdentifier PoolIdent return nil, err } - return &KubernetesIPPool{i.client, i.containerID, firstIP, pool}, nil -} - -func IPPoolName(poolIdentifier PoolIdentifier) string { - if poolIdentifier.NetworkName == UnnamedNetwork { - return normalizeRange(poolIdentifier.IpRange) - } else { - return fmt.Sprintf("%s-%s", poolIdentifier.NetworkName, normalizeRange(poolIdentifier.IpRange)) - } -} - -func normalizeRange(ipRange string) string { - // v6 filter - if ipRange[len(ipRange)-1] == ':' { - ipRange = ipRange + "0" - } - normalized := strings.ReplaceAll(ipRange, ":", "-") - - // replace subnet cidr slash - normalized = strings.ReplaceAll(normalized, "/", "-") - return normalized + return &KubernetesIPPool{i.client, firstIP, pool}, nil } func (i *KubernetesIPAM) getPool(ctx context.Context, name string, iprange string) (*whereaboutsv1alpha1.IPPool, error) { @@ -174,6 +128,26 @@ func (i *KubernetesIPAM) getPool(ctx context.Context, name string, iprange strin return pool, nil } +func IPPoolName(poolIdentifier PoolIdentifier) string { + if poolIdentifier.NetworkName == UnnamedNetwork { + return normalizeRange(poolIdentifier.IpRange) + } else { + return fmt.Sprintf("%s-%s", poolIdentifier.NetworkName, normalizeRange(poolIdentifier.IpRange)) + } +} + +func normalizeRange(ipRange string) string { + // v6 filter + if ipRange[len(ipRange)-1] == ':' { + ipRange = ipRange + "0" + } + normalized := strings.ReplaceAll(ipRange, ":", "-") + + // replace subnet cidr slash + normalized = strings.ReplaceAll(normalized, "/", "-") + return normalized +} + // Status tests connectivity to the kubernetes backend func (i *KubernetesIPAM) Status(ctx context.Context) error { _, err := i.client.WhereaboutsV1alpha1().IPPools(i.namespace).List(ctx, metav1.ListOptions{}) @@ -185,46 +159,142 @@ func (i *KubernetesIPAM) Close() error { return nil } +// KubernetesIPPool represents an IPPool resource and its parsed set of allocations +type KubernetesIPPool struct { + client wbclient.Interface + firstIP net.IP + pool *whereaboutsv1alpha1.IPPool +} + +// Allocations returns the initially retrieved set of allocations for this pool +func (p *KubernetesIPPool) Allocations() []whereaboutstypes.IPReservation { + return toIPReservationList(p.pool.Spec.Allocations, p.firstIP) +} + +// Update sets the pool allocated IP list to the given IP reservations +func (p *KubernetesIPPool) Update(ctx context.Context, reservations []whereaboutstypes.IPReservation) error { + // marshal the current pool to serve as the base for the patch creation + orig := p.pool.DeepCopy() + origBytes, err := json.Marshal(orig) + if err != nil { + return err + } + + // update the pool before marshalling once again + allocations, err := toAllocationMap(reservations, p.firstIP) + if err != nil { + return err + } + p.pool.Spec.Allocations = allocations + modBytes, err := json.Marshal(p.pool) + if err != nil { + return err + } + + // create the patch + patch, err := jsonpatch.CreatePatch(origBytes, modBytes) + if err != nil { + return err + } + + // add additional tests to the patch + ops := []jsonpatch.Operation{ + // ensure patch is applied to appropriate resource version only + {Operation: "test", Path: "/metadata/resourceVersion", Value: orig.ObjectMeta.ResourceVersion}, + } + for _, o := range patch { + // safeguard add ops -- "add" will update existing paths, this "test" ensures the path is empty + if o.Operation == "add" { + var m map[string]interface{} + ops = append(ops, jsonpatch.Operation{Operation: "test", Path: o.Path, Value: m}) + } + } + ops = append(ops, patch...) + patchData, err := json.Marshal(ops) + if err != nil { + return err + } + + // apply the patch + _, err = p.client.WhereaboutsV1alpha1().IPPools(orig.GetNamespace()).Patch(ctx, orig.GetName(), types.JSONPatchType, patchData, metav1.PatchOptions{}) + if err != nil { + if errors.IsInvalid(err) { + // expect "invalid" errors if any of the jsonpatch "test" Operations fail + return &temporaryError{err} + } + return err + } + + return nil +} + +func toIPReservationList(allocations map[string]whereaboutsv1alpha1.IPAllocation, firstip net.IP) []whereaboutstypes.IPReservation { + reservelist := []whereaboutstypes.IPReservation{} + for offset, a := range allocations { + numOffset, err := strconv.ParseInt(offset, 10, 64) + if err != nil { + // allocations that are invalid int64s should be ignored + // toAllocationMap should be the only writer of offsets, via `fmt.Sprintf("%d", ...)`` + logging.Errorf("Error decoding ip offset (backend: kubernetes): %v", err) + continue + } + ip := iphelpers.IPAddOffset(firstip, uint64(numOffset)) + reservelist = append(reservelist, whereaboutstypes.IPReservation{IP: ip, ContainerID: a.ContainerID, PodRef: a.PodRef, IfName: a.IfName}) + } + return reservelist +} + +func toAllocationMap(reservelist []whereaboutstypes.IPReservation, firstip net.IP) (map[string]whereaboutsv1alpha1.IPAllocation, error) { + allocations := make(map[string]whereaboutsv1alpha1.IPAllocation) + for _, r := range reservelist { + index, err := iphelpers.IPGetOffset(r.IP, firstip) + if err != nil { + return nil, err + } + allocations[fmt.Sprintf("%d", index)] = whereaboutsv1alpha1.IPAllocation{ContainerID: r.ContainerID, PodRef: r.PodRef, IfName: r.IfName} + } + return allocations, nil +} + // KubernetesOverlappingRangeStore represents a OverlappingRangeStore interface type KubernetesOverlappingRangeStore struct { - client wbclient.Interface - containerID string - namespace string + client wbclient.Interface + namespace string } // GetOverlappingRangeStore returns a clusterstore interface func (i *KubernetesIPAM) GetOverlappingRangeStore() (storage.OverlappingRangeStore, error) { - return &KubernetesOverlappingRangeStore{i.client, i.containerID, i.namespace}, nil + return &KubernetesOverlappingRangeStore{i.client, i.namespace}, nil } // IsAllocatedInOverlappingRange checks for IP addresses to see if they're allocated cluster wide, for overlapping -// ranges. -func (c *KubernetesOverlappingRangeStore) IsAllocatedInOverlappingRange(ctx context.Context, ip net.IP, - networkName string) (bool, error) { - normalizedIP := normalizeIP(ip, networkName) +// ranges. First return value is true if the IP is allocated, second return value is true if the IP is allocated to the +// current podRef +func (c *KubernetesOverlappingRangeStore) GetOverlappingRangeIPReservation(ctx context.Context, ip net.IP, + podRef, networkName string) (*whereaboutsv1alpha1.OverlappingRangeIPReservation, error) { + normalizedIP := NormalizeIP(ip, networkName) - logging.Debugf("OverlappingRangewide allocation check; normalized IP: %q, IP: %q, networkName: %q", + logging.Debugf("Get overlappingRangewide allocation; normalized IP: %q, IP: %q, networkName: %q", normalizedIP, ip, networkName) - _, err := c.client.WhereaboutsV1alpha1().OverlappingRangeIPReservations(c.namespace).Get(ctx, normalizedIP, metav1.GetOptions{}) + r, err := c.client.WhereaboutsV1alpha1().OverlappingRangeIPReservations(c.namespace).Get(ctx, normalizedIP, metav1.GetOptions{}) if err != nil && errors.IsNotFound(err) { // cluster ip reservation does not exist, this appears to be good news. - // logging.Debugf("IP %v is not reserved cluster wide, allowing.", ip) - return false, nil + return nil, nil } else if err != nil { logging.Errorf("k8s get OverlappingRangeIPReservation error: %s", err) - return false, fmt.Errorf("k8s get OverlappingRangeIPReservation error: %s", err) + return nil, fmt.Errorf("k8s get OverlappingRangeIPReservation error: %s", err) } logging.Debugf("Normalized IP is reserved; normalized IP: %q, IP: %q, networkName: %q", normalizedIP, ip, networkName) - return true, nil + return r, nil } // UpdateOverlappingRangeAllocation updates clusterwide allocation for overlapping ranges. func (c *KubernetesOverlappingRangeStore) UpdateOverlappingRangeAllocation(ctx context.Context, mode int, ip net.IP, - containerID, podRef, networkName string) error { - normalizedIP := normalizeIP(ip, networkName) + podRef, ifName, networkName string) error { + normalizedIP := NormalizeIP(ip, networkName) clusteripres := &whereaboutsv1alpha1.OverlappingRangeIPReservation{ ObjectMeta: metav1.ObjectMeta{Name: normalizedIP, Namespace: c.namespace}, @@ -238,8 +308,8 @@ func (c *KubernetesOverlappingRangeStore) UpdateOverlappingRangeAllocation(ctx c verb = "allocate" clusteripres.Spec = whereaboutsv1alpha1.OverlappingRangeIPReservationSpec{ - ContainerID: containerID, - PodRef: podRef, + PodRef: podRef, + IfName: ifName, } _, err = c.client.WhereaboutsV1alpha1().OverlappingRangeIPReservations(c.namespace).Create( @@ -258,9 +328,9 @@ func (c *KubernetesOverlappingRangeStore) UpdateOverlappingRangeAllocation(ctx c return nil } -// normalizeIP normalizes the IP. This is important for IPv6 which doesn't make for valid CR names. It also allows us +// NormalizeIP normalizes the IP. This is important for IPv6 which doesn't make for valid CR names. It also allows us // to add the network-name when it's different from the unnamed network. -func normalizeIP(ip net.IP, networkName string) string { +func NormalizeIP(ip net.IP, networkName string) string { ipStr := fmt.Sprint(ip) if ipStr[len(ipStr)-1] == ':' { ipStr += "0" @@ -273,76 +343,6 @@ func normalizeIP(ip net.IP, networkName string) string { return normalizedIP } -// KubernetesIPPool represents an IPPool resource and its parsed set of allocations -type KubernetesIPPool struct { - client wbclient.Interface - containerID string - firstIP net.IP - pool *whereaboutsv1alpha1.IPPool -} - -// Allocations returns the initially retrieved set of allocations for this pool -func (p *KubernetesIPPool) Allocations() []whereaboutstypes.IPReservation { - return toIPReservationList(p.pool.Spec.Allocations, p.firstIP) -} - -// Update sets the pool allocated IP list to the given IP reservations -func (p *KubernetesIPPool) Update(ctx context.Context, reservations []whereaboutstypes.IPReservation) error { - // marshal the current pool to serve as the base for the patch creation - orig := p.pool.DeepCopy() - origBytes, err := json.Marshal(orig) - if err != nil { - return err - } - - // update the pool before marshalling once again - allocations, err := toAllocationMap(reservations, p.firstIP) - if err != nil { - return err - } - p.pool.Spec.Allocations = allocations - modBytes, err := json.Marshal(p.pool) - if err != nil { - return err - } - - // create the patch - patch, err := jsonpatch.CreatePatch(origBytes, modBytes) - if err != nil { - return err - } - - // add additional tests to the patch - ops := []jsonpatch.Operation{ - // ensure patch is applied to appropriate resource version only - {Operation: "test", Path: "/metadata/resourceVersion", Value: orig.ObjectMeta.ResourceVersion}, - } - for _, o := range patch { - // safeguard add ops -- "add" will update existing paths, this "test" ensures the path is empty - if o.Operation == "add" { - var m map[string]interface{} - ops = append(ops, jsonpatch.Operation{Operation: "test", Path: o.Path, Value: m}) - } - } - ops = append(ops, patch...) - patchData, err := json.Marshal(ops) - if err != nil { - return err - } - - // apply the patch - _, err = p.client.WhereaboutsV1alpha1().IPPools(orig.GetNamespace()).Patch(ctx, orig.GetName(), types.JSONPatchType, patchData, metav1.PatchOptions{}) - if err != nil { - if errors.IsInvalid(err) { - // expect "invalid" errors if any of the jsonpatch "test" Operations fail - return &temporaryError{err} - } - return err - } - - return nil -} - // newLeaderElector creates a new leaderelection.LeaderElector and associated // channels by which to observe elections and depositions. func newLeaderElector(clientset kubernetes.Interface, namespace string, podNamespace string, podID string, leaseDuration int, renewDeadline int, retryPeriod int) (*leaderelection.LeaderElector, chan struct{}, chan struct{}) { @@ -419,7 +419,7 @@ func IPManagement(ctx context.Context, mode int, ipamConf whereaboutstypes.IPAMC return case <-leader: logging.Debugf("Elected as leader, do processing") - newips, err = IPManagementKubernetesUpdate(ctx, mode, client, ipamConf, client.containerID, ipamConf.GetPodRef()) + newips, err = IPManagementKubernetesUpdate(ctx, mode, client, ipamConf) stopM <- struct{}{} return case <-deposed: @@ -456,9 +456,8 @@ func IPManagement(ctx context.Context, mode int, ipamConf whereaboutstypes.IPAMC } // IPManagementKubernetesUpdate manages k8s updates -func IPManagementKubernetesUpdate(ctx context.Context, mode int, ipam *KubernetesIPAM, ipamConf whereaboutstypes.IPAMConfig, - containerID string, podRef string) ([]net.IPNet, error) { - logging.Debugf("IPManagement -- mode: %v / containerID: %v / podRef: %v", mode, containerID, podRef) +func IPManagementKubernetesUpdate(ctx context.Context, mode int, ipam *KubernetesIPAM, ipamConf whereaboutstypes.IPAMConfig) ([]net.IPNet, error) { + logging.Debugf("IPManagement -- mode: %d / containerID: %q / podRef: %q / ifName: %q ", mode, ipam.containerID, ipamConf.GetPodRef(), ipam.IfName) var newips []net.IPNet var newip net.IPNet @@ -485,6 +484,7 @@ func IPManagementKubernetesUpdate(ctx context.Context, mode int, ipam *Kubernete // handle the ip add/del until successful var overlappingrangeallocations []whereaboutstypes.IPReservation var ipforoverlappingrangeupdate net.IP + skipOverlappingRangeUpdate := false for _, ipRange := range ipamConf.IPRanges { RETRYLOOP: for j := 0; j < storage.DatastoreRetries; j++ { @@ -515,7 +515,7 @@ func IPManagementKubernetesUpdate(ctx context.Context, mode int, ipam *Kubernete var updatedreservelist []whereaboutstypes.IPReservation switch mode { case whereaboutstypes.Allocate: - newip, updatedreservelist, err = allocate.AssignIP(ipRange, reservelist, containerID, podRef) + newip, updatedreservelist, err = allocate.AssignIP(ipRange, reservelist, ipam.containerID, ipamConf.GetPodRef(), ipam.IfName) if err != nil { logging.Errorf("Error assigning IP: %v", err) return newips, err @@ -524,28 +524,32 @@ func IPManagementKubernetesUpdate(ctx context.Context, mode int, ipam *Kubernete // When it's allocated overlappingrange wide, we add it to a local reserved list // And we try again. if ipamConf.OverlappingRanges { - isAllocated, err := overlappingrangestore.IsAllocatedInOverlappingRange(requestCtx, newip.IP, - ipamConf.NetworkName) + overlappingRangeIPReservation, err := overlappingrangestore.GetOverlappingRangeIPReservation(requestCtx, newip.IP, + ipamConf.GetPodRef(), ipamConf.NetworkName) if err != nil { - logging.Errorf("Error checking overlappingrange allocation: %v", err) + logging.Errorf("Error getting cluster wide IP allocation: %v", err) return newips, err } - if isAllocated { - logging.Debugf("Continuing loop, IP is already allocated (possibly from another range): %v", newip) - // We create "dummy" records here for evaluation, but, we need to filter those out later. - overlappingrangeallocations = append(overlappingrangeallocations, whereaboutstypes.IPReservation{IP: newip.IP, IsAllocated: true}) - continue + if overlappingRangeIPReservation != nil { + if overlappingRangeIPReservation.Spec.PodRef != ipamConf.GetPodRef() { + logging.Debugf("Continuing loop, IP is already allocated (possibly from another range): %v", newip) + // We create "dummy" records here for evaluation, but, we need to filter those out later. + overlappingrangeallocations = append(overlappingrangeallocations, whereaboutstypes.IPReservation{IP: newip.IP, IsAllocated: true}) + continue + } + + skipOverlappingRangeUpdate = true } ipforoverlappingrangeupdate = newip.IP } case whereaboutstypes.Deallocate: - updatedreservelist, ipforoverlappingrangeupdate = allocate.DeallocateIP(reservelist, containerID) + updatedreservelist, ipforoverlappingrangeupdate = allocate.DeallocateIP(reservelist, ipam.containerID) if ipforoverlappingrangeupdate == nil { // Do not fail if allocation was not found. - logging.Debugf("Failed to find allocation for container ID: %s", containerID) + logging.Debugf("Failed to find allocation for container ID: %s", ipam.containerID) return nil, nil } } @@ -575,11 +579,13 @@ func IPManagementKubernetesUpdate(ctx context.Context, mode int, ipam *Kubernete } if ipamConf.OverlappingRanges { - err = overlappingrangestore.UpdateOverlappingRangeAllocation(requestCtx, mode, ipforoverlappingrangeupdate, - containerID, podRef, ipamConf.NetworkName) - if err != nil { - logging.Errorf("Error performing UpdateOverlappingRangeAllocation: %v", err) - return newips, err + if !skipOverlappingRangeUpdate { + err = overlappingrangestore.UpdateOverlappingRangeAllocation(requestCtx, mode, ipforoverlappingrangeupdate, + ipamConf.GetPodRef(), ipam.IfName, ipamConf.NetworkName) + if err != nil { + logging.Errorf("Error performing UpdateOverlappingRangeAllocation: %v", err) + return newips, err + } } } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 52660eee2..092c9a5a4 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -2,6 +2,7 @@ package storage import ( "context" + "github.com/k8snetworkplumbingwg/whereabouts/pkg/api/whereabouts.cni.cncf.io/v1alpha1" "net" "time" @@ -33,9 +34,8 @@ type Store interface { // OverlappingRangeStore is an interface for wrapping overlappingrange storage options type OverlappingRangeStore interface { - IsAllocatedInOverlappingRange(ctx context.Context, ip net.IP, networkName string) (bool, error) - UpdateOverlappingRangeAllocation(ctx context.Context, mode int, ip net.IP, containerID string, podRef, - networkName string) error + GetOverlappingRangeIPReservation(ctx context.Context, ip net.IP, podRef, networkName string) (*v1alpha1.OverlappingRangeIPReservation, error) + UpdateOverlappingRangeAllocation(ctx context.Context, mode int, ip net.IP, podRef, ifName, networkName string) error } type Temporary interface { diff --git a/pkg/types/types.go b/pkg/types/types.go index 519a094a5..5c218a976 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -187,7 +187,8 @@ type Address struct { type IPReservation struct { IP net.IP `json:"ip"` ContainerID string `json:"id"` - PodRef string `json:"podref,omitempty"` + PodRef string `json:"podref"` + IfName string `json:"ifName"` IsAllocated bool }