diff --git a/controllers/cloudstackisolatednetwork_controller.go b/controllers/cloudstackisolatednetwork_controller.go index 036366e5..eb97f3bd 100644 --- a/controllers/cloudstackisolatednetwork_controller.go +++ b/controllers/cloudstackisolatednetwork_controller.go @@ -85,6 +85,7 @@ func (r *CloudStackIsoNetReconciliationRunner) Reconcile() (ctrl.Result, error) if r.FailureDomain.Spec.Zone.ID == "" { return r.RequeueWithMessage("Zone ID not resolved yet.") } + if err := r.CSUser.GetOrCreateIsolatedNetwork(r.FailureDomain, r.ReconciliationSubject, r.CSCluster); err != nil { return ctrl.Result{}, err } @@ -92,17 +93,39 @@ func (r *CloudStackIsoNetReconciliationRunner) Reconcile() (ctrl.Result, error) if err := r.CSUser.AddClusterTag(cloud.ResourceTypeNetwork, r.ReconciliationSubject.Spec.ID, r.CSCluster); err != nil { return ctrl.Result{}, errors.Wrapf(err, "tagging network with id %s", r.ReconciliationSubject.Spec.ID) } - // Configure API server load balancer, if enabled and this cluster is not externally managed. + + // Assign IP and configure API server load balancer, if enabled and this cluster is not externally managed. if !annotations.IsExternallyManaged(r.CSCluster) { + pubIP, err := r.CSUser.AssociatePublicIPAddress(r.FailureDomain, r.ReconciliationSubject, r.CSCluster.Spec.ControlPlaneEndpoint.Host) + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "associating public IP address") + } + r.ReconciliationSubject.Spec.ControlPlaneEndpoint.Host = pubIP.Ipaddress + r.CSCluster.Spec.ControlPlaneEndpoint.Host = pubIP.Ipaddress + r.ReconciliationSubject.Status.PublicIPID = pubIP.Id + r.ReconciliationSubject.Status.PublicIPAddress = pubIP.Ipaddress + + if r.ReconciliationSubject.Status.APIServerLoadBalancer == nil { + r.ReconciliationSubject.Status.APIServerLoadBalancer = &infrav1.LoadBalancer{} + } + r.ReconciliationSubject.Status.APIServerLoadBalancer.IPAddressID = pubIP.Id + r.ReconciliationSubject.Status.APIServerLoadBalancer.IPAddress = pubIP.Ipaddress + if err := r.CSUser.AddClusterTag(cloud.ResourceTypeIPAddress, pubIP.Id, r.CSCluster); err != nil { + return ctrl.Result{}, errors.Wrapf(err, + "adding cluster tag to public IP address with ID %s", pubIP.Id) + } + if err := r.CSUser.ReconcileLoadBalancer(r.FailureDomain, r.ReconciliationSubject, r.CSCluster); err != nil { return ctrl.Result{}, errors.Wrap(err, "reconciling load balancer") } } + if err := csClusterPatcher.Patch(r.RequestCtx, r.CSCluster); err != nil { return ctrl.Result{}, errors.Wrap(err, "patching endpoint update to CloudStackCluster") } r.ReconciliationSubject.Status.Ready = true + return ctrl.Result{}, nil } diff --git a/pkg/cloud/isolated_network.go b/pkg/cloud/isolated_network.go index e4dc50f4..27a2ad23 100644 --- a/pkg/cloud/isolated_network.go +++ b/pkg/cloud/isolated_network.go @@ -29,7 +29,6 @@ import ( utilsnet "k8s.io/utils/net" infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" capcstrings "sigs.k8s.io/cluster-api-provider-cloudstack/pkg/utils/strings" - "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/cluster-api/util/record" ) @@ -37,9 +36,9 @@ type IsoNetworkIface interface { GetOrCreateIsolatedNetwork(*infrav1.CloudStackFailureDomain, *infrav1.CloudStackIsolatedNetwork, *infrav1.CloudStackCluster) error ReconcileLoadBalancer(*infrav1.CloudStackFailureDomain, *infrav1.CloudStackIsolatedNetwork, *infrav1.CloudStackCluster) error - AssociatePublicIPAddress(*infrav1.CloudStackFailureDomain, *infrav1.CloudStackIsolatedNetwork, *infrav1.CloudStackCluster) error + AssociatePublicIPAddress(*infrav1.CloudStackFailureDomain, *infrav1.CloudStackIsolatedNetwork, string) (*cloudstack.PublicIpAddress, error) CreateEgressFirewallRules(*infrav1.CloudStackIsolatedNetwork) error - GetPublicIP(*infrav1.CloudStackFailureDomain, *infrav1.CloudStackCluster) (*cloudstack.PublicIpAddress, error) + GetPublicIP(*infrav1.CloudStackFailureDomain, string) (*cloudstack.PublicIpAddress, error) GetLoadBalancerRules(isoNet *infrav1.CloudStackIsolatedNetwork) ([]*cloudstack.LoadBalancerRule, error) ReconcileLoadBalancerRules(isoNet *infrav1.CloudStackIsolatedNetwork, csCluster *infrav1.CloudStackCluster) error GetFirewallRules(isoNet *infrav1.CloudStackIsolatedNetwork) ([]*cloudstack.FirewallRule, error) @@ -63,53 +62,38 @@ func (c *client) getNetworkOfferingID() (string, error) { return offeringID, nil } -// AssociatePublicIPAddress Gets a PublicIP and associates the public IP to passed isolated network. +// AssociatePublicIPAddress gets a public IP and associates it to the isolated network. func (c *client) AssociatePublicIPAddress( fd *infrav1.CloudStackFailureDomain, isoNet *infrav1.CloudStackIsolatedNetwork, - csCluster *infrav1.CloudStackCluster, -) (retErr error) { + desiredIP string, +) (*cloudstack.PublicIpAddress, error) { // Check specified IP address is available or get an unused one if not specified. - publicAddress, err := c.GetPublicIP(fd, csCluster) + publicAddress, err := c.GetPublicIP(fd, desiredIP) if err != nil { - return errors.Wrapf(err, "fetching a public IP address") - } - isoNet.Spec.ControlPlaneEndpoint.Host = publicAddress.Ipaddress - if !annotations.IsExternallyManaged(csCluster) { - // Do not update the infracluster's controlPlaneEndpoint when the controlplane - // is externally managed, it is the responsibility of the externally managed - // control plane to update this. - csCluster.Spec.ControlPlaneEndpoint.Host = publicAddress.Ipaddress + return nil, errors.Wrap(err, "fetching a public IP address") } - if isoNet.Status.APIServerLoadBalancer == nil { - isoNet.Status.APIServerLoadBalancer = &infrav1.LoadBalancer{} - } - isoNet.Status.APIServerLoadBalancer.IPAddressID = publicAddress.Id - isoNet.Status.APIServerLoadBalancer.IPAddress = publicAddress.Ipaddress - // Check if the address is already associated with the network. if publicAddress.Associatednetworkid == isoNet.Spec.ID { - return nil + return publicAddress, nil } // Public IP found, but not yet associated with network -- associate it. p := c.cs.Address.NewAssociateIpAddressParams() - p.SetIpaddress(isoNet.Spec.ControlPlaneEndpoint.Host) + p.SetIpaddress(publicAddress.Ipaddress) p.SetNetworkid(isoNet.Spec.ID) if _, err := c.cs.Address.AssociateIpAddress(p); err != nil { c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err) - return errors.Wrapf(err, + return nil, errors.Wrapf(err, "associating public IP address with ID %s to network with ID %s", publicAddress.Id, isoNet.Spec.ID) - } else if err := c.AddClusterTag(ResourceTypeIPAddress, publicAddress.Id, csCluster); err != nil { - return errors.Wrapf(err, - "adding tag to public IP address with ID %s", publicAddress.Id) - } else if err := c.AddCreatedByCAPCTag(ResourceTypeIPAddress, isoNet.Status.PublicIPID); err != nil { - return errors.Wrapf(err, + } else if err := c.AddCreatedByCAPCTag(ResourceTypeIPAddress, publicAddress.Id); err != nil { + return nil, errors.Wrapf(err, "adding tag to public IP address with ID %s", publicAddress.Id) } - return nil + + return publicAddress, nil } // CreateIsolatedNetwork creates an isolated network in the relevant FailureDomain per passed network specification. @@ -172,27 +156,25 @@ func (c *client) CreateEgressFirewallRules(isoNet *infrav1.CloudStackIsolatedNet return retErr } -// GetPublicIP gets a public IP with ID for cluster endpoint. +// GetPublicIP gets a public IP. If desiredIP is empty, it will pick the next available IP. func (c *client) GetPublicIP( fd *infrav1.CloudStackFailureDomain, - csCluster *infrav1.CloudStackCluster, + desiredIP string, ) (*cloudstack.PublicIpAddress, error) { - ip := csCluster.Spec.ControlPlaneEndpoint.Host - p := c.cs.Address.NewListPublicIpAddressesParams() p.SetAllocatedonly(false) p.SetZoneid(fd.Spec.Zone.ID) - setIfNotEmpty(ip, p.SetIpaddress) + setIfNotEmpty(desiredIP, p.SetIpaddress) publicAddresses, err := c.cs.Address.ListPublicIpAddresses(p) if err != nil { c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err) return nil, err - } else if ip != "" && publicAddresses.Count == 1 { - // Endpoint specified and IP found. + } else if desiredIP != "" && publicAddresses.Count == 1 { + // Desired IP specified and IP found. // Ignore already allocated here since the IP was specified. return publicAddresses.PublicIpAddresses[0], nil } else if publicAddresses.Count > 0 { - // Endpoint not specified. Pick first available address. + // Desired IP not specified. Pick first available address. for _, v := range publicAddresses.PublicIpAddresses { if v.Allocated == "" { // Found un-allocated Public IP. return v, nil @@ -712,42 +694,6 @@ func (c *client) GetOrCreateIsolatedNetwork( isoNet.Spec.CIDR = network.CIDR } - // Tag the created network. - networkID := isoNet.Spec.ID - if err := c.AddClusterTag(ResourceTypeNetwork, networkID, csCluster); err != nil { - return errors.Wrapf(err, "tagging network with id %s", networkID) - } - - // Set the outgoing IP details in the isolated network status. - if isoNet.Status.PublicIPID == "" || isoNet.Status.PublicIPAddress == "" { - // Look up the details of the isolated network SNAT IP (outgoing IP). - p := c.cs.Address.NewListPublicIpAddressesParams() - p.SetAllocatedonly(true) - p.SetZoneid(fd.Spec.Zone.ID) - p.SetAssociatednetworkid(networkID) - p.SetIssourcenat(true) - publicAddresses, err := c.cs.Address.ListPublicIpAddresses(p) - if err != nil { - c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err) - return errors.Wrap(err, "listing public ip addresses") - } else if publicAddresses.Count == 0 || publicAddresses.Count > 1 { - c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err) - return errors.New("unexpected amount of public outgoing ip addresses found") - } - - if err := c.AddClusterTag(ResourceTypeIPAddress, publicAddresses.PublicIpAddresses[0].Id, csCluster); err != nil { - return errors.Wrapf(err, - "adding tag to public IP address with ID %s", publicAddresses.PublicIpAddresses[0].Id) - } - if err := c.AddCreatedByCAPCTag(ResourceTypeIPAddress, isoNet.Status.PublicIPID); err != nil { - return errors.Wrapf(err, - "adding tag to public IP address with ID %s", publicAddresses.PublicIpAddresses[0].Id) - } - - isoNet.Status.PublicIPAddress = publicAddresses.PublicIpAddresses[0].Ipaddress - isoNet.Status.PublicIPID = publicAddresses.PublicIpAddresses[0].Id - } - // Open the Isolated Network egress firewall. return errors.Wrap(c.CreateEgressFirewallRules(isoNet), "opening the isolated network's egress firewall") } @@ -770,12 +716,13 @@ func (c *client) ReconcileLoadBalancer( } // Associate public IP with the load balancer if enabled. + /* TODO: implement possibility for load balancer to use a different IP than isonet if csCluster.Spec.APIServerLoadBalancer.IsEnabled() { // Associate Public IP with CloudStackIsolatedNetwork if err := c.AssociatePublicIPAddress(fd, isoNet, csCluster); err != nil { return errors.Wrapf(err, "associating public IP address to csCluster") } - } + }*/ // Set up load balancing rules to map VM ports to Public IP ports. if err := c.ReconcileLoadBalancerRules(isoNet, csCluster); err != nil { diff --git a/pkg/cloud/isolated_network_test.go b/pkg/cloud/isolated_network_test.go index d66016c9..7846c45c 100644 --- a/pkg/cloud/isolated_network_test.go +++ b/pkg/cloud/isolated_network_test.go @@ -78,11 +78,6 @@ var _ = Describe("Network", func() { ns.EXPECT().GetNetworkByName(dummies.ISONet1.Name).Return(nil, 0, nil) ns.EXPECT().GetNetworkByID(dummies.ISONet1.ID).Return(nil, 0, nil) ns.EXPECT().CreateNetwork(gomock.Any()).Return(&csapi.CreateNetworkResponse{Id: dummies.ISONet1.ID}, nil) - as.EXPECT().NewListPublicIpAddressesParams().Return(&csapi.ListPublicIpAddressesParams{}) - as.EXPECT().ListPublicIpAddresses(gomock.Any()). - Return(&csapi.ListPublicIpAddressesResponse{ - Count: 1, - PublicIpAddresses: []*csapi.PublicIpAddress{{Id: dummies.PublicIPID, Ipaddress: "fakeIP"}}}, nil) fs.EXPECT().NewCreateEgressFirewallRuleParams(dummies.ISONet1.ID, gomock.Any()). DoAndReturn(func(networkid string, protocol string) *csapi.CreateEgressFirewallRuleParams { @@ -103,29 +98,19 @@ var _ = Describe("Network", func() { fs.EXPECT().CreateEgressFirewallRule(ruleParamsICMP). Return(&csapi.CreateEgressFirewallRuleResponse{}, nil)) - // Will add cluster tag once to Network and once to PublicIP. - createdByResponse := &csapi.ListTagsResponse{Tags: []*csapi.Tag{{Key: cloud.CreatedByCAPCTagName, Value: "1"}}} - rs.EXPECT().NewListTagsParams().Return(&csapi.ListTagsParams{}).Times(2) - rs.EXPECT().ListTags(gomock.Any()).Return(createdByResponse, nil).Times(2) - - // Will add creation and cluster tags to network and PublicIP. + // Will add creation tags to network. rs.EXPECT().NewCreateTagsParams(gomock.Any(), gomock.Any(), gomock.Any()). - Return(&csapi.CreateTagsParams{}).Times(4) - rs.EXPECT().CreateTags(gomock.Any()).Return(&csapi.CreateTagsResponse{}, nil).Times(4) + Return(&csapi.CreateTagsParams{}) + rs.EXPECT().CreateTags(gomock.Any()).Return(&csapi.CreateTagsResponse{}, nil) Ω(client.GetOrCreateIsolatedNetwork(dummies.CSFailureDomain1, dummies.CSISONet1, dummies.CSCluster)).Should(Succeed()) - Ω(dummies.CSISONet1.Status.PublicIPID).Should(Equal(dummies.PublicIPID)) + Ω(dummies.CSISONet1.Spec.ID).ShouldNot(BeEmpty()) }) It("resolves the existing isolated network", func() { dummies.SetClusterSpecToNet(&dummies.ISONet1) ns.EXPECT().GetNetworkByName(dummies.ISONet1.Name).Return(dummies.CAPCNetToCSAPINet(&dummies.ISONet1), 1, nil) - as.EXPECT().NewListPublicIpAddressesParams().Return(&csapi.ListPublicIpAddressesParams{}) - as.EXPECT().ListPublicIpAddresses(gomock.Any()). - Return(&csapi.ListPublicIpAddressesResponse{ - Count: 1, - PublicIpAddresses: []*csapi.PublicIpAddress{{Id: dummies.PublicIPID, Ipaddress: "fakeIP"}}}, nil) fs.EXPECT().NewCreateEgressFirewallRuleParams(dummies.ISONet1.ID, gomock.Any()). DoAndReturn(func(networkid string, protocol string) *csapi.CreateEgressFirewallRuleParams { @@ -146,19 +131,8 @@ var _ = Describe("Network", func() { fs.EXPECT().CreateEgressFirewallRule(ruleParamsICMP). Return(&csapi.CreateEgressFirewallRuleResponse{}, nil)) - // Will add cluster tag once to Network and once to PublicIP. - createdByResponse := &csapi.ListTagsResponse{Tags: []*csapi.Tag{{Key: cloud.CreatedByCAPCTagName, Value: "1"}}} - rs.EXPECT().NewListTagsParams().Return(&csapi.ListTagsParams{}).Times(2) - rs.EXPECT().ListTags(gomock.Any()).Return(createdByResponse, nil).Times(2) - - // Will add creation and cluster tags to network and PublicIP. - rs.EXPECT().NewCreateTagsParams(gomock.Any(), gomock.Any(), gomock.Any()). - Return(&csapi.CreateTagsParams{}).Times(3) - rs.EXPECT().CreateTags(gomock.Any()).Return(&csapi.CreateTagsResponse{}, nil).Times(3) - Ω(client.GetOrCreateIsolatedNetwork(dummies.CSFailureDomain1, dummies.CSISONet1, dummies.CSCluster)).Should(Succeed()) Ω(dummies.CSISONet1.Spec.ID).ShouldNot(BeEmpty()) - Ω(dummies.CSISONet1.Status.PublicIPID).Should(Equal(dummies.PublicIPID)) }) It("fails to get network offering from CloudStack", func() { @@ -226,14 +200,14 @@ var _ = Describe("Network", func() { }) Context("in an isolated network with public IPs available", func() { - It("will resolve public IP details given an endpoint spec", func() { + It("will resolve public IP details given an endpoint host", func() { as.EXPECT().NewListPublicIpAddressesParams().Return(&csapi.ListPublicIpAddressesParams{}) as.EXPECT().ListPublicIpAddresses(gomock.Any()). Return(&csapi.ListPublicIpAddressesResponse{ Count: 1, PublicIpAddresses: []*csapi.PublicIpAddress{{Id: "PublicIPID", Ipaddress: ipAddress}}, }, nil) - publicIPAddress, err := client.GetPublicIP(dummies.CSFailureDomain1, dummies.CSCluster) + publicIPAddress, err := client.GetPublicIP(dummies.CSFailureDomain1, dummies.CSCluster.Spec.ControlPlaneEndpoint.Host) Ω(err).Should(Succeed()) Ω(publicIPAddress).ShouldNot(BeNil()) Ω(publicIPAddress.Ipaddress).Should(Equal(ipAddress)) @@ -248,7 +222,7 @@ var _ = Describe("Network", func() { Count: 0, PublicIpAddresses: []*csapi.PublicIpAddress{}, }, nil) - publicIPAddress, err := client.GetPublicIP(dummies.CSFailureDomain1, dummies.CSCluster) + publicIPAddress, err := client.GetPublicIP(dummies.CSFailureDomain1, dummies.CSCluster.Spec.ControlPlaneEndpoint.Host) Ω(publicIPAddress).Should(BeNil()) Ω(err.Error()).Should(ContainSubstring("no public addresses found in available networks")) }) @@ -269,7 +243,7 @@ var _ = Describe("Network", func() { Associatednetworkid: "1", }}, }, nil) - publicIPAddress, err := client.GetPublicIP(dummies.CSFailureDomain1, dummies.CSCluster) + publicIPAddress, err := client.GetPublicIP(dummies.CSFailureDomain1, dummies.CSCluster.Spec.ControlPlaneEndpoint.Host) Ω(publicIPAddress).Should(BeNil()) Ω(err.Error()).Should(ContainSubstring("all Public IP Address(es) found were already allocated")) }) @@ -286,18 +260,14 @@ var _ = Describe("Network", func() { aip := &csapi.AssociateIpAddressParams{} as.EXPECT().NewAssociateIpAddressParams().Return(aip) as.EXPECT().AssociateIpAddress(aip).Return(&csapi.AssociateIpAddressResponse{}, nil) - // Will add cluster tag once to Network and once to PublicIP. - createdByResponse := &csapi.ListTagsResponse{Tags: []*csapi.Tag{{Key: cloud.CreatedByCAPCTagName, Value: "1"}}} - gomock.InOrder( - rs.EXPECT().NewListTagsParams().Return(&csapi.ListTagsParams{}), - rs.EXPECT().ListTags(gomock.Any()).Return(createdByResponse, nil)) // Will add creation and cluster tags to network and PublicIP. rs.EXPECT().NewCreateTagsParams(gomock.Any(), gomock.Any(), gomock.Any()). - Return(&csapi.CreateTagsParams{}).Times(2) - rs.EXPECT().CreateTags(gomock.Any()).Return(&csapi.CreateTagsResponse{}, nil).Times(2) + Return(&csapi.CreateTagsParams{}) + rs.EXPECT().CreateTags(gomock.Any()).Return(&csapi.CreateTagsResponse{}, nil) - Ω(client.AssociatePublicIPAddress(dummies.CSFailureDomain1, dummies.CSISONet1, dummies.CSCluster)).Should(Succeed()) + _, err := client.AssociatePublicIPAddress(dummies.CSFailureDomain1, dummies.CSISONet1, dummies.CSCluster.Spec.ControlPlaneEndpoint.Host) + Ω(err).Should(Succeed()) }) It("Failure Associating Public IP to Isolated network", func() { @@ -310,29 +280,25 @@ var _ = Describe("Network", func() { aip := &csapi.AssociateIpAddressParams{} as.EXPECT().NewAssociateIpAddressParams().Return(aip) as.EXPECT().AssociateIpAddress(aip).Return(nil, errors.New("Failed to allocate IP address")) - Ω(client.AssociatePublicIPAddress(dummies.CSFailureDomain1, dummies.CSISONet1, dummies.CSCluster).Error()).Should(ContainSubstring("associating public IP address with ID")) + + _, err := client.AssociatePublicIPAddress(dummies.CSFailureDomain1, dummies.CSISONet1, dummies.CSCluster.Spec.ControlPlaneEndpoint.Host) + Ω(err.Error()).Should(ContainSubstring("associating public IP address with ID")) }) }) Context("With an enabled API load balancer", func() { It("reconciles the required load balancer and firewall rules", func() { dummies.SetClusterSpecToNet(&dummies.ISONet1) + dummies.CSISONet1.Status.APIServerLoadBalancer.IPAddressID = dummies.LoadBalancerIPID - as.EXPECT().NewListPublicIpAddressesParams().Return(&csapi.ListPublicIpAddressesParams{}) - as.EXPECT().ListPublicIpAddresses(gomock.Any()). - Return(&csapi.ListPublicIpAddressesResponse{ - Count: 1, - PublicIpAddresses: []*csapi.PublicIpAddress{{Id: dummies.LoadBalancerIPID, Ipaddress: "fakeLBIP"}}}, nil) - as.EXPECT().NewAssociateIpAddressParams().Return(&csapi.AssociateIpAddressParams{}) - as.EXPECT().AssociateIpAddress(gomock.Any()) - - createdByResponse := &csapi.ListTagsResponse{Tags: []*csapi.Tag{{Key: cloud.CreatedByCAPCTagName, Value: "1"}}} - rs.EXPECT().NewListTagsParams().Return(&csapi.ListTagsParams{}) - rs.EXPECT().ListTags(gomock.Any()).Return(createdByResponse, nil) - - rs.EXPECT().NewCreateTagsParams(gomock.Any(), gomock.Any(), gomock.Any()). - Return(&csapi.CreateTagsParams{}).Times(2) - rs.EXPECT().CreateTags(gomock.Any()).Return(&csapi.CreateTagsResponse{}, nil).Times(2) + /* + as.EXPECT().NewListPublicIpAddressesParams().Return(&csapi.ListPublicIpAddressesParams{}) + as.EXPECT().ListPublicIpAddresses(gomock.Any()). + Return(&csapi.ListPublicIpAddressesResponse{ + Count: 1, + PublicIpAddresses: []*csapi.PublicIpAddress{{Id: dummies.LoadBalancerIPID, Ipaddress: "fakeLBIP"}}}, nil) + as.EXPECT().NewAssociateIpAddressParams().Return(&csapi.AssociateIpAddressParams{}) + as.EXPECT().AssociateIpAddress(gomock.Any())*/ lbs.EXPECT().NewListLoadBalancerRulesParams().Return(&csapi.ListLoadBalancerRulesParams{}) lbs.EXPECT().ListLoadBalancerRules(gomock.Any()).Return(