Skip to content

Commit

Permalink
Merge pull request #2806 from panslava/mp-integration-test
Browse files Browse the repository at this point in the history
ProviderConfig fixes and integration test
  • Loading branch information
k8s-ci-robot authored Feb 21, 2025
2 parents 28f9d4d + 2314686 commit d103dce
Show file tree
Hide file tree
Showing 11 changed files with 874 additions and 46 deletions.
20 changes: 18 additions & 2 deletions cmd/glbc/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
flag "github.com/spf13/pflag"
crdclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
informers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
Expand All @@ -43,8 +44,10 @@ import (
frontendconfigclient "k8s.io/ingress-gce/pkg/frontendconfig/client/clientset/versioned"
"k8s.io/ingress-gce/pkg/instancegroups"
"k8s.io/ingress-gce/pkg/l4lb"
multiprojectgce "k8s.io/ingress-gce/pkg/multiproject/gce"
multiprojectstart "k8s.io/ingress-gce/pkg/multiproject/start"
"k8s.io/ingress-gce/pkg/network"
providerconfigclient "k8s.io/ingress-gce/pkg/providerconfig/client/clientset/versioned"
"k8s.io/ingress-gce/pkg/psc"
"k8s.io/ingress-gce/pkg/serviceattachment"
serviceattachmentclient "k8s.io/ingress-gce/pkg/serviceattachment/client/clientset/versioned"
Expand Down Expand Up @@ -243,17 +246,28 @@ func main() {
rootLogger.Info("Multi-project mode is enabled, starting project-syncer")

runWithWg(func() {
gceCreator, err := multiprojectgce.NewDefaultGCECreator(rootLogger)
if err != nil {
klog.Fatalf("Failed to create GCE creator: %v", err)
}
providerConfigClient, err := providerconfigclient.NewForConfig(kubeConfig)
if err != nil {
klog.Fatalf("Failed to create ProviderConfig client: %v", err)
}
informersFactory := informers.NewSharedInformerFactory(kubeClient, flags.F.ResyncPeriod)
if flags.F.LeaderElection.LeaderElect {
err := multiprojectstart.StartWithLeaderElection(
context.Background(),
leaderElectKubeClient,
hostname,
kubeConfig,
rootLogger,
kubeClient,
svcNegClient,
kubeSystemUID,
eventRecorderKubeClient,
providerConfigClient,
informersFactory,
gceCreator,
namer,
stopCh,
)
Expand All @@ -262,12 +276,14 @@ func main() {
}
} else {
multiprojectstart.Start(
kubeConfig,
rootLogger,
kubeClient,
svcNegClient,
kubeSystemUID,
eventRecorderKubeClient,
providerConfigClient,
informersFactory,
gceCreator,
namer,
stopCh,
)
Expand Down
2 changes: 1 addition & 1 deletion pkg/flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ L7 load balancing. CSV values accepted. Example: -node-port-ranges=80,8080,400-5
flag.BoolVar(&F.EnableMultiProjectMode, "enable-multi-project-mode", false, "Enable running in multi-project mode.")
flag.BoolVar(&F.EnableL4ILBMixedProtocol, "enable-l4ilb-mixed-protocol", false, "Enable support for mixed protocol L4 internal load balancers.")
flag.BoolVar(&F.EnableL4NetLBMixedProtocol, "enable-l4netlb-mixed-protocol", false, "Enable support for mixed protocol L4 external load balancers.")
flag.StringVar(&F.ProviderConfigNameLabelKey, "provider-config-name-label-key", "", "The label key for provider-config name, which is used to identify the provider-config of objects in multi-project mode.")
flag.StringVar(&F.ProviderConfigNameLabelKey, "provider-config-name-label-key", "cloud.gke.io/provider-config-name", "The label key for provider-config name, which is used to identify the provider-config of objects in multi-project mode.")
flag.BoolVar(&F.EnableIPV6OnlyNEG, "enable-ipv6-only-neg", false, "Enable support for IPV6 Only NEG's.")
}

Expand Down
19 changes: 14 additions & 5 deletions pkg/multiproject/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"fmt"
"math/rand"
"runtime/debug"
"time"

"k8s.io/apimachinery/pkg/util/wait"
Expand Down Expand Up @@ -55,8 +56,14 @@ func NewProviderConfigController(manager ProviderConfigControllerManager, provid

providerConfigInformer.AddEventHandler(
cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) { pcc.providerConfigQueue.Enqueue(obj) },
UpdateFunc: func(old, cur interface{}) { pcc.providerConfigQueue.Enqueue(cur) },
AddFunc: func(obj interface{}) {
pcc.logger.V(4).Info("Enqueue add event", "object", obj)
pcc.providerConfigQueue.Enqueue(obj)
},
UpdateFunc: func(old, cur interface{}) {
pcc.logger.V(4).Info("Enqueue update event", "old", old, "new", cur)
pcc.providerConfigQueue.Enqueue(cur)
},
})

pcc.logger.Info("ProviderConfig controller created")
Expand All @@ -70,6 +77,7 @@ func (pcc *ProviderConfigController) Run() {
ctx, cancel := context.WithCancel(context.Background())
go func() {
<-pcc.stopCh
pcc.logger.Info("Stop channel closed, cancelling context")
cancel()
}()

Expand Down Expand Up @@ -98,7 +106,8 @@ func (pcc *ProviderConfigController) syncWrapper(key string) error {

defer func() {
if r := recover(); r != nil {
svcLogger.Error(fmt.Errorf("panic in ProviderConfig sync worker goroutine: %v", r), "Recovered from panic")
stack := string(debug.Stack())
svcLogger.Error(fmt.Errorf("panic in ProviderConfig sync worker goroutine: %v, stack: %s", r, stack), "Recovered from panic")
}
}()
err := pcc.sync(key, svcLogger)
Expand Down Expand Up @@ -131,12 +140,12 @@ func (pcc *ProviderConfigController) sync(key string, logger klog.Logger) error
return nil
}

logger.V(2).Info("Syncing providerConfig", "providerConfig", pc)
logger.Info("Syncing providerConfig", "providerConfig", pc)
err = pcc.manager.StartControllersForProviderConfig(pc)
if err != nil {
return fmt.Errorf("failed to start controllers for providerConfig %v: %w", pc, err)
}

logger.V(2).Info("Successfully synced providerConfig", "providerConfig", pc)
logger.Info("Successfully synced providerConfig", "providerConfig", pc)
return nil
}
104 changes: 104 additions & 0 deletions pkg/multiproject/gce/fake.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
package gce

import (
"fmt"

"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
compute "google.golang.org/api/compute/v1"
cloudgce "k8s.io/cloud-provider-gcp/providers/gce"
v1 "k8s.io/ingress-gce/pkg/apis/providerconfig/v1"
"k8s.io/ingress-gce/pkg/test"
"k8s.io/klog/v2"
)

type GCEFake struct {
defaultTestClusterValues cloudgce.TestClusterValues
clientsForProviderConfigs map[string]*cloudgce.Cloud
}

func NewGCEFake() *GCEFake {
return &GCEFake{
defaultTestClusterValues: cloudgce.DefaultTestClusterValues(),
clientsForProviderConfigs: make(map[string]*cloudgce.Cloud),
}
}

func providerConfigKey(providerConfig *v1.ProviderConfig) string {
return fmt.Sprintf("%s/%s", providerConfig.Namespace, providerConfig.Name)
}

// GCEForProviderConfig returns a new Fake GCE client for the given provider config.
// It stores the client in the GCEFake and returns it if the same provider config is requested again.
func (g *GCEFake) GCEForProviderConfig(providerConfig *v1.ProviderConfig, logger klog.Logger) (*cloudgce.Cloud, error) {
pcKey := providerConfigKey(providerConfig)
if g.clientsForProviderConfigs[pcKey] != nil {
return g.clientsForProviderConfigs[pcKey], nil
}

// Copy the default test cluster values
updatedConfig := g.defaultTestClusterValues
updatedConfig.ProjectID = providerConfig.Spec.ProjectID
updatedConfig.NetworkURL = fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/networks/%s", providerConfig.Spec.ProjectID, providerConfig.Spec.NetworkConfig.Network)
updatedConfig.SubnetworkURL = fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/subnetworks/%s", providerConfig.Spec.ProjectID, updatedConfig.Region, providerConfig.Spec.NetworkConfig.DefaultSubnetwork)
logger.Info("Creating GCEFake for provider config", "providerConfig", providerConfig.Name, "updatedConfig", updatedConfig)
fakeCloud := cloudgce.NewFakeGCECloud(updatedConfig)
_, err := createNetwork(fakeCloud, providerConfig.Spec.NetworkConfig.Network)
if err != nil {
return nil, err
}
_, err = createSubnetwork(fakeCloud, providerConfig.Spec.NetworkConfig.DefaultSubnetwork, providerConfig.Spec.NetworkConfig.Network)
if err != nil {
return nil, err
}
if err := createAndInsertNodes(fakeCloud, []string{"test-node-1"}, updatedConfig.ZoneName); err != nil {
return nil, err
}

g.clientsForProviderConfigs[pcKey] = fakeCloud
return fakeCloud, nil
}

func createAndInsertNodes(cloud *cloudgce.Cloud, nodeNames []string, zone string) error {
if _, err := test.CreateAndInsertNodes(cloud, nodeNames, zone); err != nil {
return err
}
return nil
}

func createNetwork(c *cloudgce.Cloud, networkName string) (*compute.Network, error) {
ctx, cancel := cloud.ContextWithCallTimeout()
defer cancel()

key := meta.GlobalKey(networkName)
err := c.Compute().Networks().Insert(ctx, key, &compute.Network{Name: networkName})
if err != nil {
return nil, err
}

network, err := c.Compute().Networks().Get(ctx, key)
if err != nil {
return nil, err
}
return network, nil
}

func createSubnetwork(c *cloudgce.Cloud, subnetworkName string, networkName string) (*compute.Subnetwork, error) {
ctx, cancel := cloud.ContextWithCallTimeout()
defer cancel()

key := meta.GlobalKey(subnetworkName)
err := c.Compute().Subnetworks().Insert(ctx, key, &compute.Subnetwork{Name: subnetworkName, Network: networkName})
if err != nil {
return nil, err
}

subnetwork, err := c.Compute().Subnetworks().Get(ctx, key)
if err != nil {
return nil, err
}
return subnetwork, nil
}

// assert that the GCEFake implements the GCECreator interface
var _ GCECreator = &GCEFake{}
38 changes: 38 additions & 0 deletions pkg/multiproject/gce/fake_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package gce

import (
"testing"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/ingress-gce/pkg/apis/providerconfig/v1"
"k8s.io/klog/v2"
)

func TestNewGCEForProviderConfig(t *testing.T) {
fake := NewGCEFake()

providerConfig := &v1.ProviderConfig{
ObjectMeta: metav1.ObjectMeta{
Name: "test-config",
},
Spec: v1.ProviderConfigSpec{
ProjectID: "custom-project-id",
NetworkConfig: &v1.NetworkConfig{
Network: "custom-network",
DefaultSubnetwork: "custom-subnetwork",
},
},
}

logger := klog.TODO()
cloud, err := fake.GCEForProviderConfig(providerConfig, logger)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if cloud == nil {
t.Fatal("expected cloud instance, got nil")
}
if cloud.ProjectID() != providerConfig.Spec.ProjectID {
t.Errorf("expected project id %q, got %q", providerConfig.Spec.ProjectID, cloud.ProjectID())
}
}
24 changes: 21 additions & 3 deletions pkg/multiproject/gce/gce.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,29 @@ func init() {
ini.PrettySection = true
}

// NewGCEForProviderConfig returns a new GCE client for the given project.
type GCECreator interface {
GCEForProviderConfig(providerConfig *v1.ProviderConfig, logger klog.Logger) (*cloudgce.Cloud, error)
}

type DefaultGCECreator struct {
defaultConfigFileString string
}

func NewDefaultGCECreator(logger klog.Logger) (*DefaultGCECreator, error) {
defaultGCEConfig, err := app.GCEConfString(logger)
if err != nil {
return nil, fmt.Errorf("error getting default cluster GCE config: %v", err)
}
return &DefaultGCECreator{
defaultConfigFileString: defaultGCEConfig,
}, nil
}

// GCEForProviderConfig returns a new GCE client for the given project.
// If providerConfig is nil, it returns the default cloud associated with the cluster's project.
// It modifies the default configuration when a providerConfig is provided.
func NewGCEForProviderConfig(defaultConfigContent string, providerConfig *v1.ProviderConfig, logger klog.Logger) (*cloudgce.Cloud, error) {
modifiedConfigContent, err := generateConfigForProviderConfig(defaultConfigContent, providerConfig)
func (g *DefaultGCECreator) GCEForProviderConfig(providerConfig *v1.ProviderConfig, logger klog.Logger) (*cloudgce.Cloud, error) {
modifiedConfigContent, err := generateConfigForProviderConfig(g.defaultConfigFileString, providerConfig)
if err != nil {
return nil, fmt.Errorf("failed to modify config content: %v", err)
}
Expand Down
16 changes: 12 additions & 4 deletions pkg/multiproject/manager/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"k8s.io/client-go/kubernetes"
providerconfig "k8s.io/ingress-gce/pkg/apis/providerconfig/v1"
"k8s.io/ingress-gce/pkg/multiproject/finalizer"
"k8s.io/ingress-gce/pkg/multiproject/gce"
"k8s.io/ingress-gce/pkg/multiproject/neg"
"k8s.io/ingress-gce/pkg/neg/syncers/labels"
providerconfigclient "k8s.io/ingress-gce/pkg/providerconfig/client/clientset/versioned"
Expand All @@ -35,7 +36,7 @@ type ProviderConfigControllersManager struct {
clusterNamer *namer.Namer
l4Namer *namer.L4Namer
lpConfig labels.PodLabelPropagationConfig
defaultCloudConfig string
gceCreator gce.GCECreator
globalStopCh <-chan struct{}
}

Expand All @@ -53,7 +54,7 @@ func NewProviderConfigControllerManager(
clusterNamer *namer.Namer,
l4Namer *namer.L4Namer,
lpConfig labels.PodLabelPropagationConfig,
defaultCloudConfig string,
gceCreator gce.GCECreator,
globalStopCh <-chan struct{},
logger klog.Logger,
) *ProviderConfigControllersManager {
Expand All @@ -69,7 +70,7 @@ func NewProviderConfigControllerManager(
clusterNamer: clusterNamer,
l4Namer: l4Namer,
lpConfig: lpConfig,
defaultCloudConfig: defaultCloudConfig,
gceCreator: gceCreator,
globalStopCh: globalStopCh,
}
}
Expand All @@ -83,6 +84,8 @@ func (pccm *ProviderConfigControllersManager) StartControllersForProviderConfig(
defer pccm.mu.Unlock()

pcKey := providerConfigKey(pc)

pccm.logger.Info("Starting controllers for provider config", "providerConfigId", pcKey)
if _, exists := pccm.controllers[pcKey]; exists {
pccm.logger.Info("Controllers for provider config already exist, skipping start", "providerConfigId", pcKey)
return nil
Expand All @@ -93,6 +96,11 @@ func (pccm *ProviderConfigControllersManager) StartControllersForProviderConfig(
return fmt.Errorf("failed to ensure NEG cleanup finalizer for project %s: %v", pcKey, err)
}

cloud, err := pccm.gceCreator.GCEForProviderConfig(pc, pccm.logger)
if err != nil {
return fmt.Errorf("failed to create GCE client for provider config %+v: %v", pc, err)
}

negControllerStopCh, err := neg.StartNEGController(
pccm.informersFactory,
pccm.kubeClient,
Expand All @@ -104,7 +112,7 @@ func (pccm *ProviderConfigControllersManager) StartControllersForProviderConfig(
pccm.clusterNamer,
pccm.l4Namer,
pccm.lpConfig,
pccm.defaultCloudConfig,
cloud,
pccm.globalStopCh,
pccm.logger,
pc,
Expand Down
Loading

0 comments on commit d103dce

Please sign in to comment.