Merge pull request #1588 from MartinForReal/master

Migrate msi/subnet client to track2 one

go.mod

@@ -7,11 +7,12 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.0.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0
+	github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
 	github.com/Azure/go-autorest/autorest v0.11.29
-	github.com/Azure/go-autorest/autorest/adal v0.9.24
 	github.com/container-storage-interface/spec v1.9.0
 	github.com/go-ini/ini v1.67.0
 	github.com/golang/protobuf v1.5.4
@@ -38,7 +39,7 @@ require (
 	k8s.io/pod-security-admission v0.31.1
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
 	sigs.k8s.io/cloud-provider-azure v1.31.1-0.20240914065912-f4dd79d54775
-	sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.56
+	sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.57
 	sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.27
 	sigs.k8s.io/yaml v1.4.0
 )
@@ -47,14 +48,14 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.0.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.0.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.24 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/autorest/mocks v0.4.2 // indirect
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
@@ -189,7 +190,7 @@ replace (
 	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.29.7
 	k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.29.7
 	k8s.io/endpointslice => k8s.io/endpointslice v0.29.7
-	k8s.io/kms => k8s.io/kms v0.31.1
+	k8s.io/kms => k8s.io/kms v0.29.7
 	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.29.7
 	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.29.7
 	k8s.io/kube-proxy => k8s.io/kube-proxy v0.29.7

go.sum

@@ -10,6 +10,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthoriza
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0/go.mod h1:/pz8dyNQe+Ey3yBp/XuYz7oqX8YDNWVpPB0hH3XWfbc=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 h1:zDeQI/PaWztI2tcrGO/9RIMey9NvqYbnyttf/0P3QWM=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0/go.mod h1:zflC9v4VfViJrSvcvplqws/yGXVbUEMZi/iHpZdSPWA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 h1:DWlwvVV5r/Wy1561nZ3wrpI1/vDIBRY/Wd1HWaRBZWA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0/go.mod h1:E7ltexgRDmeJ0fJWv0D/HLwY2xbDdN+uv+X2uZtOx3w=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5 v5.0.0 h1:5n7dPVqsWfVKw+ZiEKSd3Kzu7gwBkbEBkeXb8rgaE9Q=
@@ -431,8 +433,8 @@ k8s.io/csi-translation-lib v0.29.7 h1:6z1iFhTmVMK9mebK2eodvDCKv3bfL0OFu5z2C8YNvM
 k8s.io/csi-translation-lib v0.29.7/go.mod h1:+5ZOwRS5LUQOghtqv6QWWmadixbm697xNHZC318oVf4=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
-k8s.io/kms v0.31.1 h1:cGLyV3cIwb0ovpP/jtyIe2mEuQ/MkbhmeBF2IYCA9Io=
-k8s.io/kms v0.31.1/go.mod h1:OZKwl1fan3n3N5FFxnW5C4V3ygrah/3YXeJWS3O6+94=
+k8s.io/kms v0.29.7 h1:4ELQdx7T4EPKbN/QMj6SeZizrEKapza5YF8e5XtZPv0=
+k8s.io/kms v0.29.7/go.mod h1:vWVImKkJd+1BQY4tBwdfSwjQBiLrnbNtHADcDEDQFtk=
 k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e h1:OnKkExfhk4yxMqvBSPzUfhv3zQ96FWJ+UOZzLrAFyAo=
 k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e/go.mod h1:0CVn9SVo8PeW5/JgsBZZIFmmTk5noOM8WXf2e1tCihE=
 k8s.io/kubectl v0.29.7 h1:D+Jheug9M++zlt67cROZgxaIjrDdLqp9jkW/EYrXAoM=
@@ -451,8 +453,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsA
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
 sigs.k8s.io/cloud-provider-azure v1.31.1-0.20240914065912-f4dd79d54775 h1:0YqezUI2dBm+Y+XgoXA0+Atd2CDEGFq6PS/8vtgwbJI=
 sigs.k8s.io/cloud-provider-azure v1.31.1-0.20240914065912-f4dd79d54775/go.mod h1:ZMuwABqLK6ICPch/wMIeMdTs15yH1lkPlwenTVzaB2A=
-sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.56 h1:k71HScdrMkpf04udgySK7Jsw+bw90eQbaRssItA+ej4=
-sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.56/go.mod h1:kMZIHUHyI3TejvPoPVC9bPJgmOs3Wu7/dz0hxInU03o=
+sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.57 h1:Gt0aHqpju4eEtO9DoLLSZbKCjfH5fLmfCES7VGsiHHo=
+sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.57/go.mod h1:pCcUbyidPO6qrplCGARQY70n0E7ANUjmwR1xtAz/nng=
 sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.27 h1:o1LU+o0hAuY3esYQ5gzGElsCfkUNKCXmAIcBvf4CxZo=
 sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.27/go.mod h1:g/XTYItaIrR2AX3CGoFR0jIwitKedKBf6WwNJYXGoDw=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=

pkg/blob/azure.go

@@ -21,20 +21,19 @@ import (
 	"os"
 	"strings"
 
-	kv "github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault"
-	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	network "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
+	"github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets"
 	"github.com/Azure/azure-sdk-for-go/storage"
-	"github.com/Azure/go-autorest/autorest"
 	azure2 "github.com/Azure/go-autorest/autorest/azure"
 	"golang.org/x/net/context"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/ptr"
+	"sigs.k8s.io/cloud-provider-azure/pkg/azclient"
 	"sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader"
 	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
 	azure "sigs.k8s.io/cloud-provider-azure/pkg/provider"
-	providerconfig "sigs.k8s.io/cloud-provider-azure/pkg/provider/config"
-	"sigs.k8s.io/cloud-provider-azure/pkg/retry"
 )
 
 var (
@@ -147,46 +146,28 @@ func GetCloudProvider(ctx context.Context, kubeClient kubernetes.Interface, node
 
 // getKeyVaultSecretContent get content of the keyvault secret
 func (d *Driver) getKeyVaultSecretContent(ctx context.Context, vaultURL string, secretName string, secretVersion string) (content string, err error) {
-	kvClient, err := d.initializeKvClient()
+	var authProvider *azclient.AuthProvider
+	authProvider, err = azclient.NewAuthProvider(&d.cloud.AzureAuthConfig.ARMClientConfig, &d.cloud.AzureAuthConfig.AzureAuthConfig)
+	if err != nil {
+		return "", err
+	}
+	kvClient, err := azsecrets.NewClient(vaultURL, authProvider.GetAzIdentity(), nil)
 	if err != nil {
 		return "", fmt.Errorf("failed to get keyvaultClient: %w", err)
 	}
 
 	klog.V(2).Infof("get secret from vaultURL(%v), sercretName(%v), secretVersion(%v)", vaultURL, secretName, secretVersion)
-	secret, err := kvClient.GetSecret(ctx, vaultURL, secretName, secretVersion)
+	secret, err := kvClient.GetSecret(ctx, secretName, secretVersion, nil)
 	if err != nil {
 		return "", fmt.Errorf("get secret from vaultURL(%v), sercretName(%v), secretVersion(%v) failed with error: %w", vaultURL, secretName, secretVersion, err)
 	}
 	return *secret.Value, nil
 }
 
-func (d *Driver) initializeKvClient() (*kv.BaseClient, error) {
-	kvClient := kv.New()
-	token, err := d.getKeyvaultToken()
-	if err != nil {
-		return nil, err
-	}
-
-	kvClient.Authorizer = token
-	return &kvClient, nil
-}
-
-// getKeyvaultToken retrieves a new service principal token to access keyvault
-func (d *Driver) getKeyvaultToken() (authorizer autorest.Authorizer, err error) {
-	env := d.getCloudEnvironment()
-	kvEndPoint := strings.TrimSuffix(env.KeyVaultEndpoint, "/")
-	servicePrincipalToken, err := providerconfig.GetServicePrincipalToken(&d.cloud.AzureAuthConfig, &env, kvEndPoint)
-	if err != nil {
-		return nil, err
-	}
-	authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
-	return authorizer, nil
-}
-
 func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context, vnetResourceGroup, vnetName, subnetName string) ([]string, error) {
 	var vnetResourceIDs []string
-	if d.cloud.SubnetsClient == nil {
-		return vnetResourceIDs, fmt.Errorf("SubnetsClient is nil")
+	if d.networkClientFactory == nil {
+		return vnetResourceIDs, fmt.Errorf("networkClientFactory is nil")
 	}
 
 	if vnetResourceGroup == "" {
@@ -220,21 +201,21 @@ func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context, vnetResourceG
 	d.subnetLockMap.LockEntry(lockKey)
 	defer d.subnetLockMap.UnlockEntry(lockKey)
 
-	var subnets []network.Subnet
+	var subnets []*network.Subnet
 	if subnetName != "" {
 		// list multiple subnets separated by comma
 		subnetNames := strings.Split(subnetName, ",")
 		for _, sn := range subnetNames {
 			sn = strings.TrimSpace(sn)
-			subnet, rerr := d.cloud.SubnetsClient.Get(ctx, vnetResourceGroup, vnetName, sn, "")
+			subnet, rerr := d.networkClientFactory.GetSubnetClient().Get(ctx, vnetResourceGroup, vnetName, sn, nil)
 			if rerr != nil {
 				return vnetResourceIDs, fmt.Errorf("failed to get the subnet %s under rg %s vnet %s: %v", subnetName, vnetResourceGroup, vnetName, rerr.Error())
 			}
 			subnets = append(subnets, subnet)
 		}
 	} else {
-		var rerr *retry.Error
-		subnets, rerr = d.cloud.SubnetsClient.List(ctx, vnetResourceGroup, vnetName)
+		var rerr error
+		subnets, rerr = d.networkClientFactory.GetSubnetClient().List(ctx, vnetResourceGroup, vnetName)
 		if rerr != nil {
 			return vnetResourceIDs, fmt.Errorf("failed to list the subnets under rg %s vnet %s: %v", vnetResourceGroup, vnetName, rerr.Error())
 		}
@@ -249,19 +230,19 @@ func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context, vnetResourceG
 		klog.V(2).Infof("set vnetResourceID %s", vnetResourceID)
 		vnetResourceIDs = append(vnetResourceIDs, vnetResourceID)
 
-		endpointLocaions := []string{location}
-		storageServiceEndpoint := network.ServiceEndpointPropertiesFormat{
+		endpointLocaions := []*string{to.Ptr(location)}
+		storageServiceEndpoint := &network.ServiceEndpointPropertiesFormat{
 			Service:   &storageService,
-			Locations: &endpointLocaions,
+			Locations: endpointLocaions,
 		}
 		storageServiceExists := false
-		if subnet.SubnetPropertiesFormat == nil {
-			subnet.SubnetPropertiesFormat = &network.SubnetPropertiesFormat{}
+		if subnet.Properties == nil {
+			subnet.Properties = &network.SubnetPropertiesFormat{}
 		}
-		if subnet.SubnetPropertiesFormat.ServiceEndpoints == nil {
-			subnet.SubnetPropertiesFormat.ServiceEndpoints = &[]network.ServiceEndpointPropertiesFormat{}
+		if subnet.Properties.ServiceEndpoints == nil {
+			subnet.Properties.ServiceEndpoints = []*network.ServiceEndpointPropertiesFormat{}
 		}
-		serviceEndpoints := *subnet.SubnetPropertiesFormat.ServiceEndpoints
+		serviceEndpoints := subnet.Properties.ServiceEndpoints
 		for _, v := range serviceEndpoints {
 			if strings.HasPrefix(ptr.Deref(v.Service, ""), storageService) {
 				storageServiceExists = true
@@ -272,10 +253,10 @@ func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context, vnetResourceG
 
 		if !storageServiceExists {
 			serviceEndpoints = append(serviceEndpoints, storageServiceEndpoint)
-			subnet.SubnetPropertiesFormat.ServiceEndpoints = &serviceEndpoints
+			subnet.Properties.ServiceEndpoints = serviceEndpoints
 
 			klog.V(2).Infof("begin to update the subnet %s under vnet %s in rg %s", sn, vnetName, vnetResourceGroup)
-			if err := d.cloud.SubnetsClient.CreateOrUpdate(ctx, vnetResourceGroup, vnetName, sn, subnet); err != nil {
+			if _, err := d.networkClientFactory.GetSubnetClient().CreateOrUpdate(ctx, vnetResourceGroup, vnetName, sn, *subnet); err != nil {
 				return vnetResourceIDs, fmt.Errorf("failed to update the subnet %s under vnet %s: %v", sn, vnetName, err)
 			}
 		}