Skip to content

Commit bc94800

Browse files
authored
Merge pull request #1588 from MartinForReal/master
Migrate msi/subnet client to track2 one
2 parents b9cb1c0 + 2a1fa1a commit bc94800

File tree

110 files changed

+60026
-11487
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

110 files changed

+60026
-11487
lines changed

go.mod

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,12 @@ require (
77
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
88
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
99
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
10+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.0.0
1011
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
1112
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0
13+
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0
1214
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
1315
github.com/Azure/go-autorest/autorest v0.11.29
14-
github.com/Azure/go-autorest/autorest/adal v0.9.24
1516
github.com/container-storage-interface/spec v1.9.0
1617
github.com/go-ini/ini v1.67.0
1718
github.com/golang/protobuf v1.5.4
@@ -38,7 +39,7 @@ require (
3839
k8s.io/pod-security-admission v0.31.1
3940
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
4041
sigs.k8s.io/cloud-provider-azure v1.31.1-0.20240914065912-f4dd79d54775
41-
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.56
42+
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.57
4243
sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.27
4344
sigs.k8s.io/yaml v1.4.0
4445
)
@@ -47,14 +48,14 @@ require (
4748
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
4849
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 // indirect
4950
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 // indirect
51+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 // indirect
5052
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 // indirect
5153
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.0.0 // indirect
5254
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 // indirect
53-
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.0.0 // indirect
5455
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0 // indirect
55-
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0 // indirect
5656
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 // indirect
5757
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
58+
github.com/Azure/go-autorest/autorest/adal v0.9.24 // indirect
5859
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
5960
github.com/Azure/go-autorest/autorest/mocks v0.4.2 // indirect
6061
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
@@ -189,7 +190,7 @@ replace (
189190
k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.29.7
190191
k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.29.7
191192
k8s.io/endpointslice => k8s.io/endpointslice v0.29.7
192-
k8s.io/kms => k8s.io/kms v0.31.1
193+
k8s.io/kms => k8s.io/kms v0.29.7
193194
k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.29.7
194195
k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.29.7
195196
k8s.io/kube-proxy => k8s.io/kube-proxy v0.29.7

go.sum

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthoriza
1010
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0/go.mod h1:/pz8dyNQe+Ey3yBp/XuYz7oqX8YDNWVpPB0hH3XWfbc=
1111
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4=
1212
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I=
13+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 h1:zDeQI/PaWztI2tcrGO/9RIMey9NvqYbnyttf/0P3QWM=
14+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0/go.mod h1:zflC9v4VfViJrSvcvplqws/yGXVbUEMZi/iHpZdSPWA=
1315
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 h1:DWlwvVV5r/Wy1561nZ3wrpI1/vDIBRY/Wd1HWaRBZWA=
1416
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0/go.mod h1:E7ltexgRDmeJ0fJWv0D/HLwY2xbDdN+uv+X2uZtOx3w=
1517
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5 v5.0.0 h1:5n7dPVqsWfVKw+ZiEKSd3Kzu7gwBkbEBkeXb8rgaE9Q=
@@ -431,8 +433,8 @@ k8s.io/csi-translation-lib v0.29.7 h1:6z1iFhTmVMK9mebK2eodvDCKv3bfL0OFu5z2C8YNvM
431433
k8s.io/csi-translation-lib v0.29.7/go.mod h1:+5ZOwRS5LUQOghtqv6QWWmadixbm697xNHZC318oVf4=
432434
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
433435
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
434-
k8s.io/kms v0.31.1 h1:cGLyV3cIwb0ovpP/jtyIe2mEuQ/MkbhmeBF2IYCA9Io=
435-
k8s.io/kms v0.31.1/go.mod h1:OZKwl1fan3n3N5FFxnW5C4V3ygrah/3YXeJWS3O6+94=
436+
k8s.io/kms v0.29.7 h1:4ELQdx7T4EPKbN/QMj6SeZizrEKapza5YF8e5XtZPv0=
437+
k8s.io/kms v0.29.7/go.mod h1:vWVImKkJd+1BQY4tBwdfSwjQBiLrnbNtHADcDEDQFtk=
436438
k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e h1:OnKkExfhk4yxMqvBSPzUfhv3zQ96FWJ+UOZzLrAFyAo=
437439
k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e/go.mod h1:0CVn9SVo8PeW5/JgsBZZIFmmTk5noOM8WXf2e1tCihE=
438440
k8s.io/kubectl v0.29.7 h1:D+Jheug9M++zlt67cROZgxaIjrDdLqp9jkW/EYrXAoM=
@@ -451,8 +453,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsA
451453
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
452454
sigs.k8s.io/cloud-provider-azure v1.31.1-0.20240914065912-f4dd79d54775 h1:0YqezUI2dBm+Y+XgoXA0+Atd2CDEGFq6PS/8vtgwbJI=
453455
sigs.k8s.io/cloud-provider-azure v1.31.1-0.20240914065912-f4dd79d54775/go.mod h1:ZMuwABqLK6ICPch/wMIeMdTs15yH1lkPlwenTVzaB2A=
454-
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.56 h1:k71HScdrMkpf04udgySK7Jsw+bw90eQbaRssItA+ej4=
455-
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.56/go.mod h1:kMZIHUHyI3TejvPoPVC9bPJgmOs3Wu7/dz0hxInU03o=
456+
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.57 h1:Gt0aHqpju4eEtO9DoLLSZbKCjfH5fLmfCES7VGsiHHo=
457+
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.57/go.mod h1:pCcUbyidPO6qrplCGARQY70n0E7ANUjmwR1xtAz/nng=
456458
sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.27 h1:o1LU+o0hAuY3esYQ5gzGElsCfkUNKCXmAIcBvf4CxZo=
457459
sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.27/go.mod h1:g/XTYItaIrR2AX3CGoFR0jIwitKedKBf6WwNJYXGoDw=
458460
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=

pkg/blob/azure.go

Lines changed: 27 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -21,20 +21,19 @@ import (
2121
"os"
2222
"strings"
2323

24-
kv "github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault"
25-
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network"
24+
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
25+
network "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
26+
"github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets"
2627
"github.com/Azure/azure-sdk-for-go/storage"
27-
"github.com/Azure/go-autorest/autorest"
2828
azure2 "github.com/Azure/go-autorest/autorest/azure"
2929
"golang.org/x/net/context"
3030
"k8s.io/client-go/kubernetes"
3131
"k8s.io/klog/v2"
3232
"k8s.io/utils/ptr"
33+
"sigs.k8s.io/cloud-provider-azure/pkg/azclient"
3334
"sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader"
3435
azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
3536
azure "sigs.k8s.io/cloud-provider-azure/pkg/provider"
36-
providerconfig "sigs.k8s.io/cloud-provider-azure/pkg/provider/config"
37-
"sigs.k8s.io/cloud-provider-azure/pkg/retry"
3837
)
3938

4039
var (
@@ -147,46 +146,28 @@ func GetCloudProvider(ctx context.Context, kubeClient kubernetes.Interface, node
147146

148147
// getKeyVaultSecretContent get content of the keyvault secret
149148
func (d *Driver) getKeyVaultSecretContent(ctx context.Context, vaultURL string, secretName string, secretVersion string) (content string, err error) {
150-
kvClient, err := d.initializeKvClient()
149+
var authProvider *azclient.AuthProvider
150+
authProvider, err = azclient.NewAuthProvider(&d.cloud.AzureAuthConfig.ARMClientConfig, &d.cloud.AzureAuthConfig.AzureAuthConfig)
151+
if err != nil {
152+
return "", err
153+
}
154+
kvClient, err := azsecrets.NewClient(vaultURL, authProvider.GetAzIdentity(), nil)
151155
if err != nil {
152156
return "", fmt.Errorf("failed to get keyvaultClient: %w", err)
153157
}
154158

155159
klog.V(2).Infof("get secret from vaultURL(%v), sercretName(%v), secretVersion(%v)", vaultURL, secretName, secretVersion)
156-
secret, err := kvClient.GetSecret(ctx, vaultURL, secretName, secretVersion)
160+
secret, err := kvClient.GetSecret(ctx, secretName, secretVersion, nil)
157161
if err != nil {
158162
return "", fmt.Errorf("get secret from vaultURL(%v), sercretName(%v), secretVersion(%v) failed with error: %w", vaultURL, secretName, secretVersion, err)
159163
}
160164
return *secret.Value, nil
161165
}
162166

163-
func (d *Driver) initializeKvClient() (*kv.BaseClient, error) {
164-
kvClient := kv.New()
165-
token, err := d.getKeyvaultToken()
166-
if err != nil {
167-
return nil, err
168-
}
169-
170-
kvClient.Authorizer = token
171-
return &kvClient, nil
172-
}
173-
174-
// getKeyvaultToken retrieves a new service principal token to access keyvault
175-
func (d *Driver) getKeyvaultToken() (authorizer autorest.Authorizer, err error) {
176-
env := d.getCloudEnvironment()
177-
kvEndPoint := strings.TrimSuffix(env.KeyVaultEndpoint, "/")
178-
servicePrincipalToken, err := providerconfig.GetServicePrincipalToken(&d.cloud.AzureAuthConfig, &env, kvEndPoint)
179-
if err != nil {
180-
return nil, err
181-
}
182-
authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
183-
return authorizer, nil
184-
}
185-
186167
func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context, vnetResourceGroup, vnetName, subnetName string) ([]string, error) {
187168
var vnetResourceIDs []string
188-
if d.cloud.SubnetsClient == nil {
189-
return vnetResourceIDs, fmt.Errorf("SubnetsClient is nil")
169+
if d.networkClientFactory == nil {
170+
return vnetResourceIDs, fmt.Errorf("networkClientFactory is nil")
190171
}
191172

192173
if vnetResourceGroup == "" {
@@ -220,21 +201,21 @@ func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context, vnetResourceG
220201
d.subnetLockMap.LockEntry(lockKey)
221202
defer d.subnetLockMap.UnlockEntry(lockKey)
222203

223-
var subnets []network.Subnet
204+
var subnets []*network.Subnet
224205
if subnetName != "" {
225206
// list multiple subnets separated by comma
226207
subnetNames := strings.Split(subnetName, ",")
227208
for _, sn := range subnetNames {
228209
sn = strings.TrimSpace(sn)
229-
subnet, rerr := d.cloud.SubnetsClient.Get(ctx, vnetResourceGroup, vnetName, sn, "")
210+
subnet, rerr := d.networkClientFactory.GetSubnetClient().Get(ctx, vnetResourceGroup, vnetName, sn, nil)
230211
if rerr != nil {
231212
return vnetResourceIDs, fmt.Errorf("failed to get the subnet %s under rg %s vnet %s: %v", subnetName, vnetResourceGroup, vnetName, rerr.Error())
232213
}
233214
subnets = append(subnets, subnet)
234215
}
235216
} else {
236-
var rerr *retry.Error
237-
subnets, rerr = d.cloud.SubnetsClient.List(ctx, vnetResourceGroup, vnetName)
217+
var rerr error
218+
subnets, rerr = d.networkClientFactory.GetSubnetClient().List(ctx, vnetResourceGroup, vnetName)
238219
if rerr != nil {
239220
return vnetResourceIDs, fmt.Errorf("failed to list the subnets under rg %s vnet %s: %v", vnetResourceGroup, vnetName, rerr.Error())
240221
}
@@ -249,19 +230,19 @@ func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context, vnetResourceG
249230
klog.V(2).Infof("set vnetResourceID %s", vnetResourceID)
250231
vnetResourceIDs = append(vnetResourceIDs, vnetResourceID)
251232

252-
endpointLocaions := []string{location}
253-
storageServiceEndpoint := network.ServiceEndpointPropertiesFormat{
233+
endpointLocaions := []*string{to.Ptr(location)}
234+
storageServiceEndpoint := &network.ServiceEndpointPropertiesFormat{
254235
Service: &storageService,
255-
Locations: &endpointLocaions,
236+
Locations: endpointLocaions,
256237
}
257238
storageServiceExists := false
258-
if subnet.SubnetPropertiesFormat == nil {
259-
subnet.SubnetPropertiesFormat = &network.SubnetPropertiesFormat{}
239+
if subnet.Properties == nil {
240+
subnet.Properties = &network.SubnetPropertiesFormat{}
260241
}
261-
if subnet.SubnetPropertiesFormat.ServiceEndpoints == nil {
262-
subnet.SubnetPropertiesFormat.ServiceEndpoints = &[]network.ServiceEndpointPropertiesFormat{}
242+
if subnet.Properties.ServiceEndpoints == nil {
243+
subnet.Properties.ServiceEndpoints = []*network.ServiceEndpointPropertiesFormat{}
263244
}
264-
serviceEndpoints := *subnet.SubnetPropertiesFormat.ServiceEndpoints
245+
serviceEndpoints := subnet.Properties.ServiceEndpoints
265246
for _, v := range serviceEndpoints {
266247
if strings.HasPrefix(ptr.Deref(v.Service, ""), storageService) {
267248
storageServiceExists = true
@@ -272,10 +253,10 @@ func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context, vnetResourceG
272253

273254
if !storageServiceExists {
274255
serviceEndpoints = append(serviceEndpoints, storageServiceEndpoint)
275-
subnet.SubnetPropertiesFormat.ServiceEndpoints = &serviceEndpoints
256+
subnet.Properties.ServiceEndpoints = serviceEndpoints
276257

277258
klog.V(2).Infof("begin to update the subnet %s under vnet %s in rg %s", sn, vnetName, vnetResourceGroup)
278-
if err := d.cloud.SubnetsClient.CreateOrUpdate(ctx, vnetResourceGroup, vnetName, sn, subnet); err != nil {
259+
if _, err := d.networkClientFactory.GetSubnetClient().CreateOrUpdate(ctx, vnetResourceGroup, vnetName, sn, *subnet); err != nil {
279260
return vnetResourceIDs, fmt.Errorf("failed to update the subnet %s under vnet %s: %v", sn, vnetName, err)
280261
}
281262
}

0 commit comments

Comments
 (0)