Skip to content

Commit 94cce6a

Browse files
umagnusumagnus
committed
use cluster identity for azcopy
1 parent ded1639 commit 94cce6a

File tree

2 files changed

+193
-12
lines changed

2 files changed

+193
-12
lines changed

‎pkg/blob/controllerserver.go

Lines changed: 75 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"fmt"
2222
"net/url"
23+
"os"
2324
"os/exec"
2425
"strconv"
2526
"strings"
@@ -50,6 +51,12 @@ import (
5051
const (
5152
privateEndpoint = "privateendpoint"
5253

54+
azcopyAutoLoginType = "AZCOPY_AUTO_LOGIN_TYPE"
55+
azcopySPAApplicationID = "AZCOPY_SPA_APPLICATION_ID"
56+
azcopySPAClientSecret = "AZCOPY_SPA_CLIENT_SECRET"
57+
azcopyTenantID = "AZCOPY_TENANT_ID"
58+
azcopyMSIClientID = "AZCOPY_MSI_CLIENT_ID"
59+
5360
waitForCopyInterval = 5 * time.Second
5461
waitForCopyTimeout = 3 * time.Minute
5562
)
@@ -412,12 +419,11 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
412419
}
413420

414421
if req.GetVolumeContentSource() != nil {
415-
if accountKey == "" {
416-
if _, accountKey, err = d.GetStorageAccesskey(ctx, accountOptions, secrets, secretName, secretNamespace); err != nil {
417-
return nil, status.Errorf(codes.Internal, "failed to GetStorageAccesskey on account(%s) rg(%s), error: %v", accountOptions.Name, accountOptions.ResourceGroup, err)
418-
}
422+
var accountSASToken string
423+
if accountSASToken, err = d.useGenerateSASToken(ctx, accountName, accountKey, storageEndpointSuffix, accountOptions, secrets, secretName, secretNamespace); err != nil {
424+
return nil, err
419425
}
420-
if err := d.copyVolume(ctx, req, accountKey, validContainerName, storageEndpointSuffix); err != nil {
426+
if err := d.copyVolume(ctx, req, accountSASToken, validContainerName, storageEndpointSuffix); err != nil {
421427
return nil, err
422428
}
423429
} else {
@@ -712,7 +718,7 @@ func (d *Driver) DeleteBlobContainer(ctx context.Context, subsID, resourceGroupN
712718
}
713719

714720
// CopyBlobContainer copies a blob container in the same storage account
715-
func (d *Driver) copyBlobContainer(_ context.Context, req *csi.CreateVolumeRequest, accountKey, dstContainerName, storageEndpointSuffix string) error {
721+
func (d *Driver) copyBlobContainer(_ context.Context, req *csi.CreateVolumeRequest, accountSasToken, dstContainerName, storageEndpointSuffix string) error {
716722
var sourceVolumeID string
717723
if req.GetVolumeContentSource() != nil && req.GetVolumeContentSource().GetVolume() != nil {
718724
sourceVolumeID = req.GetVolumeContentSource().GetVolume().GetVolumeId()
@@ -726,10 +732,11 @@ func (d *Driver) copyBlobContainer(_ context.Context, req *csi.CreateVolumeReque
726732
return fmt.Errorf("srcContainerName(%s) or dstContainerName(%s) is empty", srcContainerName, dstContainerName)
727733
}
728734

729-
klog.V(2).Infof("generate sas token for account(%s)", accountName)
730-
accountSasToken, genErr := generateSASToken(accountName, accountKey, storageEndpointSuffix, d.sasTokenExpirationMinutes)
731-
if genErr != nil {
732-
return genErr
735+
if accountSasToken == "" {
736+
err = d.authorizeAzcopyBySecurityPrincipal()
737+
if err != nil {
738+
return err
739+
}
733740
}
734741

735742
timeAfter := time.After(waitForCopyTimeout)
@@ -768,18 +775,74 @@ func (d *Driver) copyBlobContainer(_ context.Context, req *csi.CreateVolumeReque
768775
}
769776

770777
// copyVolume copies a volume form volume or snapshot, snapshot is not supported now
771-
func (d *Driver) copyVolume(ctx context.Context, req *csi.CreateVolumeRequest, accountKey, dstContainerName, storageEndpointSuffix string) error {
778+
func (d *Driver) copyVolume(ctx context.Context, req *csi.CreateVolumeRequest, accountSASToken, dstContainerName, storageEndpointSuffix string) error {
772779
vs := req.VolumeContentSource
773780
switch vs.Type.(type) {
774781
case *csi.VolumeContentSource_Snapshot:
775782
return status.Errorf(codes.InvalidArgument, "copy volume from volumeSnapshot is not supported")
776783
case *csi.VolumeContentSource_Volume:
777-
return d.copyBlobContainer(ctx, req, accountKey, dstContainerName, storageEndpointSuffix)
784+
return d.copyBlobContainer(ctx, req, accountSASToken, dstContainerName, storageEndpointSuffix)
778785
default:
779786
return status.Errorf(codes.InvalidArgument, "%v is not a proper volume source", vs)
780787
}
781788
}
782789

790+
func (d *Driver) authorizeAzcopyBySecurityPrincipal() error {
791+
if len(d.cloud.Config.AzureAuthConfig.AADClientSecret) > 0 {
792+
klog.V(2).Infof("use service principal to authorize azcopy")
793+
if err := os.Setenv(azcopyAutoLoginType, "SPN"); err != nil {
794+
return err
795+
}
796+
if d.cloud.Config.AzureAuthConfig.AADClientID == "" {
797+
return fmt.Errorf("AADClientID and AADClientSecret must be set when use service principal")
798+
}
799+
if err := os.Setenv(azcopySPAApplicationID, d.cloud.Config.AzureAuthConfig.AADClientID); err != nil {
800+
return err
801+
}
802+
if err := os.Setenv(azcopySPAClientSecret, d.cloud.Config.AzureAuthConfig.AADClientSecret); err != nil {
803+
return err
804+
}
805+
if d.cloud.Config.AzureAuthConfig.TenantID != "" {
806+
if err := os.Setenv(azcopyTenantID, d.cloud.Config.AzureAuthConfig.TenantID); err != nil {
807+
return err
808+
}
809+
klog.V(2).Infof(fmt.Sprintf("set AZCOPY_TENANT_ID=%s successfully", d.cloud.Config.AzureAuthConfig.TenantID))
810+
}
811+
return nil
812+
}
813+
if d.cloud.Config.AzureAuthConfig.UseManagedIdentityExtension {
814+
if err := os.Setenv(azcopyAutoLoginType, "MSI"); err != nil {
815+
return err
816+
}
817+
if len(d.cloud.Config.AzureAuthConfig.UserAssignedIdentityID) > 0 {
818+
klog.V(2).Infof("use user assigned managed identity to authorize azcopy")
819+
err := os.Setenv(azcopyMSIClientID, d.cloud.Config.AzureAuthConfig.UserAssignedIdentityID)
820+
return err
821+
}
822+
klog.V(2).Infof("use system-assigned managed identity to authorize azcopy")
823+
return nil
824+
}
825+
return fmt.Errorf("AADClientSecret shouldn't be nil or useManagedIdentityExtension must be set to true")
826+
}
827+
828+
func (d *Driver) useGenerateSASToken(ctx context.Context, accountName, accountKey, storageEndpointSuffix string, accountOptions *azure.AccountOptions, secrets map[string]string, secretName, secretNamespace string) (string, error) {
829+
if len(secrets) > 0 && len(d.cloud.Config.AzureAuthConfig.AADClientSecret) == 0 && !d.cloud.Config.AzureAuthConfig.UseManagedIdentityExtension {
830+
var err error
831+
if accountKey == "" {
832+
if _, accountKey, err = d.GetStorageAccesskey(ctx, accountOptions, secrets, secretName, secretNamespace); err != nil {
833+
return "", status.Errorf(codes.Internal, "failed to GetStorageAccesskey on account(%s) rg(%s), error: %v", accountOptions.Name, accountOptions.ResourceGroup, err)
834+
}
835+
}
836+
klog.V(2).Infof("generate sas token for account(%s)", accountName)
837+
accountSASToken, err := generateSASToken(accountName, accountKey, storageEndpointSuffix, d.sasTokenExpirationMinutes)
838+
if err != nil {
839+
return "", err
840+
}
841+
return accountSASToken, nil
842+
}
843+
return "", nil
844+
}
845+
783846
// isValidVolumeCapabilities validates the given VolumeCapability array is valid
784847
func isValidVolumeCapabilities(volCaps []*csi.VolumeCapability) error {
785848
if len(volCaps) == 0 {

‎pkg/blob/controllerserver_test.go

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,11 @@ import (
3333
"k8s.io/apimachinery/pkg/util/wait"
3434
"k8s.io/utils/pointer"
3535
"sigs.k8s.io/blob-csi-driver/pkg/util"
36+
"sigs.k8s.io/cloud-provider-azure/pkg/azclient"
3637
"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/blobclient"
3738
"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/storageaccountclient/mockstorageaccountclient"
3839
azure "sigs.k8s.io/cloud-provider-azure/pkg/provider"
40+
"sigs.k8s.io/cloud-provider-azure/pkg/provider/config"
3941
"sigs.k8s.io/cloud-provider-azure/pkg/retry"
4042
)
4143

@@ -1780,3 +1782,119 @@ func Test_generateSASToken(t *testing.T) {
17801782
})
17811783
}
17821784
}
1785+
1786+
func Test_authorizeAzcopyBySecurityPrincipal(t *testing.T) {
1787+
testCases := []struct {
1788+
name string
1789+
testFunc func(t *testing.T)
1790+
}{
1791+
{
1792+
name: "use service principal to authorize azcopy",
1793+
testFunc: func(t *testing.T) {
1794+
d := NewFakeDriver()
1795+
d.cloud = &azure.Cloud{
1796+
Config: azure.Config{
1797+
AzureAuthConfig: config.AzureAuthConfig{
1798+
ARMClientConfig: azclient.ARMClientConfig{
1799+
TenantID: "TenantID",
1800+
},
1801+
AzureAuthConfig: azclient.AzureAuthConfig{
1802+
AADClientID: "AADClientID",
1803+
AADClientSecret: "AADClientSecret",
1804+
},
1805+
},
1806+
},
1807+
}
1808+
var expectedErr error
1809+
err := d.authorizeAzcopyBySecurityPrincipal()
1810+
if !reflect.DeepEqual(err, expectedErr) {
1811+
t.Errorf("Unexpected error: %v", err)
1812+
}
1813+
},
1814+
},
1815+
{
1816+
name: "use service principal to authorize azcopy but client id is empty",
1817+
testFunc: func(t *testing.T) {
1818+
d := NewFakeDriver()
1819+
d.cloud = &azure.Cloud{
1820+
Config: azure.Config{
1821+
AzureAuthConfig: config.AzureAuthConfig{
1822+
ARMClientConfig: azclient.ARMClientConfig{
1823+
TenantID: "TenantID",
1824+
},
1825+
AzureAuthConfig: azclient.AzureAuthConfig{
1826+
AADClientSecret: "AADClientSecret",
1827+
},
1828+
},
1829+
},
1830+
}
1831+
expectedErr := fmt.Errorf("AADClientID and AADClientSecret must be set when use service principal")
1832+
err := d.authorizeAzcopyBySecurityPrincipal()
1833+
if !reflect.DeepEqual(err, expectedErr) {
1834+
t.Errorf("Unexpected error: %v", err)
1835+
}
1836+
},
1837+
},
1838+
{
1839+
name: "use user assigned managed identity to authorize azcopy",
1840+
testFunc: func(t *testing.T) {
1841+
d := NewFakeDriver()
1842+
d.cloud = &azure.Cloud{
1843+
Config: azure.Config{
1844+
AzureAuthConfig: config.AzureAuthConfig{
1845+
AzureAuthConfig: azclient.AzureAuthConfig{
1846+
UseManagedIdentityExtension: true,
1847+
UserAssignedIdentityID: "UserAssignedIdentityID",
1848+
},
1849+
},
1850+
},
1851+
}
1852+
var expectedErr error
1853+
err := d.authorizeAzcopyBySecurityPrincipal()
1854+
if !reflect.DeepEqual(err, expectedErr) {
1855+
t.Errorf("Unexpected error: %v", err)
1856+
}
1857+
},
1858+
},
1859+
{
1860+
name: "use system assigned managed identity to authorize azcopy",
1861+
testFunc: func(t *testing.T) {
1862+
d := NewFakeDriver()
1863+
d.cloud = &azure.Cloud{
1864+
Config: azure.Config{
1865+
AzureAuthConfig: config.AzureAuthConfig{
1866+
AzureAuthConfig: azclient.AzureAuthConfig{
1867+
UseManagedIdentityExtension: true,
1868+
},
1869+
},
1870+
},
1871+
}
1872+
var expectedErr error
1873+
err := d.authorizeAzcopyBySecurityPrincipal()
1874+
if !reflect.DeepEqual(err, expectedErr) {
1875+
t.Errorf("Unexpected error: %v", err)
1876+
}
1877+
},
1878+
},
1879+
{
1880+
name: "AADClientSecret be nil and useManagedIdentityExtension is false",
1881+
testFunc: func(t *testing.T) {
1882+
d := NewFakeDriver()
1883+
d.cloud = &azure.Cloud{
1884+
Config: azure.Config{
1885+
AzureAuthConfig: config.AzureAuthConfig{},
1886+
},
1887+
}
1888+
expectedErr := fmt.Errorf("AADClientSecret shouldn't be nil or useManagedIdentityExtension must be set to true")
1889+
err := d.authorizeAzcopyBySecurityPrincipal()
1890+
if !reflect.DeepEqual(err, expectedErr) {
1891+
t.Errorf("Unexpected error: %v", err)
1892+
}
1893+
},
1894+
},
1895+
}
1896+
1897+
for _, tc := range testCases {
1898+
t.Run(tc.name, tc.testFunc)
1899+
}
1900+
}

0 commit comments

Comments
 (0)