Merge pull request #6686 from ywk253100/230612_kopia

Make Kopia support Azure AD

changelogs/unreleased/6686-ywk253100

@@ -0,0 +1 @@
+Make Kopia support Azure AD

go.mod

@@ -6,11 +6,15 @@ require (
 	cloud.google.com/go/storage v1.32.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-sdk-for-go v67.2.0+incompatible
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Azure/go-autorest/autorest v0.11.27
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.8
 	github.com/Azure/go-autorest/autorest/to v0.3.0
-	github.com/aws/aws-sdk-go v1.44.253
+	github.com/aws/aws-sdk-go v1.44.256
 	github.com/bombsimon/logrusr/v3 v3.0.0
 	github.com/evanphx/json-patch v5.6.0+incompatible
 	github.com/fatih/color v1.15.0
@@ -62,10 +66,7 @@ require (
 	cloud.google.com/go/compute v1.23.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect
 	github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect

go.sum

@@ -58,7 +58,10 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 h1:LNHhpdK7hzUcx/k1LIcuh
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1/go.mod h1:uE9zaUfEQT/nbQjVi2IblCG9iaLtZsuYZ8ne+PuQ02M=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0 h1:LcJtQjCXJUm1s7JpUHZvu+bpgURhCatxVNbGADXniX0=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0/go.mod h1:+OgGVo0Httq7N5oayfvaLQ/Jq+2gJdqfp++Hyyl7Tws=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4=
 github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk=
@@ -133,8 +136,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.44.253 h1:iqDd0okcH4ShfFexz2zzf4VmeDFf6NOMm07pHnEb8iY=
-github.com/aws/aws-sdk-go v1.44.253/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4=
+github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=

pkg/repository/config/azure.go

@@ -17,225 +17,34 @@ limitations under the License.
 package config
 
 import (
-	"context"
-	"fmt"
-	"os"
-	"strings"
-
-	storagemgmt "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage"
-	"github.com/Azure/go-autorest/autorest/azure"
-	"github.com/Azure/go-autorest/autorest/azure/auth"
-	"github.com/joho/godotenv"
 	"github.com/pkg/errors"
-)
-
-const (
-	subscriptionIDEnvVar = "AZURE_SUBSCRIPTION_ID"
-	cloudNameEnvVar      = "AZURE_CLOUD_NAME"
 
-	resourceGroupConfigKey = "resourceGroup"
-
-	storageAccountConfigKey          = "storageAccount"
-	storageAccountKeyEnvVarConfigKey = "storageAccountKeyEnvVar"
-	subscriptionIDConfigKey          = "subscriptionId"
-	storageDomainConfigKey           = "storageDomain"
+	"github.com/vmware-tanzu/velero/pkg/util/azure"
 )
 
-// getSubscriptionID gets the subscription ID from the 'config' map if it contains
-// it, else from the AZURE_SUBSCRIPTION_ID environment variable.
-func getSubscriptionID(config map[string]string) string {
-	if subscriptionID := config[subscriptionIDConfigKey]; subscriptionID != "" {
-		return subscriptionID
-	}
-
-	return os.Getenv(subscriptionIDEnvVar)
-}
-
-func getStorageAccountKey(config map[string]string) (string, error) {
-	credentialsFile := selectCredentialsFile(config)
-
-	if err := loadCredentialsIntoEnv(credentialsFile); err != nil {
-		return "", err
-	}
-
-	// Get Azure cloud from AZURE_CLOUD_NAME, if it exists. If the env var does not
-	// exist, parseAzureEnvironment will return azure.PublicCloud.
-	env, err := parseAzureEnvironment(os.Getenv(cloudNameEnvVar))
-	if err != nil {
-		return "", errors.Wrap(err, "unable to parse azure cloud name environment variable")
-	}
-
-	// Get storage key from secret using key config[storageAccountKeyEnvVarConfigKey]. If the config does not
-	// exist, continue obtaining it using API
-	if secretKeyEnvVar := config[storageAccountKeyEnvVarConfigKey]; secretKeyEnvVar != "" {
-		storageKey := os.Getenv(secretKeyEnvVar)
-		if storageKey == "" {
-			return "", errors.Errorf("no storage key secret with key %s found", secretKeyEnvVar)
-		}
-
-		return storageKey, nil
-	}
-
-	// get subscription ID from object store config or AZURE_SUBSCRIPTION_ID environment variable
-	subscriptionID := getSubscriptionID(config)
-	if subscriptionID == "" {
-		return "", errors.New("azure subscription ID not found in object store's config or in environment variable")
-	}
-
-	// we need config["resourceGroup"], config["storageAccount"]
-	if err := getRequiredValues(mapLookup(config), resourceGroupConfigKey, storageAccountConfigKey); err != nil {
-		return "", errors.Wrap(err, "unable to get all required config values")
-	}
-
-	// get authorizer from environment in the following order:
-	// 1. client credentials (AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET)
-	// 2. client certificate (AZURE_CERTIFICATE_PATH, AZURE_CERTIFICATE_PASSWORD)
-	// 3. username and password (AZURE_USERNAME, AZURE_PASSWORD)
-	// 4. MSI (managed service identity)
-	authorizer, err := auth.NewAuthorizerFromEnvironment()
-	if err != nil {
-		return "", errors.Wrap(err, "error getting authorizer from environment")
-	}
-
-	// get storageAccountsClient
-	storageAccountsClient := storagemgmt.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID)
-	storageAccountsClient.Authorizer = authorizer
-
-	// get storage key
-	res, err := storageAccountsClient.ListKeys(context.TODO(), config[resourceGroupConfigKey], config[storageAccountConfigKey], storagemgmt.Kerb)
-	if err != nil {
-		return "", errors.WithStack(err)
-	}
-	if res.Keys == nil || len(*res.Keys) == 0 {
-		return "", errors.New("No storage keys found")
-	}
-
-	var storageKey string
-	for _, key := range *res.Keys {
-		// The ListKeys call returns e.g. "FULL" but the storagemgmt.Full constant in the SDK is defined as "Full".
-		if strings.EqualFold(string(key.Permissions), string(storagemgmt.Full)) {
-			storageKey = *key.Value
-			break
-		}
-	}
-
-	if storageKey == "" {
-		return "", errors.New("No storage key with Full permissions found")
-	}
-
-	return storageKey, nil
-}
-
-func mapLookup(data map[string]string) func(string) string {
-	return func(key string) string {
-		return data[key]
-	}
-}
-
 // GetAzureResticEnvVars gets the environment variables that restic
 // relies on (AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY) based
 // on info in the provided object storage location config map.
 func GetAzureResticEnvVars(config map[string]string) (map[string]string, error) {
-	storageAccountKey, err := getStorageAccountKey(config)
-	if err != nil {
-		return nil, err
+	storageAccount := config[azure.BSLConfigStorageAccount]
+	if storageAccount == "" {
+		return nil, errors.New("storageAccount is required in the BSL")
 	}
 
-	if err := getRequiredValues(mapLookup(config), storageAccountConfigKey); err != nil {
-		return nil, errors.Wrap(err, "unable to get all required config values")
-	}
-
-	return map[string]string{
-		"AZURE_ACCOUNT_NAME": config[storageAccountConfigKey],
-		"AZURE_ACCOUNT_KEY":  storageAccountKey,
-	}, nil
-}
-
-// credentialsFileFromEnv retrieves the Azure credentials file from the environment.
-func credentialsFileFromEnv() string {
-	return os.Getenv("AZURE_CREDENTIALS_FILE")
-}
-
-// selectCredentialsFile selects the Azure credentials file to use, retrieving it
-// from the given config or falling back to retrieving it from the environment.
-func selectCredentialsFile(config map[string]string) string {
-	if credentialsFile, ok := config[CredentialsFileKey]; ok {
-		return credentialsFile
-	}
-
-	return credentialsFileFromEnv()
-}
-
-// loadCredentialsIntoEnv loads the variables in the given credentials
-// file into the current environment.
-func loadCredentialsIntoEnv(credentialsFile string) error {
-	if credentialsFile == "" {
-		return nil
-	}
-
-	if err := godotenv.Overload(credentialsFile); err != nil {
-		return errors.Wrapf(err, "error loading environment from credentials file (%s)", credentialsFile)
-	}
-
-	return nil
-}
-
-// ParseAzureEnvironment returns an azure.Environment for the given cloud
-// name, or azure.PublicCloud if cloudName is empty.
-func parseAzureEnvironment(cloudName string) (*azure.Environment, error) {
-	if cloudName == "" {
-		return &azure.PublicCloud, nil
-	}
-
-	env, err := azure.EnvironmentFromName(cloudName)
-	return &env, errors.WithStack(err)
-}
-
-func getRequiredValues(getValue func(string) string, keys ...string) error {
-	missing := []string{}
-	results := map[string]string{}
-
-	for _, key := range keys {
-		if val := getValue(key); val == "" {
-			missing = append(missing, key)
-		} else {
-			results[key] = val
-		}
-	}
-
-	if len(missing) > 0 {
-		return errors.Errorf("the following keys do not have values: %s", strings.Join(missing, ", "))
-	}
-
-	return nil
-}
-
-// GetAzureStorageDomain gets the Azure storage domain required by a Azure blob connection,
-// if the provided credential file doesn't have the value, get it from system's environment variables
-func GetAzureStorageDomain(config map[string]string) (string, error) {
-	credentialsFile := selectCredentialsFile(config)
-
-	if err := loadCredentialsIntoEnv(credentialsFile); err != nil {
-		return "", err
-	}
-
-	return getStorageDomainFromCloudName(os.Getenv(cloudNameEnvVar))
-}
-
-func GetAzureCredentials(config map[string]string) (string, string, error) {
-	storageAccountKey, err := getStorageAccountKey(config)
+	creds, err := azure.LoadCredentials(config)
 	if err != nil {
-		return "", "", err
+		return nil, err
 	}
 
-	return config[storageAccountConfigKey], storageAccountKey, nil
-}
-
-func getStorageDomainFromCloudName(cloudName string) (string, error) {
-	env, err := parseAzureEnvironment(cloudName)
+	// restic doesn't support Azure AD, set it as false
+	config[azure.BSLConfigUseAAD] = "false"
+	credentials, err := azure.GetStorageAccountCredentials(config, creds)
 	if err != nil {
-		return "", errors.Wrapf(err, "unable to parse azure env from cloud name %s", cloudName)
+		return nil, err
 	}
 
-	return fmt.Sprintf("blob.%s", env.StorageEndpointSuffix), nil
+	return map[string]string{
+		"AZURE_ACCOUNT_NAME": storageAccount,
+		"AZURE_ACCOUNT_KEY":  credentials[azure.CredentialKeyStorageAccountAccessKey],
+	}, nil
 }