CLOUDP-153003: Encryption at Rest Tests for GCP & Azure (#1035)

.github/workflows/test-e2e.yml

@@ -223,11 +223,11 @@ jobs:
           AWS_ACCOUNT_ARN_LIST: ${{ secrets.AWS_ACCOUNT_ARN_LIST }}
           AWS_KMS_KEY_ID: ${{ secrets.AWS_KMS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID_NEW_TEST }}
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
-          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET_NEW_TEST }}
           AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-          GCP_SA_CRED: ${{ secrets.GCP_SA_CRED }}
+          GCP_SA_CRED: ${{ secrets.GCP_SA_CRED_NEW_TEST }}
           DATADOG_KEY: ${{ secrets.DATADOG_KEY }}
         run: |
           helm version

go.mod

@@ -3,9 +3,11 @@ module github.com/mongodb/mongodb-atlas-kubernetes
 go 1.20
 
 require (
+	cloud.google.com/go/kms v1.12.1
 	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1
 	github.com/Azure/go-autorest/autorest v0.11.29
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
@@ -36,9 +38,11 @@ require (
 )
 
 require (
+	cloud.google.com/go/iam v1.1.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
+	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
 	github.com/google/s2a-go v0.1.4 // indirect

go.sum

@@ -26,6 +26,10 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94=
+cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk=
+cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U=
+cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
@@ -45,6 +49,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9Orh
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.2.0 h1:8d4U82r7ItT1Es91x3eUcAQweih36KWvUha8AZ9X0Rs=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.2.0/go.mod h1:/1bkGperHinQbAHMWivoec/Ucu6//iXo6jn5mhmqCVU=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 h1:bWh0Z2rOEDfB/ywv/l0iHN1JgyazE6kW/aIA89+CEK0=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1/go.mod h1:Bzf34hhAE9NSxailk8xVeLEZbUjOXcC+GnU1mMKdhLw=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU=
@@ -90,6 +96,7 @@ github.com/aws/aws-sdk-go v1.44.318 h1:Yl66rpbQHFUbxe9JBKLcvOvRivhVgP6+zH0b9KzAR
 github.com/aws/aws-sdk-go v1.44.318/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
+github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=

pkg/api/v1/encryption_at_rest.go

@@ -94,5 +94,6 @@ func (az AzureKeyVault) ToAtlas() mongodbatlas.AzureKeyVault {
 		KeyVaultName:      az.KeyVaultName,
 		KeyIdentifier:     az.KeyIdentifier,
 		TenantID:          az.TenantID,
+		Secret:            az.Secret,
 	}
 }

pkg/controller/atlasproject/encryption_at_rest.go

@@ -103,14 +103,16 @@ func readAndFillGoogleSecret(kubeClient client.Client, parentNs string, gkms *md
 }
 
 func readAndFillAzureSecret(kubeClient client.Client, parentNs string, azureVault *mdbv1.AzureKeyVault) (*watch.WatchedObject, error) {
-	fieldData, watchObj, err := readSecretData(kubeClient, azureVault.SecretRef, parentNs, "ClientID", "AzureEnvironment", "SubscriptionID", "ResourceGroupName", "KeyVaultName", "KeyIdentifier")
+	fieldData, watchObj, err := readSecretData(kubeClient, azureVault.SecretRef, parentNs, "ClientID", "Secret", "AzureEnvironment", "SubscriptionID", "ResourceGroupName", "KeyVaultName", "KeyIdentifier", "TenantID")
 	if err != nil {
 		return watchObj, err
 	}
 
 	azureVault.ClientID = fieldData["ClientID"]
+	azureVault.Secret = fieldData["Secret"]
 	azureVault.AzureEnvironment = fieldData["AzureEnvironment"]
 	azureVault.SubscriptionID = fieldData["SubscriptionID"]
+	azureVault.TenantID = fieldData["TenantID"]
 	azureVault.ResourceGroupName = fieldData["ResourceGroupName"]
 	azureVault.KeyVaultName = fieldData["KeyVaultName"]
 	azureVault.KeyIdentifier = fieldData["KeyIdentifier"]

pkg/controller/atlasproject/encryption_at_rest_test.go

@@ -211,8 +211,10 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) {
 	t.Run("Azure with correct secret data", func(t *testing.T) {
 		secretData := map[string][]byte{
 			"ClientID":          []byte("testClientID"),
+			"Secret":            []byte("testClientSecret"),
 			"AzureEnvironment":  []byte("testAzureEnvironment"),
 			"SubscriptionID":    []byte("testSubscriptionID"),
+			"TenantID":          []byte("testTenantID"),
 			"ResourceGroupName": []byte("testResourceGroupName"),
 			"KeyVaultName":      []byte("testKeyVaultName"),
 			"KeyIdentifier":     []byte("testKeyIdentifier"),
@@ -226,7 +228,7 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) {
 					APIVersion: "v1",
 				},
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      "gcp-secret",
+					Name:      "azure-secret",
 					Namespace: "test",
 				},
 			},
@@ -238,7 +240,7 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) {
 			AzureKeyVault: mdbv1.AzureKeyVault{
 				Enabled: toptr.MakePtr(true),
 				SecretRef: common.ResourceRefNamespaced{
-					Name: "gcp-secret",
+					Name: "azure-secret",
 				},
 			},
 		}
@@ -247,8 +249,10 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) {
 		assert.Nil(t, err)
 
 		assert.Equal(t, string(secretData["ClientID"]), encRest.AzureKeyVault.ClientID)
+		assert.Equal(t, string(secretData["Secret"]), encRest.AzureKeyVault.Secret)
 		assert.Equal(t, string(secretData["AzureEnvironment"]), encRest.AzureKeyVault.AzureEnvironment)
 		assert.Equal(t, string(secretData["SubscriptionID"]), encRest.AzureKeyVault.SubscriptionID)
+		assert.Equal(t, string(secretData["TenantID"]), encRest.AzureKeyVault.TenantID)
 		assert.Equal(t, string(secretData["ResourceGroupName"]), encRest.AzureKeyVault.ResourceGroupName)
 		assert.Equal(t, string(secretData["KeyVaultName"]), encRest.AzureKeyVault.KeyVaultName)
 		assert.Equal(t, string(secretData["KeyIdentifier"]), encRest.AzureKeyVault.KeyIdentifier)

test/e2e/actions/cloud/aws.go

@@ -1,16 +1,22 @@
 package cloud
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
+	"os"
+	"strings"
 
+	"github.com/aws/aws-sdk-go/aws"
 	aws_sdk "github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/kms"
 	"github.com/aws/aws-sdk-go/service/sts"
 	"github.com/onsi/ginkgo/v2/dsl/core"
 
 	"github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/toptr"
-	"github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/api/aws"
+	"github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/config"
 )
 
 type AwsAction struct {
@@ -25,13 +31,136 @@ type awsNetwork struct {
 	Subnets []*string
 }
 
-func NewAwsAction() *AwsAction {
-	return new(AwsAction)
+type principal struct {
+	AWS []string `json:"AWS,omitempty"`
 }
 
-func (awsAction *AwsAction) CreateKMS(region, atlasAccountArn, assumedRoleArn string) (key string, err error) {
-	session := aws.SessionAWS(region)
-	return session.GetCustomerMasterKeyID(atlasAccountArn, assumedRoleArn)
+type kmsPolicy struct {
+	Version   string      `json:"Version"`
+	Statement []statement `json:"Statement"`
+}
+
+type statement struct {
+	Sid       string    `json:"Sid"`
+	Effect    string    `json:"Effect"`
+	Principal principal `json:"Principal"`
+	Action    []string  `json:"Action"`
+	Resource  string    `json:"Resource"`
+}
+
+func (a *AwsAction) CreateKMS(region, atlasAccountArn, assumedRoleArn string) (key string, err error) {
+	a.t.Helper()
+
+	kmsClient := kms.New(a.session, aws.NewConfig().WithRegion(config.AWSRegionUS))
+
+	keyId, adminARNs, err := getKeyIDAndAdminARNs()
+	if err != nil {
+		return "", err
+	}
+
+	policyString, err := rolePolicyString(atlasAccountArn, assumedRoleArn, adminARNs)
+	if err != nil {
+		return "", err
+	}
+
+	policyInput := &kms.PutKeyPolicyInput{
+		KeyId:      &keyId,
+		PolicyName: aws_sdk.String("default"),
+		Policy:     aws_sdk.String(policyString),
+	}
+
+	_, err = kmsClient.PutKeyPolicy(policyInput)
+	if err != nil {
+		return "", err
+	}
+
+	return keyId, nil
+}
+
+func getKeyIDAndAdminARNs() (keyID string, adminARNs []string, err error) {
+	keyID = os.Getenv("AWS_KMS_KEY_ID")
+	if keyID == "" {
+		err = errors.New("AWS_KMS_KEY_ID secret is empty")
+		return
+	}
+	adminArnString := os.Getenv("AWS_ACCOUNT_ARN_LIST")
+	if adminArnString == "" {
+		err = errors.New("AWS_ACCOUNT_ARN_LIST secret is empty")
+		return
+	}
+
+	adminARNs = strings.Split(adminArnString, ",")
+	if len(adminARNs) == 0 {
+		err = errors.New("AWS_ACCOUNT_ARN_LIST wasn't parsed properly, please separate accounts via a comma")
+		return
+	}
+
+	return keyID, adminARNs, nil
+}
+
+func rolePolicyString(atlasAccountARN, assumedRoleARN string, adminARNs []string) (string, error) {
+	policy := defaultKMSPolicy(atlasAccountARN, assumedRoleARN, adminARNs)
+	byteStr, err := json.Marshal(policy)
+	if err != nil {
+		return "", err
+	}
+	return string(byteStr), nil
+}
+
+func defaultKMSPolicy(atlasAccountArn, assumedRoleArn string, adminARNs []string) kmsPolicy {
+	return kmsPolicy{
+		Version: "2012-10-17",
+		Statement: []statement{
+			{
+				Sid:    "Enable IAM User Permissions",
+				Effect: "Allow",
+				Principal: principal{
+					AWS: []string{atlasAccountArn},
+				},
+				Action:   []string{"kms:*"},
+				Resource: "*",
+			},
+			{
+				Sid:    "Allow access for Key Administrators",
+				Effect: "Allow",
+				Principal: principal{
+					AWS: adminARNs,
+				},
+				Action: []string{
+					"kms:Create*",
+					"kms:Describe*",
+					"kms:Enable*",
+					"kms:List*",
+					"kms:Put*",
+					"kms:Update*",
+					"kms:Revoke*",
+					"kms:Disable*",
+					"kms:Get*",
+					"kms:Delete*",
+					"kms:TagResource",
+					"kms:UntagResource",
+					"kms:ScheduleKeyDeletion",
+					"kms:CancelKeyDeletion",
+				},
+				Resource: "*",
+			},
+			{
+				Sid:    "Allow use of the key",
+				Effect: "Allow",
+				Principal: principal{
+					AWS: []string{assumedRoleArn},
+				},
+				Action: []string{
+					"kms:Encrypt",
+					"kms:Decrypt",
+					"kms:ReEncrypt*",
+					"kms:GenerateDataKey*",
+					"kms:DescribeKey",
+				},
+				Resource: "*",
+			},
+		},
+	}
 }
 
 func (a *AwsAction) GetAccountID() (string, error) {