diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 93d54d24d9..5536e9c3ab 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -223,11 +223,11 @@ jobs: AWS_ACCOUNT_ARN_LIST: ${{ secrets.AWS_ACCOUNT_ARN_LIST }} AWS_KMS_KEY_ID: ${{ secrets.AWS_KMS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID_NEW_TEST }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET_NEW_TEST }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - GCP_SA_CRED: ${{ secrets.GCP_SA_CRED }} + GCP_SA_CRED: ${{ secrets.GCP_SA_CRED_NEW_TEST }} DATADOG_KEY: ${{ secrets.DATADOG_KEY }} run: | helm version diff --git a/go.mod b/go.mod index a9e8af3d1a..6a5f7542ab 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,11 @@ module github.com/mongodb/mongodb-atlas-kubernetes go 1.20 require ( + cloud.google.com/go/kms v1.12.1 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 github.com/Azure/go-autorest/autorest v0.11.29 github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 @@ -36,9 +38,11 @@ require ( ) require ( + cloud.google.com/go/iam v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/s2a-go v0.1.4 // indirect diff --git a/go.sum b/go.sum index 82bed9ddf9..e71d9d1685 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,10 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94= +cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= +cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U= +cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -45,6 +49,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9Orh github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.2.0 h1:8d4U82r7ItT1Es91x3eUcAQweih36KWvUha8AZ9X0Rs= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.2.0/go.mod h1:/1bkGperHinQbAHMWivoec/Ucu6//iXo6jn5mhmqCVU= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 h1:bWh0Z2rOEDfB/ywv/l0iHN1JgyazE6kW/aIA89+CEK0= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1/go.mod h1:Bzf34hhAE9NSxailk8xVeLEZbUjOXcC+GnU1mMKdhLw= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= @@ -90,6 +96,7 @@ github.com/aws/aws-sdk-go v1.44.318 h1:Yl66rpbQHFUbxe9JBKLcvOvRivhVgP6+zH0b9KzAR github.com/aws/aws-sdk-go v1.44.318/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/pkg/api/v1/encryption_at_rest.go b/pkg/api/v1/encryption_at_rest.go index 4751c26226..13c4ba5cbc 100644 --- a/pkg/api/v1/encryption_at_rest.go +++ b/pkg/api/v1/encryption_at_rest.go @@ -94,5 +94,6 @@ func (az AzureKeyVault) ToAtlas() mongodbatlas.AzureKeyVault { KeyVaultName: az.KeyVaultName, KeyIdentifier: az.KeyIdentifier, TenantID: az.TenantID, + Secret: az.Secret, } } diff --git a/pkg/controller/atlasproject/encryption_at_rest.go b/pkg/controller/atlasproject/encryption_at_rest.go index 495c5445ae..646ba28186 100644 --- a/pkg/controller/atlasproject/encryption_at_rest.go +++ b/pkg/controller/atlasproject/encryption_at_rest.go @@ -103,14 +103,16 @@ func readAndFillGoogleSecret(kubeClient client.Client, parentNs string, gkms *md } func readAndFillAzureSecret(kubeClient client.Client, parentNs string, azureVault *mdbv1.AzureKeyVault) (*watch.WatchedObject, error) { - fieldData, watchObj, err := readSecretData(kubeClient, azureVault.SecretRef, parentNs, "ClientID", "AzureEnvironment", "SubscriptionID", "ResourceGroupName", "KeyVaultName", "KeyIdentifier") + fieldData, watchObj, err := readSecretData(kubeClient, azureVault.SecretRef, parentNs, "ClientID", "Secret", "AzureEnvironment", "SubscriptionID", "ResourceGroupName", "KeyVaultName", "KeyIdentifier", "TenantID") if err != nil { return watchObj, err } azureVault.ClientID = fieldData["ClientID"] + azureVault.Secret = fieldData["Secret"] azureVault.AzureEnvironment = fieldData["AzureEnvironment"] azureVault.SubscriptionID = fieldData["SubscriptionID"] + azureVault.TenantID = fieldData["TenantID"] azureVault.ResourceGroupName = fieldData["ResourceGroupName"] azureVault.KeyVaultName = fieldData["KeyVaultName"] azureVault.KeyIdentifier = fieldData["KeyIdentifier"] diff --git a/pkg/controller/atlasproject/encryption_at_rest_test.go b/pkg/controller/atlasproject/encryption_at_rest_test.go index 28f73969c1..b04b694900 100644 --- a/pkg/controller/atlasproject/encryption_at_rest_test.go +++ b/pkg/controller/atlasproject/encryption_at_rest_test.go @@ -211,8 +211,10 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) { t.Run("Azure with correct secret data", func(t *testing.T) { secretData := map[string][]byte{ "ClientID": []byte("testClientID"), + "Secret": []byte("testClientSecret"), "AzureEnvironment": []byte("testAzureEnvironment"), "SubscriptionID": []byte("testSubscriptionID"), + "TenantID": []byte("testTenantID"), "ResourceGroupName": []byte("testResourceGroupName"), "KeyVaultName": []byte("testKeyVaultName"), "KeyIdentifier": []byte("testKeyIdentifier"), @@ -226,7 +228,7 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) { APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "gcp-secret", + Name: "azure-secret", Namespace: "test", }, }, @@ -238,7 +240,7 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) { AzureKeyVault: mdbv1.AzureKeyVault{ Enabled: toptr.MakePtr(true), SecretRef: common.ResourceRefNamespaced{ - Name: "gcp-secret", + Name: "azure-secret", }, }, } @@ -247,8 +249,10 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) { assert.Nil(t, err) assert.Equal(t, string(secretData["ClientID"]), encRest.AzureKeyVault.ClientID) + assert.Equal(t, string(secretData["Secret"]), encRest.AzureKeyVault.Secret) assert.Equal(t, string(secretData["AzureEnvironment"]), encRest.AzureKeyVault.AzureEnvironment) assert.Equal(t, string(secretData["SubscriptionID"]), encRest.AzureKeyVault.SubscriptionID) + assert.Equal(t, string(secretData["TenantID"]), encRest.AzureKeyVault.TenantID) assert.Equal(t, string(secretData["ResourceGroupName"]), encRest.AzureKeyVault.ResourceGroupName) assert.Equal(t, string(secretData["KeyVaultName"]), encRest.AzureKeyVault.KeyVaultName) assert.Equal(t, string(secretData["KeyIdentifier"]), encRest.AzureKeyVault.KeyIdentifier) diff --git a/test/e2e/actions/cloud/aws.go b/test/e2e/actions/cloud/aws.go index 8436d7e942..f7d7ca804c 100644 --- a/test/e2e/actions/cloud/aws.go +++ b/test/e2e/actions/cloud/aws.go @@ -1,16 +1,22 @@ package cloud import ( + "encoding/json" + "errors" "fmt" + "os" + "strings" + "github.com/aws/aws-sdk-go/aws" aws_sdk "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/kms" "github.com/aws/aws-sdk-go/service/sts" "github.com/onsi/ginkgo/v2/dsl/core" "github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/toptr" - "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/api/aws" + "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/config" ) type AwsAction struct { @@ -25,13 +31,136 @@ type awsNetwork struct { Subnets []*string } -func NewAwsAction() *AwsAction { - return new(AwsAction) +type principal struct { + AWS []string `json:"AWS,omitempty"` } -func (awsAction *AwsAction) CreateKMS(region, atlasAccountArn, assumedRoleArn string) (key string, err error) { - session := aws.SessionAWS(region) - return session.GetCustomerMasterKeyID(atlasAccountArn, assumedRoleArn) +type kmsPolicy struct { + Version string `json:"Version"` + Statement []statement `json:"Statement"` +} + +type statement struct { + Sid string `json:"Sid"` + Effect string `json:"Effect"` + Principal principal `json:"Principal"` + Action []string `json:"Action"` + Resource string `json:"Resource"` +} + +func (a *AwsAction) CreateKMS(region, atlasAccountArn, assumedRoleArn string) (key string, err error) { + a.t.Helper() + + kmsClient := kms.New(a.session, aws.NewConfig().WithRegion(config.AWSRegionUS)) + + keyId, adminARNs, err := getKeyIDAndAdminARNs() + if err != nil { + return "", err + } + + policyString, err := rolePolicyString(atlasAccountArn, assumedRoleArn, adminARNs) + if err != nil { + return "", err + } + + policyInput := &kms.PutKeyPolicyInput{ + KeyId: &keyId, + PolicyName: aws_sdk.String("default"), + Policy: aws_sdk.String(policyString), + } + + _, err = kmsClient.PutKeyPolicy(policyInput) + if err != nil { + return "", err + } + + return keyId, nil +} + +func getKeyIDAndAdminARNs() (keyID string, adminARNs []string, err error) { + keyID = os.Getenv("AWS_KMS_KEY_ID") + if keyID == "" { + err = errors.New("AWS_KMS_KEY_ID secret is empty") + return + } + adminArnString := os.Getenv("AWS_ACCOUNT_ARN_LIST") + if adminArnString == "" { + err = errors.New("AWS_ACCOUNT_ARN_LIST secret is empty") + return + } + + adminARNs = strings.Split(adminArnString, ",") + if len(adminARNs) == 0 { + err = errors.New("AWS_ACCOUNT_ARN_LIST wasn't parsed properly, please separate accounts via a comma") + return + } + + return keyID, adminARNs, nil +} + +func rolePolicyString(atlasAccountARN, assumedRoleARN string, adminARNs []string) (string, error) { + policy := defaultKMSPolicy(atlasAccountARN, assumedRoleARN, adminARNs) + byteStr, err := json.Marshal(policy) + if err != nil { + return "", err + } + return string(byteStr), nil +} + +func defaultKMSPolicy(atlasAccountArn, assumedRoleArn string, adminARNs []string) kmsPolicy { + return kmsPolicy{ + Version: "2012-10-17", + Statement: []statement{ + { + Sid: "Enable IAM User Permissions", + Effect: "Allow", + Principal: principal{ + AWS: []string{atlasAccountArn}, + }, + Action: []string{"kms:*"}, + Resource: "*", + }, + { + Sid: "Allow access for Key Administrators", + Effect: "Allow", + Principal: principal{ + AWS: adminARNs, + }, + Action: []string{ + "kms:Create*", + "kms:Describe*", + "kms:Enable*", + "kms:List*", + "kms:Put*", + "kms:Update*", + "kms:Revoke*", + "kms:Disable*", + "kms:Get*", + "kms:Delete*", + "kms:TagResource", + "kms:UntagResource", + "kms:ScheduleKeyDeletion", + "kms:CancelKeyDeletion", + }, + Resource: "*", + }, + { + Sid: "Allow use of the key", + Effect: "Allow", + Principal: principal{ + AWS: []string{assumedRoleArn}, + }, + Action: []string{ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey", + }, + Resource: "*", + }, + }, + } } func (a *AwsAction) GetAccountID() (string, error) { diff --git a/test/e2e/actions/cloud/azure.go b/test/e2e/actions/cloud/azure.go index c84af42b97..2791d2b02d 100644 --- a/test/e2e/actions/cloud/azure.go +++ b/test/e2e/actions/cloud/azure.go @@ -8,18 +8,25 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" "github.com/onsi/ginkgo/v2/dsl/core" "github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/toptr" ) +const ( + AzureKeyVaultName = "ako-kms-test" +) + type AzureAction struct { t core.GinkgoTInterface resourceGroupName string network *azureNetwork + credentials *azidentity.DefaultAzureCredential - resourceFactory *armnetwork.ClientFactory + networkResourceFactory *armnetwork.ClientFactory + keyVaultResourceFactory *armkeyvault.ClientFactory } type azureNetwork struct { @@ -102,7 +109,7 @@ func (a *AzureAction) CreatePrivateEndpoint(vpcName, subnetName, endpointName, s } }) - networkClient := a.resourceFactory.NewPrivateEndpointsClient() + networkClient := a.networkResourceFactory.NewPrivateEndpointsClient() op, err := networkClient.BeginCreateOrUpdate( ctx, a.resourceGroupName, @@ -146,7 +153,7 @@ func (a *AzureAction) CreatePrivateEndpoint(vpcName, subnetName, endpointName, s func (a *AzureAction) GetPrivateEndpoint(endpointName string) (*armnetwork.PrivateEndpoint, error) { a.t.Helper() - networkClient := a.resourceFactory.NewPrivateEndpointsClient() + networkClient := a.networkResourceFactory.NewPrivateEndpointsClient() pe, err := networkClient.Get( context.Background(), a.resourceGroupName, @@ -163,7 +170,7 @@ func (a *AzureAction) GetPrivateEndpoint(endpointName string) (*armnetwork.Priva func (a *AzureAction) GetInterface(name string) (*armnetwork.Interface, error) { a.t.Helper() - interfaceClient := a.resourceFactory.NewInterfacesClient() + interfaceClient := a.networkResourceFactory.NewInterfacesClient() i, err := interfaceClient.Get( context.Background(), a.resourceGroupName, @@ -180,7 +187,7 @@ func (a *AzureAction) GetInterface(name string) (*armnetwork.Interface, error) { func (a *AzureAction) findVpc(ctx context.Context, vpcName string) (*armnetwork.VirtualNetwork, error) { a.t.Helper() - vpcClient := a.resourceFactory.NewVirtualNetworksClient() + vpcClient := a.networkResourceFactory.NewVirtualNetworksClient() vpc, err := vpcClient.Get(ctx, a.resourceGroupName, vpcName, nil) if err != nil { @@ -196,7 +203,7 @@ func (a *AzureAction) findVpc(ctx context.Context, vpcName string) (*armnetwork. func (a *AzureAction) createVpcWithSubnets(ctx context.Context, vpcName, cidr, region string, subnets map[string]string) (*armnetwork.VirtualNetwork, error) { a.t.Helper() - vpcClient := a.resourceFactory.NewVirtualNetworksClient() + vpcClient := a.networkResourceFactory.NewVirtualNetworksClient() subnetsSpec := make([]*armnetwork.Subnet, 0, len(subnets)) for name, ipRange := range subnets { @@ -245,7 +252,7 @@ func (a *AzureAction) createVpcWithSubnets(ctx context.Context, vpcName, cidr, r func (a *AzureAction) deleteVpc(ctx context.Context, vpcName string) error { a.t.Helper() - vpcClient := a.resourceFactory.NewVirtualNetworksClient() + vpcClient := a.networkResourceFactory.NewVirtualNetworksClient() op, err := vpcClient.BeginDelete( ctx, @@ -264,7 +271,7 @@ func (a *AzureAction) deleteVpc(ctx context.Context, vpcName string) error { func (a *AzureAction) createSubnet(ctx context.Context, vpcName, subnetName, ipRange string) (*armnetwork.Subnet, error) { a.t.Helper() - subnetClient := a.resourceFactory.NewSubnetsClient() + subnetClient := a.networkResourceFactory.NewSubnetsClient() op, err := subnetClient.BeginCreateOrUpdate( ctx, @@ -293,7 +300,7 @@ func (a *AzureAction) createSubnet(ctx context.Context, vpcName, subnetName, ipR func (a *AzureAction) deleteSubnet(ctx context.Context, vpcName, subnetName string) error { a.t.Helper() - subnetClient := a.resourceFactory.NewSubnetsClient() + subnetClient := a.networkResourceFactory.NewSubnetsClient() op, err := subnetClient.BeginDelete( ctx, @@ -313,7 +320,7 @@ func (a *AzureAction) deleteSubnet(ctx context.Context, vpcName, subnetName stri func (a *AzureAction) disableSubnetPENetworkPolicy(ctx context.Context, vpcName, subnetName string) (*armnetwork.Subnet, error) { a.t.Helper() - subnetClient := a.resourceFactory.NewSubnetsClient() + subnetClient := a.networkResourceFactory.NewSubnetsClient() subnet, ok := a.network.Subnets[subnetName] if !ok { @@ -343,7 +350,7 @@ func (a *AzureAction) disableSubnetPENetworkPolicy(ctx context.Context, vpcName, func (a *AzureAction) enableSubnetPENetworkPolicy(ctx context.Context, vpcName, subnetName string) (*armnetwork.Subnet, error) { a.t.Helper() - subnetClient := a.resourceFactory.NewSubnetsClient() + subnetClient := a.networkResourceFactory.NewSubnetsClient() subnet, ok := a.network.Subnets[subnetName] if !ok { @@ -373,7 +380,7 @@ func (a *AzureAction) enableSubnetPENetworkPolicy(ctx context.Context, vpcName, func (a *AzureAction) deletePrivateEndpoint(ctx context.Context, endpointName string) error { a.t.Helper() - networkClient := a.resourceFactory.NewPrivateEndpointsClient() + networkClient := a.networkResourceFactory.NewPrivateEndpointsClient() op, err := networkClient.BeginDelete( ctx, a.resourceGroupName, @@ -392,6 +399,25 @@ func (a *AzureAction) deletePrivateEndpoint(ctx context.Context, endpointName st return nil } +func (a *AzureAction) CreateKeyVault(keyName string) (string, error) { + a.t.Helper() + + ctx := context.Background() + + params := armkeyvault.KeyCreateParameters{ + Properties: &armkeyvault.KeyProperties{ + Kty: toptr.MakePtr(armkeyvault.JSONWebKeyTypeRSA), + }, + } + + r, err := a.keyVaultResourceFactory.NewKeysClient().CreateIfNotExist(ctx, a.resourceGroupName, AzureKeyVaultName, keyName, params, nil) + if err != nil { + return "", err + } + + return *r.Properties.KeyURIWithVersion, nil +} + func NewAzureAction(t core.GinkgoTInterface, subscriptionID, resourceGroupName string) (*AzureAction, error) { t.Helper() @@ -400,14 +426,21 @@ func NewAzureAction(t core.GinkgoTInterface, subscriptionID, resourceGroupName s return nil, err } - factory, err := armnetwork.NewClientFactory(subscriptionID, cred, nil) + networkFactory, err := armnetwork.NewClientFactory(subscriptionID, cred, nil) + if err != nil { + return nil, err + } + + vaultFactory, err := armkeyvault.NewClientFactory(subscriptionID, cred, nil) if err != nil { return nil, err } return &AzureAction{ - t: t, - resourceGroupName: resourceGroupName, - resourceFactory: factory, + t: t, + resourceGroupName: resourceGroupName, + networkResourceFactory: networkFactory, + keyVaultResourceFactory: vaultFactory, + credentials: cred, }, err } diff --git a/test/e2e/actions/cloud/gcp.go b/test/e2e/actions/cloud/gcp.go index 5db1ca4b39..06e8776c77 100644 --- a/test/e2e/actions/cloud/gcp.go +++ b/test/e2e/actions/cloud/gcp.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "time" "google.golang.org/api/googleapi" @@ -17,6 +18,9 @@ import ( compute "cloud.google.com/go/compute/apiv1" "cloud.google.com/go/compute/apiv1/computepb" + + kms "cloud.google.com/go/kms/apiv1" + "cloud.google.com/go/kms/apiv1/kmspb" ) type GCPAction struct { @@ -24,10 +28,11 @@ type GCPAction struct { projectID string network *gcpNetwork - networkClient *compute.NetworksClient - subnetClient *compute.SubnetworksClient - addressClient *compute.AddressesClient - forwardRuleClient *compute.ForwardingRulesClient + networkClient *compute.NetworksClient + subnetClient *compute.SubnetworksClient + addressClient *compute.AddressesClient + forwardRuleClient *compute.ForwardingRulesClient + keyManagementClient *kms.KeyManagementClient } type gcpNetwork struct { @@ -40,6 +45,7 @@ const ( GoogleProjectID = "atlasoperator" // Google Cloud Project ID googleConnectPrefix = "ao" // Private Service Connect Endpoint Prefix gcpSubnetIPMask = "10.0.0.%d" + googleKeyName = "projects/atlasoperator/locations/global/keyRings/atlas-operator-test-key-ring/cryptoKeys/encryption-at-rest-test-key" ) func (a *GCPAction) InitNetwork(vpcName, region string, subnets map[string]string, cleanup bool) (string, error) { @@ -459,6 +465,54 @@ func (a *GCPAction) deleteVPCPeering(ctx context.Context, vpcName, peerName stri return nil } +func (a *GCPAction) CreateKMS() (string, error) { + a.t.Helper() + + ctx := context.Background() + + result, err := a.keyManagementClient.CreateCryptoKeyVersion(ctx, &kmspb.CreateCryptoKeyVersionRequest{ + Parent: googleKeyName, + }) + if err != nil { + return "", err + } + + a.t.Cleanup(func() { + err = a.deleteKMS(ctx, result.Name) + if err != nil { + a.t.Error(err) + } + }) + + ver := strings.Split(result.Name, "/") + keyVersion := ver[len(ver)-1] + + _, err = a.keyManagementClient.UpdateCryptoKeyPrimaryVersion(ctx, &kmspb.UpdateCryptoKeyPrimaryVersionRequest{ + Name: googleKeyName, + CryptoKeyVersionId: keyVersion, + }) + if err != nil { + return "", err + } + + return result.Name, nil +} + +func (a *GCPAction) deleteKMS(ctx context.Context, keyName string) error { + a.t.Helper() + + req := &kmspb.DestroyCryptoKeyVersionRequest{ + Name: keyName, + } + + _, err := a.keyManagementClient.DestroyCryptoKeyVersion(ctx, req) + if err != nil { + return err + } + + return nil +} + func NewGCPAction(t core.GinkgoTInterface, projectID string) (*GCPAction, error) { t.Helper() @@ -484,13 +538,19 @@ func NewGCPAction(t core.GinkgoTInterface, projectID string) (*GCPAction, error) return nil, err } + keyManagementClient, err := kms.NewKeyManagementClient(ctx) + if err != nil { + return nil, err + } + return &GCPAction{ t: t, projectID: projectID, - networkClient: networkClient, - subnetClient: subnetClient, - addressClient: addressClient, - forwardRuleClient: forwardRuleClient, + networkClient: networkClient, + subnetClient: subnetClient, + addressClient: addressClient, + forwardRuleClient: forwardRuleClient, + keyManagementClient: keyManagementClient, }, nil } diff --git a/test/e2e/api/aws/aws.go b/test/e2e/api/aws/aws.go deleted file mode 100644 index a98564315c..0000000000 --- a/test/e2e/api/aws/aws.go +++ /dev/null @@ -1,354 +0,0 @@ -package aws - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - - "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/config" - - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/kms" -) - -type sessionAWS struct { - ec2 *ec2.EC2 - kms *kms.KMS -} - -func SessionAWS(region string) sessionAWS { - session, err := session.NewSession(&aws.Config{ - Region: aws.String(region), - }) - if err != nil { - fmt.Println(err) - } - svc := ec2.New(session) - kms := kms.New(session) - return sessionAWS{svc, kms} -} - -func (s sessionAWS) GetVPCID() (string, error) { - input := &ec2.DescribeVpcsInput{ - Filters: []*ec2.Filter{{ - Name: aws.String("tag:Name"), - Values: []*string{ - aws.String(config.TagName), - }, - }}, - } - result, err := s.ec2.DescribeVpcs(input) - if err != nil { - return "", getError(err) - } - if len(result.Vpcs) < 1 { - return "", errors.New("can not find VPC") - } - fmt.Println(result) - return *result.Vpcs[0].VpcId, nil -} - -func (s sessionAWS) CreateVPC(testID string) (string, error) { - input := &ec2.CreateVpcInput{ - AmazonProvidedIpv6CidrBlock: aws.Bool(false), - CidrBlock: aws.String("10.0.0.0/24"), - TagSpecifications: []*ec2.TagSpecification{{ - ResourceType: aws.String(ec2.ResourceTypeVpc), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String(config.TagName)}, - {Key: aws.String("Test"), Value: aws.String(testID)}, - }, - }}, - } - result, err := s.ec2.CreateVpc(input) - if err != nil { - return "", getError(err) - } - fmt.Println(result) - return *result.Vpc.VpcId, nil -} - -func (s sessionAWS) DescribeVPCStatus(vpcID string) (string, error) { - input := &ec2.DescribeVpcsInput{ - DryRun: aws.Bool(false), - VpcIds: []*string{aws.String(vpcID)}, - } - result, err := s.ec2.DescribeVpcs(input) - if err != nil { - return "", getError(err) - } - return *result.Vpcs[0].State, nil -} - -func (s sessionAWS) DeleteVPC(vpcID string) error { - input := &ec2.DeleteVpcInput{ - DryRun: aws.Bool(false), - VpcId: aws.String(vpcID), - } - _, err := s.ec2.DeleteVpc(input) - if err != nil { - return getError(err) - } - return nil -} - -func (s sessionAWS) GetOrCreateSubnetIDForVpc(vpcID string) (string, error) { - input := &ec2.DescribeSubnetsInput{ - Filters: []*ec2.Filter{ - { - Name: aws.String("vpc-id"), - Values: []*string{ - aws.String(vpcID), - }, - }, - }, - } - - result, err := s.ec2.DescribeSubnets(input) - if err != nil { - return "", getError(err) - } - - if len(result.Subnets) > 0 { - return *result.Subnets[0].SubnetId, nil - } - - return s.CreateSubnet(vpcID, "10.0.0.0/24", "pe-aws-test") -} - -func (s sessionAWS) CreateSubnet(vpcID, cidr, testID string) (string, error) { - input := &ec2.CreateSubnetInput{ - CidrBlock: aws.String(cidr), - TagSpecifications: []*ec2.TagSpecification{{ - ResourceType: aws.String(ec2.ResourceTypeSubnet), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String(config.TagName)}, - {Key: aws.String("Test"), Value: aws.String(testID)}, - }, - }}, - VpcId: aws.String(vpcID), - } - result, err := s.ec2.CreateSubnet(input) - if err != nil { - return "", getError(err) - } - fmt.Println(result) - return *result.Subnet.SubnetId, nil -} - -func (s sessionAWS) DescribeSubnetStatus(subnetID string) (string, error) { - input := &ec2.DescribeSubnetsInput{ - SubnetIds: []*string{aws.String(subnetID)}, - } - result, err := s.ec2.DescribeSubnets(input) - if err != nil { - return "", getError(err) - } - return *result.Subnets[0].State, nil -} - -func (s sessionAWS) DeleteSubnet(subnetID string) error { - input := &ec2.DeleteSubnetInput{ - SubnetId: aws.String(subnetID), - } - _, err := s.ec2.DeleteSubnet(input) - if err != nil { - return getError(err) - } - return nil -} - -func (s sessionAWS) CreatePrivateEndpoint(vpcID, subnetID, serviceName, testID string) (string, error) { - input := &ec2.CreateVpcEndpointInput{ - ServiceName: aws.String(serviceName), - SubnetIds: []*string{aws.String(subnetID)}, - TagSpecifications: []*ec2.TagSpecification{{ - ResourceType: aws.String(ec2.ResourceTypeVpcEndpoint), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String(config.TagName)}, - {Key: aws.String("Test"), Value: aws.String(testID)}, - {Key: aws.String(config.TagForTestKey), Value: aws.String(config.TagForTestValue)}, - }, - }}, - VpcEndpointType: aws.String("Interface"), - VpcId: aws.String(vpcID), - } - result, err := s.ec2.CreateVpcEndpoint(input) - if err != nil { - return "", getError(err) - } - fmt.Println(result) - return *result.VpcEndpoint.VpcEndpointId, nil -} - -func (s sessionAWS) DescribePrivateEndpointStatus(endpointID string) (string, error) { - input := &ec2.DescribeVpcEndpointsInput{ - VpcEndpointIds: []*string{aws.String(endpointID)}, - } - result, err := s.ec2.DescribeVpcEndpoints(input) - if err != nil { - return "", getError(err) - } - return *result.VpcEndpoints[0].State, nil -} - -func (s sessionAWS) DeletePrivateLink(endpointID string) error { - input := &ec2.DeleteVpcEndpointsInput{ - VpcEndpointIds: []*string{aws.String(endpointID)}, - } - _, err := s.ec2.DeleteVpcEndpoints(input) - if err != nil { - return getError(err) - } - return nil -} - -func getError(err error) error { - if aerr, ok := err.(awserr.Error); ok { //nolint - return aerr - } - return err -} - -func (s sessionAWS) GetFuncPrivateEndpointStatus(privateEndpointID string) func() string { - return func() string { - r, err := s.DescribePrivateEndpointStatus(privateEndpointID) - if err != nil { - return "" - } - return r - } -} - -func (s sessionAWS) GetCustomerMasterKeyID(atlasAccountArn, assumedRoleArn string) (keyId string, err error) { - keyId, adminARNs, err := getKeyIDAndAdminARNs() - if err != nil { - return "", err - } - - policyString, err := RolePolicyString(atlasAccountArn, assumedRoleArn, adminARNs) - if err != nil { - return "", err - } - - policyInput := &kms.PutKeyPolicyInput{ - KeyId: &keyId, - PolicyName: aws.String("default"), - Policy: aws.String(policyString), - } - _, err = s.kms.PutKeyPolicy(policyInput) - if err != nil { - return "", err - } - - return keyId, nil -} - -func getKeyIDAndAdminARNs() (keyID string, adminARNs []string, err error) { - keyID = os.Getenv("AWS_KMS_KEY_ID") - if keyID == "" { - err = errors.New("AWS_KMS_KEY_ID secret is empty") - return - } - adminArnString := os.Getenv("AWS_ACCOUNT_ARN_LIST") - if adminArnString == "" { - err = errors.New("AWS_ACCOUNT_ARN_LIST secret is empty") - return - } - - adminARNs = strings.Split(adminArnString, ",") - if len(adminARNs) == 0 { - err = errors.New("AWS_ACCOUNT_ARN_LIST wasn't parsed properly, please separate accounts via a comma") - return - } - - return keyID, adminARNs, nil -} - -func RolePolicyString(atlasAccountARN, assumedRoleARN string, adminARNs []string) (string, error) { - policy := defaultKMSPolicy(atlasAccountARN, assumedRoleARN, adminARNs) - byteStr, err := json.Marshal(policy) - if err != nil { - return "", err - } - return string(byteStr), nil -} - -func defaultKMSPolicy(atlasAccountArn, assumedRoleArn string, adminARNs []string) kmsPolicy { - return kmsPolicy{ - Version: "2012-10-17", - Statement: []statement{ - { - Sid: "Enable IAM User Permissions", - Effect: "Allow", - Principal: principal{ - AWS: []string{atlasAccountArn}, - }, - Action: []string{"kms:*"}, - Resource: "*", - }, - { - Sid: "Allow access for Key Administrators", - Effect: "Allow", - Principal: principal{ - AWS: adminARNs, - }, - Action: []string{ - "kms:Create*", - "kms:Describe*", - "kms:Enable*", - "kms:List*", - "kms:Put*", - "kms:Update*", - "kms:Revoke*", - "kms:Disable*", - "kms:Get*", - "kms:Delete*", - "kms:TagResource", - "kms:UntagResource", - "kms:ScheduleKeyDeletion", - "kms:CancelKeyDeletion", - }, - Resource: "*", - }, - { - Sid: "Allow use of the key", - Effect: "Allow", - Principal: principal{ - AWS: []string{assumedRoleArn}, - }, - Action: []string{ - "kms:Encrypt", - "kms:Decrypt", - "kms:ReEncrypt*", - "kms:GenerateDataKey*", - "kms:DescribeKey", - }, - Resource: "*", - }, - }, - } -} - -type kmsPolicy struct { - Version string `json:"Version"` - Statement []statement `json:"Statement"` -} - -type statement struct { - Sid string `json:"Sid"` - Effect string `json:"Effect"` - Principal principal `json:"Principal"` - Action []string `json:"Action"` - Resource string `json:"Resource"` -} - -type principal struct { - AWS []string `json:"AWS,omitempty"` -} diff --git a/test/e2e/encryption_at_rest_test.go b/test/e2e/encryption_at_rest_test.go index d1c941e380..281af19a31 100644 --- a/test/e2e/encryption_at_rest_test.go +++ b/test/e2e/encryption_at_rest_test.go @@ -35,11 +35,21 @@ import ( "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/utils" ) +const ( + AzureClientID = "AZURE_CLIENT_ID" + KeyVaultName = "ako-kms-test" + AzureClientSecret = "AZURE_CLIENT_SECRET" //#nosec G101 -- False positive; this is the env var, not the secret itself + AzureEnvironment = "AZURE" + KeyName = "encryption-at-rest-test-key" +) + var _ = Describe("Encryption at REST test", Label("encryption-at-rest"), func() { var testData *model.TestDataProvider _ = BeforeEach(func() { checkUpAWSEnvironment() + checkUpAzureEnvironment() + checkNSetUpGCPEnvironment() }) _ = AfterEach(func() { @@ -64,7 +74,13 @@ var _ = Describe("Encryption at REST test", Label("encryption-at-rest"), func() func(test *model.TestDataProvider, encAtRest v1.EncryptionAtRest, roles []cloudaccess.Role) { testData = test actions.ProjectCreationFlow(test) - encryptionAtRestFlow(test, encAtRest, roles) + + if roles != nil { + cloudAccessRolesFlow(test, roles) + encAtRest.AwsKms.RoleID = test.Project.Status.CloudProviderAccessRoles[0].IamAssumedRoleArn + } + + encryptionAtRestFlow(test, encAtRest) }, Entry("Test[encryption-at-rest-aws]: Can add Encryption at Rest to AWS project", Label("encryption-at-rest-aws"), model.DataProvider( @@ -76,9 +92,7 @@ var _ = Describe("Encryption at REST test", Label("encryption-at-rest"), func() v1.EncryptionAtRest{ AwsKms: v1.AwsKms{ Enabled: toptr.MakePtr(true), - // CustomerMasterKeyID: "", - Region: "US_EAST_1", - Valid: toptr.MakePtr(true), + Region: "US_EAST_1", }, }, []cloudaccess.Role{ @@ -90,20 +104,54 @@ var _ = Describe("Encryption at REST test", Label("encryption-at-rest"), func() }, }, ), + Entry("Test[encryption-at-rest-azure]: Can add Encryption at Rest to Azure project", Label("encryption-at-rest-azure"), + model.DataProvider( + "encryption-at-rest-azure", + model.NewEmptyAtlasKeyType().UseDefaultFullAccess(), + 40000, + []func(*model.TestDataProvider){}, + ).WithProject(data.DefaultProject()), + v1.EncryptionAtRest{ + AzureKeyVault: v1.AzureKeyVault{ + AzureEnvironment: AzureEnvironment, + ClientID: os.Getenv(AzureClientID), + Enabled: toptr.MakePtr(true), + KeyVaultName: KeyVaultName, + ResourceGroupName: cloud.ResourceGroupName, + Secret: os.Getenv(AzureClientSecret), + TenantID: os.Getenv(DirectoryID), + SubscriptionID: os.Getenv(SubscriptionID), + }, + }, + nil, + ), + Entry("Test[encryption-at-rest-gcp]: Can add Encryption at Rest to GCP project", Label("encryption-at-rest-gcp"), + model.DataProvider( + "encryption-at-rest-gcp", + model.NewEmptyAtlasKeyType().UseDefaultFullAccess(), + 40000, + []func(*model.TestDataProvider){}, + ).WithProject(data.DefaultProject()), + v1.EncryptionAtRest{ + GoogleCloudKms: v1.GoogleCloudKms{ + Enabled: toptr.MakePtr(true), + ServiceAccountKey: os.Getenv("GCP_SA_CRED"), + }, + }, + nil, + ), ) }) -func encryptionAtRestFlow(userData *model.TestDataProvider, encAtRest v1.EncryptionAtRest, roles []cloudaccess.Role) { - By("Add cloud access role (AWS only)", func() { - cloudAccessRolesFlow(userData, roles) - }) - +func encryptionAtRestFlow(userData *model.TestDataProvider, encAtRest v1.EncryptionAtRest) { By("Create KMS", func() { Expect(userData.K8SClient.Get(userData.Context, types.NamespacedName{Name: userData.Project.Name, Namespace: userData.Resources.Namespace}, userData.Project)).Should(Succeed()) - Expect(len(userData.Project.Status.CloudProviderAccessRoles)).NotTo(Equal(0)) - aRole := userData.Project.Status.CloudProviderAccessRoles[0] + var aRole status.CloudProviderAccessRole + if len(userData.Project.Status.CloudProviderAccessRoles) > 0 { + aRole = userData.Project.Status.CloudProviderAccessRoles[0] + } fillKMSforAWS(&encAtRest, aRole.AtlasAWSAccountArn, aRole.IamAssumedRoleArn) fillVaultforAzure(&encAtRest) @@ -146,7 +194,8 @@ func fillKMSforAWS(encAtRest *v1.EncryptionAtRest, atlasAccountArn, assumedRoleA } Expect(encAtRest.AwsKms.Region).NotTo(Equal("")) - awsAction := cloud.NewAwsAction() + awsAction, err := cloud.NewAWSAction(GinkgoT()) + Expect(err).ToNot(HaveOccurred()) CustomerMasterKeyID, err := awsAction.CreateKMS(config.AWSRegionUS, atlasAccountArn, assumedRoleArn) Expect(err).ToNot(HaveOccurred()) Expect(CustomerMasterKeyID).NotTo(Equal("")) @@ -159,7 +208,13 @@ func fillVaultforAzure(encAtRest *v1.EncryptionAtRest) { return } - // todo: fill in + azAction, err := cloud.NewAzureAction(GinkgoT(), os.Getenv(SubscriptionID), cloud.ResourceGroupName) + Expect(err).ToNot(HaveOccurred()) + + keyID, err := azAction.CreateKeyVault(KeyName) + Expect(err).ToNot(HaveOccurred()) + + encAtRest.AzureKeyVault.KeyIdentifier = keyID } func fillKMSforGCP(encAtRest *v1.EncryptionAtRest) { @@ -167,7 +222,13 @@ func fillKMSforGCP(encAtRest *v1.EncryptionAtRest) { return } - // todo: fill in + gcpAction, err := cloud.NewGCPAction(GinkgoT(), cloud.GoogleProjectID) + Expect(err).ToNot(HaveOccurred()) + + keyID, err := gcpAction.CreateKMS() + Expect(err).ToNot(HaveOccurred()) + + encAtRest.GoogleCloudKms.KeyVersionResourceID = keyID } func removeAllEncryptionsSeparately(encAtRest *v1.EncryptionAtRest) { @@ -198,7 +259,7 @@ func checkIfEncryptionsAreDisabled(projectID string) (areEmpty bool, err error) return true, nil } -var _ = Describe("Encryption at rest AWS", Label("encryption-at-rest"), func() { +var _ = Describe("Encryption at rest AWS", Label("encryption-at-rest"), Ordered, func() { var testData *model.TestDataProvider _ = BeforeEach(func() { @@ -245,7 +306,6 @@ var _ = Describe("Encryption at rest AWS", Label("encryption-at-rest"), func() { AwsKms: v1.AwsKms{ Enabled: toptr.MakePtr(true), Region: "US_EAST_1", - Valid: toptr.MakePtr(true), }, } @@ -329,7 +389,6 @@ var _ = Describe("Encryption at rest AWS", Label("encryption-at-rest"), func() { encAtRest := v1.EncryptionAtRest{ AwsKms: v1.AwsKms{ Enabled: toptr.MakePtr(true), - Valid: toptr.MakePtr(true), }, }