Cleanup orphan GCP private links

Signed-off-by: jose.vazquez <jose.vazquez@mongodb.com>
mongodb · Feb 27, 2024 · 49496af · 49496af
1 parent bfa9b72
commit 49496af
Show file tree

Hide file tree

Showing 5 changed files with 145 additions and 46 deletions.
diff --git a/.licenses-gomod.sha256 b/.licenses-gomod.sha256
@@ -1 +1 @@
-91c85f4647ba675732ee0a561dd78f2ea6b3e69ae3f1dc1971133a6d91fe6f5a
+100644 a96d3508bc189528c2722c8093490e5ddf48b99b go.mod
diff --git a/Makefile b/Makefile
@@ -96,7 +96,7 @@ SIGNATURE_REPO ?= OPERATOR_REGISTRY
 AKO_SIGN_PUBKEY = https://cosign.mongodb.com/atlas-kubernetes-operator.pem
 
 # Licenses status
-GOMOD_SHA := $(shell shasum -a 256 go.mod | awk '{print $$1}')
+GOMOD_SHA := $(shell git ls-files -s go.mod | awk '{print $$1" "$$2" "$$4}')
 LICENSES_GOMOD_SHA_FILE := .licenses-gomod.sha256
 GOMOD_LICENSES_SHA := $(shell cat $(LICENSES_GOMOD_SHA_FILE))
 
@@ -128,15 +128,15 @@ recompute-licenses: ## Recompute the licenses.csv only if needed (gomod was chan
 
 licenses-up-to-date:
 	@if [ "$(GOMOD_SHA)" != "$(GOMOD_LICENSES_SHA)" ]; then \
-	echo "licenses.csv needs ot be recalculated: run 'make licenses.csv'"; exit 1; else\
-	echo "licenses.csv is OK! (up to date)"; fi
+	echo "licenses.csv needs to be recalculated: git rebase AND run 'make licenses.csv'"; exit 1; \
+	else echo "licenses.csv is OK! (up to date)"; fi
 
 .PHONY: check-licenses
 check-licenses: go-licenses licenses-up-to-date ## Check licenses are compliant with our restrictions
 	@echo "Checking licenses not to be: $(DISALLOWED_LICENSES)"
 	@echo "============================================"
-	GOOS=linux GOARCH=amd64 $(GO_LICENSES) check --include_tests $(BASE_GO_PACKAGE)/... \
-	--disallowed_types $(DISALLOWED_LICENSES)
+	GOOS=linux GOARCH=amd64 $(GO_LICENSES) check --include_tests \
+	--disallowed_types $(DISALLOWED_LICENSES) $(BASE_GO_PACKAGE)/...
 	@echo "--------------------"
 	@echo "Licenses check: PASS"
 

diff --git a/licenses.csv b/licenses.csv
@@ -20,8 +20,8 @@ github.com/Azure/go-autorest/logger,https://github.com/Azure/go-autorest/blob/lo
 github.com/Azure/go-autorest/tracing,https://github.com/Azure/go-autorest/blob/tracing/v0.6.0/tracing/LICENSE,Apache-2.0
 github.com/AzureAD/microsoft-authentication-library-for-go/apps,https://github.com/AzureAD/microsoft-authentication-library-for-go/blob/v1.2.1/LICENSE,MIT
 github.com/Masterminds/semver,https://github.com/Masterminds/semver/blob/v1.5.0/LICENSE.txt,MIT
-github.com/aws/aws-sdk-go,https://github.com/aws/aws-sdk-go/blob/v1.50.21/LICENSE.txt,Apache-2.0
-github.com/aws/aws-sdk-go/internal/sync/singleflight,https://github.com/aws/aws-sdk-go/blob/v1.50.21/internal/sync/singleflight/LICENSE,BSD-3-Clause
+github.com/aws/aws-sdk-go,https://github.com/aws/aws-sdk-go/blob/v1.50.26/LICENSE.txt,Apache-2.0
+github.com/aws/aws-sdk-go/internal/sync/singleflight,https://github.com/aws/aws-sdk-go/blob/v1.50.26/internal/sync/singleflight/LICENSE,BSD-3-Clause
 github.com/beorn7/perks/quantile,https://github.com/beorn7/perks/blob/v1.0.1/LICENSE,MIT
 github.com/cespare/xxhash/v2,https://github.com/cespare/xxhash/blob/v2.2.0/LICENSE.txt,MIT
 github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC
@@ -51,7 +51,7 @@ github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Ap
 github.com/google/s2a-go,https://github.com/google/s2a-go/blob/v0.1.7/LICENSE.md,Apache-2.0
 github.com/google/uuid,https://github.com/google/uuid/blob/v1.6.0/LICENSE,BSD-3-Clause
 github.com/googleapis/enterprise-certificate-proxy/client,https://github.com/googleapis/enterprise-certificate-proxy/blob/v0.3.2/LICENSE,Apache-2.0
-github.com/googleapis/gax-go/v2,https://github.com/googleapis/gax-go/blob/v2.12.0/v2/LICENSE,BSD-3-Clause
+github.com/googleapis/gax-go/v2,https://github.com/googleapis/gax-go/blob/v2.12.1/v2/LICENSE,BSD-3-Clause
 github.com/imdario/mergo,https://github.com/imdario/mergo/blob/v0.3.12/LICENSE,BSD-3-Clause
 github.com/jmespath/go-jmespath,https://github.com/jmespath/go-jmespath/blob/v0.4.0/LICENSE,Apache-2.0
 github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT
@@ -91,13 +91,13 @@ go.mongodb.org/atlas-sdk/v20231115004,https://github.com/mongodb/atlas-sdk-go/bl
 go.mongodb.org/atlas/mongodbatlas,https://github.com/mongodb/go-client-mongodb-atlas/blob/v0.36.0/LICENSE,Apache-2.0
 go.mongodb.org/mongo-driver,https://github.com/mongodb/mongo-go-driver/blob/v1.14.0/LICENSE,Apache-2.0
 go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0.24.0/LICENSE,Apache-2.0
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/google.golang.org/grpc/otelgrpc/v0.47.0/instrumentation/google.golang.org/grpc/otelgrpc/LICENSE,Apache-2.0
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/net/http/otelhttp/v0.47.0/instrumentation/net/http/otelhttp/LICENSE,Apache-2.0
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/google.golang.org/grpc/otelgrpc/v0.48.0/instrumentation/google.golang.org/grpc/otelgrpc/LICENSE,Apache-2.0
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/net/http/otelhttp/v0.48.0/instrumentation/net/http/otelhttp/LICENSE,Apache-2.0
 go.opentelemetry.io/otel,https://github.com/open-telemetry/opentelemetry-go/blob/v1.23.0/LICENSE,Apache-2.0
 go.opentelemetry.io/otel/metric,https://github.com/open-telemetry/opentelemetry-go/blob/metric/v1.23.0/metric/LICENSE,Apache-2.0
 go.opentelemetry.io/otel/trace,https://github.com/open-telemetry/opentelemetry-go/blob/trace/v1.23.0/trace/LICENSE,Apache-2.0
 go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.10.0/LICENSE.txt,MIT
-go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.26.0/LICENSE.txt,MIT
+go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.27.0/LICENSE,MIT
 golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.19.0:LICENSE,BSD-3-Clause
 golang.org/x/exp,https://cs.opensource.google/go/x/exp/+/2e198f4a:LICENSE,BSD-3-Clause
 golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.21.0:LICENSE,BSD-3-Clause
@@ -108,12 +108,12 @@ golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.17.0:LICENSE,BSD-3
 golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause
 golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE,BSD-3-Clause
 gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.3.0/v2/LICENSE,Apache-2.0
-google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.165.0/LICENSE,BSD-3-Clause
-google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.165.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause
-google.golang.org/genproto/googleapis,https://github.com/googleapis/go-genproto/blob/1f4bbc51befe/LICENSE,Apache-2.0
+google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.167.0/LICENSE,BSD-3-Clause
+google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.167.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause
+google.golang.org/genproto/googleapis,https://github.com/googleapis/go-genproto/blob/31a09d347014/LICENSE,Apache-2.0
 google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/31a09d347014/googleapis/api/LICENSE,Apache-2.0
-google.golang.org/genproto/googleapis/rpc,https://github.com/googleapis/go-genproto/blob/31a09d347014/googleapis/rpc/LICENSE,Apache-2.0
-google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.61.0/LICENSE,Apache-2.0
+google.golang.org/genproto/googleapis/rpc,https://github.com/googleapis/go-genproto/blob/012b6fc9bca9/googleapis/rpc/LICENSE,Apache-2.0
+google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.61.1/LICENSE,Apache-2.0
 google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.32.0/LICENSE,BSD-3-Clause
 gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause
 gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0

diff --git a/tools/clean/atlas/atlas.go b/tools/clean/atlas/atlas.go
@@ -92,7 +92,21 @@ func (c *Cleaner) Clean(ctx context.Context, lifetime int) error {
 		c.deleteTeam(ctx, c.orgID, &t)
 	}
 
-	return nil
+	return c.cleanOrphanResources(ctx, lifetime)
+}
+
+func (c *Cleaner) cleanOrphanResources(ctx context.Context, lifetime int) error {
+	region := envOrDefault("GCP_CLEANUP_REGION", "europe-west1")
+	subnet := envOrDefault("GCP_CLEANUP_SUBNET", "atlas-operator-e2e-test-subnet1")
+	done, skipped, err := c.gcp.DeleteOrphanPrivateEndpoints(ctx, lifetime, region, subnet)
+	for _, doneMsg := range done {
+		fmt.Println(text.FgGreen.Sprintf("%s", doneMsg))
+	}
+	for _, skippedMsg := range skipped {
+		fmt.Println(text.FgYellow.Sprintf("\t%s", skippedMsg))
+	}
+
+	return err
 }
 
 func NewCleaner(aws *provider.AWS, gcp *provider.GCP, azure *provider.Azure) (*Cleaner, error) {
@@ -133,3 +147,11 @@ func NewCleaner(aws *provider.AWS, gcp *provider.GCP, azure *provider.Azure) (*C
 func isGov(url string) bool {
 	return strings.HasSuffix(url, "mongodbgov.com")
 }
+
+func envOrDefault(name, defaultValue string) string {
+	value, defined := os.LookupEnv(name)
+	if !defined {
+		return defaultValue
+	}
+	return value
+}
diff --git a/tools/clean/provider/gcp.go b/tools/clean/provider/gcp.go
@@ -5,7 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 	"strings"
+	"time"
 
 	compute "cloud.google.com/go/compute/apiv1"
 	"cloud.google.com/go/compute/apiv1/computepb"
@@ -100,6 +102,79 @@ func (gcp *GCP) DeletePrivateEndpoint(ctx context.Context, groupName, sAttachmen
 	return nil
 }
 
+func (gcp *GCP) DeleteOrphanPrivateEndpoints(ctx context.Context, lifetime int, region string, subnet string) ([]string, []string, error) {
+	addresses := gcp.addressClient.List(ctx, &computepb.ListAddressesRequest{
+		Project: gcp.projectID,
+		Region:  region,
+	})
+	done := []string{}
+	skipped := []string{}
+	for {
+		addr, err := addresses.Next()
+		if errors.Is(err, iterator.Done) {
+			break
+		}
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed iterating addresses in project %v region %v: %w",
+				gcp.projectID, region, err)
+		}
+		suffix := fmt.Sprintf("subnetworks/%s", subnet)
+		if !strings.HasSuffix(addr.GetSubnetwork(), suffix) {
+			skipped = append(skipped,
+				fmt.Sprintf("Address %s(%s) skipped, not in %s\n", addr.GetName(), addr.GetAddress(), subnet))
+			continue
+		}
+		createdAt, err := asTime(addr.GetCreationTimestamp())
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed parsing Address creation timestamp %q: %w",
+				addr.GetCreationTimestamp(), err)
+		}
+		if time.Since(createdAt) < time.Duration(lifetime)*time.Hour {
+			skipped = append(skipped, fmt.Sprintf("Address %s(%s) skipped once created less than %d hours ago\n",
+				addr.GetName(), addr.GetAddress(), lifetime))
+		}
+		frName, err := expectForwardingRule(addr.GetUsers())
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if frName != "" {
+			if err := gcp.deleteForwardingRule(ctx, frName, region); err != nil {
+				return nil, nil, fmt.Errorf("failed deleting Forwarding Rule %q in region %q: %w", region, frName, err)
+			}
+			done = append(done, fmt.Sprintf("Deleted Forwarding Rule %s for %s\n",
+				frName, addr.GetAddress()))
+		} else {
+			skipped = append(skipped,
+				fmt.Sprintf("No forwarding rule using Address %s(%s)", addr.GetName(), addr.GetAddress()))
+		}
+		if err := gcp.deleteIPAddress(ctx, addr.GetName(), region); err != nil {
+			return nil, nil, fmt.Errorf("error deleting Address %s(%s) in region %q: %w",
+				region, addr.GetName(), addr.GetAddress(), err)
+		}
+		done = append(done, fmt.Sprintf("Released orphan Address %s(%s)\n", addr.GetName(), addr.GetAddress()))
+	}
+
+	return done, skipped, nil
+}
+
+func asTime(rfc3339time string) (time.Time, error) {
+	return time.Parse(time.RFC3339, rfc3339time)
+}
+
+func expectForwardingRule(usersOfEndpointAddress []string) (string, error) {
+	if len(usersOfEndpointAddress) == 0 {
+		return "", nil
+	}
+	if len(usersOfEndpointAddress) > 1 {
+		return "", fmt.Errorf("expected a single user of an Endpoint Address, but got %v", usersOfEndpointAddress)
+	}
+	if strings.Contains(usersOfEndpointAddress[0], "/forwardingRules/") {
+		return path.Base(usersOfEndpointAddress[0]), nil
+	}
+	return "", fmt.Errorf("expected a Forwarding Rule user for Endpoint Address but got %s", usersOfEndpointAddress[0])
+}
+
 func (gcp *GCP) DeleteCryptoKey(ctx context.Context, keyName string) error {
 	_, err := gcp.keyManagementClient.GetCryptoKeyVersion(ctx, &kmspb.GetCryptoKeyVersionRequest{
 		Name: keyName,
@@ -135,32 +210,33 @@ func (gcp *GCP) deleteForwardRules(ctx context.Context, region, groupName string
 
 	it := gcp.forwardRuleClient.List(ctx, request)
 	for {
-		resp, err := it.Next()
+		fwr, err := it.Next()
 		if errors.Is(err, iterator.Done) {
 			break
 		}
 		if err != nil {
 			return err
 		}
 
-		frRequest := &computepb.DeleteForwardingRuleRequest{
-			ForwardingRule: resp.GetName(),
-			Project:        gcp.projectID,
-			Region:         resp.GetRegion(),
+		if err := gcp.deleteForwardingRule(ctx, fwr.GetName(), fwr.GetRegion()); err != nil {
+			return os.ErrClosed
 		}
+	}
 
-		op, err := gcp.forwardRuleClient.Delete(ctx, frRequest)
-		if err != nil {
-			return err
-		}
+	return nil
+}
 
-		err = op.Wait(ctx)
-		if err != nil {
-			return err
-		}
+func (gcp *GCP) deleteForwardingRule(ctx context.Context, name, region string) error {
+	op, err := gcp.forwardRuleClient.Delete(ctx, &computepb.DeleteForwardingRuleRequest{
+		ForwardingRule: name,
+		Project:        gcp.projectID,
+		Region:         region,
+	})
+	if err != nil {
+		return err
 	}
 
-	return nil
+	return op.Wait(ctx)
 }
 
 func (gcp *GCP) deleteIPAddresses(ctx context.Context, region, groupName string) error {
@@ -174,34 +250,35 @@ func (gcp *GCP) deleteIPAddresses(ctx context.Context, region, groupName string)
 
 	it := gcp.addressClient.List(ctx, request)
 	for {
-		resp, err := it.Next()
+		addr, err := it.Next()
 		if errors.Is(err, iterator.Done) {
 			break
 		}
 		if err != nil {
 			return err
 		}
 
-		aRequest := &computepb.DeleteAddressRequest{
-			Address: resp.GetName(),
-			Project: gcp.projectID,
-			Region:  resp.GetRegion(),
-		}
-
-		op, err := gcp.addressClient.Delete(ctx, aRequest)
-		if err != nil {
-			return err
-		}
-
-		err = op.Wait(ctx)
-		if err != nil {
+		if err := gcp.deleteIPAddress(ctx, addr.GetName(), addr.GetRegion()); err != nil {
 			return err
 		}
 	}
 
 	return nil
 }
 
+func (gcp *GCP) deleteIPAddress(ctx context.Context, name, region string) error {
+	op, err := gcp.addressClient.Delete(ctx, &computepb.DeleteAddressRequest{
+		Address: name,
+		Project: gcp.projectID,
+		Region:  region,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to delete address %s at %s: %v", name, region, err)
+	}
+
+	return op.Wait(ctx)
+}
+
 func NewGCPCleaner(ctx context.Context) (*GCP, error) {
 	_, defined := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS")
 	if !defined {