From 49496afc35ec671fc6bd8aea04aff503a7affcf3 Mon Sep 17 00:00:00 2001 From: "jose.vazquez" Date: Mon, 26 Feb 2024 15:31:25 +0100 Subject: [PATCH] Cleanup orphan GCP private links Signed-off-by: jose.vazquez --- .licenses-gomod.sha256 | 2 +- Makefile | 10 +-- licenses.csv | 22 +++--- tools/clean/atlas/atlas.go | 24 ++++++- tools/clean/provider/gcp.go | 133 ++++++++++++++++++++++++++++-------- 5 files changed, 145 insertions(+), 46 deletions(-) diff --git a/.licenses-gomod.sha256 b/.licenses-gomod.sha256 index 34e876f1d2..aee58094ec 100644 --- a/.licenses-gomod.sha256 +++ b/.licenses-gomod.sha256 @@ -1 +1 @@ -91c85f4647ba675732ee0a561dd78f2ea6b3e69ae3f1dc1971133a6d91fe6f5a +100644 a96d3508bc189528c2722c8093490e5ddf48b99b go.mod diff --git a/Makefile b/Makefile index 75885c69cf..0275cabc60 100644 --- a/Makefile +++ b/Makefile @@ -96,7 +96,7 @@ SIGNATURE_REPO ?= OPERATOR_REGISTRY AKO_SIGN_PUBKEY = https://cosign.mongodb.com/atlas-kubernetes-operator.pem # Licenses status -GOMOD_SHA := $(shell shasum -a 256 go.mod | awk '{print $$1}') +GOMOD_SHA := $(shell git ls-files -s go.mod | awk '{print $$1" "$$2" "$$4}') LICENSES_GOMOD_SHA_FILE := .licenses-gomod.sha256 GOMOD_LICENSES_SHA := $(shell cat $(LICENSES_GOMOD_SHA_FILE)) @@ -128,15 +128,15 @@ recompute-licenses: ## Recompute the licenses.csv only if needed (gomod was chan licenses-up-to-date: @if [ "$(GOMOD_SHA)" != "$(GOMOD_LICENSES_SHA)" ]; then \ - echo "licenses.csv needs ot be recalculated: run 'make licenses.csv'"; exit 1; else\ - echo "licenses.csv is OK! (up to date)"; fi + echo "licenses.csv needs to be recalculated: git rebase AND run 'make licenses.csv'"; exit 1; \ + else echo "licenses.csv is OK! (up to date)"; fi .PHONY: check-licenses check-licenses: go-licenses licenses-up-to-date ## Check licenses are compliant with our restrictions @echo "Checking licenses not to be: $(DISALLOWED_LICENSES)" @echo "============================================" - GOOS=linux GOARCH=amd64 $(GO_LICENSES) check --include_tests $(BASE_GO_PACKAGE)/... \ - --disallowed_types $(DISALLOWED_LICENSES) + GOOS=linux GOARCH=amd64 $(GO_LICENSES) check --include_tests \ + --disallowed_types $(DISALLOWED_LICENSES) $(BASE_GO_PACKAGE)/... @echo "--------------------" @echo "Licenses check: PASS" diff --git a/licenses.csv b/licenses.csv index 0b0f7b0c2a..d9b51c52f5 100644 --- a/licenses.csv +++ b/licenses.csv @@ -20,8 +20,8 @@ github.com/Azure/go-autorest/logger,https://github.com/Azure/go-autorest/blob/lo github.com/Azure/go-autorest/tracing,https://github.com/Azure/go-autorest/blob/tracing/v0.6.0/tracing/LICENSE,Apache-2.0 github.com/AzureAD/microsoft-authentication-library-for-go/apps,https://github.com/AzureAD/microsoft-authentication-library-for-go/blob/v1.2.1/LICENSE,MIT github.com/Masterminds/semver,https://github.com/Masterminds/semver/blob/v1.5.0/LICENSE.txt,MIT -github.com/aws/aws-sdk-go,https://github.com/aws/aws-sdk-go/blob/v1.50.21/LICENSE.txt,Apache-2.0 -github.com/aws/aws-sdk-go/internal/sync/singleflight,https://github.com/aws/aws-sdk-go/blob/v1.50.21/internal/sync/singleflight/LICENSE,BSD-3-Clause +github.com/aws/aws-sdk-go,https://github.com/aws/aws-sdk-go/blob/v1.50.26/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go/internal/sync/singleflight,https://github.com/aws/aws-sdk-go/blob/v1.50.26/internal/sync/singleflight/LICENSE,BSD-3-Clause github.com/beorn7/perks/quantile,https://github.com/beorn7/perks/blob/v1.0.1/LICENSE,MIT github.com/cespare/xxhash/v2,https://github.com/cespare/xxhash/blob/v2.2.0/LICENSE.txt,MIT github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC @@ -51,7 +51,7 @@ github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Ap github.com/google/s2a-go,https://github.com/google/s2a-go/blob/v0.1.7/LICENSE.md,Apache-2.0 github.com/google/uuid,https://github.com/google/uuid/blob/v1.6.0/LICENSE,BSD-3-Clause github.com/googleapis/enterprise-certificate-proxy/client,https://github.com/googleapis/enterprise-certificate-proxy/blob/v0.3.2/LICENSE,Apache-2.0 -github.com/googleapis/gax-go/v2,https://github.com/googleapis/gax-go/blob/v2.12.0/v2/LICENSE,BSD-3-Clause +github.com/googleapis/gax-go/v2,https://github.com/googleapis/gax-go/blob/v2.12.1/v2/LICENSE,BSD-3-Clause github.com/imdario/mergo,https://github.com/imdario/mergo/blob/v0.3.12/LICENSE,BSD-3-Clause github.com/jmespath/go-jmespath,https://github.com/jmespath/go-jmespath/blob/v0.4.0/LICENSE,Apache-2.0 github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT @@ -91,13 +91,13 @@ go.mongodb.org/atlas-sdk/v20231115004,https://github.com/mongodb/atlas-sdk-go/bl go.mongodb.org/atlas/mongodbatlas,https://github.com/mongodb/go-client-mongodb-atlas/blob/v0.36.0/LICENSE,Apache-2.0 go.mongodb.org/mongo-driver,https://github.com/mongodb/mongo-go-driver/blob/v1.14.0/LICENSE,Apache-2.0 go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0.24.0/LICENSE,Apache-2.0 -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/google.golang.org/grpc/otelgrpc/v0.47.0/instrumentation/google.golang.org/grpc/otelgrpc/LICENSE,Apache-2.0 -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/net/http/otelhttp/v0.47.0/instrumentation/net/http/otelhttp/LICENSE,Apache-2.0 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/google.golang.org/grpc/otelgrpc/v0.48.0/instrumentation/google.golang.org/grpc/otelgrpc/LICENSE,Apache-2.0 +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp,https://github.com/open-telemetry/opentelemetry-go-contrib/blob/instrumentation/net/http/otelhttp/v0.48.0/instrumentation/net/http/otelhttp/LICENSE,Apache-2.0 go.opentelemetry.io/otel,https://github.com/open-telemetry/opentelemetry-go/blob/v1.23.0/LICENSE,Apache-2.0 go.opentelemetry.io/otel/metric,https://github.com/open-telemetry/opentelemetry-go/blob/metric/v1.23.0/metric/LICENSE,Apache-2.0 go.opentelemetry.io/otel/trace,https://github.com/open-telemetry/opentelemetry-go/blob/trace/v1.23.0/trace/LICENSE,Apache-2.0 go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.10.0/LICENSE.txt,MIT -go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.26.0/LICENSE.txt,MIT +go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.27.0/LICENSE,MIT golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.19.0:LICENSE,BSD-3-Clause golang.org/x/exp,https://cs.opensource.google/go/x/exp/+/2e198f4a:LICENSE,BSD-3-Clause golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.21.0:LICENSE,BSD-3-Clause @@ -108,12 +108,12 @@ golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.17.0:LICENSE,BSD-3 golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.5.0:LICENSE,BSD-3-Clause gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.3.0/v2/LICENSE,Apache-2.0 -google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.165.0/LICENSE,BSD-3-Clause -google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.165.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause -google.golang.org/genproto/googleapis,https://github.com/googleapis/go-genproto/blob/1f4bbc51befe/LICENSE,Apache-2.0 +google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.167.0/LICENSE,BSD-3-Clause +google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.167.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause +google.golang.org/genproto/googleapis,https://github.com/googleapis/go-genproto/blob/31a09d347014/LICENSE,Apache-2.0 google.golang.org/genproto/googleapis/api,https://github.com/googleapis/go-genproto/blob/31a09d347014/googleapis/api/LICENSE,Apache-2.0 -google.golang.org/genproto/googleapis/rpc,https://github.com/googleapis/go-genproto/blob/31a09d347014/googleapis/rpc/LICENSE,Apache-2.0 -google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.61.0/LICENSE,Apache-2.0 +google.golang.org/genproto/googleapis/rpc,https://github.com/googleapis/go-genproto/blob/012b6fc9bca9/googleapis/rpc/LICENSE,Apache-2.0 +google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.61.1/LICENSE,Apache-2.0 google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.32.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 diff --git a/tools/clean/atlas/atlas.go b/tools/clean/atlas/atlas.go index eafd18d013..c9862ddbd5 100644 --- a/tools/clean/atlas/atlas.go +++ b/tools/clean/atlas/atlas.go @@ -92,7 +92,21 @@ func (c *Cleaner) Clean(ctx context.Context, lifetime int) error { c.deleteTeam(ctx, c.orgID, &t) } - return nil + return c.cleanOrphanResources(ctx, lifetime) +} + +func (c *Cleaner) cleanOrphanResources(ctx context.Context, lifetime int) error { + region := envOrDefault("GCP_CLEANUP_REGION", "europe-west1") + subnet := envOrDefault("GCP_CLEANUP_SUBNET", "atlas-operator-e2e-test-subnet1") + done, skipped, err := c.gcp.DeleteOrphanPrivateEndpoints(ctx, lifetime, region, subnet) + for _, doneMsg := range done { + fmt.Println(text.FgGreen.Sprintf("%s", doneMsg)) + } + for _, skippedMsg := range skipped { + fmt.Println(text.FgYellow.Sprintf("\t%s", skippedMsg)) + } + + return err } func NewCleaner(aws *provider.AWS, gcp *provider.GCP, azure *provider.Azure) (*Cleaner, error) { @@ -133,3 +147,11 @@ func NewCleaner(aws *provider.AWS, gcp *provider.GCP, azure *provider.Azure) (*C func isGov(url string) bool { return strings.HasSuffix(url, "mongodbgov.com") } + +func envOrDefault(name, defaultValue string) string { + value, defined := os.LookupEnv(name) + if !defined { + return defaultValue + } + return value +} diff --git a/tools/clean/provider/gcp.go b/tools/clean/provider/gcp.go index ddef331875..f2eeb4a196 100644 --- a/tools/clean/provider/gcp.go +++ b/tools/clean/provider/gcp.go @@ -5,7 +5,9 @@ import ( "errors" "fmt" "os" + "path" "strings" + "time" compute "cloud.google.com/go/compute/apiv1" "cloud.google.com/go/compute/apiv1/computepb" @@ -100,6 +102,79 @@ func (gcp *GCP) DeletePrivateEndpoint(ctx context.Context, groupName, sAttachmen return nil } +func (gcp *GCP) DeleteOrphanPrivateEndpoints(ctx context.Context, lifetime int, region string, subnet string) ([]string, []string, error) { + addresses := gcp.addressClient.List(ctx, &computepb.ListAddressesRequest{ + Project: gcp.projectID, + Region: region, + }) + done := []string{} + skipped := []string{} + for { + addr, err := addresses.Next() + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + return nil, nil, fmt.Errorf("failed iterating addresses in project %v region %v: %w", + gcp.projectID, region, err) + } + suffix := fmt.Sprintf("subnetworks/%s", subnet) + if !strings.HasSuffix(addr.GetSubnetwork(), suffix) { + skipped = append(skipped, + fmt.Sprintf("Address %s(%s) skipped, not in %s\n", addr.GetName(), addr.GetAddress(), subnet)) + continue + } + createdAt, err := asTime(addr.GetCreationTimestamp()) + if err != nil { + return nil, nil, fmt.Errorf("failed parsing Address creation timestamp %q: %w", + addr.GetCreationTimestamp(), err) + } + if time.Since(createdAt) < time.Duration(lifetime)*time.Hour { + skipped = append(skipped, fmt.Sprintf("Address %s(%s) skipped once created less than %d hours ago\n", + addr.GetName(), addr.GetAddress(), lifetime)) + } + frName, err := expectForwardingRule(addr.GetUsers()) + if err != nil { + return nil, nil, err + } + + if frName != "" { + if err := gcp.deleteForwardingRule(ctx, frName, region); err != nil { + return nil, nil, fmt.Errorf("failed deleting Forwarding Rule %q in region %q: %w", region, frName, err) + } + done = append(done, fmt.Sprintf("Deleted Forwarding Rule %s for %s\n", + frName, addr.GetAddress())) + } else { + skipped = append(skipped, + fmt.Sprintf("No forwarding rule using Address %s(%s)", addr.GetName(), addr.GetAddress())) + } + if err := gcp.deleteIPAddress(ctx, addr.GetName(), region); err != nil { + return nil, nil, fmt.Errorf("error deleting Address %s(%s) in region %q: %w", + region, addr.GetName(), addr.GetAddress(), err) + } + done = append(done, fmt.Sprintf("Released orphan Address %s(%s)\n", addr.GetName(), addr.GetAddress())) + } + + return done, skipped, nil +} + +func asTime(rfc3339time string) (time.Time, error) { + return time.Parse(time.RFC3339, rfc3339time) +} + +func expectForwardingRule(usersOfEndpointAddress []string) (string, error) { + if len(usersOfEndpointAddress) == 0 { + return "", nil + } + if len(usersOfEndpointAddress) > 1 { + return "", fmt.Errorf("expected a single user of an Endpoint Address, but got %v", usersOfEndpointAddress) + } + if strings.Contains(usersOfEndpointAddress[0], "/forwardingRules/") { + return path.Base(usersOfEndpointAddress[0]), nil + } + return "", fmt.Errorf("expected a Forwarding Rule user for Endpoint Address but got %s", usersOfEndpointAddress[0]) +} + func (gcp *GCP) DeleteCryptoKey(ctx context.Context, keyName string) error { _, err := gcp.keyManagementClient.GetCryptoKeyVersion(ctx, &kmspb.GetCryptoKeyVersionRequest{ Name: keyName, @@ -135,7 +210,7 @@ func (gcp *GCP) deleteForwardRules(ctx context.Context, region, groupName string it := gcp.forwardRuleClient.List(ctx, request) for { - resp, err := it.Next() + fwr, err := it.Next() if errors.Is(err, iterator.Done) { break } @@ -143,24 +218,25 @@ func (gcp *GCP) deleteForwardRules(ctx context.Context, region, groupName string return err } - frRequest := &computepb.DeleteForwardingRuleRequest{ - ForwardingRule: resp.GetName(), - Project: gcp.projectID, - Region: resp.GetRegion(), + if err := gcp.deleteForwardingRule(ctx, fwr.GetName(), fwr.GetRegion()); err != nil { + return os.ErrClosed } + } - op, err := gcp.forwardRuleClient.Delete(ctx, frRequest) - if err != nil { - return err - } + return nil +} - err = op.Wait(ctx) - if err != nil { - return err - } +func (gcp *GCP) deleteForwardingRule(ctx context.Context, name, region string) error { + op, err := gcp.forwardRuleClient.Delete(ctx, &computepb.DeleteForwardingRuleRequest{ + ForwardingRule: name, + Project: gcp.projectID, + Region: region, + }) + if err != nil { + return err } - return nil + return op.Wait(ctx) } func (gcp *GCP) deleteIPAddresses(ctx context.Context, region, groupName string) error { @@ -174,7 +250,7 @@ func (gcp *GCP) deleteIPAddresses(ctx context.Context, region, groupName string) it := gcp.addressClient.List(ctx, request) for { - resp, err := it.Next() + addr, err := it.Next() if errors.Is(err, iterator.Done) { break } @@ -182,19 +258,7 @@ func (gcp *GCP) deleteIPAddresses(ctx context.Context, region, groupName string) return err } - aRequest := &computepb.DeleteAddressRequest{ - Address: resp.GetName(), - Project: gcp.projectID, - Region: resp.GetRegion(), - } - - op, err := gcp.addressClient.Delete(ctx, aRequest) - if err != nil { - return err - } - - err = op.Wait(ctx) - if err != nil { + if err := gcp.deleteIPAddress(ctx, addr.GetName(), addr.GetRegion()); err != nil { return err } } @@ -202,6 +266,19 @@ func (gcp *GCP) deleteIPAddresses(ctx context.Context, region, groupName string) return nil } +func (gcp *GCP) deleteIPAddress(ctx context.Context, name, region string) error { + op, err := gcp.addressClient.Delete(ctx, &computepb.DeleteAddressRequest{ + Address: name, + Project: gcp.projectID, + Region: region, + }) + if err != nil { + return fmt.Errorf("failed to delete address %s at %s: %v", name, region, err) + } + + return op.Wait(ctx) +} + func NewGCPCleaner(ctx context.Context) (*GCP, error) { _, defined := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS") if !defined {