Skip to content

feat(gcp): redesign region model, add Secret Manager, recover Init panics #1062

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
james00012 wants to merge 1 commit into
base: master
Choose a base branch
from
+849 −205
Open

feat(gcp): redesign region model, add Secret Manager, recover Init panics #1062

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 49 additions & 53 deletions commands/gcp_commands.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
// These functions implement the CLI commands for GCP operations

// gcpNuke is the main command handler for nuking (deleting) GCP resources.
// It supports region filtering, resource type filtering, time-based filtering,
// It supports location filtering, resource type filtering, time-based filtering,
// and config file overrides.
func gcpNuke(c *cli.Context) error {
defer telemetry.TrackCommandLifecycle("gcp")()
Expand All @@ -30,39 +30,15 @@ func gcpNuke(c *cli.Context) error {
return err
}

// Load config file if provided
configObj, err := loadConfigFile(c.String(FlagConfig))
configObj, query, err := generateGcpQuery(c)
if err != nil {
return errors.WithStackTrace(err)
}

query := &gcp.Query{
ProjectID: c.String(FlagProjectID),
Regions: c.StringSlice(FlagRegion),
ExcludeRegions: c.StringSlice(FlagExcludeRegion),
ResourceTypes: c.StringSlice(FlagResourceType),
ExcludeResourceTypes: c.StringSlice(FlagExcludeResourceType),
ExcludeFirstSeen: c.Bool(FlagExcludeFirstSeen),
}

// Apply timeout to config
if err := parseAndApplyTimeout(c, &configObj); err != nil {
return err
}

// Apply time filters to config
if err := parseAndApplyTimeFilters(c, &configObj); err != nil {
return err
}

// Get output preferences
outputFormat := c.String(FlagOutputFormat)
outputFile := c.String(FlagOutputFile)

if err := query.Validate(); err != nil {
return err
}

return gcpNukeHelper(c, configObj, query, outputFormat, outputFile)
}

Expand All @@ -81,39 +57,15 @@ func gcpInspect(c *cli.Context) error {
return err
}

query := &gcp.Query{
ProjectID: c.String(FlagProjectID),
Regions: c.StringSlice(FlagRegion),
ExcludeRegions: c.StringSlice(FlagExcludeRegion),
ResourceTypes: c.StringSlice(FlagResourceType),
ExcludeResourceTypes: c.StringSlice(FlagExcludeResourceType),
ExcludeFirstSeen: c.Bool(FlagExcludeFirstSeen),
}

// Load config file if provided
configObj, err := loadConfigFile(c.String(FlagConfig))
configObj, query, err := generateGcpQuery(c)
if err != nil {
return errors.WithStackTrace(err)
}

// Apply timeout to config
if err := parseAndApplyTimeout(c, &configObj); err != nil {
return err
}

// Apply time filters to config
if err := parseAndApplyTimeFilters(c, &configObj); err != nil {
return err
}

// Get output preferences
outputFormat := c.String(FlagOutputFormat)
outputFile := c.String(FlagOutputFile)

if err := query.Validate(); err != nil {
return err
}

// Retrieve and display resources without deleting them
_, err = handleGetGcpResourcesWithFormat(c, configObj, query, outputFormat, outputFile)
return err
Expand All @@ -122,6 +74,50 @@ func gcpInspect(c *cli.Context) error {
// Helper Functions
// These functions contain shared logic used by multiple command handlers

// generateGcpQuery parses CLI flags into a validated gcp.Query and config.Config.
func generateGcpQuery(c *cli.Context) (config.Config, *gcp.Query, error) {
configObj, err := loadConfigFile(c.String(FlagConfig))
if err != nil {
return config.Config{}, nil, errors.WithStackTrace(err)
}

excludeAfter, err := parseDurationParam(FlagOlderThan, c.String(FlagOlderThan))
if err != nil {
return config.Config{}, nil, errors.WithStackTrace(err)
}

includeAfter, err := parseDurationParam(FlagNewerThan, c.String(FlagNewerThan))
if err != nil {
return config.Config{}, nil, errors.WithStackTrace(err)
}

timeout, err := parseTimeoutDurationParam(FlagTimeout, c.String(FlagTimeout))
if err != nil {
return config.Config{}, nil, errors.WithStackTrace(err)
}

configObj.ApplyTimeFilters(excludeAfter, includeAfter)
if timeout != nil {
configObj.AddTimeout(timeout)
}

query := &gcp.Query{
ProjectID: c.String(FlagProjectID),
Locations: c.StringSlice(FlagRegion),
ExcludeLocations: c.StringSlice(FlagExcludeRegion),
ResourceTypes: c.StringSlice(FlagResourceType),
ExcludeResourceTypes: c.StringSlice(FlagExcludeResourceType),
ExcludeFirstSeen: c.Bool(FlagExcludeFirstSeen),
Timeout: timeout,
}

if err := query.Validate(c.Context); err != nil {
return config.Config{}, nil, errors.WithStackTrace(err)
}

return configObj, query, nil
}

// gcpNukeHelper is the core logic for nuking GCP resources.
// It retrieves resources, confirms deletion with the user, and executes the nuke operation.
func gcpNukeHelper(c *cli.Context, configObj config.Config, query *gcp.Query, outputFormat string, outputFile string) error {
Expand All @@ -145,14 +141,14 @@ func gcpNukeHelper(c *cli.Context, configObj config.Config, query *gcp.Query, ou
collector.Emit(reporting.ScanComplete{})

// Confirm with user before proceeding (unless --force or --dry-run is set)
shouldProceed, err := confirmNuke(c, len(account.Resources) > 0)
shouldProceed, err := confirmNuke(c, account.TotalResourceCount() > 0)
if err != nil {
return err
}

// Execute the nuke operation if confirmed
if shouldProceed {
if err := gcp.NukeAllResources(c.Context, account, query.Regions, collector); err != nil {
if err := gcp.NukeAllResources(c.Context, account, collector); err != nil {
return err
}
}
Expand Down
2 changes: 2 additions & 0 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ type Config struct {
CloudFunction ResourceType `yaml:"CloudFunction"`
ArtifactRegistry ResourceType `yaml:"ArtifactRegistry"`
GcpPubSubTopic ResourceType `yaml:"GcpPubSubTopic"`
GcpSecretManager ResourceType `yaml:"GcpSecretManager"`
}

// allResourceTypes returns pointers to the embedded ResourceType for every
Expand Down Expand Up @@ -295,6 +296,7 @@ func (c *Config) allResourceTypes() []*ResourceType {
&c.CloudFunction,
&c.ArtifactRegistry,
&c.GcpPubSubTopic,
&c.GcpSecretManager,
}
}

Expand Down
134 changes: 65 additions & 69 deletions gcp/gcp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ import (
"github.com/hashicorp/go-multierror"
)

// projectKey is the single map key used for GcpProjectResources.Resources.
// GCP resources are project-scoped; location is a filter hint, not a scope.
const projectKey = "project"

// IsNukeable checks whether a resource type should be nuked based on the
// requested resource types and exclude lists. An empty include list or the
// special value "all" means nuke everything, minus any excluded types.
Expand All @@ -34,67 +38,71 @@ func IsNukeable(resourceType string, resourceTypes []string, excludeResourceType
}

// GetAllResources lists all GCP resources that can be deleted.
// Each resource is called exactly once; location filtering is handled internally by each resource's Lister.
func GetAllResources(ctx context.Context, query *Query, configObj config.Config, collector *reporting.Collector) (*GcpProjectResources, error) {
allResources := GcpProjectResources{
ProjectID: query.ProjectID,
Resources: map[string]GcpResources{},
}

ctx = context.WithValue(ctx, util.ExcludeFirstSeenTagKey, query.ExcludeFirstSeen)

for _, region := range query.Regions {
cfg := resources.GcpConfig{ProjectID: query.ProjectID, Region: region}
regionResources := GetAndInitRegisteredResources(cfg, region)
cfg := resources.GcpConfig{
ProjectID: query.ProjectID,
Locations: query.Locations,
ExcludeLocations: query.ExcludeLocations,
}
allRegistered := GetAndInitRegisteredResources(cfg)

for _, res := range allRegistered {
resourceName := (*res).ResourceName()

for _, res := range regionResources {
resourceName := (*res).ResourceName()
if !IsNukeable(resourceName, query.ResourceTypes, query.ExcludeResourceTypes) {
continue
}

// Emit scan progress event
collector.Emit(reporting.ScanProgress{
ResourceType: resourceName,
Region: query.ProjectID,
})

if !IsNukeable(resourceName, query.ResourceTypes, query.ExcludeResourceTypes) {
// Get all resource identifiers
identifiers, err := (*res).GetAndSetIdentifiers(ctx, configObj)
if err != nil {
if isServiceDisabledError(err) && !collections.ListContainsElement(query.ResourceTypes, resourceName) {
logging.Debugf("Skipping %s: API is disabled in this project", resourceName)
continue
}

// Emit scan progress event
collector.Emit(reporting.ScanProgress{
logging.Debugf("Error getting identifiers for %s: %v", resourceName, err)
collector.Emit(reporting.GeneralError{
ResourceType: resourceName,
Region: region,
Description: fmt.Sprintf("Unable to retrieve %s", resourceName),
Error: err.Error(),
})
continue
}

// Get all resource identifiers
identifiers, err := (*res).GetAndSetIdentifiers(ctx, configObj)
if err != nil {
if isServiceDisabledError(err) && !collections.ListContainsElement(query.ResourceTypes, resourceName) {
logging.Debugf("Skipping %s: API is disabled in this project", resourceName)
continue
}
logging.Debugf("Error getting identifiers for %s: %v", resourceName, err)
collector.Emit(reporting.GeneralError{
ResourceType: resourceName,
Description: fmt.Sprintf("Unable to retrieve %s", resourceName),
Error: err.Error(),
})
continue
// Only append if we have non-empty identifiers
if len(identifiers) > 0 {
logging.Infof("Found %d %s resources", len(identifiers), resourceName)
allResources.Resources[projectKey] = GcpResources{
Resources: append(allResources.Resources[projectKey].Resources, res),
}

// Only append if we have non-empty identifiers
if len(identifiers) > 0 {
logging.Infof("Found %d %s resources", len(identifiers), resourceName)
allResources.Resources[region] = GcpResources{
Resources: append(allResources.Resources[region].Resources, res),
}

// Emit ResourceFound events for each identifier
for _, id := range identifiers {
nukable, reason := true, ""
if _, err := (*res).IsNukable(id); err != nil {
nukable, reason = false, err.Error()
}
collector.Emit(reporting.ResourceFound{
ResourceType: resourceName,
Region: region,
Identifier: id,
Nukable: nukable,
Reason: reason,
})
// Emit ResourceFound events for each identifier
for _, id := range identifiers {
nukable, reason := true, ""
if _, err := (*res).IsNukable(id); err != nil {
nukable, reason = false, err.Error()
}
collector.Emit(reporting.ResourceFound{
ResourceType: resourceName,
Region: query.ProjectID,
Identifier: id,
Nukable: nukable,
Reason: reason,
})
}
}
}
Expand All @@ -105,16 +113,18 @@ func GetAllResources(ctx context.Context, query *Query, configObj config.Config,
return &allResources, nil
}

// NukeAllResources nukes all GCP resources across the given regions.
func NukeAllResources(ctx context.Context, account *GcpProjectResources, regions []string, collector *reporting.Collector) error {
// NukeAllResources nukes all GCP resources in the project.
func NukeAllResources(ctx context.Context, account *GcpProjectResources, collector *reporting.Collector) error {
// Emit NukeStarted event (CLIRenderer will initialize progress bar)
collector.Emit(reporting.NukeStarted{Total: account.TotalResourceCount()})

var allErrors *multierror.Error

for _, region := range regions {
if err := nukeAllResourcesInRegion(ctx, account, region, collector); err != nil {
allErrors = multierror.Append(allErrors, err)
for _, gcpResources := range account.Resources {
for _, gcpResource := range gcpResources.Resources {
if err := nukeResource(ctx, gcpResource, account.ProjectID, collector); err != nil {
allErrors = multierror.Append(allErrors, err)
}
}
}

Expand All @@ -124,22 +134,8 @@ func NukeAllResources(ctx context.Context, account *GcpProjectResources, regions
return allErrors.ErrorOrNil()
}

// nukeAllResourcesInRegion nukes all resources in a single region.
func nukeAllResourcesInRegion(ctx context.Context, account *GcpProjectResources, region string, collector *reporting.Collector) error {
var allErrors *multierror.Error

resourcesInRegion := account.Resources[region]
for _, gcpResource := range resourcesInRegion.Resources {
if err := nukeResource(ctx, gcpResource, region, collector); err != nil {
allErrors = multierror.Append(allErrors, err)
}
}

return allErrors.ErrorOrNil()
}

// nukeResource nukes a single GCP resource type
func nukeResource(ctx context.Context, gcpResource *GcpResource, region string, collector *reporting.Collector) error {
// nukeResource nukes a single GCP resource type within a project scope.
func nukeResource(ctx context.Context, gcpResource *GcpResource, scope string, collector *reporting.Collector) error {
// Filter to only nukable resources
var nukableIdentifiers []string
for _, id := range (*gcpResource).ResourceIdentifiers() {
Expand All @@ -164,7 +160,7 @@ func nukeResource(ctx context.Context, gcpResource *GcpResource, region string,
// Emit progress event (CLIRenderer updates its progress bar)
collector.Emit(reporting.NukeProgress{
ResourceType: (*gcpResource).ResourceName(),
Region: region,
Region: scope,
BatchSize: len(batch),
})

Expand All @@ -178,7 +174,7 @@ func nukeResource(ctx context.Context, gcpResource *GcpResource, region string,
}
collector.Emit(reporting.ResourceDeleted{
ResourceType: (*gcpResource).ResourceName(),
Region: region,
Region: scope,
Identifier: result.Identifier,
Success: result.Error == nil,
Error: errStr,
Expand All @@ -194,13 +190,13 @@ func nukeResource(ctx context.Context, gcpResource *GcpResource, region string,
continue
}

allErrors = multierror.Append(allErrors, fmt.Errorf("[%s] %s: %w", region, (*gcpResource).ResourceName(), err))
allErrors = multierror.Append(allErrors, fmt.Errorf("[%s] %s: %w", scope, (*gcpResource).ResourceName(), err))

// Report to telemetry - aggregated metrics of failures per resources.
telemetry.TrackEvent(commonTelemetry.EventContext{
EventName: fmt.Sprintf("error:Nuke:%s", (*gcpResource).ResourceName()),
}, map[string]interface{}{
"region": region,
"scope": scope,
})
}

Expand Down
Loading
Loading