From d4b7f8485592350668d6c5078fc8e3019fde8648 Mon Sep 17 00:00:00 2001 From: Victor Moraes Date: Wed, 7 Feb 2024 21:43:42 -0300 Subject: [PATCH] ENG-13394: Add Dmap Scanner library (#1) * Add base structure * Remove hello files * Update awsClient AssumeRole * Add config validation * Update scanAWSRepositories to use go routines * Update Repository structure * Add appendError to Scanner * Handle pagination for AWS requests * Implement repository converters * Create aws and model packages * Add Scanner interface * Remove unnecessary IsAWSConfigured and rename alias for aws types * Refactor go routines in AWS Scan * Update cluster instance check in scanRDSInstanceRepositories * Refactor getDynamoDBTables * Update error wrapping * Add interfaces for AWS service clients * Add tests * Update RepoType values * Remove ScanManager * Update AWSScanner to scan regions concurrently * Fix imports * Add description to exported types and functions * Fix newRepositoryFromDynamoDBTable to use Table ARN as Id * Move scan type definitions to scanner.go file * Add IAM Role validation * Update scanErrors * Refactor go routines * Extract tag formatting into function * Add lint --- Makefile | 17 ++ aws/client.go | 231 ++++++++++++++++++++++ aws/config.go | 55 ++++++ aws/config_test.go | 70 +++++++ aws/repository.go | 91 +++++++++ aws/scan.go | 122 ++++++++++++ aws/scanner.go | 157 +++++++++++++++ aws/scanner_integration_test.go | 44 +++++ aws/scanner_test.go | 331 ++++++++++++++++++++++++++++++++ go.mod | 30 +++ go.sum | 55 ++++++ scan/hello.go | 5 - scan/hello_test.go | 26 --- scan/scanner.go | 41 ++++ testutil/mock/aws.go | 164 ++++++++++++++++ testutil/mock/scanner.go | 18 ++ 16 files changed, 1426 insertions(+), 31 deletions(-) create mode 100644 Makefile create mode 100644 aws/client.go create mode 100644 aws/config.go create mode 100644 aws/config_test.go create mode 100644 aws/repository.go create mode 100644 aws/scan.go create mode 100644 aws/scanner.go create mode 100644 aws/scanner_integration_test.go create mode 100644 aws/scanner_test.go delete mode 100644 scan/hello.go delete mode 100644 scan/hello_test.go create mode 100644 scan/scanner.go create mode 100644 testutil/mock/aws.go create mode 100644 testutil/mock/scanner.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..569d168 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +all: tidy lint test + +tidy: + go mod tidy + +lint: + golangci-lint run + +# Using --count=1 disables test caching +test: + go test -v -race ./... --count=1 + +integration-test: + go test -v -race ./... --count=1 --tags=integration + +clean: + go clean -i ./... diff --git a/aws/client.go b/aws/client.go new file mode 100644 index 0000000..81fadea --- /dev/null +++ b/aws/client.go @@ -0,0 +1,231 @@ +package aws + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + ddbTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/aws/aws-sdk-go-v2/service/rds" + rdsTypes "github.com/aws/aws-sdk-go-v2/service/rds/types" + "github.com/aws/aws-sdk-go-v2/service/redshift" + rsTypes "github.com/aws/aws-sdk-go-v2/service/redshift/types" +) + +type rdsClient interface { + DescribeDBClusters( + ctx context.Context, + params *rds.DescribeDBClustersInput, + optFns ...func(*rds.Options), + ) (*rds.DescribeDBClustersOutput, error) + DescribeDBInstances( + ctx context.Context, + params *rds.DescribeDBInstancesInput, + optFns ...func(*rds.Options), + ) (*rds.DescribeDBInstancesOutput, error) +} + +type redshiftClient interface { + DescribeClusters( + ctx context.Context, + params *redshift.DescribeClustersInput, + optFns ...func(*redshift.Options), + ) (*redshift.DescribeClustersOutput, error) +} + +type dynamoDBClient interface { + ListTables( + ctx context.Context, + params *dynamodb.ListTablesInput, + optFns ...func(*dynamodb.Options), + ) (*dynamodb.ListTablesOutput, error) + DescribeTable( + ctx context.Context, + params *dynamodb.DescribeTableInput, + optFns ...func(*dynamodb.Options), + ) (*dynamodb.DescribeTableOutput, error) + ListTagsOfResource( + ctx context.Context, + params *dynamodb.ListTagsOfResourceInput, + optFns ...func(*dynamodb.Options), + ) (*dynamodb.ListTagsOfResourceOutput, error) +} + +type awsClient struct { + config aws.Config + rds rdsClient + redshift redshiftClient + dynamodb dynamoDBClient +} + +type awsClientConstructor func(awsConfig aws.Config) *awsClient + +func newAWSClient(awsConfig aws.Config) *awsClient { + return &awsClient{ + config: awsConfig, + rds: rds.NewFromConfig(awsConfig), + redshift: redshift.NewFromConfig(awsConfig), + dynamodb: dynamodb.NewFromConfig(awsConfig), + } +} + +func (c *awsClient) getRDSClusters( + ctx context.Context, +) ([]rdsTypes.DBCluster, error) { + var clusters []rdsTypes.DBCluster + // Used for pagination + var marker *string + for { + output, err := c.rds.DescribeDBClusters( + ctx, + &rds.DescribeDBClustersInput{ + Marker: marker, + }, + ) + if err != nil { + return nil, err + } + + clusters = append(clusters, output.DBClusters...) + + if output.Marker == nil { + break + } else { + marker = output.Marker + } + } + return clusters, nil +} + +func (c *awsClient) getRDSInstances( + ctx context.Context, +) ([]rdsTypes.DBInstance, error) { + var instances []rdsTypes.DBInstance + // Used for pagination + var marker *string + for { + output, err := c.rds.DescribeDBInstances( + ctx, + &rds.DescribeDBInstancesInput{ + Marker: marker, + }, + ) + if err != nil { + return nil, err + } + + instances = append(instances, output.DBInstances...) + + if output.Marker == nil { + break + } else { + marker = output.Marker + } + } + return instances, nil +} + +func (c *awsClient) getRedshiftClusters( + ctx context.Context, +) ([]rsTypes.Cluster, error) { + var clusters []rsTypes.Cluster + // Used for pagination + var marker *string + for { + output, err := c.redshift.DescribeClusters( + ctx, + &redshift.DescribeClustersInput{ + Marker: marker, + }, + ) + if err != nil { + return nil, err + } + + clusters = append(clusters, output.Clusters...) + + if output.Marker == nil { + break + } else { + marker = output.Marker + } + } + return clusters, nil +} + +type dynamoDBTable struct { + Table ddbTypes.TableDescription + Tags []ddbTypes.Tag +} + +func (c *awsClient) getDynamoDBTables( + ctx context.Context, +) ([]dynamoDBTable, error) { + var tableNames []string + // Used for pagination + var exclusiveStartTableName *string + for { + output, err := c.dynamodb.ListTables( + ctx, + &dynamodb.ListTablesInput{ + ExclusiveStartTableName: exclusiveStartTableName, + }, + ) + if err != nil { + return nil, err + } + + tableNames = append(tableNames, output.TableNames...) + + if output.LastEvaluatedTableName == nil { + break + } else { + exclusiveStartTableName = output.LastEvaluatedTableName + } + } + + tables := make([]dynamoDBTable, 0, len(tableNames)) + for i := range tableNames { + tableName := tableNames[i] + describeTableOutput, err := c.dynamodb.DescribeTable( + ctx, + &dynamodb.DescribeTableInput{ + TableName: &tableName, + }, + ) + if err != nil { + return nil, err + } + table := describeTableOutput.Table + + var tableTags []ddbTypes.Tag + // Used for pagination + var nextToken *string + for { + tagsOutput, err := c.dynamodb.ListTagsOfResource( + ctx, + &dynamodb.ListTagsOfResourceInput{ + ResourceArn: table.TableArn, + NextToken: nextToken, + }, + ) + if err != nil { + return nil, err + } + + tableTags = append(tableTags, tagsOutput.Tags...) + + if tagsOutput.NextToken == nil { + break + } else { + nextToken = tagsOutput.NextToken + } + } + + tables = append(tables, dynamoDBTable{ + Table: *table, + Tags: tableTags, + }) + } + return tables, nil +} diff --git a/aws/config.go b/aws/config.go new file mode 100644 index 0000000..3f515db --- /dev/null +++ b/aws/config.go @@ -0,0 +1,55 @@ +package aws + +import ( + "fmt" + "regexp" +) + +// ScannerConfig represents an AWSScanner configuration. It allows defining the +// AWS regions that should be scanned and an optional AssumeRoleConfig that +// contains the configuration for assuming an IAM Role during the scan. If +// AssumeRoleConfig is nil, the AWS default external configuration will be used +// instead. +type ScannerConfig struct { + Regions []string + AssumeRole *AssumeRoleConfig +} + +// AssumeRoleConfig represents the information of an IAM Role to be assumed by +// the AWSScanner when performing request to the AWS services during the data +// repositories scan. +type AssumeRoleConfig struct { + // The ARN of the IAM Role to be assumed. + IAMRoleARN string + // Optional External ID to be used as part of the assume role process. + ExternalID string +} + +// Validate validates the ScannerConfig configuration. +func (config *ScannerConfig) Validate() error { + if len(config.Regions) == 0 { + return fmt.Errorf("AWS regions are required") + } + for _, region := range config.Regions { + if region == "" { + return fmt.Errorf("AWS region can't be empty") + } + } + if config.AssumeRole != nil { + iamRolePatern := "^arn:aws:iam::\\d{12}:role/.*$" + match, err := regexp.MatchString( + iamRolePatern, + config.AssumeRole.IAMRoleARN, + ) + if err != nil { + return fmt.Errorf("error verifying IAM Role format: %w", err) + } + if !match { + return fmt.Errorf( + "invalid IAM Role: must match format '%s'", + iamRolePatern, + ) + } + } + return nil +} diff --git a/aws/config_test.go b/aws/config_test.go new file mode 100644 index 0000000..d16a521 --- /dev/null +++ b/aws/config_test.go @@ -0,0 +1,70 @@ +package aws + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type ScannerConfigTestSuite struct { + suite.Suite +} + +func TestScannerConfig(t *testing.T) { + s := new(ScannerConfigTestSuite) + suite.Run(t, s) +} + +func (s *ScannerConfigTestSuite) TestValidate() { + type TestCase struct { + description string + config ScannerConfig + expectedErrorMsg string + } + tests := []TestCase{ + { + description: "No regions should return error", + config: ScannerConfig{}, + expectedErrorMsg: "AWS regions are required", + }, + { + description: "Empty region should return error", + config: ScannerConfig{ + Regions: []string{""}, + }, + expectedErrorMsg: "AWS region can't be empty", + }, + { + description: "Invalid IAM Role format should return error", + config: ScannerConfig{ + Regions: []string{"us-east-1"}, + AssumeRole: &AssumeRoleConfig{ + IAMRoleARN: "invalid-iam-role-format", + }, + }, + expectedErrorMsg: "invalid IAM Role: must match format " + + "'^arn:aws:iam::\\d{12}:role/.*$'", + }, + { + description: "Valid config should return nil error", + config: ScannerConfig{ + Regions: []string{"us-east-1"}, + AssumeRole: &AssumeRoleConfig{ + IAMRoleARN: "arn:aws:iam::123456789012:role/SomeIAMRole", + }, + }, + }, + } + + for _, test := range tests { + s.T().Run(test.description, func(t *testing.T) { + err := test.config.Validate() + if test.expectedErrorMsg == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, test.expectedErrorMsg) + } + }) + } +} diff --git a/aws/repository.go b/aws/repository.go new file mode 100644 index 0000000..77eac94 --- /dev/null +++ b/aws/repository.go @@ -0,0 +1,91 @@ +package aws + +import ( + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + rdsTypes "github.com/aws/aws-sdk-go-v2/service/rds/types" + redshiftTypes "github.com/aws/aws-sdk-go-v2/service/redshift/types" + + "github.com/cyralinc/dmap/scan" +) + +func newRepositoryFromRDSCluster( + cluster rdsTypes.DBCluster, +) scan.Repository { + tags := make([]string, 0, len(cluster.TagList)) + for _, tag := range cluster.TagList { + tags = append(tags, formatTag(tag.Key, tag.Value)) + } + + return scan.Repository{ + Id: aws.ToString(cluster.DBClusterArn), + Name: aws.ToString(cluster.DBClusterIdentifier), + CreatedAt: aws.ToTime(cluster.ClusterCreateTime), + Type: scan.RepoTypeRDS, + Tags: tags, + Properties: cluster, + } +} + +func newRepositoryFromRDSInstance( + instance rdsTypes.DBInstance, +) scan.Repository { + tags := make([]string, 0, len(instance.TagList)) + for _, tag := range instance.TagList { + tags = append(tags, formatTag(tag.Key, tag.Value)) + } + + return scan.Repository{ + Id: aws.ToString(instance.DBInstanceArn), + Name: aws.ToString(instance.DBInstanceIdentifier), + CreatedAt: aws.ToTime(instance.InstanceCreateTime), + Type: scan.RepoTypeRDS, + Tags: tags, + Properties: instance, + } +} + +func newRepositoryFromRedshiftCluster( + cluster redshiftTypes.Cluster, +) scan.Repository { + tags := make([]string, 0, len(cluster.Tags)) + for _, tag := range cluster.Tags { + tags = append(tags, formatTag(tag.Key, tag.Value)) + } + + return scan.Repository{ + Id: aws.ToString(cluster.ClusterNamespaceArn), + Name: aws.ToString(cluster.ClusterIdentifier), + CreatedAt: aws.ToTime(cluster.ClusterCreateTime), + Type: scan.RepoTypeRedshift, + Tags: tags, + Properties: cluster, + } +} + +func newRepositoryFromDynamoDBTable( + table dynamoDBTable, +) scan.Repository { + tags := make([]string, 0, len(table.Tags)) + for _, tag := range table.Tags { + tags = append(tags, formatTag(tag.Key, tag.Value)) + } + + return scan.Repository{ + Id: aws.ToString(table.Table.TableArn), + Name: aws.ToString(table.Table.TableName), + CreatedAt: aws.ToTime(table.Table.CreationDateTime), + Type: scan.RepoTypeDynamoDB, + Tags: tags, + Properties: table.Table, + } +} + +func formatTag(key, value *string) string { + return fmt.Sprintf( + "%s:%s", + aws.ToString(key), + aws.ToString(value), + ) +} diff --git a/aws/scan.go b/aws/scan.go new file mode 100644 index 0000000..64fcb83 --- /dev/null +++ b/aws/scan.go @@ -0,0 +1,122 @@ +package aws + +import ( + "context" + "fmt" + + "github.com/cyralinc/dmap/scan" +) + +type scanFunction func( + ctx context.Context, + awsClient *awsClient, +) scanResponse + +type scanResponse struct { + repositories []scan.Repository + scanErrors []error +} + +func scanRDSClusterRepositories( + ctx context.Context, + awsClient *awsClient, +) scanResponse { + repositories := []scan.Repository{} + var scanErrors []error + rdsClusters, err := awsClient.getRDSClusters(ctx) + if err != nil { + scanErrors = append(scanErrors, fmt.Errorf( + "error scanning RDS clusters: %w", + err, + )) + } + for _, cluster := range rdsClusters { + repositories = append( + repositories, + newRepositoryFromRDSCluster(cluster), + ) + } + return scanResponse{ + repositories: repositories, + scanErrors: scanErrors, + } +} + +func scanRDSInstanceRepositories( + ctx context.Context, + awsClient *awsClient, +) scanResponse { + repositories := []scan.Repository{} + var scanErrors []error + rdsInstances, err := awsClient.getRDSInstances(ctx) + if err != nil { + scanErrors = append(scanErrors, fmt.Errorf( + "error scanning RDS instances: %w", + err, + )) + } + for _, instance := range rdsInstances { + // Skip cluster instances, since they were already added when retrieving + // the RDS clusters. + if instance.DBClusterIdentifier == nil { + repositories = append( + repositories, + newRepositoryFromRDSInstance(instance), + ) + } + } + return scanResponse{ + repositories: repositories, + scanErrors: scanErrors, + } +} + +func scanRedshiftRepositories( + ctx context.Context, + awsClient *awsClient, +) scanResponse { + repositories := []scan.Repository{} + var scanErrors []error + redshiftClusters, err := awsClient.getRedshiftClusters(ctx) + if err != nil { + scanErrors = append(scanErrors, fmt.Errorf( + "error scanning Redshift clusters: %w", + err, + )) + } + for _, cluster := range redshiftClusters { + repositories = append( + repositories, + newRepositoryFromRedshiftCluster(cluster), + ) + } + return scanResponse{ + repositories: repositories, + scanErrors: scanErrors, + } +} + +func scanDynamoDBRepositories( + ctx context.Context, + awsClient *awsClient, +) scanResponse { + repositories := []scan.Repository{} + var scanErrors []error + dynamodbTables, err := awsClient.getDynamoDBTables(ctx) + if err != nil { + scanErrors = append(scanErrors, fmt.Errorf( + "error scanning DynamoDB tables: %w", + err, + )) + } + for _, table := range dynamodbTables { + repositories = append( + repositories, + newRepositoryFromDynamoDBTable(table), + ) + } + return scanResponse{ + repositories: repositories, + scanErrors: scanErrors, + } +} diff --git a/aws/scanner.go b/aws/scanner.go new file mode 100644 index 0000000..a42a6db --- /dev/null +++ b/aws/scanner.go @@ -0,0 +1,157 @@ +package aws + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/sts" + + "github.com/cyralinc/dmap/scan" +) + +// AWSScanner is an implementation of the Scanner interface for the AWS cloud +// provider. It supports scanning data repositories from multiple AWS regions, +// including RDS clusters and instances, Redshift clusters and DynamoDB tables. +type AWSScanner struct { + scannerConfig ScannerConfig + awsConfig aws.Config + awsClientConstructor awsClientConstructor +} + +// AWSScanner implements scan.Scanner +var _ scan.Scanner = (*AWSScanner)(nil) + +// NewAWSScanner creates a new instance of AWSScanner based on the ScannerConfig. +// If AssumeRoleConfig is specified, the AWSScanner will assume this IAM Role +// and use it during service requests. If AssumeRoleConfig is nil, the AWSScanner +// will use the AWS default external configuration. +func NewAWSScanner( + ctx context.Context, + scannerConfig ScannerConfig, +) (*AWSScanner, error) { + if err := scannerConfig.Validate(); err != nil { + return nil, fmt.Errorf("invalid scanner config: %w", err) + } + awsConfig, err := awsconfig.LoadDefaultConfig(ctx) + if err != nil { + return nil, err + } + s := &AWSScanner{ + scannerConfig: scannerConfig, + awsConfig: awsConfig, + } + if s.scannerConfig.AssumeRole != nil { + if err := s.assumeRole(ctx); err != nil { + return nil, fmt.Errorf("error assuming IAM role: %w", err) + } + } + s.awsClientConstructor = newAWSClient + return s, nil +} + +// Scan performs a scan across all the AWS regions configured and return a scan +// results, containing a list of data repositories that includes: RDS clusters +// and instances, Redshift clusters and DynamoDB tables. +func (s *AWSScanner) Scan(ctx context.Context) (*scan.ScanResults, error) { + repositories := []scan.Repository{} + var scanErrors []error + + responseChan := make(chan scanResponse) + var wg sync.WaitGroup + wg.Add(len(s.scannerConfig.Regions)) + + for i := range s.scannerConfig.Regions { + go func(region string) { + defer wg.Done() + response := scanRegion(ctx, s.awsConfig, region, s.awsClientConstructor) + responseChan <- response + }(s.scannerConfig.Regions[i]) + } + + go func() { + wg.Wait() + close(responseChan) + }() + + for response := range responseChan { + repositories = append(repositories, response.repositories...) + scanErrors = append(scanErrors, response.scanErrors...) + } + + scanResults := &scan.ScanResults{ + Repositories: repositories, + } + + return scanResults, errors.Join(scanErrors...) +} + +func scanRegion( + ctx context.Context, + awsConfig aws.Config, + region string, + newAWSClient awsClientConstructor, +) scanResponse { + repositories := []scan.Repository{} + var scanErrors []error + + awsConfig.Region = region + awsClient := newAWSClient(awsConfig) + + scanFunctions := []scanFunction{ + scanRDSClusterRepositories, + scanRDSInstanceRepositories, + scanRedshiftRepositories, + scanDynamoDBRepositories, + } + + responseChan := make(chan scanResponse) + var wg sync.WaitGroup + wg.Add(len(scanFunctions)) + + for i := range scanFunctions { + go func(scanFunc scanFunction) { + defer wg.Done() + response := scanFunc(ctx, awsClient) + responseChan <- response + }(scanFunctions[i]) + } + + go func() { + wg.Wait() + close(responseChan) + }() + + for response := range responseChan { + repositories = append(repositories, response.repositories...) + scanErrors = append(scanErrors, response.scanErrors...) + } + + return scanResponse{ + repositories: repositories, + scanErrors: scanErrors, + } +} + +func (s *AWSScanner) assumeRole( + ctx context.Context, +) error { + stsClient := sts.NewFromConfig(s.awsConfig) + credsProvider := stscreds.NewAssumeRoleProvider( + stsClient, + s.scannerConfig.AssumeRole.IAMRoleARN, + func(o *stscreds.AssumeRoleOptions) { + o.ExternalID = &s.scannerConfig.AssumeRole.ExternalID + }, + ) + s.awsConfig.Credentials = aws.NewCredentialsCache(credsProvider) + // Validate AWS credentials provider. + if _, err := s.awsConfig.Credentials.Retrieve(ctx); err != nil { + return fmt.Errorf("failed to retrieve AWS credentials: %w", err) + } + return nil +} diff --git a/aws/scanner_integration_test.go b/aws/scanner_integration_test.go new file mode 100644 index 0000000..4a8e6d7 --- /dev/null +++ b/aws/scanner_integration_test.go @@ -0,0 +1,44 @@ +//go:build integration + +package aws + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type AWSScannerIntegrationTestSuite struct { + suite.Suite + scanner *AWSScanner +} + +func (s *AWSScannerIntegrationTestSuite) SetupSuite() { + ctx := context.Background() + scanner, err := NewAWSScanner(ctx, ScannerConfig{ + Regions: []string{ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + }, + }) + require.NoError(s.T(), err) + s.scanner = scanner +} + +func TestIntegrationAWSScanner(t *testing.T) { + s := new(AWSScannerIntegrationTestSuite) + suite.Run(t, s) +} + +func (s *AWSScannerIntegrationTestSuite) TestScan() { + ctx := context.Background() + results, scanErrors := s.scanner.Scan(ctx) + fmt.Printf("Num. Repositories: %v\n", len(results.Repositories)) + fmt.Printf("Repositories: %v\n", results.Repositories) + fmt.Printf("Scan Erros: %v\n", scanErrors) +} diff --git a/aws/scanner_test.go b/aws/scanner_test.go new file mode 100644 index 0000000..dd78a05 --- /dev/null +++ b/aws/scanner_test.go @@ -0,0 +1,331 @@ +package aws + +import ( + "context" + "fmt" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + rdsTypes "github.com/aws/aws-sdk-go-v2/service/rds/types" + redshiftTypes "github.com/aws/aws-sdk-go-v2/service/redshift/types" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/cyralinc/dmap/scan" + "github.com/cyralinc/dmap/testutil/mock" +) + +type AWSScannerTestSuite struct { + suite.Suite + dummyRDSClusters []rdsTypes.DBCluster + dummyRDSInstances []rdsTypes.DBInstance + dummyRedshiftClusters []redshiftTypes.Cluster + dummyDynamoDBTableNames []string + dummyDynamoDBTable map[string]*types.TableDescription + dummyDynamoDBTags []types.Tag +} + +func (s *AWSScannerTestSuite) SetupSuite() { + s.dummyRDSClusters = []rdsTypes.DBCluster{ + { + DBClusterArn: aws.String("dummy-arn-1"), + DBClusterIdentifier: aws.String("rds-cluster-1"), + }, + { + DBClusterArn: aws.String("dummy-arn-2"), + DBClusterIdentifier: aws.String("rds-cluster-2"), + }, + { + DBClusterArn: aws.String("dummy-arn-3"), + DBClusterIdentifier: aws.String("rds-cluster-3"), + }, + } + s.dummyRDSInstances = []rdsTypes.DBInstance{ + { + DBInstanceArn: aws.String("dummy-arn-1"), + DBInstanceIdentifier: aws.String("rds-instance-1"), + }, + { + DBInstanceArn: aws.String("dummy-arn-2"), + DBInstanceIdentifier: aws.String("rds-instance-2"), + }, + { + DBInstanceArn: aws.String("dummy-arn-3"), + DBInstanceIdentifier: aws.String("rds-instance-3"), + }, + } + s.dummyRedshiftClusters = []redshiftTypes.Cluster{ + { + ClusterNamespaceArn: aws.String("dummy-arn-1"), + ClusterIdentifier: aws.String("redshift-cluster-1"), + }, + { + ClusterNamespaceArn: aws.String("dummy-arn-2"), + ClusterIdentifier: aws.String("redshift-cluster-2"), + }, + { + ClusterNamespaceArn: aws.String("dummy-arn-3"), + ClusterIdentifier: aws.String("redshift-cluster-3"), + }, + } + s.dummyDynamoDBTableNames = []string{ + "dynamodb-table-1", + "dynamodb-table-2", + "dynamodb-table-3", + } + s.dummyDynamoDBTable = map[string]*types.TableDescription{ + s.dummyDynamoDBTableNames[0]: { + TableArn: aws.String("dummy-arn-1"), + TableName: aws.String(s.dummyDynamoDBTableNames[0]), + }, + s.dummyDynamoDBTableNames[1]: { + TableArn: aws.String("dummy-arn-2"), + TableName: aws.String(s.dummyDynamoDBTableNames[1]), + }, + s.dummyDynamoDBTableNames[2]: { + TableArn: aws.String("dummy-arn-3"), + TableName: aws.String(s.dummyDynamoDBTableNames[2]), + }, + } + s.dummyDynamoDBTags = []types.Tag{ + { + Key: aws.String("tag1"), + Value: aws.String("value1"), + }, + { + Key: aws.String("tag2"), + Value: aws.String("value2"), + }, + { + Key: aws.String("tag3"), + Value: aws.String("value3"), + }, + } +} + +func TestAWSScanner(t *testing.T) { + s := new(AWSScannerTestSuite) + suite.Run(t, s) +} + +func (s *AWSScannerTestSuite) TestScan() { + awsScanner := AWSScanner{ + scannerConfig: ScannerConfig{ + Regions: []string{ + "us-east-1", + }, + AssumeRole: &AssumeRoleConfig{ + IAMRoleARN: "arn:aws:iam::123456789012:role/SomeIAMRole", + ExternalID: "some-external-id-12345", + }, + }, + awsConfig: aws.Config{}, + awsClientConstructor: func(awsConfig aws.Config) *awsClient { + return &awsClient{ + config: awsConfig, + rds: &mock.MockRDSClient{ + DBClusters: s.dummyRDSClusters, + DBInstances: s.dummyRDSInstances, + }, + redshift: &mock.MockRedshiftClient{ + Clusters: s.dummyRedshiftClusters, + }, + dynamodb: &mock.MockDynamoDBClient{ + TableNames: s.dummyDynamoDBTableNames, + Table: s.dummyDynamoDBTable, + Tags: s.dummyDynamoDBTags, + }, + } + }, + } + ctx := context.Background() + results, err := awsScanner.Scan(ctx) + + expectedResults := &scan.ScanResults{ + Repositories: []scan.Repository{ + { + Id: *s.dummyRDSClusters[0].DBClusterArn, + Name: *s.dummyRDSClusters[0].DBClusterIdentifier, + Type: scan.RepoTypeRDS, + Tags: []string{}, + Properties: s.dummyRDSClusters[0], + }, + { + Id: *s.dummyRDSClusters[1].DBClusterArn, + Name: *s.dummyRDSClusters[1].DBClusterIdentifier, + Type: scan.RepoTypeRDS, + Tags: []string{}, + Properties: s.dummyRDSClusters[1], + }, + { + Id: *s.dummyRDSClusters[2].DBClusterArn, + Name: *s.dummyRDSClusters[2].DBClusterIdentifier, + Type: scan.RepoTypeRDS, + Tags: []string{}, + Properties: s.dummyRDSClusters[2], + }, + { + Id: *s.dummyRDSInstances[0].DBInstanceArn, + Name: *s.dummyRDSInstances[0].DBInstanceIdentifier, + Type: scan.RepoTypeRDS, + Tags: []string{}, + Properties: s.dummyRDSInstances[0], + }, + { + Id: *s.dummyRDSInstances[1].DBInstanceArn, + Name: *s.dummyRDSInstances[1].DBInstanceIdentifier, + Type: scan.RepoTypeRDS, + Tags: []string{}, + Properties: s.dummyRDSInstances[1], + }, + { + Id: *s.dummyRDSInstances[2].DBInstanceArn, + Name: *s.dummyRDSInstances[2].DBInstanceIdentifier, + Type: scan.RepoTypeRDS, + Tags: []string{}, + Properties: s.dummyRDSInstances[2], + }, + { + Id: *s.dummyRedshiftClusters[0].ClusterNamespaceArn, + Name: *s.dummyRedshiftClusters[0].ClusterIdentifier, + Type: scan.RepoTypeRedshift, + Tags: []string{}, + Properties: s.dummyRedshiftClusters[0], + }, + { + Id: *s.dummyRedshiftClusters[1].ClusterNamespaceArn, + Name: *s.dummyRedshiftClusters[1].ClusterIdentifier, + Type: scan.RepoTypeRedshift, + Tags: []string{}, + Properties: s.dummyRedshiftClusters[1], + }, + { + Id: *s.dummyRedshiftClusters[2].ClusterNamespaceArn, + Name: *s.dummyRedshiftClusters[2].ClusterIdentifier, + Type: scan.RepoTypeRedshift, + Tags: []string{}, + Properties: s.dummyRedshiftClusters[2], + }, + { + Id: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[0]].TableArn, + Name: s.dummyDynamoDBTableNames[0], + Type: scan.RepoTypeDynamoDB, + Tags: []string{ + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[0].Key, *s.dummyDynamoDBTags[0].Value, + ), + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[1].Key, *s.dummyDynamoDBTags[1].Value, + ), + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[2].Key, *s.dummyDynamoDBTags[2].Value, + ), + }, + Properties: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[0]], + }, + { + Id: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[1]].TableArn, + Name: s.dummyDynamoDBTableNames[1], + Type: scan.RepoTypeDynamoDB, + Tags: []string{ + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[0].Key, *s.dummyDynamoDBTags[0].Value, + ), + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[1].Key, *s.dummyDynamoDBTags[1].Value, + ), + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[2].Key, *s.dummyDynamoDBTags[2].Value, + ), + }, + Properties: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[1]], + }, + { + Id: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[2]].TableArn, + Name: s.dummyDynamoDBTableNames[2], + Type: scan.RepoTypeDynamoDB, + Tags: []string{ + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[0].Key, *s.dummyDynamoDBTags[0].Value, + ), + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[1].Key, *s.dummyDynamoDBTags[1].Value, + ), + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[2].Key, *s.dummyDynamoDBTags[2].Value, + ), + }, + Properties: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[2]], + }, + }, + } + + require.ElementsMatch( + s.T(), + expectedResults.Repositories, + results.Repositories, + ) + require.NoError(s.T(), err) +} + +func (s *AWSScannerTestSuite) TestScan_WithErrors() { + dummyError := fmt.Errorf("dummy-error") + awsScanner := AWSScanner{ + scannerConfig: ScannerConfig{ + Regions: []string{ + "us-east-1", + "us-east-2", + }, + AssumeRole: &AssumeRoleConfig{ + IAMRoleARN: "arn:aws:iam::123456789012:role/SomeIAMRole", + ExternalID: "some-external-id-12345", + }, + }, + awsConfig: aws.Config{}, + awsClientConstructor: func(awsConfig aws.Config) *awsClient { + return &awsClient{ + config: awsConfig, + rds: &mock.MockRDSClient{ + Errors: map[string]error{ + "DescribeDBClusters": dummyError, + "DescribeDBInstances": dummyError, + }, + }, + redshift: &mock.MockRedshiftClient{ + Errors: map[string]error{ + "DescribeClusters": dummyError, + }, + }, + dynamodb: &mock.MockDynamoDBClient{ + Errors: map[string]error{ + "ListTables": dummyError, + }, + }, + } + }, + } + + ctx := context.Background() + results, err := awsScanner.Scan(ctx) + + expectedResults := &scan.ScanResults{ + Repositories: []scan.Repository{}, + } + + require.ElementsMatch( + s.T(), + expectedResults.Repositories, + results.Repositories, + ) + require.ErrorIs(s.T(), err, dummyError) +} diff --git a/go.mod b/go.mod index e20305b..7c16c46 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,33 @@ module github.com/cyralinc/dmap go 1.21 + +require ( + github.com/aws/aws-sdk-go-v2 v1.24.1 + github.com/aws/aws-sdk-go-v2/config v1.26.6 + github.com/aws/aws-sdk-go-v2/credentials v1.16.16 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.0 + github.com/aws/aws-sdk-go-v2/service/rds v1.69.0 + github.com/aws/aws-sdk-go-v2/service/redshift v1.39.8 + github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 + github.com/stretchr/testify v1.8.4 +) + +require ( + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect + github.com/aws/smithy-go v1.19.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index e69de29..2ed3193 100644 --- a/go.sum +++ b/go.sum @@ -0,0 +1,55 @@ +github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= +github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU6HuGmfPL/o= +github.com/aws/aws-sdk-go-v2/config v1.26.6/go.mod h1:uKU6cnDmYCvJ+pxO9S4cWDb2yWWIH5hra+32hVh1MI4= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.0 h1:e/HPLjLas04wKnmCUSSXD44cYdVjT/Dcd9CkmlYNyNU= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.0/go.mod h1:N5tqZcYMM0N1PN7UQYJNWuGyO886OfnMhf/3MAbqMcI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.11 h1:e9AVb17H4x5FTE5KWIP5M1Du+9M86pS+Hw0lBUdN8EY= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.11/go.mod h1:B90ZQJa36xo0ph9HsoteI1+r8owgQH/U1QNfqZQkj1Q= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= +github.com/aws/aws-sdk-go-v2/service/rds v1.69.0 h1:vnB7v2ZiKOYOXcu1xamRx9OyPJW9daWXUbysKrY3V/A= +github.com/aws/aws-sdk-go-v2/service/rds v1.69.0/go.mod h1:N/ijzTwR4cOG2P8Kvos/QOCetpDTtconhvDOheqnrTw= +github.com/aws/aws-sdk-go-v2/service/redshift v1.39.8 h1:oXLgjBNdlrQkcz8n2m5XJzJLsLmyDadJ1GVJRMh+db4= +github.com/aws/aws-sdk-go-v2/service/redshift v1.39.8/go.mod h1:FjYkfyM8Zq2ddSX2y1hb1rOhEERLzCTidT0VBQOKFss= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scan/hello.go b/scan/hello.go deleted file mode 100644 index 8d249b6..0000000 --- a/scan/hello.go +++ /dev/null @@ -1,5 +0,0 @@ -package scan - -func hello() string { - return "Hello, World!" -} diff --git a/scan/hello_test.go b/scan/hello_test.go deleted file mode 100644 index c5fbd68..0000000 --- a/scan/hello_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package scan - -import ( - "testing" -) - -func Test_hello(t *testing.T) { - tests := []struct { - name string - want string - }{ - { - name: "success", - want: "Hello, World!", - }, - } - for _, tt := range tests { - t.Run( - tt.name, func(t *testing.T) { - if got := hello(); got != tt.want { - t.Errorf("hello() = %v, want %v", got, tt.want) - } - }, - ) - } -} diff --git a/scan/scanner.go b/scan/scanner.go new file mode 100644 index 0000000..25421be --- /dev/null +++ b/scan/scanner.go @@ -0,0 +1,41 @@ +package scan + +import ( + "context" + "time" +) + +// Scanner is an interface that should be implemented for a specific cloud +// provider (e.g. AWS, GCP, etc). It defines the Scan method responsible for +// scanning the existing data repositories of the corresponding cloud provider +// environment. +type Scanner interface { + Scan(ctx context.Context) (*ScanResults, error) +} + +// RepoType defines the AWS data repository types supported (e.g. RDS, Redshift, +// DynamoDB, etc). +type RepoType string + +const ( + // Repo types + RepoTypeRDS RepoType = "REPO_TYPE_RDS" + RepoTypeRedshift RepoType = "REPO_TYPE_REDSHIFT" + RepoTypeDynamoDB RepoType = "REPO_TYPE_DYNAMODB" +) + +// Repository represents a scanned data repository. +type Repository struct { + Id string + Name string + Type RepoType + CreatedAt time.Time + Tags []string + Properties any +} + +// ScanResults represents the results of a repository scan, including all the +// data repositories that were scanned. +type ScanResults struct { + Repositories []Repository +} diff --git a/testutil/mock/aws.go b/testutil/mock/aws.go new file mode 100644 index 0000000..d840a79 --- /dev/null +++ b/testutil/mock/aws.go @@ -0,0 +1,164 @@ +package mock + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + dynamodbTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/aws/aws-sdk-go-v2/service/rds" + rdsTypes "github.com/aws/aws-sdk-go-v2/service/rds/types" + "github.com/aws/aws-sdk-go-v2/service/redshift" + redshiftTypes "github.com/aws/aws-sdk-go-v2/service/redshift/types" +) + +type MockRDSClient struct { + DBClusters []rdsTypes.DBCluster + DBInstances []rdsTypes.DBInstance + Errors map[string]error +} + +func (m *MockRDSClient) DescribeDBClusters( + ctx context.Context, + params *rds.DescribeDBClustersInput, + optFns ...func(*rds.Options), +) (*rds.DescribeDBClustersOutput, error) { + if m.Errors["DescribeDBClusters"] != nil { + return nil, m.Errors["DescribeDBClusters"] + } + if params.Marker == nil { + return &rds.DescribeDBClustersOutput{ + DBClusters: []rdsTypes.DBCluster{ + m.DBClusters[0], + m.DBClusters[1], + }, + Marker: aws.String("2"), + }, nil + } + return &rds.DescribeDBClustersOutput{ + DBClusters: []rdsTypes.DBCluster{ + m.DBClusters[2], + }, + }, nil +} + +func (m *MockRDSClient) DescribeDBInstances( + ctx context.Context, + params *rds.DescribeDBInstancesInput, + optFns ...func(*rds.Options), +) (*rds.DescribeDBInstancesOutput, error) { + if m.Errors["DescribeDBInstances"] != nil { + return nil, m.Errors["DescribeDBInstances"] + } + if params.Marker == nil { + return &rds.DescribeDBInstancesOutput{ + DBInstances: []rdsTypes.DBInstance{ + m.DBInstances[0], + m.DBInstances[1], + }, + Marker: aws.String("2"), + }, nil + } + return &rds.DescribeDBInstancesOutput{ + DBInstances: []rdsTypes.DBInstance{ + m.DBInstances[2], + }, + }, nil +} + +type MockRedshiftClient struct { + Clusters []redshiftTypes.Cluster + Errors map[string]error +} + +func (m *MockRedshiftClient) DescribeClusters( + ctx context.Context, + params *redshift.DescribeClustersInput, + optFns ...func(*redshift.Options), +) (*redshift.DescribeClustersOutput, error) { + if m.Errors["DescribeClusters"] != nil { + return nil, m.Errors["DescribeClusters"] + } + if params.Marker == nil { + return &redshift.DescribeClustersOutput{ + Clusters: []redshiftTypes.Cluster{ + m.Clusters[0], + m.Clusters[1], + }, + Marker: aws.String("2"), + }, nil + } + return &redshift.DescribeClustersOutput{ + Clusters: []redshiftTypes.Cluster{ + m.Clusters[2], + }, + }, nil +} + +type MockDynamoDBClient struct { + TableNames []string + Table map[string]*dynamodbTypes.TableDescription + Tags []dynamodbTypes.Tag + Errors map[string]error +} + +func (m *MockDynamoDBClient) ListTables( + ctx context.Context, + params *dynamodb.ListTablesInput, + optFns ...func(*dynamodb.Options), +) (*dynamodb.ListTablesOutput, error) { + if m.Errors["ListTables"] != nil { + return nil, m.Errors["ListTables"] + } + if params.ExclusiveStartTableName == nil { + return &dynamodb.ListTablesOutput{ + TableNames: []string{ + m.TableNames[0], + m.TableNames[1], + }, + LastEvaluatedTableName: aws.String(m.TableNames[1]), + }, nil + } + return &dynamodb.ListTablesOutput{ + TableNames: []string{ + m.TableNames[2], + }, + }, nil +} + +func (m *MockDynamoDBClient) DescribeTable( + ctx context.Context, + params *dynamodb.DescribeTableInput, + optFns ...func(*dynamodb.Options), +) (*dynamodb.DescribeTableOutput, error) { + if m.Errors["DescribeTable"] != nil { + return nil, m.Errors["DescribeTable"] + } + return &dynamodb.DescribeTableOutput{ + Table: m.Table[*params.TableName], + }, nil +} + +func (m *MockDynamoDBClient) ListTagsOfResource( + ctx context.Context, + params *dynamodb.ListTagsOfResourceInput, + optFns ...func(*dynamodb.Options), +) (*dynamodb.ListTagsOfResourceOutput, error) { + if m.Errors["ListTagsOfResource"] != nil { + return nil, m.Errors["ListTagsOfResource"] + } + if params.NextToken == nil { + return &dynamodb.ListTagsOfResourceOutput{ + Tags: []dynamodbTypes.Tag{ + m.Tags[0], + m.Tags[1], + }, + NextToken: aws.String("2"), + }, nil + } + return &dynamodb.ListTagsOfResourceOutput{ + Tags: []dynamodbTypes.Tag{ + m.Tags[2], + }, + }, nil +} diff --git a/testutil/mock/scanner.go b/testutil/mock/scanner.go new file mode 100644 index 0000000..bd62bbd --- /dev/null +++ b/testutil/mock/scanner.go @@ -0,0 +1,18 @@ +package mock + +import ( + "context" + + "github.com/cyralinc/dmap/scan" +) + +type MockScanner struct { + Repositories []scan.Repository + Err error +} + +func (m *MockScanner) Scan( + ctx context.Context, +) ([]scan.Repository, error) { + return m.Repositories, m.Err +}