diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a36faac --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +all: tidy test + +tidy: + go mod tidy + +# Using --count=1 disables test caching +test: + go test -v -race ./... --count=1 + +integration-test: + go test -v -race ./... --count=1 --tags=integration + +clean: + go clean -i ./... diff --git a/aws/aws_test.go b/aws/aws_test.go new file mode 100644 index 0000000..be226cc --- /dev/null +++ b/aws/aws_test.go @@ -0,0 +1,290 @@ +package aws + +import ( + "context" + "fmt" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + + // ddbTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + rdsTypes "github.com/aws/aws-sdk-go-v2/service/rds/types" + redshiftTypes "github.com/aws/aws-sdk-go-v2/service/redshift/types" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/cyralinc/dmap/config" + "github.com/cyralinc/dmap/model" + "github.com/cyralinc/dmap/testutil/mock" +) + +type AWSScannerTestSuite struct { + suite.Suite + dummyRDSClusters []rdsTypes.DBCluster + dummyRDSInstances []rdsTypes.DBInstance + dummyRedshiftClusters []redshiftTypes.Cluster + dummyDynamoDBTableNames []string + dummyDynamoDBTable map[string]*types.TableDescription + dummyDynamoDBTags []types.Tag +} + +func (s *AWSScannerTestSuite) SetupSuite() { + s.dummyRDSClusters = []rdsTypes.DBCluster{ + { + DBClusterIdentifier: aws.String("rds-cluster-1"), + }, + { + DBClusterIdentifier: aws.String("rds-cluster-2"), + }, + { + DBClusterIdentifier: aws.String("rds-cluster-3"), + }, + } + s.dummyRDSInstances = []rdsTypes.DBInstance{ + { + DBInstanceIdentifier: aws.String("rds-instance-1"), + }, + { + DBInstanceIdentifier: aws.String("rds-instance-2"), + }, + { + DBInstanceIdentifier: aws.String("rds-instance-3"), + }, + } + s.dummyRedshiftClusters = []redshiftTypes.Cluster{ + { + ClusterIdentifier: aws.String("redshift-cluster-1"), + }, + { + ClusterIdentifier: aws.String("redshift-cluster-2"), + }, + { + ClusterIdentifier: aws.String("redshift-cluster-3"), + }, + } + s.dummyDynamoDBTableNames = []string{ + "dynamodb-table-1", + "dynamodb-table-2", + "dynamodb-table-3", + } + s.dummyDynamoDBTable = map[string]*types.TableDescription{ + s.dummyDynamoDBTableNames[0]: &types.TableDescription{ + TableName: aws.String(s.dummyDynamoDBTableNames[0]), + }, + s.dummyDynamoDBTableNames[1]: &types.TableDescription{ + TableName: aws.String(s.dummyDynamoDBTableNames[1]), + }, + s.dummyDynamoDBTableNames[2]: &types.TableDescription{ + TableName: aws.String(s.dummyDynamoDBTableNames[2]), + }, + } + s.dummyDynamoDBTags = []types.Tag{ + { + Key: aws.String("tag1"), + Value: aws.String("value1"), + }, + { + Key: aws.String("tag2"), + Value: aws.String("value2"), + }, + { + Key: aws.String("tag3"), + Value: aws.String("value3"), + }, + } +} + +func TestAWSScanner(t *testing.T) { + s := new(AWSScannerTestSuite) + suite.Run(t, s) +} + +func (s *AWSScannerTestSuite) TestScan() { + region := "us-east-1" + awsScanner := AWSScanner{ + scanConfig: config.AWSConfig{ + Regions: []string{region}, + AssumeRole: &config.AWSAssumeRoleConfig{ + IAMRoleARN: "arn:aws:iam::123456789012:role/SomeIAMRole", + ExternalID: "some-external-id-12345", + }, + }, + awsConfig: aws.Config{ + Region: region, + }, + rdsClient: &mock.MockRDSClient{ + DBClusters: s.dummyRDSClusters, + DBInstances: s.dummyRDSInstances, + }, + redshiftClient: &mock.MockRedshiftClient{ + Clusters: s.dummyRedshiftClusters, + }, + dynamodbClient: &mock.MockDynamoDBClient{ + TableNames: s.dummyDynamoDBTableNames, + Table: s.dummyDynamoDBTable, + Tags: s.dummyDynamoDBTags, + }, + } + ctx := context.Background() + repositories, err := awsScanner.Scan(ctx) + + expectedRepositories := []model.Repository{ + { + Name: *s.dummyRDSClusters[0].DBClusterIdentifier, + Type: model.RepoTypeRDS, + Tags: []string{}, + Properties: s.dummyRDSClusters[0], + }, + { + Name: *s.dummyRDSClusters[1].DBClusterIdentifier, + Type: model.RepoTypeRDS, + Tags: []string{}, + Properties: s.dummyRDSClusters[1], + }, + { + Name: *s.dummyRDSClusters[2].DBClusterIdentifier, + Type: model.RepoTypeRDS, + Tags: []string{}, + Properties: s.dummyRDSClusters[2], + }, + { + Name: *s.dummyRDSInstances[0].DBInstanceIdentifier, + Type: model.RepoTypeRDS, + Tags: []string{}, + Properties: s.dummyRDSInstances[0], + }, + { + Name: *s.dummyRDSInstances[1].DBInstanceIdentifier, + Type: model.RepoTypeRDS, + Tags: []string{}, + Properties: s.dummyRDSInstances[1], + }, + { + Name: *s.dummyRDSInstances[2].DBInstanceIdentifier, + Type: model.RepoTypeRDS, + Tags: []string{}, + Properties: s.dummyRDSInstances[2], + }, + { + Name: *s.dummyRedshiftClusters[0].ClusterIdentifier, + Type: model.RepoTypeRedshift, + Tags: []string{}, + Properties: s.dummyRedshiftClusters[0], + }, + { + Name: *s.dummyRedshiftClusters[1].ClusterIdentifier, + Type: model.RepoTypeRedshift, + Tags: []string{}, + Properties: s.dummyRedshiftClusters[1], + }, + { + Name: *s.dummyRedshiftClusters[2].ClusterIdentifier, + Type: model.RepoTypeRedshift, + Tags: []string{}, + Properties: s.dummyRedshiftClusters[2], + }, + { + Name: s.dummyDynamoDBTableNames[0], + Type: model.RepoTypeDynamoDB, + Tags: []string{ + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[0].Key, *s.dummyDynamoDBTags[0].Value, + ), + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[1].Key, *s.dummyDynamoDBTags[1].Value, + ), + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[2].Key, *s.dummyDynamoDBTags[2].Value, + ), + }, + Properties: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[0]], + }, + { + Name: s.dummyDynamoDBTableNames[1], + Type: model.RepoTypeDynamoDB, + Tags: []string{ + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[0].Key, *s.dummyDynamoDBTags[0].Value, + ), + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[1].Key, *s.dummyDynamoDBTags[1].Value, + ), + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[2].Key, *s.dummyDynamoDBTags[2].Value, + ), + }, + Properties: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[1]], + }, + { + Name: s.dummyDynamoDBTableNames[2], + Type: model.RepoTypeDynamoDB, + Tags: []string{ + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[0].Key, *s.dummyDynamoDBTags[0].Value, + ), + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[1].Key, *s.dummyDynamoDBTags[1].Value, + ), + fmt.Sprintf( + "%s:%s", + *s.dummyDynamoDBTags[2].Key, *s.dummyDynamoDBTags[2].Value, + ), + }, + Properties: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[2]], + }, + } + + require.ElementsMatch(s.T(), expectedRepositories, repositories) + require.NoError(s.T(), err) +} + +func (s *AWSScannerTestSuite) TestScan_WithErrors() { + region := "us-east-1" + dummyError := fmt.Errorf("dummy-error") + awsScanner := AWSScanner{ + scanConfig: config.AWSConfig{ + Regions: []string{region}, + AssumeRole: &config.AWSAssumeRoleConfig{ + IAMRoleARN: "arn:aws:iam::123456789012:role/SomeIAMRole", + ExternalID: "some-external-id-12345", + }, + }, + awsConfig: aws.Config{ + Region: region, + }, + rdsClient: &mock.MockRDSClient{ + Errors: map[string]error{ + "DescribeDBClusters": dummyError, + "DescribeDBInstances": dummyError, + }, + }, + redshiftClient: &mock.MockRedshiftClient{ + Errors: map[string]error{ + "DescribeClusters": dummyError, + }, + }, + dynamodbClient: &mock.MockDynamoDBClient{ + Errors: map[string]error{ + "ListTables": dummyError, + }, + }, + } + ctx := context.Background() + repositories, err := awsScanner.Scan(ctx) + + expectedRepositories := []model.Repository{} + expectedErrorSubstring := dummyError.Error() + + require.ElementsMatch(s.T(), expectedRepositories, repositories) + require.ErrorContains(s.T(), err, expectedErrorSubstring) +} diff --git a/scan/scan_integration_test.go b/scan/scan_integration_test.go new file mode 100644 index 0000000..4653676 --- /dev/null +++ b/scan/scan_integration_test.go @@ -0,0 +1,47 @@ +//go:build integration + +package scan + +import ( + "context" + "fmt" + "testing" + + "github.com/cyralinc/dmap/config" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type ScanManagerIntegrationTestSuite struct { + suite.Suite + scanner *ScanManager +} + +func (s *ScanManagerIntegrationTestSuite) SetupSuite() { + ctx := context.Background() + scanner, err := NewScanManager(ctx, config.Config{ + AWS: &config.AWSConfig{ + Regions: []string{ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + }, + }, + }) + require.NoError(s.T(), err) + s.scanner = scanner +} + +func TestIntegrationScanManager(t *testing.T) { + s := new(ScanManagerIntegrationTestSuite) + suite.Run(t, s) +} + +func (s *ScanManagerIntegrationTestSuite) TestScanRepositories() { + ctx := context.Background() + results, scanErrors := s.scanner.ScanRepositories(ctx) + fmt.Printf("Num. Repositories: %v\n", len(results.Repositories)) + fmt.Printf("Repositories: %v\n", results.Repositories) + fmt.Printf("Scan Erros: %v\n", scanErrors) +} diff --git a/scan/scan_test.go b/scan/scan_test.go new file mode 100644 index 0000000..43dee57 --- /dev/null +++ b/scan/scan_test.go @@ -0,0 +1,122 @@ +package scan + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + ddbTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + rdsTypes "github.com/aws/aws-sdk-go-v2/service/rds/types" + redshiftTypes "github.com/aws/aws-sdk-go-v2/service/redshift/types" + + "github.com/cyralinc/dmap/config" + "github.com/cyralinc/dmap/model" + "github.com/cyralinc/dmap/testutil/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type ScanManagerTestSuite struct { + suite.Suite + dummyRepos []model.Repository +} + +func (s *ScanManagerTestSuite) SetupSuite() { + s.dummyRepos = []model.Repository{ + { + Id: "1", + Name: "some-rds-instance", + Type: model.RepoTypeRDS, + CreatedAt: time.Now(), + Tags: []string{"tag1", "tag2"}, + Properties: rdsTypes.DBInstance{ + DBInstanceIdentifier: aws.String("rds-instance-1"), + }, + }, + { + Id: "2", + Name: "some-redshift-cluster", + Type: model.RepoTypeRedshift, + CreatedAt: time.Now(), + Tags: []string{"tag1"}, + Properties: redshiftTypes.Cluster{ + ClusterIdentifier: aws.String("redshift-cluster-1"), + }, + }, + { + Id: "3", + Name: "some-rds-cluster", + Type: model.RepoTypeRDS, + CreatedAt: time.Now(), + Tags: []string{}, + Properties: rdsTypes.DBCluster{ + DBClusterIdentifier: aws.String("rds-cluster-1"), + }, + }, + { + Id: "4", + Name: "some-dynamodb-table", + Type: model.RepoTypeDynamoDB, + CreatedAt: time.Now(), + Tags: nil, + Properties: ddbTypes.TableDescription{ + TableName: aws.String("dynamodb-table-1"), + }, + }, + } +} + +func TestScanManager(t *testing.T) { + s := new(ScanManagerTestSuite) + suite.Run(t, s) +} + +func (s *ScanManagerTestSuite) TestScanRepositories() { + err1 := fmt.Errorf("Error during scanner 1") + err2 := fmt.Errorf("Error during scanner 2") + manager := ScanManager{ + config: config.Config{ + AWS: &config.AWSConfig{ + Regions: []string{ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + }, + AssumeRole: &config.AWSAssumeRoleConfig{ + IAMRoleARN: "arn:aws:iam::123456789012:role/SomeIAMRole", + ExternalID: "some-external-id-12345", + }, + }, + }, + scanners: []Scanner{ + &mock.MockScanner{ + Repositories: []model.Repository{ + s.dummyRepos[0], + s.dummyRepos[1], + }, + Err: err1, + }, + &mock.MockScanner{ + Repositories: []model.Repository{ + s.dummyRepos[2], + s.dummyRepos[3], + }, + Err: err2, + }, + }, + } + ctx := context.Background() + scanResults, err := manager.ScanRepositories(ctx) + + expectedscanResults := &ScanResults{ + Repositories: s.dummyRepos, + } + expectedError := errors.Join(err1, err2) + + require.Equal(s.T(), expectedscanResults, scanResults) + require.EqualError(s.T(), err, expectedError.Error()) +} diff --git a/testutil/mock/aws.go b/testutil/mock/aws.go new file mode 100644 index 0000000..d840a79 --- /dev/null +++ b/testutil/mock/aws.go @@ -0,0 +1,164 @@ +package mock + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + dynamodbTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/aws/aws-sdk-go-v2/service/rds" + rdsTypes "github.com/aws/aws-sdk-go-v2/service/rds/types" + "github.com/aws/aws-sdk-go-v2/service/redshift" + redshiftTypes "github.com/aws/aws-sdk-go-v2/service/redshift/types" +) + +type MockRDSClient struct { + DBClusters []rdsTypes.DBCluster + DBInstances []rdsTypes.DBInstance + Errors map[string]error +} + +func (m *MockRDSClient) DescribeDBClusters( + ctx context.Context, + params *rds.DescribeDBClustersInput, + optFns ...func(*rds.Options), +) (*rds.DescribeDBClustersOutput, error) { + if m.Errors["DescribeDBClusters"] != nil { + return nil, m.Errors["DescribeDBClusters"] + } + if params.Marker == nil { + return &rds.DescribeDBClustersOutput{ + DBClusters: []rdsTypes.DBCluster{ + m.DBClusters[0], + m.DBClusters[1], + }, + Marker: aws.String("2"), + }, nil + } + return &rds.DescribeDBClustersOutput{ + DBClusters: []rdsTypes.DBCluster{ + m.DBClusters[2], + }, + }, nil +} + +func (m *MockRDSClient) DescribeDBInstances( + ctx context.Context, + params *rds.DescribeDBInstancesInput, + optFns ...func(*rds.Options), +) (*rds.DescribeDBInstancesOutput, error) { + if m.Errors["DescribeDBInstances"] != nil { + return nil, m.Errors["DescribeDBInstances"] + } + if params.Marker == nil { + return &rds.DescribeDBInstancesOutput{ + DBInstances: []rdsTypes.DBInstance{ + m.DBInstances[0], + m.DBInstances[1], + }, + Marker: aws.String("2"), + }, nil + } + return &rds.DescribeDBInstancesOutput{ + DBInstances: []rdsTypes.DBInstance{ + m.DBInstances[2], + }, + }, nil +} + +type MockRedshiftClient struct { + Clusters []redshiftTypes.Cluster + Errors map[string]error +} + +func (m *MockRedshiftClient) DescribeClusters( + ctx context.Context, + params *redshift.DescribeClustersInput, + optFns ...func(*redshift.Options), +) (*redshift.DescribeClustersOutput, error) { + if m.Errors["DescribeClusters"] != nil { + return nil, m.Errors["DescribeClusters"] + } + if params.Marker == nil { + return &redshift.DescribeClustersOutput{ + Clusters: []redshiftTypes.Cluster{ + m.Clusters[0], + m.Clusters[1], + }, + Marker: aws.String("2"), + }, nil + } + return &redshift.DescribeClustersOutput{ + Clusters: []redshiftTypes.Cluster{ + m.Clusters[2], + }, + }, nil +} + +type MockDynamoDBClient struct { + TableNames []string + Table map[string]*dynamodbTypes.TableDescription + Tags []dynamodbTypes.Tag + Errors map[string]error +} + +func (m *MockDynamoDBClient) ListTables( + ctx context.Context, + params *dynamodb.ListTablesInput, + optFns ...func(*dynamodb.Options), +) (*dynamodb.ListTablesOutput, error) { + if m.Errors["ListTables"] != nil { + return nil, m.Errors["ListTables"] + } + if params.ExclusiveStartTableName == nil { + return &dynamodb.ListTablesOutput{ + TableNames: []string{ + m.TableNames[0], + m.TableNames[1], + }, + LastEvaluatedTableName: aws.String(m.TableNames[1]), + }, nil + } + return &dynamodb.ListTablesOutput{ + TableNames: []string{ + m.TableNames[2], + }, + }, nil +} + +func (m *MockDynamoDBClient) DescribeTable( + ctx context.Context, + params *dynamodb.DescribeTableInput, + optFns ...func(*dynamodb.Options), +) (*dynamodb.DescribeTableOutput, error) { + if m.Errors["DescribeTable"] != nil { + return nil, m.Errors["DescribeTable"] + } + return &dynamodb.DescribeTableOutput{ + Table: m.Table[*params.TableName], + }, nil +} + +func (m *MockDynamoDBClient) ListTagsOfResource( + ctx context.Context, + params *dynamodb.ListTagsOfResourceInput, + optFns ...func(*dynamodb.Options), +) (*dynamodb.ListTagsOfResourceOutput, error) { + if m.Errors["ListTagsOfResource"] != nil { + return nil, m.Errors["ListTagsOfResource"] + } + if params.NextToken == nil { + return &dynamodb.ListTagsOfResourceOutput{ + Tags: []dynamodbTypes.Tag{ + m.Tags[0], + m.Tags[1], + }, + NextToken: aws.String("2"), + }, nil + } + return &dynamodb.ListTagsOfResourceOutput{ + Tags: []dynamodbTypes.Tag{ + m.Tags[2], + }, + }, nil +} diff --git a/testutil/mock/scanner.go b/testutil/mock/scanner.go new file mode 100644 index 0000000..62a188b --- /dev/null +++ b/testutil/mock/scanner.go @@ -0,0 +1,18 @@ +package mock + +import ( + "context" + + "github.com/cyralinc/dmap/model" +) + +type MockScanner struct { + Repositories []model.Repository + Err error +} + +func (m *MockScanner) Scan( + ctx context.Context, +) ([]model.Repository, error) { + return m.Repositories, m.Err +}