ENG-13394: Add Dmap Scanner library (#1)

* Add base structure

* Remove hello files

* Update awsClient AssumeRole

* Add config validation

* Update scanAWSRepositories to use go routines

* Update Repository structure

* Add appendError to Scanner

* Handle pagination for AWS requests

* Implement repository converters

* Create aws and model packages

* Add Scanner interface

* Remove unnecessary IsAWSConfigured and rename alias for aws types

* Refactor go routines in AWS Scan

* Update cluster instance check in scanRDSInstanceRepositories

* Refactor getDynamoDBTables

* Update error wrapping

* Add interfaces for AWS service clients

* Add tests

* Update RepoType values

* Remove ScanManager

* Update AWSScanner to scan regions concurrently

* Fix imports

* Add description to exported types and functions

* Fix newRepositoryFromDynamoDBTable to use Table ARN as Id

* Move scan type definitions to scanner.go file

* Add IAM Role validation

* Update scanErrors

* Refactor go routines

* Extract tag formatting into function

* Add lint

Makefile

@@ -0,0 +1,17 @@
+all: tidy lint test
+
+tidy:
+	go mod tidy
+
+lint:
+	golangci-lint run
+
+# Using --count=1 disables test caching
+test:
+	go test -v -race ./... --count=1
+
+integration-test:
+	go test -v -race ./... --count=1 --tags=integration
+
+clean:
+	go clean -i ./...

aws/client.go

@@ -0,0 +1,231 @@
+package aws
+
+import (
+	"context"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
+	ddbTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
+	"github.com/aws/aws-sdk-go-v2/service/rds"
+	rdsTypes "github.com/aws/aws-sdk-go-v2/service/rds/types"
+	"github.com/aws/aws-sdk-go-v2/service/redshift"
+	rsTypes "github.com/aws/aws-sdk-go-v2/service/redshift/types"
+)
+
+type rdsClient interface {
+	DescribeDBClusters(
+		ctx context.Context,
+		params *rds.DescribeDBClustersInput,
+		optFns ...func(*rds.Options),
+	) (*rds.DescribeDBClustersOutput, error)
+	DescribeDBInstances(
+		ctx context.Context,
+		params *rds.DescribeDBInstancesInput,
+		optFns ...func(*rds.Options),
+	) (*rds.DescribeDBInstancesOutput, error)
+}
+
+type redshiftClient interface {
+	DescribeClusters(
+		ctx context.Context,
+		params *redshift.DescribeClustersInput,
+		optFns ...func(*redshift.Options),
+	) (*redshift.DescribeClustersOutput, error)
+}
+
+type dynamoDBClient interface {
+	ListTables(
+		ctx context.Context,
+		params *dynamodb.ListTablesInput,
+		optFns ...func(*dynamodb.Options),
+	) (*dynamodb.ListTablesOutput, error)
+	DescribeTable(
+		ctx context.Context,
+		params *dynamodb.DescribeTableInput,
+		optFns ...func(*dynamodb.Options),
+	) (*dynamodb.DescribeTableOutput, error)
+	ListTagsOfResource(
+		ctx context.Context,
+		params *dynamodb.ListTagsOfResourceInput,
+		optFns ...func(*dynamodb.Options),
+	) (*dynamodb.ListTagsOfResourceOutput, error)
+}
+
+type awsClient struct {
+	config   aws.Config
+	rds      rdsClient
+	redshift redshiftClient
+	dynamodb dynamoDBClient
+}
+
+type awsClientConstructor func(awsConfig aws.Config) *awsClient
+
+func newAWSClient(awsConfig aws.Config) *awsClient {
+	return &awsClient{
+		config:   awsConfig,
+		rds:      rds.NewFromConfig(awsConfig),
+		redshift: redshift.NewFromConfig(awsConfig),
+		dynamodb: dynamodb.NewFromConfig(awsConfig),
+	}
+}
+
+func (c *awsClient) getRDSClusters(
+	ctx context.Context,
+) ([]rdsTypes.DBCluster, error) {
+	var clusters []rdsTypes.DBCluster
+	// Used for pagination
+	var marker *string
+	for {
+		output, err := c.rds.DescribeDBClusters(
+			ctx,
+			&rds.DescribeDBClustersInput{
+				Marker: marker,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		clusters = append(clusters, output.DBClusters...)
+
+		if output.Marker == nil {
+			break
+		} else {
+			marker = output.Marker
+		}
+	}
+	return clusters, nil
+}
+
+func (c *awsClient) getRDSInstances(
+	ctx context.Context,
+) ([]rdsTypes.DBInstance, error) {
+	var instances []rdsTypes.DBInstance
+	// Used for pagination
+	var marker *string
+	for {
+		output, err := c.rds.DescribeDBInstances(
+			ctx,
+			&rds.DescribeDBInstancesInput{
+				Marker: marker,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		instances = append(instances, output.DBInstances...)
+
+		if output.Marker == nil {
+			break
+		} else {
+			marker = output.Marker
+		}
+	}
+	return instances, nil
+}
+
+func (c *awsClient) getRedshiftClusters(
+	ctx context.Context,
+) ([]rsTypes.Cluster, error) {
+	var clusters []rsTypes.Cluster
+	// Used for pagination
+	var marker *string
+	for {
+		output, err := c.redshift.DescribeClusters(
+			ctx,
+			&redshift.DescribeClustersInput{
+				Marker: marker,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		clusters = append(clusters, output.Clusters...)
+
+		if output.Marker == nil {
+			break
+		} else {
+			marker = output.Marker
+		}
+	}
+	return clusters, nil
+}
+
+type dynamoDBTable struct {
+	Table ddbTypes.TableDescription
+	Tags  []ddbTypes.Tag
+}
+
+func (c *awsClient) getDynamoDBTables(
+	ctx context.Context,
+) ([]dynamoDBTable, error) {
+	var tableNames []string
+	// Used for pagination
+	var exclusiveStartTableName *string
+	for {
+		output, err := c.dynamodb.ListTables(
+			ctx,
+			&dynamodb.ListTablesInput{
+				ExclusiveStartTableName: exclusiveStartTableName,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		tableNames = append(tableNames, output.TableNames...)
+
+		if output.LastEvaluatedTableName == nil {
+			break
+		} else {
+			exclusiveStartTableName = output.LastEvaluatedTableName
+		}
+	}
+
+	tables := make([]dynamoDBTable, 0, len(tableNames))
+	for i := range tableNames {
+		tableName := tableNames[i]
+		describeTableOutput, err := c.dynamodb.DescribeTable(
+			ctx,
+			&dynamodb.DescribeTableInput{
+				TableName: &tableName,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
+		table := describeTableOutput.Table
+
+		var tableTags []ddbTypes.Tag
+		// Used for pagination
+		var nextToken *string
+		for {
+			tagsOutput, err := c.dynamodb.ListTagsOfResource(
+				ctx,
+				&dynamodb.ListTagsOfResourceInput{
+					ResourceArn: table.TableArn,
+					NextToken:   nextToken,
+				},
+			)
+			if err != nil {
+				return nil, err
+			}
+
+			tableTags = append(tableTags, tagsOutput.Tags...)
+
+			if tagsOutput.NextToken == nil {
+				break
+			} else {
+				nextToken = tagsOutput.NextToken
+			}
+		}
+
+		tables = append(tables, dynamoDBTable{
+			Table: *table,
+			Tags:  tableTags,
+		})
+	}
+	return tables, nil
+}

aws/config.go

@@ -0,0 +1,55 @@
+package aws
+
+import (
+	"fmt"
+	"regexp"
+)
+
+// ScannerConfig represents an AWSScanner configuration. It allows defining the
+// AWS regions that should be scanned and an optional AssumeRoleConfig that
+// contains the configuration for assuming an IAM Role during the scan. If
+// AssumeRoleConfig is nil, the AWS default external configuration will be used
+// instead.
+type ScannerConfig struct {
+	Regions    []string
+	AssumeRole *AssumeRoleConfig
+}
+
+// AssumeRoleConfig represents the information of an IAM Role to be assumed by
+// the AWSScanner when performing request to the AWS services during the data
+// repositories scan.
+type AssumeRoleConfig struct {
+	// The ARN of the IAM Role to be assumed.
+	IAMRoleARN string
+	// Optional External ID to be used as part of the assume role process.
+	ExternalID string
+}
+
+// Validate validates the ScannerConfig configuration.
+func (config *ScannerConfig) Validate() error {
+	if len(config.Regions) == 0 {
+		return fmt.Errorf("AWS regions are required")
+	}
+	for _, region := range config.Regions {
+		if region == "" {
+			return fmt.Errorf("AWS region can't be empty")
+		}
+	}
+	if config.AssumeRole != nil {
+		iamRolePatern := "^arn:aws:iam::\\d{12}:role/.*$"
+		match, err := regexp.MatchString(
+			iamRolePatern,
+			config.AssumeRole.IAMRoleARN,
+		)
+		if err != nil {
+			return fmt.Errorf("error verifying IAM Role format: %w", err)
+		}
+		if !match {
+			return fmt.Errorf(
+				"invalid IAM Role: must match format '%s'",
+				iamRolePatern,
+			)
+		}
+	}
+	return nil
+}

aws/config_test.go

@@ -0,0 +1,70 @@
+package aws
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+)
+
+type ScannerConfigTestSuite struct {
+	suite.Suite
+}
+
+func TestScannerConfig(t *testing.T) {
+	s := new(ScannerConfigTestSuite)
+	suite.Run(t, s)
+}
+
+func (s *ScannerConfigTestSuite) TestValidate() {
+	type TestCase struct {
+		description      string
+		config           ScannerConfig
+		expectedErrorMsg string
+	}
+	tests := []TestCase{
+		{
+			description:      "No regions should return error",
+			config:           ScannerConfig{},
+			expectedErrorMsg: "AWS regions are required",
+		},
+		{
+			description: "Empty region should return error",
+			config: ScannerConfig{
+				Regions: []string{""},
+			},
+			expectedErrorMsg: "AWS region can't be empty",
+		},
+		{
+			description: "Invalid IAM Role format should return error",
+			config: ScannerConfig{
+				Regions: []string{"us-east-1"},
+				AssumeRole: &AssumeRoleConfig{
+					IAMRoleARN: "invalid-iam-role-format",
+				},
+			},
+			expectedErrorMsg: "invalid IAM Role: must match format " +
+				"'^arn:aws:iam::\\d{12}:role/.*$'",
+		},
+		{
+			description: "Valid config should return nil error",
+			config: ScannerConfig{
+				Regions: []string{"us-east-1"},
+				AssumeRole: &AssumeRoleConfig{
+					IAMRoleARN: "arn:aws:iam::123456789012:role/SomeIAMRole",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		s.T().Run(test.description, func(t *testing.T) {
+			err := test.config.Validate()
+			if test.expectedErrorMsg == "" {
+				require.NoError(t, err)
+			} else {
+				require.ErrorContains(t, err, test.expectedErrorMsg)
+			}
+		})
+	}
+}