Increase default AWS max retry attempts and make it configurable

Signed-off-by: Matt Welke <matt.welke@spectrocloud.com>

Author: mattwelke

README.md

@@ -40,6 +40,10 @@ Authentication details for the AWS validator controller are provided within each
 > [!NOTE]
 > See [values.yaml](https://github.com/validator-labs/validator-plugin-aws/tree/main/chart/validator-plugin-aws/values.yaml) for additional configuration details for each authentication option.
 
+# Configuring AWS SDK
+
+* The max attempts setting for retries is set to 25. Use `auth.MaxAttempts` to increase this further or set any other desired value. Valid values are those gte 0.
+
 ### Minimal AWS managed IAM permission policies by validation type
 For validation to succeed, certain AWS managed permission policies must be attached to the principal used and/or assumed by the AWS validator controller. The minimal required IAM policies, broken out by validation category, are as follows:
 * IAM

api/v1alpha1/awsvalidator_types.go

@@ -69,7 +69,7 @@ func (s AwsValidatorSpec) ResultCount() int {
 		len(s.IamUserRules) + len(s.ServiceQuotaRules) + len(s.TagRules)
 }
 
-// AwsAuth defines authentication configuration for an AwsValidator.
+// AwsAuth defines authentication and AWS SDK configuration for an AwsValidator.
 type AwsAuth struct {
 	// If true, the AwsValidator will use the AWS SDK's default credential chain to authenticate.
 	// Set to true if using node instance IAM role or IAM roles for Service Accounts.
@@ -82,6 +82,11 @@ type AwsAuth struct {
 	Credentials *Credentials `json:"credentials,omitempty" yaml:"credentials,omitempty"`
 	// STS authentication properties (optional)
 	StsAuth *AwsSTSAuth `json:"stsAuth,omitempty" yaml:"stsAuth,omitempty"`
+	// MaxAttempts is the number of times the AWS SDK should retry retryable operations. If
+	// specified, overrides the setting the plugin uses by default for this AwsValidator. Set to 0
+	// to disable retrying.
+	// +kubebuilder:validation:Minimum=0
+	MaxAttempts *int `json:"maxAttempts,omitempty" yaml:"maxAttempts,omitempty"`
 }
 
 // Credentials is the credentials to use when running in direct mode.

api/v1alpha1/zz_generated.deepcopy.go

@@ -87,7 +87,8 @@ spec:
                 - message: AmiRules must have unique names
                   rule: self.all(e, size(self.filter(x, x.name == e.name)) == 1)
               auth:
-                description: AwsAuth defines authentication configuration for an AwsValidator.
+                description: AwsAuth defines authentication and AWS SDK configuration
+                  for an AwsValidator.
                 properties:
                   credentials:
                     description: The credentials to use when running in direct mode.
@@ -107,6 +108,13 @@ spec:
                       If true, the AwsValidator will use the AWS SDK's default credential chain to authenticate.
                       Set to true if using node instance IAM role or IAM roles for Service Accounts.
                     type: boolean
+                  maxAttempts:
+                    description: |-
+                      MaxAttempts is the number of times the AWS SDK should retry retryable operations. If
+                      specified, overrides the setting the plugin uses by default for this AwsValidator. Set to 0
+                      to disable retrying.
+                    minimum: 0
+                    type: integer
                   secretName:
                     description: |-
                       Name of a Secret in the same namespace as the AwsValidator that contains AWS credentials.

build

@@ -87,7 +87,8 @@ spec:
                 - message: AmiRules must have unique names
                   rule: self.all(e, size(self.filter(x, x.name == e.name)) == 1)
               auth:
-                description: AwsAuth defines authentication configuration for an AwsValidator.
+                description: AwsAuth defines authentication and AWS SDK configuration
+                  for an AwsValidator.
                 properties:
                   credentials:
                     description: The credentials to use when running in direct mode.
@@ -107,6 +108,13 @@ spec:
                       If true, the AwsValidator will use the AWS SDK's default credential chain to authenticate.
                       Set to true if using node instance IAM role or IAM roles for Service Accounts.
                     type: boolean
+                  maxAttempts:
+                    description: |-
+                      MaxAttempts is the number of times the AWS SDK should retry retryable operations. If
+                      specified, overrides the setting the plugin uses by default for this AwsValidator. Set to 0
+                      to disable retrying.
+                    minimum: 0
+                    type: integer
                   secretName:
                     description: |-
                       Name of a Secret in the same namespace as the AwsValidator that contains AWS credentials.

chart/validator-plugin-aws/crds/validation.spectrocloud.labs_awsvalidators.yaml

@@ -28,4 +28,8 @@ const (
 
 	// IAMWildcard is the wildcard used in IAM policies.
 	IAMWildcard string = "*"
+
+	// RetryMaxAttemptsDefault is the default we use for the max retries setting of the AWS SDK.
+	// It's greater than the SDK's default.
+	RetryMaxAttemptsDefault int = 25
 )

config/crd/bases/validation.spectrocloud.labs_awsvalidators.yaml

@@ -22,9 +22,6 @@ import (
 	"github.com/validator-labs/validator-plugin-aws/pkg/validators/tag"
 )
 
-var errInvalidAccessKeyID = errors.New("access key ID is invalid, must be a non-empty string")
-var errInvalidSecretAccessKey = errors.New("secret access key is invalid, must be a non-empty string")
-
 // Validate validates the AwsValidatorSpec and returns a ValidationResponse.
 func Validate(spec v1alpha1.AwsValidatorSpec, log logr.Logger) types.ValidationResponse {
 	resp := types.ValidationResponse{
@@ -155,10 +152,14 @@ func validateAuth(auth v1alpha1.AwsAuth) error {
 	var validationErrors []error
 
 	if auth.Credentials.AccessKeyID == "" {
-		validationErrors = append(validationErrors, errInvalidAccessKeyID)
+		validationErrors = append(validationErrors, errors.New("access key ID is invalid, must be a non-empty string"))
 	}
 	if auth.Credentials.SecretAccessKey == "" {
-		validationErrors = append(validationErrors, errInvalidSecretAccessKey)
+		validationErrors = append(validationErrors, errors.New("secret access key is invalid, must be a non-empty string"))
+	}
+
+	if auth.MaxAttempts != nil && *auth.MaxAttempts < 0 {
+		validationErrors = append(validationErrors, fmt.Errorf("invalid max retries setting (%d), must be gte 0", *auth.MaxAttempts))
 	}
 
 	if len(validationErrors) > 0 {
@@ -179,13 +180,20 @@ func configureAuth(auth v1alpha1.AwsAuth, log logr.Logger) error {
 	nonSecretData := map[string]string{
 		"accessKeyId": auth.Credentials.AccessKeyID,
 	}
-	log.Info("Determined AWS auth data.", "nonSecretData", nonSecretData)
+	if auth.MaxAttempts != nil {
+		nonSecretData["maxAttempts"] = fmt.Sprintf("%d", *auth.MaxAttempts)
+	}
+	log.Info("Determined AWS auth and SDK config data.", "nonSecretData", nonSecretData)
 
 	// Use collected and validated values to set env vars.
 	data := map[string]string{
 		"AWS_ACCESS_KEY_ID":     auth.Credentials.AccessKeyID,
 		"AWS_SECRET_ACCESS_KEY": auth.Credentials.SecretAccessKey,
 	}
+	data["AWS_MAX_ATTEMPTS"] = fmt.Sprintf("%d", constants.RetryMaxAttemptsDefault)
+	if auth.MaxAttempts != nil {
+		data["AWS_MAX_ATTEMPTS"] = fmt.Sprintf("%d", *auth.MaxAttempts)
+	}
 	for k, v := range data {
 		if err := os.Setenv(k, v); err != nil {
 			return err

pkg/constants/constants.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/validator-labs/validator-plugin-aws/api/v1alpha1"
+	"github.com/validator-labs/validator/pkg/util"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -19,6 +20,13 @@ func Test_validateAuth(t *testing.T) {
 		args    args
 		wantErr bool
 	}{
+		{
+			name: "No panic for nil auth.Credentials",
+			args: args{
+				auth: v1alpha1.AwsAuth{},
+			},
+			wantErr: true,
+		},
 		{
 			name: "No error for valid inline auth data",
 			args: args{
@@ -27,16 +35,22 @@ func Test_validateAuth(t *testing.T) {
 						AccessKeyID:     "a",
 						SecretAccessKey: "b",
 					},
+					MaxAttempts: util.Ptr(0),
 				},
 			},
 			wantErr: false,
 		},
 		{
-			name: "No panic for nil auth.Credentials",
+			name: "No error for omitted max attempts",
 			args: args{
-				auth: v1alpha1.AwsAuth{},
+				auth: v1alpha1.AwsAuth{
+					Credentials: &v1alpha1.Credentials{
+						AccessKeyID:     "a",
+						SecretAccessKey: "b",
+					},
+				},
 			},
-			wantErr: true,
+			wantErr: false,
 		},
 		{
 			name: "Error for invalid access key ID",
@@ -46,6 +60,7 @@ func Test_validateAuth(t *testing.T) {
 						AccessKeyID:     "",
 						SecretAccessKey: "b",
 					},
+					MaxAttempts: util.Ptr(0),
 				},
 			},
 			wantErr: true,
@@ -62,6 +77,19 @@ func Test_validateAuth(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "Error for invalid max attempts",
+			args: args{
+				auth: v1alpha1.AwsAuth{
+					Credentials: &v1alpha1.Credentials{
+						AccessKeyID:     "a",
+						SecretAccessKey: "b",
+					},
+					MaxAttempts: util.Ptr(-1),
+				},
+			},
+			wantErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -91,12 +119,14 @@ func Test_configureAuth(t *testing.T) {
 						AccessKeyID:     "a",
 						SecretAccessKey: "b",
 					},
+					MaxAttempts: util.Ptr(0),
 				},
 			},
 			wantErr: false,
 			wantEnvVars: map[string]string{
 				"AWS_ACCESS_KEY_ID":     "a",
 				"AWS_SECRET_ACCESS_KEY": "b",
+				"AWS_MAX_ATTEMPTS":      "0",
 			},
 		},
 		{