[Integration] Add support for Table-Level filtering

This commit adds support for new attributes DataFilter and Description.
The resource can now also modify IntegrationName.

Co-authored-by: Justin Chen <jutchen@amazon.com>

.gitignore

@@ -11,3 +11,5 @@ env/
 .gradle
 /gradle/
 /wrapper/
+# unit test artifacts
+aws-rds-cfn-common/src/main/resources/version.properties

README.md

@@ -5,3 +5,51 @@ The CloudFormation Resource Provider Package For Amazon Relational Database Serv
 ## License
 
 This library is licensed under the Apache 2.0 License.
+
+### Generate testsAccountsConfig.yml for contract tests
+
+See [Uluru wiki](https://w.amazon.com/bin/view/AWS/CloudFormation/Teams/ProviderEx/RP-Framework/Projects/UluruContractTests#HCanIrunCTv2inpipelineusingmyownaccounts3F)
+
+Uluru allows service teams to run contract tests on their own accounts. This way, the test process is completely visible
+to the service team -- any errors can be easily debugged in Step Functions (instead of S3), any stuck dependency stacks
+can be freely removed and retried, and contract tests can reuse the same prefab resources as integration tests.
+
+File generation is only needed if: 1) RDS adds a new control plane region, 2) RDS adds a new CFN resource
+
+1. (One-time) Install jq and yq
+   ```
+   brew install jq yq
+   ```
+2. Run command to generate testsAccountsConfig.yml and copy the generated file to all projects' **contract-tests-artifacts** directories
+   ```
+   brazil-build generateTestAccountsConfig
+   ```
+3. Examine `git diff` to make sure the changes are expected
+4. CR the changes
+
+## IntelliJ Setup
+
+As long as you are using the latest BlackCaiman, it should "just work".
+1. Run `brazil-build` once at the root level.
+1. Open it just like any other Java package.
+1. Use the menu Brazil -> Sync from workspace (Enhanced)
+
+It's using the [Enhanced Sync from Workspace](https://builderhub.corp.amazon.com/docs/black-caiman/user-guide/enhanced-sync-from-workspace.html#override) so you must use
+a BlackCaiman version 2023.3.186.0.2023.2 or more recent.
+
+If you are adding a new resource type, you must edit the file `.bemol.toml` to add the project source and test files.
+You can simply copy an existing example.
+
+### Running unit tests
+
+It should just work. The unit nest needs "aws-rds-RESOURCE/target/schema" specifically in the classpath (handled by the bemol config file).
+
+### My local state is really strange!
+
+First make sure all your changes are saved/committed. There is no going back.
+
+There are many files that are in .gitignore. To truly clean your workspace, run the following:
+```
+brazil ws clean
+git clean -dfx
+```

aws-rds-cfn-common/pom.xml

@@ -28,12 +28,12 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>utils</artifactId>
-            <version>2.24.13</version>
+            <version>2.25.12</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>rds</artifactId>
-            <version>2.24.13</version>
+            <version>2.25.12</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.cloudformation</groupId>

aws-rds-customdbengineversion/docs/README.md

@@ -160,7 +160,7 @@ _Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormati
 
 #### UseAwsProvidedLatestImage
 
-A value that indicates whether AWS provided latest image is applied automatically to the Custom Engine Version. By default, AWS provided latest image is applied automatically.
+A value that indicates whether AWS provided latest image is applied automatically to the Custom Engine Version. By default, AWS provided latest image is applied automatically. This value is only applied on create.
 
 _Required_: No
 

aws-rds-customdbengineversion/pom.xml

@@ -23,7 +23,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>rds</artifactId>
-            <version>2.24.13</version>
+            <version>2.25.12</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.rds.common</groupId>

aws-rds-dbcluster/docs/README.md

@@ -750,3 +750,7 @@ Returns the <code>Address</code> value.
 #### SecretArn
 
 Returns the <code>SecretArn</code> value.
+
+#### StorageThroughput
+
+Specifies the storage throughput value for the DB cluster. This setting applies only to the gp3 storage type.

aws-rds-dbcluster/pom.xml

@@ -30,7 +30,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>rds</artifactId>
-            <version>2.24.13</version>
+            <version>2.25.12</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>

aws-rds-dbclusterendpoint/pom.xml

@@ -30,7 +30,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>rds</artifactId>
-            <version>2.24.13</version>
+            <version>2.25.12</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/software.amazon.cloudformation/aws-cloudformation-rpdk-java-plugin -->
         <dependency>

aws-rds-dbclusterparametergroup/pom.xml

@@ -30,7 +30,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>rds</artifactId>
-            <version>2.24.13</version>
+            <version>2.25.12</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/software.amazon.cloudformation/aws-cloudformation-rpdk-java-plugin -->
         <dependency>

aws-rds-dbinstance/pom.xml

@@ -33,7 +33,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>rds</artifactId>
-            <version>2.24.13</version>
+            <version>2.25.12</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>

aws-rds-dbparametergroup/pom.xml

@@ -23,7 +23,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>rds</artifactId>
-            <version>2.24.13</version>
+            <version>2.25.12</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/software.amazon.cloudformation/aws-cloudformation-rpdk-java-plugin -->
         <dependency>

aws-rds-dbsubnetgroup/pom.xml

@@ -23,7 +23,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>rds</artifactId>
-            <version>2.24.13</version>
+            <version>2.25.12</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/software.amazon.cloudformation/aws-cloudformation-rpdk-java-plugin -->
         <dependency>

aws-rds-eventsubscription/pom.xml

@@ -29,7 +29,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>rds</artifactId>
-            <version>2.24.13</version>
+            <version>2.25.12</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/software.amazon.cloudformation/aws-cloudformation-rpdk-java-plugin -->
         <dependency>

aws-rds-globalcluster/pom.xml

@@ -28,7 +28,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>rds</artifactId>
-            <version>2.24.13</version>
+            <version>2.25.12</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/software.amazon.cloudformation/aws-cloudformation-rpdk-java-plugin -->
         <dependency>

aws-rds-integration/aws-rds-integration.json

@@ -9,6 +9,12 @@
       "minLength": 1,
       "maxLength": 64
     },
+    "Description": {
+      "type": "string",
+      "description": "The description of the integration.",
+      "minLength": 1,
+      "maxLength": 1000
+    },
     "Tags": {
       "type": "array",
       "maxItems": 50,
@@ -19,6 +25,13 @@
         "$ref": "#/definitions/Tag"
       }
     },
+    "DataFilter": {
+      "type": "string",
+      "description": "The data filter for the integration.",
+      "minLength": 1,
+      "maxLength": 25600,
+      "pattern": "[a-zA-Z0-9_ \"\\\\\\-$,*.:?+\\/]*"
+    },
     "SourceArn": {
       "type": "string",
       "description": "The Amazon Resource Name (ARN) of the Aurora DB cluster to use as the source for replication."
@@ -100,8 +113,7 @@
     "/properties/SourceArn",
     "/properties/TargetArn",
     "/properties/KMSKeyId",
-    "/properties/AdditionalEncryptionContext",
-    "/properties/IntegrationName"
+    "/properties/AdditionalEncryptionContext"
   ],
   "readOnlyProperties": [
     "/properties/IntegrationArn",
@@ -130,7 +142,8 @@
       "permissions": [
         "rds:DescribeIntegrations",
         "rds:AddTagsToResource",
-        "rds:RemoveTagsFromResource"
+        "rds:RemoveTagsFromResource",
+        "rds:ModifyIntegration"
       ]
     },
     "delete": {

aws-rds-integration/docs/README.md

@@ -13,7 +13,9 @@ To declare this entity in your AWS CloudFormation template, use the following sy
     "Type" : "AWS::RDS::Integration",
     "Properties" : {
         "<a href="#integrationname" title="IntegrationName">IntegrationName</a>" : <i>String</i>,
+        "<a href="#description" title="Description">Description</a>" : <i>String</i>,
         "<a href="#tags" title="Tags">Tags</a>" : <i>[ <a href="tag.md">Tag</a>, ... ]</i>,
+        "<a href="#datafilter" title="DataFilter">DataFilter</a>" : <i>String</i>,
         "<a href="#sourcearn" title="SourceArn">SourceArn</a>" : <i>String</i>,
         "<a href="#targetarn" title="TargetArn">TargetArn</a>" : <i>String</i>,
         "<a href="#kmskeyid" title="KMSKeyId">KMSKeyId</a>" : <i>String</i>,
@@ -28,8 +30,10 @@ To declare this entity in your AWS CloudFormation template, use the following sy
 Type: AWS::RDS::Integration
 Properties:
     <a href="#integrationname" title="IntegrationName">IntegrationName</a>: <i>String</i>
+    <a href="#description" title="Description">Description</a>: <i>String</i>
     <a href="#tags" title="Tags">Tags</a>: <i>
       - <a href="tag.md">Tag</a></i>
+    <a href="#datafilter" title="DataFilter">DataFilter</a>: <i>String</i>
     <a href="#sourcearn" title="SourceArn">SourceArn</a>: <i>String</i>
     <a href="#targetarn" title="TargetArn">TargetArn</a>: <i>String</i>
     <a href="#kmskeyid" title="KMSKeyId">KMSKeyId</a>: <i>String</i>
@@ -50,7 +54,21 @@ _Minimum Length_: <code>1</code>
 
 _Maximum Length_: <code>64</code>
 
-_Update requires_: [Replacement](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-replacement)
+_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt)
+
+#### Description
+
+The description of the integration.
+
+_Required_: No
+
+_Type_: String
+
+_Minimum Length_: <code>1</code>
+
+_Maximum Length_: <code>1000</code>
+
+_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt)
 
 #### Tags
 
@@ -62,6 +80,22 @@ _Type_: List of <a href="tag.md">Tag</a>
 
 _Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt)
 
+#### DataFilter
+
+The data filter for the integration.
+
+_Required_: No
+
+_Type_: String
+
+_Minimum Length_: <code>1</code>
+
+_Maximum Length_: <code>25600</code>
+
+_Pattern_: <code>[a-zA-Z0-9_ "\\\-$,*.:?+\/]*</code>
+
+_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt)
+
 #### SourceArn
 
 The Amazon Resource Name (ARN) of the Aurora DB cluster to use as the source for replication.

aws-rds-integration/pom.xml

@@ -22,7 +22,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>rds</artifactId>
-            <version>2.24.13</version>
+            <version>2.25.12</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/software.amazon.cloudformation/aws-cloudformation-rpdk-java-plugin -->
         <dependency>

aws-rds-integration/resource-role.yaml

@@ -36,6 +36,7 @@ Resources:
                 - "rds:CreateIntegration"
                 - "rds:DeleteIntegration"
                 - "rds:DescribeIntegrations"
+                - "rds:ModifyIntegration"
                 - "rds:RemoveTagsFromResource"
                 - "redshift:CreateInboundIntegration"
                 Resource: "*"

aws-rds-integration/src/main/java/software/amazon/rds/integration/BaseHandlerStd.java

@@ -6,6 +6,7 @@
 import software.amazon.awssdk.services.rds.model.IntegrationNotFoundException;
 import software.amazon.awssdk.services.rds.model.IntegrationQuotaExceededException;
 import software.amazon.awssdk.services.rds.model.IntegrationStatus;
+import software.amazon.awssdk.services.rds.model.InvalidIntegrationStateException;
 import software.amazon.awssdk.services.rds.model.KmsKeyNotAccessibleException;
 import software.amazon.awssdk.services.rds.model.Tag;
 import software.amazon.cloudformation.exceptions.CfnNotStabilizedException;
@@ -30,13 +31,18 @@
 import java.util.Optional;
 
 public abstract class BaseHandlerStd extends BaseHandler<CallbackContext> {
-    protected final static String INTEGRATION_NAME_CONFLICT_ERROR_MESSAGE = "Integration names must be unique within an account";
     /** If this message is given with IntegrationConflictOperationFault, then it is a retriable error.
      * This is thrown when the underlying Redshift cluster is busy with other operations, but they are always
-     * transient, unless something is really wrong. */
+     * transient, unless something is really wrong.
+     * Applies to IntegrationConflictOperationException */
     protected final static String INTEGRATION_RETRIABLE_CONFLICT_MESSAGE = "because another operation is in progress for the " +
             "Amazon Redshift data warehouse specified by the Amazon Resource Name (ARN). " +
             "Try again after the current operation completes.";
+    /**
+     * Applies to InvalidIntegrationStateException, similar to INTEGRATION_RETRIABLE_CONFLICT_MESSAGE
+     */
+    protected final static String INTEGRATION_RETRIABLE_CONFLICTING_STATE_MESSAGE = "because it is not in a valid state. " +
+            "Wait until the integration is in a valid state and try again.";
     protected static final String STACK_NAME = "rds";
     protected static final String RESOURCE_IDENTIFIER = "integration";
     protected static final int MAX_LENGTH_INTEGRATION = 63;
@@ -61,23 +67,24 @@ public abstract class BaseHandlerStd extends BaseHandler<CallbackContext> {
             .withErrorClasses(ErrorStatus.failWith(HandlerErrorCode.NotFound),
                     IntegrationNotFoundException.class)
             .withErrorClasses(ErrorStatus.conditional((e) -> {
-                // this condition happens on create-integration.
-                // it's a little strange that IntegrationConflictOperationException is thrown instead of AlreadyExists exception
-                // we need to override the default error handling because in this case we need to tell CFN that it's an AlreadyExists.
-                // otherwise,
                 String nonNullErrorMessage = Optional.ofNullable(e.getMessage()).orElse("");
-                if (nonNullErrorMessage.contains(INTEGRATION_NAME_CONFLICT_ERROR_MESSAGE)) {
-                    return ErrorStatus.failWith(HandlerErrorCode.AlreadyExists);
-                } else if (nonNullErrorMessage.contains(INTEGRATION_RETRIABLE_CONFLICT_MESSAGE)) {
+                if (nonNullErrorMessage.contains(INTEGRATION_RETRIABLE_CONFLICT_MESSAGE)) {
                     // this tells the cfn framework to come back in a bit.
                     return ErrorStatus.retry(CALLBACK_DELAY);
                 } else {
-                    // this shouldn't happen but it's a good fallback.
+                    // it's unclear if it's safe to retry otherwise.
                     return ErrorStatus.failWith(HandlerErrorCode.ResourceConflict);
                 }
             }), IntegrationConflictOperationException.class)
-            .withErrorClasses(ErrorStatus.failWith(HandlerErrorCode.ResourceConflict),
-                    IntegrationConflictOperationException.class)
+            .withErrorClasses(ErrorStatus.conditional((e) -> {
+                String nonNullErrorMessage = Optional.ofNullable(e.getMessage()).orElse("");
+                if (nonNullErrorMessage.contains(INTEGRATION_RETRIABLE_CONFLICTING_STATE_MESSAGE)) {
+                    return ErrorStatus.retry(CALLBACK_DELAY);
+                } else {
+                    // it's unclear if it's safe to retry otherwise.
+                    return ErrorStatus.failWith(HandlerErrorCode.ResourceConflict);
+                }
+            }), InvalidIntegrationStateException.class)
             .withErrorClasses(ErrorStatus.failWith(HandlerErrorCode.ServiceLimitExceeded),
                     IntegrationQuotaExceededException.class)
             .withErrorClasses(ErrorStatus.failWith(HandlerErrorCode.AccessDenied),
@@ -133,7 +140,7 @@ protected ProgressEvent<ResourceModel, CallbackContext> handleRequest(
     ) {
         this.requestLogger = requestLogger;
         return handleRequest(proxy, proxyClient, request, context);
-    };
+    }
 
 
     /**