China region support (#318)

* handle case with no webui

* made region generci for china

* added PR number

* fix backwards compatibility for CR

* fixed redeploy-containers to work with china

* incremented version number

* remove use of grep due to difference in OS version

* updated python to 3.9

* improved name of function

* added explanation

* added negative test

* fix syntax error

* rule to check region supports Cognito

* fixed typo

Co-authored-by: Matteo Figus <matteofigus@gmail.com>

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Change Log
 
-## v0.52 (unreleased)
+## v0.52
+
+- [#318](https://github.com/awslabs/amazon-s3-find-and-forget/pull/318): Added
+  support for AWS China
 
 - [#324](https://github.com/awslabs/amazon-s3-find-and-forget/pull/324): Upgrade
   frontend dependencies

Makefile

@@ -109,12 +109,14 @@ backend/ecs_tasks/python_3.9-slim.tar:
 
 redeploy-containers:
 	$(eval ACCOUNT_ID := $(shell aws sts get-caller-identity --query Account --output text))
-	$(eval REGION := $(shell aws configure get region))
+	$(eval API_URL := $(shell aws cloudformation describe-stacks --stack-name S3F2 --query 'Stacks[0].Outputs[?OutputKey==`ApiUrl`].OutputValue' --output text))
+	$(eval REGION := $(shell echo $(API_URL) | cut -d'.' -f3))
 	$(eval ECR_REPOSITORY := $(shell aws cloudformation describe-stacks --stack-name S3F2 --query 'Stacks[0].Outputs[?OutputKey==`ECRRepository`].OutputValue' --output text))
+	$(eval REPOSITORY_URI := $(shell aws ecr describe-repositories --repository-names $(ECR_REPOSITORY) --query 'repositories[0].repositoryUri' --output text))
 	$(shell aws ecr get-login --no-include-email --region $(REGION))
 	docker build -t $(ECR_REPOSITORY) -f backend/ecs_tasks/delete_files/Dockerfile .
-	docker tag $(ECR_REPOSITORY):latest $(ACCOUNT_ID).dkr.ecr.$(REGION).amazonaws.com/$(ECR_REPOSITORY):latest
-	docker push $(ACCOUNT_ID).dkr.ecr.$(REGION).amazonaws.com/$(ECR_REPOSITORY):latest
+	docker tag $(ECR_REPOSITORY):latest $(REPOSITORY_URI):latest
+	docker push $(REPOSITORY_URI):latest
 
 redeploy-frontend:
 	$(eval WEBUI_BUCKET := $(shell aws cloudformation describe-stacks --stack-name S3F2 --query 'Stacks[0].Outputs[?OutputKey==`WebUIBucket`].OutputValue' --output text))
@@ -158,8 +160,9 @@ backend/lambda_layers/%/requirements-installed.sentinel: backend/lambda_layers/%
 	touch $@
 
 setup-frontend-local-dev:
+	$(eval WEBUI_URL := $(shell aws cloudformation describe-stacks --stack-name S3F2 --query 'Stacks[0].Outputs[?OutputKey==`WebUIUrl`].OutputValue' --output text))
 	$(eval WEBUI_BUCKET := $(shell aws cloudformation describe-stacks --stack-name S3F2 --query 'Stacks[0].Outputs[?OutputKey==`WebUIBucket`].OutputValue' --output text))
-	aws s3 cp s3://$(WEBUI_BUCKET)/settings.js frontend/public/settings.js
+	$(if $(filter none, $(WEBUI_URL)), @echo "WebUI not deployed.", aws s3 cp s3://$(WEBUI_BUCKET)/settings.js frontend/public/settings.js)
 
 setup-predeploy:
 	virtualenv venv

backend/ecs_tasks/delete_files/main.py

@@ -255,8 +255,8 @@ def kill_handler(msgs, process_pool):
 
 def get_queue(queue_url, **resource_kwargs):
     if not resource_kwargs.get("endpoint_url") and os.getenv("AWS_DEFAULT_REGION"):
-        resource_kwargs["endpoint_url"] = "https://sqs.{}.amazonaws.com".format(
-            os.getenv("AWS_DEFAULT_REGION")
+        resource_kwargs["endpoint_url"] = "https://sqs.{}.{}".format(
+            os.getenv("AWS_DEFAULT_REGION"), os.getenv("AWS_URL_SUFFIX")
         )
     sqs = boto3.resource("sqs", **resource_kwargs)
     return sqs.Queue(queue_url)

backend/lambdas/custom_resources/copy_build_artefact.py

@@ -16,6 +16,9 @@ def create(event, context):
     version = props.get("Version")
     destination_artefact = props.get("ArtefactName")
     destination_bucket = props.get("CodeBuildArtefactBucket")
+    destination_bucket_arn = props.get(
+        "CodeBuildArtefactBucketArn", "arn:aws:s3:::{}".format(destination_bucket)
+    )
     source_bucket = props.get("PreBuiltArtefactsBucket")
     source_artefact = "{}/amazon-s3-find-and-forget/{}/build.zip".format(
         source_bucket, version
@@ -25,7 +28,7 @@ def create(event, context):
         Bucket=destination_bucket, CopySource=source_artefact, Key=destination_artefact
     )
 
-    return "arn:aws:s3:::{}/{}".format(destination_bucket, destination_artefact)
+    return "{}/{}".format(destination_bucket_arn, destination_artefact)
 
 
 @with_logging

backend/lambdas/custom_resources/get_vpce_subnets.py

@@ -0,0 +1,45 @@
+#############################################################
+# This Custom Resource is required since VPC Endpoint names #
+# and subnets are not consistant in the China region        #
+#############################################################
+
+import boto3
+from crhelper import CfnResource
+from decorators import with_logging
+
+helper = CfnResource(json_logging=False, log_level="DEBUG", boto_level="CRITICAL")
+
+ec2_client = boto3.client("ec2")
+
+
+@with_logging
+@helper.create
+@helper.update
+def create(event, context):
+    props = event.get("ResourceProperties", None)
+    service_name = props.get("ServiceName")
+    subnet_ids = props.get("SubnetIds")
+    vpc_endpoint_type = props.get("VpcEndpointType")
+    describe_subnets = ec2_client.describe_subnets(SubnetIds=subnet_ids)
+    subnet_dict = {
+        s["AvailabilityZone"]: s["SubnetId"] for s in describe_subnets["Subnets"]
+    }
+    endpoint_service = ec2_client.describe_vpc_endpoint_services(
+        Filters=[
+            {"Name": "service-name", "Values": [f"cn.{service_name}", service_name]},
+            {"Name": "service-type", "Values": [vpc_endpoint_type]},
+        ]
+    )
+    service_details = endpoint_service["ServiceDetails"][0]
+    helper.Data["ServiceName"] = service_details["ServiceName"]
+    return ",".join([subnet_dict[s] for s in service_details["AvailabilityZones"]])
+
+
+@with_logging
+@helper.delete
+def delete(event, context):
+    return None
+
+
+def handler(event, context):
+    helper(event, context)

docker_run_with_creds.sh

@@ -17,10 +17,11 @@ DLQ_URL=$(aws cloudformation describe-stacks \
   --query 'Stacks[0].Outputs[?OutputKey==`DLQUrl`].OutputValue' \
   --output text)
 ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+PARTITION=$(aws sts get-caller-identity --query Arn --output text | cut -d':' -f2)
 # Assume IAM Role to be passed to container
 SESSION_DATA=$(aws sts assume-role \
   --role-session-name s3f2-local \
-  --role-arn arn:aws:iam::"${ACCOUNT_ID}":role/"${ROLE_NAME}" \
+  --role-arn arn:"${PARTITION}":iam::"${ACCOUNT_ID}":role/"${ROLE_NAME}" \
   --query Credentials \
   --output json)
 AWS_ACCESS_KEY_ID=$(echo "${SESSION_DATA}" | jq -r ".AccessKeyId")

templates/api.yaml

@@ -1266,10 +1266,10 @@ Outputs:
       - !Ref WebUIOrigin
       - !Ref AccessControlAllowOriginOverride
   ApiArn:
-    Value: !Sub arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${Api}/${Api.Stage}/*/*
+    Value: !Sub arn:${AWS::Partition}:execute-api:${AWS::Region}:${AWS::AccountId}:${Api}/${Api.Stage}/*/*
   ApiUrl:
     Description: API endpoint URL for Prod environment
-    Value: !Sub https://${Api}.execute-api.${AWS::Region}.amazonaws.com/${Api.Stage}/
+    Value: !Sub https://${Api}.execute-api.${AWS::Region}.${AWS::URLSuffix}/${Api.Stage}/
   PutDataMapperRole:
     Description: Role used by the PutDataMapper API
     Value: !Ref PutDataMapperRole

templates/auth.yaml

@@ -59,7 +59,7 @@ Resources:
     Type: AWS::IAM::Role
     Properties:
       ManagedPolicyArns:
-        - arn:aws:iam::aws:policy/AmazonAPIGatewayInvokeFullAccess
+        - !Sub arn:${AWS::Partition}:iam::aws:policy/AmazonAPIGatewayInvokeFullAccess
       AssumeRolePolicyDocument:
         Statement:
           - Effect: Allow

templates/deletion_flow.yaml

@@ -81,7 +81,7 @@ Resources:
               - sts:AssumeRole
       Path: /
       ManagedPolicyArns:
-      - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
+      - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
 
   DelObjQ:
     Type: AWS::SQS::Queue
@@ -128,7 +128,7 @@ Resources:
       ContainerDefinitions:
         - Name: !Sub ${ResourcePrefix}_DeleteTask
           Essential: true
-          Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ECRRepository}:latest
+          Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/${ECRRepository}:latest
           LogConfiguration:
             LogDriver: awslogs
             Options:
@@ -138,6 +138,8 @@ Resources:
           Environment:
             - Name: AWS_STS_REGIONAL_ENDPOINTS
               Value: regional
+            - Name: AWS_URL_SUFFIX
+              Value: !Ref AWS::URLSuffix
             - Name: DELETE_OBJECTS_QUEUE
               Value: !Ref DelObjQ
             - Name: ECS_ENABLE_CONTAINER_METADATA
@@ -185,13 +187,13 @@ Resources:
             - logs:CreateLogStream
             - logs:PutLogEvents
             Effect: Allow
-            Resource: "arn:aws:logs:*:*:*"
+            Resource: !Sub "arn:${AWS::Partition}:logs:*:*:*"
           - Action: sts:AssumeRole
             Effect: Allow
             Resource: !Sub "arn:${AWS::Partition}:iam::*:role/S3F2DataAccessRole"
           - Action: s3:GetObject*
             Effect: Allow
-            Resource: !Sub arn:aws:s3:::${ManifestsBucket}/manifests/*
+            Resource: !Sub arn:${AWS::Partition}:s3:::${ManifestsBucket}/manifests/*
           - !If
             - WithKMS
             - Action:

templates/deployment_helper.yaml

@@ -78,17 +78,17 @@ Resources:
                 Action:
                   - logs:CreateLogGroup
                   - ecr:GetAuthorizationToken
-              - Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:*
+              - Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:*
                 Effect: Allow
                 Action:
                   - logs:CreateLogStream
                   - logs:PutLogEvents
-              - Resource: !Sub arn:aws:s3:::${CodeBuildArtefactBucket}/*
+              - Resource: !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}/*
                 Effect: Allow
                 Action:
                   - s3:Get*
                   - s3:List*
-              - Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository}
+              - Resource: !Sub arn:${AWS::Partition}:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository}
                 Effect: Allow
                 Action:
                   - ecr:GetDownloadUrlForLayer
@@ -116,26 +116,26 @@ Resources:
               - Resource: "*"
                 Effect: Allow
                 Action: logs:CreateLogGroup
-              - Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:*
+              - Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:*
                 Effect: Allow
                 Action:
                   - logs:CreateLogStream
                   - logs:PutLogEvents
-              - Resource: !Sub arn:aws:s3:::${CodeBuildArtefactBucket}/*
+              - Resource: !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}/*
                 Effect: Allow
                 Action:
                   - s3:Get*
                   - s3:List*
               - !If
                 - ShouldDeployWebUI
-                - Resource: !Sub arn:aws:s3:::${WebUIBucket}/*
+                - Resource: !Sub arn:${AWS::Partition}:s3:::${WebUIBucket}/*
                   Effect: Allow
                   Action: s3:PutObject*
                 - !Ref AWS::NoValue
               - !If
                 - WithoutCloudFront
                 - !Ref AWS::NoValue
-                - Resource: !Sub arn:aws:cloudfront::${AWS::AccountId}:distribution/${CloudFrontDistribution}
+                - Resource: !Sub arn:${AWS::Partition}:cloudfront::${AWS::AccountId}:distribution/${CloudFrontDistribution}
                   Effect: Allow
                   Action: cloudfront:CreateInvalidation
 
@@ -169,7 +169,7 @@ Resources:
           - Name: AWS_DEFAULT_REGION
             Value: !Ref AWS::Region
           - Name: REPOSITORY_URI
-            Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ECRRepository}
+            Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/${ECRRepository}
       Name: !Sub ${ResourcePrefix}BackendBuild
       ServiceRole: !Ref CodeBuildBackendServiceRole
 
@@ -256,26 +256,26 @@ Resources:
                   - ecs:UpdateService
               - Effect: Allow
                 Resource:
-                  - !Sub arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${CodeBuildBackend}
+                  - !Sub arn:${AWS::Partition}:codebuild:${AWS::Region}:${AWS::AccountId}:project/${CodeBuildBackend}
                 Action:
                   - codebuild:BatchGetBuilds
                   - codebuild:StartBuild
               - !If
                 - ShouldDeployWebUI
                 - Effect: Allow
                   Resource:
-                    - !Sub arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${CodeBuildFrontend}
+                    - !Sub arn:${AWS::Partition}:codebuild:${AWS::Region}:${AWS::AccountId}:project/${CodeBuildFrontend}
                   Action:
                     - codebuild:BatchGetBuilds
                     - codebuild:StartBuild
                 - !Ref AWS::NoValue
               - Effect: Allow
-                Resource: !Sub arn:aws:s3:::${CodeBuildArtefactBucket}
+                Resource: !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}
                 Action:
                   - s3:GetBucket*
                   - s3:List*
               - Effect: Allow
-                Resource: !Sub arn:aws:s3:::${CodeBuildArtefactBucket}/*
+                Resource: !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}/*
                 Action:
                   - s3:GetObject
                   - s3:GetObjectVersion
@@ -347,10 +347,10 @@ Resources:
         - Statement:
           - Effect: Allow
             Action: s3:PutObject*
-            Resource: !Sub arn:aws:s3:::${CodeBuildArtefactBucket}/*
+            Resource: !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}/*
           - Effect: Allow
             Action: s3:GetObject
-            Resource: !Sub arn:aws:s3:::${PreBuiltArtefactsBucket}/*
+            Resource: !Sub arn:${AWS::Partition}:s3:::${PreBuiltArtefactsBucket}/*
 
   CleanupBucketFunction:
     Type: AWS::Serverless::Function
@@ -367,10 +367,10 @@ Resources:
               - s3:ListBucket*
               - s3:ListObject*
             Resource:
-              - !Sub arn:aws:s3:::${CodeBuildArtefactBucket}
-              - !Sub arn:aws:s3:::${CodeBuildArtefactBucket}/*
-              - !Sub arn:aws:s3:::${WebUIBucket}
-              - !Sub arn:aws:s3:::${WebUIBucket}/*
+              - !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}
+              - !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}/*
+              - !Sub arn:${AWS::Partition}:s3:::${WebUIBucket}
+              - !Sub arn:${AWS::Partition}:s3:::${WebUIBucket}/*
 
   CleanupRepositoryFunction:
     Type: AWS::Serverless::Function
@@ -384,7 +384,7 @@ Resources:
             Action:
               - ecr:BatchDeleteImage
               - ecr:ListImages
-            Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository}
+            Resource: !Sub arn:${AWS::Partition}:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository}
 
   WaitForContainerBuildFunction:
     Type: AWS::Serverless::Function
@@ -405,10 +405,10 @@ Resources:
             Resource: "*"
           - Effect: Allow
             Action: s3:GetObject*
-            Resource: !Sub "arn:aws:s3:::${CodeBuildArtefactBucket}/*"
+            Resource: !Sub "arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}/*"
           - Effect: Allow
             Action: ecr:DescribeImages
-            Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository}
+            Resource: !Sub arn:${AWS::Partition}:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository}
 
 
   RedeployApiFunction:
@@ -424,7 +424,7 @@ Resources:
               - apigateway:POST
             Resource:
               - !Sub
-                - arn:aws:apigateway:${AWS::Region}::/restapis/${ApiId}/deployments
+                - arn:${AWS::Partition}:apigateway:${AWS::Region}::/restapis/${ApiId}/deployments
                 - ApiId: !Select [0, !Split [".", !Select [2, !Split ["/", !Ref ApiUrl]]]]
 
 
@@ -440,7 +440,7 @@ Resources:
             Action:
               - codepipeline:StartPipelineExecution
             Resource:
-              - !Sub arn:aws:codepipeline:${AWS::Region}:${AWS::AccountId}:${CodePipeline}
+              - !Sub arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:${CodePipeline}
 
 
   CleanupCodeBuildArtefactBucket:
@@ -472,6 +472,7 @@ Resources:
       ServiceToken: !GetAtt CopyBuildArtefactFunction.Arn
       ArtefactName: !Ref ArtefactName
       CodeBuildArtefactBucket: !Ref CodeBuildArtefactBucket
+      CodeBuildArtefactBucketArn: !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}
       LogLevel: !Ref LogLevel
       Region: !Ref AWS::Region
       PreBuiltArtefactsBucket: !Ref PreBuiltArtefactsBucket

templates/manifests.yaml

@@ -41,8 +41,8 @@ Resources:
             Action: '*'
             Effect: Deny
             Resource:
-              - !Sub arn:aws:s3:::${ManifestsBucket}
-              - !Sub arn:aws:s3:::${ManifestsBucket}/*
+              - !Sub arn:${AWS::Partition}:s3:::${ManifestsBucket}
+              - !Sub arn:${AWS::Partition}:s3:::${ManifestsBucket}/*
             Principal: '*'
             Condition:
               Bool: