diff --git a/CHANGELOG.md b/CHANGELOG.md index 59343708..0361f0d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Change Log -## v0.52 (unreleased) +## v0.52 + +- [#318](https://github.com/awslabs/amazon-s3-find-and-forget/pull/318): Added + support for AWS China - [#324](https://github.com/awslabs/amazon-s3-find-and-forget/pull/324): Upgrade frontend dependencies diff --git a/Makefile b/Makefile index 54b5e71f..c8b890b9 100644 --- a/Makefile +++ b/Makefile @@ -109,12 +109,14 @@ backend/ecs_tasks/python_3.9-slim.tar: redeploy-containers: $(eval ACCOUNT_ID := $(shell aws sts get-caller-identity --query Account --output text)) - $(eval REGION := $(shell aws configure get region)) + $(eval API_URL := $(shell aws cloudformation describe-stacks --stack-name S3F2 --query 'Stacks[0].Outputs[?OutputKey==`ApiUrl`].OutputValue' --output text)) + $(eval REGION := $(shell echo $(API_URL) | cut -d'.' -f3)) $(eval ECR_REPOSITORY := $(shell aws cloudformation describe-stacks --stack-name S3F2 --query 'Stacks[0].Outputs[?OutputKey==`ECRRepository`].OutputValue' --output text)) + $(eval REPOSITORY_URI := $(shell aws ecr describe-repositories --repository-names $(ECR_REPOSITORY) --query 'repositories[0].repositoryUri' --output text)) $(shell aws ecr get-login --no-include-email --region $(REGION)) docker build -t $(ECR_REPOSITORY) -f backend/ecs_tasks/delete_files/Dockerfile . - docker tag $(ECR_REPOSITORY):latest $(ACCOUNT_ID).dkr.ecr.$(REGION).amazonaws.com/$(ECR_REPOSITORY):latest - docker push $(ACCOUNT_ID).dkr.ecr.$(REGION).amazonaws.com/$(ECR_REPOSITORY):latest + docker tag $(ECR_REPOSITORY):latest $(REPOSITORY_URI):latest + docker push $(REPOSITORY_URI):latest redeploy-frontend: $(eval WEBUI_BUCKET := $(shell aws cloudformation describe-stacks --stack-name S3F2 --query 'Stacks[0].Outputs[?OutputKey==`WebUIBucket`].OutputValue' --output text)) @@ -158,8 +160,9 @@ backend/lambda_layers/%/requirements-installed.sentinel: backend/lambda_layers/% touch $@ setup-frontend-local-dev: + $(eval WEBUI_URL := $(shell aws cloudformation describe-stacks --stack-name S3F2 --query 'Stacks[0].Outputs[?OutputKey==`WebUIUrl`].OutputValue' --output text)) $(eval WEBUI_BUCKET := $(shell aws cloudformation describe-stacks --stack-name S3F2 --query 'Stacks[0].Outputs[?OutputKey==`WebUIBucket`].OutputValue' --output text)) - aws s3 cp s3://$(WEBUI_BUCKET)/settings.js frontend/public/settings.js + $(if $(filter none, $(WEBUI_URL)), @echo "WebUI not deployed.", aws s3 cp s3://$(WEBUI_BUCKET)/settings.js frontend/public/settings.js) setup-predeploy: virtualenv venv diff --git a/backend/ecs_tasks/delete_files/main.py b/backend/ecs_tasks/delete_files/main.py index a53dd898..e4f5a41a 100644 --- a/backend/ecs_tasks/delete_files/main.py +++ b/backend/ecs_tasks/delete_files/main.py @@ -255,8 +255,8 @@ def kill_handler(msgs, process_pool): def get_queue(queue_url, **resource_kwargs): if not resource_kwargs.get("endpoint_url") and os.getenv("AWS_DEFAULT_REGION"): - resource_kwargs["endpoint_url"] = "https://sqs.{}.amazonaws.com".format( - os.getenv("AWS_DEFAULT_REGION") + resource_kwargs["endpoint_url"] = "https://sqs.{}.{}".format( + os.getenv("AWS_DEFAULT_REGION"), os.getenv("AWS_URL_SUFFIX") ) sqs = boto3.resource("sqs", **resource_kwargs) return sqs.Queue(queue_url) diff --git a/backend/lambdas/custom_resources/copy_build_artefact.py b/backend/lambdas/custom_resources/copy_build_artefact.py index db091924..3a8daf67 100644 --- a/backend/lambdas/custom_resources/copy_build_artefact.py +++ b/backend/lambdas/custom_resources/copy_build_artefact.py @@ -16,6 +16,9 @@ def create(event, context): version = props.get("Version") destination_artefact = props.get("ArtefactName") destination_bucket = props.get("CodeBuildArtefactBucket") + destination_bucket_arn = props.get( + "CodeBuildArtefactBucketArn", "arn:aws:s3:::{}".format(destination_bucket) + ) source_bucket = props.get("PreBuiltArtefactsBucket") source_artefact = "{}/amazon-s3-find-and-forget/{}/build.zip".format( source_bucket, version @@ -25,7 +28,7 @@ def create(event, context): Bucket=destination_bucket, CopySource=source_artefact, Key=destination_artefact ) - return "arn:aws:s3:::{}/{}".format(destination_bucket, destination_artefact) + return "{}/{}".format(destination_bucket_arn, destination_artefact) @with_logging diff --git a/backend/lambdas/custom_resources/get_vpce_subnets.py b/backend/lambdas/custom_resources/get_vpce_subnets.py new file mode 100644 index 00000000..a45900f7 --- /dev/null +++ b/backend/lambdas/custom_resources/get_vpce_subnets.py @@ -0,0 +1,45 @@ +############################################################# +# This Custom Resource is required since VPC Endpoint names # +# and subnets are not consistant in the China region # +############################################################# + +import boto3 +from crhelper import CfnResource +from decorators import with_logging + +helper = CfnResource(json_logging=False, log_level="DEBUG", boto_level="CRITICAL") + +ec2_client = boto3.client("ec2") + + +@with_logging +@helper.create +@helper.update +def create(event, context): + props = event.get("ResourceProperties", None) + service_name = props.get("ServiceName") + subnet_ids = props.get("SubnetIds") + vpc_endpoint_type = props.get("VpcEndpointType") + describe_subnets = ec2_client.describe_subnets(SubnetIds=subnet_ids) + subnet_dict = { + s["AvailabilityZone"]: s["SubnetId"] for s in describe_subnets["Subnets"] + } + endpoint_service = ec2_client.describe_vpc_endpoint_services( + Filters=[ + {"Name": "service-name", "Values": [f"cn.{service_name}", service_name]}, + {"Name": "service-type", "Values": [vpc_endpoint_type]}, + ] + ) + service_details = endpoint_service["ServiceDetails"][0] + helper.Data["ServiceName"] = service_details["ServiceName"] + return ",".join([subnet_dict[s] for s in service_details["AvailabilityZones"]]) + + +@with_logging +@helper.delete +def delete(event, context): + return None + + +def handler(event, context): + helper(event, context) diff --git a/docker_run_with_creds.sh b/docker_run_with_creds.sh index 423aa214..11fa687c 100755 --- a/docker_run_with_creds.sh +++ b/docker_run_with_creds.sh @@ -17,10 +17,11 @@ DLQ_URL=$(aws cloudformation describe-stacks \ --query 'Stacks[0].Outputs[?OutputKey==`DLQUrl`].OutputValue' \ --output text) ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +PARTITION=$(aws sts get-caller-identity --query Arn --output text | cut -d':' -f2) # Assume IAM Role to be passed to container SESSION_DATA=$(aws sts assume-role \ --role-session-name s3f2-local \ - --role-arn arn:aws:iam::"${ACCOUNT_ID}":role/"${ROLE_NAME}" \ + --role-arn arn:"${PARTITION}":iam::"${ACCOUNT_ID}":role/"${ROLE_NAME}" \ --query Credentials \ --output json) AWS_ACCESS_KEY_ID=$(echo "${SESSION_DATA}" | jq -r ".AccessKeyId") diff --git a/templates/api.yaml b/templates/api.yaml index 803c6b63..dc669eeb 100644 --- a/templates/api.yaml +++ b/templates/api.yaml @@ -1266,10 +1266,10 @@ Outputs: - !Ref WebUIOrigin - !Ref AccessControlAllowOriginOverride ApiArn: - Value: !Sub arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${Api}/${Api.Stage}/*/* + Value: !Sub arn:${AWS::Partition}:execute-api:${AWS::Region}:${AWS::AccountId}:${Api}/${Api.Stage}/*/* ApiUrl: Description: API endpoint URL for Prod environment - Value: !Sub https://${Api}.execute-api.${AWS::Region}.amazonaws.com/${Api.Stage}/ + Value: !Sub https://${Api}.execute-api.${AWS::Region}.${AWS::URLSuffix}/${Api.Stage}/ PutDataMapperRole: Description: Role used by the PutDataMapper API Value: !Ref PutDataMapperRole diff --git a/templates/auth.yaml b/templates/auth.yaml index 60ce2f3f..8803f74f 100644 --- a/templates/auth.yaml +++ b/templates/auth.yaml @@ -59,7 +59,7 @@ Resources: Type: AWS::IAM::Role Properties: ManagedPolicyArns: - - arn:aws:iam::aws:policy/AmazonAPIGatewayInvokeFullAccess + - !Sub arn:${AWS::Partition}:iam::aws:policy/AmazonAPIGatewayInvokeFullAccess AssumeRolePolicyDocument: Statement: - Effect: Allow diff --git a/templates/deletion_flow.yaml b/templates/deletion_flow.yaml index 816346e0..ff8ac3c1 100644 --- a/templates/deletion_flow.yaml +++ b/templates/deletion_flow.yaml @@ -81,7 +81,7 @@ Resources: - sts:AssumeRole Path: / ManagedPolicyArns: - - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy DelObjQ: Type: AWS::SQS::Queue @@ -128,7 +128,7 @@ Resources: ContainerDefinitions: - Name: !Sub ${ResourcePrefix}_DeleteTask Essential: true - Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ECRRepository}:latest + Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/${ECRRepository}:latest LogConfiguration: LogDriver: awslogs Options: @@ -138,6 +138,8 @@ Resources: Environment: - Name: AWS_STS_REGIONAL_ENDPOINTS Value: regional + - Name: AWS_URL_SUFFIX + Value: !Ref AWS::URLSuffix - Name: DELETE_OBJECTS_QUEUE Value: !Ref DelObjQ - Name: ECS_ENABLE_CONTAINER_METADATA @@ -185,13 +187,13 @@ Resources: - logs:CreateLogStream - logs:PutLogEvents Effect: Allow - Resource: "arn:aws:logs:*:*:*" + Resource: !Sub "arn:${AWS::Partition}:logs:*:*:*" - Action: sts:AssumeRole Effect: Allow Resource: !Sub "arn:${AWS::Partition}:iam::*:role/S3F2DataAccessRole" - Action: s3:GetObject* Effect: Allow - Resource: !Sub arn:aws:s3:::${ManifestsBucket}/manifests/* + Resource: !Sub arn:${AWS::Partition}:s3:::${ManifestsBucket}/manifests/* - !If - WithKMS - Action: diff --git a/templates/deployment_helper.yaml b/templates/deployment_helper.yaml index 406d06a6..27196fc8 100644 --- a/templates/deployment_helper.yaml +++ b/templates/deployment_helper.yaml @@ -78,17 +78,17 @@ Resources: Action: - logs:CreateLogGroup - ecr:GetAuthorizationToken - - Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:* + - Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:* Effect: Allow Action: - logs:CreateLogStream - logs:PutLogEvents - - Resource: !Sub arn:aws:s3:::${CodeBuildArtefactBucket}/* + - Resource: !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}/* Effect: Allow Action: - s3:Get* - s3:List* - - Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository} + - Resource: !Sub arn:${AWS::Partition}:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository} Effect: Allow Action: - ecr:GetDownloadUrlForLayer @@ -116,26 +116,26 @@ Resources: - Resource: "*" Effect: Allow Action: logs:CreateLogGroup - - Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:* + - Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:* Effect: Allow Action: - logs:CreateLogStream - logs:PutLogEvents - - Resource: !Sub arn:aws:s3:::${CodeBuildArtefactBucket}/* + - Resource: !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}/* Effect: Allow Action: - s3:Get* - s3:List* - !If - ShouldDeployWebUI - - Resource: !Sub arn:aws:s3:::${WebUIBucket}/* + - Resource: !Sub arn:${AWS::Partition}:s3:::${WebUIBucket}/* Effect: Allow Action: s3:PutObject* - !Ref AWS::NoValue - !If - WithoutCloudFront - !Ref AWS::NoValue - - Resource: !Sub arn:aws:cloudfront::${AWS::AccountId}:distribution/${CloudFrontDistribution} + - Resource: !Sub arn:${AWS::Partition}:cloudfront::${AWS::AccountId}:distribution/${CloudFrontDistribution} Effect: Allow Action: cloudfront:CreateInvalidation @@ -169,7 +169,7 @@ Resources: - Name: AWS_DEFAULT_REGION Value: !Ref AWS::Region - Name: REPOSITORY_URI - Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ECRRepository} + Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/${ECRRepository} Name: !Sub ${ResourcePrefix}BackendBuild ServiceRole: !Ref CodeBuildBackendServiceRole @@ -256,7 +256,7 @@ Resources: - ecs:UpdateService - Effect: Allow Resource: - - !Sub arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${CodeBuildBackend} + - !Sub arn:${AWS::Partition}:codebuild:${AWS::Region}:${AWS::AccountId}:project/${CodeBuildBackend} Action: - codebuild:BatchGetBuilds - codebuild:StartBuild @@ -264,18 +264,18 @@ Resources: - ShouldDeployWebUI - Effect: Allow Resource: - - !Sub arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${CodeBuildFrontend} + - !Sub arn:${AWS::Partition}:codebuild:${AWS::Region}:${AWS::AccountId}:project/${CodeBuildFrontend} Action: - codebuild:BatchGetBuilds - codebuild:StartBuild - !Ref AWS::NoValue - Effect: Allow - Resource: !Sub arn:aws:s3:::${CodeBuildArtefactBucket} + Resource: !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket} Action: - s3:GetBucket* - s3:List* - Effect: Allow - Resource: !Sub arn:aws:s3:::${CodeBuildArtefactBucket}/* + Resource: !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}/* Action: - s3:GetObject - s3:GetObjectVersion @@ -347,10 +347,10 @@ Resources: - Statement: - Effect: Allow Action: s3:PutObject* - Resource: !Sub arn:aws:s3:::${CodeBuildArtefactBucket}/* + Resource: !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}/* - Effect: Allow Action: s3:GetObject - Resource: !Sub arn:aws:s3:::${PreBuiltArtefactsBucket}/* + Resource: !Sub arn:${AWS::Partition}:s3:::${PreBuiltArtefactsBucket}/* CleanupBucketFunction: Type: AWS::Serverless::Function @@ -367,10 +367,10 @@ Resources: - s3:ListBucket* - s3:ListObject* Resource: - - !Sub arn:aws:s3:::${CodeBuildArtefactBucket} - - !Sub arn:aws:s3:::${CodeBuildArtefactBucket}/* - - !Sub arn:aws:s3:::${WebUIBucket} - - !Sub arn:aws:s3:::${WebUIBucket}/* + - !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket} + - !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}/* + - !Sub arn:${AWS::Partition}:s3:::${WebUIBucket} + - !Sub arn:${AWS::Partition}:s3:::${WebUIBucket}/* CleanupRepositoryFunction: Type: AWS::Serverless::Function @@ -384,7 +384,7 @@ Resources: Action: - ecr:BatchDeleteImage - ecr:ListImages - Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository} + Resource: !Sub arn:${AWS::Partition}:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository} WaitForContainerBuildFunction: Type: AWS::Serverless::Function @@ -405,10 +405,10 @@ Resources: Resource: "*" - Effect: Allow Action: s3:GetObject* - Resource: !Sub "arn:aws:s3:::${CodeBuildArtefactBucket}/*" + Resource: !Sub "arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket}/*" - Effect: Allow Action: ecr:DescribeImages - Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository} + Resource: !Sub arn:${AWS::Partition}:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository} RedeployApiFunction: @@ -424,7 +424,7 @@ Resources: - apigateway:POST Resource: - !Sub - - arn:aws:apigateway:${AWS::Region}::/restapis/${ApiId}/deployments + - arn:${AWS::Partition}:apigateway:${AWS::Region}::/restapis/${ApiId}/deployments - ApiId: !Select [0, !Split [".", !Select [2, !Split ["/", !Ref ApiUrl]]]] @@ -440,7 +440,7 @@ Resources: Action: - codepipeline:StartPipelineExecution Resource: - - !Sub arn:aws:codepipeline:${AWS::Region}:${AWS::AccountId}:${CodePipeline} + - !Sub arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:${CodePipeline} CleanupCodeBuildArtefactBucket: @@ -472,6 +472,7 @@ Resources: ServiceToken: !GetAtt CopyBuildArtefactFunction.Arn ArtefactName: !Ref ArtefactName CodeBuildArtefactBucket: !Ref CodeBuildArtefactBucket + CodeBuildArtefactBucketArn: !Sub arn:${AWS::Partition}:s3:::${CodeBuildArtefactBucket} LogLevel: !Ref LogLevel Region: !Ref AWS::Region PreBuiltArtefactsBucket: !Ref PreBuiltArtefactsBucket diff --git a/templates/manifests.yaml b/templates/manifests.yaml index 9b09eaa4..ff50f6ed 100644 --- a/templates/manifests.yaml +++ b/templates/manifests.yaml @@ -41,8 +41,8 @@ Resources: Action: '*' Effect: Deny Resource: - - !Sub arn:aws:s3:::${ManifestsBucket} - - !Sub arn:aws:s3:::${ManifestsBucket}/* + - !Sub arn:${AWS::Partition}:s3:::${ManifestsBucket} + - !Sub arn:${AWS::Partition}:s3:::${ManifestsBucket}/* Principal: '*' Condition: Bool: diff --git a/templates/state_machine.yaml b/templates/state_machine.yaml index 75246afb..ca41265f 100644 --- a/templates/state_machine.yaml +++ b/templates/state_machine.yaml @@ -100,7 +100,7 @@ Resources: - "events:PutTargets" - "events:PutRule" - "events:DescribeRule" - Resource: !Sub arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/* + Resource: !Sub arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:rule/* - Effect: Allow Action: "states:ListStateMachines" Resource: "*" @@ -644,7 +644,7 @@ Resources: }, "Start Fargate Workflow": { "Type":"Task", - "Resource":"arn:aws:states:::states:startExecution.sync", + "Resource":"arn:${AWS::Partition}:states:::states:startExecution.sync", "Parameters":{ "Input":{ "AWS_STEP_FUNCTIONS_STARTED_BY_EXECUTION_ID.$": "$$.Execution.Id", @@ -796,7 +796,7 @@ Resources: Condition: ForAnyValue:StringEquals: aws:CalledVia: - - athena.amazonaws.com + - !Sub athena.${AWS::URLSuffix} - Action: - "kms:Encrypt" - "kms:Decrypt" @@ -808,7 +808,7 @@ Resources: Condition: ForAnyValue:StringEquals: aws:CalledVia: - - athena.amazonaws.com + - !Sub athena.${AWS::URLSuffix} CheckQueryStatus: Type: AWS::Serverless::Function @@ -1007,7 +1007,7 @@ Resources: Properties: FunctionName: !Ref EmitEvent Action: "lambda:InvokeFunction" - Principal: "events.amazonaws.com" + Principal: !Sub "events.${AWS::URLSuffix}" SourceArn: !GetAtt EventRule.Arn Outputs: diff --git a/templates/stream_processor.yaml b/templates/stream_processor.yaml index dda1e319..95becfec 100644 --- a/templates/stream_processor.yaml +++ b/templates/stream_processor.yaml @@ -82,7 +82,7 @@ Resources: Resource: !Ref StateMachineArn - Effect: Allow Action: s3:GetObject* - Resource: !Sub arn:aws:s3:::${ManifestsBucket}/manifests/* + Resource: !Sub arn:${AWS::Partition}:s3:::${ManifestsBucket}/manifests/* - Effect: Allow Action: glue:BatchDeletePartition Resource: diff --git a/templates/template.yaml b/templates/template.yaml index 6c6bc70c..7ba8585c 100644 --- a/templates/template.yaml +++ b/templates/template.yaml @@ -1,6 +1,6 @@ AWSTemplateFormatVersion: "2010-09-09" Transform: AWS::Serverless-2016-10-31 -Description: Amazon S3 Find and Forget (uksb-1q2j8beb0) (version:v0.51) +Description: Amazon S3 Find and Forget (uksb-1q2j8beb0) (version:v0.52) Parameters: AccessControlAllowOriginOverride: @@ -159,6 +159,32 @@ Rules: - AssertDescription: IAM Auth cannot be chosen when deploying the WebUI. Assert: !Equals [!Ref AuthMethod, "Cognito"] + ValidateRegion: + RuleCondition: !Not + - !Contains + - - ap-northeast-1 + - ap-northeast-2 + - ap-south-1 + - ap-southeast-1 + - ap-southeast-2 + - ca-central-1 + - eu-central-1 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - me-south-1 + - sa-east-1 + - us-east-1 + - us-east-2 + - us-gov-west-1 + - us-west-1 + - us-west-2 + - !Ref AWS::Region + Assertions: + - AssertDescription: Cognito is not supported in this region please select IAM authentication. + Assert: !Not [!Equals [!Ref AuthMethod, "Cognito"]] + Conditions: DefaultPreBuiltArtefactsBucket: !Equals [!Ref PreBuiltArtefactsBucketOverride, "false"] ShouldDeployVpc: !Equals [!Ref DeployVpc, "true"] @@ -168,7 +194,7 @@ Conditions: Mappings: Solution: Constants: - Version: 'v0.51' + Version: 'v0.52' Resources: TempBucket: @@ -197,8 +223,8 @@ Resources: Action: '*' Effect: Deny Resource: - - !Sub arn:aws:s3:::${TempBucket} - - !Sub arn:aws:s3:::${TempBucket}/* + - !Sub arn:${AWS::Partition}:s3:::${TempBucket} + - !Sub arn:${AWS::Partition}:s3:::${TempBucket}/* Principal: '*' Condition: Bool: @@ -383,6 +409,12 @@ Resources: FlowLogsGroup: !Ref FlowLogsGroup FlowLogsRoleArn: !Ref FlowLogsRoleArn KMSKeyArns: !Ref KMSKeyArns + CommonLayers: !Join + - "," + - - !GetAtt LayersStack.Outputs.AWSSDKLayer + - !GetAtt LayersStack.Outputs.BotoUtils + - !GetAtt LayersStack.Outputs.CustomResourceHelper + - !GetAtt LayersStack.Outputs.Decorators WebUIStack: Type: AWS::CloudFormation::Stack Properties: diff --git a/templates/vpc.yaml b/templates/vpc.yaml index 3e56d07c..d8427625 100644 --- a/templates/vpc.yaml +++ b/templates/vpc.yaml @@ -1,6 +1,13 @@ AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 Description: Amazon S3 Find and Forget VPC (uksb-1qjminsd5) +Globals: + Function: + Runtime: python3.9 + Timeout: 30 + Layers: !Ref CommonLayers + Parameters: FlowLogsGroup: Type: String @@ -21,6 +28,8 @@ Parameters: Type: String Default: 10.0.0.0/16 AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(1[6-9]|2[0-8]))$ + CommonLayers: + Type: CommaDelimitedList Mappings: Regions: @@ -56,6 +65,8 @@ Mappings: HasThreeAZs: false us-west-2: HasThreeAZs: true + cn-north-1: + HasThreeAZs: true Conditions: HasThreeAZs: !Equals [!FindInMap [Regions, !Ref "AWS::Region", HasThreeAZs], true] @@ -63,6 +74,7 @@ Conditions: - !Not [!Equals [!Ref FlowLogsGroup, ""]] - !Not [!Equals [!Ref FlowLogsRoleArn, ""]] WithKMS: !Not [!Equals [!Ref KMSKeyArns, ""]] + ChinaRegion: !Equals [!Select [0, !Split ["-", !Ref "AWS::Region"]], "cn"] Resources: VPC: @@ -213,58 +225,136 @@ Resources: ToPort: 443 CidrIp: !Ref VpcIpBlock + GetEndpointSubnetFunction: + Type: AWS::Serverless::Function + Properties: + Handler: get_vpce_subnets.handler + CodeUri: ../backend/lambdas/custom_resources/ + Description: Custom Lambda resource for the Amazon S3 Find and Forget Cloudformation Stack + Policies: + - Statement: + - Effect: Allow + Action: + - ec2:DescribeSubnets + - ec2:DescribeVpcEndpointServices + Resource: '*' + # Endpoints - CloudWatchEndpoint: - Type: AWS::EC2::VPCEndpoint + CloudWatchEndpointSubnets: + Type: Custom::Setup + Condition: ChinaRegion Properties: - PrivateDnsEnabled: true - SecurityGroupIds: [!Ref SecurityGroup] + ServiceToken: !GetAtt GetEndpointSubnetFunction.Arn ServiceName: !Sub 'com.amazonaws.${AWS::Region}.monitoring' SubnetIds: - !Ref PrivateSubnet1 - !Ref PrivateSubnet2 - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] VpcEndpointType: Interface - VpcId: !Ref VPC - - CloudWatchLogsEndpoint: + CloudWatchEndpoint: Type: AWS::EC2::VPCEndpoint Properties: PrivateDnsEnabled: true SecurityGroupIds: [!Ref SecurityGroup] + ServiceName: !If + - ChinaRegion + - !GetAtt CloudWatchEndpointSubnets.ServiceName + - !Sub 'com.amazonaws.${AWS::Region}.monitoring' + SubnetIds: !If + - ChinaRegion + - !Split [',', !Ref CloudWatchEndpointSubnets] + - - !Ref PrivateSubnet1 + - !Ref PrivateSubnet2 + - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] + VpcEndpointType: Interface + VpcId: !Ref VPC + + CloudWatchLogsEndpointSubnets: + Type: Custom::Setup + Condition: ChinaRegion + Properties: + ServiceToken: !GetAtt GetEndpointSubnetFunction.Arn ServiceName: !Sub 'com.amazonaws.${AWS::Region}.logs' SubnetIds: - !Ref PrivateSubnet1 - !Ref PrivateSubnet2 - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] VpcEndpointType: Interface - VpcId: !Ref VPC - - ECREndpoint: + CloudWatchLogsEndpoint: Type: AWS::EC2::VPCEndpoint Properties: PrivateDnsEnabled: true SecurityGroupIds: [!Ref SecurityGroup] + ServiceName: !If + - ChinaRegion + - !GetAtt CloudWatchLogsEndpointSubnets.ServiceName + - !Sub 'com.amazonaws.${AWS::Region}.logs' + SubnetIds: !If + - ChinaRegion + - !Split [',', !Ref CloudWatchLogsEndpointSubnets] + - - !Ref PrivateSubnet1 + - !Ref PrivateSubnet2 + - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] + VpcEndpointType: Interface + VpcId: !Ref VPC + + ECREndpointSubnets: + Type: Custom::Setup + Condition: ChinaRegion + Properties: + ServiceToken: !GetAtt GetEndpointSubnetFunction.Arn ServiceName: !Sub 'com.amazonaws.${AWS::Region}.ecr.dkr' SubnetIds: - !Ref PrivateSubnet1 - !Ref PrivateSubnet2 - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] VpcEndpointType: Interface - VpcId: !Ref VPC - - ECRAPIEndpoint: + ECREndpoint: Type: AWS::EC2::VPCEndpoint Properties: PrivateDnsEnabled: true SecurityGroupIds: [!Ref SecurityGroup] + ServiceName: !If + - ChinaRegion + - !GetAtt ECREndpointSubnets.ServiceName + - !Sub 'com.amazonaws.${AWS::Region}.ecr.dkr' + SubnetIds: !If + - ChinaRegion + - !Split [',', !Ref ECREndpointSubnets] + - - !Ref PrivateSubnet1 + - !Ref PrivateSubnet2 + - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] + VpcEndpointType: Interface + VpcId: !Ref VPC + + ECRAPIEndpointSubnets: + Type: Custom::Setup + Condition: ChinaRegion + Properties: + ServiceToken: !GetAtt GetEndpointSubnetFunction.Arn ServiceName: !Sub 'com.amazonaws.${AWS::Region}.ecr.api' SubnetIds: - !Ref PrivateSubnet1 - !Ref PrivateSubnet2 - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] VpcEndpointType: Interface + ECRAPIEndpoint: + Type: AWS::EC2::VPCEndpoint + Properties: + PrivateDnsEnabled: true + SecurityGroupIds: [!Ref SecurityGroup] + ServiceName: !If + - ChinaRegion + - !GetAtt ECRAPIEndpointSubnets.ServiceName + - !Sub 'com.amazonaws.${AWS::Region}.ecr.api' + SubnetIds: !If + - ChinaRegion + - !Split [',', !Ref ECRAPIEndpointSubnets] + - - !Ref PrivateSubnet1 + - !Ref PrivateSubnet2 + - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] + VpcEndpointType: Interface VpcId: !Ref VPC S3Endpoint: @@ -276,18 +366,34 @@ Resources: - !If [HasThreeAZs, !Ref PrivateRouteTable3, !Ref 'AWS::NoValue'] ServiceName: !Sub 'com.amazonaws.${AWS::Region}.s3' VpcId: !Ref VPC - - SQSEndpoint: - Type: AWS::EC2::VPCEndpoint + + SQSEndpointSubnets: + Type: Custom::Setup + Condition: ChinaRegion Properties: - PrivateDnsEnabled: true - SecurityGroupIds: [!Ref SecurityGroup] + ServiceToken: !GetAtt GetEndpointSubnetFunction.Arn ServiceName: !Sub 'com.amazonaws.${AWS::Region}.sqs' SubnetIds: - !Ref PrivateSubnet1 - !Ref PrivateSubnet2 - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] VpcEndpointType: Interface + SQSEndpoint: + Type: AWS::EC2::VPCEndpoint + Properties: + PrivateDnsEnabled: true + SecurityGroupIds: [!Ref SecurityGroup] + ServiceName: !If + - ChinaRegion + - !GetAtt SQSEndpointSubnets.ServiceName + - !Sub 'com.amazonaws.${AWS::Region}.sqs' + SubnetIds: !If + - ChinaRegion + - !Split [',', !Ref SQSEndpointSubnets] + - - !Ref PrivateSubnet1 + - !Ref PrivateSubnet2 + - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] + VpcEndpointType: Interface VpcId: !Ref VPC DynamoEndpoint: @@ -300,31 +406,63 @@ Resources: ServiceName: !Sub 'com.amazonaws.${AWS::Region}.dynamodb' VpcId: !Ref VPC - STSEndpoint: - Type: AWS::EC2::VPCEndpoint + STSEndpointSubnets: + Type: Custom::Setup + Condition: ChinaRegion Properties: - PrivateDnsEnabled: true - SecurityGroupIds: [!Ref SecurityGroup] + ServiceToken: !GetAtt GetEndpointSubnetFunction.Arn ServiceName: !Sub 'com.amazonaws.${AWS::Region}.sts' SubnetIds: - !Ref PrivateSubnet1 - !Ref PrivateSubnet2 - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] VpcEndpointType: Interface - VpcId: !Ref VPC - - KMSEndpoint: + STSEndpoint: Type: AWS::EC2::VPCEndpoint - Condition: WithKMS Properties: PrivateDnsEnabled: true SecurityGroupIds: [!Ref SecurityGroup] + ServiceName: !If + - ChinaRegion + - !GetAtt STSEndpointSubnets.ServiceName + - !Sub 'com.amazonaws.${AWS::Region}.sts' + SubnetIds: !If + - ChinaRegion + - !Split [',', !Ref STSEndpointSubnets] + - - !Ref PrivateSubnet1 + - !Ref PrivateSubnet2 + - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] + VpcEndpointType: Interface + VpcId: !Ref VPC + + KMSEndpointSubnets: + Type: Custom::Setup + Condition: ChinaRegion + Properties: + ServiceToken: !GetAtt GetEndpointSubnetFunction.Arn ServiceName: !Sub 'com.amazonaws.${AWS::Region}.kms' SubnetIds: - !Ref PrivateSubnet1 - !Ref PrivateSubnet2 - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] VpcEndpointType: Interface + KMSEndpoint: + Type: AWS::EC2::VPCEndpoint + Condition: WithKMS + Properties: + PrivateDnsEnabled: true + SecurityGroupIds: [!Ref SecurityGroup] + ServiceName: !If + - ChinaRegion + - !GetAtt KMSEndpointSubnets.ServiceName + - !Sub 'com.amazonaws.${AWS::Region}.kms' + SubnetIds: !If + - ChinaRegion + - !Split [',', !Ref KMSEndpointSubnets] + - - !Ref PrivateSubnet1 + - !Ref PrivateSubnet2 + - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] + VpcEndpointType: Interface VpcId: !Ref VPC Outputs: diff --git a/templates/web_ui.yaml b/templates/web_ui.yaml index 41decf9a..c269fd34 100644 --- a/templates/web_ui.yaml +++ b/templates/web_ui.yaml @@ -54,8 +54,8 @@ Resources: Action: '*' Effect: Deny Resource: - - !Sub arn:aws:s3:::${WebUIBucket} - - !Sub arn:aws:s3:::${WebUIBucket}/* + - !Sub arn:${AWS::Partition}:s3:::${WebUIBucket} + - !Sub arn:${AWS::Partition}:s3:::${WebUIBucket}/* Principal: '*' Condition: Bool: @@ -65,7 +65,7 @@ Resources: - Sid: CloudFrontOriginOnly Action: s3:GetObject Effect: Allow - Resource: !Sub arn:aws:s3:::${WebUIBucket}/* + Resource: !Sub arn:${AWS::Partition}:s3:::${WebUIBucket}/* Principal: CanonicalUser: !GetAtt CloudFrontOriginAccessIdentity.S3CanonicalUserId - !Ref AWS::NoValue @@ -107,7 +107,7 @@ Resources: CloudFrontDefaultCertificate: true Logging: !If - WithAccessLogs - - Bucket: !Sub ${AccessLogsBucket}.s3.amazonaws.com + - Bucket: !Sub ${AccessLogsBucket}.s3.${AWS::URLSuffix} IncludeCookies: false Prefix: !Sub ${ResourcePrefix}/ - !Ref AWS::NoValue diff --git a/tests/acceptance/conftest.py b/tests/acceptance/conftest.py index 99120936..2c812853 100644 --- a/tests/acceptance/conftest.py +++ b/tests/acceptance/conftest.py @@ -103,6 +103,13 @@ def s3_resource(): return boto3.resource("s3") +@pytest.fixture(scope="session") +def arn_partition(): + return boto3.session.Session().get_partition_for_region( + getenv("AWS_DEFAULT_REGION", "eu-west-1") + ) + + @pytest.fixture(scope="session") def sf_client(): return boto3.client("stepfunctions") @@ -584,7 +591,7 @@ def empty_lake(dummy_lake): @pytest.fixture(scope="session") -def dummy_lake(s3_resource, stack, data_access_role): +def dummy_lake(s3_resource, stack, data_access_role, arn_partition): # Lake Config bucket_name = "test-" + str(uuid4()) # Create the bucket and Glue table @@ -610,8 +617,8 @@ def dummy_lake(s3_resource, stack, data_access_role): "Principal": {"AWS": roles}, "Action": "s3:*", "Resource": [ - "arn:aws:s3:::{}".format(bucket_name), - "arn:aws:s3:::{}/*".format(bucket_name), + "arn:{}:s3:::{}".format(arn_partition, bucket_name), + "arn:{}:s3:::{}/*".format(arn_partition, bucket_name), ], } ], diff --git a/tests/acceptance/test_job_cognito.py b/tests/acceptance/test_job_cognito.py index d96e0e07..e870047f 100644 --- a/tests/acceptance/test_job_cognito.py +++ b/tests/acceptance/test_job_cognito.py @@ -1040,6 +1040,7 @@ def test_it_handles_find_permission_issues( job_table, policy_changer, stack, + arn_partition, ): # Arrange glue_data_mapper_factory( @@ -1061,8 +1062,8 @@ def test_it_handles_find_permission_issues( "Principal": {"AWS": [stack["AthenaExecutionRoleArn"]]}, "Action": "s3:*", "Resource": [ - "arn:aws:s3:::{}".format(bucket_name), - "arn:aws:s3:::{}/*".format(bucket_name), + "arn:{}:s3:::{}".format(arn_partition, bucket_name), + "arn:{}:s3:::{}/*".format(arn_partition, bucket_name), ], } ], @@ -1091,6 +1092,7 @@ def test_it_handles_forget_permission_issues( job_table, policy_changer, stack, + arn_partition, ): # Arrange glue_data_mapper_factory( @@ -1110,8 +1112,8 @@ def test_it_handles_forget_permission_issues( "Principal": {"AWS": [stack["DeleteTaskRoleArn"]]}, "Action": "s3:*", "Resource": [ - "arn:aws:s3:::{}".format(bucket_name), - "arn:aws:s3:::{}/*".format(bucket_name), + "arn:{}:s3:::{}".format(arn_partition, bucket_name), + "arn:{}:s3:::{}/*".format(arn_partition, bucket_name), ], } ) @@ -1137,13 +1139,14 @@ def test_it_handles_forget_invalid_role( data_loader, job_finished_waiter, job_table, + arn_partition, ): # Arrange glue_data_mapper_factory( "test", partition_keys=["year", "month", "day"], partitions=[["2019", "08", "20"]], - role_arn="arn:aws:iam::invalid:role/DoesntExist", + role_arn="arn:{}:iam::invalid:role/DoesntExist".format(arn_partition), ) item = del_queue_factory("12345") object_key = "test/2019/08/20/test.parquet" diff --git a/tests/acceptance/test_job_iam.py b/tests/acceptance/test_job_iam.py index d272c5d0..44881d75 100644 --- a/tests/acceptance/test_job_iam.py +++ b/tests/acceptance/test_job_iam.py @@ -1030,6 +1030,7 @@ def test_it_handles_find_permission_issues( job_table, policy_changer, stack, + arn_partition, ): # Arrange glue_data_mapper_factory( @@ -1051,8 +1052,8 @@ def test_it_handles_find_permission_issues( "Principal": {"AWS": [stack["AthenaExecutionRoleArn"]]}, "Action": "s3:*", "Resource": [ - "arn:aws:s3:::{}".format(bucket_name), - "arn:aws:s3:::{}/*".format(bucket_name), + "arn:{}:s3:::{}".format(arn_partition, bucket_name), + "arn:{}:s3:::{}/*".format(arn_partition, bucket_name), ], } ], @@ -1081,6 +1082,7 @@ def test_it_handles_forget_permission_issues( job_table, policy_changer, stack, + arn_partition, ): # Arrange glue_data_mapper_factory( @@ -1100,8 +1102,8 @@ def test_it_handles_forget_permission_issues( "Principal": {"AWS": [stack["DeleteTaskRoleArn"]]}, "Action": "s3:*", "Resource": [ - "arn:aws:s3:::{}".format(bucket_name), - "arn:aws:s3:::{}/*".format(bucket_name), + "arn:{}:s3:::{}".format(arn_partition, bucket_name), + "arn:{}:s3:::{}/*".format(arn_partition, bucket_name), ], } ) @@ -1127,13 +1129,14 @@ def test_it_handles_forget_invalid_role( data_loader, job_finished_waiter, job_table, + arn_partition, ): # Arrange glue_data_mapper_factory( "test", partition_keys=["year", "month", "day"], partitions=[["2019", "08", "20"]], - role_arn="arn:aws:iam::invalid:role/DoesntExist", + role_arn="arn:{}:iam::invalid:role/DoesntExist".format(arn_partition), ) item = del_queue_factory("12345") object_key = "test/2019/08/20/test.parquet" diff --git a/tests/unit/crs/test_cr_copy_build_artefact.py b/tests/unit/crs/test_cr_copy_build_artefact.py index b8cbbafa..b8301ba7 100644 --- a/tests/unit/crs/test_cr_copy_build_artefact.py +++ b/tests/unit/crs/test_cr_copy_build_artefact.py @@ -12,6 +12,7 @@ def test_it_copies_file(mock_client): "ResourceProperties": { "ArtefactName": "build/s3f2.zip", "CodeBuildArtefactBucket": "codebuild-bucket", + "CodeBuildArtefactBucketArn": "arn:aws:s3:::codebuild-bucket", "PreBuiltArtefactsBucket": "source-bucket-eu-west-1", "Version": "1.0", } diff --git a/tests/unit/crs/test_cr_get_vpce_subnets.py b/tests/unit/crs/test_cr_get_vpce_subnets.py new file mode 100644 index 00000000..344cf7c9 --- /dev/null +++ b/tests/unit/crs/test_cr_get_vpce_subnets.py @@ -0,0 +1,125 @@ +import pytest +from mock import patch, MagicMock + +from backend.lambdas.custom_resources.get_vpce_subnets import ( + create, + delete, + handler, +) + +pytestmark = [pytest.mark.unit, pytest.mark.task] + + +@patch("backend.lambdas.custom_resources.get_vpce_subnets.ec2_client") +def test_it_returns_valid_subnets(mock_client): + event = { + "ResourceProperties": { + "ServiceName": "com.amazonaws.eu-west-2.monitoring", + "SubnetIds": ["subnet-0123456789abcdef0", "subnet-0123456789abcdef1"], + "VpcEndpointType": "Interface", + } + } + + mock_client.describe_subnets.return_value = { + "Subnets": [ + { + "AvailabilityZone": "eu-west-2a", + "SubnetId": "subnet-0123456789abcdef0", + }, + { + "AvailabilityZone": "eu-west-2b", + "SubnetId": "subnet-0123456789abcdef1", + }, + ] + } + + mock_client.describe_vpc_endpoint_services.return_value = { + "ServiceDetails": [ + { + "ServiceName": "com.amazonaws.eu-west-2.monitoring", + "AvailabilityZones": [ + "eu-west-2a", + "eu-west-2b", + ], + } + ] + } + + resp = create(event, MagicMock()) + + mock_client.describe_subnets.assert_called_with( + SubnetIds=["subnet-0123456789abcdef0", "subnet-0123456789abcdef1"] + ) + mock_client.describe_vpc_endpoint_services.assert_called_with( + Filters=[ + { + "Name": "service-name", + "Values": [ + "cn.com.amazonaws.eu-west-2.monitoring", + "com.amazonaws.eu-west-2.monitoring", + ], + }, + {"Name": "service-type", "Values": ["Interface"]}, + ] + ) + + assert resp == "subnet-0123456789abcdef0,subnet-0123456789abcdef1" + + +@patch("backend.lambdas.custom_resources.get_vpce_subnets.ec2_client") +def test_it_raises_exception(mock_client): + event = { + "ResourceProperties": { + "ServiceName": "com.amazonaws.eu-west-2.dummy", + "SubnetIds": [], + "VpcEndpointType": "Interface", + } + } + + mock_client.describe_subnets.return_value = { + "Subnets": [ + { + "AvailabilityZone": "eu-west-2a", + "SubnetId": "subnet-0123456789abcdef0", + }, + { + "AvailabilityZone": "eu-west-2b", + "SubnetId": "subnet-0123456789abcdef1", + }, + ] + } + + mock_client.describe_vpc_endpoint_services.return_value = {"ServiceDetails": []} + + with pytest.raises(Exception) as e_info: + create(event, MagicMock()) + + mock_client.describe_subnets.assert_called_with(SubnetIds=[]) + mock_client.describe_vpc_endpoint_services.assert_called_with( + Filters=[ + { + "Name": "service-name", + "Values": [ + "cn.com.amazonaws.eu-west-2.dummy", + "com.amazonaws.eu-west-2.dummy", + ], + }, + {"Name": "service-type", "Values": ["Interface"]}, + ] + ) + + assert e_info.typename == "IndexError" + + +@patch("backend.lambdas.custom_resources.get_vpce_subnets.ec2_client") +def test_it_does_nothing_on_delete(mock_client): + resp = delete({}, MagicMock()) + + mock_client.assert_not_called() + assert not resp + + +@patch("backend.lambdas.custom_resources.get_vpce_subnets.helper") +def test_it_delegates_to_cr_helper(cr_helper): + handler(1, 2) + cr_helper.assert_called_with(1, 2) diff --git a/tests/unit/ecs_tasks/test_main.py b/tests/unit/ecs_tasks/test_main.py index 394d3cb1..6d40fd58 100644 --- a/tests/unit/ecs_tasks/test_main.py +++ b/tests/unit/ecs_tasks/test_main.py @@ -934,6 +934,7 @@ def test_it_inits_arg_parser_with_defaults(): @patch("backend.ecs_tasks.delete_files.main.boto3") @patch.dict(os.environ, {"AWS_DEFAULT_REGION": "eu-west-2"}) +@patch.dict(os.environ, {"AWS_URL_SUFFIX": "amazonaws.com"}) def test_it_inits_queue_with_regional_url(mock_boto): get_queue("https://queue/rule") mock_boto.resource.assert_called_with(