From e76479ef72666f99322aaa876f287a25d4c35e06 Mon Sep 17 00:00:00 2001 From: Noah Paige Date: Mon, 9 Sep 2024 23:11:30 -0400 Subject: [PATCH 1/3] add option to configure log retention --- .../modules/mlstudio/cdk/mlstudio_extension.py | 3 ++- deploy/stacks/albfront_stack.py | 3 ++- deploy/stacks/albfront_stage.py | 2 ++ deploy/stacks/backend_stack.py | 4 ++++ deploy/stacks/backend_stage.py | 2 ++ deploy/stacks/container.py | 5 ++++- deploy/stacks/lambda_api.py | 11 ++++++++--- deploy/stacks/opensearch.py | 2 ++ deploy/stacks/pipeline.py | 2 ++ deploy/stacks/vpc.py | 3 +++ template_cdk.json | 3 ++- 11 files changed, 33 insertions(+), 7 deletions(-) diff --git a/backend/dataall/modules/mlstudio/cdk/mlstudio_extension.py b/backend/dataall/modules/mlstudio/cdk/mlstudio_extension.py index 617ed4454..5d2bf330c 100644 --- a/backend/dataall/modules/mlstudio/cdk/mlstudio_extension.py +++ b/backend/dataall/modules/mlstudio/cdk/mlstudio_extension.py @@ -2,6 +2,7 @@ Extends an environment stack for SageMaker Studio Domain """ +import os import logging from aws_cdk import ( @@ -63,7 +64,7 @@ def extent(setup: EnvironmentSetup): setup, f'SageMakerStudio{_environment.name}', log_group_name=f'/{_environment.resourcePrefix}/{_environment.name}/vpc/sagemakerstudio', - retention=logs.RetentionDays.ONE_MONTH, + retention=getattr(logs.RetentionDays, os.environ.get('LOG_RETENTION', 'TWO_YEARS')), removal_policy=RemovalPolicy.DESTROY, ) vpc_flow_role = iam.Role( diff --git a/deploy/stacks/albfront_stack.py b/deploy/stacks/albfront_stack.py index 2caef4a13..0f6c340d5 100644 --- a/deploy/stacks/albfront_stack.py +++ b/deploy/stacks/albfront_stack.py @@ -32,6 +32,7 @@ def __init__( ip_ranges=None, custom_auth=None, backend_region=None, + log_retention_duration='TWO_YEARS', **kwargs, ): super().__init__(scope, id, **kwargs) @@ -321,8 +322,8 @@ def create_log_group(self, envname, resource_prefix, log_group_name): self, f'ECSLogGroup{log_group_name}{envname}', log_group_name=f'/{resource_prefix}/{envname}/ecs/{log_group_name}', - retention=logs.RetentionDays.ONE_MONTH, removal_policy=RemovalPolicy.DESTROY, + retention=getattr(logs.RetentionDays, self.log_retention_duration) ) return log_group diff --git a/deploy/stacks/albfront_stage.py b/deploy/stacks/albfront_stage.py index 53ecb8ee3..b19272eb4 100644 --- a/deploy/stacks/albfront_stage.py +++ b/deploy/stacks/albfront_stage.py @@ -18,6 +18,7 @@ def __init__( ip_ranges=None, custom_auth=None, backend_region=None, + log_retention_duration='TWO_YEARS', **kwargs, ): super().__init__(scope, id, **kwargs) @@ -33,6 +34,7 @@ def __init__( ip_ranges=ip_ranges, custom_auth=custom_auth, backend_region=backend_region, + log_retention_duration=log_retention_duration, ) Tags.of(albfront_stack).add('Application', f'{resource_prefix}-{envname}') diff --git a/deploy/stacks/backend_stack.py b/deploy/stacks/backend_stack.py index 0dd9f4350..bba5a9111 100644 --- a/deploy/stacks/backend_stack.py +++ b/deploy/stacks/backend_stack.py @@ -59,6 +59,7 @@ def __init__( custom_waf_rules=None, with_approval_tests=False, allowed_origins='*', + log_retention_duration='TWO_YEARS', **kwargs, ): super().__init__(scope, id, **kwargs) @@ -75,6 +76,7 @@ def __init__( vpc_endpoints_sg=vpc_endpoints_sg, vpc_id=vpc_id, restricted_nacl=vpc_restricted_nacls, + log_retention_duration=log_retention_duration, **kwargs, ) vpc = self.vpc_stack.vpc @@ -200,6 +202,7 @@ def __init__( custom_auth=custom_auth, custom_waf_rules=custom_waf_rules, allowed_origins=allowed_origins, + log_retention_duration=log_retention_duration, **kwargs, ) @@ -224,6 +227,7 @@ def __init__( email_custom_domain=ses_stack.ses_identity.email_identity_name if ses_stack is not None else None, ses_configuration_set=ses_stack.configuration_set.configuration_set_name if ses_stack is not None else None, custom_domain=custom_domain, + log_retention_duration=log_retention_duration, **kwargs, ) diff --git a/deploy/stacks/backend_stage.py b/deploy/stacks/backend_stage.py index dee8009bc..39a22cc58 100644 --- a/deploy/stacks/backend_stage.py +++ b/deploy/stacks/backend_stage.py @@ -37,6 +37,7 @@ def __init__( custom_waf_rules=None, with_approval_tests=False, allowed_origins='*', + log_retention_duration='TWO_YEARS', **kwargs, ): super().__init__(scope, id, **kwargs) @@ -71,6 +72,7 @@ def __init__( custom_waf_rules=custom_waf_rules, with_approval_tests=with_approval_tests, allowed_origins=allowed_origins, + log_retention_duration=log_retention_duration, **kwargs, ) diff --git a/deploy/stacks/container.py b/deploy/stacks/container.py index 5150e08b1..65e419b57 100644 --- a/deploy/stacks/container.py +++ b/deploy/stacks/container.py @@ -36,11 +36,13 @@ def __init__( email_custom_domain=None, ses_configuration_set=None, custom_domain=None, + log_retention_duration='TWO_YEARS', **kwargs, ): super().__init__(scope, id, **kwargs) self._envname = envname self._resource_prefix = resource_prefix + self.log_retention_duration = log_retention_duration if self.node.try_get_context('image_tag'): image_tag = self.node.try_get_context('image_tag') @@ -101,6 +103,7 @@ def __init__( ecs.CfnTaskDefinition.KeyValuePairProperty(name='envname', value=envname), ecs.CfnTaskDefinition.KeyValuePairProperty(name='LOGLEVEL', value='DEBUG'), ecs.CfnTaskDefinition.KeyValuePairProperty(name='config_location', value='/config.json'), + ecs.CfnTaskDefinition.KeyValuePairProperty(name='LOG_RETENTION', value=self.log_retention_duration), ], essential=True, log_configuration=ecs.CfnTaskDefinition.LogConfigurationProperty( @@ -713,8 +716,8 @@ def create_log_group(self, envname, resource_prefix, log_group_name): self, f'ECSLogGroup{log_group_name}{envname}', log_group_name=f'/{resource_prefix}/{envname}/ecs/{log_group_name}', - retention=logs.RetentionDays.ONE_MONTH, removal_policy=RemovalPolicy.DESTROY, + retention=getattr(logs.RetentionDays, self.log_retention_duration) ) return log_group diff --git a/deploy/stacks/lambda_api.py b/deploy/stacks/lambda_api.py index 738e28126..e95650115 100644 --- a/deploy/stacks/lambda_api.py +++ b/deploy/stacks/lambda_api.py @@ -58,10 +58,13 @@ def __init__( custom_domain=None, custom_auth=None, allowed_origins='*', + log_retention_duration='TWO_YEARS', **kwargs, ): super().__init__(scope, id, **kwargs) + self.log_retention_duration = log_retention_duration + if self.node.try_get_context('image_tag'): image_tag = self.node.try_get_context('image_tag') @@ -105,7 +108,7 @@ def __init__( 'ElasticSearchProxyHandler', function_name=f'{resource_prefix}-{envname}-esproxy', log_group=logs.LogGroup( - self, 'esproxyloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-esproxy' + self, 'esproxyloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-esproxy', retention=getattr(logs.RetentionDays, self.log_retention_duration) ), description='dataall es search function', role=self.create_function_role(envname, resource_prefix, 'esproxy', pivot_role_name, vpc), @@ -142,7 +145,7 @@ def __init__( 'LambdaGraphQL', function_name=f'{resource_prefix}-{envname}-graphql', log_group=logs.LogGroup( - self, 'graphqlloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-graphql' + self, 'graphqlloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-graphql', retention=getattr(logs.RetentionDays, self.log_retention_duration) ), description='dataall graphql function', role=self.create_function_role(envname, resource_prefix, 'graphql', pivot_role_name, vpc), @@ -173,7 +176,7 @@ def __init__( 'AWSWorker', function_name=f'{resource_prefix}-{envname}-awsworker', log_group=logs.LogGroup( - self, 'awsworkerloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-awsworker' + self, 'awsworkerloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-awsworker', retention=getattr(logs.RetentionDays, self.log_retention_duration) ), description='dataall aws worker for aws asynchronous tasks function', role=self.create_function_role(envname, resource_prefix, 'awsworker', pivot_role_name, vpc), @@ -245,6 +248,7 @@ def __init__( self, 'customauthorizerloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-custom-authorizer', + retention=getattr(logs.RetentionDays, self.log_retention_duration) ), handler='custom_authorizer_lambda.lambda_handler', code=_lambda.Code.from_asset( @@ -762,6 +766,7 @@ def set_up_graphql_api_gateway( f'{resource_prefix}/{envname}/apigateway', log_group_name=f'{resource_prefix}/{envname}/apigateway', removal_policy=RemovalPolicy.DESTROY, + retention=getattr(logs.RetentionDays, self.log_retention_duration), ) iam_policy = iam.PolicyDocument( diff --git a/deploy/stacks/opensearch.py b/deploy/stacks/opensearch.py index 09f1e3e33..160654113 100644 --- a/deploy/stacks/opensearch.py +++ b/deploy/stacks/opensearch.py @@ -38,6 +38,7 @@ def __init__( lambdas=None, ecs_security_groups: [ec2.SecurityGroup] = None, prod_sizing=False, + log_retention_duration='TWO_YEARS', **kwargs, ): super().__init__(scope, id) @@ -67,6 +68,7 @@ def __init__( id='EsAppLogGroup', log_group_name=f'/{resource_prefix}/{envname}/opensearch', removal_policy=RemovalPolicy.DESTROY, + retention=getattr(logs.RetentionDays, log_retention_duration), ) self.domain = opensearch.Domain( diff --git a/deploy/stacks/pipeline.py b/deploy/stacks/pipeline.py index 804792923..3cfcbead3 100644 --- a/deploy/stacks/pipeline.py +++ b/deploy/stacks/pipeline.py @@ -652,6 +652,7 @@ def set_backend_stage(self, target_env, repository_name): custom_waf_rules=target_env.get('custom_waf_rules', None), with_approval_tests=target_env.get('with_approval_tests', False), allowed_origins=target_env.get('allowed_origins', '*'), + log_retention_duration=target_env.get('log_retention_duration', 'TWO_YEARS') ) ) return backend_stage @@ -890,6 +891,7 @@ def set_albfront_stage(self, target_env, repository_name): resource_prefix=self.resource_prefix, custom_auth=target_env.get('custom_auth', None), backend_region=target_env.get('region', self.region), + log_retention_duration=target_env.get('log_retention_duration', 'TWO_YEARS'), ), pre=[ pipelines.CodeBuildStep( diff --git a/deploy/stacks/vpc.py b/deploy/stacks/vpc.py index 497736dcb..a56fc5608 100644 --- a/deploy/stacks/vpc.py +++ b/deploy/stacks/vpc.py @@ -23,9 +23,11 @@ def __init__( cidr=None, resource_prefix=None, restricted_nacl=False, + log_retention_duration='TWO_YEARS', **kwargs, ): super().__init__(scope, id, **kwargs) + self.log_retention_duration=log_retention_duration if vpc_id: self.vpc = ec2.Vpc.from_lookup(self, 'vpc', vpc_id=vpc_id) @@ -179,6 +181,7 @@ def create_new_vpc(self, cidr, envname, resource_prefix, restricted_nacl): f'{resource_prefix}/{envname}/flowlogs', log_group_name=f'{resource_prefix}/{envname}/flowlogs', removal_policy=RemovalPolicy.DESTROY, + retention=getattr(logs.RetentionDays, self.log_retention_duration) ) iam_policy = iam.PolicyDocument( assign_sids=True, diff --git a/template_cdk.json b/template_cdk.json index 9ddf62e3d..35bba4300 100644 --- a/template_cdk.json +++ b/template_cdk.json @@ -58,7 +58,8 @@ "user_id": "string_USER_ID_CLAIM_NAME_MAPPING_FOR_EXTERNAL_IDP|DEFAULT=None", "email": "string_EMAIL_ID_CLAIM_NAME_MAPPING_FOR_EXTERNAL_IDP|DEFAULT=None" } - } + }, + "log_retention_duration": "string_LOG_RETENTION_DURATION|DEFAULT=TWO_YEARS" } ] } From 3d4c86a1ea4f1ac97c7f613aaadaa527217cfacc Mon Sep 17 00:00:00 2001 From: Noah Paige Date: Mon, 9 Sep 2024 23:14:03 -0400 Subject: [PATCH 2/3] ruff format --- deploy/stacks/albfront_stack.py | 2 +- deploy/stacks/container.py | 6 ++++-- deploy/stacks/lambda_api.py | 19 ++++++++++++++----- deploy/stacks/opensearch.py | 2 +- deploy/stacks/pipeline.py | 2 +- deploy/stacks/vpc.py | 4 ++-- 6 files changed, 23 insertions(+), 12 deletions(-) diff --git a/deploy/stacks/albfront_stack.py b/deploy/stacks/albfront_stack.py index 0f6c340d5..0de9b0b1f 100644 --- a/deploy/stacks/albfront_stack.py +++ b/deploy/stacks/albfront_stack.py @@ -323,7 +323,7 @@ def create_log_group(self, envname, resource_prefix, log_group_name): f'ECSLogGroup{log_group_name}{envname}', log_group_name=f'/{resource_prefix}/{envname}/ecs/{log_group_name}', removal_policy=RemovalPolicy.DESTROY, - retention=getattr(logs.RetentionDays, self.log_retention_duration) + retention=getattr(logs.RetentionDays, self.log_retention_duration), ) return log_group diff --git a/deploy/stacks/container.py b/deploy/stacks/container.py index 65e419b57..8f99e4a52 100644 --- a/deploy/stacks/container.py +++ b/deploy/stacks/container.py @@ -103,7 +103,9 @@ def __init__( ecs.CfnTaskDefinition.KeyValuePairProperty(name='envname', value=envname), ecs.CfnTaskDefinition.KeyValuePairProperty(name='LOGLEVEL', value='DEBUG'), ecs.CfnTaskDefinition.KeyValuePairProperty(name='config_location', value='/config.json'), - ecs.CfnTaskDefinition.KeyValuePairProperty(name='LOG_RETENTION', value=self.log_retention_duration), + ecs.CfnTaskDefinition.KeyValuePairProperty( + name='LOG_RETENTION', value=self.log_retention_duration + ), ], essential=True, log_configuration=ecs.CfnTaskDefinition.LogConfigurationProperty( @@ -717,7 +719,7 @@ def create_log_group(self, envname, resource_prefix, log_group_name): f'ECSLogGroup{log_group_name}{envname}', log_group_name=f'/{resource_prefix}/{envname}/ecs/{log_group_name}', removal_policy=RemovalPolicy.DESTROY, - retention=getattr(logs.RetentionDays, self.log_retention_duration) + retention=getattr(logs.RetentionDays, self.log_retention_duration), ) return log_group diff --git a/deploy/stacks/lambda_api.py b/deploy/stacks/lambda_api.py index e95650115..071b330ea 100644 --- a/deploy/stacks/lambda_api.py +++ b/deploy/stacks/lambda_api.py @@ -108,7 +108,10 @@ def __init__( 'ElasticSearchProxyHandler', function_name=f'{resource_prefix}-{envname}-esproxy', log_group=logs.LogGroup( - self, 'esproxyloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-esproxy', retention=getattr(logs.RetentionDays, self.log_retention_duration) + self, + 'esproxyloggroup', + log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-esproxy', + retention=getattr(logs.RetentionDays, self.log_retention_duration), ), description='dataall es search function', role=self.create_function_role(envname, resource_prefix, 'esproxy', pivot_role_name, vpc), @@ -145,7 +148,10 @@ def __init__( 'LambdaGraphQL', function_name=f'{resource_prefix}-{envname}-graphql', log_group=logs.LogGroup( - self, 'graphqlloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-graphql', retention=getattr(logs.RetentionDays, self.log_retention_duration) + self, + 'graphqlloggroup', + log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-graphql', + retention=getattr(logs.RetentionDays, self.log_retention_duration), ), description='dataall graphql function', role=self.create_function_role(envname, resource_prefix, 'graphql', pivot_role_name, vpc), @@ -176,7 +182,10 @@ def __init__( 'AWSWorker', function_name=f'{resource_prefix}-{envname}-awsworker', log_group=logs.LogGroup( - self, 'awsworkerloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-awsworker', retention=getattr(logs.RetentionDays, self.log_retention_duration) + self, + 'awsworkerloggroup', + log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-awsworker', + retention=getattr(logs.RetentionDays, self.log_retention_duration), ), description='dataall aws worker for aws asynchronous tasks function', role=self.create_function_role(envname, resource_prefix, 'awsworker', pivot_role_name, vpc), @@ -248,7 +257,7 @@ def __init__( self, 'customauthorizerloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-custom-authorizer', - retention=getattr(logs.RetentionDays, self.log_retention_duration) + retention=getattr(logs.RetentionDays, self.log_retention_duration), ), handler='custom_authorizer_lambda.lambda_handler', code=_lambda.Code.from_asset( @@ -766,7 +775,7 @@ def set_up_graphql_api_gateway( f'{resource_prefix}/{envname}/apigateway', log_group_name=f'{resource_prefix}/{envname}/apigateway', removal_policy=RemovalPolicy.DESTROY, - retention=getattr(logs.RetentionDays, self.log_retention_duration), + retention=getattr(logs.RetentionDays, self.log_retention_duration), ) iam_policy = iam.PolicyDocument( diff --git a/deploy/stacks/opensearch.py b/deploy/stacks/opensearch.py index 160654113..d0e9f10b7 100644 --- a/deploy/stacks/opensearch.py +++ b/deploy/stacks/opensearch.py @@ -68,7 +68,7 @@ def __init__( id='EsAppLogGroup', log_group_name=f'/{resource_prefix}/{envname}/opensearch', removal_policy=RemovalPolicy.DESTROY, - retention=getattr(logs.RetentionDays, log_retention_duration), + retention=getattr(logs.RetentionDays, log_retention_duration), ) self.domain = opensearch.Domain( diff --git a/deploy/stacks/pipeline.py b/deploy/stacks/pipeline.py index 3cfcbead3..79e315591 100644 --- a/deploy/stacks/pipeline.py +++ b/deploy/stacks/pipeline.py @@ -652,7 +652,7 @@ def set_backend_stage(self, target_env, repository_name): custom_waf_rules=target_env.get('custom_waf_rules', None), with_approval_tests=target_env.get('with_approval_tests', False), allowed_origins=target_env.get('allowed_origins', '*'), - log_retention_duration=target_env.get('log_retention_duration', 'TWO_YEARS') + log_retention_duration=target_env.get('log_retention_duration', 'TWO_YEARS'), ) ) return backend_stage diff --git a/deploy/stacks/vpc.py b/deploy/stacks/vpc.py index a56fc5608..1cb3d5f69 100644 --- a/deploy/stacks/vpc.py +++ b/deploy/stacks/vpc.py @@ -27,7 +27,7 @@ def __init__( **kwargs, ): super().__init__(scope, id, **kwargs) - self.log_retention_duration=log_retention_duration + self.log_retention_duration = log_retention_duration if vpc_id: self.vpc = ec2.Vpc.from_lookup(self, 'vpc', vpc_id=vpc_id) @@ -181,7 +181,7 @@ def create_new_vpc(self, cidr, envname, resource_prefix, restricted_nacl): f'{resource_prefix}/{envname}/flowlogs', log_group_name=f'{resource_prefix}/{envname}/flowlogs', removal_policy=RemovalPolicy.DESTROY, - retention=getattr(logs.RetentionDays, self.log_retention_duration) + retention=getattr(logs.RetentionDays, self.log_retention_duration), ) iam_policy = iam.PolicyDocument( assign_sids=True, From 87484bb5c302df6aefbabc2431b26a8c4c412e98 Mon Sep 17 00:00:00 2001 From: Noah Paige Date: Tue, 10 Sep 2024 09:55:36 -0400 Subject: [PATCH 3/3] Touch ups log retention and make global config --- deploy/stacks/albfront_stack.py | 4 +++- deploy/stacks/albfront_stage.py | 2 +- deploy/stacks/backend_stack.py | 3 ++- deploy/stacks/backend_stage.py | 2 +- deploy/stacks/container.py | 2 +- deploy/stacks/lambda_api.py | 2 +- deploy/stacks/opensearch.py | 2 +- deploy/stacks/pipeline.py | 9 +++++++-- deploy/stacks/vpc.py | 2 +- template_cdk.json | 4 ++-- 10 files changed, 20 insertions(+), 12 deletions(-) diff --git a/deploy/stacks/albfront_stack.py b/deploy/stacks/albfront_stack.py index 0de9b0b1f..46632e2da 100644 --- a/deploy/stacks/albfront_stack.py +++ b/deploy/stacks/albfront_stack.py @@ -32,11 +32,13 @@ def __init__( ip_ranges=None, custom_auth=None, backend_region=None, - log_retention_duration='TWO_YEARS', + log_retention_duration=None, **kwargs, ): super().__init__(scope, id, **kwargs) + self.log_retention_duration = log_retention_duration + if self.node.try_get_context('image_tag'): image_tag = self.node.try_get_context('image_tag') diff --git a/deploy/stacks/albfront_stage.py b/deploy/stacks/albfront_stage.py index b19272eb4..fd690e700 100644 --- a/deploy/stacks/albfront_stage.py +++ b/deploy/stacks/albfront_stage.py @@ -18,7 +18,7 @@ def __init__( ip_ranges=None, custom_auth=None, backend_region=None, - log_retention_duration='TWO_YEARS', + log_retention_duration=None, **kwargs, ): super().__init__(scope, id, **kwargs) diff --git a/deploy/stacks/backend_stack.py b/deploy/stacks/backend_stack.py index bba5a9111..2bd964394 100644 --- a/deploy/stacks/backend_stack.py +++ b/deploy/stacks/backend_stack.py @@ -59,7 +59,7 @@ def __init__( custom_waf_rules=None, with_approval_tests=False, allowed_origins='*', - log_retention_duration='TWO_YEARS', + log_retention_duration=None, **kwargs, ): super().__init__(scope, id, **kwargs) @@ -374,6 +374,7 @@ def __init__( 'ecs_security_groups': self.ecs_stack.ecs_security_groups, 'ecs_task_role': self.ecs_stack.ecs_task_role, 'prod_sizing': prod_sizing, + 'log_retention_duration': log_retention_duration, **kwargs, } if enable_opensearch_serverless: diff --git a/deploy/stacks/backend_stage.py b/deploy/stacks/backend_stage.py index 39a22cc58..4185c5623 100644 --- a/deploy/stacks/backend_stage.py +++ b/deploy/stacks/backend_stage.py @@ -37,7 +37,7 @@ def __init__( custom_waf_rules=None, with_approval_tests=False, allowed_origins='*', - log_retention_duration='TWO_YEARS', + log_retention_duration=None, **kwargs, ): super().__init__(scope, id, **kwargs) diff --git a/deploy/stacks/container.py b/deploy/stacks/container.py index 8f99e4a52..7fdddd855 100644 --- a/deploy/stacks/container.py +++ b/deploy/stacks/container.py @@ -36,7 +36,7 @@ def __init__( email_custom_domain=None, ses_configuration_set=None, custom_domain=None, - log_retention_duration='TWO_YEARS', + log_retention_duration=None, **kwargs, ): super().__init__(scope, id, **kwargs) diff --git a/deploy/stacks/lambda_api.py b/deploy/stacks/lambda_api.py index 071b330ea..7a32dc62f 100644 --- a/deploy/stacks/lambda_api.py +++ b/deploy/stacks/lambda_api.py @@ -58,7 +58,7 @@ def __init__( custom_domain=None, custom_auth=None, allowed_origins='*', - log_retention_duration='TWO_YEARS', + log_retention_duration=None, **kwargs, ): super().__init__(scope, id, **kwargs) diff --git a/deploy/stacks/opensearch.py b/deploy/stacks/opensearch.py index d0e9f10b7..a1b9cdb55 100644 --- a/deploy/stacks/opensearch.py +++ b/deploy/stacks/opensearch.py @@ -38,7 +38,7 @@ def __init__( lambdas=None, ecs_security_groups: [ec2.SecurityGroup] = None, prod_sizing=False, - log_retention_duration='TWO_YEARS', + log_retention_duration=None, **kwargs, ): super().__init__(scope, id) diff --git a/deploy/stacks/pipeline.py b/deploy/stacks/pipeline.py index 79e315591..3dee744d5 100644 --- a/deploy/stacks/pipeline.py +++ b/deploy/stacks/pipeline.py @@ -8,6 +8,7 @@ from aws_cdk import aws_codecommit as codecommit from aws_cdk import aws_ec2 as ec2 from aws_cdk import aws_iam as iam +from aws_cdk import aws_logs as logs from aws_cdk import aws_s3 as s3 from aws_cdk import pipelines from aws_cdk.aws_codebuild import BuildEnvironmentVariable, BuildEnvironmentVariableType @@ -43,6 +44,9 @@ def __init__( self.target_envs = target_envs self.repo_string = repo_string self.repo_connection_arn = repo_connection_arn + self.log_retention_duration = ( + self.node.try_get_context('log_retention_duration') or logs.RetentionDays.TWO_YEARS.value + ) self.vpc_stack = VpcStack( self, @@ -52,6 +56,7 @@ def __init__( resource_prefix=resource_prefix, vpc_id=self.node.try_get_context('tooling_vpc_id'), restricted_nacl=self.node.try_get_context('tooling_vpc_restricted_nacl'), + log_retention_duration=self.log_retention_duration, **kwargs, ) self.vpc = self.vpc_stack.vpc @@ -652,7 +657,7 @@ def set_backend_stage(self, target_env, repository_name): custom_waf_rules=target_env.get('custom_waf_rules', None), with_approval_tests=target_env.get('with_approval_tests', False), allowed_origins=target_env.get('allowed_origins', '*'), - log_retention_duration=target_env.get('log_retention_duration', 'TWO_YEARS'), + log_retention_duration=self.log_retention_duration, ) ) return backend_stage @@ -891,7 +896,7 @@ def set_albfront_stage(self, target_env, repository_name): resource_prefix=self.resource_prefix, custom_auth=target_env.get('custom_auth', None), backend_region=target_env.get('region', self.region), - log_retention_duration=target_env.get('log_retention_duration', 'TWO_YEARS'), + log_retention_duration=self.log_retention_duration, ), pre=[ pipelines.CodeBuildStep( diff --git a/deploy/stacks/vpc.py b/deploy/stacks/vpc.py index 1cb3d5f69..a29105324 100644 --- a/deploy/stacks/vpc.py +++ b/deploy/stacks/vpc.py @@ -23,7 +23,7 @@ def __init__( cidr=None, resource_prefix=None, restricted_nacl=False, - log_retention_duration='TWO_YEARS', + log_retention_duration=None, **kwargs, ): super().__init__(scope, id, **kwargs) diff --git a/template_cdk.json b/template_cdk.json index 35bba4300..7b2ebe0d5 100644 --- a/template_cdk.json +++ b/template_cdk.json @@ -14,6 +14,7 @@ "repository_source": "string_VERSION_CONTROL_SERVICE|(codecommit, codestar_connection) DEFAULT=codecommit", "repo_string": "string_REPOSITORY_IN_GITHUB_OWNER/REPOSITORY|DEFAULT=awslabs/aws-dataall, REQUIRED if repository_source=codestar_connection", "repo_connection_arn": "string_CODESTAR_SOURCE_CONNECTION_ARN_FOR_GITHUB_arn:aws:codestar-connections:region:account-id:connection/connection-id|DEFAULT=None, REQUIRED if repository_source=codestar_connection", + "log_retention_duration": "string_LOG_RETENTION_DURATION|DEFAULT=TWO_YEARS", "DeploymentEnvironments": [ { "envname": "string_ENVIRONMENT_NAME|REQUIRED", @@ -58,8 +59,7 @@ "user_id": "string_USER_ID_CLAIM_NAME_MAPPING_FOR_EXTERNAL_IDP|DEFAULT=None", "email": "string_EMAIL_ID_CLAIM_NAME_MAPPING_FOR_EXTERNAL_IDP|DEFAULT=None" } - }, - "log_retention_duration": "string_LOG_RETENTION_DURATION|DEFAULT=TWO_YEARS" + } } ] }