Config log retention

backend/dataall/modules/mlstudio/cdk/mlstudio_extension.py

@@ -2,6 +2,7 @@
 Extends an environment stack for SageMaker Studio Domain
 """
 
+import os
 import logging
 
 from aws_cdk import (
@@ -63,7 +64,7 @@ def extent(setup: EnvironmentSetup):
                     setup,
                     f'SageMakerStudio{_environment.name}',
                     log_group_name=f'/{_environment.resourcePrefix}/{_environment.name}/vpc/sagemakerstudio',
-                    retention=logs.RetentionDays.ONE_MONTH,
+                    retention=getattr(logs.RetentionDays, os.environ.get('LOG_RETENTION', 'TWO_YEARS')),
                     removal_policy=RemovalPolicy.DESTROY,
                 )
                 vpc_flow_role = iam.Role(

deploy/stacks/albfront_stack.py

@@ -32,10 +32,13 @@ def __init__(
         ip_ranges=None,
         custom_auth=None,
         backend_region=None,
+        log_retention_duration=None,
         **kwargs,
     ):
         super().__init__(scope, id, **kwargs)
 
+        self.log_retention_duration = log_retention_duration
+
         if self.node.try_get_context('image_tag'):
             image_tag = self.node.try_get_context('image_tag')
 
@@ -321,8 +324,8 @@ def create_log_group(self, envname, resource_prefix, log_group_name):
             self,
             f'ECSLogGroup{log_group_name}{envname}',
             log_group_name=f'/{resource_prefix}/{envname}/ecs/{log_group_name}',
-            retention=logs.RetentionDays.ONE_MONTH,
             removal_policy=RemovalPolicy.DESTROY,
+            retention=getattr(logs.RetentionDays, self.log_retention_duration),
         )
         return log_group
 

deploy/stacks/albfront_stage.py

@@ -18,6 +18,7 @@ def __init__(
         ip_ranges=None,
         custom_auth=None,
         backend_region=None,
+        log_retention_duration=None,
         **kwargs,
     ):
         super().__init__(scope, id, **kwargs)
@@ -33,6 +34,7 @@ def __init__(
             ip_ranges=ip_ranges,
             custom_auth=custom_auth,
             backend_region=backend_region,
+            log_retention_duration=log_retention_duration,
         )
 
         Tags.of(albfront_stack).add('Application', f'{resource_prefix}-{envname}')

deploy/stacks/backend_stack.py

@@ -60,6 +60,7 @@ def __init__(
         custom_waf_rules=None,
         with_approval_tests=False,
         allowed_origins='*',
+        log_retention_duration=None,
         **kwargs,
     ):
         super().__init__(scope, id, **kwargs)
@@ -76,6 +77,7 @@ def __init__(
             vpc_endpoints_sg=vpc_endpoints_sg,
             vpc_id=vpc_id,
             restricted_nacl=vpc_restricted_nacls,
+            log_retention_duration=log_retention_duration,
             **kwargs,
         )
         vpc = self.vpc_stack.vpc
@@ -202,6 +204,7 @@ def __init__(
             custom_auth=custom_auth,
             custom_waf_rules=custom_waf_rules,
             allowed_origins=allowed_origins,
+            log_retention_duration=log_retention_duration,
             **kwargs,
         )
 
@@ -226,6 +229,7 @@ def __init__(
             email_custom_domain=ses_stack.ses_identity.email_identity_name if ses_stack is not None else None,
             ses_configuration_set=ses_stack.configuration_set.configuration_set_name if ses_stack is not None else None,
             custom_domain=custom_domain,
+            log_retention_duration=log_retention_duration,
             **kwargs,
         )
 
@@ -372,6 +376,7 @@ def __init__(
             'ecs_security_groups': self.ecs_stack.ecs_security_groups,
             'ecs_task_role': self.ecs_stack.ecs_task_role,
             'prod_sizing': prod_sizing,
+            'log_retention_duration': log_retention_duration,
             **kwargs,
         }
         if enable_opensearch_serverless:

deploy/stacks/backend_stage.py

@@ -37,6 +37,7 @@ def __init__(
         custom_waf_rules=None,
         with_approval_tests=False,
         allowed_origins='*',
+        log_retention_duration=None,
         **kwargs,
     ):
         super().__init__(scope, id, **kwargs)
@@ -71,6 +72,7 @@ def __init__(
             custom_waf_rules=custom_waf_rules,
             with_approval_tests=with_approval_tests,
             allowed_origins=allowed_origins,
+            log_retention_duration=log_retention_duration,
             **kwargs,
         )
 

deploy/stacks/container.py

@@ -37,11 +37,13 @@ def __init__(
         email_custom_domain=None,
         ses_configuration_set=None,
         custom_domain=None,
+        log_retention_duration=None,
         **kwargs,
     ):
         super().__init__(scope, id, **kwargs)
         self._envname = envname
         self._resource_prefix = resource_prefix
+        self.log_retention_duration = log_retention_duration
 
         if self.node.try_get_context('image_tag'):
             image_tag = self.node.try_get_context('image_tag')
@@ -102,6 +104,9 @@ def __init__(
                         ecs.CfnTaskDefinition.KeyValuePairProperty(name='envname', value=envname),
                         ecs.CfnTaskDefinition.KeyValuePairProperty(name='LOGLEVEL', value='DEBUG'),
                         ecs.CfnTaskDefinition.KeyValuePairProperty(name='config_location', value='/config.json'),
+                        ecs.CfnTaskDefinition.KeyValuePairProperty(
+                            name='LOG_RETENTION', value=self.log_retention_duration
+                        ),
                     ],
                     essential=True,
                     log_configuration=ecs.CfnTaskDefinition.LogConfigurationProperty(
@@ -719,8 +724,8 @@ def create_log_group(self, envname, resource_prefix, log_group_name):
             self,
             f'ECSLogGroup{log_group_name}{envname}',
             log_group_name=f'/{resource_prefix}/{envname}/ecs/{log_group_name}',
-            retention=logs.RetentionDays.ONE_MONTH,
             removal_policy=RemovalPolicy.DESTROY,
+            retention=getattr(logs.RetentionDays, self.log_retention_duration),
         )
         return log_group
 

deploy/stacks/lambda_api.py

@@ -58,10 +58,13 @@ def __init__(
         custom_domain=None,
         custom_auth=None,
         allowed_origins='*',
+        log_retention_duration=None,
         **kwargs,
     ):
         super().__init__(scope, id, **kwargs)
 
+        self.log_retention_duration = log_retention_duration
+
         if self.node.try_get_context('image_tag'):
             image_tag = self.node.try_get_context('image_tag')
 
@@ -105,7 +108,10 @@ def __init__(
             'ElasticSearchProxyHandler',
             function_name=f'{resource_prefix}-{envname}-esproxy',
             log_group=logs.LogGroup(
-                self, 'esproxyloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-esproxy'
+                self,
+                'esproxyloggroup',
+                log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-esproxy',
+                retention=getattr(logs.RetentionDays, self.log_retention_duration),
             ),
             description='dataall es search function',
             role=self.create_function_role(envname, resource_prefix, 'esproxy', pivot_role_name, vpc),
@@ -142,7 +148,10 @@ def __init__(
             'LambdaGraphQL',
             function_name=f'{resource_prefix}-{envname}-graphql',
             log_group=logs.LogGroup(
-                self, 'graphqlloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-graphql'
+                self,
+                'graphqlloggroup',
+                log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-graphql',
+                retention=getattr(logs.RetentionDays, self.log_retention_duration),
             ),
             description='dataall graphql function',
             role=self.create_function_role(envname, resource_prefix, 'graphql', pivot_role_name, vpc),
@@ -173,7 +182,10 @@ def __init__(
             'AWSWorker',
             function_name=f'{resource_prefix}-{envname}-awsworker',
             log_group=logs.LogGroup(
-                self, 'awsworkerloggroup', log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-awsworker'
+                self,
+                'awsworkerloggroup',
+                log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-backend-awsworker',
+                retention=getattr(logs.RetentionDays, self.log_retention_duration),
             ),
             description='dataall aws worker for aws asynchronous tasks function',
             role=self.create_function_role(envname, resource_prefix, 'awsworker', pivot_role_name, vpc),
@@ -245,6 +257,7 @@ def __init__(
                     self,
                     'customauthorizerloggroup',
                     log_group_name=f'/aws/lambda/{resource_prefix}-{envname}-custom-authorizer',
+                    retention=getattr(logs.RetentionDays, self.log_retention_duration),
                 ),
                 handler='custom_authorizer_lambda.lambda_handler',
                 code=_lambda.Code.from_asset(
@@ -762,6 +775,7 @@ def set_up_graphql_api_gateway(
             f'{resource_prefix}/{envname}/apigateway',
             log_group_name=f'{resource_prefix}/{envname}/apigateway',
             removal_policy=RemovalPolicy.DESTROY,
+            retention=getattr(logs.RetentionDays, self.log_retention_duration),
         )
 
         iam_policy = iam.PolicyDocument(

deploy/stacks/opensearch.py

@@ -38,6 +38,7 @@ def __init__(
         lambdas=None,
         ecs_security_groups: [ec2.SecurityGroup] = None,
         prod_sizing=False,
+        log_retention_duration=None,
         **kwargs,
     ):
         super().__init__(scope, id)
@@ -67,6 +68,7 @@ def __init__(
             id='EsAppLogGroup',
             log_group_name=f'/{resource_prefix}/{envname}/opensearch',
             removal_policy=RemovalPolicy.DESTROY,
+            retention=getattr(logs.RetentionDays, log_retention_duration),
         )
 
         self.domain = opensearch.Domain(

deploy/stacks/pipeline.py

@@ -7,6 +7,7 @@
 from aws_cdk import aws_codecommit as codecommit
 from aws_cdk import aws_ec2 as ec2
 from aws_cdk import aws_iam as iam
+from aws_cdk import aws_logs as logs
 from aws_cdk import aws_kms as kms
 from aws_cdk import aws_s3 as s3
 from aws_cdk import pipelines
@@ -43,6 +44,9 @@ def __init__(
         self.target_envs = target_envs
         self.repo_string = repo_string
         self.repo_connection_arn = repo_connection_arn
+        self.log_retention_duration = (
+            self.node.try_get_context('log_retention_duration') or logs.RetentionDays.TWO_YEARS.value
+        )
 
         self.vpc_stack = VpcStack(
             self,
@@ -52,6 +56,7 @@ def __init__(
             resource_prefix=resource_prefix,
             vpc_id=self.node.try_get_context('tooling_vpc_id'),
             restricted_nacl=self.node.try_get_context('tooling_vpc_restricted_nacl'),
+            log_retention_duration=self.log_retention_duration,
             **kwargs,
         )
         self.vpc = self.vpc_stack.vpc
@@ -672,6 +677,7 @@ def set_backend_stage(self, target_env, repository_name):
                 custom_waf_rules=target_env.get('custom_waf_rules', None),
                 with_approval_tests=target_env.get('with_approval_tests', False),
                 allowed_origins=target_env.get('allowed_origins', '*'),
+                log_retention_duration=self.log_retention_duration,
             )
         )
         return backend_stage
@@ -913,6 +919,7 @@ def set_albfront_stage(self, target_env, repository_name):
                 resource_prefix=self.resource_prefix,
                 custom_auth=target_env.get('custom_auth', None),
                 backend_region=target_env.get('region', self.region),
+                log_retention_duration=self.log_retention_duration,
             ),
             pre=[
                 pipelines.CodeBuildStep(

deploy/stacks/vpc.py

@@ -23,9 +23,11 @@ def __init__(
         cidr=None,
         resource_prefix=None,
         restricted_nacl=False,
+        log_retention_duration=None,
         **kwargs,
     ):
         super().__init__(scope, id, **kwargs)
+        self.log_retention_duration = log_retention_duration
 
         if vpc_id:
             self.vpc = ec2.Vpc.from_lookup(self, 'vpc', vpc_id=vpc_id)
@@ -179,6 +181,7 @@ def create_new_vpc(self, cidr, envname, resource_prefix, restricted_nacl):
             f'{resource_prefix}/{envname}/flowlogs',
             log_group_name=f'{resource_prefix}/{envname}/flowlogs',
             removal_policy=RemovalPolicy.DESTROY,
+            retention=getattr(logs.RetentionDays, self.log_retention_duration),
         )
         iam_policy = iam.PolicyDocument(
             assign_sids=True,

template_cdk.json

@@ -14,6 +14,7 @@
     "repository_source": "string_VERSION_CONTROL_SERVICE|(codecommit, codestar_connection) DEFAULT=codecommit",
     "repo_string": "string_REPOSITORY_IN_GITHUB_OWNER/REPOSITORY|DEFAULT=awslabs/aws-dataall, REQUIRED if repository_source=codestar_connection",
     "repo_connection_arn": "string_CODESTAR_SOURCE_CONNECTION_ARN_FOR_GITHUB_arn:aws:codestar-connections:region:account-id:connection/connection-id|DEFAULT=None, REQUIRED if repository_source=codestar_connection",
+    "log_retention_duration": "string_LOG_RETENTION_DURATION|DEFAULT=TWO_YEARS",
     "DeploymentEnvironments": [
       {
         "envname": "string_ENVIRONMENT_NAME|REQUIRED",