Add DLQ support, and new variable for cloudwatch integration

.github/workflows/Changelog.yaml

@@ -15,15 +15,15 @@ jobs:
     steps:
       - name: Get PR labels
         id: pr-labels
-        uses: joerick/pr-labels-action@v1.0.8
+        uses: joerick/pr-labels-action@v1.0.9
 
   check-changelog-updates:
     if: "${{ needs.get-label.outputs.labels != ' skip-changelog ' }}"
     runs-on: ubuntu-latest
     needs: get-label
     name: Check changelog update
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Get changed files

.github/workflows/release.yml

@@ -19,7 +19,7 @@ jobs:
     if: github.repository_owner == 'coralogix'
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
           fetch-depth: 0

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
       packages: ${{ env.packages }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
           fetch-depth: 0
@@ -36,7 +36,7 @@ jobs:
         package: ${{ fromJSON(needs.check.outputs.packages) }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Check for missing test directories
         run: |

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## v1.0.98
+#### **coralogix-aws-shipper**
+### 💡 Enhancements 
+- Add support for DLQ
+- Add log_group_prefix variable to avoid limitation of number of log groups
+- Update versions for github actions to avoid node.js 16 issue
+
 ## v1.0.97
 #### firehose-metrics
 ### 💡 Enhancements 

examples/coralogix-aws-shipper/variables.tf

@@ -90,6 +90,12 @@ variable "log_groups" {
   default     = []
 }
 
+variable "log_group_prefix" {
+  description = "Prefix of the CloudWatch log groups that will trigger the lambda"
+  type        = list(string)
+  default     = null
+}
+
 # vpc variables
 
 variable "subnet_ids" {
@@ -128,6 +134,32 @@ variable "cpu_arch" {
   }
 }
 
+# DLQ configuration
+
+variable "enable_dlq" {
+  description = "Enable Dead Letter Queue for the Lambda function"
+  type        = bool
+  default     = false
+}
+
+variable "dlq_retry_limit" {
+  description = "The maximum number of times to retry the function execution in case of failure"
+  type        = number
+  default     = 3
+}
+
+variable "dlq_retry_delay" {
+  description = "The delay in seconds between retries"
+  type        = number
+  default     = 900
+}
+
+variable "dlq_s3_bucket" {
+  description = "The S3 bucket to store the DLQ failed messages after retry limit is reached"
+  type        = string
+  default     = null
+}
+
 # Integration Generic Config (Optional)
 
 variable "notification_email" {
@@ -242,7 +274,7 @@ variable "msk_cluster_arn" {
 
 variable "msk_topic_name" {
   description = "List of names of the Kafka topic used to store records in your Kafka cluster ( [\"topic1\", \"topic2\",])"
-  type        = list
+  type        = list(any)
   default     = null
 }
 

modules/coralogix-aws-shipper/CloudWatch.tf

@@ -1,10 +1,10 @@
 resource "aws_lambda_permission" "cloudwatch_trigger_premission" {
   depends_on    = [module.lambda]
-  for_each      = local.log_groups
+  for_each      =  var.log_group_prefix == null ? local.log_groups : local.log_group_prefix 
   action        = "lambda:InvokeFunction"
   function_name = local.integration_info.integration.lambda_name == null ? module.locals.integration.function_name : local.integration_info.integration.lambda_name
   principal     = "logs.amazonaws.com"
-  source_arn    = "${data.aws_cloudwatch_log_group.this[each.key].arn}:*"
+  source_arn    = var.log_group_prefix == null ? "${data.aws_cloudwatch_log_group.this[each.key].arn}:*" : "arn:aws:logs:${data.aws_region.this.name}:${data.aws_caller_identity.this.account_id}:log-group:${local.log_group_prefix[each.value]}*:*"
 }
 
 resource "aws_cloudwatch_log_subscription_filter" "this" {

modules/coralogix-aws-shipper/README.md

@@ -79,6 +79,7 @@ If you want to avoid this issue, you can deploy in other ways:
 | Name | Description | Type | Default | Required | 
 |------|-------------|------|---------|:--------:|
 | <a name="input_log_groups"></a> [log\_groups](#input\_log\_groups) | Provide a comma-separated list of CloudWatch log group names to monitor, for example, (log-group1, log-group2, log-group3). | `list(string)` | n/a | yes |
+| <a name="input_log_group_prefix"></a> [log\_group\_prefix](#input\_log\_group\_prefix) | Prefix of the CloudWatch log groups that will trigger the lambda, in case that your log groups are `log-group1, log-group2, log-group3` then you can set the value to `log-group`. When using this variable you will not be able to see the log groups as trigger for the lambda. | `list(string)` | n/a | no |
 
 ### SNS Configuration
 
@@ -150,6 +151,19 @@ When using this variable you will need to create an S3 bucket in the region wher
 | <a name="input_subnet_ids"></a> [vpc\_subnet\_ids](#input\_subnet\_ids) | Specify the ID of the subnet where the integration should be deployed. | `list(string)` | n/a | no |
 | <a name="input_security_group_ids"></a> [security\_group\_ids](#input\_security\_group\_ids) | Specify the ID of the Security Group where the integration should be deployed. | `list(string)` | n/a | no |
 
+### DLQ
+
+A Dead Letter Queue (DLQ) is a queue where messages are sent if they cannot be processed by the Lambda function. This is useful for debugging and monitoring.
+
+ To enable the DLQ, you must provide the required parameters outlined below.
+
+| Parameter       | Description                                                                   | Default Value | Required |
+|-----------------|-------------------------------------------------------------------------------|---------------|----------|
+| enable_dlq      | Enable the Dead Letter Queue for the Lambda function.                         | false         | yes      |
+| dlq_s3_bucket   | An S3 bucket used to store all failure events that have exhausted retries.    |               | yes      |
+| dlq_retry_limit | The number of times a failed event should be retried before being saved in S3 | 3             | yes      |
+| dlq_retry_delay | The delay in seconds between retries of failed events                         | 900           | yes      |
+
 **AWS PrivateLink**
 
 If you want to bypass using the public internet, you can use AWS PrivateLink to facilitate secure connections between your VPCs and AWS Services. This option is available under [VPC Configuration](#vpc-configuration-optional). For additional instructions on AWS PrivateLink, please [follow our dedicated tutorial](https://coralogix.com/docs/coralogix-amazon-web-services-aws-privatelink-endpoints/).

modules/coralogix-aws-shipper/Sns.tf

@@ -1,4 +1,5 @@
 resource "aws_s3_bucket_notification" "topic_notification" {
+  depends_on = [ module.lambda ]
   count  = local.sns_enable == true && (var.integration_type == "S3" || var.integration_type == "CloudTrail") ? 1 : 0
   bucket = data.aws_s3_bucket.this[0].bucket
   topic {

modules/coralogix-aws-shipper/Sqs.tf

@@ -1,4 +1,5 @@
 resource "aws_s3_bucket_notification" "sqs_notification" {
+  depends_on = [ module.lambda ]
   count  = var.sqs_name != null && (var.integration_type == "S3" || var.integration_type == "CloudTrail") ? 1 : 0
   bucket = data.aws_s3_bucket.this[0].bucket
   queue {

modules/coralogix-aws-shipper/data.tf

@@ -11,11 +11,17 @@ data "aws_cloudwatch_log_group" "this" {
   for_each = local.log_groups
   name     = each.key
 }
+
 data "aws_s3_bucket" "this" {
   count  = var.s3_bucket_name == null ? 0 : 1
   bucket = var.s3_bucket_name
 }
 
+data "aws_s3_bucket" "dlq_bucket" {
+  count  = var.enable_dlq ? 1 : 0
+  bucket = var.dlq_s3_bucket
+}
+
 data "aws_sns_topic" "sns_topic" {
   count = local.sns_enable ? 1 : 0
   name  = var.sns_topic_name

modules/coralogix-aws-shipper/local.tf

@@ -10,6 +10,12 @@ locals {
     for group in var.log_groups : group =>
     length(group) > 100 ? "${substr(replace(group, "/", "_"), 0, 95)}_${substr(sha256(group), 0, 4)}" : replace(group, "/", "_")
   }
+
+  log_group_prefix = var.log_group_prefix != null ? {
+    # Need to convert the log group prefix to a map so we could use it in the for_each in CloudWatch file
+    for group in var.log_group_prefix : group =>
+    group
+  } : {}
 
   api_key_is_arn = replace(var.api_key, ":", "") != var.api_key ? true : false
 

modules/coralogix-aws-shipper/main.tf

@@ -32,7 +32,7 @@ resource "null_resource" "s3_bucket_copy" {
 module "lambda" {
   for_each = var.integration_info != null ? var.integration_info : local.integration_info
 
-  depends_on             = [null_resource.s3_bucket_copy]
+  depends_on             = [null_resource.s3_bucket_copy,aws_sqs_queue.DLQ]
   source                 = "terraform-aws-modules/lambda/aws"
   function_name          = each.value.lambda_name == null ? module.locals[each.key].function_name : each.value.lambda_name
   description            = "Send logs to Coralogix."
@@ -46,6 +46,8 @@ module "lambda" {
   destination_on_failure = aws_sns_topic.this[each.key].arn
   vpc_subnet_ids         = var.subnet_ids
   vpc_security_group_ids = var.security_group_ids
+  dead_letter_target_arn    = var.enable_dlq ? aws_sqs_queue.DLQ[0].arn : null
+  # attach_dead_letter_policy = var.enable_dlq
   environment_variables = {
     CORALOGIX_ENDPOINT = var.custom_domain != "" ? "https://ingress.${var.custom_domain}" : var.subnet_ids == null ? "https://ingress.${lookup(module.locals[each.key].coralogix_domains, var.coralogix_region, "EU1")}" : "https://ingress.private.${lookup(module.locals[each.key].coralogix_domains, var.coralogix_region, "EU1")}"
     INTEGRATION_TYPE   = each.value.integration_type
@@ -59,6 +61,10 @@ module "lambda" {
     ADD_METADATA       = var.add_metadata
     CUSTOM_METADATA    = var.custom_metadata
     CUSTOM_CSV_HEADER  = var.custom_csv_header
+    DLQ_ARN            = var.enable_dlq ? aws_sqs_queue.DLQ[0].arn : null
+    DLQ_RETRY_LIMIT    = var.enable_dlq ? var.dlq_retry_limit : null
+    DLQ_S3_BUCKET      = var.enable_dlq ? var.dlq_s3_bucket : null
+    DLQ_URL            = var.enable_dlq ? aws_sqs_queue.DLQ[0].url : null
   }
   s3_existing_package = {
     bucket = var.custom_s3_bucket == "" ? "coralogix-serverless-repo-${data.aws_region.this.name}" : var.custom_s3_bucket
@@ -74,6 +80,24 @@ module "lambda" {
   create_role                             = var.msk_cluster_arn != null ? false : true
   lambda_role                             = var.msk_cluster_arn != null ? aws_iam_role.role_for_msk[0].arn : ""
   policy_statements = {
+    dlq_sqs_permissions = var.enable_dlq ? {
+      effect    = "Allow"
+      actions   = ["sqs:SendMessage","sqs:ReceiveMessage","sqs:DeleteMessage","sqs:GetQueueAttributes"]
+      resources = [aws_sqs_queue.DLQ[0].arn]
+    } : {
+      effect    = "Deny"
+      actions   = ["rds:DescribeAccountAttributes"]
+      resources = ["*"]
+    }
+    dlq_s3_permissions = var.enable_dlq ? {
+      effect    = "Allow"
+      actions   = ["s3:PutObject","s3:PutObjectAcl","s3:AbortMultipartUpload","s3:DeleteObject","s3:PutObjectTagging","s3:PutObjectVersionTagging"]
+      resources = ["${data.aws_s3_bucket.dlq_bucket[0].arn}/*", data.aws_s3_bucket.dlq_bucket[0].arn]
+    } : {
+      effect    = "Deny"
+      actions   = ["rds:DescribeAccountAttributes"]
+      resources = ["*"]
+    }
     secret_access_policy = var.store_api_key_in_secrets_manager || local.api_key_is_arn ? {
       effect    = "Allow"
       actions   = ["secretsmanager:GetSecretValue"]
@@ -246,4 +270,20 @@ resource "aws_vpc_endpoint" "secretsmanager" {
   subnet_ids          = var.subnet_ids
   security_group_ids  = var.security_group_ids
   private_dns_enabled = true
+}
+
+resource "aws_sqs_queue" "DLQ" {
+  count = var.enable_dlq ? 1 : 0
+  name                       = "coralogix-aws-shipper-dlq-${random_string.this.result}"
+  message_retention_seconds  = 1209600
+  delay_seconds              = var.dlq_retry_delay
+  visibility_timeout_seconds = var.timeout
+}
+
+resource "aws_lambda_event_source_mapping" "dlq_sqs" {
+  depends_on       = [module.lambda]
+  count            = var.enable_dlq ? 1 : 0
+  event_source_arn = aws_sqs_queue.DLQ[0].arn
+  function_name    = local.integration_info.integration.lambda_name == null ? module.locals.integration.function_name : local.integration_info.integration.lambda_name
+  enabled          = true
 }

modules/coralogix-aws-shipper/variables.tf

@@ -90,6 +90,12 @@ variable "log_groups" {
   default     = []
 }
 
+variable "log_group_prefix" {
+  description = "Prefix of the CloudWatch log groups that will trigger the lambda"
+  type        = list(string)
+  default     = null
+}
+
 # vpc variables
 
 variable "subnet_ids" {
@@ -128,6 +134,32 @@ variable "cpu_arch" {
   }
 }
 
+# DLQ configuration
+
+variable "enable_dlq" {
+  description = "Enable Dead Letter Queue for the Lambda function"
+  type        = bool
+  default     = false
+}
+
+variable "dlq_retry_limit" {
+  description = "The maximum number of times to retry the function execution in case of failure"
+  type        = number
+  default     = 3
+}
+
+variable "dlq_retry_delay" {
+  description = "The delay in seconds between retries"
+  type        = number
+  default     = 900
+}
+
+variable "dlq_s3_bucket" {
+  description = "The S3 bucket to store the DLQ failed messages after retry limit is reached"
+  type        = string
+  default     = null
+}
+
 # Integration Generic Config (Optional)
 
 variable "notification_email" {

modules/firehose-metrics/README.md

@@ -212,7 +212,9 @@ then the CloudWatch metric stream must be configured with the same format, confi
 
 ## Inputs - Custom Resource Naming
 If there are conflicts with existing resources, the following variables can be used to customize the names of the resources created by this module.
-
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
 | <a name="input_cloudwatch_metric_stream_custom_name"></a> [cloudwatch_metric_stream_custom_name](#input\_cloudwatch_metric_stream_custom_name) | Set the name of the CloudWatch metric stream, otherwise variable 'firehose_stream' will be used | `string` | `null` | no |
 | <a name="input_s3_backup_custom_name"></a> [s3_backup_custom_name](#input\_s3_backup_custom_name) | Set the name of the S3 backup bucket, otherwise variable '{firehose_stream}-backup-metrics' will be used | `string` | `null` | no |
 | <a name="input_lambda_processor_custom_name"></a> [lambda_processor_custom_name](#input\_lambda_processor_custom_name) | Set the name of the lambda processor function, otherwise variable '{firehose_stream}-metrics-tags-processor' will be used | `string` | `null` | no |