From af8fe9e5f3563cf42560c3dfaa1a8011362e3832 Mon Sep 17 00:00:00 2001
From: guyrenny <guy.renny@coralogix.com>
Date: Thu, 2 May 2024 10:27:25 +0300
Subject: [PATCH 01/12] add new variable to allow the use to use an existing s3
 bucket

---
 modules/firehose-logs/main.tf         | 14 ++++++++++----
 modules/firehose-logs/variables.tf    |  6 ++++++
 modules/firehose-metrics/README.md    |  4 +++-
 modules/firehose-metrics/main.tf      | 22 ++++++++++++++--------
 modules/firehose-metrics/variables.tf |  6 ++++++
 test.tf                               | 22 ++++++++++++++++++++++
 6 files changed, 61 insertions(+), 13 deletions(-)
 create mode 100644 test.tf

diff --git a/modules/firehose-logs/main.tf b/modules/firehose-logs/main.tf
index 198baa96..f435400b 100644
--- a/modules/firehose-logs/main.tf
+++ b/modules/firehose-logs/main.tf
@@ -32,6 +32,11 @@ locals {
 data "aws_caller_identity" "current_identity" {}
 data "aws_region" "current_region" {}
 
+data "aws_s3_bucket" "s3_bucket" {
+  depends_on = [ aws_s3_bucket.firehose_bucket ]
+  bucket     = var.s3_existing_backup_bucket != null ? var.s3_existing_backup_bucket : aws_s3_bucket.firehose_bucket.id
+}
+
 resource "random_string" "this" {
   length  = 6
   special = false
@@ -58,12 +63,13 @@ resource "aws_cloudwatch_log_stream" "firehose_logstream_backup" {
 }
 
 resource "aws_s3_bucket" "firehose_bucket" {
+  count = var.s3_existing_backup_bucket != null ? 0 : 1
   tags   = merge(local.tags, { Name = local.s3_logs_backup_bucket_name })
   bucket = local.s3_logs_backup_bucket_name
 }
 
 resource "aws_s3_bucket_public_access_block" "firehose_bucket_bucket_access" {
-  bucket = aws_s3_bucket.firehose_bucket.id
+  bucket = data.aws_s3_bucket.s3_bucket.id
 
   block_public_acls       = true
   block_public_policy     = true
@@ -102,8 +108,8 @@ resource "aws_iam_role" "firehose_to_coralogix" {
             "s3:PutObject"
           ],
           "Resource" = [
-            aws_s3_bucket.firehose_bucket.arn,
-            "${aws_s3_bucket.firehose_bucket.arn}/*"
+            data.aws_s3_bucket.s3_bucket.arn,
+            "${data.aws_s3_bucket.s3_bucket.arn}/*"
           ]
         },
         {
@@ -159,7 +165,7 @@ resource "aws_kinesis_firehose_delivery_stream" "coralogix_stream_logs" {
 
     s3_configuration {
       role_arn           = aws_iam_role.firehose_to_coralogix.arn
-      bucket_arn         = aws_s3_bucket.firehose_bucket.arn
+      bucket_arn         = data.aws_s3_bucket.s3_bucket.arn
       buffering_size     = 5
       buffering_interval = 300
       compression_format = "GZIP"
diff --git a/modules/firehose-logs/variables.tf b/modules/firehose-logs/variables.tf
index eff608c5..92acb693 100644
--- a/modules/firehose-logs/variables.tf
+++ b/modules/firehose-logs/variables.tf
@@ -77,3 +77,9 @@ variable "s3_backup_custom_name" {
   type        = string
   default     = null
 }
+
+variable "s3_existing_backup_bucket" {
+  description = "Set the name of an existing S3 backup bucket to use, otherwise a new bucket will be created"
+  type        = string
+  default     = null
+}
diff --git a/modules/firehose-metrics/README.md b/modules/firehose-metrics/README.md
index 3b1e0a43..9f88cca3 100644
--- a/modules/firehose-metrics/README.md
+++ b/modules/firehose-metrics/README.md
@@ -212,7 +212,9 @@ then the CloudWatch metric stream must be configured with the same format, confi
 
 ## Inputs - Custom Resource Naming
 If there are conflicts with existing resources, the following variables can be used to customize the names of the resources created by this module.
-
+ 
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
 | <a name="input_cloudwatch_metric_stream_custom_name"></a> [cloudwatch_metric_stream_custom_name](#input\_cloudwatch_metric_stream_custom_name) | Set the name of the CloudWatch metric stream, otherwise variable 'firehose_stream' will be used | `string` | `null` | no |
 | <a name="input_s3_backup_custom_name"></a> [s3_backup_custom_name](#input\_s3_backup_custom_name) | Set the name of the S3 backup bucket, otherwise variable '{firehose_stream}-backup-metrics' will be used | `string` | `null` | no |
 | <a name="input_lambda_processor_custom_name"></a> [lambda_processor_custom_name](#input\_lambda_processor_custom_name) | Set the name of the lambda processor function, otherwise variable '{firehose_stream}-metrics-tags-processor' will be used | `string` | `null` | no |
diff --git a/modules/firehose-metrics/main.tf b/modules/firehose-metrics/main.tf
index d27f3647..20fbfc57 100644
--- a/modules/firehose-metrics/main.tf
+++ b/modules/firehose-metrics/main.tf
@@ -27,13 +27,18 @@ locals {
 
   # default namings
   cloud_watch_metric_stream_name = var.cloudwatch_metric_stream_custom_name != null ? var.cloudwatch_metric_stream_custom_name : var.firehose_stream
-  s3_backup_bucket_name          = var.s3_backup_custom_name != null ? var.s3_backup_custom_name : "${var.firehose_stream}-backup-metrics"
+  s3_backup_bucket_name          = var.s3_backup_custom_name != null ? var.s3_backup_custom_name : var.s3_existing_backup_bucket != null ? var.s3_existing_backup_bucket : "${var.firehose_stream}-backup-metrics"
   lambda_processor_name          = var.lambda_processor_custom_name != null ? var.lambda_processor_custom_name : "${var.firehose_stream}-metrics-transform"
 }
 
 data "aws_caller_identity" "current_identity" {}
 data "aws_region" "current_region" {}
 
+data "aws_s3_bucket" "s3_bucket" {
+  depends_on = [ aws_s3_bucket.firehose_bucket ]
+  bucket     = var.s3_existing_backup_bucket != null ? var.s3_existing_backup_bucket : aws_s3_bucket.firehose_bucket.id
+}
+
 resource "random_string" "this" {
   length  = 6
   special = false
@@ -60,12 +65,13 @@ resource "aws_cloudwatch_log_stream" "firehose_logstream_backup" {
 }
 
 resource "aws_s3_bucket" "firehose_bucket" {
+  count = var.s3_existing_backup_bucket != null ? 0 : 1
   tags   = merge(local.tags, { Name = local.s3_backup_bucket_name })
   bucket = local.s3_backup_bucket_name
 }
 
 resource "aws_s3_bucket_public_access_block" "firehose_bucket_bucket_access" {
-  bucket = aws_s3_bucket.firehose_bucket.id
+  bucket = data.aws_s3_bucket.s3_bucket.id
 
   block_public_acls       = true
   block_public_policy     = true
@@ -104,8 +110,8 @@ resource "aws_iam_role" "firehose_to_coralogix" {
             "s3:PutObject"
           ],
           "Resource" = [
-            aws_s3_bucket.firehose_bucket.arn,
-            "${aws_s3_bucket.firehose_bucket.arn}/*"
+            data.aws_s3_bucket.s3_bucket.arn,
+            "${data.aws_s3_bucket.s3_bucket.arn}/*"
           ]
         },
         {
@@ -155,8 +161,8 @@ resource "aws_iam_policy" "firehose_to_coralogix_metric_policy" {
                 "s3:PutObject"
             ],
             "Resource": [
-                "${aws_s3_bucket.firehose_bucket.arn}",
-                "${aws_s3_bucket.firehose_bucket.arn}/*"
+                "${data.aws_s3_bucket.s3_bucket.arn}",
+                "${data.aws_s3_bucket.s3_bucket.arn}/*"
             ]
         },
         {
@@ -173,7 +179,7 @@ resource "aws_iam_policy" "firehose_to_coralogix_metric_policy" {
                    "kms:ViaService": "s3.${data.aws_region.current_region.name}.amazonaws.com"
                },
                "StringLike": {
-                   "kms:EncryptionContext:aws:s3:arn": "${aws_s3_bucket.firehose_bucket.arn}/prefix*"
+                   "kms:EncryptionContext:aws:s3:arn": "${data.aws_s3_bucket.s3_bucket.arn}/prefix*"
                }
            }
         },
@@ -325,7 +331,7 @@ resource "aws_kinesis_firehose_delivery_stream" "coralogix_stream_metrics" {
 
     s3_configuration {
       role_arn           = aws_iam_role.firehose_to_coralogix.arn
-      bucket_arn         = aws_s3_bucket.firehose_bucket.arn
+      bucket_arn         = data.aws_s3_bucket.s3_bucket.arn
       buffering_size     = 5
       buffering_interval = 300
       compression_format = "GZIP"
diff --git a/modules/firehose-metrics/variables.tf b/modules/firehose-metrics/variables.tf
index 005684ed..ad273e7c 100644
--- a/modules/firehose-metrics/variables.tf
+++ b/modules/firehose-metrics/variables.tf
@@ -163,3 +163,9 @@ variable "s3_backup_custom_name" {
   type        = string
   default     = null
 }
+
+variable "s3_existing_backup_bucket" {
+  description = "Set the name of an existing S3 backup bucket to use, otherwise a new bucket will be created"
+  type        = string
+  default     = null
+}
diff --git a/test.tf b/test.tf
new file mode 100644
index 00000000..896b846b
--- /dev/null
+++ b/test.tf
@@ -0,0 +1,22 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.32"
+    }
+  }
+}
+
+# Configure the AWS Provider
+provider "aws" {
+  region = "us-east-1"
+}
+
+module "cloudwatch_firehose_logs_coralogix" {
+  source                         = "coralogix/aws/coralogix//modules/firehose-logs"
+  firehose_stream                = var.coralogix_firehose_stream_name
+  private_key                    = ""
+  coralogix_region               = "Europe"
+  integration_type_logs          = "RawText"
+  source_type_logs               = "DirectPut"
+}
\ No newline at end of file

From 02e9e25e6b56e0c9a0b4a89a238ab690ba0dac33 Mon Sep 17 00:00:00 2001
From: guyrenny <guy.renny@coralogix.com>
Date: Wed, 1 May 2024 16:46:56 +0300
Subject: [PATCH 02/12] add support for dlq in the shipper module

---
 modules/coralogix-aws-shipper/data.tf      |  6 ++++
 modules/coralogix-aws-shipper/main.tf      | 41 +++++++++++++++++++++-
 modules/coralogix-aws-shipper/variables.tf | 26 ++++++++++++++
 3 files changed, 72 insertions(+), 1 deletion(-)

diff --git a/modules/coralogix-aws-shipper/data.tf b/modules/coralogix-aws-shipper/data.tf
index ac12d2ed..04159961 100644
--- a/modules/coralogix-aws-shipper/data.tf
+++ b/modules/coralogix-aws-shipper/data.tf
@@ -11,11 +11,17 @@ data "aws_cloudwatch_log_group" "this" {
   for_each = local.log_groups
   name     = each.key
 }
+
 data "aws_s3_bucket" "this" {
   count  = var.s3_bucket_name == null ? 0 : 1
   bucket = var.s3_bucket_name
 }
 
+data "aws_s3_bucket" "dlq_bucket" {
+  count  = var.enable_dlq ? 1 : 0
+  bucket = var.dlq_s3_bucket
+}
+
 data "aws_sns_topic" "sns_topic" {
   count = local.sns_enable ? 1 : 0
   name  = var.sns_topic_name
diff --git a/modules/coralogix-aws-shipper/main.tf b/modules/coralogix-aws-shipper/main.tf
index fab71ecc..793f3220 100644
--- a/modules/coralogix-aws-shipper/main.tf
+++ b/modules/coralogix-aws-shipper/main.tf
@@ -32,7 +32,7 @@ resource "null_resource" "s3_bucket_copy" {
 module "lambda" {
   for_each = var.integration_info != null ? var.integration_info : local.integration_info
 
-  depends_on             = [null_resource.s3_bucket_copy]
+  depends_on             = [null_resource.s3_bucket_copy,aws_sqs_queue.DLQ]
   source                 = "terraform-aws-modules/lambda/aws"
   function_name          = each.value.lambda_name == null ? module.locals[each.key].function_name : each.value.lambda_name
   description            = "Send logs to Coralogix."
@@ -59,6 +59,11 @@ module "lambda" {
     ADD_METADATA       = var.add_metadata
     CUSTOM_METADATA    = var.custom_metadata
     CUSTOM_CSV_HEADER  = var.custom_csv_header
+    DLQ_ARN            = var.enable_dlq ? aws_sqs_queue.DLQ[0].arn : null
+    DLQ_RETRY_LIMIT    = var.enable_dlq ? var.dlq_retry_limit : null
+    DLQ_S3_BUCKET      = var.enable_dlq ? var.dlq_s3_bucket : null
+    DLQ_URL            = var.enable_dlq ? aws_sqs_queue.DLQ[0].url : null
+
   }
   s3_existing_package = {
     bucket = var.custom_s3_bucket == "" ? "coralogix-serverless-repo-${data.aws_region.this.name}" : var.custom_s3_bucket
@@ -74,6 +79,24 @@ module "lambda" {
   create_role                             = var.msk_cluster_arn != null ? false : true
   lambda_role                             = var.msk_cluster_arn != null ? aws_iam_role.role_for_msk[0].arn : ""
   policy_statements = {
+    dlq_sqs_permissions = var.enable_dlq ? {
+      effect    = "Allow"
+      actions   = ["sqs:SendMessage","sqs:ReceiveMessage","sqs:DeleteMessage","sqs:GetQueueAttributes"]
+      resources = [aws_sqs_queue.DLQ[0].arn]
+    } : {
+      effect    = "Deny"
+      actions   = ["rds:DescribeAccountAttributes"]
+      resources = ["*"]
+    }
+    dlq_s3_permissions = var.enable_dlq ? {
+      effect    = "Allow"
+      actions   = ["s3:PutObject","s3:PutObjectAcl","s3:AbortMultipartUpload","s3:DeleteObject","s3:PutObjectTagging","s3:PutObjectVersionTagging"]
+      resources = ["${data.aws_s3_bucket.dlq_bucket[0].arn}/*", data.aws_s3_bucket.dlq_bucket[0].arn]
+    } : {
+      effect    = "Deny"
+      actions   = ["rds:DescribeAccountAttributes"]
+      resources = ["*"]
+    }
     secret_access_policy = var.store_api_key_in_secrets_manager || local.api_key_is_arn ? {
       effect    = "Allow"
       actions   = ["secretsmanager:GetSecretValue"]
@@ -246,4 +269,20 @@ resource "aws_vpc_endpoint" "secretsmanager" {
   subnet_ids          = var.subnet_ids
   security_group_ids  = var.security_group_ids
   private_dns_enabled = true
+}
+
+resource "aws_sqs_queue" "DLQ" {
+  count = var.enable_dlq ? 1 : 0
+  name                       = "coralogix-aws-shipper-dlq-${random_string.this.result}"
+  message_retention_seconds  = 1209600
+  delay_seconds              = var.dlq_retry_delay
+  visibility_timeout_seconds = var.timeout
+}
+
+resource "aws_lambda_event_source_mapping" "dlq_sqs" {
+  depends_on       = [module.lambda]
+  count            = var.enable_dlq ? 1 : 0
+  event_source_arn = aws_sqs_queue.DLQ[0].arn
+  function_name    = local.integration_info.integration.lambda_name == null ? module.locals.integration.function_name : local.integration_info.integration.lambda_name
+  enabled          = true
 }
\ No newline at end of file
diff --git a/modules/coralogix-aws-shipper/variables.tf b/modules/coralogix-aws-shipper/variables.tf
index a55968ad..eef4bcca 100644
--- a/modules/coralogix-aws-shipper/variables.tf
+++ b/modules/coralogix-aws-shipper/variables.tf
@@ -128,6 +128,32 @@ variable "cpu_arch" {
   }
 }
 
+# DLQ configuration
+
+variable "enable_dlq" {
+  description = "Enable Dead Letter Queue for the Lambda function"
+  type        = bool
+  default     = false
+}
+
+variable "dlq_retry_limit" {
+  description = "The maximum number of times to retry the function execution in case of failure"
+  type        = number
+  default     = 3
+}
+
+variable "dlq_retry_delay" {
+  description = "The delay in seconds between retries"
+  type        = number
+  default     = 900
+}
+
+variable "dlq_s3_bucket" {
+  description = "The S3 bucket to store the DLQ failed messages after retry limit is reached"
+  type        = string
+  default     = null
+}
+
 # Integration Generic Config (Optional)
 
 variable "notification_email" {

From e8bbe703c43f07a155b7022232098992e84491c2 Mon Sep 17 00:00:00 2001
From: guyrenny <guy.renny@coralogix.com>
Date: Thu, 18 Apr 2024 16:42:30 +0300
Subject: [PATCH 03/12] add option to use dynamic permission for lambda trigger

---
 modules/coralogix-aws-shipper/CloudWatch.tf | 4 ++--
 modules/coralogix-aws-shipper/README.md     | 1 +
 modules/coralogix-aws-shipper/local.tf      | 5 +++++
 modules/coralogix-aws-shipper/variables.tf  | 6 ++++++
 4 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/modules/coralogix-aws-shipper/CloudWatch.tf b/modules/coralogix-aws-shipper/CloudWatch.tf
index d0f63ddb..f306dc07 100644
--- a/modules/coralogix-aws-shipper/CloudWatch.tf
+++ b/modules/coralogix-aws-shipper/CloudWatch.tf
@@ -1,10 +1,10 @@
 resource "aws_lambda_permission" "cloudwatch_trigger_premission" {
   depends_on    = [module.lambda]
-  for_each      = local.log_groups
+  for_each      =  var.log_group_prefix == null ? local.log_groups : local.log_group_prefix 
   action        = "lambda:InvokeFunction"
   function_name = local.integration_info.integration.lambda_name == null ? module.locals.integration.function_name : local.integration_info.integration.lambda_name
   principal     = "logs.amazonaws.com"
-  source_arn    = "${data.aws_cloudwatch_log_group.this[each.key].arn}:*"
+  source_arn    = var.log_group_prefix == null ? "${data.aws_cloudwatch_log_group.this[each.key].arn}:*" : "arn:aws:logs:${data.aws_region.this.name}:${data.aws_caller_identity.this.account_id}:log-group:${local.log_group_prefix[each.value]}*:*"
 }
 
 resource "aws_cloudwatch_log_subscription_filter" "this" {
diff --git a/modules/coralogix-aws-shipper/README.md b/modules/coralogix-aws-shipper/README.md
index cf72ea32..47ecc307 100644
--- a/modules/coralogix-aws-shipper/README.md
+++ b/modules/coralogix-aws-shipper/README.md
@@ -79,6 +79,7 @@ If you want to avoid this issue, you can deploy in other ways:
 | Name | Description | Type | Default | Required | 
 |------|-------------|------|---------|:--------:|
 | <a name="input_log_groups"></a> [log\_groups](#input\_log\_groups) | Provide a comma-separated list of CloudWatch log group names to monitor, for example, (log-group1, log-group2, log-group3). | `list(string)` | n/a | yes |
+| <a name="input_log_group_prefix"></a> [log\_group\_prefix](#input\_log\_group\_prefix) | Prefix of the CloudWatch log groups that will trigger the lambda, in case that your log groups are `log-group1, log-group2, log-group3` then you can set the value to `log-group`. When using this variable you will not be able to see the log groups as trigger for the lambda. | `list(string)` | n/a | no |
 
 ### SNS Configuration
 
diff --git a/modules/coralogix-aws-shipper/local.tf b/modules/coralogix-aws-shipper/local.tf
index 7643d57a..d3917d36 100644
--- a/modules/coralogix-aws-shipper/local.tf
+++ b/modules/coralogix-aws-shipper/local.tf
@@ -10,6 +10,11 @@ locals {
     for group in var.log_groups : group =>
     length(group) > 100 ? "${substr(replace(group, "/", "_"), 0, 95)}_${substr(sha256(group), 0, 4)}" : replace(group, "/", "_")
   }
+  
+  log_group_prefix = var.log_group_prefix != null ? {
+    for group in var.log_group_prefix : group =>
+    length(group) > 100 ? "${substr(replace(group, "/", "_"), 0, 95)}_${substr(sha256(group), 0, 4)}" : replace(group, "/", "_")
+  } : {}
 
   api_key_is_arn = replace(var.api_key, ":", "") != var.api_key ? true : false
 
diff --git a/modules/coralogix-aws-shipper/variables.tf b/modules/coralogix-aws-shipper/variables.tf
index eef4bcca..bd95d3d3 100644
--- a/modules/coralogix-aws-shipper/variables.tf
+++ b/modules/coralogix-aws-shipper/variables.tf
@@ -90,6 +90,12 @@ variable "log_groups" {
   default     = []
 }
 
+variable "log_group_prefix" {
+  description = "Prefix of the CloudWatch log groups that will trigger the lambda"
+  type        = list(string)
+  default     = null
+}
+
 # vpc variables
 
 variable "subnet_ids" {

From 5ba5a6e11768e06761324239d4d08164acc0b5ae Mon Sep 17 00:00:00 2001
From: guyrenny <guy.renny@coralogix.com>
Date: Tue, 30 Apr 2024 13:59:27 +0300
Subject: [PATCH 04/12] pull for master and add depend_on for sns and sqs

---
 modules/coralogix-aws-shipper/Sns.tf | 1 +
 modules/coralogix-aws-shipper/Sqs.tf | 1 +
 2 files changed, 2 insertions(+)

diff --git a/modules/coralogix-aws-shipper/Sns.tf b/modules/coralogix-aws-shipper/Sns.tf
index 42626c5d..e25e17bc 100644
--- a/modules/coralogix-aws-shipper/Sns.tf
+++ b/modules/coralogix-aws-shipper/Sns.tf
@@ -1,4 +1,5 @@
 resource "aws_s3_bucket_notification" "topic_notification" {
+  depends_on = [ module.lambda ]
   count  = local.sns_enable == true && (var.integration_type == "S3" || var.integration_type == "CloudTrail") ? 1 : 0
   bucket = data.aws_s3_bucket.this[0].bucket
   topic {
diff --git a/modules/coralogix-aws-shipper/Sqs.tf b/modules/coralogix-aws-shipper/Sqs.tf
index f2b5047a..5199bc40 100644
--- a/modules/coralogix-aws-shipper/Sqs.tf
+++ b/modules/coralogix-aws-shipper/Sqs.tf
@@ -1,4 +1,5 @@
 resource "aws_s3_bucket_notification" "sqs_notification" {
+  depends_on = [ module.lambda ]
   count  = var.sqs_name != null && (var.integration_type == "S3" || var.integration_type == "CloudTrail") ? 1 : 0
   bucket = data.aws_s3_bucket.this[0].bucket
   queue {

From 44bcb9925a6387a90c81f68ebf569c94897b968e Mon Sep 17 00:00:00 2001
From: guyrenny <guy.renny@coralogix.com>
Date: Tue, 30 Apr 2024 14:58:07 +0300
Subject: [PATCH 05/12] update local variable

---
 modules/coralogix-aws-shipper/local.tf | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/coralogix-aws-shipper/local.tf b/modules/coralogix-aws-shipper/local.tf
index d3917d36..964ab1b3 100644
--- a/modules/coralogix-aws-shipper/local.tf
+++ b/modules/coralogix-aws-shipper/local.tf
@@ -12,8 +12,9 @@ locals {
   }
   
   log_group_prefix = var.log_group_prefix != null ? {
+    # Need to convert the log group prefix to a map so we could use it in the for_each in CloudWatch file
     for group in var.log_group_prefix : group =>
-    length(group) > 100 ? "${substr(replace(group, "/", "_"), 0, 95)}_${substr(sha256(group), 0, 4)}" : replace(group, "/", "_")
+    group
   } : {}
 
   api_key_is_arn = replace(var.api_key, ":", "") != var.api_key ? true : false

From 678790cccf0c20a3cda6e1d0553a9d666f265445 Mon Sep 17 00:00:00 2001
From: guyrenny <guy.renny@coralogix.com>
Date: Thu, 2 May 2024 10:57:14 +0300
Subject: [PATCH 06/12] update actions version to avoid node.js 16 issue

---
 .github/workflows/Changelog.yaml |  4 ++--
 .github/workflows/release.yml    |  2 +-
 .github/workflows/test.yml       |  4 ++--
 test.tf                          | 22 ----------------------
 4 files changed, 5 insertions(+), 27 deletions(-)
 delete mode 100644 test.tf

diff --git a/.github/workflows/Changelog.yaml b/.github/workflows/Changelog.yaml
index f613ad2c..c9d9e2fa 100644
--- a/.github/workflows/Changelog.yaml
+++ b/.github/workflows/Changelog.yaml
@@ -15,7 +15,7 @@ jobs:
     steps:
       - name: Get PR labels
         id: pr-labels
-        uses: joerick/pr-labels-action@v1.0.8
+        uses: joerick/pr-labels-action@v1.0.9
         
   check-changelog-updates:
     if: "${{ needs.get-label.outputs.labels != ' skip-changelog ' }}"
@@ -23,7 +23,7 @@ jobs:
     needs: get-label
     name: Check changelog update
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Get changed files
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0c1a789f..97add490 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -19,7 +19,7 @@ jobs:
     if: github.repository_owner == 'coralogix'
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
           fetch-depth: 0
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0423e4d9..937e5cb6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -15,7 +15,7 @@ jobs:
       packages: ${{ env.packages }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
           fetch-depth: 0
@@ -36,7 +36,7 @@ jobs:
         package: ${{ fromJSON(needs.check.outputs.packages) }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         
       - name: Check for missing test directories
         run: |
diff --git a/test.tf b/test.tf
deleted file mode 100644
index 896b846b..00000000
--- a/test.tf
+++ /dev/null
@@ -1,22 +0,0 @@
-terraform {
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = "~> 5.32"
-    }
-  }
-}
-
-# Configure the AWS Provider
-provider "aws" {
-  region = "us-east-1"
-}
-
-module "cloudwatch_firehose_logs_coralogix" {
-  source                         = "coralogix/aws/coralogix//modules/firehose-logs"
-  firehose_stream                = var.coralogix_firehose_stream_name
-  private_key                    = ""
-  coralogix_region               = "Europe"
-  integration_type_logs          = "RawText"
-  source_type_logs               = "DirectPut"
-}
\ No newline at end of file

From e67e8599ccbb99db3ee52e5453bc507d4b49c0bd Mon Sep 17 00:00:00 2001
From: guyrenny <guy.renny@coralogix.com>
Date: Wed, 8 May 2024 11:51:53 +0300
Subject: [PATCH 07/12] fix issue with dlq

---
 examples/coralogix-aws-shipper/variables.tf | 34 ++++++++++++++++++++-
 modules/coralogix-aws-shipper/main.tf       |  2 ++
 test.tf                                     | 19 ++++++++++++
 3 files changed, 54 insertions(+), 1 deletion(-)
 create mode 100644 test.tf

diff --git a/examples/coralogix-aws-shipper/variables.tf b/examples/coralogix-aws-shipper/variables.tf
index d64492b6..bd95d3d3 100644
--- a/examples/coralogix-aws-shipper/variables.tf
+++ b/examples/coralogix-aws-shipper/variables.tf
@@ -90,6 +90,12 @@ variable "log_groups" {
   default     = []
 }
 
+variable "log_group_prefix" {
+  description = "Prefix of the CloudWatch log groups that will trigger the lambda"
+  type        = list(string)
+  default     = null
+}
+
 # vpc variables
 
 variable "subnet_ids" {
@@ -128,6 +134,32 @@ variable "cpu_arch" {
   }
 }
 
+# DLQ configuration
+
+variable "enable_dlq" {
+  description = "Enable Dead Letter Queue for the Lambda function"
+  type        = bool
+  default     = false
+}
+
+variable "dlq_retry_limit" {
+  description = "The maximum number of times to retry the function execution in case of failure"
+  type        = number
+  default     = 3
+}
+
+variable "dlq_retry_delay" {
+  description = "The delay in seconds between retries"
+  type        = number
+  default     = 900
+}
+
+variable "dlq_s3_bucket" {
+  description = "The S3 bucket to store the DLQ failed messages after retry limit is reached"
+  type        = string
+  default     = null
+}
+
 # Integration Generic Config (Optional)
 
 variable "notification_email" {
@@ -242,7 +274,7 @@ variable "msk_cluster_arn" {
 
 variable "msk_topic_name" {
   description = "List of names of the Kafka topic used to store records in your Kafka cluster ( [\"topic1\", \"topic2\",])"
-  type        = list
+  type        = list(any)
   default     = null
 }
 
diff --git a/modules/coralogix-aws-shipper/main.tf b/modules/coralogix-aws-shipper/main.tf
index 793f3220..95ae0905 100644
--- a/modules/coralogix-aws-shipper/main.tf
+++ b/modules/coralogix-aws-shipper/main.tf
@@ -46,6 +46,8 @@ module "lambda" {
   destination_on_failure = aws_sns_topic.this[each.key].arn
   vpc_subnet_ids         = var.subnet_ids
   vpc_security_group_ids = var.security_group_ids
+  dead_letter_target_arn    = var.enable_dlq ? aws_sqs_queue.DLQ[0].arn : null
+  # attach_dead_letter_policy = var.enable_dlq
   environment_variables = {
     CORALOGIX_ENDPOINT = var.custom_domain != "" ? "https://ingress.${var.custom_domain}" : var.subnet_ids == null ? "https://ingress.${lookup(module.locals[each.key].coralogix_domains, var.coralogix_region, "EU1")}" : "https://ingress.private.${lookup(module.locals[each.key].coralogix_domains, var.coralogix_region, "EU1")}"
     INTEGRATION_TYPE   = each.value.integration_type
diff --git a/test.tf b/test.tf
new file mode 100644
index 00000000..4ee58e6b
--- /dev/null
+++ b/test.tf
@@ -0,0 +1,19 @@
+provider "aws" {
+}
+
+module "coralogix-shipper" {
+  source = "./modules/coralogix-aws-shipper"
+
+  coralogix_region = "EU1"
+  api_key          = "123456778901234"
+  application_name = "TF-cloudwatch"
+  subsystem_name   = "TF-cloudwatch"
+  integration_type = "CloudWatch"
+  log_groups       = ["gr-test"]
+  enable_dlq       = true
+  dlq_s3_bucket    = "gr-integrations-aws-testing"
+  dlq_retry_delay  = 30
+  dlq_retry_limit  = 3
+  timeout          = 30
+  # log_level        = "DEBUG"
+}
\ No newline at end of file

From 0638d42b1548968934cec84767e4b92f4b7c7ad1 Mon Sep 17 00:00:00 2001
From: guyrenny <guy.renny@coralogix.com>
Date: Wed, 15 May 2024 10:50:51 +0300
Subject: [PATCH 08/12]  add dlq to readme

---
 modules/coralogix-aws-shipper/README.md | 13 +++++++++++++
 modules/coralogix-aws-shipper/main.tf   |  2 +-
 modules/firehose-logs/main.tf           | 16 +++++-----------
 modules/firehose-logs/variables.tf      |  8 +-------
 modules/firehose-metrics/main.tf        | 24 +++++++++---------------
 modules/firehose-metrics/variables.tf   |  8 +-------
 test.tf                                 | 19 -------------------
 7 files changed, 30 insertions(+), 60 deletions(-)
 delete mode 100644 test.tf

diff --git a/modules/coralogix-aws-shipper/README.md b/modules/coralogix-aws-shipper/README.md
index 47ecc307..5945c556 100644
--- a/modules/coralogix-aws-shipper/README.md
+++ b/modules/coralogix-aws-shipper/README.md
@@ -151,6 +151,19 @@ When using this variable you will need to create an S3 bucket in the region wher
 | <a name="input_subnet_ids"></a> [vpc\_subnet\_ids](#input\_subnet\_ids) | Specify the ID of the subnet where the integration should be deployed. | `list(string)` | n/a | no |
 | <a name="input_security_group_ids"></a> [security\_group\_ids](#input\_security\_group\_ids) | Specify the ID of the Security Group where the integration should be deployed. | `list(string)` | n/a | no |
 
+### DLQ
+
+A Dead Letter Queue (DLQ) is a queue where messages are sent if they cannot be processed by the Lambda function. This is useful for debugging and monitoring.
+
+ To enable the DLQ, you must provide the required parameters outlined below.
+
+| Parameter       | Description                                                                   | Default Value | Required           |
+|-----------------|-------------------------------------------------------------------------------|---------------|--------------------|
+| enable_dlq      | Enable the Dead Letter Queue for the Lambda function.                         | false         | :heavy_check_mark: |
+| dlq_s3_bucket   | An S3 bucket used to store all failure events that have exhausted retries.    |               | :heavy_check_mark: |
+| dlq_retry_limit | The number of times a failed event should be retried before being saved in S3 | 3             | :heavy_check_mark: |
+| dlq_retry_delay | The delay in seconds between retries of failed events                         | 900           | :heavy_check_mark: |
+
 **AWS PrivateLink**
 
 If you want to bypass using the public internet, you can use AWS PrivateLink to facilitate secure connections between your VPCs and AWS Services. This option is available under [VPC Configuration](#vpc-configuration-optional). For additional instructions on AWS PrivateLink, please [follow our dedicated tutorial](https://coralogix.com/docs/coralogix-amazon-web-services-aws-privatelink-endpoints/).
diff --git a/modules/coralogix-aws-shipper/main.tf b/modules/coralogix-aws-shipper/main.tf
index 95ae0905..5bb0f51f 100644
--- a/modules/coralogix-aws-shipper/main.tf
+++ b/modules/coralogix-aws-shipper/main.tf
@@ -65,7 +65,7 @@ module "lambda" {
     DLQ_RETRY_LIMIT    = var.enable_dlq ? var.dlq_retry_limit : null
     DLQ_S3_BUCKET      = var.enable_dlq ? var.dlq_s3_bucket : null
     DLQ_URL            = var.enable_dlq ? aws_sqs_queue.DLQ[0].url : null
-
+    NO_COLOR           = 1
   }
   s3_existing_package = {
     bucket = var.custom_s3_bucket == "" ? "coralogix-serverless-repo-${data.aws_region.this.name}" : var.custom_s3_bucket
diff --git a/modules/firehose-logs/main.tf b/modules/firehose-logs/main.tf
index f435400b..9645783a 100644
--- a/modules/firehose-logs/main.tf
+++ b/modules/firehose-logs/main.tf
@@ -32,11 +32,6 @@ locals {
 data "aws_caller_identity" "current_identity" {}
 data "aws_region" "current_region" {}
 
-data "aws_s3_bucket" "s3_bucket" {
-  depends_on = [ aws_s3_bucket.firehose_bucket ]
-  bucket     = var.s3_existing_backup_bucket != null ? var.s3_existing_backup_bucket : aws_s3_bucket.firehose_bucket.id
-}
-
 resource "random_string" "this" {
   length  = 6
   special = false
@@ -63,13 +58,12 @@ resource "aws_cloudwatch_log_stream" "firehose_logstream_backup" {
 }
 
 resource "aws_s3_bucket" "firehose_bucket" {
-  count = var.s3_existing_backup_bucket != null ? 0 : 1
   tags   = merge(local.tags, { Name = local.s3_logs_backup_bucket_name })
   bucket = local.s3_logs_backup_bucket_name
 }
 
 resource "aws_s3_bucket_public_access_block" "firehose_bucket_bucket_access" {
-  bucket = data.aws_s3_bucket.s3_bucket.id
+  bucket = aws_s3_bucket.firehose_bucket.id
 
   block_public_acls       = true
   block_public_policy     = true
@@ -108,8 +102,8 @@ resource "aws_iam_role" "firehose_to_coralogix" {
             "s3:PutObject"
           ],
           "Resource" = [
-            data.aws_s3_bucket.s3_bucket.arn,
-            "${data.aws_s3_bucket.s3_bucket.arn}/*"
+            aws_s3_bucket.firehose_bucket.arn,
+            "${aws_s3_bucket.firehose_bucket.arn}/*"
           ]
         },
         {
@@ -165,7 +159,7 @@ resource "aws_kinesis_firehose_delivery_stream" "coralogix_stream_logs" {
 
     s3_configuration {
       role_arn           = aws_iam_role.firehose_to_coralogix.arn
-      bucket_arn         = data.aws_s3_bucket.s3_bucket.arn
+      bucket_arn         = aws_s3_bucket.firehose_bucket.arn
       buffering_size     = 5
       buffering_interval = 300
       compression_format = "GZIP"
@@ -220,4 +214,4 @@ resource "aws_iam_role_policy_attachment" "additional_policy_attachment_1" {
 resource "aws_iam_role_policy_attachment" "additional_policy_attachment_2" {
   role       = aws_iam_role.firehose_to_coralogix.name
   policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
-}
+}
\ No newline at end of file
diff --git a/modules/firehose-logs/variables.tf b/modules/firehose-logs/variables.tf
index 92acb693..16453fc7 100644
--- a/modules/firehose-logs/variables.tf
+++ b/modules/firehose-logs/variables.tf
@@ -76,10 +76,4 @@ variable "s3_backup_custom_name" {
   description = "Set the name of the S3 backup bucket, otherwise variable '{firehose_stream}-backup-logs' will be used"
   type        = string
   default     = null
-}
-
-variable "s3_existing_backup_bucket" {
-  description = "Set the name of an existing S3 backup bucket to use, otherwise a new bucket will be created"
-  type        = string
-  default     = null
-}
+}
\ No newline at end of file
diff --git a/modules/firehose-metrics/main.tf b/modules/firehose-metrics/main.tf
index 20fbfc57..0b3079dd 100644
--- a/modules/firehose-metrics/main.tf
+++ b/modules/firehose-metrics/main.tf
@@ -27,18 +27,13 @@ locals {
 
   # default namings
   cloud_watch_metric_stream_name = var.cloudwatch_metric_stream_custom_name != null ? var.cloudwatch_metric_stream_custom_name : var.firehose_stream
-  s3_backup_bucket_name          = var.s3_backup_custom_name != null ? var.s3_backup_custom_name : var.s3_existing_backup_bucket != null ? var.s3_existing_backup_bucket : "${var.firehose_stream}-backup-metrics"
+  s3_backup_bucket_name          = var.s3_backup_custom_name != null ? var.s3_backup_custom_name : "${var.firehose_stream}-backup-metrics"
   lambda_processor_name          = var.lambda_processor_custom_name != null ? var.lambda_processor_custom_name : "${var.firehose_stream}-metrics-transform"
 }
 
 data "aws_caller_identity" "current_identity" {}
 data "aws_region" "current_region" {}
 
-data "aws_s3_bucket" "s3_bucket" {
-  depends_on = [ aws_s3_bucket.firehose_bucket ]
-  bucket     = var.s3_existing_backup_bucket != null ? var.s3_existing_backup_bucket : aws_s3_bucket.firehose_bucket.id
-}
-
 resource "random_string" "this" {
   length  = 6
   special = false
@@ -65,13 +60,12 @@ resource "aws_cloudwatch_log_stream" "firehose_logstream_backup" {
 }
 
 resource "aws_s3_bucket" "firehose_bucket" {
-  count = var.s3_existing_backup_bucket != null ? 0 : 1
   tags   = merge(local.tags, { Name = local.s3_backup_bucket_name })
   bucket = local.s3_backup_bucket_name
 }
 
 resource "aws_s3_bucket_public_access_block" "firehose_bucket_bucket_access" {
-  bucket = data.aws_s3_bucket.s3_bucket.id
+  bucket = aws_s3_bucket.firehose_bucket.id
 
   block_public_acls       = true
   block_public_policy     = true
@@ -110,8 +104,8 @@ resource "aws_iam_role" "firehose_to_coralogix" {
             "s3:PutObject"
           ],
           "Resource" = [
-            data.aws_s3_bucket.s3_bucket.arn,
-            "${data.aws_s3_bucket.s3_bucket.arn}/*"
+            aws_s3_bucket.firehose_bucket.arn,
+            "${aws_s3_bucket.firehose_bucket.arn}/*"
           ]
         },
         {
@@ -161,8 +155,8 @@ resource "aws_iam_policy" "firehose_to_coralogix_metric_policy" {
                 "s3:PutObject"
             ],
             "Resource": [
-                "${data.aws_s3_bucket.s3_bucket.arn}",
-                "${data.aws_s3_bucket.s3_bucket.arn}/*"
+                "${aws_s3_bucket.firehose_bucket.arn}",
+                "${aws_s3_bucket.firehose_bucket.arn}/*"
             ]
         },
         {
@@ -179,7 +173,7 @@ resource "aws_iam_policy" "firehose_to_coralogix_metric_policy" {
                    "kms:ViaService": "s3.${data.aws_region.current_region.name}.amazonaws.com"
                },
                "StringLike": {
-                   "kms:EncryptionContext:aws:s3:arn": "${data.aws_s3_bucket.s3_bucket.arn}/prefix*"
+                   "kms:EncryptionContext:aws:s3:arn": "${aws_s3_bucket.firehose_bucket.arn}/prefix*"
                }
            }
         },
@@ -331,7 +325,7 @@ resource "aws_kinesis_firehose_delivery_stream" "coralogix_stream_metrics" {
 
     s3_configuration {
       role_arn           = aws_iam_role.firehose_to_coralogix.arn
-      bucket_arn         = data.aws_s3_bucket.s3_bucket.arn
+      bucket_arn         = aws_s3_bucket.firehose_bucket.arn
       buffering_size     = 5
       buffering_interval = 300
       compression_format = "GZIP"
@@ -480,4 +474,4 @@ resource "aws_cloudwatch_metric_stream" "cloudwatch_metric_stream" {
       }
     }
   }
-}
+}
\ No newline at end of file
diff --git a/modules/firehose-metrics/variables.tf b/modules/firehose-metrics/variables.tf
index ad273e7c..e461bef4 100644
--- a/modules/firehose-metrics/variables.tf
+++ b/modules/firehose-metrics/variables.tf
@@ -162,10 +162,4 @@ variable "s3_backup_custom_name" {
   description = "Set the name of the S3 backup bucket, otherwise variable '{firehose_stream}-backup' will be used"
   type        = string
   default     = null
-}
-
-variable "s3_existing_backup_bucket" {
-  description = "Set the name of an existing S3 backup bucket to use, otherwise a new bucket will be created"
-  type        = string
-  default     = null
-}
+}
\ No newline at end of file
diff --git a/test.tf b/test.tf
deleted file mode 100644
index 4ee58e6b..00000000
--- a/test.tf
+++ /dev/null
@@ -1,19 +0,0 @@
-provider "aws" {
-}
-
-module "coralogix-shipper" {
-  source = "./modules/coralogix-aws-shipper"
-
-  coralogix_region = "EU1"
-  api_key          = "123456778901234"
-  application_name = "TF-cloudwatch"
-  subsystem_name   = "TF-cloudwatch"
-  integration_type = "CloudWatch"
-  log_groups       = ["gr-test"]
-  enable_dlq       = true
-  dlq_s3_bucket    = "gr-integrations-aws-testing"
-  dlq_retry_delay  = 30
-  dlq_retry_limit  = 3
-  timeout          = 30
-  # log_level        = "DEBUG"
-}
\ No newline at end of file

From badfc501c89e9360e7d01d2c7d2db9ba737d7eb1 Mon Sep 17 00:00:00 2001
From: guyrenny <guy.renny@coralogix.com>
Date: Wed, 15 May 2024 12:14:24 +0300
Subject: [PATCH 09/12] revert firehose changes

---
 modules/coralogix-aws-shipper/README.md | 12 ++++++------
 modules/firehose-logs/main.tf           |  2 +-
 modules/firehose-logs/variables.tf      |  2 +-
 modules/firehose-metrics/main.tf        |  2 +-
 modules/firehose-metrics/variables.tf   |  4 ++--
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/modules/coralogix-aws-shipper/README.md b/modules/coralogix-aws-shipper/README.md
index 5945c556..d7391a92 100644
--- a/modules/coralogix-aws-shipper/README.md
+++ b/modules/coralogix-aws-shipper/README.md
@@ -157,12 +157,12 @@ A Dead Letter Queue (DLQ) is a queue where messages are sent if they cannot be p
 
  To enable the DLQ, you must provide the required parameters outlined below.
 
-| Parameter       | Description                                                                   | Default Value | Required           |
-|-----------------|-------------------------------------------------------------------------------|---------------|--------------------|
-| enable_dlq      | Enable the Dead Letter Queue for the Lambda function.                         | false         | :heavy_check_mark: |
-| dlq_s3_bucket   | An S3 bucket used to store all failure events that have exhausted retries.    |               | :heavy_check_mark: |
-| dlq_retry_limit | The number of times a failed event should be retried before being saved in S3 | 3             | :heavy_check_mark: |
-| dlq_retry_delay | The delay in seconds between retries of failed events                         | 900           | :heavy_check_mark: |
+| Parameter       | Description                                                                   | Default Value | Required |
+|-----------------|-------------------------------------------------------------------------------|---------------|----------|
+| enable_dlq      | Enable the Dead Letter Queue for the Lambda function.                         | false         | yes      |
+| dlq_s3_bucket   | An S3 bucket used to store all failure events that have exhausted retries.    |               | yes      |
+| dlq_retry_limit | The number of times a failed event should be retried before being saved in S3 | 3             | yes      |
+| dlq_retry_delay | The delay in seconds between retries of failed events                         | 900           | yes      |
 
 **AWS PrivateLink**
 
diff --git a/modules/firehose-logs/main.tf b/modules/firehose-logs/main.tf
index 9645783a..198baa96 100644
--- a/modules/firehose-logs/main.tf
+++ b/modules/firehose-logs/main.tf
@@ -214,4 +214,4 @@ resource "aws_iam_role_policy_attachment" "additional_policy_attachment_1" {
 resource "aws_iam_role_policy_attachment" "additional_policy_attachment_2" {
   role       = aws_iam_role.firehose_to_coralogix.name
   policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
-}
\ No newline at end of file
+}
diff --git a/modules/firehose-logs/variables.tf b/modules/firehose-logs/variables.tf
index 16453fc7..eff608c5 100644
--- a/modules/firehose-logs/variables.tf
+++ b/modules/firehose-logs/variables.tf
@@ -76,4 +76,4 @@ variable "s3_backup_custom_name" {
   description = "Set the name of the S3 backup bucket, otherwise variable '{firehose_stream}-backup-logs' will be used"
   type        = string
   default     = null
-}
\ No newline at end of file
+}
diff --git a/modules/firehose-metrics/main.tf b/modules/firehose-metrics/main.tf
index 0b3079dd..d27f3647 100644
--- a/modules/firehose-metrics/main.tf
+++ b/modules/firehose-metrics/main.tf
@@ -474,4 +474,4 @@ resource "aws_cloudwatch_metric_stream" "cloudwatch_metric_stream" {
       }
     }
   }
-}
\ No newline at end of file
+}
diff --git a/modules/firehose-metrics/variables.tf b/modules/firehose-metrics/variables.tf
index e461bef4..96388002 100644
--- a/modules/firehose-metrics/variables.tf
+++ b/modules/firehose-metrics/variables.tf
@@ -45,7 +45,7 @@ variable "custom_domain" {
 variable "integration_type_metrics" {
   description = "The integration type of the firehose delivery stream: 'CloudWatch_Metrics_OpenTelemetry070' or 'CloudWatch_Metrics_OpenTelemetry070_WithAggregations'"
   type        = string
-  default     = "CloudWatch_Metrics_OpenTelemetry070_WithAggregations"
+  default     = "CloudWatch_Metrics_OpenTelemetry070"
 }
 
 variable "output_format" {
@@ -162,4 +162,4 @@ variable "s3_backup_custom_name" {
   description = "Set the name of the S3 backup bucket, otherwise variable '{firehose_stream}-backup' will be used"
   type        = string
   default     = null
-}
\ No newline at end of file
+}

From 0920fcaa16cc8f9e2b67d848f0ef9b768003bff9 Mon Sep 17 00:00:00 2001
From: guyrenny <guy.renny@coralogix.com>
Date: Wed, 15 May 2024 10:50:51 +0300
Subject: [PATCH 10/12]  add dlq to readme

---
 modules/firehose-logs/main.tf    | 2 +-
 modules/firehose-metrics/main.tf | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/firehose-logs/main.tf b/modules/firehose-logs/main.tf
index 198baa96..9645783a 100644
--- a/modules/firehose-logs/main.tf
+++ b/modules/firehose-logs/main.tf
@@ -214,4 +214,4 @@ resource "aws_iam_role_policy_attachment" "additional_policy_attachment_1" {
 resource "aws_iam_role_policy_attachment" "additional_policy_attachment_2" {
   role       = aws_iam_role.firehose_to_coralogix.name
   policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
-}
+}
\ No newline at end of file
diff --git a/modules/firehose-metrics/main.tf b/modules/firehose-metrics/main.tf
index d27f3647..0b3079dd 100644
--- a/modules/firehose-metrics/main.tf
+++ b/modules/firehose-metrics/main.tf
@@ -474,4 +474,4 @@ resource "aws_cloudwatch_metric_stream" "cloudwatch_metric_stream" {
       }
     }
   }
-}
+}
\ No newline at end of file

From af6296d5cab3970217ff9a55e37ea7338d8fa6cb Mon Sep 17 00:00:00 2001
From: guyrenny <guy.renny@coralogix.com>
Date: Wed, 15 May 2024 12:21:37 +0300
Subject: [PATCH 11/12] add changelog

---
 CHANGELOG.md                          | 7 +++++++
 modules/coralogix-aws-shipper/main.tf | 1 -
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a8141157..debbc73a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,12 @@
 # Changelog
 
+## v1.0.98
+#### **coralogix-aws-shipper**
+### 💡 Enhancements 
+- Add support for DLQ
+- Add log_group_prefix variable to avoid limitation of number of log groups
+- Update versions for github actions to avoid node.js 16 issue
+
 ## v1.0.97
 #### firehose-metrics
 ### 💡 Enhancements 
diff --git a/modules/coralogix-aws-shipper/main.tf b/modules/coralogix-aws-shipper/main.tf
index 5bb0f51f..ed0eaa51 100644
--- a/modules/coralogix-aws-shipper/main.tf
+++ b/modules/coralogix-aws-shipper/main.tf
@@ -65,7 +65,6 @@ module "lambda" {
     DLQ_RETRY_LIMIT    = var.enable_dlq ? var.dlq_retry_limit : null
     DLQ_S3_BUCKET      = var.enable_dlq ? var.dlq_s3_bucket : null
     DLQ_URL            = var.enable_dlq ? aws_sqs_queue.DLQ[0].url : null
-    NO_COLOR           = 1
   }
   s3_existing_package = {
     bucket = var.custom_s3_bucket == "" ? "coralogix-serverless-repo-${data.aws_region.this.name}" : var.custom_s3_bucket

From 00b066262b8bf502c86c241e86a204c29046d4b4 Mon Sep 17 00:00:00 2001
From: guyrenny <guy.renny@coralogix.com>
Date: Wed, 15 May 2024 13:31:02 +0300
Subject: [PATCH 12/12] revert firehose cahnges

---
 modules/firehose-logs/main.tf         | 2 +-
 modules/firehose-metrics/main.tf      | 2 +-
 modules/firehose-metrics/variables.tf | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/firehose-logs/main.tf b/modules/firehose-logs/main.tf
index 9645783a..198baa96 100644
--- a/modules/firehose-logs/main.tf
+++ b/modules/firehose-logs/main.tf
@@ -214,4 +214,4 @@ resource "aws_iam_role_policy_attachment" "additional_policy_attachment_1" {
 resource "aws_iam_role_policy_attachment" "additional_policy_attachment_2" {
   role       = aws_iam_role.firehose_to_coralogix.name
   policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
-}
\ No newline at end of file
+}
diff --git a/modules/firehose-metrics/main.tf b/modules/firehose-metrics/main.tf
index 0b3079dd..d27f3647 100644
--- a/modules/firehose-metrics/main.tf
+++ b/modules/firehose-metrics/main.tf
@@ -474,4 +474,4 @@ resource "aws_cloudwatch_metric_stream" "cloudwatch_metric_stream" {
       }
     }
   }
-}
\ No newline at end of file
+}
diff --git a/modules/firehose-metrics/variables.tf b/modules/firehose-metrics/variables.tf
index 96388002..005684ed 100644
--- a/modules/firehose-metrics/variables.tf
+++ b/modules/firehose-metrics/variables.tf
@@ -45,7 +45,7 @@ variable "custom_domain" {
 variable "integration_type_metrics" {
   description = "The integration type of the firehose delivery stream: 'CloudWatch_Metrics_OpenTelemetry070' or 'CloudWatch_Metrics_OpenTelemetry070_WithAggregations'"
   type        = string
-  default     = "CloudWatch_Metrics_OpenTelemetry070"
+  default     = "CloudWatch_Metrics_OpenTelemetry070_WithAggregations"
 }
 
 variable "output_format" {