From 847990dee2e80015a1ff630b86ed365a6970d51f Mon Sep 17 00:00:00 2001 From: jbeemster Date: Tue, 21 Nov 2023 14:10:58 +0700 Subject: [PATCH] Add support for the SQS sink and buffer (closes #42) --- README.md | 108 +++++++++++++++++++++++-- main.tf | 152 +++++++++++++++++++++++++++--------- templates/config.hocon.tmpl | 6 +- templates/user-data.sh.tmpl | 2 +- variables.tf | 39 +++++++-- 5 files changed, 259 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 3a87af7..7cfd0d4 100644 --- a/README.md +++ b/README.md @@ -108,10 +108,10 @@ module "collector_kinesis" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [bad\_stream\_name](#input\_bad\_stream\_name) | The name of the bad kinesis stream that the collector will insert data into | `string` | n/a | yes | +| [bad\_stream\_name](#input\_bad\_stream\_name) | The name of the bad kinesis/sqs stream that the collector will insert data into | `string` | n/a | yes | | [collector\_lb\_sg\_id](#input\_collector\_lb\_sg\_id) | The ID of the load-balancer security group that sits upstream of the webserver | `string` | n/a | yes | | [collector\_lb\_tg\_id](#input\_collector\_lb\_tg\_id) | The ID of the load-balancer target group to direct traffic from the load-balancer to the webserver | `string` | n/a | yes | -| [good\_stream\_name](#input\_good\_stream\_name) | The name of the good kinesis stream that the collector will insert data into | `string` | n/a | yes | +| [good\_stream\_name](#input\_good\_stream\_name) | The name of the good kinesis/sqs stream that the collector will insert data into | `string` | n/a | yes | | [ingress\_port](#input\_ingress\_port) | The port that the collector will be bound to and expose over HTTP | `number` | n/a | yes | | [name](#input\_name) | A name which will be pre-pended to the resources created | `string` | n/a | yes | | [ssh\_key\_name](#input\_ssh\_key\_name) | The name of the preexisting SSH key-pair to attach to all EC2 nodes deployed | `string` | n/a | yes | @@ -120,7 +120,8 @@ module "collector_kinesis" { | [amazon\_linux\_2\_ami\_id](#input\_amazon\_linux\_2\_ami\_id) | The AMI ID to use which must be based of of Amazon Linux 2; by default the latest community version is used | `string` | `""` | no | | [app\_version](#input\_app\_version) | App version to use. This variable facilitates dev flow, the modules may not work with anything other than the default value. | `string` | `"2.9.2"` | no | | [associate\_public\_ip\_address](#input\_associate\_public\_ip\_address) | Whether to assign a public ip address to this instance | `bool` | `true` | no | -| [byte\_limit](#input\_byte\_limit) | The amount of bytes to buffer events before pushing them to Kinesis | `number` | `1000000` | no | +| [bad\_sqs\_buffer\_name](#input\_bad\_sqs\_buffer\_name) | The name of the bad sqs queue to use as an overflow buffer for kinesis | `string` | `""` | no | +| [byte\_limit](#input\_byte\_limit) | The amount of bytes to buffer events before pushing them downstream | `number` | `1000000` | no | | [cloudwatch\_logs\_enabled](#input\_cloudwatch\_logs\_enabled) | Whether application logs should be reported to CloudWatch | `bool` | `true` | no | | [cloudwatch\_logs\_retention\_days](#input\_cloudwatch\_logs\_retention\_days) | The length of time in days to retain logs for | `number` | `7` | no | | [config\_override\_b64](#input\_config\_override\_b64) | App config uploaded as a base64 encoded blob. This variable facilitates dev flow, if config is incorrect this can break the deployment. | `string` | `""` | no | @@ -128,22 +129,119 @@ module "collector_kinesis" { | [cookie\_enabled](#input\_cookie\_enabled) | Whether server side cookies are enabled or not | `bool` | `true` | no | | [custom\_paths](#input\_custom\_paths) | Optional custom paths that the collector will respond to, typical paths to override are '/com.snowplowanalytics.snowplow/tp2', '/com.snowplowanalytics.iglu/v1' and '/r/tp2'. e.g. { "/custom/path/" : "/com.snowplowanalytics.snowplow/tp2"} | `map(string)` | `{}` | no | | [enable\_auto\_scaling](#input\_enable\_auto\_scaling) | Whether to enable auto-scaling policies for the service | `bool` | `true` | no | +| [enable\_sqs\_buffer](#input\_enable\_sqs\_buffer) | Whether to enable the optional sqs overflow buffer for kinesis (note: only works when 'sink\_type' is 'kinesis') | `bool` | `false` | no | +| [good\_sqs\_buffer\_name](#input\_good\_sqs\_buffer\_name) | The name of the good sqs queue to use as an overflow buffer for kinesis | `string` | `""` | no | | [iam\_permissions\_boundary](#input\_iam\_permissions\_boundary) | The permissions boundary ARN to set on IAM roles created | `string` | `""` | no | | [instance\_type](#input\_instance\_type) | The instance type to use | `string` | `"t3a.micro"` | no | | [java\_opts](#input\_java\_opts) | Custom JAVA Options | `string` | `"-Dcom.amazonaws.sdk.disableCbor -XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75"` | no | | [max\_size](#input\_max\_size) | The maximum number of servers in this server-group | `number` | `2` | no | | [min\_size](#input\_min\_size) | The minimum number of servers in this server-group | `number` | `1` | no | -| [record\_limit](#input\_record\_limit) | The number of events to buffer before pushing them to Kinesis | `number` | `500` | no | +| [record\_limit](#input\_record\_limit) | The number of events to buffer before pushing them downstream | `number` | `500` | no | | [scale\_down\_cooldown\_sec](#input\_scale\_down\_cooldown\_sec) | Time (in seconds) until another scale-down action can occur | `number` | `600` | no | | [scale\_down\_cpu\_threshold\_percentage](#input\_scale\_down\_cpu\_threshold\_percentage) | The average CPU percentage that we must be below to scale-down | `number` | `20` | no | | [scale\_down\_eval\_minutes](#input\_scale\_down\_eval\_minutes) | The number of consecutive minutes that we must be below the threshold to scale-down | `number` | `60` | no | | [scale\_up\_cooldown\_sec](#input\_scale\_up\_cooldown\_sec) | Time (in seconds) until another scale-up action can occur | `number` | `180` | no | | [scale\_up\_cpu\_threshold\_percentage](#input\_scale\_up\_cpu\_threshold\_percentage) | The average CPU percentage that must be exceeded to scale-up | `number` | `60` | no | | [scale\_up\_eval\_minutes](#input\_scale\_up\_eval\_minutes) | The number of consecutive minutes that the threshold must be breached to scale-up | `number` | `5` | no | +| [sink\_type](#input\_sink\_type) | The stream technology to push messages into (either 'kinesis' or 'sqs') | `string` | `"kinesis"` | no | | [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | 
[
  "0.0.0.0/0"
]
| no | | [tags](#input\_tags) | The tags to append to this resource | `map(string)` | `{}` | no | | [telemetry\_enabled](#input\_telemetry\_enabled) | Whether or not to send telemetry information back to Snowplow Analytics Ltd | `bool` | `true` | no | -| [time\_limit\_ms](#input\_time\_limit\_ms) | The amount of time to buffer events before pushing them to Kinesis | `number` | `500` | no | +| [time\_limit\_ms](#input\_time\_limit\_ms) | The amount of time to buffer events before pushing them downstream | `number` | `500` | no | +| [user\_provided\_id](#input\_user\_provided\_id) | An optional unique identifier to identify the telemetry events emitted by this stack | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [asg\_id](#output\_asg\_id) | ID of the ASG | +| [asg\_name](#output\_asg\_name) | Name of the ASG | +| [sg\_id](#output\_sg\_id) | ID of the security group attached to the Collector Server node | +jbeemster@Joshuas-MBP in .../Github/terraform-aws-collector-kinesis-ec2 $ tfdocs +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [aws](#requirement\_aws) | >= 3.72.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | 5.17.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance\_type\_metrics](#module\_instance\_type\_metrics) | snowplow-devops/ec2-instance-type-metrics/aws | 0.1.2 | +| [service](#module\_service) | snowplow-devops/service-ec2/aws | 0.2.0 | +| [telemetry](#module\_telemetry) | snowplow-devops/telemetry/snowplow | 0.5.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_iam_instance_profile.instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_policy.iam_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_security_group.sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.egress_tcp_443](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.egress_tcp_80](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.egress_udp_123](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.ingress_tcp_22](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.ingress_tcp_webserver](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.lb_egress_tcp_webserver](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [bad\_stream\_name](#input\_bad\_stream\_name) | The name of the bad kinesis/sqs stream that the collector will insert data into | `string` | n/a | yes | +| [collector\_lb\_sg\_id](#input\_collector\_lb\_sg\_id) | The ID of the load-balancer security group that sits upstream of the webserver | `string` | n/a | yes | +| [collector\_lb\_tg\_id](#input\_collector\_lb\_tg\_id) | The ID of the load-balancer target group to direct traffic from the load-balancer to the webserver | `string` | n/a | yes | +| [good\_stream\_name](#input\_good\_stream\_name) | The name of the good kinesis/sqs stream that the collector will insert data into | `string` | n/a | yes | +| [ingress\_port](#input\_ingress\_port) | The port that the collector will be bound to and expose over HTTP | `number` | n/a | yes | +| [name](#input\_name) | A name which will be pre-pended to the resources created | `string` | n/a | yes | +| [ssh\_key\_name](#input\_ssh\_key\_name) | The name of the preexisting SSH key-pair to attach to all EC2 nodes deployed | `string` | n/a | yes | +| [subnet\_ids](#input\_subnet\_ids) | The list of at least two subnets in different availability zones to deploy the collector across | `list(string)` | n/a | yes | +| [vpc\_id](#input\_vpc\_id) | The VPC to deploy the collector within | `string` | n/a | yes | +| [amazon\_linux\_2\_ami\_id](#input\_amazon\_linux\_2\_ami\_id) | The AMI ID to use which must be based of of Amazon Linux 2; by default the latest community version is used | `string` | `""` | no | +| [app\_version](#input\_app\_version) | App version to use. This variable facilitates dev flow, the modules may not work with anything other than the default value. | `string` | `"2.9.2"` | no | +| [associate\_public\_ip\_address](#input\_associate\_public\_ip\_address) | Whether to assign a public ip address to this instance | `bool` | `true` | no | +| [bad\_sqs\_buffer\_name](#input\_bad\_sqs\_buffer\_name) | The name of the bad sqs queue to use as an overflow buffer for kinesis | `string` | `""` | no | +| [byte\_limit](#input\_byte\_limit) | The amount of bytes to buffer events before pushing them downstream | `number` | `1000000` | no | +| [cloudwatch\_logs\_enabled](#input\_cloudwatch\_logs\_enabled) | Whether application logs should be reported to CloudWatch | `bool` | `true` | no | +| [cloudwatch\_logs\_retention\_days](#input\_cloudwatch\_logs\_retention\_days) | The length of time in days to retain logs for | `number` | `7` | no | +| [config\_override\_b64](#input\_config\_override\_b64) | App config uploaded as a base64 encoded blob. This variable facilitates dev flow, if config is incorrect this can break the deployment. | `string` | `""` | no | +| [cookie\_domain](#input\_cookie\_domain) | Optional first party cookie domain for the collector to set cookies on (e.g. acme.com) | `string` | `""` | no | +| [cookie\_enabled](#input\_cookie\_enabled) | Whether server side cookies are enabled or not | `bool` | `true` | no | +| [custom\_paths](#input\_custom\_paths) | Optional custom paths that the collector will respond to, typical paths to override are '/com.snowplowanalytics.snowplow/tp2', '/com.snowplowanalytics.iglu/v1' and '/r/tp2'. e.g. { "/custom/path/" : "/com.snowplowanalytics.snowplow/tp2"} | `map(string)` | `{}` | no | +| [enable\_auto\_scaling](#input\_enable\_auto\_scaling) | Whether to enable auto-scaling policies for the service | `bool` | `true` | no | +| [enable\_sqs\_buffer](#input\_enable\_sqs\_buffer) | Whether to enable the optional sqs overflow buffer for kinesis (note: only works when 'sink\_type' is 'kinesis') | `bool` | `false` | no | +| [good\_sqs\_buffer\_name](#input\_good\_sqs\_buffer\_name) | The name of the good sqs queue to use as an overflow buffer for kinesis | `string` | `""` | no | +| [iam\_permissions\_boundary](#input\_iam\_permissions\_boundary) | The permissions boundary ARN to set on IAM roles created | `string` | `""` | no | +| [instance\_type](#input\_instance\_type) | The instance type to use | `string` | `"t3a.micro"` | no | +| [java\_opts](#input\_java\_opts) | Custom JAVA Options | `string` | `"-Dcom.amazonaws.sdk.disableCbor -XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75"` | no | +| [max\_size](#input\_max\_size) | The maximum number of servers in this server-group | `number` | `2` | no | +| [min\_size](#input\_min\_size) | The minimum number of servers in this server-group | `number` | `1` | no | +| [record\_limit](#input\_record\_limit) | The number of events to buffer before pushing them downstream | `number` | `500` | no | +| [scale\_down\_cooldown\_sec](#input\_scale\_down\_cooldown\_sec) | Time (in seconds) until another scale-down action can occur | `number` | `600` | no | +| [scale\_down\_cpu\_threshold\_percentage](#input\_scale\_down\_cpu\_threshold\_percentage) | The average CPU percentage that we must be below to scale-down | `number` | `20` | no | +| [scale\_down\_eval\_minutes](#input\_scale\_down\_eval\_minutes) | The number of consecutive minutes that we must be below the threshold to scale-down | `number` | `60` | no | +| [scale\_up\_cooldown\_sec](#input\_scale\_up\_cooldown\_sec) | Time (in seconds) until another scale-up action can occur | `number` | `180` | no | +| [scale\_up\_cpu\_threshold\_percentage](#input\_scale\_up\_cpu\_threshold\_percentage) | The average CPU percentage that must be exceeded to scale-up | `number` | `60` | no | +| [scale\_up\_eval\_minutes](#input\_scale\_up\_eval\_minutes) | The number of consecutive minutes that the threshold must be breached to scale-up | `number` | `5` | no | +| [sink\_type](#input\_sink\_type) | The stream technology to push messages into (either 'kinesis' or 'sqs') | `string` | `"kinesis"` | no | +| [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | 
[
  "0.0.0.0/0"
]
| no | +| [tags](#input\_tags) | The tags to append to this resource | `map(string)` | `{}` | no | +| [telemetry\_enabled](#input\_telemetry\_enabled) | Whether or not to send telemetry information back to Snowplow Analytics Ltd | `bool` | `true` | no | +| [time\_limit\_ms](#input\_time\_limit\_ms) | The amount of time to buffer events before pushing them downstream | `number` | `500` | no | | [user\_provided\_id](#input\_user\_provided\_id) | An optional unique identifier to identify the telemetry events emitted by this stack | `string` | `""` | no | ## Outputs diff --git a/main.tf b/main.tf index b28e734..b81ad78 100644 --- a/main.tf +++ b/main.tf @@ -1,5 +1,5 @@ locals { - module_name = "collector-kinesis-ec2" + module_name = "collector-${var.sink_type}-ec2" module_version = "0.7.0" app_name = "stream-collector" @@ -73,40 +73,115 @@ EOF permissions_boundary = var.iam_permissions_boundary } -resource "aws_iam_policy" "iam_policy" { - name = var.name +locals { + region = data.aws_region.current.name + account_id = data.aws_caller_identity.current.account_id + + kinesis_arn_list = formatlist( + "arn:aws:kinesis:%s:%s:stream/%s", local.region, local.account_id, + compact(tolist([ + var.good_stream_name, + var.bad_stream_name + ])) + ) - policy = <