From 4e1317cbd8bd0abfb7dbde9994ea1ce68f85fb12 Mon Sep 17 00:00:00 2001 From: Soren Martius Date: Fri, 12 Apr 2024 18:36:36 -0700 Subject: [PATCH 01/11] chore: remove github templates --- .github/ISSUE_TEMPLATE/bug_report.md | 33 ----------------------- .github/ISSUE_TEMPLATE/feature_request.md | 22 --------------- 2 files changed, 55 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index ca10044..0000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -name: Bug report -about: Create a bug report to help us improve. -title: '' -labels: bug -assignees: '' - ---- - - - -**Describe the bug** -A clear and concise description of what the bug is. - -**To Reproduce** -Steps to reproduce the behavior including the relevant Terraform/OpenTofu/Terragrunt/Packer version number and any code snippets and module inputs you used. - -```hcl -// paste code snippets here -``` - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Nice to have** -- [ ] Terminal output -- [ ] Screenshots - -**Additional context** -Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 023a330..0000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -name: Feature request -about: Submit a feature request for this repo. -title: '' -labels: enhancement -assignees: '' - ---- - - - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. From f0280dba69b830c2acf4e3638870b1ffb15f4dd6 Mon Sep 17 00:00:00 2001 From: Soren Martius Date: Fri, 12 Apr 2024 18:37:01 -0700 Subject: [PATCH 02/11] feat: update CODEOWNERS --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 898ab61..db48573 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @brikis98 @denis256 \ No newline at end of file +* @terramate-io/engineering From e759654999960ccad98b8513fed48d03f48fb000 Mon Sep 17 00:00:00 2001 From: Soren Martius Date: Fri, 12 Apr 2024 18:37:19 -0700 Subject: [PATCH 03/11] feat: update LICENSE --- LICENSE.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/LICENSE.txt b/LICENSE.txt index 77300ed..083730f 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -187,6 +187,7 @@ identification within third-party archives. Copyright 2016 Gruntwork, Inc + Copyright 2024 Terramate GmbH Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From fb4233dd591c9fc7f73c8e2670cc74a53b9cf94d Mon Sep 17 00:00:00 2001 From: Soren Martius Date: Fri, 12 Apr 2024 18:37:30 -0700 Subject: [PATCH 04/11] ci: update permissions in workflows --- .github/workflows/deploy.yml | 7 +++++-- .github/workflows/drift-detection.yml | 5 ++++- .github/workflows/preview.yml | 6 ++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6359a67..4e167eb 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -16,6 +16,7 @@ jobs: id-token: write contents: read pull-requests: read + checks: read runs-on: ubuntu-latest @@ -89,7 +90,8 @@ jobs: --cloud-sync-deployment \ --cloud-sync-terraform-plan-file=out.tfplan \ --terragrunt \ - -- terragrunt apply -input=false -auto-approve -lock-timeout=5m out.tfplan + -- \ + terragrunt apply -input=false -auto-approve -lock-timeout=5m out.tfplan env: TF_VAR_master_password: ${{ secrets.MYSQL_PROD_MASTER_PASSWORD }} @@ -101,7 +103,8 @@ jobs: --cloud-sync-drift-status \ --cloud-sync-terraform-plan-file=drift.tfplan \ --terragrunt \ - -- terragrunt plan -out drift.tfplan -detailed-exitcode + -- \ + terragrunt plan -out drift.tfplan -detailed-exitcode env: TF_VAR_master_password: ${{ secrets.MYSQL_PROD_MASTER_PASSWORD }} diff --git a/.github/workflows/drift-detection.yml b/.github/workflows/drift-detection.yml index 309ec3c..f6fa7d7 100644 --- a/.github/workflows/drift-detection.yml +++ b/.github/workflows/drift-detection.yml @@ -16,6 +16,7 @@ jobs: id-token: write contents: read pull-requests: read + checks: read runs-on: ubuntu-latest @@ -67,6 +68,7 @@ jobs: id: drift-detect run: | terramate run \ + --parallel 5 --cloud-sync-drift-status \ --cloud-sync-terraform-plan-file=drift.tfplan \ --terragrunt \ @@ -93,6 +95,7 @@ jobs: --cloud-sync-deployment \ --cloud-sync-terraform-plan-file=drift.tfplan \ --terragrunt \ - -- terragrunt apply -input=false -auto-approve -lock-timeout=5m drift.tfplan + -- \ + terragrunt apply -input=false -auto-approve -lock-timeout=5m drift.tfplan env: TF_VAR_master_password: ${{ secrets.MYSQL_PROD_MASTER_PASSWORD }} diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index eb30750..cdf05fa 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -99,7 +99,8 @@ jobs: --debug-preview-url preview_url.txt \ --continue-on-error \ --terragrunt \ - -- terragrunt plan -out out.tfplan \ + -- \ + terragrunt plan -out out.tfplan \ -detailed-exitcode \ -lock=false env: @@ -127,7 +128,8 @@ jobs: terramate run \ --changed \ --terragrunt \ - -- terragrunt show -no-color out.tfplan \ + -- \ + terragrunt show -no-color out.tfplan \ |& dd bs=1024 count=248 >>pr-comment.txt [ "${PIPESTATUS[0]}" == "141" ] && sed -i 's/#### Terragrunt Plan/#### :warning: Terragrunt Plan truncated: please check console output :warning:/' pr-comment.txt echo >>pr-comment.txt '```' From 5d5c2bd4dd13004d4a26a8bea8d9f404b9cc6575 Mon Sep 17 00:00:00 2001 From: Soren Martius Date: Tue, 16 Apr 2024 03:38:25 +0200 Subject: [PATCH 05/11] feat: upgrade terramate to v0.6.1 --- .tool-versions | 2 +- terramate.tm.hcl | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.tool-versions b/.tool-versions index 159e425..4dd105d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ terragrunt 0.55.14 -terramate 0.6.0 +terramate 0.6.1 terraform 1.7.5 diff --git a/terramate.tm.hcl b/terramate.tm.hcl index 0d4162c..dff012c 100644 --- a/terramate.tm.hcl +++ b/terramate.tm.hcl @@ -5,10 +5,6 @@ terramate { organization = "terramate-demo" } - experiments = [ - "terragrunt", - ] - run { env { TG_BUCKET_PREFIX = "tmcd-" From 7091718ad61f5a79a4876b22620e9ff0f61bd9f1 Mon Sep 17 00:00:00 2001 From: Soren Martius Date: Tue, 16 Apr 2024 03:48:05 +0200 Subject: [PATCH 06/11] fix: drift detection command --- .github/workflows/drift-detection.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/drift-detection.yml b/.github/workflows/drift-detection.yml index f6fa7d7..f603115 100644 --- a/.github/workflows/drift-detection.yml +++ b/.github/workflows/drift-detection.yml @@ -68,7 +68,7 @@ jobs: id: drift-detect run: | terramate run \ - --parallel 5 + --parallel 5 \ --cloud-sync-drift-status \ --cloud-sync-terraform-plan-file=drift.tfplan \ --terragrunt \ From 949b98608f3f62ee0d19f7252b0ff537ec9c0371 Mon Sep 17 00:00:00 2001 From: Soren Martius Date: Tue, 16 Apr 2024 06:21:35 +0200 Subject: [PATCH 07/11] feat: add new readme.md docs: add new readme --- README.md | 156 +++++------------------------------------------------- 1 file changed, 12 insertions(+), 144 deletions(-) diff --git a/README.md b/README.md index 8374ae6..d39c42d 100644 --- a/README.md +++ b/README.md @@ -1,149 +1,17 @@ -[![Maintained by Gruntwork.io](https://img.shields.io/badge/maintained%20by-gruntwork.io-%235849a6.svg)](https://gruntwork.io/?ref=repo_terragrunt-infra-live-example) +# Example of how to integrate Terragrunt with Terramate -# Example infrastructure-live for Terragrunt +This repository is a fork of the [Terragrunt reference architecture](https://github.com/gruntwork-io/terragrunt-infrastructure-live-example) +to demonstrate how to enhance Terragrunt with Terramate by adding: -This repo, along with the [terragrunt-infrastructure-modules-example -repo](https://github.com/gruntwork-io/terragrunt-infrastructure-modules-example), show an example file/folder structure -you can use with [Terragrunt](https://github.com/gruntwork-io/terragrunt) to keep your -[Terraform](https://www.terraform.io) and [OpenTofu](https://opentofu.org/) code DRY. For background information, -check out the [Keep your code DRY](https://github.com/gruntwork-io/terragrunt#keep-your-terraform-code-dry) -section of the Terragrunt documentation. +- **Orchestration with Change Detection** to automatically detect and execute modules that contain changes only using +`terragrunt plan` and `terragrunt apply`. +- **GitOps automation workflows in GitHub Actions** (or any other CI/CD) to automate Terragrunt with plan previews in +Pull Requests in your CI/CD without requiring any additional tooling such as Atlantis. +- **Drift detection and reconciliation** to keep your Terragrunt modules drift-free with scheduled workflows in GitHub actions. +- **Observability and Visibility** to understand the health and infrastructure managed in your modules. -This repo shows an example of how to use the modules from the `terragrunt-infrastructure-modules-example` repo to -deploy an Auto Scaling Group (ASG) and a MySQL DB across three environments (qa, stage, prod) and two AWS accounts -(non-prod, prod), all with minimal duplication of code. That's because there is just a single copy of -the code, defined in the `terragrunt-infrastructure-modules-example` repo, and in this repo, we solely define -`terragrunt.hcl` files that reference that code (at a specific version, too!) and fill in variables specific to each -environment. +Please read our [Terramate and Terragrunt guide](https://terramate.io/rethinking-iac/how-terramate-adds-superpowers-to-terragrunt-in-just-5-minutes/) to learn more about how this repository works and how you can use Terramate to supercharge Terragrunt. -Be sure to read through [the Terragrunt documentation on DRY -Architectures](https://terragrunt.gruntwork.io/docs/features/keep-your-terragrunt-architecture-dry/) to understand the -features of Terragrunt used in this folder organization. +## Terragrunt reference architecture documentation -Note: This code is solely for demonstration purposes. This is not production-ready code, so use at your own risk. If -you are interested in battle-tested, production-ready Terraform code, check out [Gruntwork](http://www.gruntwork.io/). - - - - -## How do you deploy the infrastructure in this repo? - - -### Pre-requisites - -1. Install [OpenTofu](https://opentofu.org/) version `1.6.0` or newer and - [Terragrunt](https://github.com/gruntwork-io/terragrunt) version `v0.52.0` or newer. -2. Update the `bucket` parameter in the root `terragrunt.hcl`. We use S3 [as a Terraform - backend](https://opentofu.org/docs/language/settings/backends/s3/) to store your - state, and S3 bucket names must be globally unique. The name currently in - the file is already taken, so you'll have to specify your own. Alternatives, you can - set the environment variable `TG_BUCKET_PREFIX` to set a custom prefix. -3. Update the `account_name` and `aws_account_id` parameters in [`non-prod/account.hcl`](/non-prod/account.hcl) and - [`prod/account.hcl`](/prod/account.hcl) with the names and IDs of accounts you want to use for non production and - production workloads, respectively. -4. Configure your AWS credentials using one of the supported [authentication - mechanisms](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html). - - -### Deploying a single module - -1. `cd` into the module's folder (e.g. `cd non-prod/us-east-1/qa/mysql`). -1. Note: if you're deploying the MySQL DB, you'll need to configure your DB password as an environment variable: - `export TF_VAR_master_password=(...)`. -1. Run `terragrunt plan` to see the changes you're about to apply. -1. If the plan looks good, run `terragrunt apply`. - - -### Deploying all modules in a region - -1. `cd` into the region folder (e.g. `cd non-prod/us-east-1`). -1. Configure the password for the MySQL DB as an environment variable: `export TF_VAR_master_password=(...)`. -1. Run `terragrunt run-all plan` to see all the changes you're about to apply. -1. If the plan looks good, run `terragrunt run-all apply`. - - -### Testing the infrastructure after it's deployed - -After each module is finished deploying, it will write a bunch of outputs to the screen. For example, the ASG will -output something like the following: - -``` -Outputs: - -asg_name = tf-asg-00343cdb2415e9d5f20cda6620 -asg_security_group_id = sg-d27df1a3 -elb_dns_name = webserver-example-prod-1234567890.us-east-1.elb.amazonaws.com -elb_security_group_id = sg-fe62ee8f -url = http://webserver-example-prod-1234567890.us-east-1.elb.amazonaws.com:80 -``` - -A minute or two after the deployment finishes, and the servers in the ASG have passed their health checks, you should -be able to test the `url` output in your browser or with `curl`: - -``` -curl http://webserver-example-prod-1234567890.us-east-1.elb.amazonaws.com:80 - -Hello, World -``` - -Similarly, the MySQL module produces outputs that will look something like this: - -``` -Outputs: - -arn = arn:aws:rds:us-east-1:1234567890:db:tofu-00d7a11c1e02cf617f80bbe301 -db_name = mysql_prod -endpoint = tofu-1234567890.abcdefghijklmonp.us-east-1.rds.amazonaws.com:3306 -``` - -You can use the `endpoint` and `db_name` outputs with any MySQL client: - -``` -mysql --host=tofu-1234567890.abcdefghijklmonp.us-east-1.rds.amazonaws.com:3306 --user=admin --password mysql_prod -``` - - - - - - -## How is the code in this repo organized? - -The code in this repo uses the following folder hierarchy: - -``` -account - └ _global - └ region - └ _global - └ environment - └ resource -``` - -Where: - -* **Account**: At the top level are each of your AWS accounts, such as `stage-account`, `prod-account`, `mgmt-account`, - etc. If you have everything deployed in a single AWS account, there will just be a single folder at the root (e.g. - `main-account`). - -* **Region**: Within each account, there will be one or more [AWS - regions](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html), such as - `us-east-1`, `eu-west-1`, and `ap-southeast-2`, where you've deployed resources. There may also be a `_global` - folder that defines resources that are available across all the AWS regions in this account, such as IAM users, - Route 53 hosted zones, and CloudTrail. - -* **Environment**: Within each region, there will be one or more "environments", such as `qa`, `stage`, etc. Typically, - an environment will correspond to a single [AWS Virtual Private Cloud (VPC)](https://aws.amazon.com/vpc/), which - isolates that environment from everything else in that AWS account. There may also be a `_global` folder - that defines resources that are available across all the environments in this AWS region, such as Route 53 A records, - SNS topics, and ECR repos. - -* **Resource**: Within each environment, you deploy all the resources for that environment, such as EC2 Instances, Auto - Scaling Groups, ECS Clusters, Databases, Load Balancers, and so on. Note that the code for most of these - resources lives in the [terragrunt-infrastructure-modules-example repo](https://github.com/gruntwork-io/terragrunt-infrastructure-modules-example). - -## Creating and using root (account) level variables - -In the situation where you have multiple AWS accounts or regions, you often have to pass common variables down to each -of your modules. Rather than copy/pasting the same variables into each `terragrunt.hcl` file, in every region, and in -every environment, you can inherit them from the `inputs` defined in the root `terragrunt.hcl` file. +To learn more about Terragrunt and the infrastructure-live example, please look at the [terragrunt-infrastructure-live-example](https://github.com/gruntwork-io/terragrunt-infrastructure-live-example) repository. From 53e06738f408d9e929cf3a9cfc42411b85d8a055 Mon Sep 17 00:00:00 2001 From: Soren Martius Date: Tue, 16 Apr 2024 06:27:36 +0200 Subject: [PATCH 08/11] chore: remove PR template --- .github/pull_request_template.md | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 6b100e4..0000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,25 +0,0 @@ - - -## Description - -Fixes #000. - - - -## TODOs - -Read the [Gruntwork contribution guidelines](https://gruntwork.notion.site/Gruntwork-Coding-Methodology-02fdcd6e4b004e818553684760bf691e). - -- [ ] Update the docs. -- [ ] Run the relevant tests successfully, including pre-commit checks. -- [ ] Ensure any 3rd party code adheres with our [license policy](https://www.notion.so/gruntwork/Gruntwork-licenses-and-open-source-usage-policy-f7dece1f780341c7b69c1763f22b1378) or delete this line if its not applicable. -- [ ] Include release notes. If this PR is backward incompatible, include a migration guide. - -## Release Notes (draft) - - -Added / Removed / Updated [X]. - -### Migration Guide - - From 2834f030f6078aaed608b65724f22a417a67e35d Mon Sep 17 00:00:00 2001 From: Soren Martius Date: Wed, 17 Apr 2024 10:38:35 +0200 Subject: [PATCH 09/11] feat: increase storage of the prod database to 120gb --- prod/us-east-1/prod/mysql/terragrunt.hcl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prod/us-east-1/prod/mysql/terragrunt.hcl b/prod/us-east-1/prod/mysql/terragrunt.hcl index be74778..69ba4af 100644 --- a/prod/us-east-1/prod/mysql/terragrunt.hcl +++ b/prod/us-east-1/prod/mysql/terragrunt.hcl @@ -30,7 +30,7 @@ terraform { # For production, we want to specify bigger instance classes and storage, so we specify override parameters here. These # inputs get merged with the common inputs from the root and the envcommon terragrunt.hcl -# inputs = { -# instance_class = "db.m5d.large" -# allocated_storage = 120 -# } +inputs = { + # instance_class = "db.m5d.large" + allocated_storage = 120 +} From 0c8999735374d3564460b76f7dc261d7d508c27d Mon Sep 17 00:00:00 2001 From: Soren Martius Date: Tue, 30 Apr 2024 11:59:23 +0200 Subject: [PATCH 10/11] feat: upgrade dependencies --- .github/workflows/deploy.yml | 8 ++++---- .github/workflows/drift-detection.yml | 4 ++-- .github/workflows/preview.yml | 4 ++-- .tool-versions | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4e167eb..76b7f16 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -87,8 +87,8 @@ jobs: run: | terramate run \ --changed \ - --cloud-sync-deployment \ - --cloud-sync-terraform-plan-file=out.tfplan \ + --sync-deployment \ + --sync-terraform-plan-file=out.tfplan \ --terragrunt \ -- \ terragrunt apply -input=false -auto-approve -lock-timeout=5m out.tfplan @@ -100,8 +100,8 @@ jobs: run: | terramate run \ --changed \ - --cloud-sync-drift-status \ - --cloud-sync-terraform-plan-file=drift.tfplan \ + --sync-drift-status \ + --sync-terraform-plan-file=drift.tfplan \ --terragrunt \ -- \ terragrunt plan -out drift.tfplan -detailed-exitcode diff --git a/.github/workflows/drift-detection.yml b/.github/workflows/drift-detection.yml index f603115..885ee3b 100644 --- a/.github/workflows/drift-detection.yml +++ b/.github/workflows/drift-detection.yml @@ -69,8 +69,8 @@ jobs: run: | terramate run \ --parallel 5 \ - --cloud-sync-drift-status \ - --cloud-sync-terraform-plan-file=drift.tfplan \ + --sync-drift-status \ + --sync-terraform-plan-file=drift.tfplan \ --terragrunt \ -- terragrunt plan -out drift.tfplan -detailed-exitcode -lock=false env: diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index cdf05fa..18e0aeb 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -94,8 +94,8 @@ jobs: terramate run \ --parallel 5 \ --changed \ - --cloud-sync-preview \ - --cloud-sync-terraform-plan-file=out.tfplan \ + --sync-preview \ + --sync-terraform-plan-file=out.tfplan \ --debug-preview-url preview_url.txt \ --continue-on-error \ --terragrunt \ diff --git a/.tool-versions b/.tool-versions index 4dd105d..63ee42f 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ -terragrunt 0.55.14 -terramate 0.6.1 -terraform 1.7.5 +terragrunt 0.57.13 +terramate 0.8.1 +terraform 1.8.2 From c88af111dd96c08cdda35f24509cf119c928e7e0 Mon Sep 17 00:00:00 2001 From: Soren Martius Date: Tue, 30 Apr 2024 12:01:57 +0200 Subject: [PATCH 11/11] fix: drift reconciliation workflow --- .github/workflows/drift-detection.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/drift-detection.yml b/.github/workflows/drift-detection.yml index 885ee3b..e21d269 100644 --- a/.github/workflows/drift-detection.yml +++ b/.github/workflows/drift-detection.yml @@ -69,8 +69,8 @@ jobs: run: | terramate run \ --parallel 5 \ - --sync-drift-status \ - --sync-terraform-plan-file=drift.tfplan \ + --cloud-sync-drift-status \ + --cloud-sync-terraform-plan-file=drift.tfplan \ --terragrunt \ -- terragrunt plan -out drift.tfplan -detailed-exitcode -lock=false env: @@ -91,7 +91,7 @@ jobs: run: | terramate run \ --cloud-status=drifted \ - --tags reconcile, + --tags reconcile \ --cloud-sync-deployment \ --cloud-sync-terraform-plan-file=drift.tfplan \ --terragrunt \