Skip to content

Commit

Permalink
Add Terraform for production environment [#3641]
Browse files Browse the repository at this point in the history
* Remove old IaC configuration
* Manage Terraform Service Accounts in Terraform scripts
* Add terraform validation workflow using Workload Identity Federation
* Remove jupyterhub disks and instance group resources
* Add makefile to run Terraform locally

---------

Signed-off-by: Doc Ritezel <doc@ministryofvelocity.com>
Co-authored-by: Erika Pacheco <erika@ministryofvelocity.com>
  • Loading branch information
ohrite and erikamov authored Feb 20, 2025
1 parent 09cb4a1 commit f3be70c
Show file tree
Hide file tree
Showing 119 changed files with 16,630 additions and 208 deletions.
106 changes: 106 additions & 0 deletions .github/workflows/terraform-apply.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
name: Terraform Apply

on:
push:
branches:
- 'main'
paths:
- 'iac/*'

permissions:
contents: read
pull-requests: write

jobs:
targets:
name: Find targets

runs-on: ubuntu-latest

outputs:
staging: ${{ steps.staging.outputs.paths }}
production: ${{ steps.production.outputs.paths }}

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Staging Terraform targets
id: staging
run: echo "paths=$(find iac -name 'provider.tf' | grep cal-itp-data-infra-staging/ | xargs dirname | jq --raw-input --slurp --compact-output 'split("\n")[:-1]')" >> ${GITHUB_OUTPUT}

- name: Production Terraform targets
id: production
run: echo "paths=$(find iac -name 'provider.tf' | grep cal-itp-data-infra/ | xargs dirname | jq --raw-input --slurp --compact-output 'split("\n")[:-1]')" >> ${GITHUB_OUTPUT}

staging:
name: Staging

needs: targets

runs-on: ubuntu-latest

permissions:
contents: 'read'
id-token: 'write'

strategy:
fail-fast: false
matrix:
path: ${{ fromJson(needs.targets.outputs.staging) }}

steps:
- name: Checkout
uses: actions/checkout@v4

- uses: 'google-github-actions/auth@v2'
with:
create_credentials_file: 'true'
project_id: cal-itp-data-infra-staging
workload_identity_provider: 'projects/473674835135/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider'
service_account: 'github-actions-terraform@cal-itp-data-infra-staging.iam.gserviceaccount.com'

- uses: google-github-actions/setup-gcloud@v2

- name: Terraform Apply
uses: dflook/terraform-apply@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
path: ${{ matrix.path }}

production:
name: Production

needs: targets

runs-on: ubuntu-latest

permissions:
contents: 'read'
id-token: 'write'

strategy:
fail-fast: false
matrix:
path: ${{ fromJson(needs.targets.outputs.production) }}

steps:
- name: Checkout
uses: actions/checkout@v4

- uses: 'google-github-actions/auth@v2'
with:
create_credentials_file: 'true'
project_id: cal-itp-data-infra
workload_identity_provider: 'projects/1005246706141/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider'
service_account: 'github-actions-terraform@cal-itp-data-infra.iam.gserviceaccount.com'

- uses: google-github-actions/setup-gcloud@v2

- name: Terraform Apply
uses: dflook/terraform-apply@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
path: ${{ matrix.path }}
48 changes: 0 additions & 48 deletions .github/workflows/terraform-deploy.yml

This file was deleted.

122 changes: 122 additions & 0 deletions .github/workflows/terraform-plan.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
name: Terraform Plan

on:
pull_request:
paths:
- 'iac/*'

jobs:
targets:
name: Find targets

runs-on: ubuntu-latest

outputs:
staging: ${{ steps.staging.outputs.paths }}
production: ${{ steps.production.outputs.paths }}

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Staging Terraform targets
id: staging
run: echo "paths=$(find iac -name 'provider.tf' | grep cal-itp-data-infra-staging/ | xargs dirname | jq --raw-input --slurp --compact-output 'split("\n")[:-1]')" >> ${GITHUB_OUTPUT}

- name: Production Terraform targets
id: production
run: echo "paths=$(find iac -name 'provider.tf' | grep cal-itp-data-infra/ | xargs dirname | jq --raw-input --slurp --compact-output 'split("\n")[:-1]')" >> ${GITHUB_OUTPUT}

staging:
name: Staging

needs: targets

runs-on: ubuntu-latest

permissions:
contents: 'read'
id-token: 'write'

strategy:
fail-fast: false
matrix:
path: ${{ fromJson(needs.targets.outputs.staging) }}

steps:
- name: Checkout
uses: actions/checkout@v4

- uses: 'google-github-actions/auth@v2'
with:
create_credentials_file: 'true'
project_id: cal-itp-data-infra-staging
workload_identity_provider: 'projects/473674835135/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider'
service_account: 'github-actions-terraform@cal-itp-data-infra-staging.iam.gserviceaccount.com'

- uses: google-github-actions/setup-gcloud@v2

- name: Terraform Formatting
uses: dflook/terraform-fmt-check@v1
with:
path: ${{ matrix.path }}

- name: Terraform Validation
uses: dflook/terraform-validate@v1
with:
path: ${{ matrix.path }}

- name: Terraform Plan
uses: dflook/terraform-plan@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
add_github_comment: changes-only
path: ${{ matrix.path }}

production:
name: Production

needs: targets

runs-on: ubuntu-latest

permissions:
contents: 'read'
id-token: 'write'

strategy:
fail-fast: false
matrix:
path: ${{ fromJson(needs.targets.outputs.production) }}

steps:
- name: Checkout
uses: actions/checkout@v4

- uses: 'google-github-actions/auth@v2'
with:
create_credentials_file: 'true'
project_id: cal-itp-data-infra
workload_identity_provider: 'projects/1005246706141/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider'
service_account: 'github-actions-terraform@cal-itp-data-infra.iam.gserviceaccount.com'

- uses: google-github-actions/setup-gcloud@v2

- name: Terraform Format Check
uses: dflook/terraform-fmt-check@v1
with:
path: ${{ matrix.path }}

- name: Terraform Validate
uses: dflook/terraform-validate@v1
with:
path: ${{ matrix.path }}

- name: Terraform Plan
uses: dflook/terraform-plan@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
add_github_comment: changes-only
path: ${{ matrix.path }}
70 changes: 0 additions & 70 deletions .github/workflows/terraform-report.yml

This file was deleted.

1 change: 0 additions & 1 deletion iac/.engine
Submodule .engine deleted from 3175c1
6 changes: 6 additions & 0 deletions iac/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
.terraform/
.terraform.tfstate.*.backup
terraform.tfstate.backup
terraform.tfstate
/.terraform.lock.hcl
/provider.tf
9 changes: 9 additions & 0 deletions iac/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
TARGETS := init plan apply fmt migrate-state

PATHS := $(wildcard */.)

$(TARGETS): $(PATHS)
$(PATHS):
$(MAKE) -C $@ $(MAKECMDGOALS)

.PHONY: $(TARGETS) $(PATHS)
Loading

0 comments on commit f3be70c

Please sign in to comment.