diff --git a/.github/labeler.yml b/.github/labeler.yml index 0055ace..27eeef3 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -3,13 +3,12 @@ # this file is for the labeler workflow job # Documentation https://github.com/marketplace/actions/labeler -'type: documentation': - - assets/**/* - - .github/* - - ./*.md - -'type: maintenance': - - .github/**/* - - tests/**/* +"type: docs": + - changed-files: + - any-glob-to-any-file: [assets/**/*, .github/*, ./*.md] + +"type: maintenance": + - changed-files: + - any-glob-to-any-file: .github/**/* ... diff --git a/.github/workflows/deploy-release.yaml b/.github/workflows/deploy-release.yaml index 403b94d..3a6c033 100644 --- a/.github/workflows/deploy-release.yaml +++ b/.github/workflows/deploy-release.yaml @@ -2,8 +2,7 @@ name: 📌 Deploy release on: release: - types: - - released + types: [published] workflow_dispatch: inputs: tag_name: @@ -20,6 +19,7 @@ jobs: image: runs-on: ubuntu-latest name: Release Actions + steps: - name: 📦 Checkout uses: actions/checkout@v3 diff --git a/gh-action-terragrunt-apply/README.md b/gh-action-terragrunt-apply/README.md index 782d465..6d8bc83 100644 --- a/gh-action-terragrunt-apply/README.md +++ b/gh-action-terragrunt-apply/README.md @@ -8,12 +8,14 @@ If the plan is not found or has changed, then the apply action will fail. This i You can instead set `auto_approve: true` which will generate a plan and apply it immediately, without looking for a plan attached to a PR. -**NOTE:** This github action uses default terragrunt cache folder `.terragrunt-cache` to create plan and then to read it. -Don't use terragrunt_download setting in your terragrunt code and also don't clear cache. Otherwise the action won't work. +>**NOTE:** +>There are two apply strategies. Read about them bellow in Inputs section. + +This github action uses --terragrunt-download-dir option to redirect cache in `/tmp/tg_cache_dir`. ## Inputs -These input values must be the same as any wayofdev/gh-action-terragrunt-plan for the same configuration. (unless auto_approve: true) +These input values must be the same as any wayofdev/gh-action-terragrunt-plan for the same configuration, except strategy because it is actual only for apply command. (unless auto_approve: true) * `path` @@ -73,10 +75,20 @@ These input values must be the same as any wayofdev/gh-action-terragrunt-plan fo The default is false, which requires plans to have been approved through a pull request. - - Type: bool + - Type: boolean - Optional - Default: false +* `strategy` + + When set to **parallel**(default) `terragrunt run-all apply` will be executed in provided `path`. And terragrunt will execute multiple plans in parallel accordint to parallelism settings. Plan will be applied even if there is no changes for particular module. + + When set to **sequential** terragrunt will change into each module directory individually and execute `terragrunt run-all apply` and only when there are changes in the plan. So it will skip modules without changes and apply changes one by one. + + - Type: string + - Optional + - Default: parallel + ## Environment Variables * `GITHUB_TOKEN` diff --git a/gh-action-terragrunt-apply/action.yaml b/gh-action-terragrunt-apply/action.yaml index 5aa94a0..e708046 100644 --- a/gh-action-terragrunt-apply/action.yaml +++ b/gh-action-terragrunt-apply/action.yaml @@ -31,6 +31,10 @@ inputs: description: Automatically approve and apply plans required: false default: "false" + strategy: + description: "What strategy to use when applying: parallel or sequential" + required: false + default: "parallel" runs: using: docker diff --git a/image/actions.sh b/image/actions.sh index 624723f..a665976 100755 --- a/image/actions.sh +++ b/image/actions.sh @@ -80,24 +80,35 @@ function set_common_plan_args() { function plan() { # shellcheck disable=SC2086 - debug_log terragrunt run-all plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PARALLEL_ARG -out=plan.out '$PLAN_ARGS' # don't expand PLAN_ARGS + debug_log terragrunt run-all plan --terragrunt-download-dir $TG_CACHE_DIR -input=false -no-color -detailed-exitcode -lock-timeout=300s $PARALLEL_ARG -out=plan.out '$PLAN_ARGS' # don't expand PLAN_ARGS - MODULE_PATHS=$(find $INPUT_PATH -mindepth 2 -name terragrunt.hcl -exec dirname {} \;) + # Get a list of all modules in the provided path + MODULE_PATHS=$(terragrunt output-module-groups --terragrunt-working-dir $INPUT_PATH|jq -r 'to_entries | .[].value[]') + export MODULE_PATHS + + start_group "List of modules found in the provided input path" + for p in $MODULE_PATHS; do + echo "- ${INPUT_PATH}${p#*${INPUT_PATH#./}}" + done + end_group set +e # shellcheck disable=SC2086 start_group "Generating plan" - (cd "$INPUT_PATH" && terragrunt run-all plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PARALLEL_ARG -out=plan.out $PLAN_ARGS) \ - 2>"$STEP_TMP_DIR/terraform_plan.stderr" \ - | $TFMASK + ( + (cd "$INPUT_PATH" && terragrunt run-all plan --terragrunt-download-dir $TG_CACHE_DIR -input=false -no-color -detailed-exitcode -lock-timeout=300s $PARALLEL_ARG -out=plan.out $PLAN_ARGS) \ + 2>"$STEP_TMP_DIR/terraform_plan.stderr" \ + | $TFMASK + wait + ) end_group + # Generate text file for each plan start_group "Generating plan it text format" # shellcheck disable=SC2034 - for i in $MODULE_PATHS; do - plan_name=${i//.\//} - plan_name=${plan_name//\//___} - terragrunt show plan.out --terragrunt-working-dir $i -no-color 2>"$STEP_TMP_DIR/terraform_show_plan.stderr" \ + for i in $MODULE_PATHS; do + plan_name=${i//\//___} + terragrunt show plan.out --terragrunt-working-dir $i -no-color --terragrunt-download-dir $TG_CACHE_DIR 2>"$STEP_TMP_DIR/terraform_show_plan.stderr" \ |tee $PLAN_OUT_DIR/$plan_name done end_group @@ -105,17 +116,46 @@ function plan() { } function apply() { - + + # shellcheck disable=SC2086 + debug_log terragrunt run-all apply --terragrunt-download-dir $TG_CACHE_DIR -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG '$PLAN_ARGS' plan.out + + set +e + start_group "Applying plan sequentially" + ( + for i in $MODULE_PATHS; do + plan_name=${i//\//___} + if grep -q "No changes." $PLAN_OUT_DIR/$plan_name; then + echo "There is no changes in the module ${INPUT_PATH}${i#*${INPUT_PATH#./}}, skiping plan apply for it" + continue + else + (cd $i && terragrunt run-all apply --terragrunt-download-dir $TG_CACHE_DIR -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS plan.out) \ + 2>"$STEP_TMP_DIR/terraform_apply_error/${plan_name}.stderr" \ + | $TFMASK \ + | tee /dev/fd/3 "$STEP_TMP_DIR/terraform_apply_stdout/${plan_name}.stdout" + fi + done + wait + ) + end_group + set -e +} + +function apply_all() { + # shellcheck disable=SC2086 - debug_log terragrunt run-all apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG '$PLAN_ARGS' + debug_log terragrunt run-all apply --terragrunt-download-dir $TG_CACHE_DIR -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG '$PLAN_ARGS' plan.out set +e - start_group "Applying plan" + start_group "Applying plan parallel" # shellcheck disable=SC2086 - (cd "$INPUT_PATH" && terragrunt run-all apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) \ - 2>"$STEP_TMP_DIR/terraform_apply.stderr" \ - | $TFMASK \ - | tee /dev/fd/3 "$STEP_TMP_DIR/terraform_apply.stdout" + ( + (cd "$INPUT_PATH" && terragrunt run-all apply --terragrunt-download-dir $TG_CACHE_DIR -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS plan.out) \ + 2>"$STEP_TMP_DIR/terraform_apply.stderr" \ + | $TFMASK \ + | tee /dev/fd/3 "$STEP_TMP_DIR/terraform_apply.stdout" + wait + ) end_group set -e } @@ -177,10 +217,13 @@ function fix_owners() { # Every file written to disk should use one of these directories STEP_TMP_DIR="/tmp" PLAN_OUT_DIR="/tmp/plan" +TG_CACHE_DIR="/tmp/tg_cache_dir" JOB_TMP_DIR="$HOME/.gh-actions-terragrunt" WORKSPACE_TMP_DIR=".gh-actions-terragrunt/$(random_string)" -mkdir -p $PLAN_OUT_DIR -readonly STEP_TMP_DIR JOB_TMP_DIR WORKSPACE_TMP_DIR PLAN_OUT_DIR -export STEP_TMP_DIR JOB_TMP_DIR WORKSPACE_TMP_DIR PLAN_OUT_DIR +mkdir -p $PLAN_OUT_DIR $TG_CACHE_DIR +mkdir -p $STEP_TMP_DIR/terraform_apply_stdout +mkdir -p $STEP_TMP_DIR/terraform_apply_error +readonly STEP_TMP_DIR JOB_TMP_DIR WORKSPACE_TMP_DIR PLAN_OUT_DIR TG_CACHE_DIR +export STEP_TMP_DIR JOB_TMP_DIR WORKSPACE_TMP_DIR PLAN_OUT_DIR TG_CACHE_DIR trap fix_owners EXIT diff --git a/image/entrypoints/tg_apply_all.sh b/image/entrypoints/tg_apply_all.sh index c88a9b9..65a1ff8 100755 --- a/image/entrypoints/tg_apply_all.sh +++ b/image/entrypoints/tg_apply_all.sh @@ -26,15 +26,19 @@ end_group # Check if state is locked if lock-info "$STEP_TMP_DIR/terraform_plan.stderr"; then - update_status ":x: Error applying plan in $(job_markdown_ref)(State is locked)" + update_status ":x: Error applying plan in $(job_markdown_ref) (State is locked)" exit 1 fi -### Apply the plan +# Apply the plan if [[ "$INPUT_AUTO_APPROVE" == "true" ]]; then echo "Automatically approving plan" - apply + if [[ "$INPUT_STRATEGY" == "parallel" ]]; then + apply_all + else + apply + fi else if [[ "$GITHUB_EVENT_NAME" != "push" && "$GITHUB_EVENT_NAME" != "pull_request" && "$GITHUB_EVENT_NAME" != "issue_comment" && "$GITHUB_EVENT_NAME" != "pull_request_review_comment" && "$GITHUB_EVENT_NAME" != "pull_request_target" && "$GITHUB_EVENT_NAME" != "pull_request_review" && "$GITHUB_EVENT_NAME" != "repository_dispatch" ]]; then @@ -50,29 +54,81 @@ else fi if github_pr_comment approved; then - apply + if [[ "$INPUT_STRATEGY" == "parallel" ]]; then + apply_all + else + apply + fi else exit 1 fi fi -start_group "Content of terraform_apply.stderr" -cat "$STEP_TMP_DIR/terraform_apply.stderr" -end_group -start_group "Content of terraform_apply.stdout" -cat "$STEP_TMP_DIR/terraform_apply.stdout" -end_group +if [[ "$INPUT_STRATEGY" == "parallel" ]]; then + start_group "Content of terraform_apply.stderr" + cat >&2 "$STEP_TMP_DIR/terraform_apply.stderr" + end_group + + start_group "Content of terraform_apply.stdout" + cat >&2 "$STEP_TMP_DIR/terraform_apply.stdout" + end_group + + # check if there are errors in terraform_apply.stderr + if lock-info "$STEP_TMP_DIR/terraform_apply.stderr"; then + update_status ":x: Error applying plan in $(job_markdown_ref) (State is locked)" + exit 1 + else + for code in $(tac $STEP_TMP_DIR/terraform_apply.stderr | awk '/^[[:space:]]*\*/{flag=1; print} flag && /^[[:space:]]*time=/{exit}' | awk '{print $5}'); do + if [[ $code -eq 1 ]]; then + update_status ":x: Error applying plan in $(job_markdown_ref)" + exit 1 + fi + done + fi -# check if there are errors in terraform_apply.stderr -if lock-info "$STEP_TMP_DIR/terraform_apply.stderr"; then - update_status ":x: Error applying plan in $(job_markdown_ref)(State is locked)" - exit 1 else - for code in $(tac $STEP_TMP_DIR/terraform_apply.stderr | awk '/^[[:space:]]*\*/{flag=1; print} flag && /^[[:space:]]*time=/{exit}' | awk '{print $5}'); do - if [[ $code -eq 1 ]]; then - update_status ":x: Error applying plan in $(job_markdown_ref)" + # If there is no files in terraform_apply_error and in terraform_apply_stdout, then there is no changes in the plan + if [[ ! "$(ls $STEP_TMP_DIR/terraform_apply_error/*.stderr 2>/dev/null)" ]] && [[ ! "$(ls $STEP_TMP_DIR/terraform_apply_error/*.stdout 2>/dev/null)" ]]; then + echo "No changes in the plan, skipping apply" + update_status ":white_check_mark: No changes to apply in $(job_markdown_ref)" + exit 0 + fi + + echo "Apply errors by module:" + for file in $STEP_TMP_DIR/terraform_apply_error/*; do + if [[ -s $file ]]; then + filename=$(basename "$file") + filename=${filename//___/\/} + start_group "${INPUT_PATH}${filename#*${INPUT_PATH#./}}" + cat $file + end_group + fi + done + + echo "Apply output by module:" + for file in $STEP_TMP_DIR/terraform_apply_stdout/*; do + if [[ -s $file ]]; then + filename=$(basename "$file") + filename=${filename//___/\/} + start_group "${INPUT_PATH}${filename#*${INPUT_PATH#./}}" + cat $file + end_group + fi + done + + # check if there are errors in terraform_apply.stderr + for file in $STEP_TMP_DIR/terraform_apply_error/*; do + if lock-info "$file"; then + update_status ":x: Error applying plan in $(job_markdown_ref) (State is locked)" exit 1 + else + for code in $(tac $file | awk '/^[[:space:]]*\*/{flag=1; print} flag && /^[[:space:]]*time=/{exit}' | awk '{print $5}'); do + if [[ $code -eq 1 ]]; then + update_status ":x: Error applying plan in $(job_markdown_ref)" + exit 1 + fi + done fi done fi diff --git a/image/entrypoints/tg_plan_all.sh b/image/entrypoints/tg_plan_all.sh index a6ce06c..867b4b6 100755 --- a/image/entrypoints/tg_plan_all.sh +++ b/image/entrypoints/tg_plan_all.sh @@ -20,12 +20,6 @@ start_group "Content of terraform_show_plan.stderr" cat >&2 "$STEP_TMP_DIR/terraform_show_plan.stderr" end_group -# Check if state is locked -if lock-info "$STEP_TMP_DIR/terraform_plan.stderr"; then - update_status ":x: Failed to generate plan in $(job_markdown_ref)(State is locked)" - exit 1 -fi - if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_comment" || "$GITHUB_EVENT_NAME" == "pull_request_review_comment" || "$GITHUB_EVENT_NAME" == "pull_request_target" || "$GITHUB_EVENT_NAME" == "pull_request_review" || "$GITHUB_EVENT_NAME" == "repository_dispatch" ]]; then if [[ "$INPUT_ADD_GITHUB_COMMENT" == "true" || "$INPUT_ADD_GITHUB_COMMENT" == "changes-only" ]]; then @@ -36,17 +30,21 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c exit 1 fi - STATUS=":memo: Plan generated in $(job_markdown_ref)" + # Check if state is locked + if lock-info "$STEP_TMP_DIR/terraform_plan.stderr"; then + STATUS=":x: Failed to generate plan in $(job_markdown_ref) (State is locked)" github_pr_comment plan + exit 1 + fi # Checking plan exit codes for code in $(tac $STEP_TMP_DIR/terraform_plan.stderr | awk '/^[[:space:]]*\*/{flag=1; print} flag && /^[[:space:]]*time=/{exit}' | awk '{print $5}'); do if [[ $code -eq 1 ]]; then - STATUS=":x: Failed to generate plan in $(job_markdown_ref)" + STATUS=":x: Failed to generate plan in $(job_markdown_ref)" github_pr_comment plan + exit 1 fi done - export STATUS - if ! github_pr_comment plan ; then + if ! STATUS=":memo: Plan generated in $(job_markdown_ref)" github_pr_comment plan ; then exit 1 fi fi diff --git a/image/src/github_actions/inputs.py b/image/src/github_actions/inputs.py index 1ebaee1..d3ca867 100644 --- a/image/src/github_actions/inputs.py +++ b/image/src/github_actions/inputs.py @@ -39,6 +39,7 @@ class Plan(PlanPrInputs): class Apply(PlanPrInputs): """Input variables for the terraform-apply action""" INPUT_AUTO_APPROVE: str + INPUT_STRATEGY: str class Check(PlanInputs): diff --git a/image/src/github_pr_comment/__main__.py b/image/src/github_pr_comment/__main__.py index a8564c7..5693c17 100644 --- a/image/src/github_pr_comment/__main__.py +++ b/image/src/github_pr_comment/__main__.py @@ -113,8 +113,6 @@ def create_plan_hashes(folder_path: str, salt: str) -> Optional[List[dict]]: hash_section['plan_hash'] = plan_hash(file_path.read_text().strip(), salt) plan_hashes.append(hash_section) - print("PRINTING HASHES") - print(plan_hashes) return plan_hashes