release-internal.yml

# What?
# Tag and release an arbitrary ref. Uploads to an internal archive for further processing.
#
# How?
# After checking out and testing the provided ref, the image is built and uploaded.
#
# When?
# Manual trigger.
name: "Release to Cloud"
run-name: "Release to Cloud off of ${{ inputs.ref }}"

on:
  workflow_dispatch:
    inputs:
      ref:
        description: "The ref (sha or branch name) to use"
        type: string
        default: "main"
        required: true
      package_test_command:
        description: "Package test command"
        type: string
        default: "python -c \"import dbt.adapters.redshift\""
        required: true

defaults:
  run:
    shell: "bash"

env:
  PYTHON_TARGET_VERSION: 3.8
  NOTIFICATION_PREFIX: "[Internal Archive Release]"
  TEMP_PROFILE_NAME: "temp_aws_profile"

jobs:
  job-setup:
    name: Job Setup
    runs-on: ubuntu-latest
    steps:
      - name: "[DEBUG] Print Variables"
        run: |
          echo The release ref:                    ${{ inputs.ref }}
          echo Package test command:               ${{ inputs.package_test_command }}

      - name: "Checkout provided ref, default to branch main"
        uses: actions/checkout@v4
        with:
          ref: "${{ inputs.ref }}"

  unit-tests:
    name: 'Unit Tests'
    runs-on: ubuntu-latest
    needs: job-setup
    env:
      TOXENV: unit

    steps:
      - name: "Checkout provided ref, default to branch main"
        uses: actions/checkout@v4
        with:
          ref: "${{ inputs.ref }}"

      - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}"
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_TARGET_VERSION }}

      - name: "Install Python Dependencies"
        run: |
          python -m pip install --user --upgrade pip
          python -m pip install tox
          python -m pip --version
          python -m tox --version

      - name: "Run Tests"
        run: tox

  integration-tests:
    name: 'Integration Tests'
    runs-on: ubuntu-latest
    needs: unit-tests
    env:
      TOXENV: integration-redshift
      PYTEST_ADDOPTS: "-v --color=yes -n4"
      DBT_INVOCATION_ENV: github-actions
    steps:
      - name: "Checkout provided ref, default to branch main"
        uses: actions/checkout@v4
        with:
          ref: "${{ inputs.ref }}"

      - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}"
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_TARGET_VERSION }}

      - name: "Install Python Dependencies"
        run: |
          python -m pip install --user --upgrade pip
          python -m pip --version
          python -m pip install tox
          tox --version

      - name: "Create AWS IAM profiles"
        run: |
          aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID
          aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY
          aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION
          aws configure --profile $AWS_USER_PROFILE set output json

          aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID
          aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY
          aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION
          aws configure --profile $AWS_SOURCE_PROFILE set output json

          aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE
          aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN
          aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION
          aws configure --profile $AWS_ROLE_PROFILE set output json
        env:
          AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }}
          AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }}
          AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }}
          AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user
          AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}
          AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }}
          AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }}
          AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }}
          AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }}

      - name: "Run tox"
        env:
          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
          REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }}
          REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }}
          REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }}
          REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }}
          REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }}
          REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}
          DBT_TEST_USER_1: dbt_test_user_1
          DBT_TEST_USER_2: dbt_test_user_2
          DBT_TEST_USER_3: dbt_test_user_3
        run: tox -- -m "not flaky"

  integration-tests-flaky:
    name: 'Integration Tests (flaky)'
    runs-on: ubuntu-latest
    needs: integration-tests

    env:
      TOXENV: integration-redshift
      PYTEST_ADDOPTS: "-v --color=yes -n1"
      DBT_INVOCATION_ENV: github-actions

    steps:
      - name: "Checkout provided ref, default to branch main"
        uses: actions/checkout@v4
        with:
          ref: "${{ inputs.ref }}"

      - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}"
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_TARGET_VERSION }}

      - name: "Install Python Dependencies"
        run: |
          python -m pip install --user --upgrade pip
          python -m pip --version
          python -m pip install tox
          tox --version

      - name: "Run tox"
        env:
          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
          REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }}
          REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }}
          REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }}
          REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }}
          REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }}
          REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}
          DBT_TEST_USER_1: dbt_test_user_1
          DBT_TEST_USER_2: dbt_test_user_2
          DBT_TEST_USER_3: dbt_test_user_3
        run: tox -- -m flaky

  call-release-workflow:
    needs: [unit-tests, integration-tests, integration-tests-flaky]
    name: "Create cloud release"
    uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main"

    with:
      package_test_command: "${{ inputs.package_test_command }}"
      dbms_name: "redshift"
      ref: "${{ inputs.ref }}"
      # Always skip tests since we run them above
      skip_tests: true

    secrets: "inherit"