diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 6e4b4bb..693952e 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -25,8 +25,6 @@ jobs: - name: Checkout repository code uses: actions/checkout@v4 - - run: sh variable_substitution.sh - - run: sh setup.sh - run: dbt compile @@ -40,6 +38,7 @@ jobs: id-token: 'write' env: DBT_PROJECT_NAME: dbt_demo + DBT_WAREHOUSE_PROVIDER: 'snowflake' DBT_PROFILES_DIR: '.' SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} @@ -60,6 +59,7 @@ jobs: id-token: 'write' env: DBT_PROJECT_NAME: dbt_demo + DBT_WAREHOUSE_PROVIDER: 'snowflake' DBT_PROFILES_DIR: '.' SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} @@ -71,8 +71,6 @@ jobs: - name: Checkout repository code uses: actions/checkout@v4 - - run: sh variable_substitution.sh - - run: sh setup.sh - run: dbt test @@ -84,6 +82,7 @@ jobs: id-token: 'write' env: DBT_PROJECT_NAME: dbt_demo + DBT_WAREHOUSE_PROVIDER: 'snowflake' DBT_PROFILES_DIR: '.' SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} @@ -112,17 +111,6 @@ jobs: - name: Checkout repository code uses: actions/checkout@v4 - - run: sh variable_substitution.sh - - run: sh setup.sh - run: dbt run - - - - - - - - - diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..aab52db --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "python.defaultInterpreterPath": "${cwd}/.venv/bin/python", + "python.terminal.activateEnvironment": true +} \ No newline at end of file diff --git a/dbt_profiles.yml b/dbt_profiles.snowflake.yml similarity index 100% rename from dbt_profiles.yml rename to dbt_profiles.snowflake.yml diff --git a/models/example/schema.yml b/models/example/schema.yml index 2a53081..e2aef43 100644 --- a/models/example/schema.yml +++ b/models/example/schema.yml @@ -9,7 +9,6 @@ models: description: "The primary key for this table" tests: - unique - - not_null - name: my_second_dbt_model description: "A starter dbt model" diff --git a/setup.sh b/setup.sh index 3804861..47b3b86 100644 --- a/setup.sh +++ b/setup.sh @@ -1,24 +1,50 @@ ## -## Setup runner configuration +## Validate runner configuration variables ## -if [[ -z "${DBT_PROJECT_NAME}" ]]; then +if [ -z "${DBT_PROJECT_NAME}" ]; then echo "Environment variable DBT_PROJECT_NAME is not set" exit 1 fi -if [[ -z "${DBT_PROFILES_DIR}" ]]; then +if [ -z "${DBT_PROFILES_DIR}" ]; then # Note: the value used by dbt when --profiles-dir is not is "/Users//.dbt" echo "Environment variable DBT_PROFILES_DIR is not set" exit 1 fi -if [[ -z "${DBT_WAREHOUSE_PROVIDER}" ]]; then +if [ -z "${DBT_WAREHOUSE_PROVIDER}" ]; then echo "Environment variable DBT_WAREHOUSE_PROVIDER is not set" exit 1 fi +## +## Perform Environment-Specific variable subtitutions +## + +if [ "$DBT_WAREHOUSE_PROVIDER" = "snowflake" ]; then + cp dbt_profiles.snowflake.yml profiles.yml + + sed -i "s/__SNOWFLAKE_ACCOUNT__/${SNOWFLAKE_ACCOUNT}/g" profiles.yml + sed -i "s/__SNOWFLAKE_DATABASE__/${SNOWFLAKE_DATABASE}/g" profiles.yml + sed -i "s/__SNOWFLAKE_PASSWORD__/${SNOWFLAKE_PASSWORD}/g" profiles.yml + sed -i "s/__SNOWFLAKE_ROLE__/${SNOWFLAKE_ROLE}/g" profiles.yml + sed -i "s/__SNOWFLAKE_USER__/${SNOWFLAKE_USER}/g" profiles.yml + sed -i "s/__SNOWFLAKE_WAREHOUSE__/${SNOWFLAKE_WAREHOUSE}/g" profiles.yml + +elif [ "$DBT_WAREHOUSE_PROVIDER" = "bigquery" ]; then + cp dbt_profiles.bigquery.yml profiles.yml + + # TODO: + # sed -i "s/__SNOWFLAKE_ACCOUNT__/${SNOWFLAKE_ACCOUNT}/g" profiles.yml + # sed -i "s/__SNOWFLAKE_DATABASE__/${SNOWFLAKE_DATABASE}/g" profiles.yml + # sed -i "s/__SNOWFLAKE_PASSWORD__/${SNOWFLAKE_PASSWORD}/g" profiles.yml + # sed -i "s/__SNOWFLAKE_ROLE__/${SNOWFLAKE_ROLE}/g" profiles.yml + # sed -i "s/__SNOWFLAKE_USER__/${SNOWFLAKE_USER}/g" profiles.yml + # sed -i "s/__SNOWFLAKE_WAREHOUSE__/${SNOWFLAKE_WAREHOUSE}/g" profiles.yml +fi + ## ## Install system dependencies ## @@ -49,32 +75,6 @@ elif [ "$DBT_WAREHOUSE_PROVIDER" = "bigquery" ]; then pip install dbt-bigquery fi -## -## Perform Environment-Specific variable subtitutions -## - -if [ "$DBT_WAREHOUSE_PROVIDER" = "snowflake" ]; then - cp dbt_profiles.snowflake.yml profiles.yml - - sed -i "s/__SNOWFLAKE_ACCOUNT__/${SNOWFLAKE_ACCOUNT}/g" profiles.yml - sed -i "s/__SNOWFLAKE_DATABASE__/${SNOWFLAKE_DATABASE}/g" profiles.yml - sed -i "s/__SNOWFLAKE_PASSWORD__/${SNOWFLAKE_PASSWORD}/g" profiles.yml - sed -i "s/__SNOWFLAKE_ROLE__/${SNOWFLAKE_ROLE}/g" profiles.yml - sed -i "s/__SNOWFLAKE_USER__/${SNOWFLAKE_USER}/g" profiles.yml - sed -i "s/__SNOWFLAKE_WAREHOUSE__/${SNOWFLAKE_WAREHOUSE}/g" profiles.yml - -elif [ "$DBT_WAREHOUSE_PROVIDER" = "bigquery" ]; then - cp dbt_profiles.bigquery.yml profiles.yml - - # TODO: - # sed -i "s/__SNOWFLAKE_ACCOUNT__/${SNOWFLAKE_ACCOUNT}/g" profiles.yml - # sed -i "s/__SNOWFLAKE_DATABASE__/${SNOWFLAKE_DATABASE}/g" profiles.yml - # sed -i "s/__SNOWFLAKE_PASSWORD__/${SNOWFLAKE_PASSWORD}/g" profiles.yml - # sed -i "s/__SNOWFLAKE_ROLE__/${SNOWFLAKE_ROLE}/g" profiles.yml - # sed -i "s/__SNOWFLAKE_USER__/${SNOWFLAKE_USER}/g" profiles.yml - # sed -i "s/__SNOWFLAKE_WAREHOUSE__/${SNOWFLAKE_WAREHOUSE}/g" profiles.yml -fi - # Confirm that the dbt installation was successful. dbt --version