diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index c6e836a..9b004b2 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -4,6 +4,8 @@ on: push: branches: - main + schedule: + - cron: '0 0 * * *' jobs: build: @@ -93,6 +95,7 @@ jobs: - run: echo "Deploy" run: + name: Run ELT runs-on: ubuntu-latest permissions: contents: 'read' @@ -113,3 +116,28 @@ jobs: - run: sh setup.sh - run: dbt run + + + scheduled-run: + name: Run ELT (scheduled) + if: ${{ github.ref_name == 'main' }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + env: + DBT_PROJECT_NAME: dbt_demo + DBT_WAREHOUSE_PROVIDER: 'snowflake' + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + steps: + - name: Checkout repository code + uses: actions/checkout@v4 + + - run: sh setup.sh + + - run: dbt run \ No newline at end of file diff --git a/README.md b/README.md index d79c2c8..9c945d2 100644 --- a/README.md +++ b/README.md @@ -3,17 +3,17 @@ A project template for building, testing, scheduling, and running an ELT data pipeline as a SQL workflow using: -- dbt-core -- Github Actions as pipeline runner, scheduler, and orchestrator. -- Python virtual environment. -- Snowflake or BigQuery as data warehouse -- [sqlfluff](https://sqlfluff.com/) for linting SQL files -- [sqlfmt](https://sqlfmt.com/) for formatting SQL files +- [dbt-core](https://github.com/dbt-labs/dbt-core) +- [Github Actions](https://docs.github.com/en/actions) used as runner, scheduler, and orchestrator. +- [Python virtual environment](https://docs.python.org/3/library/venv.html). +- [Snowflake](https://docs.snowflake.com/) as a data warehouse option. +- [BigQuery](https://cloud.google.com/bigquery/docs) as a data warehouse option. +- [sqlfluff](https://sqlfluff.com/) used for linting SQL files +- [sqlfmt](https://sqlfmt.com/) used for formatting SQL files ## Getting started -Run `bash setup.sh` once to install project dependencies and configure the -desired data warehouse and agent connection. +Run `bash setup.sh` once to install project dependencies and configure the desired data warehouse and agent connection. Run the following target commands to execute the desired SQL workflow operation: - `dbt compile` @@ -28,8 +28,11 @@ Run the following target commands to execute the desired SQL workflow operation: The `main` branch is read-only. +`git checkout -b ` -`git checkout -b ` +Raise a PR + +Merge PR to main branch ## Code Style Guide diff --git a/dbt_profiles.bigquery.yml b/dbt_profiles.bigquery.yml index 60150f8..e31b57f 100644 --- a/dbt_profiles.bigquery.yml +++ b/dbt_profiles.bigquery.yml @@ -19,4 +19,4 @@ dbt_demo: OPTIONAL_CONFIG: VALUE - keyfile: __GCP_SVC_ACC_KEY_PATH__ \ No newline at end of file + keyfile: __GCP_SVC_ACC_KEY_PATH__ diff --git a/dbt_profiles.snowflake.yml b/dbt_profiles.snowflake.yml index 0415ea9..806f981 100644 --- a/dbt_profiles.snowflake.yml +++ b/dbt_profiles.snowflake.yml @@ -6,15 +6,15 @@ ## ## See dbt docs on Connection profiles: https://docs.getdbt.com/docs/core/connect-data-platform/connection-profiles dbt_demo: + target: dev outputs: dev: - account: __SNOWFLAKE_ACCOUNT__ - database: __SNOWFLAKE_DATABASE__ - password: __SNOWFLAKE_PASSWORD__ - role: __SNOWFLAKE_ROLE__ + account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') }}" + password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + role: "{{ env_var('SNOWFLAKE_ROLE') }}" schema: PUBLIC threads: 10 type: snowflake - user: __SNOWFLAKE_USER__ - warehouse: __SNOWFLAKE_WAREHOUSE__ - target: dev + user: "{{ env_var('SNOWFLAKE_USER') }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" \ No newline at end of file diff --git a/docs/git-branching-model.drawio.svg b/docs/git-branching-model.drawio.svg index 30754eb..06fd3cf 100644 --- a/docs/git-branching-model.drawio.svg +++ b/docs/git-branching-model.drawio.svg @@ -1,6 +1,4 @@ - + - - - + \ No newline at end of file diff --git a/models/example/schema.yml b/models/example/schema.yml index 1a49561..fddc3f9 100644 --- a/models/example/schema.yml +++ b/models/example/schema.yml @@ -1,5 +1,18 @@ --- version: 2 + +## +## DAG Sources +## See dbt docs: https://docs.getdbt.com/docs/build/sources +## +sources: + - name: jaffle_shop + database: raw + schema: jaffle_shop + tables: + - name: orders + - name: customers + models: - name: my_first_dbt_model description: A starter dbt model @@ -13,3 +26,33 @@ models: - name: id description: The primary key for this table tests: [unique, not_null] + +## +## Exposures define downstream uses of the dbt project, +## See dbt docs: https://docs.getdbt.com/docs/build/exposures +## +exposures: + - name: weekly_jaffle_metrics + label: Weekly Jaffle Metrics + type: dashboard + + # Indicates the level of confidence (stability) in the exposure. + # Options: + # - high (i.e. for a well-established dashboard, widely used and trusted within your organizatio) + # - medium + # - low (i.e. for a new or experimental analysis) + maturity: high + + url: https://bi.tool/dashboards/1 + + description: > + Did someone say "exponential growth"? + + depends_on: + - ref('my_first_dbt_model') + - source('jaffle_shop', 'orders') + # - metric('count_orders') + + owner: + name: Jo Lares + email: jolares@jaffleshop.com \ No newline at end of file diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..8f0e8f7 --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,6 @@ +packages: +- package: dbt-labs/codegen + version: 0.12.1 +- package: dbt-labs/dbt_utils + version: 1.1.1 +sha1_hash: acdde602cd2e228b1dac1cf1a7e5f98f12c16b3b diff --git a/packages.yml b/packages.yml new file mode 100644 index 0000000..c610ca4 --- /dev/null +++ b/packages.yml @@ -0,0 +1,8 @@ +packages: + # See https://hub.getdbt.com/dbt-labs/codegen/latest/ + - package: dbt-labs/codegen + version: 0.12.1 + + # See https://github.com/tnightengale/dbt-meta-testing/tree/0.3.6/ + - package: tnightengale/dbt_meta_testing + version: 0.3.6 \ No newline at end of file diff --git a/setup.sh b/setup.sh index 6af8218..4ad8051 100644 --- a/setup.sh +++ b/setup.sh @@ -4,12 +4,12 @@ ## if [ -z "${DBT_PROJECT_NAME}" ]; then - echo "Environment variable DBT_PROJECT_NAME is not set" + echo "[Error] Environment variable DBT_PROJECT_NAME is not set" exit 1 fi if [ -z "${DBT_WAREHOUSE_PROVIDER}" ]; then - echo "Environment variable DBT_WAREHOUSE_PROVIDER is not set" + echo "[Error] Environment vartarget='dev'iable DBT_WAREHOUSE_PROVIDER is not set" exit 1 fi @@ -19,24 +19,8 @@ fi if [ "$DBT_WAREHOUSE_PROVIDER" = "snowflake" ]; then cp dbt_profiles.snowflake.yml profiles.yml - - sed -i "s/__SNOWFLAKE_ACCOUNT__/${SNOWFLAKE_ACCOUNT}/g" profiles.yml - sed -i "s/__SNOWFLAKE_DATABASE__/${SNOWFLAKE_DATABASE}/g" profiles.yml - sed -i "s/__SNOWFLAKE_PASSWORD__/${SNOWFLAKE_PASSWORD}/g" profiles.yml - sed -i "s/__SNOWFLAKE_ROLE__/${SNOWFLAKE_ROLE}/g" profiles.yml - sed -i "s/__SNOWFLAKE_USER__/${SNOWFLAKE_USER}/g" profiles.yml - sed -i "s/__SNOWFLAKE_WAREHOUSE__/${SNOWFLAKE_WAREHOUSE}/g" profiles.yml - elif [ "$DBT_WAREHOUSE_PROVIDER" = "bigquery" ]; then cp dbt_profiles.bigquery.yml profiles.yml - - # TODO: - # sed -i "s/__SNOWFLAKE_ACCOUNT__/${SNOWFLAKE_ACCOUNT}/g" profiles.yml - # sed -i "s/__SNOWFLAKE_DATABASE__/${SNOWFLAKE_DATABASE}/g" profiles.yml - # sed -i "s/__SNOWFLAKE_PASSWORD__/${SNOWFLAKE_PASSWORD}/g" profiles.yml - # sed -i "s/__SNOWFLAKE_ROLE__/${SNOWFLAKE_ROLE}/g" profiles.yml - # sed -i "s/__SNOWFLAKE_USER__/${SNOWFLAKE_USER}/g" profiles.yml - # sed -i "s/__SNOWFLAKE_WAREHOUSE__/${SNOWFLAKE_WAREHOUSE}/g" profiles.yml fi ## @@ -72,16 +56,11 @@ fi # Confirm that the dbt installation was successful. dbt --version -python -m pip install -r requirements.txt - -# dbt init $DBT_PROJECT_NAME +# Install dbt packages specified in package.yml +dbt deps -## -## Setup dbt connection profile to data warehouse -## - -# TODO -# ... +# Install Python packages +python -m pip install -r requirements.txt # Verify dbt connection profile to data warehouse -dbt debug \ No newline at end of file +dbt debug