chore: Add example dbt exposures

.github/workflows/cicd.yml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - main
+  schedule:
+    - cron: '0 0 * * *'
 
 jobs:
   build:
@@ -93,6 +95,7 @@ jobs:
       - run: echo "Deploy"
 
   run:
+    name: Run ELT
     runs-on: ubuntu-latest
     permissions:
       contents: 'read'
@@ -113,3 +116,28 @@ jobs:
       - run: sh setup.sh
 
       - run: dbt run
+
+
+  scheduled-run:
+    name: Run ELT (scheduled)
+    if: ${{ github.ref_name == 'main' }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    env:
+      DBT_PROJECT_NAME: dbt_demo
+      DBT_WAREHOUSE_PROVIDER: 'snowflake'
+      SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+      SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
+      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+      SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }}
+      SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+      SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v4
+
+      - run: sh setup.sh
+
+      - run: dbt run

README.md

@@ -3,17 +3,17 @@
 A project template for building, testing, scheduling, and running an ELT data
 pipeline as a SQL workflow using:
 
-- dbt-core
-- Github Actions as pipeline runner, scheduler, and orchestrator.
-- Python virtual environment.
-- Snowflake or BigQuery as data warehouse
-- [sqlfluff](https://sqlfluff.com/) for linting SQL files
-- [sqlfmt](https://sqlfmt.com/) for formatting SQL files
+- [dbt-core](https://github.com/dbt-labs/dbt-core)
+- [Github Actions](https://docs.github.com/en/actions) used as runner, scheduler, and orchestrator.
+- [Python virtual environment](https://docs.python.org/3/library/venv.html).
+- [Snowflake](https://docs.snowflake.com/) as a data warehouse option.
+- [BigQuery](https://cloud.google.com/bigquery/docs) as a data warehouse option.
+- [sqlfluff](https://sqlfluff.com/) used for linting SQL files
+- [sqlfmt](https://sqlfmt.com/) used for formatting SQL files
 
 ## Getting started
 
-Run `bash setup.sh` once to install project dependencies and configure the
-desired data warehouse and agent connection.
+Run `bash setup.sh` once to install project dependencies and configure the desired data warehouse and agent connection.
 
 Run the following target commands to execute the desired SQL workflow operation:
 - `dbt compile`
@@ -28,8 +28,11 @@ Run the following target commands to execute the desired SQL workflow operation:
 
 The `main` branch is read-only.
 
+`git checkout -b <feature-name>`
 
-`git checkout -b <branch-name>`
+Raise a PR
+
+Merge PR to main branch
 
 
 ## Code Style Guide

dbt_profiles.snowflake.yml

@@ -6,15 +6,15 @@
 ##
 ## See dbt docs on Connection profiles: https://docs.getdbt.com/docs/core/connect-data-platform/connection-profiles
 dbt_demo:
+  target: dev
   outputs:
     dev:
-      account: __SNOWFLAKE_ACCOUNT__
-      database: __SNOWFLAKE_DATABASE__
-      password: __SNOWFLAKE_PASSWORD__
-      role: __SNOWFLAKE_ROLE__
+      account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}"
+      database: "{{ env_var('SNOWFLAKE_DATABASE') }}"
+      password: "{{ env_var('SNOWFLAKE_PASSWORD') }}"
+      role: "{{ env_var('SNOWFLAKE_ROLE') }}"
       schema: PUBLIC
       threads: 10
       type: snowflake
-      user: __SNOWFLAKE_USER__
-      warehouse: __SNOWFLAKE_WAREHOUSE__
-  target: dev
+      user: "{{ env_var('SNOWFLAKE_USER') }}"
+      warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}"

models/example/schema.yml

@@ -1,5 +1,18 @@
 ---
 version: 2
+
+##
+## DAG Sources
+## See dbt docs: https://docs.getdbt.com/docs/build/sources
+##
+sources:
+  - name: jaffle_shop
+    database: raw  
+    schema: jaffle_shop  
+    tables:
+      - name: orders
+      - name: customers
+
 models:
   - name: my_first_dbt_model
     description: A starter dbt model
@@ -13,3 +26,33 @@ models:
       - name: id
         description: The primary key for this table
         tests: [unique, not_null]
+
+##
+## Exposures define downstream uses of the dbt project,
+## See dbt docs: https://docs.getdbt.com/docs/build/exposures
+##
+exposures:
+  - name: weekly_jaffle_metrics
+    label: Weekly Jaffle Metrics
+    type: dashboard
+
+    # Indicates the level of confidence (stability) in the exposure.
+    # Options:
+    # - high (i.e. for a well-established dashboard, widely used and trusted within your organizatio)
+    # - medium
+    # - low (i.e. for a new or experimental analysis)
+    maturity: high
+
+    url: https://bi.tool/dashboards/1
+
+    description: >
+      Did someone say "exponential growth"?
+  
+    depends_on:
+      - ref('my_first_dbt_model')
+      - source('jaffle_shop', 'orders')
+      # - metric('count_orders')
+
+    owner:
+      name: Jo Lares
+      email: jolares@jaffleshop.com

package-lock.yml

@@ -0,0 +1,6 @@
+packages:
+- package: dbt-labs/codegen
+  version: 0.12.1
+- package: dbt-labs/dbt_utils
+  version: 1.1.1
+sha1_hash: acdde602cd2e228b1dac1cf1a7e5f98f12c16b3b

packages.yml

@@ -0,0 +1,8 @@
+packages:
+  # See https://hub.getdbt.com/dbt-labs/codegen/latest/
+  - package: dbt-labs/codegen
+    version: 0.12.1
+
+  # See https://github.com/tnightengale/dbt-meta-testing/tree/0.3.6/
+  - package: tnightengale/dbt_meta_testing
+    version: 0.3.6

setup.sh

@@ -4,12 +4,12 @@
 ##
 
 if [ -z "${DBT_PROJECT_NAME}" ]; then
-  echo "Environment variable DBT_PROJECT_NAME is not set"
+  echo "[Error] Environment variable DBT_PROJECT_NAME is not set"
   exit 1
 fi
 
 if [ -z "${DBT_WAREHOUSE_PROVIDER}" ]; then
-  echo "Environment variable DBT_WAREHOUSE_PROVIDER is not set"
+  echo "[Error] Environment vartarget='dev'iable DBT_WAREHOUSE_PROVIDER is not set"
   exit 1
 fi
 
@@ -19,24 +19,8 @@ fi
 
 if [ "$DBT_WAREHOUSE_PROVIDER" = "snowflake" ]; then
   cp dbt_profiles.snowflake.yml profiles.yml
-
-  sed -i "s/__SNOWFLAKE_ACCOUNT__/${SNOWFLAKE_ACCOUNT}/g" profiles.yml
-  sed -i "s/__SNOWFLAKE_DATABASE__/${SNOWFLAKE_DATABASE}/g" profiles.yml
-  sed -i "s/__SNOWFLAKE_PASSWORD__/${SNOWFLAKE_PASSWORD}/g" profiles.yml
-  sed -i "s/__SNOWFLAKE_ROLE__/${SNOWFLAKE_ROLE}/g" profiles.yml
-  sed -i "s/__SNOWFLAKE_USER__/${SNOWFLAKE_USER}/g" profiles.yml
-  sed -i "s/__SNOWFLAKE_WAREHOUSE__/${SNOWFLAKE_WAREHOUSE}/g" profiles.yml
-
 elif [ "$DBT_WAREHOUSE_PROVIDER" = "bigquery" ]; then
   cp dbt_profiles.bigquery.yml profiles.yml
-
-  # TODO:
-  # sed -i "s/__SNOWFLAKE_ACCOUNT__/${SNOWFLAKE_ACCOUNT}/g" profiles.yml
-  # sed -i "s/__SNOWFLAKE_DATABASE__/${SNOWFLAKE_DATABASE}/g" profiles.yml
-  # sed -i "s/__SNOWFLAKE_PASSWORD__/${SNOWFLAKE_PASSWORD}/g" profiles.yml
-  # sed -i "s/__SNOWFLAKE_ROLE__/${SNOWFLAKE_ROLE}/g" profiles.yml
-  # sed -i "s/__SNOWFLAKE_USER__/${SNOWFLAKE_USER}/g" profiles.yml
-  # sed -i "s/__SNOWFLAKE_WAREHOUSE__/${SNOWFLAKE_WAREHOUSE}/g" profiles.yml
 fi
 
 ##
@@ -72,16 +56,11 @@ fi
 # Confirm that the dbt installation was successful.
 dbt --version
 
-python -m pip install -r requirements.txt
-
-# dbt init $DBT_PROJECT_NAME
+# Install dbt packages specified in package.yml
+dbt deps
 
-##
-## Setup dbt connection profile to data warehouse
-##
-
-# TODO
-# ...
+# Install Python packages
+python -m pip install -r requirements.txt
 
 # Verify dbt connection profile to data warehouse
-dbt debug
+dbt debug