chore: Setup linter and formatter

.sqlfluff

@@ -0,0 +1,37 @@
+[sqlfluff]
+dialect = snowflake
+templater = dbt
+runaway_limit = 10
+max_line_length = 80
+indent_unit = space
+
+[sqlfluff:indentation]
+tab_space_size = 4
+
+[sqlfluff:layout:type:comma]
+spacing_before = touch
+line_position = trailing
+
+[sqlfluff:rules:capitalisation.keywords] 
+capitalisation_policy = lower
+
+[sqlfluff:rules:aliasing.table]
+aliasing = explicit
+
+[sqlfluff:rules:aliasing.column]
+aliasing = explicit
+
+[sqlfluff:rules:aliasing.expression]
+allow_scalar = False
+
+[sqlfluff:rules:capitalisation.identifiers]
+extended_capitalisation_policy = lower
+
+[sqlfluff:rules:capitalisation.functions]
+capitalisation_policy = lower
+
+[sqlfluff:rules:capitalisation.literals]
+capitalisation_policy = lower
+
+[sqlfluff:rules:ambiguous.column_references]  # Number in group by
+group_by_and_order_by_style = implicit

.sqlfluffignore

@@ -0,0 +1,5 @@
+reports
+target
+dist
+dbt_packages
+macros

README.md

@@ -7,6 +7,8 @@ pipeline as a SQL workflow using:
 - Github Actions as pipeline runner, scheduler, and orchestrator.
 - Python virtual environment.
 - Snowflake or BigQuery as data warehouse
+- [sqlfluff](https://sqlfluff.com/) for linting SQL files
+- [sqlfmt](https://sqlfmt.com/) for formatting SQL files
 
 ## Getting started
 

dbt_project.yml

@@ -9,7 +9,7 @@ config-version: 2
 # This setting configures which "profile" dbt uses for this project.
 profile: 'dbt_demo'
 
-# These configurations specify where dbt should look for different types of files.
+# These configurations specify WHERE dbt should look for different types of files.
 # The `model-paths` config, for example, states that models in this project can be
 # found in the "models/" directory. You probably won't need to change these!
 model-paths: ["models"]

docs/dbt.md

@@ -27,7 +27,7 @@ dbt allows us to configure:
 
 `tags` to support easy categorization and graph selection
 custom schemas to split your models across multiple schemas
-aliases if your view/table name should differ from the filename
+aliases if your view/table name should differ FROM the filename
 Snippets of SQL to run at the start or end of a model, known as hooks
 Warehouse-specific configurations for performance (e.g. sort and dist keys on Redshift, partitions on BigQuery)
 
@@ -71,13 +71,13 @@ TODO: See https://docs.getdbt.com/docs/deploy/deployments#monitor-jobs-and-alert
 - See dbt docs [Snowflake setup](https://docs.getdbt.com/docs/core/connect-data-platform/snowflake-setup#account)
 - See Snowflake's [BEST PRACTICES FOR OPTIMIZING YOUR DBT AND SNOWFLAKE DEPLOYMENT](https://www.snowflake.com/wp-content/uploads/2021/10/Best-Practices-for-Optimizing-Your-dbt-and-Snowflake-Deployment.pdf)
 - See post [Snowflake Queries: How Much Is Each One Costing You?](https://blog.sundeck.io/snowflake-queries-how-much-is-each-one-costing-you-83f40a61dd90)
-- See Select post [Calculating cost per query in Snowflake](https://select.dev/posts/cost-per-query#complete-sql-query)
+- See Select post [Calculating cost per query in Snowflake](https://SELECT.dev/posts/cost-per-query#complete-sql-query)
 
 ### Cost Monitoring tools
 
 - See Snowflake docs on [Understanding overall cost](https://docs.snowflake.com/en/user-guide/cost-understanding-overall)
-- See [Select](https://select.dev/docs/dbt-snowflake-monitoring)
-    - [dbt-snowflake-monitoring](https://github.com/get-select/dbt-snowflake-monitoring)
+- See [Select](https://SELECT.dev/docs/dbt-snowflake-monitoring)
+    - [dbt-snowflake-monitoring](https://github.com/get-SELECT/dbt-snowflake-monitoring)
 
 ## Notes for BigQuery
 

format.sh

@@ -0,0 +1,9 @@
+# Formats SQL files
+sqlfmt */**/*.sql
+
+# Formats Python files
+# TODO: do not run if there is no python files
+# black */**/*.py
+
+# Format yml files
+yamlfix */**/*.y*ml 

lint.sh

@@ -0,0 +1 @@
+sqlfluff lint */**/*.sql --dialect $DBT_WAREHOUSE_PROVIDER --disregard-sqlfluffignores

models/example/my_first_dbt_model.sql

@@ -1,27 +1,27 @@
-
 /*
  * Welcome to your first dbt model!
  * Did you know that you can also configure models directly within SQL files?
  * This will override configurations stated in dbt_project.yml
  *
  * Try changing "table" to "view" below
 */
+{{ config(materialized="table") }}
 
-{{ config(materialized='table') }}
-
-WITH source_data AS (
+with
+    source_data as (
 
-    SELECT 1 AS id
-    UNION ALL
-    SELECT null AS id
+        select 1 as id
+        union all
+        select null as id
 
-)
+    )
 
-SELECT *
-FROM source_data
+select *
+from
+    source_data
 
-/*
+    /*
  * Uncomment the line below to remove records WITH null `id` values
  */
-
--- WHERE id IS NOT null
+    -- WHERE id IS NOT null
+    

models/example/my_second_dbt_model.sql

@@ -1,6 +1,2 @@
-
--- Use the `ref` function to select from other models
-
-SELECT *
-FROM {{ ref('my_first_dbt_model') }}
-WHERE id = 1
+-- Use the `ref` function to SELECT FROM other models
+select * from {{ ref("my_first_dbt_model") }} where id = 1

models/example/schema.yml

@@ -1,20 +1,15 @@
-
+---
 version: 2
-
 models:
   - name: my_first_dbt_model
-    description: "A starter dbt model"
+    description: A starter dbt model
     columns:
       - name: id
-        description: "The primary key for this table"
-        tests:
-          - unique
-
+        description: The primary key for this table
+        tests: [unique]
   - name: my_second_dbt_model
-    description: "A starter dbt model"
+    description: A starter dbt model
     columns:
       - name: id
-        description: "The primary key for this table"
-        tests:
-          - unique
-          - not_null
+        description: The primary key for this table
+        tests: [unique, not_null]

requirements.txt

@@ -0,0 +1,4 @@
+sqlfluff-templater-dbt==2.3.5
+shandy-sqlfmt[jinjafmt]
+black==24.2.0
+yamlfix==1.16.0

setup.sh

@@ -72,6 +72,8 @@ fi
 # Confirm that the dbt installation was successful.
 dbt --version
 
+python -m pip install -r requirements.txt
+
 # dbt init $DBT_PROJECT_NAME
 
 ##