Feature/#6 database config (#7)

* Add dbt_dq_tool_database variable * #6 Move tests of log table to integration tests * #6 fix dbt version in pr pipeline * #6 fix hook error * #6 update test script and add security * #6 update config for test * #6 comment in dbt_dq_tool_database for test * #6 fix dbt_dq_tool_database missing deafult * #6 extend test to other dbt versions * # isolate schema for PR with dbt version * #6 fix DBT_SCHEMA in pr ci pipeline --------- Co-authored-by: Dat Nguyen <dat@infinitelambda>
infinitelambda · Aug 1, 2023 · 82ed511 · 82ed511
1 parent 9ee745c
commit 82ed511
Show file tree

Hide file tree

Showing 19 changed files with 131 additions and 77 deletions.
diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml
@@ -11,7 +11,7 @@ jobs:
       max-parallel: 1
       matrix:
         adapter: ["snowflake"]
-        version: ["1.3.0"] #"1.4.1", "1.5.2"
+        version: ["1.3.0", "1.4.1", "1.5.2"]
     container:
       image: "ghcr.io/dbt-labs/dbt-${{ matrix.adapter }}:${{ matrix.version }}"
     steps:
@@ -20,7 +20,8 @@ jobs:
       - name: Run integration test
         run: |
           chmod +x run_test.sh
-          cp -fr ./integration_tests/packages_${{ matrix.version }}.yml ./integration_tests/packages.yml
+          dbtversion=${{ matrix.version }}
+          cp -fr ./integration_tests/packages_${dbtversion%.*}.yml ./integration_tests/packages.yml
           ./run_test.sh ${{ matrix.adapter }}
         env:
           DBT_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_TEST_ACCOUNT }}
@@ -29,15 +30,15 @@ jobs:
           DBT_SNOWFLAKE_TEST_ROLE: ${{ secrets.DBT_SNOWFLAKE_TEST_ROLE }}
           DBT_SNOWFLAKE_TEST_DATABASE: ${{ secrets.DBT_SNOWFLAKE_TEST_DATABASE }}
           DBT_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.DBT_SNOWFLAKE_TEST_WAREHOUSE }}
-          DBT_SCHEMA: PRCI_${{ github.sha }}
+          DBT_SCHEMA: PRCI_${{ matrix.version }}_${{ github.sha }}
 
   poseidon:
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 1
       matrix:
         adapter: ["bigquery"]
-        version: ["1.3.0"] #"1.4.1", "1.5.3"
+        version: ["1.3.0", "1.4.1", "1.5.3"]
     container:
       image: "ghcr.io/dbt-labs/dbt-${{ matrix.adapter }}:${{ matrix.version }}"
     steps:
@@ -52,9 +53,10 @@ jobs:
       - name: Run integration test
         run: |
           chmod +x run_test.sh
-          cp -fr ./integration_tests/packages_${{ matrix.version }}.yml ./integration_tests/packages.yml
+          dbtversion=${{ matrix.version }}
+          cp -fr ./integration_tests/packages_${dbtversion%.*}.yml ./integration_tests/packages.yml
           ./run_test.sh ${{ matrix.adapter }}
         env:
           DBT_ENV_SECRET_BIGQUERY_SERVICE_KEY_PATH: /opt/bigquery_service_key.json
           DBT_BIGQUERY_TEST_DATABASE: ${{ secrets.DBT_BIGQUERY_TEST_DATABASE }}
-          DBT_SCHEMA: PRCI_${{ github.sha }}
+          DBT_SCHEMA: PRCI_DQTOOLS_${{ matrix.version }}_${{ github.sha }}
diff --git a/SECURITY.md b/SECURITY.md
@@ -0,0 +1,24 @@
+# Security Policy
+
+## Supported Versions
+
+Use this section to tell people about which versions of your project are
+currently being supported with security updates.
+
+| dbt Core    | dq_tools 1.1       | dq_tools 1.2       |
+| ----------- | ------------------ | ------------------ |
+| 1.5.x       | :warning:          | :white_check_mark: |
+| 1.4.x       | :warning:          | :white_check_mark: |
+| 1.3.x       | :white_check_mark: | :white_check_mark: |
+| 1.2.x       | :white_check_mark: | :x:                |
+| 1.1.x       | :white_check_mark: | :x:                |
+| 1.0.x       | :white_check_mark: | :x:                |
+| < 1.0       | :x:                | :x:                |
+
+## Reporting a Vulnerability
+
+Use this section to tell people how to report a vulnerability.
+
+Tell them where to go, how often they can expect to get an update on a
+reported vulnerability, what to expect if the vulnerability is accepted or
+declined, etc.
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -16,4 +16,5 @@ models:
 
 vars:
   # dbt_dq_tool_schema: dq_raw
+  # dbt_dq_tool_database: dq_tools
   # dq_tools_enable_store_test_results: true
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -31,15 +31,16 @@ on-run-start:
     {% if var('fresh', false) %}
       {{ "drop schema if exists " ~ target.schema }};
       {{ "create schema " ~ target.schema }};
-      {{ "drop schema if exists " ~ var("dbt_dq_tool_schema", target.schema) }};
-      {{ "create schema " ~ var("dbt_dq_tool_schema", target.schema) }};
+      {{ "drop schema if exists " ~ var("dbt_dq_tool_database", target.database) ~ "." ~ var("dbt_dq_tool_schema", target.schema) }};
+      {{ "create schema "  ~ var("dbt_dq_tool_database", target.database) ~ "."  ~ var("dbt_dq_tool_schema", target.schema) }};
     {% endif %}
 
 on-run-end:
   - '{{ dq_tools.store_test_results(results) }}'
 
 vars:
   dbt_dq_tool_schema: dq_raw
+  # dbt_dq_tool_database: dq_tools
 
 seeds:
   dq_tools_integration_tests:

diff --git a/integration_tests/models/source_dq_tools_test.yml b/integration_tests/models/source_dq_tools_test.yml
@@ -0,0 +1,69 @@
+version: 2
+
+sources:
+  - name: dq_tools_test
+    database: "{{ var('dbt_dq_tool_database', target.database) }}"
+    schema: "{{ var('dbt_dq_tool_schema') }}"
+    tables:
+      - name: dq_issue_log
+        tests:
+          - dq_tools.expression_is_true_db:
+              where: coalesce(ref_column,'') != ''
+              expression: coalesce(ref_table,'') != ''
+        columns:
+          - name: check_timestamp
+            tests:
+              - dq_tools.not_null_where_db
+          - name: table_name
+            tests:
+              - dq_tools.not_null_where_db
+          - name: dq_issue_type
+            tests:
+              - dq_tools.not_null_where_db
+          - name: invocation_id
+            tests:
+              - dq_tools.not_null_where_db
+          - name: dq_model
+            tests:
+              - dq_tools.not_null_where_db
+              - dq_tools.expression_is_true_db:  # singular test error should not exists
+                  expression: dq_model != 'assert_intentional_error'
+              - dq_tools.expression_is_true_db: # generic test error should not exists
+                  where: table_name like '%.test_failures' and coalesce(column_name,'') = ''
+                  expression: dq_model not like 'dq_tools_expression_is_true_db_test_failures_1_0%'
+          - name: severity
+            tests:
+              - dq_tools.not_null_where_db
+              - dq_tools.accepted_values_where_db:
+                  values: [
+                    'warn',
+                    'error'
+                  ]
+          - name: kpi_category
+            tests:
+              - dq_tools.not_null_where_db
+              - dq_tools.accepted_values_where_db:
+                  values: [
+                    'Other',
+                    'Uniqueness',
+                    'Consistency',
+                    'Validity',
+                    'Timeliness',
+                    'Completeness',
+                    'Accuracy',
+                  ]
+          - name: no_of_records
+            tests:
+              - dq_tools.not_null_where_db:
+                  where: coalesce(table_name,'') != ''
+              - dq_tools.expression_is_true_db:
+                  where: coalesce(ref_table,'') != ''
+                  expression: no_of_records >= 0
+          - name: no_of_records_failed
+            tests:
+              - dq_tools.not_null_where_db
+              - dq_tools.expression_is_true_db:
+                  expression: no_of_records_failed >= 0
+              - dq_tools.expression_is_true_db:
+                  where: lower(dq_model) like '%failure%'
+                  expression: no_of_records_failed > 0
diff --git a/integration_tests/packages_1.5.yml b/integration_tests/packages_1.5.yml
@@ -0,0 +1,4 @@
+packages:
+  - local: ../
+  - package: dbt-labs/dbt_utils
+    version: [">=1.0.0", "<2.0.0"]
diff --git a/macros/artifacts/test/store_test_results.sql b/macros/artifacts/test/store_test_results.sql
@@ -21,7 +21,7 @@
     {{ return('') -}}
   {% endif -%}
 
-  {%- set log_tbl %} {{ target.database }}.{{ var('dbt_dq_tool_schema') }}.dq_issue_log {% endset -%}
+  {%- set log_tbl %} {{ var('dbt_dq_tool_database', target.database) }}.{{ var('dbt_dq_tool_schema') }}.dq_issue_log {% endset -%}
 
   {{ log("Centralizing " ~ test_results|length ~ " test results in " + log_tbl, true) if execute -}}
   {{- dq_tools.create_dq_issue_log() -}}

diff --git a/macros/create_table_dq_issue_log.sql b/macros/create_table_dq_issue_log.sql
@@ -9,8 +9,8 @@
 
 {%- if target.type | lower == 'snowflake'  %}
   {%- set query %}
-    create schema if not exists {{ target.database }}.{{ var('dbt_dq_tool_schema') }};
-    create transient table if not exists {{ target.database }}.{{ var('dbt_dq_tool_schema') }}.dq_issue_log
+    create schema if not exists {{ var('dbt_dq_tool_database', target.database) }}.{{ var('dbt_dq_tool_schema') }};
+    create transient table if not exists {{ var('dbt_dq_tool_database', target.database) }}.{{ var('dbt_dq_tool_schema') }}.dq_issue_log
     (
       CHECK_TIMESTAMP TIMESTAMP_NTZ(9),
       TABLE_NAME VARCHAR(16777216),
@@ -31,7 +31,7 @@
 
 {%- elif target.type | lower == 'bigquery' %}
   {%- set query %}
-    create table if not exists {{ target.project }}.{{ var('dbt_dq_tool_schema') }}.dq_issue_log
+    create table if not exists {{ var('dbt_dq_tool_database', target.database) }}.{{ var('dbt_dq_tool_schema') }}.dq_issue_log
     (
       CHECK_TIMESTAMP TIMESTAMP,
       TABLE_NAME STRING,

diff --git a/macros/generic_tests/test_accepted_values_where_db.sql b/macros/generic_tests/test_accepted_values_where_db.sql
@@ -35,7 +35,7 @@
 
    {{ dq_tools.transaction_begin() }}
 
-    insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
+    insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
     select
           {{ dq_tools.current_timestamp() }} check_timestamp
        , '{{ model_text }}' table_name

diff --git a/macros/generic_tests/test_equal_rowcount_where_db.sql b/macros/generic_tests/test_equal_rowcount_where_db.sql
@@ -24,7 +24,7 @@
 
     {{ dq_tools.transaction_begin() }}
 
-    insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, dq_issue_type, ref_table, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
+    insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, dq_issue_type, ref_table, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
       (
         with left_table as (
 

diff --git a/macros/generic_tests/test_equality_where_db.sql b/macros/generic_tests/test_equality_where_db.sql
@@ -31,7 +31,7 @@
 
     {{ dq_tools.transaction_begin() }}
 
-    insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, dq_issue_type, ref_table, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
+    insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, dq_issue_type, ref_table, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
       (
         with left_table as (
 

diff --git a/macros/generic_tests/test_expression_is_true_db.sql b/macros/generic_tests/test_expression_is_true_db.sql
@@ -23,7 +23,7 @@
 
    {{ dq_tools.transaction_begin() }}
 
-    insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
+    insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
     select
           {{ dq_tools.current_timestamp() }} check_timestamp
        , '{{ model_text }}' table_name

diff --git a/macros/generic_tests/test_not_null_where_db.sql b/macros/generic_tests/test_not_null_where_db.sql
@@ -24,7 +24,7 @@
 
     {{ dq_tools.transaction_begin() }}
 
-    insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
+    insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
     select
           {{ dq_tools.current_timestamp() }} check_timestamp
        , '{{ model_text }}' table_name

diff --git a/macros/generic_tests/test_recency_db.sql b/macros/generic_tests/test_recency_db.sql
@@ -23,7 +23,7 @@
 
    {{ dq_tools.transaction_begin() }}
 
-    insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
+    insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
     (
       with max_date as
       (

diff --git a/macros/generic_tests/test_relationships_where_db.sql b/macros/generic_tests/test_relationships_where_db.sql
@@ -26,7 +26,7 @@
 
     {{ dq_tools.transaction_begin() }}
 
-    insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, ref_table, ref_column, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
+    insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, ref_table, ref_column, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
       (
         with left_table as (
 

diff --git a/macros/generic_tests/test_unique_where_db.sql b/macros/generic_tests/test_unique_where_db.sql
@@ -24,7 +24,7 @@
 
     {{ dq_tools.transaction_begin() }}
 
-    insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
+    insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed)
       (
 
         with duplications as (

diff --git a/macros/singular_tests/dq_exec_custom_test_db.sql b/macros/singular_tests/dq_exec_custom_test_db.sql
@@ -15,7 +15,7 @@
 
     begin;
 
-    insert into {{ target.database }}.DBT.DQ_BUSINESS_CHECK_LOG (business_check_code, business_check_comment, check_timestamp, no_of_records, invocation_id, dq_model, severity, sql_cmd)
+    insert into {{ var('dbt_dq_tool_database', target.database) }}.DBT.DQ_BUSINESS_CHECK_LOG (business_check_code, business_check_comment, check_timestamp, no_of_records, invocation_id, dq_model, severity, sql_cmd)
 
     with subsql as
     (
@@ -46,7 +46,7 @@ commit;
    {% do run_query(sql_cmd) %}
 
    select 1
-   from {{ target.database }}.DBT.DQ_BUSINESS_CHECK_LOG
+   from {{ var('dbt_dq_tool_database', target.database) }}.DBT.DQ_BUSINESS_CHECK_LOG
    where invocation_id='{{ p_invocation_id }}'
          and dq_model = '{{ this }}'
 {% endif %}

diff --git a/models/01_lake/source_dq_tools.yml b/models/01_lake/source_dq_tools.yml
@@ -2,22 +2,15 @@ version: 2
 
 sources:
   - name: dq_tools
+    database: "{{ var('dbt_dq_tool_database', target.database) }}"
     schema: "{{ var('dbt_dq_tool_schema') }}"
     tables:
       - name: dq_issue_log
-        tests:
-          - dq_tools.expression_is_true_db:
-              where: coalesce(ref_column,'') != ''
-              expression: coalesce(ref_table,'') != ''
         columns:
           - name: check_timestamp
             description: Test run timestamp
-            tests:
-              - dq_tools.not_null_where_db
           - name: table_name
             description: table scope which test runs against
-            tests:
-              - dq_tools.not_null_where_db
           - name: column_name
             description: column scope which test runs against. can be empty as the test can run against multiple columns
           - name: ref_table
@@ -37,30 +30,12 @@ sources:
                 - equal row count
                 - equality
                 - uniqueness
-            tests:
-              - dq_tools.not_null_where_db
           - name: invocation_id
             description: invocation uuid generated by dbt
-            tests:
-              - dq_tools.not_null_where_db
           - name: dq_model
             description: the dbt model name or its sub query if having `where` parameter
-            tests:
-              - dq_tools.not_null_where_db
-              - dq_tools.expression_is_true_db:  # singular test error should not exists
-                  expression: dq_model != 'assert_intentional_error'
-              - dq_tools.expression_is_true_db: # generic test error should not exists
-                  where: table_name like '%.test_failures' and coalesce(column_name,'') = ''
-                  expression: dq_model not like 'dq_tools_expression_is_true_db_test_failures_1_0%'
           - name: severity
-            description: warn or error
-            tests:
-              - dq_tools.not_null_where_db
-              - dq_tools.accepted_values_where_db:
-                  values: [
-                    'warn',
-                    'error'
-                  ]
+            description: test severity level, values are `warn` or `error`
           - name: kpi_category
             description: |
               data quality kpis
@@ -73,30 +48,7 @@ sources:
                 - consistency
                 - uniqueness
                 - accuracy
-            tests:
-              - dq_tools.not_null_where_db
-              - dq_tools.accepted_values_where_db:
-                  values: [
-                    'Other',
-                    'Uniqueness',
-                    'Consistency',
-                    'Validity',
-                    'Timeliness',
-                    'Completeness',
-                    'Accuracy',
-                  ]
           - name: no_of_records
-            tests:
-              - dq_tools.not_null_where_db:
-                  where: coalesce(table_name,'') != ''
-              - dq_tools.expression_is_true_db:
-                  where: coalesce(ref_table,'') != ''
-                  expression: no_of_records >= 0
+            description: number of rows scanned by the test
           - name: no_of_records_failed
-            tests:
-              - dq_tools.not_null_where_db
-              - dq_tools.expression_is_true_db:
-                  expression: no_of_records_failed >= 0
-              - dq_tools.expression_is_true_db:
-                  where: lower(dq_model) like '%failure%'
-                  expression: no_of_records_failed > 0
+            description: number of rows failed by the test