From 82ed5115144ce4eea5915c11287f3b7f2c635160 Mon Sep 17 00:00:00 2001 From: Dat Nguyen <103571964+il-dat@users.noreply.github.com> Date: Tue, 1 Aug 2023 08:34:52 +0700 Subject: [PATCH] Feature/#6 database config (#7) * Add dbt_dq_tool_database variable * #6 Move tests of log table to integration tests * #6 fix dbt version in pr pipeline * #6 fix hook error * #6 update test script and add security * #6 update config for test * #6 comment in dbt_dq_tool_database for test * #6 fix dbt_dq_tool_database missing deafult * #6 extend test to other dbt versions * # isolate schema for PR with dbt version * #6 fix DBT_SCHEMA in pr ci pipeline --------- Co-authored-by: Dat Nguyen --- .github/workflows/ci-pr.yml | 14 ++-- SECURITY.md | 24 +++++++ dbt_project.yml | 1 + integration_tests/dbt_project.yml | 5 +- .../models/source_dq_tools_test.yml | 69 +++++++++++++++++++ integration_tests/packages_1.5.yml | 4 ++ macros/artifacts/test/store_test_results.sql | 2 +- macros/create_table_dq_issue_log.sql | 6 +- .../test_accepted_values_where_db.sql | 2 +- .../test_equal_rowcount_where_db.sql | 2 +- .../generic_tests/test_equality_where_db.sql | 2 +- .../test_expression_is_true_db.sql | 2 +- .../generic_tests/test_not_null_where_db.sql | 2 +- macros/generic_tests/test_recency_db.sql | 2 +- .../test_relationships_where_db.sql | 2 +- macros/generic_tests/test_unique_where_db.sql | 2 +- .../singular_tests/dq_exec_custom_test_db.sql | 4 +- models/01_lake/source_dq_tools.yml | 56 ++------------- run_test.sh | 7 +- 19 files changed, 131 insertions(+), 77 deletions(-) create mode 100644 SECURITY.md create mode 100644 integration_tests/models/source_dq_tools_test.yml create mode 100644 integration_tests/packages_1.5.yml diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 1c878e1..f6ffb14 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -11,7 +11,7 @@ jobs: max-parallel: 1 matrix: adapter: ["snowflake"] - version: ["1.3.0"] #"1.4.1", "1.5.2" + version: ["1.3.0", "1.4.1", "1.5.2"] container: image: "ghcr.io/dbt-labs/dbt-${{ matrix.adapter }}:${{ matrix.version }}" steps: @@ -20,7 +20,8 @@ jobs: - name: Run integration test run: | chmod +x run_test.sh - cp -fr ./integration_tests/packages_${{ matrix.version }}.yml ./integration_tests/packages.yml + dbtversion=${{ matrix.version }} + cp -fr ./integration_tests/packages_${dbtversion%.*}.yml ./integration_tests/packages.yml ./run_test.sh ${{ matrix.adapter }} env: DBT_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_TEST_ACCOUNT }} @@ -29,7 +30,7 @@ jobs: DBT_SNOWFLAKE_TEST_ROLE: ${{ secrets.DBT_SNOWFLAKE_TEST_ROLE }} DBT_SNOWFLAKE_TEST_DATABASE: ${{ secrets.DBT_SNOWFLAKE_TEST_DATABASE }} DBT_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.DBT_SNOWFLAKE_TEST_WAREHOUSE }} - DBT_SCHEMA: PRCI_${{ github.sha }} + DBT_SCHEMA: PRCI_${{ matrix.version }}_${{ github.sha }} poseidon: runs-on: ubuntu-latest @@ -37,7 +38,7 @@ jobs: max-parallel: 1 matrix: adapter: ["bigquery"] - version: ["1.3.0"] #"1.4.1", "1.5.3" + version: ["1.3.0", "1.4.1", "1.5.3"] container: image: "ghcr.io/dbt-labs/dbt-${{ matrix.adapter }}:${{ matrix.version }}" steps: @@ -52,9 +53,10 @@ jobs: - name: Run integration test run: | chmod +x run_test.sh - cp -fr ./integration_tests/packages_${{ matrix.version }}.yml ./integration_tests/packages.yml + dbtversion=${{ matrix.version }} + cp -fr ./integration_tests/packages_${dbtversion%.*}.yml ./integration_tests/packages.yml ./run_test.sh ${{ matrix.adapter }} env: DBT_ENV_SECRET_BIGQUERY_SERVICE_KEY_PATH: /opt/bigquery_service_key.json DBT_BIGQUERY_TEST_DATABASE: ${{ secrets.DBT_BIGQUERY_TEST_DATABASE }} - DBT_SCHEMA: PRCI_${{ github.sha }} \ No newline at end of file + DBT_SCHEMA: PRCI_DQTOOLS_${{ matrix.version }}_${{ github.sha }} \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..603af12 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,24 @@ +# Security Policy + +## Supported Versions + +Use this section to tell people about which versions of your project are +currently being supported with security updates. + +| dbt Core | dq_tools 1.1 | dq_tools 1.2 | +| ----------- | ------------------ | ------------------ | +| 1.5.x | :warning: | :white_check_mark: | +| 1.4.x | :warning: | :white_check_mark: | +| 1.3.x | :white_check_mark: | :white_check_mark: | +| 1.2.x | :white_check_mark: | :x: | +| 1.1.x | :white_check_mark: | :x: | +| 1.0.x | :white_check_mark: | :x: | +| < 1.0 | :x: | :x: | + +## Reporting a Vulnerability + +Use this section to tell people how to report a vulnerability. + +Tell them where to go, how often they can expect to get an update on a +reported vulnerability, what to expect if the vulnerability is accepted or +declined, etc. \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index b258339..a56f351 100755 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -16,4 +16,5 @@ models: vars: # dbt_dq_tool_schema: dq_raw + # dbt_dq_tool_database: dq_tools # dq_tools_enable_store_test_results: true \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index c4dcdae..5cf59b7 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -31,8 +31,8 @@ on-run-start: {% if var('fresh', false) %} {{ "drop schema if exists " ~ target.schema }}; {{ "create schema " ~ target.schema }}; - {{ "drop schema if exists " ~ var("dbt_dq_tool_schema", target.schema) }}; - {{ "create schema " ~ var("dbt_dq_tool_schema", target.schema) }}; + {{ "drop schema if exists " ~ var("dbt_dq_tool_database", target.database) ~ "." ~ var("dbt_dq_tool_schema", target.schema) }}; + {{ "create schema " ~ var("dbt_dq_tool_database", target.database) ~ "." ~ var("dbt_dq_tool_schema", target.schema) }}; {% endif %} on-run-end: @@ -40,6 +40,7 @@ on-run-end: vars: dbt_dq_tool_schema: dq_raw + # dbt_dq_tool_database: dq_tools seeds: dq_tools_integration_tests: diff --git a/integration_tests/models/source_dq_tools_test.yml b/integration_tests/models/source_dq_tools_test.yml new file mode 100644 index 0000000..ae42201 --- /dev/null +++ b/integration_tests/models/source_dq_tools_test.yml @@ -0,0 +1,69 @@ +version: 2 + +sources: + - name: dq_tools_test + database: "{{ var('dbt_dq_tool_database', target.database) }}" + schema: "{{ var('dbt_dq_tool_schema') }}" + tables: + - name: dq_issue_log + tests: + - dq_tools.expression_is_true_db: + where: coalesce(ref_column,'') != '' + expression: coalesce(ref_table,'') != '' + columns: + - name: check_timestamp + tests: + - dq_tools.not_null_where_db + - name: table_name + tests: + - dq_tools.not_null_where_db + - name: dq_issue_type + tests: + - dq_tools.not_null_where_db + - name: invocation_id + tests: + - dq_tools.not_null_where_db + - name: dq_model + tests: + - dq_tools.not_null_where_db + - dq_tools.expression_is_true_db: # singular test error should not exists + expression: dq_model != 'assert_intentional_error' + - dq_tools.expression_is_true_db: # generic test error should not exists + where: table_name like '%.test_failures' and coalesce(column_name,'') = '' + expression: dq_model not like 'dq_tools_expression_is_true_db_test_failures_1_0%' + - name: severity + tests: + - dq_tools.not_null_where_db + - dq_tools.accepted_values_where_db: + values: [ + 'warn', + 'error' + ] + - name: kpi_category + tests: + - dq_tools.not_null_where_db + - dq_tools.accepted_values_where_db: + values: [ + 'Other', + 'Uniqueness', + 'Consistency', + 'Validity', + 'Timeliness', + 'Completeness', + 'Accuracy', + ] + - name: no_of_records + tests: + - dq_tools.not_null_where_db: + where: coalesce(table_name,'') != '' + - dq_tools.expression_is_true_db: + where: coalesce(ref_table,'') != '' + expression: no_of_records >= 0 + - name: no_of_records_failed + tests: + - dq_tools.not_null_where_db + - dq_tools.expression_is_true_db: + expression: no_of_records_failed >= 0 + - dq_tools.expression_is_true_db: + where: lower(dq_model) like '%failure%' + expression: no_of_records_failed > 0 diff --git a/integration_tests/packages_1.5.yml b/integration_tests/packages_1.5.yml new file mode 100644 index 0000000..337b780 --- /dev/null +++ b/integration_tests/packages_1.5.yml @@ -0,0 +1,4 @@ +packages: + - local: ../ + - package: dbt-labs/dbt_utils + version: [">=1.0.0", "<2.0.0"] \ No newline at end of file diff --git a/macros/artifacts/test/store_test_results.sql b/macros/artifacts/test/store_test_results.sql index 0eab646..a031eb2 100644 --- a/macros/artifacts/test/store_test_results.sql +++ b/macros/artifacts/test/store_test_results.sql @@ -21,7 +21,7 @@ {{ return('') -}} {% endif -%} - {%- set log_tbl %} {{ target.database }}.{{ var('dbt_dq_tool_schema') }}.dq_issue_log {% endset -%} + {%- set log_tbl %} {{ var('dbt_dq_tool_database', target.database) }}.{{ var('dbt_dq_tool_schema') }}.dq_issue_log {% endset -%} {{ log("Centralizing " ~ test_results|length ~ " test results in " + log_tbl, true) if execute -}} {{- dq_tools.create_dq_issue_log() -}} diff --git a/macros/create_table_dq_issue_log.sql b/macros/create_table_dq_issue_log.sql index 82de3f9..b0b348b 100644 --- a/macros/create_table_dq_issue_log.sql +++ b/macros/create_table_dq_issue_log.sql @@ -9,8 +9,8 @@ {%- if target.type | lower == 'snowflake' %} {%- set query %} - create schema if not exists {{ target.database }}.{{ var('dbt_dq_tool_schema') }}; - create transient table if not exists {{ target.database }}.{{ var('dbt_dq_tool_schema') }}.dq_issue_log + create schema if not exists {{ var('dbt_dq_tool_database', target.database) }}.{{ var('dbt_dq_tool_schema') }}; + create transient table if not exists {{ var('dbt_dq_tool_database', target.database) }}.{{ var('dbt_dq_tool_schema') }}.dq_issue_log ( CHECK_TIMESTAMP TIMESTAMP_NTZ(9), TABLE_NAME VARCHAR(16777216), @@ -31,7 +31,7 @@ {%- elif target.type | lower == 'bigquery' %} {%- set query %} - create table if not exists {{ target.project }}.{{ var('dbt_dq_tool_schema') }}.dq_issue_log + create table if not exists {{ var('dbt_dq_tool_database', target.database) }}.{{ var('dbt_dq_tool_schema') }}.dq_issue_log ( CHECK_TIMESTAMP TIMESTAMP, TABLE_NAME STRING, diff --git a/macros/generic_tests/test_accepted_values_where_db.sql b/macros/generic_tests/test_accepted_values_where_db.sql index 3289e6f..4ca01b6 100644 --- a/macros/generic_tests/test_accepted_values_where_db.sql +++ b/macros/generic_tests/test_accepted_values_where_db.sql @@ -35,7 +35,7 @@ {{ dq_tools.transaction_begin() }} - insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) + insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) select {{ dq_tools.current_timestamp() }} check_timestamp , '{{ model_text }}' table_name diff --git a/macros/generic_tests/test_equal_rowcount_where_db.sql b/macros/generic_tests/test_equal_rowcount_where_db.sql index 02dad94..b32d0a4 100644 --- a/macros/generic_tests/test_equal_rowcount_where_db.sql +++ b/macros/generic_tests/test_equal_rowcount_where_db.sql @@ -24,7 +24,7 @@ {{ dq_tools.transaction_begin() }} - insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, dq_issue_type, ref_table, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) + insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, dq_issue_type, ref_table, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) ( with left_table as ( diff --git a/macros/generic_tests/test_equality_where_db.sql b/macros/generic_tests/test_equality_where_db.sql index a625865..d014d07 100644 --- a/macros/generic_tests/test_equality_where_db.sql +++ b/macros/generic_tests/test_equality_where_db.sql @@ -31,7 +31,7 @@ {{ dq_tools.transaction_begin() }} - insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, dq_issue_type, ref_table, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) + insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, dq_issue_type, ref_table, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) ( with left_table as ( diff --git a/macros/generic_tests/test_expression_is_true_db.sql b/macros/generic_tests/test_expression_is_true_db.sql index 187b9a2..47b4870 100644 --- a/macros/generic_tests/test_expression_is_true_db.sql +++ b/macros/generic_tests/test_expression_is_true_db.sql @@ -23,7 +23,7 @@ {{ dq_tools.transaction_begin() }} - insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) + insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) select {{ dq_tools.current_timestamp() }} check_timestamp , '{{ model_text }}' table_name diff --git a/macros/generic_tests/test_not_null_where_db.sql b/macros/generic_tests/test_not_null_where_db.sql index 3d00c7f..e3316a5 100644 --- a/macros/generic_tests/test_not_null_where_db.sql +++ b/macros/generic_tests/test_not_null_where_db.sql @@ -24,7 +24,7 @@ {{ dq_tools.transaction_begin() }} - insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) + insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) select {{ dq_tools.current_timestamp() }} check_timestamp , '{{ model_text }}' table_name diff --git a/macros/generic_tests/test_recency_db.sql b/macros/generic_tests/test_recency_db.sql index 69864f3..308c608 100644 --- a/macros/generic_tests/test_recency_db.sql +++ b/macros/generic_tests/test_recency_db.sql @@ -23,7 +23,7 @@ {{ dq_tools.transaction_begin() }} - insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) + insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) ( with max_date as ( diff --git a/macros/generic_tests/test_relationships_where_db.sql b/macros/generic_tests/test_relationships_where_db.sql index 9fc2bc2..0347eb8 100644 --- a/macros/generic_tests/test_relationships_where_db.sql +++ b/macros/generic_tests/test_relationships_where_db.sql @@ -26,7 +26,7 @@ {{ dq_tools.transaction_begin() }} - insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, ref_table, ref_column, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) + insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, ref_table, ref_column, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) ( with left_table as ( diff --git a/macros/generic_tests/test_unique_where_db.sql b/macros/generic_tests/test_unique_where_db.sql index 3ae6dce..a7e0e31 100644 --- a/macros/generic_tests/test_unique_where_db.sql +++ b/macros/generic_tests/test_unique_where_db.sql @@ -24,7 +24,7 @@ {{ dq_tools.transaction_begin() }} - insert into {{ target.database }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) + insert into {{ var('dbt_dq_tool_database', target.database) }}.{{var('dbt_dq_tool_schema', target.schema)}}.dq_issue_log (check_timestamp, table_name, column_name, dq_issue_type, invocation_id, dq_model, severity, kpi_category, no_of_records, no_of_records_failed) ( with duplications as ( diff --git a/macros/singular_tests/dq_exec_custom_test_db.sql b/macros/singular_tests/dq_exec_custom_test_db.sql index 2111abf..b00b8a7 100644 --- a/macros/singular_tests/dq_exec_custom_test_db.sql +++ b/macros/singular_tests/dq_exec_custom_test_db.sql @@ -15,7 +15,7 @@ begin; - insert into {{ target.database }}.DBT.DQ_BUSINESS_CHECK_LOG (business_check_code, business_check_comment, check_timestamp, no_of_records, invocation_id, dq_model, severity, sql_cmd) + insert into {{ var('dbt_dq_tool_database', target.database) }}.DBT.DQ_BUSINESS_CHECK_LOG (business_check_code, business_check_comment, check_timestamp, no_of_records, invocation_id, dq_model, severity, sql_cmd) with subsql as ( @@ -46,7 +46,7 @@ commit; {% do run_query(sql_cmd) %} select 1 - from {{ target.database }}.DBT.DQ_BUSINESS_CHECK_LOG + from {{ var('dbt_dq_tool_database', target.database) }}.DBT.DQ_BUSINESS_CHECK_LOG where invocation_id='{{ p_invocation_id }}' and dq_model = '{{ this }}' {% endif %} diff --git a/models/01_lake/source_dq_tools.yml b/models/01_lake/source_dq_tools.yml index adc2906..6c260c5 100644 --- a/models/01_lake/source_dq_tools.yml +++ b/models/01_lake/source_dq_tools.yml @@ -2,22 +2,15 @@ version: 2 sources: - name: dq_tools + database: "{{ var('dbt_dq_tool_database', target.database) }}" schema: "{{ var('dbt_dq_tool_schema') }}" tables: - name: dq_issue_log - tests: - - dq_tools.expression_is_true_db: - where: coalesce(ref_column,'') != '' - expression: coalesce(ref_table,'') != '' columns: - name: check_timestamp description: Test run timestamp - tests: - - dq_tools.not_null_where_db - name: table_name description: table scope which test runs against - tests: - - dq_tools.not_null_where_db - name: column_name description: column scope which test runs against. can be empty as the test can run against multiple columns - name: ref_table @@ -37,30 +30,12 @@ sources: - equal row count - equality - uniqueness - tests: - - dq_tools.not_null_where_db - name: invocation_id description: invocation uuid generated by dbt - tests: - - dq_tools.not_null_where_db - name: dq_model description: the dbt model name or its sub query if having `where` parameter - tests: - - dq_tools.not_null_where_db - - dq_tools.expression_is_true_db: # singular test error should not exists - expression: dq_model != 'assert_intentional_error' - - dq_tools.expression_is_true_db: # generic test error should not exists - where: table_name like '%.test_failures' and coalesce(column_name,'') = '' - expression: dq_model not like 'dq_tools_expression_is_true_db_test_failures_1_0%' - name: severity - description: warn or error - tests: - - dq_tools.not_null_where_db - - dq_tools.accepted_values_where_db: - values: [ - 'warn', - 'error' - ] + description: test severity level, values are `warn` or `error` - name: kpi_category description: | data quality kpis @@ -73,30 +48,7 @@ sources: - consistency - uniqueness - accuracy - tests: - - dq_tools.not_null_where_db - - dq_tools.accepted_values_where_db: - values: [ - 'Other', - 'Uniqueness', - 'Consistency', - 'Validity', - 'Timeliness', - 'Completeness', - 'Accuracy', - ] - name: no_of_records - tests: - - dq_tools.not_null_where_db: - where: coalesce(table_name,'') != '' - - dq_tools.expression_is_true_db: - where: coalesce(ref_table,'') != '' - expression: no_of_records >= 0 + description: number of rows scanned by the test - name: no_of_records_failed - tests: - - dq_tools.not_null_where_db - - dq_tools.expression_is_true_db: - expression: no_of_records_failed >= 0 - - dq_tools.expression_is_true_db: - where: lower(dq_model) like '%failure%' - expression: no_of_records_failed > 0 + description: number of rows failed by the test diff --git a/run_test.sh b/run_test.sh index 0a455e0..bd53dda 100755 --- a/run_test.sh +++ b/run_test.sh @@ -10,6 +10,7 @@ dbt --version cd integration_tests cp ci/sample.profiles.yml profiles.yml export DBT_PROFILES_DIR=. +export DBT_SCHEMA=${DBT_SCHEMA//./_} # Show the location of the profiles directory and test the connection dbt debug --target $1 @@ -36,10 +37,10 @@ echo -e "${BLUE}1: Perform Failures intentionally inc Errors / w Fresh enviromen dbt seed --target $1 $_models || exit 1 echo -e "${BLUE}2: Verify macros / Turn warns as errors${NC}" -dbt --warn-error build --exclude source:dq_tools+ tag:failed --vars '{dq_tools_enable_store_test_results: true}' --target $1 $_models || exit 1 +dbt --warn-error build --exclude source:dq_tools_test+ tag:failed --vars '{dq_tools_enable_store_test_results: true}' --target $1 $_models || exit 1 echo -e "${BLUE}2: Enable & Verify models / Turn warns as errors${NC}" -dbt --warn-error build --select +tag:dq+ --exclude source:dq_tools --vars '{dq_tools_enable_store_test_results: true}' --target $1 $_models || exit 1 +dbt --warn-error build --select +tag:dq+ --exclude source:dq_tools_test --vars '{dq_tools_enable_store_test_results: true}' --target $1 $_models || exit 1 echo -e "${BLUE}3: Verify log table / Turn warns as errors${NC}" -dbt --warn-error test --select source:dq_tools --target $1 || exit 1 \ No newline at end of file +dbt --warn-error test --select source:dq_tools_test --target $1 || exit 1 \ No newline at end of file