diff --git a/integration_tests/models/behavior/verify_if_masking_policies_applied_correctly.yml b/integration_tests/models/behavior/verify_if_masking_policies_applied_correctly.yml index 14d14e9..4a94667 100644 --- a/integration_tests/models/behavior/verify_if_masking_policies_applied_correctly.yml +++ b/integration_tests/models/behavior/verify_if_masking_policies_applied_correctly.yml @@ -1,5 +1,5 @@ models: - name: verify_if_masking_policies_applied_correctly - tests: + data_tests: - dbt_utils.expression_is_true: expression: ifnull(adapter_tag, '') = dbt_project_tag diff --git a/integration_tests/models/behavior/verify_if_masking_policies_created_correctly.yml b/integration_tests/models/behavior/verify_if_masking_policies_created_correctly.yml index 56c2d3a..d3b7f38 100644 --- a/integration_tests/models/behavior/verify_if_masking_policies_created_correctly.yml +++ b/integration_tests/models/behavior/verify_if_masking_policies_created_correctly.yml @@ -1,5 +1,5 @@ models: - name: verify_if_masking_policies_created_correctly - tests: + data_tests: - dbt_utils.expression_is_true: expression: ifnull(adapter_masking_policy, '') = dbt_project_masking_policy diff --git a/integration_tests/models/behavior/verify_if_tags_applied_correctly.yml b/integration_tests/models/behavior/verify_if_tags_applied_correctly.yml index aee0e4f..288cdb9 100644 --- a/integration_tests/models/behavior/verify_if_tags_applied_correctly.yml +++ b/integration_tests/models/behavior/verify_if_tags_applied_correctly.yml @@ -1,5 +1,5 @@ models: - name: verify_if_tags_applied_correctly - tests: + data_tests: - dbt_utils.expression_is_true: expression: ifnull(actual_tag_value, '') = config_tag_value diff --git a/integration_tests/models/behavior/verify_if_tags_created_correctly.yml b/integration_tests/models/behavior/verify_if_tags_created_correctly.yml index 2103975..74cde37 100644 --- a/integration_tests/models/behavior/verify_if_tags_created_correctly.yml +++ b/integration_tests/models/behavior/verify_if_tags_created_correctly.yml @@ -1,5 +1,5 @@ models: - name: verify_if_tags_created_correctly - tests: + data_tests: - dbt_utils.expression_is_true: expression: adapter_tag is not null diff --git a/integration_tests/models/customers.yml b/integration_tests/models/customers.yml index e49c36b..b346c95 100644 --- a/integration_tests/models/customers.yml +++ b/integration_tests/models/customers.yml @@ -8,7 +8,7 @@ models: columns: - name: customer_id description: This is a unique identifier for a customer - tests: + data_tests: - unique - not_null diff --git a/integration_tests/models/orders.yml b/integration_tests/models/orders.yml index b1a0aa3..51b1d88 100644 --- a/integration_tests/models/orders.yml +++ b/integration_tests/models/orders.yml @@ -4,51 +4,30 @@ models: columns: - name: order_id - tests: - - unique - - not_null description: This is a unique identifier for an order - name: customer_id description: Foreign key to the customers table - tests: - - not_null - - relationships: - to: ref('customers') - field: customer_id - name: order_date description: Date (UTC) that the order was placed - name: status description: Order status - tests: - - accepted_values: - values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] - name: amount description: Total amount (AUD) of the order tags: - pii_amount - tests: - - not_null - name: credit_card_amount description: Amount of the order (AUD) paid for by credit card - tests: - - not_null - name: coupon_amount description: Amount of the order (AUD) paid for by coupon - tests: - - not_null - name: bank_transfer_amount description: Amount of the order (AUD) paid for by bank transfer - tests: - - not_null - name: gift_card_amount description: Amount of the order (AUD) paid for by gift card - tests: - - not_null diff --git a/integration_tests/tests/cleanup/verify_if_drop_tags_correctly.sql b/integration_tests/tests/cleanup/verify_if_drop_tags_correctly.sql new file mode 100644 index 0000000..4348d90 --- /dev/null +++ b/integration_tests/tests/cleanup/verify_if_drop_tags_correctly.sql @@ -0,0 +1,9 @@ +{{ + config( + tags = ['cleanup'], + ) +}} +select * +from {{ ref('verify_if_tags_created_correctly') }} +where dbt_project_tag in ('{{ var("dbt_tags__allowed_tags", []) | join("','") }}') + and adapter_tag is not null diff --git a/integration_tests/tests/cleanup/verify_if_unapply_mps_correctly.sql b/integration_tests/tests/cleanup/verify_if_unapply_mps_correctly.sql new file mode 100644 index 0000000..12a0f2e --- /dev/null +++ b/integration_tests/tests/cleanup/verify_if_unapply_mps_correctly.sql @@ -0,0 +1,9 @@ +{{ + config( + tags = ['cleanup'], + ) +}} + +select * +from {{ ref('verify_if_masking_policies_applied_correctly') }} +where adapter_tag is not null diff --git a/macros/resources/tags/drop_tags.sql b/macros/resources/tags/drop_tags.sql index 64b317d..a400cb6 100644 --- a/macros/resources/tags/drop_tags.sql +++ b/macros/resources/tags/drop_tags.sql @@ -6,20 +6,33 @@ {% set ns = dbt_tags.get_resource_ns() %} {% set adapter_tags = dbt_tags.get_dbt_tags(debug=debug) %} + {% set policy_data_types_list = var('dbt_tags__policy_data_types', none) -%} {% set query -%} - {% for item in adapter_tags %} - {%- if dbt_tags.get_masking_policy_for_tag(item.tag) %} - {% for policy_data_types in policy_data_types_list if item.tag in policy_data_types.keys() %} - {% for datatype in policy_data_types.values() | first %} - alter tag {{ ns }}.{{ item.tag }} unset masking policy {{ ns }}.{{ item.tag }}_{{ datatype }}; - {% endfor %} - {% else %} - alter tag {{ ns }}.{{ item.tag }} unset masking policy {{ ns }}.{{ item.tag }}; - {% endfor %} + {% set unique_tags = [] -%} + {% for item in adapter_tags -%} + {%- if dbt_tags.get_masking_policy_for_tag(item.tag) -%} + {% for policy_data_types in policy_data_types_list if item.tag in policy_data_types.keys() -%} + {% for datatype in policy_data_types.values() | first -%} + {% set tag_info = (item.tag, item.tag ~ "_" ~ datatype) -%} + {% do unique_tags.append(tag_info) if tag_info not in unique_tags -%} + {% endfor -%} + {% else -%} + {% set tag_info = (item.tag, item.tag) -%} + {% do unique_tags.append(tag_info) if tag_info not in unique_tags -%} + {% endfor -%} + {% else -%} + {% set tag_info = (item.tag, none) -%} + {% do unique_tags.append(tag_info) if tag_info not in unique_tags -%} {%- endif -%} - drop tag {{ item.tag }}; - {% endfor %} + {% endfor -%} + + {%- for item in unique_tags if item[1] is not none %} + alter tag {{ ns }}.{{ item[0] }} unset masking policy {{ ns }}.{{ item[1] }}; + {%- endfor %} + {%- for item in unique_tags %} + drop tag if exists {{ ns }}.{{ item[0] }}; + {%- endfor %} {%- endset %} diff --git a/poetry.lock b/poetry.lock index 7f5de44..38bf20a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "agate" @@ -435,13 +435,13 @@ test = ["hypothesis (>=6.87,<7.0)", "pytest (>=7.3,<8.0)", "pytest-cov (>=4.1,<5 [[package]] name = "dbt-core" -version = "1.8.3" +version = "1.8.4" description = "With dbt, data analysts and engineers can build analytics the way engineers build applications." optional = false python-versions = ">=3.8" files = [ - {file = "dbt_core-1.8.3-py3-none-any.whl", hash = "sha256:29c4d3ed4385090492ea48e88c58a405e4d646aabde81e7945bedddb2a55e86e"}, - {file = "dbt_core-1.8.3.tar.gz", hash = "sha256:e98ea11f0c91f086e9df9ac4d2e6093cf30a94e6fd419541a32374ddd6b92d9e"}, + {file = "dbt_core-1.8.4-py3-none-any.whl", hash = "sha256:d748d51b9d38c0b93b7f047cc32bdd97a5933cb89aec33bf17a47172c12fda03"}, + {file = "dbt_core-1.8.4.tar.gz", hash = "sha256:3eb37273f7ceaa070a8e3e42bfd806954be769f87292971829ce8e5baa9abad5"}, ] [package.dependencies] @@ -551,13 +551,13 @@ optimize = ["orjson"] [[package]] name = "diff-cover" -version = "9.1.0" +version = "9.1.1" description = "Run coverage and linting reports on diffs" optional = false python-versions = "<4.0.0,>=3.8.10" files = [ - {file = "diff_cover-9.1.0-py3-none-any.whl", hash = "sha256:9b95204e6b98ec2815087b0ad1d504422a41fe36d7c778497eba128f267618c4"}, - {file = "diff_cover-9.1.0.tar.gz", hash = "sha256:c6c9f7f1eacd9019d118cafb75908445caa41a79f65f36f9f506c9626dbcbc70"}, + {file = "diff_cover-9.1.1-py3-none-any.whl", hash = "sha256:2d520d6c4f41674c7e3010ce5e0f637bd2fab4dc2f8e3e174ad39e0364318310"}, + {file = "diff_cover-9.1.1.tar.gz", hash = "sha256:b5ed20955b3ebdee94476e429cfd9f1324e1c19a04c4aae32a893b11c3673f1e"}, ] [package.dependencies] @@ -1142,13 +1142,13 @@ pyyaml = ">=5.1" [[package]] name = "mkdocs-material" -version = "9.5.29" +version = "9.5.30" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.29-py3-none-any.whl", hash = "sha256:afc1f508e2662ded95f0a35a329e8a5acd73ee88ca07ba73836eb6fcdae5d8b4"}, - {file = "mkdocs_material-9.5.29.tar.gz", hash = "sha256:3e977598ec15a4ddad5c4dfc9e08edab6023edb51e88f0729bd27be77e3d322a"}, + {file = "mkdocs_material-9.5.30-py3-none-any.whl", hash = "sha256:fc070689c5250a180e9b9d79d8491ef9a3a7acb240db0728728d6c31eeb131d4"}, + {file = "mkdocs_material-9.5.30.tar.gz", hash = "sha256:3fd417dd42d679e3ba08b9e2d72cd8b8af142cc4a3969676ad6b00993dd182ec"}, ] [package.dependencies] @@ -1639,17 +1639,17 @@ extra = ["pygments (>=2.12)"] [[package]] name = "pyopenssl" -version = "24.1.0" +version = "24.2.1" description = "Python wrapper module around the OpenSSL library" optional = false python-versions = ">=3.7" files = [ - {file = "pyOpenSSL-24.1.0-py3-none-any.whl", hash = "sha256:17ed5be5936449c5418d1cd269a1a9e9081bc54c17aed272b45856a3d3dc86ad"}, - {file = "pyOpenSSL-24.1.0.tar.gz", hash = "sha256:cabed4bfaa5df9f1a16c0ef64a0cb65318b5cd077a7eda7d6970131ca2f41a6f"}, + {file = "pyOpenSSL-24.2.1-py3-none-any.whl", hash = "sha256:967d5719b12b243588573f39b0c677637145c7a1ffedcd495a487e58177fbb8d"}, + {file = "pyopenssl-24.2.1.tar.gz", hash = "sha256:4247f0dbe3748d560dcbb2ff3ea01af0f9a1a001ef5f7c4c647956ed8cbf0e95"}, ] [package.dependencies] -cryptography = ">=41.0.5,<43" +cryptography = ">=41.0.5,<44" [package.extras] docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx-rtd-theme"] @@ -1671,13 +1671,13 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "8.2.2" +version = "8.3.1" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, - {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, + {file = "pytest-8.3.1-py3-none-any.whl", hash = "sha256:e9600ccf4f563976e2c99fa02c7624ab938296551f280835ee6516df8bc4ae8c"}, + {file = "pytest-8.3.1.tar.gz", hash = "sha256:7e8e5c5abd6e93cb1cc151f23e57adc31fcf8cfd2a3ff2da63e23f732de35db6"}, ] [package.dependencies] @@ -1685,7 +1685,7 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" -pluggy = ">=1.5,<2.0" +pluggy = ">=1.5,<2" tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] diff --git a/pyproject.toml b/pyproject.toml index 5879379..67b225f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,12 +83,32 @@ dbt-tags-verify = [ {cmd = "dbt deps --project-dir integration_tests"}, {cmd = "dbt debug --project-dir integration_tests"}, ] -dbt-tags-test = [ - {cmd = "dbt deps --project-dir integration_tests"}, +dbt-tags-tagging = [ {cmd = "dbt run-operation create_tags --project-dir integration_tests"}, {cmd = "dbt run-operation create_masking_policies --project-dir integration_tests"}, {cmd = "dbt run-operation apply_mps_to_tags --project-dir integration_tests"}, - {cmd = "dbt build --project-dir integration_tests"}, +] +dbt-tags-test-tagging = [ + {cmd = "dbt deps --project-dir integration_tests"}, + {cmd = "poe dbt-tags-tagging"}, + {cmd = "dbt build --exclude tag:cleanup --project-dir integration_tests"}, +] +dbt-tags-test-unapply-mps = [ + {cmd = "poe dbt-tags-tagging"}, + {cmd = "dbt run-operation unapply_mps_from_tags --project-dir integration_tests"}, + {cmd = "dbt run -s verify_if_masking_policies_applied_correctly --project-dir integration_tests"}, + {cmd = "dbt test -s verify_if_unapply_mps_correctly --project-dir integration_tests"}, +] +dbt-tags-test-drop-tags = [ + {cmd = "poe dbt-tags-tagging"}, + {cmd = "dbt run-operation drop_tags --project-dir integration_tests"}, + {cmd = "dbt run -s verify_if_tags_created_correctly --project-dir integration_tests"}, + {cmd = "dbt test -s verify_if_drop_tags_correctly --project-dir integration_tests"}, +] +dbt-tags-test = [ + {cmd = "poe dbt-tags-test-tagging"}, + {cmd = "poe dbt-tags-test-unapply-mps"}, + {cmd = "poe dbt-tags-test-drop-tags"}, ] build-doc = [ {cmd = "dbt deps --project-dir integration_tests"},