From 394559c5b78e92a00c369c84cbfab2686aa8d6f7 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 25 Oct 2021 17:41:57 +0200 Subject: [PATCH] Take another look at logic --- materialized-views/README.md | 41 ++++++++++--------- .../integration_tests/.gitignore | 1 + .../integration_tests/dbt_project.yml | 4 ++ .../integration_tests/macros/overrides.sql | 12 ------ .../integration_tests/packages.yml | 6 +-- .../macros/default/adapters.sql | 6 +-- .../macros/default/materialized_view.sql | 8 +++- .../macros/redshift/adapters.sql | 8 ++-- .../macros/snowflake/materialized_view.sql | 5 ++- 9 files changed, 47 insertions(+), 44 deletions(-) diff --git a/materialized-views/README.md b/materialized-views/README.md index 9a928d6..52bbb0c 100644 --- a/materialized-views/README.md +++ b/materialized-views/README.md @@ -14,27 +14,30 @@ If you're here, you may also like the [dbt-materialize](https://github.com/Mater ### General installation: -You can install the materialized-view funcionality using one of the following methods. +You can install the materialized-view project as a package, to get access to it in your own project, either as a `local` package (by cloning this repository to your machine) or as a `git` package referencing the `materialized-views` subdirectory: +```yml +# packages.yml +packages: + - git: https://github.com/dbt-labs/dbt-labs-experimental-features.git + subdirectory: materialized-views +``` + +If you do this, you'll need to set the `dispatch` project config ([docs](https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch)), since some functionality in this package requires overriding built-in dbt macros: +```yml +# dbt_project.yml +dispatch: + - macro_namespace: dbt + search_order: ['dbt_labs_materialized_views', 'dbt'] +``` -- Install this project as a package ([package-management docs](https://docs.getdbt.com/docs/building-a-dbt-project/package-management)) - - [Local package](https://docs.getdbt.com/docs/building-a-dbt-project/package-management#local-packages): by referencing the [`materialized-views`](https://github.com/dbt-labs/dbt-labs-experimental-features/tree/master/materialized-views) folder. - - [Git package](https://docs.getdbt.com/docs/building-a-dbt-project/package-management#git-packages) using [project subdirectories](https://docs.getdbt.com/docs/building-a-dbt-project/package-management#git-packages): again by referencing the [`materialized-views`](https://github.com/dbt-labs/dbt-labs-experimental-features/tree/master/materialized-views) folder. -- Copy-paste the files from `macros/` (specifically `default` and your adapter) into your own project. +You're also welcome to copy, paste, and edit the files from `macros/` (specifically `default` and your adapter) in your own project's `macros/` directory. If you find spots for improvement, we welcome PRs back to this repository. ### Extra installation steps for Postgres and Redshift The Postgres and Redshift implementations both require overriding the builtin versions of some adapter macros. If you've installed `dbt_labs_materialized_views` as a local package, you can achieve this override by creating a file `macros/*.sql` in your project with the following contents: ```sql -{# postgres and redshift #} - -{% macro drop_relation(relation) -%} - {{ return(dbt_labs_materialized_views.drop_relation(relation)) }} -{% endmacro %} - -{% macro postgres__list_relations_without_caching(schema_relation) %} - {{ return(dbt_labs_materialized_views.postgres__list_relations_without_caching(schema_relation)) }} -{% endmacro %} +{# postgres + redshift #} {% macro postgres_get_relations() %} {{ return(dbt_labs_materialized_views.postgres_get_relations()) }} @@ -42,12 +45,12 @@ The Postgres and Redshift implementations both require overriding the builtin ve {# redshift only #} -{% macro redshift__list_relations_without_caching(schema_relation) %} - {{ return(dbt_labs_materialized_views.redshift__list_relations_without_caching(schema_relation)) }} -{% endmacro %} - {% macro load_relation(relation) %} - {{ return(dbt_labs_materialized_views.redshift_load_relation_or_mv(relation)) }} + {% if adapter.type() == 'redshift' %} + {{ return(dbt_labs_materialized_views.redshift_load_relation_or_mv(relation)) }} + {% else %} + {{ return(dbt.load_relation(relation)) }} + {% endif %} {% endmacro %} ``` diff --git a/materialized-views/integration_tests/.gitignore b/materialized-views/integration_tests/.gitignore index dad33a4..bd45744 100644 --- a/materialized-views/integration_tests/.gitignore +++ b/materialized-views/integration_tests/.gitignore @@ -1,4 +1,5 @@ target/ dbt_modules/ +dbt_packages/ logs/ diff --git a/materialized-views/integration_tests/dbt_project.yml b/materialized-views/integration_tests/dbt_project.yml index 2a4edd1..594ca02 100644 --- a/materialized-views/integration_tests/dbt_project.yml +++ b/materialized-views/integration_tests/dbt_project.yml @@ -22,3 +22,7 @@ quoting: seeds: quote_columns: false + +dispatch: + - macro_namespace: dbt + search_order: ['dbt_labs_materialized_views', 'dbt'] diff --git a/materialized-views/integration_tests/macros/overrides.sql b/materialized-views/integration_tests/macros/overrides.sql index 576777c..aa92788 100644 --- a/materialized-views/integration_tests/macros/overrides.sql +++ b/materialized-views/integration_tests/macros/overrides.sql @@ -1,23 +1,11 @@ {# postgres + redshift #} -{% macro drop_relation(relation) -%} - {{ return(dbt_labs_materialized_views.drop_relation(relation)) }} -{% endmacro %} - -{% macro postgres__list_relations_without_caching(schema_relation) %} - {{ return(dbt_labs_materialized_views.postgres__list_relations_without_caching(schema_relation)) }} -{% endmacro %} - {% macro postgres_get_relations() %} {{ return(dbt_labs_materialized_views.postgres_get_relations()) }} {% endmacro %} {# redshift only #} -{% macro redshift__list_relations_without_caching(schema_relation) %} - {{ return(dbt_labs_materialized_views.redshift__list_relations_without_caching(schema_relation)) }} -{% endmacro %} - {% macro load_relation(relation) %} {% if adapter.type() == 'redshift' %} {{ return(dbt_labs_materialized_views.redshift_load_relation_or_mv(relation)) }} diff --git a/materialized-views/integration_tests/packages.yml b/materialized-views/integration_tests/packages.yml index 38bc149..dbe16b6 100644 --- a/materialized-views/integration_tests/packages.yml +++ b/materialized-views/integration_tests/packages.yml @@ -1,4 +1,4 @@ packages: - - local: ../ - - package: fishtown-analytics/dbt_utils - version: 0.6.4 + - local: ../ + - package: dbt-labs/dbt_utils + version: 0.8.2 diff --git a/materialized-views/macros/default/adapters.sql b/materialized-views/macros/default/adapters.sql index ba604e4..7b4c465 100644 --- a/materialized-views/macros/default/adapters.sql +++ b/materialized-views/macros/default/adapters.sql @@ -1,5 +1,5 @@ {% macro create_materialized_view_as(relation, sql, config) %} - {{ return(adapter.dispatch('create_materialized_view_as', macro_namespace = 'dbt_labs_materialized_views')(relation, sql, config)) }} + {{ return(adapter.dispatch('create_materialized_view_as', 'dbt')(relation, sql, config)) }} {% endmacro %} {% macro default__create_materialized_view_as(relation, sql, config) -%} @@ -11,7 +11,7 @@ {% endmacro %} {% macro refresh_materialized_view(relation, config) %} - {{ return(adapter.dispatch('refresh_materialized_view', macro_namespace = 'dbt_labs_materialized_views')(relation, config)) }} + {{ return(adapter.dispatch('refresh_materialized_view', 'dbt')(relation, config)) }} {% endmacro %} {% macro default__refresh_materialized_view(relation, config) -%} @@ -21,7 +21,7 @@ {% endmacro %} {# override builtin behavior of adapter.drop_relation #} -{% macro drop_relation(relation) -%} +{% macro default__drop_relation(relation) -%} {% set relation_type = 'materialized view' if relation.type == 'materializedview' else relation.type %} {% call statement('drop_relation', auto_begin=False) -%} drop {{ relation_type }} if exists {{ relation }} cascade diff --git a/materialized-views/macros/default/materialized_view.sql b/materialized-views/macros/default/materialized_view.sql index 4f74891..4d4b88c 100644 --- a/materialized-views/macros/default/materialized_view.sql +++ b/materialized-views/macros/default/materialized_view.sql @@ -18,8 +18,12 @@ {% elif full_refresh_mode or existing_relation.type != 'materializedview' %} {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #} - {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %} - {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %} + {%- set backup_identifier = model['name'] + '__dbt_backup' -%} + {%- set backup_relation_type = 'materializedview' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = api.Relation.create(identifier=backup_identifier, + schema=schema, + database=database, + type=backup_relation_type) -%} {% do adapter.drop_relation(backup_relation) %} {% do adapter.rename_relation(target_relation, backup_relation) %} diff --git a/materialized-views/macros/redshift/adapters.sql b/materialized-views/macros/redshift/adapters.sql index c50e6b0..96a685a 100644 --- a/materialized-views/macros/redshift/adapters.sql +++ b/materialized-views/macros/redshift/adapters.sql @@ -24,7 +24,7 @@ {% macro redshift__refresh_materialized_view(relation, config) -%} - {%- set is_auto_refresh = config.get('auto_refresh', true) %} + {%- set is_auto_refresh = config.get('auto_refresh', false) %} {%- if is_auto_refresh == false -%} {# manual refresh #} @@ -84,9 +84,9 @@ {% if rel.type == 'materializedview' and execute %} - {# materialized views are not properly registered in pg_depend, - so the cache can miss that they've been dropped - https://github.com/awslabs/amazon-redshift-utils/issues/499 #} + {#-- materialized views are not properly registered in pg_depend, --#} + {#-- so the cache can miss that they've been dropped --#} + {#-- see: https://github.com/awslabs/amazon-redshift-utils/issues/499 --#} {% set hard_check_mv_sql %} diff --git a/materialized-views/macros/snowflake/materialized_view.sql b/materialized-views/macros/snowflake/materialized_view.sql index fcf6f82..ac6e77c 100644 --- a/materialized-views/macros/snowflake/materialized_view.sql +++ b/materialized-views/macros/snowflake/materialized_view.sql @@ -12,7 +12,10 @@ {% if (existing_relation is none or full_refresh_mode) %} {% set build_sql = dbt_labs_materialized_views.create_materialized_view_as(target_relation, sql, config) %} - {% elif existing_relation.is_view or existing_relation.is_table %} + {#-- elif existing_reation.is_view --#} + {#-- Snowflake `show terse objects` does not distinguish between views and materialized views, so our adapter cache can't either. --#} + {#-- Document as a limitation that switching between views and materialized views requires manual `drop` on Snowflake --#} + {% elif existing_relation.is_table %} {#-- Can't overwrite a view with a table - we must drop --#} {{ log("Dropping relation " ~ target_relation ~ " because it is a " ~ existing_relation.type ~ " and this model is a materialized view.") }} {% do adapter.drop_relation(existing_relation) %}