Revisit materialized views

materialized-views/README.md

@@ -14,40 +14,43 @@ If you're here, you may also like the [dbt-materialize](https://github.com/Mater
 
 ### General installation:
 
-You can install the materialized-view funcionality using one of the following methods.
+You can install the materialized-view project as a package, to get access to it in your own project, either as a `local` package (by cloning this repository to your machine) or as a `git` package referencing the `materialized-views` subdirectory:
+```yml
+# packages.yml
+packages:
+  - git: https://github.com/dbt-labs/dbt-labs-experimental-features.git
+    subdirectory: materialized-views
+```
+
+If you do this, you'll need to set the `dispatch` project config ([docs](https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch)), since some functionality in this package requires overriding built-in dbt macros:
+```yml
+# dbt_project.yml
+dispatch:
+  - macro_namespace: dbt
+    search_order: ['dbt_labs_materialized_views', 'dbt']
+```
 
-- Install this project as a package ([package-management docs](https://docs.getdbt.com/docs/building-a-dbt-project/package-management))
-  - [Local package](https://docs.getdbt.com/docs/building-a-dbt-project/package-management#local-packages): by referencing the [`materialized-views`](https://github.com/dbt-labs/dbt-labs-experimental-features/tree/master/materialized-views) folder.
-  - [Git package](https://docs.getdbt.com/docs/building-a-dbt-project/package-management#git-packages) using [project subdirectories](https://docs.getdbt.com/docs/building-a-dbt-project/package-management#git-packages): again by referencing the [`materialized-views`](https://github.com/dbt-labs/dbt-labs-experimental-features/tree/master/materialized-views) folder.
-- Copy-paste the files from `macros/` (specifically `default` and your adapter) into your own project.
+You're also welcome to copy, paste, and edit the files from `macros/` (specifically `default` and your adapter) in your own project's `macros/` directory. If you find spots for improvement, we welcome PRs back to this repository.
 
 ### Extra installation steps for Postgres and Redshift
 
 The Postgres and Redshift implementations both require overriding the builtin versions of some adapter macros. If you've installed `dbt_labs_materialized_views` as a local package, you can achieve this override by creating a file `macros/*.sql` in your project with the following contents:
 
 ```sql
-{# postgres and redshift #}
-
-{% macro drop_relation(relation) -%}
-  {{ return(dbt_labs_materialized_views.drop_relation(relation)) }}
-{% endmacro %}
-
-{% macro postgres__list_relations_without_caching(schema_relation) %}
-  {{ return(dbt_labs_materialized_views.postgres__list_relations_without_caching(schema_relation)) }}
-{% endmacro %}
+{# postgres + redshift #}
 
 {% macro postgres_get_relations() %}
   {{ return(dbt_labs_materialized_views.postgres_get_relations()) }}
 {% endmacro %}
 
 {# redshift only #}
 
-{% macro redshift__list_relations_without_caching(schema_relation) %}
-  {{ return(dbt_labs_materialized_views.redshift__list_relations_without_caching(schema_relation)) }}
-{% endmacro %}
-
 {% macro load_relation(relation) %}
-  {{ return(dbt_labs_materialized_views.redshift_load_relation_or_mv(relation)) }}
+  {% if adapter.type() == 'redshift' %}
+    {{ return(dbt_labs_materialized_views.redshift_load_relation_or_mv(relation)) }}
+  {% else %}
+    {{ return(dbt.load_relation(relation)) }}
+  {% endif %}
 {% endmacro %}
 ```
 

materialized-views/integration_tests/.gitignore

@@ -1,4 +1,5 @@
 
 target/
 dbt_modules/
+dbt_packages/
 logs/

materialized-views/integration_tests/dbt_project.yml

@@ -22,3 +22,7 @@ quoting:
 
 seeds:
   quote_columns: false
+
+dispatch:
+  - macro_namespace: dbt
+    search_order: ['dbt_labs_materialized_views', 'dbt']

materialized-views/integration_tests/macros/overrides.sql

@@ -1,23 +1,11 @@
 {# postgres + redshift #}
 
-{% macro drop_relation(relation) -%}
-  {{ return(dbt_labs_materialized_views.drop_relation(relation)) }}
-{% endmacro %}
-
-{% macro postgres__list_relations_without_caching(schema_relation) %}
-  {{ return(dbt_labs_materialized_views.postgres__list_relations_without_caching(schema_relation)) }}
-{% endmacro %}
-
 {% macro postgres_get_relations() %}
   {{ return(dbt_labs_materialized_views.postgres_get_relations()) }}
 {% endmacro %}
 
 {# redshift only #}
 
-{% macro redshift__list_relations_without_caching(schema_relation) %}
-  {{ return(dbt_labs_materialized_views.redshift__list_relations_without_caching(schema_relation)) }}
-{% endmacro %}
-
 {% macro load_relation(relation) %}
   {% if adapter.type() == 'redshift' %}
     {{ return(dbt_labs_materialized_views.redshift_load_relation_or_mv(relation)) }}

materialized-views/integration_tests/packages.yml

@@ -1,4 +1,4 @@
 packages:
-    - local: ../
-    - package: fishtown-analytics/dbt_utils
-      version: 0.6.4
+  - local: ../
+  - package: dbt-labs/dbt_utils
+    version: 0.8.2

materialized-views/macros/default/adapters.sql

@@ -1,5 +1,5 @@
 {% macro create_materialized_view_as(relation, sql, config) %}
-    {{ return(adapter.dispatch('create_materialized_view_as', macro_namespace = 'dbt_labs_materialized_views')(relation, sql, config)) }}
+    {{ return(adapter.dispatch('create_materialized_view_as', 'dbt')(relation, sql, config)) }}
 {% endmacro %}
 
 {% macro default__create_materialized_view_as(relation, sql, config) -%}
@@ -11,7 +11,7 @@
 {% endmacro %}
 
 {% macro refresh_materialized_view(relation, config) %}
-    {{ return(adapter.dispatch('refresh_materialized_view', macro_namespace = 'dbt_labs_materialized_views')(relation, config)) }}
+    {{ return(adapter.dispatch('refresh_materialized_view', 'dbt')(relation, config)) }}
 {% endmacro %}
 
 {% macro default__refresh_materialized_view(relation, config) -%}
@@ -21,7 +21,7 @@
 {% endmacro %}
 
 {# override builtin behavior of adapter.drop_relation #}
-{% macro drop_relation(relation) -%}
+{% macro default__drop_relation(relation) -%}
   {% set relation_type = 'materialized view' if relation.type == 'materializedview' else relation.type %}
   {% call statement('drop_relation', auto_begin=False) -%}
     drop {{ relation_type }} if exists {{ relation }} cascade

materialized-views/macros/default/materialized_view.sql

@@ -18,8 +18,12 @@
 
   {% elif full_refresh_mode or existing_relation.type != 'materializedview' %}
       {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
-      {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
-      {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
+      {%- set backup_identifier = model['name'] + '__dbt_backup' -%}
+      {%- set backup_relation_type = 'materializedview' if existing_relation is none else existing_relation.type -%}
+      {%- set backup_relation = api.Relation.create(identifier=backup_identifier,
+                                                schema=schema,
+                                                database=database,
+                                                type=backup_relation_type) -%}
       {% do adapter.drop_relation(backup_relation) %}
 
       {% do adapter.rename_relation(target_relation, backup_relation) %}

materialized-views/macros/redshift/adapters.sql

@@ -24,7 +24,7 @@
 
 {% macro redshift__refresh_materialized_view(relation, config) -%}
 
-    {%- set is_auto_refresh = config.get('auto_refresh', true) %}
+    {%- set is_auto_refresh = config.get('auto_refresh', false) %}
 
     {%- if is_auto_refresh == false -%} {# manual refresh #}
 
@@ -84,9 +84,9 @@
 
   {% if rel.type == 'materializedview' and execute %}
 
-    {# materialized views are not properly registered in pg_depend,
-       so the cache can miss that they've been dropped
-       https://github.com/awslabs/amazon-redshift-utils/issues/499 #}
+    {#-- materialized views are not properly registered in pg_depend,     --#}
+    {#-- so the cache can miss that they've been dropped                  --#}
+    {#-- see: https://github.com/awslabs/amazon-redshift-utils/issues/499 --#}
 
     {% set hard_check_mv_sql %}
 

materialized-views/macros/snowflake/materialized_view.sql

@@ -12,7 +12,10 @@
 
   {% if (existing_relation is none or full_refresh_mode) %}
       {% set build_sql = dbt_labs_materialized_views.create_materialized_view_as(target_relation, sql, config) %}
-  {% elif existing_relation.is_view or existing_relation.is_table %}
+  {#-- elif existing_reation.is_view --#}
+      {#-- Snowflake `show terse objects` does not distinguish between views and materialized views, so our adapter cache can't either. --#}
+      {#-- Document as a limitation that switching between views and materialized views requires manual `drop` on Snowflake --#}
+  {% elif existing_relation.is_table %}
       {#-- Can't overwrite a view with a table - we must drop --#}
       {{ log("Dropping relation " ~ target_relation ~ " because it is a " ~ existing_relation.type ~ " and this model is a materialized view.") }}
       {% do adapter.drop_relation(existing_relation) %}