|
6 | 6 | {%- materialization materialized_view, adapter='clickhouse' -%} |
7 | 7 |
|
8 | 8 | {%- set target_relation = this.incorporate(type='table') -%} |
9 | | - {%- set mv_relation = target_relation.derivative('_mv', 'materialized_view') -%} |
10 | 9 | {%- set cluster_clause = on_cluster_clause(target_relation) -%} |
11 | 10 |
|
12 | 11 | {# look for an existing relation for the target table and create backup relations if necessary #} |
|
35 | 34 | -- `BEGIN` happens here: |
36 | 35 | {{ run_hooks(pre_hooks, inside_transaction=True) }} |
37 | 36 |
|
| 37 | + -- extract the names of the materialized views from the sql |
| 38 | + {% set view_names = modules.re.findall('--([^:]+):begin', sql) %} |
| 39 | + |
| 40 | + -- extract the sql for each of the materialized view into a map |
| 41 | + {% set views = {} %} |
| 42 | + {% if view_names %} |
| 43 | + {% for view_name in view_names %} |
| 44 | + {% set view_sql = modules.re.findall('--' + view_name + ':begin(.*)--' + view_name + ':end', sql, flags=modules.re.DOTALL)[0] %} |
| 45 | + {%- set _ = views.update({view_name: view_sql}) -%} |
| 46 | + {% endfor %} |
| 47 | + {% else %} |
| 48 | + {%- set _ = views.update({"mv": sql}) -%} |
| 49 | + {% endif %} |
| 50 | + |
38 | 51 | {% if backup_relation is none %} |
39 | 52 | {{ log('Creating new materialized view ' + target_relation.name )}} |
40 | | - {% call statement('main') -%} |
41 | | - {{ clickhouse__get_create_materialized_view_as_sql(target_relation, sql) }} |
42 | | - {%- endcall %} |
| 53 | + {{ clickhouse__get_create_materialized_view_as_sql(target_relation, sql, views) }} |
43 | 54 | {% elif existing_relation.can_exchange %} |
44 | 55 | {{ log('Replacing existing materialized view ' + target_relation.name) }} |
45 | | - {% call statement('drop existing materialized view') %} |
46 | | - drop view if exists {{ mv_relation }} {{ cluster_clause }} |
47 | | - {% endcall %} |
| 56 | + -- in this section, we look for mvs that has the same pattern as this model, but for some reason, |
| 57 | + -- are not listed in the model. This might happen when using multiple mv, and renaming one of the mv in the model. |
| 58 | + -- In case such mv found, we raise a warning to the user, that they might need to drop the mv manually. |
| 59 | + {{ log('Searching for existing materialized views with the pattern of ' + target_relation.name) }} |
| 60 | + {{ log('Views dictionary contents: ' + views | string) }} |
| 61 | + |
| 62 | + {% set tables_query %} |
| 63 | + select table_name |
| 64 | + from information_schema.tables |
| 65 | + where table_schema = '{{ existing_relation.schema }}' |
| 66 | + and table_name like '%{{ target_relation.name }}%' |
| 67 | + and table_type = 'VIEW' |
| 68 | + {% endset %} |
| 69 | + |
| 70 | + {% set tables_result = run_query(tables_query) %} |
| 71 | + {% if tables_result is not none %} |
| 72 | + {% set tables = tables_result.columns[0].values() %} |
| 73 | + {{ log('Current mvs found in ClickHouse are: ' + tables | join(', ')) }} |
| 74 | + {% set mv_names = [] %} |
| 75 | + {% for key in views.keys() %} |
| 76 | + {% do mv_names.append(target_relation.name ~ "_" ~ key) %} |
| 77 | + {% endfor %} |
| 78 | + {{ log('Model mvs to replace ' + mv_names | string) }} |
| 79 | + {% for table in tables %} |
| 80 | + {% if table not in mv_names %} |
| 81 | + {{ log('Warning - Table "' + table + '" was detected with the same pattern as model name "' + target_relation.name + '" but was not found in this run. In case it is a renamed mv that was previously part of this model, drop it manually (!!!)') }} |
| 82 | + {% endif %} |
| 83 | + {% endfor %} |
| 84 | + {% else %} |
| 85 | + {{ log('No existing mvs found matching the pattern. continuing..', info=True) }} |
| 86 | + {% endif %} |
| 87 | + {{ clickhouse__drop_mvs(target_relation, cluster_clause, views) }} |
48 | 88 | {% if should_full_refresh() %} |
49 | 89 | {% call statement('main') -%} |
50 | 90 | {{ get_create_table_as_sql(False, backup_relation, sql) }} |
|
56 | 96 | select 1 |
57 | 97 | {%- endcall %} |
58 | 98 | {% endif %} |
59 | | - {% call statement('create new materialized view') %} |
60 | | - {{ clickhouse__create_mv_sql(mv_relation, existing_relation, cluster_clause, sql) }} |
61 | | - {% endcall %} |
| 99 | + {{ clickhouse__create_mvs(existing_relation, cluster_clause, views) }} |
62 | 100 | {% else %} |
63 | 101 | {{ log('Replacing existing materialized view ' + target_relation.name) }} |
64 | | - {{ clickhouse__replace_mv(target_relation, existing_relation, intermediate_relation, backup_relation, sql) }} |
| 102 | + {{ clickhouse__replace_mv(target_relation, existing_relation, intermediate_relation, backup_relation, sql, views) }} |
65 | 103 | {% endif %} |
66 | 104 |
|
67 | 105 | -- cleanup |
|
78 | 116 |
|
79 | 117 | {{ run_hooks(post_hooks, inside_transaction=False) }} |
80 | 118 |
|
81 | | - {{ return({'relations': [target_relation, mv_relation]}) }} |
| 119 | + {% set relations = [target_relation] %} |
| 120 | + {% for view in views %} |
| 121 | + {{ relations.append(target_relation.derivative('_' + view, 'materialized_view')) }} |
| 122 | + {% endfor %} |
| 123 | + |
| 124 | + {{ return({'relations': relations}) }} |
82 | 125 |
|
83 | 126 | {%- endmaterialization -%} |
84 | 127 |
|
|
89 | 132 | 2. Create a materialized view using the SQL in the model that inserts |
90 | 133 | data into the table creating during step 1 |
91 | 134 | #} |
92 | | -{% macro clickhouse__get_create_materialized_view_as_sql(relation, sql) -%} |
93 | | - {% call statement('create_target_table') %} |
| 135 | +{% macro clickhouse__get_create_materialized_view_as_sql(relation, sql, views) -%} |
| 136 | + {% call statement('main') %} |
94 | 137 | {{ get_create_table_as_sql(False, relation, sql) }} |
95 | 138 | {% endcall %} |
96 | 139 | {%- set cluster_clause = on_cluster_clause(relation) -%} |
97 | 140 | {%- set mv_relation = relation.derivative('_mv', 'materialized_view') -%} |
98 | | - {{ clickhouse__create_mv_sql(mv_relation, relation, cluster_clause, sql) }} |
| 141 | + {{ clickhouse__create_mvs(relation, cluster_clause, views) }} |
99 | 142 | {%- endmacro %} |
100 | 143 |
|
| 144 | +{% macro clickhouse__drop_mv(mv_relation, cluster_clause) -%} |
| 145 | + drop view if exists {{ mv_relation }} {{ cluster_clause }} |
| 146 | +{%- endmacro %}u |
101 | 147 |
|
102 | | -{% macro clickhouse__create_mv_sql(mv_relation, target_table, cluster_clause, sql) -%} |
| 148 | +{% macro clickhouse__create_mv(mv_relation, target_table, cluster_clause, sql) -%} |
103 | 149 | create materialized view if not exists {{ mv_relation }} {{ cluster_clause }} |
104 | 150 | to {{ target_table }} |
105 | 151 | as {{ sql }} |
106 | 152 | {%- endmacro %} |
107 | 153 |
|
| 154 | +{% macro clickhouse__drop_mvs(target_relation, cluster_clause, views) -%} |
| 155 | + {% for view in views.keys() %} |
| 156 | + {%- set mv_relation = target_relation.derivative('_' + view, 'materialized_view') -%} |
| 157 | + {% call statement('drop existing mv: ' + view) -%} |
| 158 | + {{ clickhouse__drop_mv(mv_relation, cluster_clause) }}; |
| 159 | + {% endcall %} |
| 160 | + {% endfor %} |
| 161 | +{%- endmacro %} |
| 162 | + |
| 163 | +{% macro clickhouse__create_mvs(target_relation, cluster_clause, views) -%} |
| 164 | + {% for view, view_sql in views.items() %} |
| 165 | + {%- set mv_relation = target_relation.derivative('_' + view, 'materialized_view') -%} |
| 166 | + {% call statement('create existing mv: ' + view) -%} |
| 167 | + {{ clickhouse__create_mv(mv_relation, target_relation, cluster_clause, view_sql) }}; |
| 168 | + {% endcall %} |
| 169 | + {% endfor %} |
| 170 | +{%- endmacro %} |
108 | 171 |
|
109 | | -{% macro clickhouse__replace_mv(target_relation, existing_relation, intermediate_relation, backup_relation, sql) %} |
| 172 | +{% macro clickhouse__replace_mv(target_relation, existing_relation, intermediate_relation, backup_relation, sql, views) %} |
110 | 173 | {# drop existing materialized view while we recreate the target table #} |
111 | 174 | {%- set cluster_clause = on_cluster_clause(target_relation) -%} |
112 | | - {%- set mv_relation = target_relation.derivative('_mv', 'materialized_view') -%} |
113 | | - {% call statement('drop existing mv') -%} |
114 | | - drop view if exists {{ mv_relation }} {{ cluster_clause }} |
115 | | - {%- endcall %} |
| 175 | + {{ clickhouse__drop_mvs(target_relation, cluster_clause, views) }} |
116 | 176 |
|
117 | 177 | {# recreate the target table #} |
118 | 178 | {% call statement('main') -%} |
|
122 | 182 | {{ adapter.rename_relation(intermediate_relation, target_relation) }} |
123 | 183 |
|
124 | 184 | {# now that the target table is recreated, we can finally create our new view #} |
125 | | - {{ clickhouse__create_mv_sql(mv_relation, target_relation, cluster_clause, sql) }} |
| 185 | + {{ clickhouse__create_mvs(target_relation, cluster_clause, views) }} |
126 | 186 | {% endmacro %} |
| 187 | + |
0 commit comments