Merge pull request #412 from ClickHouse/support-replace-view

Support replace view

Author: BentsiLeviav

CHANGELOG.md

@@ -1,7 +1,7 @@
-### Release [x.x.x]
+### Release [1.8.8], 2025-02-05
 ### Improvements
-* Ignores incompatible settings based on the configured Engine.
 * Materialized view now attempts to use `ALTER TABLE...MODIFY QUERY` to update existing materialized views. This is an atomic operation so data is not lost. ([#390](https://github.com/ClickHouse/dbt-clickhouse/pull/390))
+* Make view materialization updates atomic. ([#412](https://github.com/ClickHouse/dbt-clickhouse/pull/412))
 
 
 #### New Features

dbt/adapters/clickhouse/__version__.py

@@ -1 +1 @@
-version = '1.8.7'
+version = '1.8.8'

dbt/include/clickhouse/macros/materializations/view.sql

@@ -2,52 +2,24 @@
 
   {%- set existing_relation = load_cached_relation(this) -%}
   {%- set target_relation = this.incorporate(type='view') -%}
-  {%- set backup_relation = none -%}
-  {%- set preexisting_backup_relation = none -%}
-  {%- set preexisting_intermediate_relation = none -%}
-
-  {% if existing_relation is not none %}
-    {%- set backup_relation_type = existing_relation.type -%}
-    {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
-    {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
-    {% if not existing_relation.can_exchange %}
-      {%- set intermediate_relation =  make_intermediate_relation(target_relation) -%}
-      {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}
-    {% endif %}
-  {% endif %}
 
   {% set grant_config = config.get('grants') %}
 
   {{ run_hooks(pre_hooks, inside_transaction=False) }}
 
-  -- drop the temp relations if they exist already in the database
-  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
-  {{ drop_relation_if_exists(preexisting_backup_relation) }}
-
   -- `BEGIN` happens here:
   {{ run_hooks(pre_hooks, inside_transaction=True) }}
 
-  {% if backup_relation is none %}
+  {% if existing_relation is none %}
     {{ log('Creating new relation ' + target_relation.name )}}
-    -- There is not existing relation, so we can just create
-    {% call statement('main') -%}
-      {{ get_create_view_as_sql(target_relation, sql) }}
-    {%- endcall %}
-  {% elif existing_relation.can_exchange %}
-    -- We can do an atomic exchange, so no need for an intermediate
-    {% call statement('main') -%}
-      {{ get_create_view_as_sql(backup_relation, sql) }}
-    {%- endcall %}
-    {% do exchange_tables_atomic(backup_relation, existing_relation) %}
   {% else %}
-    -- We have to use an intermediate and rename accordingly
-    {% call statement('main') -%}
-      {{ get_create_view_as_sql(intermediate_relation, sql) }}
-    {%- endcall %}
-    {{ adapter.rename_relation(existing_relation, backup_relation) }}
-    {{ adapter.rename_relation(intermediate_relation, target_relation) }}
+    {{ log('Relation ' + target_relation.name + ' already exists, replacing it' )}}
   {% endif %}
 
+  {% call statement('main') -%}
+    {{ get_create_view_as_sql(target_relation, sql) }}
+  {%- endcall %}
+
   -- cleanup
   {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
   {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
@@ -58,8 +30,6 @@
 
   {{ adapter.commit() }}
 
-  {{ drop_relation_if_exists(backup_relation) }}
-
   {{ run_hooks(post_hooks, inside_transaction=False) }}
 
   {{ return({'relations': [target_relation]}) }}
@@ -71,7 +41,7 @@
   {%- set sql_header = config.get('sql_header', none) -%}
   {{ sql_header if sql_header is not none }}
 
-  create view {{ relation.include(database=False) }} {{ on_cluster_clause(relation)}}
+  create or replace view {{ relation.include(database=False) }} {{ on_cluster_clause(relation)}}
     {% set contract_config = config.get('contract') %}
     {% if contract_config.enforced %}
       {{ get_assert_columns_equivalent(sql) }}

tests/integration/adapter/view/test_view.py

@@ -0,0 +1,70 @@
+"""
+Test ClickHouse view materialization in dbt-clickhouse
+"""
+
+import json
+
+import pytest
+from dbt.tests.util import run_dbt
+
+PEOPLE_SEED_CSV = """
+id,name,age,department
+1231,Dade,33,engineering
+6666,Ksenia,48,engineering
+8888,Kate,50,engineering
+""".lstrip()
+
+PEOPLE_VIEW_MODEL = """
+{{ config(
+       materialized='view'
+) }}
+
+{% if var('run_type', '') == '' %}
+    select id, name, age from {{ source('raw', 'people') }}
+{% elif var('run_type', '') == 'update_view' %}
+    select id, name, age, department from {{ source('raw', 'people') }}
+{% endif %}
+"""
+
+
+SEED_SCHEMA_YML = """
+version: 2
+
+sources:
+  - name: raw
+    schema: "{{ target.schema }}"
+    tables:
+      - name: people
+"""
+
+
+class TestClickHouseView:
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {
+            "people.csv": PEOPLE_SEED_CSV,
+            "schema.yml": SEED_SCHEMA_YML,
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {"people_view.sql": PEOPLE_VIEW_MODEL}
+
+    def test_create_view(self, project):
+        # Load seed data
+        run_dbt(["seed"])
+
+        # Run dbt to create the view
+        run_dbt()
+
+        # Query the view and check if it returns expected data
+        result = project.run_sql("SELECT COUNT(*) FROM people_view", fetch="one")
+        assert result[0] == 3  # 3 records in the seed data
+
+        # Run dbt again to apply the update
+        run_dbt(["run", "--vars", json.dumps({"run_type": "update_view"})])
+
+        # Verify the new column is present
+        result = project.run_sql("DESCRIBE TABLE people_view", fetch="all")
+        columns = {row[0] for row in result}
+        assert "department" in columns  # New column should be present

tests/integration/docker-compose.yml

@@ -22,6 +22,7 @@ services:
       - SERVER_INDEX=1
       - SHARD_NUM=${SHARD_NUM:-1}
       - REPLICA_NUM=${REPLICA_NUM:-1}
+      - CLICKHOUSE_SKIP_USER_SETUP=1
     ports:
       - "8123:8123"
       - "8443:8443"
@@ -37,13 +38,15 @@ services:
       - SERVER_INDEX=2
       - SHARD_NUM=${SHARD_NUM:-2}
       - REPLICA_NUM=${REPLICA_NUM:-2}
+      - CLICKHOUSE_SKIP_USER_SETUP=1
     <<: *ch-common
   ch2:
     image: clickhouse/clickhouse-server:${DBT_CH_TEST_CH_VERSION:-latest}
     environment:
       - SERVER_INDEX=3
       - SHARD_NUM=${SHARD_NUM:-3}
       - REPLICA_NUM=${REPLICA_NUM:-3}
+      - CLICKHOUSE_SKIP_USER_SETUP=1
     <<: *ch-common
 
 networks: