Release 1.3.3 (#130)

* Add some tests, including for ephemeral models

* Fix lint

* Update readme to show ephemeral validated

* Add ephemeral limitation to readme

* Use copy of custom settings to keep new connections clean

* Remove debug client

* Ensure that our unique session id isn't stomped on in custom settings

* Add seed column types to seeds test

* Update docs with limitations and thread info

CHANGELOG.md

@@ -1,3 +1,15 @@
+### Release [1.3.3], 2023-01-18
+#### Documentation Update
+- The documentation has been updated to reflect that dbt-clickhouse does support ephemeral models, and ephemeral model tests do pass.
+However, due to a [ClickHouse limitation](https://github.com/ClickHouse/ClickHouse/issues/30323), CTEs will not work directly
+with INSERT statements so table models will fail if they include ephemeral models in the SELECT.  View models and other SQL 
+statements using ephemeral models should work correctly.
+
+#### Bug Fix
+- Client connections would incorrectly reuse a previous session_id when initializing, causing session locked errors.  This has been fixed.  This
+closes https://github.com/ClickHouse/dbt-clickhouse/issues/127.  Multiple threads should now work correctly in dbt projects,
+and dbt-clickhouse automated tests now use `threads=4` (soon to be the dbt default).
+
 ### Release [1.3.2], 2022-12-23
 #### Improvements
 - Added *experimental* support for the `delete+insert` incremental strategy.  In most cases this strategy should be significantly faster
@@ -207,6 +219,7 @@ for Replicated tables that use the `{uuid}` macro in the path to avoid name conf
 ### Release [0.19.0], 2021-02-14
 #### Initial Release
 
+[1.3.3]: https://github.com/ClickHouse/dbt-clickhouse/compare/v1.3.2..v1.3.3
 [1.3.2]: https://github.com/ClickHouse/dbt-clickhouse/compare/v1.3.1..v1.3.2
 [1.3.1]: https://github.com/ClickHouse/dbt-clickhouse/compare/v1.3.0..v1.3.1
 [1.3.0]: https://github.com/ClickHouse/dbt-clickhouse/compare/v1.2.1..v1.3.0

README.md

@@ -28,7 +28,7 @@ pip install dbt-clickhouse
 - [x] Tests
 - [x] Snapshots
 - [x] Most dbt-utils macros (now included in dbt-core)  
-- [ ] Ephemeral materialization
+- [x] Ephemeral materialization (but incompatible with table materializations)
 
 # Usage Notes
 
@@ -85,6 +85,12 @@ your_profile_name:
 | inserts_only         | If set to True for an incremental model, incremental updates will be inserted directly to the target table without creating intermediate table. It has been deprecated in favor of the `append` incremental `strategy`, which operates in the same way | Optional                          |
 | incremental_strategy | Incremental model update strategy of `delete+insert` or `append`.  See the following Incremental Model Strategies                                                                                                                                      | Optional (default: `default`)     |
 
+## Known Limitations
+
+* Replicated tables (combined with the `cluster` profile setting) are available using the `on_cluster_clause` macro but are not included in the test suite and not formally tested. 
+* Ephemeral models/CTEs don't work with INSERT statements in ClickHouse, so table materializations that incorporate an ephemeral model will fail.
+See https://github.com/ClickHouse/ClickHouse/issues/30323.  View models and other SQL statements using ephemeral models should work correctly.
+
 ## Incremental Model Strategies
 
 As of version 1.3.2, dbt-clickhouse supports three incremental model strategies.
@@ -146,10 +152,6 @@ See the [S3 test file](https://github.com/ClickHouse/dbt-clickhouse/blob/main/te
 # Running Tests
 
 This adapter passes all of dbt basic tests as presented in dbt's official docs: https://docs.getdbt.com/docs/contributing/testing-a-new-adapter#testing-your-adapter.
-
-Notes:  Ephemeral materializations are not supported and not tested.  Replicated tables (combined with the `cluster` profile setting) are available
-using the `on_cluster_clause` macro but are not included in the test suite and not formally tested. 
-
 Use `pytest tests` to run tests.
 
 You can customize the test environment via environment variables. We recommend doing so with the pytest `pytest-dotenv` plugin combined with root level `test.env`

dbt/adapters/clickhouse/__version__.py

@@ -1 +1 @@
-version = '1.3.2'
+version = '1.3.3'

dbt/adapters/clickhouse/dbclient.py

@@ -57,7 +57,9 @@ class ChRetryableException(Exception):
 class ChClientWrapper(ABC):
     def __init__(self, credentials: ClickHouseCredentials):
         self.database = credentials.schema
-        self._conn_settings = credentials.custom_settings or {}
+        custom_settings = credentials.custom_settings or {}
+        self._conn_settings = custom_settings.copy()
+        self._conn_settings['session_id'] = f'dbt::{uuid.uuid4()}'
         if credentials.cluster_mode or credentials.database_engine == 'Replicated':
             self._conn_settings['database_replicated_enforce_synchronous_settings'] = '1'
             self._conn_settings['insert_quorum'] = 'auto'

dbt/adapters/clickhouse/httpclient.py

@@ -1,5 +1,3 @@
-import uuid
-
 import clickhouse_connect
 from clickhouse_connect.driver.exceptions import DatabaseError, OperationalError
 from dbt.exceptions import DatabaseException as DBTDatabaseException
@@ -45,10 +43,9 @@ def _create_client(self, credentials):
                 compress=False if credentials.compression == '' else bool(credentials.compression),
                 connect_timeout=credentials.connect_timeout,
                 send_receive_timeout=credentials.send_receive_timeout,
-                client_name=f'cc-dbt-{dbt_version}',
+                client_name=f'dbt/{dbt_version}',
                 verify=credentials.verify,
                 query_limit=0,
-                session_id='dbt::' + str(uuid.uuid4()),
                 settings=self._conn_settings,
             )
         except OperationalError as ex:

dev_requirements.txt

@@ -1,9 +1,9 @@
-dbt-core~=1.3
+dbt-core~=1.3.2
 clickhouse-connect>=0.4.7
 clickhouse-driver>=0.2.3
 pytest>=7.2.0
 pytest-dotenv==0.5.2
-dbt-tests-adapter~=1.3
+dbt-tests-adapter~=1.3.2
 black==22.3.0
 isort==5.10.1
 mypy==0.960

tests/integration/adapter/test_basic.py

@@ -3,6 +3,7 @@
 from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod
 from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations
 from dbt.tests.adapter.basic.test_empty import BaseEmpty
+from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral
 from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests
 from dbt.tests.adapter.basic.test_incremental import BaseIncremental, BaseIncrementalNotSchemaChange
 from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests
@@ -21,16 +22,27 @@
 
 # CSV content with empty fields.
 seeds_empty_csv = """
-key,val1,val2
-abc,1,1
-abc,1,0
-def,1,0
-hij,1,1
-hij,1,
-klm,1,0
-klm,1,
+key,val1,val2,str1
+abc,1,1,some_str
+abc,1,0,"another string"
+def,1,0,
+hij,1,1,Caps
+hij,1,,"second string"
+klm,1,0,"test"
+klm,1,,"test4"
 """.lstrip()
 
+seeds_schema_yml = """
+version: 2
+
+seeds:
+  - name: empty
+    config:
+      column_types:
+        val2: Nullable(UInt32)
+        str1: Nullable(String)
+"""
+
 
 class TestBaseSimpleMaterializations(BaseSimpleMaterializations):
     pass
@@ -44,6 +56,10 @@ class TestIncremental(BaseIncremental):
     pass
 
 
+class TestEphemeral(BaseEphemeral):
+    pass
+
+
 class TestSnapshotTimestamp(BaseSnapshotTimestamp):
     pass
 
@@ -112,12 +128,19 @@ def models(self):
 class TestCSVSeed:
     @pytest.fixture(scope="class")
     def seeds(self):
-        return {"boolean.csv": seeds_boolean_csv, "empty.csv": seeds_empty_csv}
+        return {
+            "schema.yml": seeds_schema_yml,
+            "boolean.csv": seeds_boolean_csv,
+            "empty.csv": seeds_empty_csv,
+        }
 
     def test_seed(self, project):
         # seed command
         results = run_dbt(["seed"])
         assert len(results) == 2
+        columns = project.run_sql("DESCRIBE TABLE empty", fetch='all')
+        assert columns[2][1] == 'Nullable(UInt32)'
+        assert columns[3][1] == 'Nullable(String)'
 
 
 incremental_not_schema_change_sql = """

tests/integration/adapter/test_singular.py

@@ -0,0 +1,10 @@
+from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests
+from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral
+
+
+class TestSingularTests(BaseSingularTests):
+    pass
+
+
+class TestSingularTestsEphemeral(BaseSingularTestsEphemeral):
+    pass

tests/integration/conftest.py

@@ -99,7 +99,7 @@ def test_config(ch_test_users):
 def dbt_profile_target(test_config):
     return {
         'type': 'clickhouse',
-        'threads': 1,
+        'threads': 4,
         'driver': test_config['driver'],
         'host': test_config['host'],
         'user': test_config['user'],