dbt-materialize: support enforcing contracts for pseudo-types

Author: morsapaes

misc/dbt-materialize/CHANGELOG.md

@@ -1,5 +1,11 @@
 # dbt-materialize Changelog
 
+## Unreleased
+
+* Support enforcing model contracts for the [`map`](https://materialize.com/docs/sql/types/map/),
+  [`list`](https://materialize.com/docs/sql/types/list/),
+  and [`record`](https://materialize.com/docs/sql/types/record/) pseudo-types.
+
 ## 1.7.8 - 2024-05-06
 
 * Fix permission management in blue/green automation macros for non-admin users

misc/dbt-materialize/README.md

@@ -19,7 +19,7 @@ pip install dbt-materialize      # install the adapter
 ## Requirements
 
 <!-- If you update this, bump the constraint in connections.py too. -->
-`dbt-materialize` requires Materialize v0.68.0+.
+`dbt-materialize` requires Materialize v0.100.0+.
 
 ## Configuring your profile
 

misc/dbt-materialize/dbt/adapters/materialize/connections.py

@@ -26,7 +26,7 @@
 from dbt.semver import versions_compatible
 
 # If you bump this version, bump it in README.md too.
-SUPPORTED_MATERIALIZE_VERSIONS = ">=0.68.0"
+SUPPORTED_MATERIALIZE_VERSIONS = ">=0.100.0"
 
 logger = AdapterLogger("Materialize")
 

misc/dbt-materialize/dbt/include/materialize/macros/columns.sql

@@ -22,3 +22,28 @@
         {{ select_sql }}
     ) as __dbt_sbq
 {% endmacro %}
+
+{% macro materialize__get_empty_schema_sql(columns) %}
+    {%- set col_err = [] -%}
+    {%- set col_naked_numeric = [] -%}
+    select
+    {% for i in columns %}
+      {%- set col = columns[i] -%}
+      {%- if col['data_type'] is not defined -%}
+        {%- do col_err.append(col['name']) -%}
+      {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}
+      {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}
+        {%- do col_naked_numeric.append(col['name']) -%}
+      {%- endif -%}
+      {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}
+      -- NOTE(morsapaes): in a future release of dbt, the default macro will use
+      -- the global cast macro, as modified here, and we can remove this custom
+      -- override. See: https://github.com/dbt-labs/dbt-adapters/pull/165
+      {{ cast("null", col['data_type']) }} as {{ col_name }}{{ ", " if not loop.last }}
+    {%- endfor -%}
+    {%- if (col_err | length) > 0 -%}
+      {{ exceptions.column_type_missing(column_names=col_err) }}
+    {%- elif (col_naked_numeric | length) > 0 -%}
+      {{ exceptions.warn("Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: " ~ col_naked_numeric ~ "`") }}
+    {%- endif -%}
+{% endmacro %}

misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql

@@ -0,0 +1,27 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License in the LICENSE file at the
+-- root of this repository, or online at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+{% macro materialize__cast(expression, data_type) -%}
+    {#-- Handle types that don't support cast(NULL as type) --#}
+    {%- if expression.strip().lower() == "null" and data_type.strip().lower() == "map" -%}
+      MAP[NULL=>NULL]
+    {%- elif expression.strip().lower() == "null" and data_type.strip().lower() == "list" -%}
+      LIST[NULL]
+    {%- elif expression.strip().lower() == "null" and data_type.strip().lower() == "record" -%}
+      ROW(NULL)
+    {%- else -%}
+      cast({{ expression }} as {{ data_type }})
+    {%- endif -%}
+{%- endmacro %}

misc/dbt-materialize/tests/adapter/fixtures.py

@@ -299,3 +299,25 @@
       - name: c
         data_type: string
 """
+
+contract_pseudo_types_yml = """
+version: 2
+models:
+  - name: test_pseudo_types
+    config:
+      contract:
+        enforced: true
+    columns:
+      - name: a
+        data_type: map
+      - name: b
+        data_type: record
+      - name: c
+        data_type: list
+"""
+
+test_pseudo_types = """
+{{ config(materialized='view') }}
+
+    SELECT MAP['a' => 1, 'b' => 2] AS a, ROW(1, 2) AS b, LIST[[1,2],[3]] AS c
+"""

misc/dbt-materialize/tests/adapter/test_constraints.py

@@ -32,8 +32,10 @@
 from dbt.tests.util import run_dbt, run_sql_with_adapter
 from fixtures import (
     contract_invalid_cluster_schema_yml,
+    contract_pseudo_types_yml,
     nullability_assertions_schema_yml,
     test_materialized_view,
+    test_pseudo_types,
     test_view,
 )
 
@@ -162,3 +164,18 @@ def test_materialize_drop_quickstart(self, project):
         run_dbt(["run", "--models", "contract_invalid_cluster"], expect_pass=True)
 
         project.run_sql("CREATE CLUSTER quickstart SIZE = '1'")
+
+
+class TestContractMaterializeTypes:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "contract_pseudo_types.yml": contract_pseudo_types_yml,
+            "contract_pseudo_types.sql": test_pseudo_types,
+        }
+
+    # Pseudotypes in Materialize cannot be cast using the cast() function, so we
+    # special-handle their NULL casting for contract validation.
+    # See #17870: https://github.com/MaterializeInc/materialize/issues/17870
+    def test_test_pseudo_types(self, project):
+        run_dbt(["run", "--models", "contract_pseudo_types"], expect_pass=True)

misc/dbt-materialize/tests/adapter/test_utils.py

@@ -23,6 +23,7 @@
 from dbt.tests.adapter.utils.fixture_get_intervals_between import (
     models__test_get_intervals_between_yml,
 )
+from dbt.tests.adapter.utils.test_cast import BaseCast
 from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText
 from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampAware
 from dbt.tests.adapter.utils.test_date_spine import BaseDateSpine
@@ -96,6 +97,9 @@
   11 as expected
 """
 
+class TestCast(BaseCast):
+    pass
+
 # The `cast_bool_to_text` macro works as expected, but we must alter the test case
 # because set operation type conversions do not work properly.
 # See https://github.com/MaterializeInc/materialize/issues/3331