Merge branch 'main' into 1.6.latest

.github/workflows/integration.yml

@@ -3,6 +3,7 @@ on:
   push:
     paths-ignore:
       - "**.MD"
+      - "**.md"
 jobs:
   run-tox-tests-uc-cluster:
     runs-on: ubuntu-latest

.github/workflows/main.yml

@@ -22,9 +22,11 @@ on:
       - "releases/*"
     paths-ignore:
       - "**.MD"
+      - "**.md"
   pull_request:
     paths-ignore:
       - "**.MD"
+      - "**.md"
   workflow_dispatch:
 
 permissions: read-all

CHANGELOG.md

@@ -1,5 +1,12 @@
 ## dbt-databricks 1.6.x (Release TBD)
 
+## dbt-databricks 1.6.4 (September 14, 2023)
+
+### Fixes
+
+- Fixed an issue with AWS OAuth M2M flow ([#445](https://github.com/databricks/dbt-databricks/pull/445))
+- Fixed an issue where every table in hive_metastore would get described ([#446](https://github.com/databricks/dbt-databricks/pull/446))
+
 ## dbt-databricks 1.6.3 (September 8, 2023)
 
 ### Fixes

dbt/adapters/databricks/__version__.py

@@ -1 +1 @@
-version: str = "1.6.3"
+version: str = "1.6.4"

dbt/adapters/databricks/auth.py

@@ -40,12 +40,12 @@ def noop_credentials(_: Any):  # type: ignore
 
         config = Config(host=host, credentials_provider=noop_credentials)
         oidc = config.oidc_endpoints
-        scopes = ["offline_access", "all-apis"]
+        scopes = ["all-apis"]
         if not oidc:
             raise ValueError(f"{host} does not support OAuth")
         if config.is_azure:
             # Azure AD only supports full access to Azure Databricks.
-            scopes = [f"{config.effective_azure_login_app_id}/.default", "offline_access"]
+            scopes = [f"{config.effective_azure_login_app_id}/.default"]
         self._token_source = ClientCredentials(
             client_id=client_id,
             client_secret=client_secret,

dbt/adapters/databricks/impl.py

@@ -264,6 +264,8 @@ def typeFromNames(
                         if view_names[name]
                         else DatabricksRelationType.View
                     )
+                elif database is None or database == "hive_metastore":
+                    return DatabricksRelationType.Table
                 else:
                     # not a view so it might be a streaming table
                     # get extended information to determine

tests/integration/avoid_describe_extended/models/new_model.sql

@@ -0,0 +1,7 @@
+{{ config(
+    materialized = 'table'
+) }}
+
+select cast(1 as bigint) as id, 'hello' as msg
+union all
+select cast(2 as bigint) as id, 'goodbye' as msg

tests/integration/avoid_describe_extended/seeds/preexisting_data.csv

@@ -0,0 +1,5 @@
+id,msg
+1,hello
+2,goodbye
+2,yo
+3,anyway

tests/integration/avoid_describe_extended/test_avoid_describe_extended.py

@@ -0,0 +1,33 @@
+from tests.integration.base import DBTIntegrationTest, use_profile
+
+
+class TestAvoidDescribeExtended(DBTIntegrationTest):
+    """Tests in this class exist to ensure we don't call describe extended unnecessarily.
+    This became a problem due to needing to discern tables from streaming tables, which is not
+    relevant on hive, but users on hive were having all of their tables describe extended-ed.
+    We only need to call describe extended if we are using a UC catalog and we can't determine the
+    type of the materialization."""
+
+    @property
+    def schema(self):
+        return "schema"
+
+    @property
+    def models(self):
+        return "models"
+
+    def _test_avoid_describe_extended(self):
+        # Add some existing data to ensure we don't try to 'describe extended' it.
+        self.run_dbt(["seed"])
+        _, log_output = self.run_dbt_and_capture(["run"])
+        self.assertNotIn("describe extended", log_output)
+
+    @use_profile("databricks_cluster")
+    def test_avoid_describe_extended_databricks_cluster(self):
+        """When UC is not enabled, we can assumed that all tables are regular tables"""
+        self._test_avoid_describe_extended()
+
+    @use_profile("databricks_uc_sql_endpoint")
+    def test_avoid_describe_extended_databricks_uc_sql_endpoint(self):
+        """When UC is enabled, regular tables are marked as such"""
+        self._test_avoid_describe_extended()