Skip to content

Commit

Permalink
Merge branch '1.6.6' into 1.6.latest
Browse files Browse the repository at this point in the history
  • Loading branch information
benc-db committed Oct 9, 2023
2 parents 23d8689 + c566584 commit ebdd191
Show file tree
Hide file tree
Showing 10 changed files with 77 additions and 29 deletions.
13 changes: 12 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,15 @@
## dbt-databricks 1.6.x (Release TBD)
## dbt-databricks 1.7.x (TBD)

## dbt-databricks 1.6.6 (October 9, 2023)

### Fixes

- Optimize now runs after creating / updating liquid clustering tables ([463](https://github.com/databricks/dbt-databricks/pull/463))
- Fixing an issue where the new python library install from index behavior breaks users who were already customizing their installs ([472](https://github.com/databricks/dbt-databricks/pull/472))

### Under the Hood

- fix Pylance import errors (thanks @dataders) ([471](https://github.com/databricks/dbt-databricks/pull/471))

## dbt-databricks 1.6.5 (September 26, 2023)

Expand Down
3 changes: 3 additions & 0 deletions dbt/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
from pkgutil import extend_path

__path__ = extend_path(__path__, __name__)
2 changes: 1 addition & 1 deletion dbt/adapters/databricks/__version__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
version: str = "1.6.5"
version: str = "1.6.6"
7 changes: 5 additions & 2 deletions dbt/adapters/databricks/python_submissions.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,14 +96,17 @@ def _submit_job(self, path: str, cluster_spec: dict) -> str:
packages = self.parsed_model["config"].get("packages", [])

# custom index URL or default
index_url = self.parsed_model["config"].get("index_url", "https://pypi.org/simple")
index_url = self.parsed_model["config"].get("index_url", None)

# additional format of packages
additional_libs = self.parsed_model["config"].get("additional_libs", [])
libraries = []

for package in packages:
libraries.append({"pypi": {"package": package, "repo": index_url}})
if index_url:
libraries.append({"pypi": {"package": package, "repo": index_url}})
else:
libraries.append({"pypi": {"package": package}})

for lib in additional_libs:
libraries.append(lib)
Expand Down
52 changes: 29 additions & 23 deletions dbt/include/databricks/macros/adapters.sql
Original file line number Diff line number Diff line change
Expand Up @@ -368,32 +368,38 @@
{{ return(adapter.dispatch('optimize', 'dbt')(relation)) }}
{% endmacro %}

{% macro databricks__optimize(relation) %}
{% if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}
{% if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and var('databricks_skip_optimize', 'false')|lower != 'true' %}
{% call statement('run_optimize_stmt') %}
{%- macro databricks__optimize(relation) -%}
{%- if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and
var('databricks_skip_optimize', 'false')|lower != 'true' and
config.get('file_format', 'delta') == 'delta' -%}
{%- if (config.get('zorder', False) or config.get('liquid_clustered_by', False)) -%}
{%- call statement('run_optimize_stmt') -%}
{{ get_optimize_sql(relation) }}
{% endcall %}
{% endif %}
{% endif %}
{% endmacro %}
{%- endcall -%}
{%- endif -%}
{%- endif -%}
{%- endmacro -%}

{% macro get_optimize_sql(relation) %}
{% if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}
{%- set zorder = config.get('zorder', none) -%}
optimize {{ relation }}
{# TODO: predicates here? WHERE ... #}
{% if zorder is sequence and zorder is not string %}
zorder by (
{%- for col in zorder -%}
{%- macro get_optimize_sql(relation) %}
optimize {{ relation }}
{%- if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}
{%- if config.get('liquid_clustered_by', False) %}
{{ exceptions.warn("Both zorder and liquid_clustered_by are set but they are incompatible. zorder will be ignored.") }}
{%- else %}
{%- set zorder = config.get('zorder', none) %}
{# TODO: predicates here? WHERE ... #}
{%- if zorder is sequence and zorder is not string %}
zorder by (
{%- for col in zorder %}
{{ col }}{% if not loop.last %}, {% endif %}
{%- endfor -%}
)
{% else %}
zorder by ({{zorder}})
{% endif %}
{% endif %}
{% endmacro %}
{%- endfor %}
)
{%- else %}
zorder by ({{zorder}})
{%- endif %}
{%- endif %}
{%- endif %}
{%- endmacro -%}

{% macro databricks__list_relations_without_caching(schema_relation) %}
{{ return(adapter.get_relations_without_caching(schema_relation)) }}
Expand Down
6 changes: 6 additions & 0 deletions tests/integration/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -605,6 +605,12 @@ def _assertTablesEqualSql(self, relation_a, relation_b, columns=None):

return sql

def assert_in_log(self, message: str) -> None:
log_file = os.path.join(self._logs_dir, "dbt.log")
with open(log_file, "r") as f:
log = f.read()
assert message.lower() in log.lower()

def assertTablesEqual(
self,
table_a,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{{ config(materialized='incremental', liquid_clustered_by='id') }}
select 1 as id, 'Joe' as name
16 changes: 16 additions & 0 deletions tests/integration/liquid_clustering/test_liquid_clustering.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
from tests.integration.base import DBTIntegrationTest, use_profile


class TestLiquidClustering(DBTIntegrationTest):
@property
def schema(self):
return "liquid"

@property
def models(self):
return "models"

@use_profile("databricks_uc_sql_endpoint")
def test_liquid_clustering_databricks_uc_sql_endpoint(self):
self.run_dbt()
self.assert_in_log("optimize")
3 changes: 2 additions & 1 deletion tests/integration/zorder/test_zorder.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ def project_config(self):

def _test_zorder(self):
self.run_dbt(["run"])
self.run_dbt(["run"]) # make sure it also run in incremental
self.run_dbt(["run"])
self.assert_in_log("zorder by") # make sure it also run in incremental

@use_profile("databricks_cluster")
def test_zorder_databricks_cluster(self):
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/macros/test_adapters_macros.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ def test_macro_get_optimize_sql_multiple_args(self):

self.assertEqual(
sql,
("optimize " "`some_database`.`some_schema`.`some_table` " "zorder by (foo, bar)"),
("optimize " "`some_database`.`some_schema`.`some_table` " "zorder by ( foo, bar )"),
)

def test_macros_optimize_with_extraneous_info(self):
Expand Down

0 comments on commit ebdd191

Please sign in to comment.