From 34c1cc4bbcf28ccb0243807441bf642e801ad158 Mon Sep 17 00:00:00 2001 From: guillesd <74136033+guillesd@users.noreply.github.com> Date: Fri, 24 Mar 2023 10:00:27 +0100 Subject: [PATCH] Create schema for external tables when it is not there. (#167) --- macros/common/create_external_schema.sql | 11 +++++++++++ macros/plugins/bigquery/get_external_build_plan.sql | 5 ++++- .../plugins/snowflake/get_external_build_plan.sql | 6 +++++- macros/plugins/spark/get_external_build_plan.sql | 1 + macros/plugins/sqlserver/create_external_schema.sql | 13 +++++++++++++ .../plugins/sqlserver/get_external_build_plan.sql | 1 + 6 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 macros/common/create_external_schema.sql create mode 100644 macros/plugins/sqlserver/create_external_schema.sql diff --git a/macros/common/create_external_schema.sql b/macros/common/create_external_schema.sql new file mode 100644 index 00000000..c96cbcda --- /dev/null +++ b/macros/common/create_external_schema.sql @@ -0,0 +1,11 @@ +{% macro create_external_schema(source_node) %} + {{ adapter.dispatch('create_external_schema', 'dbt_external_tables')(source_node) }} +{% endmacro %} + +{% macro default__create_external_schema(source_node) %} + {% set ddl %} + create schema if not exists {{ source_node.schema }} + {% endset %} + + {{return(ddl)}} +{% endmacro %} diff --git a/macros/plugins/bigquery/get_external_build_plan.sql b/macros/plugins/bigquery/get_external_build_plan.sql index fe7d7838..f90c1bfa 100644 --- a/macros/plugins/bigquery/get_external_build_plan.sql +++ b/macros/plugins/bigquery/get_external_build_plan.sql @@ -11,7 +11,10 @@ {% set create_or_replace = (old_relation is none or var('ext_full_refresh', false)) %} {% if create_or_replace %} - {% set build_plan = build_plan + [dbt_external_tables.create_external_table(source_node)] %} + {% set build_plan = build_plan + [ + dbt_external_tables.create_external_schema(source_node), + dbt_external_tables.create_external_table(source_node) + ] %} {% else %} {% set build_plan = build_plan + dbt_external_tables.refresh_external_table(source_node) %} {% endif %} diff --git a/macros/plugins/snowflake/get_external_build_plan.sql b/macros/plugins/snowflake/get_external_build_plan.sql index 0f73a0b2..807cbb5d 100644 --- a/macros/plugins/snowflake/get_external_build_plan.sql +++ b/macros/plugins/snowflake/get_external_build_plan.sql @@ -14,6 +14,7 @@ {% if create_or_replace %} {% set build_plan = build_plan + [ + dbt_external_tables.create_external_schema(source_node), dbt_external_tables.snowflake_create_empty_table(source_node), dbt_external_tables.snowflake_get_copy_sql(source_node, explicit_transaction=true), dbt_external_tables.snowflake_create_snowpipe(source_node) @@ -25,7 +26,10 @@ {% else %} {% if create_or_replace %} - {% set build_plan = build_plan + [dbt_external_tables.create_external_table(source_node)] %} + {% set build_plan = build_plan + [ + dbt_external_tables.create_external_schema(source_node), + dbt_external_tables.create_external_table(source_node) + ] %} {% else %} {% set build_plan = build_plan + dbt_external_tables.refresh_external_table(source_node) %} {% endif %} diff --git a/macros/plugins/spark/get_external_build_plan.sql b/macros/plugins/spark/get_external_build_plan.sql index 38afdac6..cab1ca29 100644 --- a/macros/plugins/spark/get_external_build_plan.sql +++ b/macros/plugins/spark/get_external_build_plan.sql @@ -12,6 +12,7 @@ {% if create_or_replace %} {% set build_plan = build_plan + [ + dbt_external_tables.create_external_schema(source_node), dbt_external_tables.dropif(source_node), dbt_external_tables.create_external_table(source_node) ] %} diff --git a/macros/plugins/sqlserver/create_external_schema.sql b/macros/plugins/sqlserver/create_external_schema.sql new file mode 100644 index 00000000..c1ef48be --- /dev/null +++ b/macros/plugins/sqlserver/create_external_schema.sql @@ -0,0 +1,13 @@ +{% macro sqlserver__create_external_schema(source_node) %} + {# https://learn.microsoft.com/en-us/sql/t-sql/statements/create-schema-transact-sql?view=sql-server-ver16 #} + + {% set ddl %} + IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ source_node.schema }}') + BEGIN + EXEC('CREATE SCHEMA [{{ source_node.schema }}]') + END + {% endset %} + + {{return(ddl)}} + +{% endmacro %} diff --git a/macros/plugins/sqlserver/get_external_build_plan.sql b/macros/plugins/sqlserver/get_external_build_plan.sql index 4e974100..7a161a0d 100644 --- a/macros/plugins/sqlserver/get_external_build_plan.sql +++ b/macros/plugins/sqlserver/get_external_build_plan.sql @@ -12,6 +12,7 @@ {% if create_or_replace %} {% set build_plan = build_plan + [ + dbt_external_tables.create_external_schema(source_node), dbt_external_tables.dropif(source_node), dbt_external_tables.create_external_table(source_node) ] %}