From 55830eb94e57fd6c5b751d1808f3068fcc9fd40b Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Thu, 8 Aug 2024 14:13:48 -0700 Subject: [PATCH 1/4] v1.8.1 - oracledb python driver upgraded to 2.3.0 - Added support for BLOB column type in dbt model contracts - Fixed method signature for create schema --- Makefile | 2 +- dbt/include/oracle/macros/adapters.sql | 4 +++- .../macros/materializations/snapshot/snapshot.sql | 2 +- dbt_adbs_test_project/dbt_project.yml | 1 + dbt_adbs_test_project/models/kafka.sql | 2 +- dbt_adbs_test_project/models/promotion_costs.sql | 2 +- dbt_adbs_test_project/models/schema.yml | 5 +++++ dbt_adbs_test_project/profiles.yml | 2 +- dbt_adbs_test_project/seeds/kafka_message.csv | 10 +++++----- setup.cfg | 4 ++-- setup.py | 4 ++-- 11 files changed, 23 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 0fa583f..d36d2ae 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Configuration variables -VERSION=1.8.0 +VERSION=1.8.1 PROJ_DIR?=$(shell pwd) VENV_DIR?=${PROJ_DIR}/.bldenv BUILD_DIR=${PROJ_DIR}/build diff --git a/dbt/include/oracle/macros/adapters.sql b/dbt/include/oracle/macros/adapters.sql index 6d3b8af..84ab91a 100644 --- a/dbt/include/oracle/macros/adapters.sql +++ b/dbt/include/oracle/macros/adapters.sql @@ -42,6 +42,8 @@ {%- endif -%} {%- if col['data_type'] | lower == 'clob' -%} empty_clob() as {{ col['name'] }}{{ ", " if not loop.last }} + {%- elif col['data_type'] | lower == 'blob' -%} + empty_blob() as {{ col['name'] }}{{ ", " if not loop.last }} {%- else -%} cast(null as {{ col['data_type'] }}) as {{ col['name'] }}{{ ", " if not loop.last }} {%- endif -%} @@ -63,7 +65,7 @@ ) model_subq {%- endmacro %} -{% macro oracle__create_schema(relation, schema_name) -%} +{% macro oracle__create_schema(relation) -%} {% if relation.database -%} {{ adapter.verify_database(relation.database) }} {%- endif -%} diff --git a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql index 39d03c5..9036757 100644 --- a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql @@ -215,7 +215,7 @@ {% endif %} {% if not adapter.check_schema_exists(model_database, model.schema) %} - {% do create_schema(model_database, model.schema) %} + {% do create_schema(model.schema) %} {% endif %} {% set target_relation_exists, target_relation = get_or_create_relation( diff --git a/dbt_adbs_test_project/dbt_project.yml b/dbt_adbs_test_project/dbt_project.yml index 4298fc7..93805f7 100644 --- a/dbt_adbs_test_project/dbt_project.yml +++ b/dbt_adbs_test_project/dbt_project.yml @@ -21,6 +21,7 @@ seeds: kafka_message: +column_types: message: CLOB + blob_message: BLOB seed_with_empty_col: +column_types: id: number diff --git a/dbt_adbs_test_project/models/kafka.sql b/dbt_adbs_test_project/models/kafka.sql index 512e40e..e4e03dd 100644 --- a/dbt_adbs_test_project/models/kafka.sql +++ b/dbt_adbs_test_project/models/kafka.sql @@ -1 +1 @@ -select * from {{ ref('kafka_message') }} \ No newline at end of file +select * from {{ ref('kafka_message') }} diff --git a/dbt_adbs_test_project/models/promotion_costs.sql b/dbt_adbs_test_project/models/promotion_costs.sql index 18cf043..112adbc 100644 --- a/dbt_adbs_test_project/models/promotion_costs.sql +++ b/dbt_adbs_test_project/models/promotion_costs.sql @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. #} -{{ config(materialized='table')}} +{{ config(materialized='table', grants = {'select': ['dbt_test_1']})}} with all_promo_costs as( select * from {{ source('sh_database', 'promotions') }} ) diff --git a/dbt_adbs_test_project/models/schema.yml b/dbt_adbs_test_project/models/schema.yml index 2a577d0..2f3da77 100644 --- a/dbt_adbs_test_project/models/schema.yml +++ b/dbt_adbs_test_project/models/schema.yml @@ -43,6 +43,11 @@ models: data_type: CLOB constraints: - type: not_null + - name: blob_message + description: Kafka message + data_type: BLOB + constraints: + - type: not_null - name: people diff --git a/dbt_adbs_test_project/profiles.yml b/dbt_adbs_test_project/profiles.yml index faeceed..c837a56 100644 --- a/dbt_adbs_test_project/profiles.yml +++ b/dbt_adbs_test_project/profiles.yml @@ -9,7 +9,7 @@ dbt_test: host: "{{ env_var('DBT_ORACLE_HOST') }}" port: 1522 service: "{{ env_var('DBT_ORACLE_SERVICE') }}" -# database: "{{ env_var('DBT_ORACLE_DATABASE') }}" + database: "{{ env_var('DBT_ORACLE_DATABASE') }}" schema: "{{ env_var('DBT_ORACLE_SCHEMA') }}" oml_cloud_service_url: "{{ env_var('DBT_ORACLE_OML_CLOUD_SERVICE_URL')}}" session_info: diff --git a/dbt_adbs_test_project/seeds/kafka_message.csv b/dbt_adbs_test_project/seeds/kafka_message.csv index 427bcc7..5d5f34e 100644 --- a/dbt_adbs_test_project/seeds/kafka_message.csv +++ b/dbt_adbs_test_project/seeds/kafka_message.csv @@ -1,5 +1,5 @@ -message -Jack-Hunter -Jack-Hunter -Jack-HunterJackHunter -Jack-Hunter +message,blob_message +Jack-Hunter,1234 +Jack-Hunter,123124 +Jack-HunterJackHunter,123123 +Jack-Hunter,123143 diff --git a/setup.cfg b/setup.cfg index 7e2a88e..c87a843 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dbt-oracle -version = 1.8.0 +version = 1.8.1 description = dbt (data build tool) adapter for Oracle Autonomous Database long_description = file: README.md long_description_content_type = text/markdown @@ -36,7 +36,7 @@ install_requires = dbt-common>=1.1.0,<2.0 dbt-adapters>=1.2.1,<2.0 dbt-core~=1.8,<1.9 - oracledb==2.2.1 + oracledb==2.3.0 test_suite=tests test_requires = dbt-tests-adapter~=1.8,<1.9 diff --git a/setup.py b/setup.py index 8239965..31fed00 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ "dbt-common>=1.1.0,<2.0", "dbt-adapters>=1.2.1,<2.0", "dbt-core~=1.8,<1.9", - "oracledb==2.2.1" + "oracledb==2.3.0" ] test_requirements = [ @@ -61,7 +61,7 @@ url = 'https://github.com/oracle/dbt-oracle' -VERSION = '1.8.0' +VERSION = '1.8.1' setup( author="Oracle", python_requires='>=3.8', From 0433ccd56625e147002199256c204b1233a01796 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Thu, 8 Aug 2024 14:27:38 -0700 Subject: [PATCH 2/4] Upgraded oracledb version in requirements.txt --- dbt/adapters/oracle/__version__.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/oracle/__version__.py b/dbt/adapters/oracle/__version__.py index 170c215..57528fd 100644 --- a/dbt/adapters/oracle/__version__.py +++ b/dbt/adapters/oracle/__version__.py @@ -14,4 +14,4 @@ See the License for the specific language governing permissions and limitations under the License. """ -version = "1.8.0" +version = "1.8.1" diff --git a/requirements.txt b/requirements.txt index 7741b37..9183567 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ dbt-common>=1.1.0,<2.0 dbt-adapters>=1.2.1,<2.0 dbt-core>=1.8.1,<2.0 -oracledb==2.2.1 +oracledb==2.3.0 From 41ffb0a8930d21ec51962c636e4309a572e2db04 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Tue, 20 Aug 2024 11:27:23 -0700 Subject: [PATCH 3/4] Fixed Oracle limit SQL --- dbt/include/oracle/macros/show.sql | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dbt/include/oracle/macros/show.sql b/dbt/include/oracle/macros/show.sql index 6b93b99..2b9463b 100644 --- a/dbt/include/oracle/macros/show.sql +++ b/dbt/include/oracle/macros/show.sql @@ -1,5 +1,5 @@ {# - Copyright (c) 2023, Oracle and/or its affiliates. + Copyright (c) 2024, Oracle and/or its affiliates. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,3 +21,12 @@ ) fetch first {{ limit }} rows only {% endmacro %} + + +{% macro oracle__get_limit_sql(sql, limit) %} + select * + from ( + {{ sql }} + ) + fetch first {{ limit }} rows only +{% endmacro %} From ff9378972051f17592ea60e0d7374923607b53fa Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Tue, 20 Aug 2024 14:37:41 -0700 Subject: [PATCH 4/4] Fixed oracle__get_limit_sql --- dbt/include/oracle/macros/show.sql | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dbt/include/oracle/macros/show.sql b/dbt/include/oracle/macros/show.sql index 2b9463b..8c1158c 100644 --- a/dbt/include/oracle/macros/show.sql +++ b/dbt/include/oracle/macros/show.sql @@ -24,9 +24,8 @@ {% macro oracle__get_limit_sql(sql, limit) %} - select * - from ( - {{ sql }} - ) - fetch first {{ limit }} rows only + {{ compiled_code }} + {% if limit is not none %} + fetch first {{ limit }} rows only + {%- endif -%} {% endmacro %}