From 1a7706e86fe6e9ac933852d860278e7aadeaad82 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Wed, 5 Jun 2024 15:25:17 -0700 Subject: [PATCH 1/5] Added changes for dbt-oracle 1.8 --- Makefile | 2 +- dbt/adapters/oracle/connection_helper.py | 10 +-- dbt/adapters/oracle/connections.py | 25 +++--- dbt/adapters/oracle/impl.py | 86 ++++++++----------- dbt/adapters/oracle/python_submissions.py | 22 ++--- dbt/adapters/oracle/relation.py | 5 +- dbt/adapters/oracle/relation_configs/base.py | 2 +- .../relation_configs/materialized_view.py | 13 +-- .../oracle/relation_configs/policies.py | 4 +- requirements.txt | 6 +- requirements_dev.txt | 2 +- setup.cfg | 10 ++- setup.py | 10 ++- 13 files changed, 98 insertions(+), 99 deletions(-) diff --git a/Makefile b/Makefile index ade00f9..0fa583f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Configuration variables -VERSION=1.7.5 +VERSION=1.8.0 PROJ_DIR?=$(shell pwd) VENV_DIR?=${PROJ_DIR}/.bldenv BUILD_DIR=${PROJ_DIR}/build diff --git a/dbt/adapters/oracle/connection_helper.py b/dbt/adapters/oracle/connection_helper.py index 758459a..f863810 100644 --- a/dbt/adapters/oracle/connection_helper.py +++ b/dbt/adapters/oracle/connection_helper.py @@ -17,10 +17,10 @@ import enum import os -import dbt.exceptions -from dbt.events import AdapterLogger +import dbt_common.exceptions +from dbt.adapters.events.logging import AdapterLogger -from dbt.ui import warning_tag, yellow, red +from dbt_common.ui import warning_tag, yellow, red logger = AdapterLogger("oracle") @@ -129,5 +129,5 @@ class OracleDriverType(str, enum.Enum): SQLNET_ORA_CONFIG = OracleNetConfig.from_env() logger.info("Running in thin mode") else: - raise dbt.exceptions.DbtRuntimeError("Invalid value set for ORA_PYTHON_DRIVER_TYPE\n" - "Use any one of 'cx', 'thin', or 'thick'") + raise dbt_common.exceptions.DbtRuntimeError("Invalid value set for ORA_PYTHON_DRIVER_TYPE\n" + "Use any one of 'cx', 'thin', or 'thick'") diff --git a/dbt/adapters/oracle/connections.py b/dbt/adapters/oracle/connections.py index ff91a96..f66dbc1 100644 --- a/dbt/adapters/oracle/connections.py +++ b/dbt/adapters/oracle/connections.py @@ -23,15 +23,16 @@ import uuid import platform -import dbt.exceptions -from dbt.adapters.base import Credentials +import dbt_common.exceptions +from dbt.adapters.contracts.connection import AdapterResponse, Credentials +from dbt.adapters.exceptions.connection import FailedToConnectError from dbt.adapters.sql import SQLConnectionManager -from dbt.contracts.connection import AdapterResponse -from dbt.events.functions import fire_event -from dbt.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus -from dbt.events import AdapterLogger -from dbt.events.contextvars import get_node_info -from dbt.utils import cast_to_str +from dbt.adapters.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus +from dbt.adapters.events.logging import AdapterLogger + +from dbt_common.events.functions import fire_event +from dbt_common.events.contextvars import get_node_info +from dbt_common.utils import cast_to_str from dbt.version import __version__ as dbt_version from dbt.adapters.oracle.connection_helper import oracledb, SQLNET_ORA_CONFIG @@ -256,7 +257,7 @@ def open(cls, connection): connection.handle = None connection.state = 'fail' - raise dbt.exceptions.FailedToConnectError(str(e)) + raise FailedToConnectError(str(e)) return connection @@ -302,18 +303,18 @@ def exception_handler(self, sql): logger.info("Failed to release connection!") pass - raise dbt.exceptions.DbtDatabaseError(str(e).strip()) from e + raise dbt_common.exceptions.DbtDatabaseError(str(e).strip()) from e except Exception as e: logger.info("Rolling back transaction.") self.release() - if isinstance(e, dbt.exceptions.DbtRuntimeError): + if isinstance(e, dbt_common.exceptions.DbtRuntimeError): # during a sql query, an internal to dbt exception was raised. # this sounds a lot like a signal handler and probably has # useful information, so raise it without modification. raise e - raise dbt.exceptions.DbtRuntimeError(str(e)) from e + raise dbt_common.exceptions.DbtRuntimeError(str(e)) from e @classmethod def get_credentials(cls, credentials): diff --git a/dbt/adapters/oracle/impl.py b/dbt/adapters/oracle/impl.py index ceaeb7a..4109a2d 100644 --- a/dbt/adapters/oracle/impl.py +++ b/dbt/adapters/oracle/impl.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2023, Oracle and/or its affiliates. +Copyright (c) 2024, Oracle and/or its affiliates. Copyright (c) 2020, Vitor Avancini Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,7 @@ """ import datetime from typing import ( - Optional, List, Set + Optional, List, Set, FrozenSet, Tuple, Iterable ) from itertools import chain from typing import ( @@ -27,24 +27,23 @@ import agate import requests -import dbt.exceptions +import dbt_common.exceptions +from dbt_common.contracts.constraints import ConstraintType +from dbt_common.utils import filter_null_values + from dbt.adapters.base.relation import BaseRelation, InformationSchema from dbt.adapters.base.impl import GET_CATALOG_MACRO_NAME, ConstraintSupport, GET_CATALOG_RELATIONS_MACRO_NAME, _expect_row_value +from dbt.adapters.contracts.relation import RelationConfig +from dbt.adapters.events.logging import AdapterLogger from dbt.adapters.sql import SQLAdapter from dbt.adapters.base.meta import available from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability + from dbt.adapters.oracle import OracleAdapterConnectionManager from dbt.adapters.oracle.column import OracleColumn from dbt.adapters.oracle.relation import OracleRelation -from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.graph.nodes import ConstraintType -from dbt.events import AdapterLogger - -from dbt.utils import filter_null_values - from dbt.adapters.oracle.keyword_catalog import KEYWORDS from dbt.adapters.oracle.python_submissions import OracleADBSPythonJob -from dbt.adapters.oracle.connections import AdapterResponse logger = AdapterLogger("oracle") @@ -139,7 +138,7 @@ def verify_database(self, database): database = database.strip('"') expected = self.config.credentials.database if expected and database.lower() != expected.lower(): - raise dbt.exceptions.DbtRuntimeError( + raise dbt_common.exceptions.DbtRuntimeError( 'Cross-db references not allowed in {} ({} vs {})' .format(self.type(), database, expected) ) @@ -210,55 +209,44 @@ def _get_one_catalog( self, information_schema: InformationSchema, schemas: Set[str], - manifest: Manifest, + used_schemas: FrozenSet[Tuple[str, str]], ) -> agate.Table: - + logger.info(f"GET ONE CATALOG =====> {schemas}") + logger.info(f"GET ONE CATALOG =====> {information_schema}") + logger.info(f"GET ONE CATALOG =====> {used_schemas}") kwargs = {"information_schema": information_schema, "schemas": schemas} - table = self.execute_macro( - GET_CATALOG_MACRO_NAME, - kwargs=kwargs, - # pass in the full manifest so we get any local project - # overrides - manifest=manifest, - ) - # In case database is not defined, we can use the the configured database which we set as part of credentials - for node in chain(manifest.nodes.values(), manifest.sources.values()): - if not node.database or node.database == 'None': - node.database = self.config.credentials.database - - results = self._catalog_filter_table(table, manifest) + table = self.execute_macro(GET_CATALOG_MACRO_NAME, kwargs=kwargs) + results = self._catalog_filter_table(table, used_schemas=used_schemas) + logger.info(f"GET ONE CATALOG =====> {results}") return results def _get_one_catalog_by_relations( self, information_schema: InformationSchema, relations: List[BaseRelation], - manifest: Manifest, + used_schemas: FrozenSet[Tuple[str, str]], ) -> agate.Table: - + logger.info(f"GET ONE _get_one_catalog_by_relations =====> {relations}") + logger.info(f"GET ONE _get_one_catalog_by_relations =====> {information_schema}") + logger.info(f"GET ONE _get_one_catalog_by_relations =====> {used_schemas}") kwargs = { "information_schema": information_schema, "relations": relations, } - table = self.execute_macro( - GET_CATALOG_RELATIONS_MACRO_NAME, - kwargs=kwargs, - # pass in the full manifest, so we get any local project - # overrides - manifest=manifest, - ) - - # In case database is not defined, we can use the the configured database which we set as part of credentials - for node in chain(manifest.nodes.values(), manifest.sources.values()): - if not node.database or node.database == 'None': - node.database = self.config.credentials.database - - results = self._catalog_filter_table(table, manifest) # type: ignore[arg-type] + table = self.execute_macro(GET_CATALOG_RELATIONS_MACRO_NAME, kwargs=kwargs) + results = self._catalog_filter_table(table, used_schemas) # type: ignore[arg-type] + logger.info(f"GET ONE _get_one_catalog_by_relations =====> {results}") return results def get_filtered_catalog( - self, manifest: Manifest, relations: Optional[Set[BaseRelation]] = None + self, + relation_configs: Iterable[RelationConfig], + used_schemas: FrozenSet[Tuple[str, str]], + relations: Optional[Set[BaseRelation]] = None ): + logger.info(f"GET ONE get_filtered_catalog =====> {relations}") + logger.info(f"GET ONE get_filtered_catalog =====> {relations}") + logger.info(f"GET ONE get_filtered_catalog =====> {used_schemas}") catalogs: agate.Table if ( relations is None @@ -266,11 +254,11 @@ def get_filtered_catalog( or not self.supports(Capability.SchemaMetadataByRelations) ): # Do it the traditional way. We get the full catalog. - catalogs, exceptions = self.get_catalog(manifest) + catalogs, exceptions = self.get_catalog(relation_configs, used_schemas) else: # Do it the new way. We try to save time by selecting information # only for the exact set of relations we are interested in. - catalogs, exceptions = self.get_catalog_by_relations(manifest, relations) + catalogs, exceptions = self.get_catalog_by_relations(used_schemas, relations) if relations and catalogs: relation_map = { @@ -388,8 +376,8 @@ def quote_seed_column( elif quote_config is None: pass else: - raise dbt.exceptions.CompilationError(f'The seed configuration value of "quote_columns" ' - f'has an invalid type {type(quote_config)}') + raise dbt_common.exceptions.CompilationError(f'The seed configuration value of "quote_columns" ' + f'has an invalid type {type(quote_config)}') if quote_columns: return self.quote(column) @@ -417,7 +405,7 @@ def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) def get_oml_auth_token(self) -> str: if self.config.credentials.oml_auth_token_uri is None: - raise dbt.exceptions.DbtRuntimeError("oml_auth_token_uri should be set to run dbt-py models") + raise dbt_common.exceptions.DbtRuntimeError("oml_auth_token_uri should be set to run dbt-py models") data = { "grant_type": "password", "username": self.config.credentials.user, @@ -428,7 +416,7 @@ def get_oml_auth_token(self) -> str: json=data) r.raise_for_status() except requests.exceptions.RequestException: - raise dbt.exceptions.DbtRuntimeError("Error getting OML OAuth2.0 token") + raise dbt_common.exceptions.DbtRuntimeError("Error getting OML OAuth2.0 token") else: return r.json()["accessToken"] diff --git a/dbt/adapters/oracle/python_submissions.py b/dbt/adapters/oracle/python_submissions.py index 52b5bcb..0ce4e59 100644 --- a/dbt/adapters/oracle/python_submissions.py +++ b/dbt/adapters/oracle/python_submissions.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2023, Oracle and/or its affiliates. +Copyright (c) 2024, Oracle and/or its affiliates. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,10 +23,10 @@ import requests import time -import dbt.exceptions +import dbt_common.exceptions from dbt.adapters.oracle import OracleAdapterCredentials -from dbt.events import AdapterLogger -from dbt.ui import red, green +from dbt.adapters.events.logging import AdapterLogger +from dbt_common.ui import red, green from dbt.version import __version__ as dbt_version # ADB-S OML Rest API minimum timeout is 1800 seconds @@ -170,7 +170,7 @@ def schedule_async_job_and_wait_for_completion(self, data): r.raise_for_status() except requests.exceptions.RequestException as e: logger.error(red(f"Error {e} scheduling async Python job for model {self.identifier}")) - raise dbt.exceptions.DbtRuntimeError(f"Error scheduling Python model {self.identifier}") + raise dbt_common.exceptions.DbtRuntimeError(f"Error scheduling Python model {self.identifier}") job_location = r.headers["location"] logger.info(f"Started async job {job_location}") @@ -192,24 +192,24 @@ def schedule_async_job_and_wait_for_completion(self, data): job_result_json = job_result.json() if 'errorMessage' in job_result_json: logger.error(red(f"FAILURE - Python model {self.identifier} Job failure is: {job_result_json}")) - raise dbt.exceptions.DbtRuntimeError(f"Error running Python model {self.identifier}") + raise dbt_common.exceptions.DbtRuntimeError(f"Error running Python model {self.identifier}") job_result.raise_for_status() logger.info(green(f"SUCCESS - Python model {self.identifier} Job result is: {job_result_json}")) return elif job_status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR: logger.error(red(f"FAILURE - Job status is: {job_status.json()}")) - raise dbt.exceptions.DbtRuntimeError(f"Error running Python model {self.identifier}") + raise dbt_common.exceptions.DbtRuntimeError(f"Error running Python model {self.identifier}") else: logger.debug(f"Python model {self.identifier} job status is: {job_status.json()}") job_status.raise_for_status() except requests.exceptions.RequestException as e: logger.error(red(f"Error {e} checking status of Python job {job_location} for model {self.identifier}")) - raise dbt.exceptions.DbtRuntimeError(f"Error checking status for job {job_location}") + raise dbt_common.exceptions.DbtRuntimeError(f"Error checking status for job {job_location}") time.sleep(DEFAULT_DELAY_BETWEEN_POLL_IN_SECONDS) logger.error(red(f"Timeout error for Python model {self.identifier}")) - raise dbt.exceptions.DbtRuntimeError(f"Timeout error for Python model {self.identifier}") + raise dbt_common.exceptions.DbtRuntimeError(f"Timeout error for Python model {self.identifier}") def __call__(self, *args, **kwargs): data = { @@ -234,10 +234,10 @@ def __call__(self, *args, **kwargs): job_result = r.json() if 'errorMessage' in job_result: logger.error(red(f"FAILURE - Python model {self.identifier} Job failure is: {job_result}")) - raise dbt.exceptions.DbtRuntimeError(f"Error running Python model {self.identifier}") + raise dbt_common.exceptions.DbtRuntimeError(f"Error running Python model {self.identifier}") r.raise_for_status() logger.info(green(f"SUCCESS - Python model {self.identifier} Job result is: {job_result}")) except requests.exceptions.RequestException as e: logger.error(red(f"Error {e} running Python model {self.identifier}")) - raise dbt.exceptions.DbtRuntimeError(f"Error running Python model {self.identifier}") + raise dbt_common.exceptions.DbtRuntimeError(f"Error running Python model {self.identifier}") diff --git a/dbt/adapters/oracle/relation.py b/dbt/adapters/oracle/relation.py index 52b9c04..e0a8ec4 100644 --- a/dbt/adapters/oracle/relation.py +++ b/dbt/adapters/oracle/relation.py @@ -19,6 +19,7 @@ from dataclasses import dataclass, field from dbt.adapters.base.relation import BaseRelation +from dbt.adapters.events.logging import AdapterLogger from dbt.adapters.relation_configs import ( RelationConfigBase, RelationConfigChangeAction, @@ -26,7 +27,7 @@ ) from dbt.context.providers import RuntimeConfigObject from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import RelationType +from dbt.adapters.base import RelationType from dbt.exceptions import DbtRuntimeError from dbt.adapters.oracle.relation_configs import ( @@ -40,7 +41,7 @@ OracleQuotePolicy, OracleIncludePolicy) -from dbt.events import AdapterLogger + logger = AdapterLogger("oracle") diff --git a/dbt/adapters/oracle/relation_configs/base.py b/dbt/adapters/oracle/relation_configs/base.py index 94bb12b..1327697 100644 --- a/dbt/adapters/oracle/relation_configs/base.py +++ b/dbt/adapters/oracle/relation_configs/base.py @@ -24,7 +24,7 @@ RelationResults, ) from dbt.contracts.graph.nodes import ModelNode -from dbt.contracts.relation import ComponentName +from dbt.adapters.contracts.relation import ComponentName from dbt.adapters.oracle.relation_configs.policies import ( OracleQuotePolicy, diff --git a/dbt/adapters/oracle/relation_configs/materialized_view.py b/dbt/adapters/oracle/relation_configs/materialized_view.py index 0afec3e..08e24ce 100644 --- a/dbt/adapters/oracle/relation_configs/materialized_view.py +++ b/dbt/adapters/oracle/relation_configs/materialized_view.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2023, Oracle and/or its affiliates. +Copyright (c) 2024, Oracle and/or its affiliates. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,20 +15,20 @@ """ from dataclasses import dataclass -from typing import Optional, Set +from typing import Optional import agate from dbt.adapters.oracle.relation_configs.base import OracleRelationConfigBase from dbt.adapters.relation_configs import RelationResults, RelationConfigChange -from dbt.contracts.relation import ComponentName +from dbt.adapters.contracts.relation import ComponentName from dbt.contracts.graph.nodes import ModelNode -from dbt.exceptions import DbtRuntimeError -from dbt.events import AdapterLogger +from dbt.adapters.events.logging import AdapterLogger logger = AdapterLogger("oracle") + @dataclass(frozen=True, eq=True, unsafe_hash=True) class OracleMaterializedViewConfig(OracleRelationConfigBase): """ @@ -143,6 +143,7 @@ class OracleRefreshMethodConfigChange(RelationConfigChange): def requires_full_refresh(self) -> bool: return True + @dataclass(frozen=True, eq=True, unsafe_hash=True) class OracleBuildModeConfigChange(RelationConfigChange): context: Optional[str] = None @@ -151,6 +152,7 @@ class OracleBuildModeConfigChange(RelationConfigChange): def requires_full_refresh(self) -> bool: return False + @dataclass(frozen=True, eq=True, unsafe_hash=True) class OracleQueryRewriteConfigChange(RelationConfigChange): context: Optional[str] = None @@ -159,6 +161,7 @@ class OracleQueryRewriteConfigChange(RelationConfigChange): def requires_full_refresh(self) -> bool: return False + @dataclass(frozen=True, eq=True, unsafe_hash=True) class OracleQueryConfigChange(RelationConfigChange): context: Optional[str] = None diff --git a/dbt/adapters/oracle/relation_configs/policies.py b/dbt/adapters/oracle/relation_configs/policies.py index 6cf15a7..ff0f64d 100644 --- a/dbt/adapters/oracle/relation_configs/policies.py +++ b/dbt/adapters/oracle/relation_configs/policies.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2023, Oracle and/or its affiliates. +Copyright (c) 2024, Oracle and/or its affiliates. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ from dataclasses import dataclass from dbt.adapters.base.relation import Policy -from dbt.dataclass_schema import StrEnum +from dbt_common.dataclass_schema import StrEnum @dataclass diff --git a/requirements.txt b/requirements.txt index a3aa617..7741b37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ -dbt-core~=1.7,<1.8 -oracledb==2.2.0 +dbt-common>=1.1.0,<2.0 +dbt-adapters>=1.2.1,<2.0 +dbt-core>=1.8.1,<2.0 +oracledb==2.2.1 diff --git a/requirements_dev.txt b/requirements_dev.txt index fc9f974..eb14ecc 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -6,4 +6,4 @@ tox coverage twine pytest -dbt-tests-adapter~=1.7,<1.8 +dbt-tests-adapter~=1.8,<1.9 diff --git a/setup.cfg b/setup.cfg index 79ca43b..7e2a88e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dbt-oracle -version = 1.7.5 +version = 1.8.0 description = dbt (data build tool) adapter for Oracle Autonomous Database long_description = file: README.md long_description_content_type = text/markdown @@ -33,11 +33,13 @@ zip_safe = False packages = find_namespace: include_package_data = True install_requires = - dbt-core~=1.7,<1.8 - oracledb==2.2.0 + dbt-common>=1.1.0,<2.0 + dbt-adapters>=1.2.1,<2.0 + dbt-core~=1.8,<1.9 + oracledb==2.2.1 test_suite=tests test_requires = - dbt-tests-adapter~=1.7,<1.8 + dbt-tests-adapter~=1.8,<1.9 pytest scripts = bin/create-pem-from-p12 diff --git a/setup.py b/setup.py index 06d45a3..8239965 100644 --- a/setup.py +++ b/setup.py @@ -40,12 +40,14 @@ requirements = [ - "dbt-core~=1.7,<1.8", - "oracledb==2.2.0" + "dbt-common>=1.1.0,<2.0", + "dbt-adapters>=1.2.1,<2.0", + "dbt-core~=1.8,<1.9", + "oracledb==2.2.1" ] test_requirements = [ - "dbt-tests-adapter~=1.7,<1.8", + "dbt-tests-adapter~=1.8,<1.9", "pytest" ] @@ -59,7 +61,7 @@ url = 'https://github.com/oracle/dbt-oracle' -VERSION = '1.7.5' +VERSION = '1.8.0' setup( author="Oracle", python_requires='>=3.8', From 6d224123766b678a54a8e664050a5f9ccd4a8a41 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Wed, 5 Jun 2024 16:25:38 -0700 Subject: [PATCH 2/5] fixed dbt-tests-adapter version installed for dbt-oracle 1.8 --- .github/workflows/oracle-xe-adapter-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/oracle-xe-adapter-tests.yml b/.github/workflows/oracle-xe-adapter-tests.yml index ce1aef9..e6544ab 100644 --- a/.github/workflows/oracle-xe-adapter-tests.yml +++ b/.github/workflows/oracle-xe-adapter-tests.yml @@ -48,7 +48,7 @@ jobs: - name: Install dbt-oracle with core dependencies run: | python -m pip install --upgrade pip - pip install pytest 'dbt-tests-adapter~=1.7,<1.8' + pip install pytest 'dbt-tests-adapter~=1.8,<1.9' pip install -r requirements.txt pip install -e . From 4311eb59b4b381955d558c45ef8e4bb34f15d485 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Wed, 5 Jun 2024 16:32:19 -0700 Subject: [PATCH 3/5] Fixed SQL generation for delete+insert incremental append strategy --- .../oracle/macros/materializations/incremental/strategies.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/oracle/macros/materializations/incremental/strategies.sql b/dbt/include/oracle/macros/materializations/incremental/strategies.sql index 97be12b..dad5005 100644 --- a/dbt/include/oracle/macros/materializations/incremental/strategies.sql +++ b/dbt/include/oracle/macros/materializations/incremental/strategies.sql @@ -193,7 +193,7 @@ from {{ temp_relation }})'; END; {%- else -%} - insert {%- if parallel -%} /*+parallel({{ parallel }})*/ {%- endif -%} into {{ target_relation }} ({{ dest_cols_csv }}) + insert {% if parallel %} /*+parallel({{ parallel }})*/ {% endif %} into {{ target_relation }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} from {{ temp_relation }} From 880dab9508ea4d0caf682c5102c30bce3d4ed6fd Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Tue, 18 Jun 2024 20:52:36 -0700 Subject: [PATCH 4/5] Catalog generation fix and warning for database name --- dbt/adapters/oracle/__init__.py | 3 +- dbt/adapters/oracle/__version__.py | 2 +- dbt/adapters/oracle/connection_helper.py | 2 +- dbt/adapters/oracle/impl.py | 71 ++++++++++++++---------- 4 files changed, 46 insertions(+), 32 deletions(-) diff --git a/dbt/adapters/oracle/__init__.py b/dbt/adapters/oracle/__init__.py index bd9c17d..972b1c2 100644 --- a/dbt/adapters/oracle/__init__.py +++ b/dbt/adapters/oracle/__init__.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2022, Oracle and/or its affiliates. +Copyright (c) 2024, Oracle and/or its affiliates. Copyright (c) 2020, Vitor Avancini Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +17,6 @@ from dbt.adapters.oracle.connections import OracleAdapterConnectionManager from dbt.adapters.oracle.connections import OracleAdapterCredentials from dbt.adapters.oracle.impl import OracleAdapter - from dbt.adapters.base import AdapterPlugin from dbt.include import oracle diff --git a/dbt/adapters/oracle/__version__.py b/dbt/adapters/oracle/__version__.py index 0fdcd4c..170c215 100644 --- a/dbt/adapters/oracle/__version__.py +++ b/dbt/adapters/oracle/__version__.py @@ -14,4 +14,4 @@ See the License for the specific language governing permissions and limitations under the License. """ -version = "1.7.14" +version = "1.8.0" diff --git a/dbt/adapters/oracle/connection_helper.py b/dbt/adapters/oracle/connection_helper.py index f863810..30a4db2 100644 --- a/dbt/adapters/oracle/connection_helper.py +++ b/dbt/adapters/oracle/connection_helper.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2023, Oracle and/or its affiliates. +Copyright (c) 2024, Oracle and/or its affiliates. Copyright (c) 2020, Vitor Avancini Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/dbt/adapters/oracle/impl.py b/dbt/adapters/oracle/impl.py index 4109a2d..0358a07 100644 --- a/dbt/adapters/oracle/impl.py +++ b/dbt/adapters/oracle/impl.py @@ -18,7 +18,6 @@ from typing import ( Optional, List, Set, FrozenSet, Tuple, Iterable ) -from itertools import chain from typing import ( Any, Callable, @@ -31,8 +30,9 @@ from dbt_common.contracts.constraints import ConstraintType from dbt_common.utils import filter_null_values +from dbt.adapters.base.connections import Connection from dbt.adapters.base.relation import BaseRelation, InformationSchema -from dbt.adapters.base.impl import GET_CATALOG_MACRO_NAME, ConstraintSupport, GET_CATALOG_RELATIONS_MACRO_NAME, _expect_row_value +from dbt.adapters.base.impl import ConstraintSupport, GET_CATALOG_RELATIONS_MACRO_NAME, _expect_row_value from dbt.adapters.contracts.relation import RelationConfig from dbt.adapters.events.logging import AdapterLogger from dbt.adapters.sql import SQLAdapter @@ -45,6 +45,8 @@ from dbt.adapters.oracle.keyword_catalog import KEYWORDS from dbt.adapters.oracle.python_submissions import OracleADBSPythonJob +from dbt_common.ui import warning_tag, yellow + logger = AdapterLogger("oracle") # Added 6 random hex letters (56c36b) to table_a and table_b to avoid ORA-32031. @@ -81,6 +83,15 @@ LIST_RELATIONS_MACRO_NAME = 'list_relations_without_caching' GET_DATABASE_MACRO_NAME = 'get_database_name' +MISSING_DATABASE_NAME_FOR_CATALOG_WARNING_MESSAGE = ( + "database key is missing from the target-profile in the file profiles.yml " + "\n Starting with dbt-oracle 1.8 database name is needed for catalog generation " + "\n Without database key in the target profile the generated catalog will be empty " + "\n \t i.e. `dbt docs generate` command will generate an empty catalog json " + "\n Make the following entry in dbt profile.yml file for the target profile " + "\n database: {0}" +) + class OracleAdapter(SQLAdapter): ConnectionManager = OracleAdapterConnectionManager @@ -137,7 +148,7 @@ def verify_database(self, database): if database.startswith('"'): database = database.strip('"') expected = self.config.credentials.database - if expected and database.lower() != expected.lower(): + if expected and database.lower() != 'none' and database.lower() != expected.lower(): raise dbt_common.exceptions.DbtRuntimeError( 'Cross-db references not allowed in {} ({} vs {})' .format(self.type(), database, expected) @@ -205,37 +216,18 @@ def get_relation(self, database: str, schema: str, identifier: str) -> Optional[ database = self.config.credentials.database return super().get_relation(database, schema, identifier) - def _get_one_catalog( + def _get_one_catalog_by_relations( self, information_schema: InformationSchema, - schemas: Set[str], + relations: List[BaseRelation], used_schemas: FrozenSet[Tuple[str, str]], - ) -> agate.Table: - logger.info(f"GET ONE CATALOG =====> {schemas}") - logger.info(f"GET ONE CATALOG =====> {information_schema}") - logger.info(f"GET ONE CATALOG =====> {used_schemas}") - kwargs = {"information_schema": information_schema, "schemas": schemas} - table = self.execute_macro(GET_CATALOG_MACRO_NAME, kwargs=kwargs) - results = self._catalog_filter_table(table, used_schemas=used_schemas) - logger.info(f"GET ONE CATALOG =====> {results}") - return results - - def _get_one_catalog_by_relations( - self, - information_schema: InformationSchema, - relations: List[BaseRelation], - used_schemas: FrozenSet[Tuple[str, str]], - ) -> agate.Table: - logger.info(f"GET ONE _get_one_catalog_by_relations =====> {relations}") - logger.info(f"GET ONE _get_one_catalog_by_relations =====> {information_schema}") - logger.info(f"GET ONE _get_one_catalog_by_relations =====> {used_schemas}") + ) -> "agate.Table": kwargs = { "information_schema": information_schema, "relations": relations, } table = self.execute_macro(GET_CATALOG_RELATIONS_MACRO_NAME, kwargs=kwargs) results = self._catalog_filter_table(table, used_schemas) # type: ignore[arg-type] - logger.info(f"GET ONE _get_one_catalog_by_relations =====> {results}") return results def get_filtered_catalog( @@ -244,10 +236,21 @@ def get_filtered_catalog( used_schemas: FrozenSet[Tuple[str, str]], relations: Optional[Set[BaseRelation]] = None ): - logger.info(f"GET ONE get_filtered_catalog =====> {relations}") - logger.info(f"GET ONE get_filtered_catalog =====> {relations}") - logger.info(f"GET ONE get_filtered_catalog =====> {used_schemas}") catalogs: agate.Table + + def is_database_none(database): + return database is None or database == 'None' + + def populate_database(database): + if not is_database_none(database): + return database + return self.config.credentials.database + + # In case database is not defined, we can use database set in credentials object + if any(is_database_none(database) for database, schema in used_schemas): + used_schemas = frozenset([(populate_database(database).casefold(), schema) + for database, schema in used_schemas]) + if ( relations is None or len(relations) > 100 @@ -446,3 +449,15 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str): response, _ = self.execute(sql=py_q_drop_script) logger.info(response) return response + + def acquire_connection(self, name=None) -> Connection: + connection = self.connections.set_connection_name(name) + if connection.credentials.database is None or connection.credentials.database.lower() == 'none': + with connection.handle.cursor() as cr: + cr.execute("select SYS_CONTEXT('userenv', 'DB_NAME') FROM DUAL") + r = cr.fetchone() + database = r[0] + logger.warning(warning_tag(yellow(MISSING_DATABASE_NAME_FOR_CATALOG_WARNING_MESSAGE.format(database)))) + self.config.credentials.database = database + connection.credentials.database = database + return connection From 2b745738a3db15b3b1702c460391ea0471cef2a0 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Wed, 19 Jun 2024 23:44:47 -0700 Subject: [PATCH 5/5] Changed warning message --- dbt/adapters/oracle/impl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/oracle/impl.py b/dbt/adapters/oracle/impl.py index 0358a07..5691957 100644 --- a/dbt/adapters/oracle/impl.py +++ b/dbt/adapters/oracle/impl.py @@ -84,10 +84,10 @@ GET_DATABASE_MACRO_NAME = 'get_database_name' MISSING_DATABASE_NAME_FOR_CATALOG_WARNING_MESSAGE = ( - "database key is missing from the target-profile in the file profiles.yml " + "database key is missing from the target profile in the file profiles.yml " "\n Starting with dbt-oracle 1.8 database name is needed for catalog generation " "\n Without database key in the target profile the generated catalog will be empty " - "\n \t i.e. `dbt docs generate` command will generate an empty catalog json " + "\n i.e. `dbt docs generate` command will generate an empty catalog json " "\n Make the following entry in dbt profile.yml file for the target profile " "\n database: {0}" )