Remove get_columns implementation for column metadata generation (#940

)

* remove get_columns behavior flag, associated functionality, and skip associated tests

.changes/unreleased/Fixes-20241101-150335.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Remove `redshift_connector`'s `get_columns` method for column metadata generation,
+  avoiding a warning on every run
+time: 2024-11-01T15:03:35.44697-04:00
+custom:
+  Author: "mikealfare"
+  Issue: "914"

dbt/adapters/redshift/connections.py

@@ -3,7 +3,6 @@
 from contextlib import contextmanager
 from typing import Any, Callable, Dict, Tuple, Union, Optional, List, TYPE_CHECKING
 from dataclasses import dataclass, field
-import time
 
 import sqlparse
 import redshift_connector
@@ -13,14 +12,10 @@
 from dbt.adapters.sql import SQLConnectionManager
 from dbt.adapters.contracts.connection import AdapterResponse, Connection, Credentials
 from dbt.adapters.events.logging import AdapterLogger
-from dbt.adapters.events.types import SQLQuery, SQLQueryStatus
 from dbt_common.contracts.util import Replaceable
 from dbt_common.dataclass_schema import dbtClassMixin, StrEnum, ValidationError
-from dbt_common.events.contextvars import get_node_info
-from dbt_common.events.functions import fire_event
 from dbt_common.helper_types import Port
 from dbt_common.exceptions import DbtRuntimeError, CompilationError, DbtDatabaseError
-from dbt_common.utils import cast_to_str
 
 if TYPE_CHECKING:
     # Indirectly imported via agate_helper, which is lazy loaded further downfile.
@@ -465,51 +460,3 @@ def _initialize_sqlparse_lexer():
 
         if hasattr(Lexer, "get_default_instance"):
             Lexer.get_default_instance()
-
-    def columns_in_relation(self, relation) -> List[Dict[str, Any]]:
-        connection = self.get_thread_connection()
-
-        fire_event(
-            SQLQuery(
-                conn_name=cast_to_str(connection.name),
-                sql=f"call redshift_connector.Connection.get_columns({relation.database}, {relation.schema}, {relation.identifier})",
-                node_info=get_node_info(),
-            )
-        )
-
-        pre = time.perf_counter()
-
-        cursor = connection.handle.cursor()
-        columns = cursor.get_columns(
-            catalog=relation.database,
-            schema_pattern=relation.schema,
-            tablename_pattern=relation.identifier,
-        )
-
-        fire_event(
-            SQLQueryStatus(
-                status=str(self.get_response(cursor)),
-                elapsed=time.perf_counter() - pre,
-                node_info=get_node_info(),
-            )
-        )
-
-        return [self._parse_column_results(column) for column in columns]
-
-    @staticmethod
-    def _parse_column_results(record: Tuple[Any, ...]) -> Dict[str, Any]:
-        _, _, _, column_name, dtype_code, dtype_name, column_size, _, decimals, *_ = record
-
-        char_dtypes = [1, 12]
-        num_dtypes = [2, 3, 4, 5, 6, 7, 8, -5, 2003]
-
-        if dtype_code in char_dtypes:
-            return {"column": column_name, "dtype": dtype_name, "char_size": column_size}
-        elif dtype_code in num_dtypes:
-            return {
-                "column": column_name,
-                "dtype": dtype_name,
-                "numeric_precision": column_size,
-                "numeric_scale": decimals,
-            }
-        return {"column": column_name, "dtype": dtype_name, "char_size": column_size}

dbt/adapters/redshift/impl.py

@@ -1,12 +1,10 @@
 import os
 from dataclasses import dataclass
 
-from dbt_common.behavior_flags import BehaviorFlag
 from dbt_common.contracts.constraints import ConstraintType
-from typing import Optional, Set, Any, Dict, Type, TYPE_CHECKING, List
+from typing import Optional, Set, Any, Dict, Type, TYPE_CHECKING
 from collections import namedtuple
 from dbt.adapters.base import PythonJobHelper
-from dbt.adapters.base.column import Column
 from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport
 from dbt.adapters.base.meta import available
 from dbt.adapters.capability import Capability, CapabilityDict, CapabilitySupport, Support
@@ -68,23 +66,6 @@ class RedshiftAdapter(SQLAdapter):
         }
     )
 
-    @property
-    def _behavior_flags(self) -> List[BehaviorFlag]:
-        return [
-            {
-                "name": "restrict_direct_pg_catalog_access",
-                "default": False,
-                "description": (
-                    "The dbt-redshift adapter is migrating from using pg_ tables "
-                    "to using Redshift Metadata API and information_schema tables "
-                    "in order to support additional Redshift functionalities.\n"
-                    "We do not expect this to impact your dbt experience. "
-                    "Please report any issues using this GitHub discussion: https://github.com/dbt-labs/dbt-redshift/discussions/921"
-                ),
-                "docs_url": "https://docs.getdbt.com/reference/global-configs/behavior-changes#redshift-restrict_direct_pg_catalog_access",
-            }
-        ]
-
     @classmethod
     def date_function(cls):
         return "getdate()"
@@ -107,12 +88,6 @@ def drop_relation(self, relation):
         with self.connections.fresh_transaction():
             return super().drop_relation(relation)
 
-    def get_columns_in_relation(self, relation) -> List[Column]:
-        if self.behavior.restrict_direct_pg_catalog_access:
-            column_configs = self.connections.columns_in_relation(relation)
-            return [Column(**column) for column in column_configs]
-        return super().get_columns_in_relation(relation)
-
     @classmethod
     def convert_text_type(cls, agate_table: "agate.Table", col_idx):
         column = agate_table.columns[col_idx]

tests/functional/test_columns_in_relation.py

@@ -1,5 +1,3 @@
-import os
-
 from dbt.adapters.base import Column
 from dbt.tests.util import run_dbt, run_dbt_and_capture
 import pytest
@@ -47,6 +45,12 @@ def expected_columns(self):
         ]
 
 
+@pytest.mark.skip(
+    """
+    There is a discrepancy between our custom query and the get_columns SDK call.
+    This test should be skipped for now, but re-enabled once get_columns is implemented.
+"""
+)
 class TestColumnsInRelationBehaviorFlagOn(ColumnsInRelation):
     @pytest.fixture(scope="class")
     def project_config_update(self):
@@ -74,6 +78,12 @@ def expected_columns(self):
 """
 
 
+@pytest.mark.skip(
+    """
+    There is a discrepancy between our custom query and the get_columns SDK call.
+    This test should be skipped for now, but re-enabled once get_columns is implemented.
+"""
+)
 class TestBehaviorFlagFiresOnce:
     @pytest.fixture(scope="class")
     def project_config_update(self):