Merge branch 'main' into retryInterfaceError

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.9.0a1
+current_version = 1.9.0b1
 parse = (?P<major>[\d]+) # major version number
 	\.(?P<minor>[\d]+) # minor version number
 	\.(?P<patch>[\d]+) # patch version number

.changes/1.9.0-b1.md

@@ -0,0 +1,42 @@
+## dbt-redshift 1.9.0-b1 - October 01, 2024
+
+### Features
+
+- Support IAM user auth via direct parameters, in addition to the existing profile method ([#760](https://github.com/dbt-labs/dbt-redshift/issues/760))
+- Add support for IAM Role auth ([#623](https://github.com/dbt-labs/dbt-redshift/issues/623))
+- Add tests for cross-database `cast` macro ([#804](https://github.com/dbt-labs/dbt-redshift/issues/804))
+- Cross-database `date` macro ([#808](https://github.com/dbt-labs/dbt-redshift/issues/808))
+- Remove `pg_catalog` from metadata queries ([#555](https://github.com/dbt-labs/dbt-redshift/issues/555))
+
+### Fixes
+
+- Stop adding aliases to subqueries when calling with `--empty` ([#782](https://github.com/dbt-labs/dbt-redshift/issues/782))
+- Support IAM Role authentication for Redshift Serverless ([#835](https://github.com/dbt-labs/dbt-redshift/issues/835))
+- Fix config change detection not working for multiple sortkey in materialized views ([#838](https://github.com/dbt-labs/dbt-redshift/issues/838))
+- update pin range for redshift-connector to allow 2.1.0 ([#844](https://github.com/dbt-labs/dbt-redshift/issues/844))
+- Handle unit test fixtures where typing goes wrong from first value in column being Null.  ([#821](https://github.com/dbt-labs/dbt-redshift/issues/821))
+- Fix materialized views comment syntax ([#837](https://github.com/dbt-labs/dbt-redshift/issues/837))
+- Support DROP CASCADE for materialized views; fixes bug that occurs when running dbt on materialized views that reference other materialized views ([#642](https://github.com/dbt-labs/dbt-redshift/issues/642))
+
+### Under the Hood
+
+- Lazy load agate ([#745](https://github.com/dbt-labs/dbt-redshift/issues/745))
+- Remove `freezegun` as a testing dependency; this package is no longer used ([#1136](https://github.com/dbt-labs/dbt-redshift/issues/1136))
+
+### Dependencies
+
+- Update freezegun requirement from ~=1.3 to ~=1.4 ([#695](https://github.com/dbt-labs/dbt-redshift/pull/695))
+- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#762](https://github.com/dbt-labs/dbt-redshift/pull/762))
+- Bump actions/checkout from 3 to 4 ([#802](https://github.com/dbt-labs/dbt-redshift/pull/802))
+- Bump pre-commit from 3.7.0 to 3.7.1 ([#867](https://github.com/dbt-labs/dbt-redshift/pull/867))
+- Update tox requirement from ~=4.11 to ~=4.16 ([#870](https://github.com/dbt-labs/dbt-redshift/pull/870))
+- Update twine requirement from ~=4.0 to ~=5.1 ([#876](https://github.com/dbt-labs/dbt-redshift/pull/876))
+
+### Contributors
+- [@dwreeves](https://github.com/dwreeves) ([#745](https://github.com/dbt-labs/dbt-redshift/issues/745))
+- [@fleid](https://github.com/fleid) ([#835](https://github.com/dbt-labs/dbt-redshift/issues/835))
+- [@jeremyyeo](https://github.com/jeremyyeo) ([#837](https://github.com/dbt-labs/dbt-redshift/issues/837))
+- [@jiezhen-chen](https://github.com/jiezhen-chen) ([#555](https://github.com/dbt-labs/dbt-redshift/issues/555))
+- [@lvitti](https://github.com/lvitti) ([#838](https://github.com/dbt-labs/dbt-redshift/issues/838))
+- [@mikealfare,](https://github.com/mikealfare,) ([#555](https://github.com/dbt-labs/dbt-redshift/issues/555))
+- [@mikealfare,abbywh](https://github.com/mikealfare,abbywh) ([#623](https://github.com/dbt-labs/dbt-redshift/issues/623))

.changes/1.9.0/Features-20240826-123954.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Remove `pg_catalog` from metadata queries
+time: 2024-08-26T12:39:54.481505-04:00
+custom:
+  Author: mikealfare, jiezhen-chen
+  Issue: "555"

.changes/1.9.0/Fixes-20240610-104114.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix config change detection not working for multiple sortkey in materialized views
+time: 2024-06-10T10:41:14.721411-03:00
+custom:
+    Author: lvitti
+    Issue: "838"

.changes/1.9.0/Fixes-20240715-143507.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix materialized views comment syntax
+time: 2024-07-15T14:35:07.412274+12:00
+custom:
+    Author: jeremyyeo
+    Issue: "837"

.changes/1.9.0/Fixes-20240906-102642.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Support DROP CASCADE for materialized views; fixes bug that occurs when running
+  dbt on materialized views that reference other materialized views
+time: 2024-09-06T10:26:42.501014-04:00
+custom:
+  Author: mikealfare
+  Issue: "642"

.changes/unreleased/Breaking Changes-20241016-190600.yaml

@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: Drop support for Python 3.8
+time: 2024-10-16T19:06:00.668396-04:00
+custom:
+  Author: mikealfare
+  Issue: "931"

.changes/unreleased/Fixes-20240920-184812.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Restrict behavior change warnings to firing once per run to avoid noisy logs
+time: 2024-09-20T18:48:12.064324-04:00
+custom:
+  Author: mikealfare
+  Issue: "915"

.github/workflows/integration.yml

@@ -91,7 +91,7 @@ jobs:
       max-parallel: 3
       matrix:
         os: [ubuntu-22.04, macos-12, windows-2022]
-        python-version: ["3.8"]
+        python-version: ["3.9"]
         include:
           - os: ubuntu-22.04
             python-version: "3.9"
@@ -236,7 +236,7 @@ jobs:
       fail-fast: false
       max-parallel: 1
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     env:
       TOXENV: integration-redshift

.github/workflows/main.yml

@@ -50,7 +50,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -70,7 +70,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
     env:
       TOXENV: "unit"
@@ -127,7 +127,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -174,7 +174,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-22.04, macos-12, windows-2022]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}

.github/workflows/release-internal.yml

@@ -28,7 +28,7 @@ defaults:
     shell: "bash"
 
 env:
-  PYTHON_TARGET_VERSION: 3.8
+  PYTHON_TARGET_VERSION: 3.9
   NOTIFICATION_PREFIX: "[Internal Archive Release]"
   TEMP_PROFILE_NAME: "temp_aws_profile"
 

.github/workflows/release-prep.yml

@@ -84,7 +84,7 @@ defaults:
     shell: bash
 
 env:
-  PYTHON_TARGET_VERSION: 3.8
+  PYTHON_TARGET_VERSION: 3.9
   NOTIFICATION_PREFIX: "[Release Preparation]"
 
 jobs:

.pre-commit-config.yaml

@@ -23,7 +23,6 @@ repos:
     -   id: black
         args:
         -   --line-length=99
-        -   --target-version=py38
         -   --target-version=py39
         -   --target-version=py310
         -   --target-version=py311

CHANGELOG.md

@@ -5,10 +5,52 @@
 - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
 - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md#adding-changelog-entry)
 
+## dbt-redshift 1.9.0-b1 - October 01, 2024
+
+### Features
+
+- Support IAM user auth via direct parameters, in addition to the existing profile method ([#760](https://github.com/dbt-labs/dbt-redshift/issues/760))
+- Add support for IAM Role auth ([#623](https://github.com/dbt-labs/dbt-redshift/issues/623))
+- Add tests for cross-database `cast` macro ([#804](https://github.com/dbt-labs/dbt-redshift/issues/804))
+- Cross-database `date` macro ([#808](https://github.com/dbt-labs/dbt-redshift/issues/808))
+- Remove `pg_catalog` from metadata queries ([#555](https://github.com/dbt-labs/dbt-redshift/issues/555))
+
+### Fixes
+
+- Stop adding aliases to subqueries when calling with `--empty` ([#782](https://github.com/dbt-labs/dbt-redshift/issues/782))
+- Support IAM Role authentication for Redshift Serverless ([#835](https://github.com/dbt-labs/dbt-redshift/issues/835))
+- Fix config change detection not working for multiple sortkey in materialized views ([#838](https://github.com/dbt-labs/dbt-redshift/issues/838))
+- update pin range for redshift-connector to allow 2.1.0 ([#844](https://github.com/dbt-labs/dbt-redshift/issues/844))
+- Handle unit test fixtures where typing goes wrong from first value in column being Null.  ([#821](https://github.com/dbt-labs/dbt-redshift/issues/821))
+- Fix materialized views comment syntax ([#837](https://github.com/dbt-labs/dbt-redshift/issues/837))
+- Support DROP CASCADE for materialized views; fixes bug that occurs when running dbt on materialized views that reference other materialized views ([#642](https://github.com/dbt-labs/dbt-redshift/issues/642))
+
+### Under the Hood
+
+- Lazy load agate ([#745](https://github.com/dbt-labs/dbt-redshift/issues/745))
+- Remove `freezegun` as a testing dependency; this package is no longer used ([#1136](https://github.com/dbt-labs/dbt-redshift/issues/1136))
+
+### Dependencies
+
+- Update freezegun requirement from ~=1.3 to ~=1.4 ([#695](https://github.com/dbt-labs/dbt-redshift/pull/695))
+- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#762](https://github.com/dbt-labs/dbt-redshift/pull/762))
+- Bump actions/checkout from 3 to 4 ([#802](https://github.com/dbt-labs/dbt-redshift/pull/802))
+- Bump pre-commit from 3.7.0 to 3.7.1 ([#867](https://github.com/dbt-labs/dbt-redshift/pull/867))
+- Update tox requirement from ~=4.11 to ~=4.16 ([#870](https://github.com/dbt-labs/dbt-redshift/pull/870))
+- Update twine requirement from ~=4.0 to ~=5.1 ([#876](https://github.com/dbt-labs/dbt-redshift/pull/876))
+
+### Contributors
+- [@dwreeves](https://github.com/dwreeves) ([#745](https://github.com/dbt-labs/dbt-redshift/issues/745))
+- [@fleid](https://github.com/fleid) ([#835](https://github.com/dbt-labs/dbt-redshift/issues/835))
+- [@jeremyyeo](https://github.com/jeremyyeo) ([#837](https://github.com/dbt-labs/dbt-redshift/issues/837))
+- [@jiezhen-chen](https://github.com/jiezhen-chen) ([#555](https://github.com/dbt-labs/dbt-redshift/issues/555))
+- [@lvitti](https://github.com/lvitti) ([#838](https://github.com/dbt-labs/dbt-redshift/issues/838))
+- [@mikealfare,](https://github.com/mikealfare,) ([#555](https://github.com/dbt-labs/dbt-redshift/issues/555))
+- [@mikealfare,abbywh](https://github.com/mikealfare,abbywh) ([#623](https://github.com/dbt-labs/dbt-redshift/issues/623))
+
+
 ## Previous Releases
 For information on prior major and minor releases, see their changelogs:
-- [1.8](https://github.com/dbt-labs/dbt-redshift/blob/1.8.latest/CHANGELOG.md)
-- [1.7](https://github.com/dbt-labs/dbt-redshift/blob/1.7.latest/CHANGELOG.md)
 - [1.6](https://github.com/dbt-labs/dbt-redshift/blob/1.6.latest/CHANGELOG.md)
 - [1.5](https://github.com/dbt-labs/dbt-redshift/blob/1.5.latest/CHANGELOG.md)
 - [1.4](https://github.com/dbt-labs/dbt-redshift/blob/1.4.latest/CHANGELOG.md)

CONTRIBUTING.md

@@ -70,7 +70,7 @@ $EDITOR test.env
 There are a few methods for running tests locally.
 
 #### `tox`
-`tox` takes care of managing Python virtualenvs and installing dependencies in order to run tests. You can also run tests in parallel. For example, you can run unit tests for Python 3.8, Python 3.9, Python 3.10, and `flake8` checks in parallel with `tox -p`. Also, you can run unit tests for specific python versions with `tox -e py38`. The configuration of these tests are located in `tox.ini`.
+`tox` takes care of managing Python virtualenvs and installing dependencies in order to run tests. You can also run tests in parallel. For example, you can run unit tests for Python 3.9 and Python 3.10, and `flake8` checks in parallel with `tox -p`. Also, you can run unit tests for specific python versions with `tox -e py39`. The configuration of these tests are located in `tox.ini`.
 
 #### `pytest`
 Finally, you can also run a specific test or group of tests using `pytest` directly. With a Python virtualenv active and dev dependencies installed you can do things like:

Makefile

@@ -38,22 +38,22 @@ linecheck: ## Checks for all Python lines 100 characters or more
 	find dbt -type f -name "*.py" -exec grep -I -r -n '.\{100\}' {} \;
 
 .PHONY: unit
-unit: ## Runs unit tests with py38.
+unit: ## Runs unit tests with py39.
 	@\
-	tox -e py38
+	tox -e py39
 
 .PHONY: test
 test: ## Runs unit tests with py38 and code checks against staged changes.
 	@\
-	tox -p -e py38; \
+	tox -p -e py39; \
 	pre-commit run black-check --hook-stage manual | grep -v "INFO"; \
 	pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \
 	pre-commit run mypy-check --hook-stage manual | grep -v "INFO"
 
 .PHONY: integration
 integration: ## Runs redshift integration tests with py38.
 	@\
-	tox -e py38-redshift --
+	tox -e py39-redshift --
 
 .PHONY: clean
 	@echo "cleaning repo"

dbt/adapters/redshift/__version__.py

@@ -1 +1 @@
-version = "1.9.0a1"
+version = "1.9.0b1"

dbt/adapters/redshift/connections.py

@@ -3,6 +3,7 @@
 from contextlib import contextmanager
 from typing import Any, Callable, Dict, Tuple, Union, Optional, List, TYPE_CHECKING
 from dataclasses import dataclass, field
+import time
 
 import sqlparse
 import redshift_connector
@@ -12,10 +13,14 @@
 from dbt.adapters.sql import SQLConnectionManager
 from dbt.adapters.contracts.connection import AdapterResponse, Connection, Credentials
 from dbt.adapters.events.logging import AdapterLogger
+from dbt.adapters.events.types import SQLQuery, SQLQueryStatus
 from dbt_common.contracts.util import Replaceable
 from dbt_common.dataclass_schema import dbtClassMixin, StrEnum, ValidationError
+from dbt_common.events.contextvars import get_node_info
+from dbt_common.events.functions import fire_event
 from dbt_common.helper_types import Port
 from dbt_common.exceptions import DbtRuntimeError, CompilationError, DbtDatabaseError
+from dbt_common.utils import cast_to_str
 
 if TYPE_CHECKING:
     # Indirectly imported via agate_helper, which is lazy loaded further downfile.
@@ -461,3 +466,51 @@ def _initialize_sqlparse_lexer():
 
         if hasattr(Lexer, "get_default_instance"):
             Lexer.get_default_instance()
+
+    def columns_in_relation(self, relation) -> List[Dict[str, Any]]:
+        connection = self.get_thread_connection()
+
+        fire_event(
+            SQLQuery(
+                conn_name=cast_to_str(connection.name),
+                sql=f"call redshift_connector.Connection.get_columns({relation.database}, {relation.schema}, {relation.identifier})",
+                node_info=get_node_info(),
+            )
+        )
+
+        pre = time.perf_counter()
+
+        cursor = connection.handle.cursor()
+        columns = cursor.get_columns(
+            catalog=relation.database,
+            schema_pattern=relation.schema,
+            tablename_pattern=relation.identifier,
+        )
+
+        fire_event(
+            SQLQueryStatus(
+                status=str(self.get_response(cursor)),
+                elapsed=time.perf_counter() - pre,
+                node_info=get_node_info(),
+            )
+        )
+
+        return [self._parse_column_results(column) for column in columns]
+
+    @staticmethod
+    def _parse_column_results(record: Tuple[Any, ...]) -> Dict[str, Any]:
+        _, _, _, column_name, dtype_code, dtype_name, column_size, _, decimals, *_ = record
+
+        char_dtypes = [1, 12]
+        num_dtypes = [2, 3, 4, 5, 6, 7, 8, -5, 2003]
+
+        if dtype_code in char_dtypes:
+            return {"column": column_name, "dtype": dtype_name, "char_size": column_size}
+        elif dtype_code in num_dtypes:
+            return {
+                "column": column_name,
+                "dtype": dtype_name,
+                "numeric_precision": column_size,
+                "numeric_scale": decimals,
+            }
+        return {"column": column_name, "dtype": dtype_name, "char_size": column_size}

dbt/adapters/redshift/impl.py

@@ -1,9 +1,12 @@
 import os
 from dataclasses import dataclass
+
+from dbt_common.behavior_flags import BehaviorFlag
 from dbt_common.contracts.constraints import ConstraintType
-from typing import Optional, Set, Any, Dict, Type, TYPE_CHECKING
+from typing import Optional, Set, Any, Dict, Type, TYPE_CHECKING, List
 from collections import namedtuple
 from dbt.adapters.base import PythonJobHelper
+from dbt.adapters.base.column import Column
 from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport
 from dbt.adapters.base.meta import available
 from dbt.adapters.capability import Capability, CapabilityDict, CapabilitySupport, Support
@@ -65,6 +68,23 @@ class RedshiftAdapter(SQLAdapter):
         }
     )
 
+    @property
+    def _behavior_flags(self) -> List[BehaviorFlag]:
+        return [
+            {
+                "name": "restrict_direct_pg_catalog_access",
+                "default": False,
+                "description": (
+                    "The dbt-redshift adapter is migrating from using pg_ tables "
+                    "to using Redshift Metadata API and information_schema tables "
+                    "in order to support additional Redshift functionalities.\n"
+                    "We do not expect this to impact your dbt experience. "
+                    "Please report any issues using this GitHub discussion: https://github.com/dbt-labs/dbt-redshift/discussions/921"
+                ),
+                "docs_url": "https://docs.getdbt.com/reference/global-configs/behavior-changes#redshift-restrict_direct_pg_catalog_access",
+            }
+        ]
+
     @classmethod
     def date_function(cls):
         return "getdate()"
@@ -87,6 +107,12 @@ def drop_relation(self, relation):
         with self.connections.fresh_transaction():
             return super().drop_relation(relation)
 
+    def get_columns_in_relation(self, relation) -> List[Column]:
+        if self.behavior.restrict_direct_pg_catalog_access:
+            column_configs = self.connections.columns_in_relation(relation)
+            return [Column(**column) for column in column_configs]
+        return super().get_columns_in_relation(relation)
+
     @classmethod
     def convert_text_type(cls, agate_table: "agate.Table", col_idx):
         column = agate_table.columns[col_idx]