From de3f2de571dba76ba9ea9e4b53deb04be4ce79d2 Mon Sep 17 00:00:00 2001 From: michelley-an <101600153+michelley-an@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:03:12 -0700 Subject: [PATCH] testing #445 --- .github/workflows/ci_test_package.yml | 83 ++++++++++++++++++- .github/workflows/main_test_package.yml | 44 +++++++++- README.md | 2 +- dbt_project.yml | 2 +- integration_test_project/profiles.yml | 3 +- models/dim_dbt__current_models.sql | 2 + models/staging/stg_dbt__model_executions.sql | 1 + models/staging/stg_dbt__models.sql | 1 + models/staging/stg_dbt__seed_executions.sql | 1 + models/staging/stg_dbt__seeds.sql | 1 + .../staging/stg_dbt__snapshot_executions.sql | 1 + models/staging/stg_dbt__sources.sql | 1 + tox.ini | 18 ++-- 13 files changed, 148 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index 5eb359fd..6b30e14e 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -113,7 +113,7 @@ jobs: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: - warehouse: ["snowflake", "bigquery", "postgres", "sqlserver"] + warehouse: ["snowflake", "bigquery", "postgres"] # When supporting a new version, update the list here version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0", "1_8_0"] runs-on: ubuntu-latest @@ -193,3 +193,84 @@ jobs: # env: # DBT_VERSION: '' # run: tox -e integration_databricks + + integration-sqlserver: + strategy: + fail-fast: false # Don't fail one DWH if the others fail + runs-on: ubuntu-latest + environment: + name: Approve Integration Tests + + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.8.x" + architecture: "x64" + - name: Install SQL Server + uses: Particular/install-sql-server-action@v1.2.0 + with: + connection-string-env-var: SQL_SERVER_CONNECTION_STRING + catalog: dbt_artifact_integrationtests + - name: Create DBT User + shell: pwsh + run: | + echo "Create dbt login with sysadmin" + sqlcmd -Q "CREATE LOGIN dbt WITH PASSWORD = '123', CHECK_POLICY = OFF, CHECK_EXPIRATION = OFF" -d "dbt_artifact_integrationtests" + sqlcmd -Q "ALTER SERVER ROLE sysadmin ADD MEMBER dbt" -d "dbt_artifact_integrationtests" + - name: Install tox + run: python3 -m pip install tox + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR + + - name: Run Tests on PR + env: + DBT_VERSION: ${{ matrix.version }} + run: tox -e integration_sqlserver + + integration-sqlserver-single-run: + strategy: + fail-fast: false # Don't fail one DWH if the others fail + matrix: + # When supporting a new version, update the list here + version: ["1_3_0", "1_4_0", "1_7_0", "1_8_0"] + runs-on: ubuntu-latest + environment: + name: Approve Integration Tests + + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.8.x" + architecture: "x64" + - name: Install SQL Server + uses: Particular/install-sql-server-action@v1.2.0 + with: + connection-string-env-var: SQL_SERVER_CONNECTION_STRING + catalog: dbt_artifact_integrationtests + - name: Create DBT User + shell: pwsh + run: | + echo "Create dbt login with sysadmin" + sqlcmd -Q "CREATE LOGIN dbt WITH PASSWORD = '123', CHECK_POLICY = OFF, CHECK_EXPIRATION = OFF" -d "dbt_artifact_integrationtests" + sqlcmd -Q "ALTER SERVER ROLE sysadmin ADD MEMBER dbt" -d "dbt_artifact_integrationtests" + - name: Install tox + run: python3 -m pip install tox + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR + + - name: Run Tests on PR + env: + DBT_VERSION: ${{ matrix.version }} + run: tox -e integration_sqlserver_${{ matrix.version }} diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 1c55194e..6f0e8015 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -34,7 +34,7 @@ jobs: integration: strategy: matrix: - warehouse: ["snowflake", "bigquery", "postgres", "sqlserver"] + warehouse: ["snowflake", "bigquery", "postgres"] version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0", "1_8_0"] runs-on: ubuntu-latest permissions: @@ -102,3 +102,45 @@ jobs: # - name: Run Databricks Tests # run: tox -e integration_databricks + +integration-sqlserver: + strategy: + fail-fast: false # Don't fail one DWH if the others fail + matrix: + # When supporting a new version, update the list here + version: ["1_3_0", "1_4_0", "1_7_0", "1_8_0"] + runs-on: ubuntu-latest + environment: + name: Approve Integration Tests + + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.8.x" + architecture: "x64" + - name: Install SQL Server + uses: Particular/install-sql-server-action@v1.2.0 + with: + connection-string-env-var: SQL_SERVER_CONNECTION_STRING + catalog: dbt_artifact_integrationtests + - name: Create DBT User + shell: pwsh + run: | + echo "Create dbt login with sysadmin" + sqlcmd -Q "CREATE LOGIN dbt WITH PASSWORD = '123', CHECK_POLICY = OFF, CHECK_EXPIRATION = OFF" -d "dbt_artifact_integrationtests" + sqlcmd -Q "ALTER SERVER ROLE sysadmin ADD MEMBER dbt" -d "dbt_artifact_integrationtests" + - name: Install tox + run: python3 -m pip install tox + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR + + - name: Run Tests on PR + env: + DBT_VERSION: ${{ matrix.version }} + run: tox -e integration_sqlserver_${{ matrix.version }} diff --git a/README.md b/README.md index 4569b8ed..2a575c78 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ See the generated [dbt docs site](https://brooklyn-data.github.io/dbt_artifacts/ ``` packages: - package: brooklyn-data/dbt_artifacts - version: 2.6.4 + version: 2.7.1 ``` :construction_worker: Make sure to fix at least the **minor** version, to avoid issues when a new release is open. See the notes on upgrading below for more detail. diff --git a/dbt_project.yml b/dbt_project.yml index 4c05e3ae..9cba8340 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: "dbt_artifacts" -version: "2.6.4" +version: "2.7.1" config-version: 2 require-dbt-version: [">=1.3.0", "<1.9.0"] profile: "dbt_artifacts" diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index 33a1117a..fee862ad 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -61,5 +61,6 @@ dbt_artifacts: schema: dbo windows_login: False trust_cert: True - user: sa + Encrypt: False + user: dbt password: "123" diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index 4e6ceb1b..d94aa9b2 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -1,6 +1,8 @@ with base as (select * from {{ ref("stg_dbt__models") }}), + model_executions as (select * from {{ ref("stg_dbt__model_executions") }}), + latest_models as ( /* Retrieves the models present in the most recent run */ diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index ad2c4f50..c67f5551 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("model_executions") }}), + enhanced as ( select diff --git a/models/staging/stg_dbt__models.sql b/models/staging/stg_dbt__models.sql index d8184183..d90cdf71 100644 --- a/models/staging/stg_dbt__models.sql +++ b/models/staging/stg_dbt__models.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("models") }}), + enhanced as ( select diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql index 90f52870..9a6e5582 100644 --- a/models/staging/stg_dbt__seed_executions.sql +++ b/models/staging/stg_dbt__seed_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("seed_executions") }}), + enhanced as ( select diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql index 0e5dcb1d..4c68ac7a 100644 --- a/models/staging/stg_dbt__seeds.sql +++ b/models/staging/stg_dbt__seeds.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("seeds") }}), + enhanced as ( select diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index 8794b99b..15678a3b 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("snapshot_executions") }}), + enhanced as ( select diff --git a/models/staging/stg_dbt__sources.sql b/models/staging/stg_dbt__sources.sql index 1bd8cef1..86e83c33 100644 --- a/models/staging/stg_dbt__sources.sql +++ b/models/staging/stg_dbt__sources.sql @@ -1,5 +1,6 @@ with base as (select * from {{ ref("sources") }}), + enhanced as ( select diff --git a/tox.ini b/tox.ini index 50e79ce7..95c276df 100644 --- a/tox.ini +++ b/tox.ini @@ -355,32 +355,36 @@ commands = [testenv:integration_sqlserver] changedir = integration_test_project -deps = dbt-sqlserver~=1.8.4 +deps = dbt-sqlserver~=1.8.0 commands = dbt clean dbt deps dbt build --target sqlserver -[testenv:integration_sqlserver_1_4_3] +[testenv:integration_sqlserver_1_3_0] changedir = integration_test_project -deps = dbt-sqlserver~=1.4.3 +deps = dbt-sqlserver~=1.3.0 commands = dbt clean dbt deps dbt build --target sqlserver -[testenv:integration_sqlserver_1_7_4] +[testenv:integration_sqlserver_1_4_0] changedir = integration_test_project -deps = dbt-sqlserver~=1.7.4 +deps = dbt-sqlserver~=1.4.0 commands = dbt clean dbt deps dbt build --target sqlserver -[testenv:integration_sqlserver_1_8_4] +[testenv:integration_sqlserver_1_7_0] changedir = integration_test_project -deps = dbt-sqlserver~=1.8.4 +deps = dbt-sqlserver~=1.7.0 commands = dbt clean dbt deps dbt build --target sqlserver + +[testenv:integration_sqlserver_1_8_0] +changedir = integration_test_project +deps = dbt-sqlserver~=1.8.0