Merge pull request #39 from fivetran/feature/performance-enhancement

feature/performance-enhancement

.buildkite/hooks/pre-command

@@ -21,4 +21,7 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_SQL_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_SQL_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_TOKEN" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -69,5 +69,21 @@ steps:
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
       bash .buildkite/scripts/run_models.sh databricks
+
+  - label: ":databricks: :database: Run Tests - Databricks SQL Warehouse"
+    key: "run_dbt_databricks_sql"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_SQL_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_SQL_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks-sql

.buildkite/scripts/run_models.sh

@@ -16,9 +16,22 @@ db=$1
 echo `pwd`
 cd integration_tests
 dbt deps
+
+if [ "$db" = "databricks-sql" ]; then
+dbt seed --vars '{jira_schema: jira_source_integrations_tests_sqlw}' --target "$db" --full-refresh
+dbt compile --vars '{jira_schema: jira_source_integrations_tests_sqlw}' --target "$db"
+dbt run --vars '{jira_schema: jira_source_integrations_tests_sqlw}' --target "$db" --full-refresh
+dbt test --vars '{jira_schema: jira_source_integrations_tests_sqlw}' --target "$db"
+dbt run --vars '{jira_schema: jira_source_integrations_tests_sqlw, jira_using_priorities: false, jira_using_sprints: false, jira_using_components: false, jira_using_versions: false, jira_field_grain: 'field_name'}' --target "$db" --full-refresh
+dbt test --vars '{jira_schema: jira_source_integrations_tests_sqlw, jira_using_priorities: false, jira_using_sprints: false, jira_using_components: false, jira_using_versions: false, jira_field_grain: 'field_name'}' --target "$db"
+
+else
 dbt seed --target "$db" --full-refresh
+dbt compile --target "$db"
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{jira_using_sprints: false, jira_using_components: false, jira_using_versions: false}' --target "$db" --full-refresh
-dbt test --target "$db"
-dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
+dbt run --vars '{jira_using_priorities: false, jira_using_sprints: false, jira_using_components: false, jira_using_versions: false, jira_field_grain: 'field_name'}' --target "$db" --full-refresh
+dbt test --vars '{jira_using_priorities: false, jira_using_sprints: false, jira_using_components: false, jira_using_versions: false, jira_field_grain: 'field_name'}' --target "$db"
+fi
+
+dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"

.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md

@@ -4,48 +4,27 @@
 **This PR will result in the following new package version:**
 <!--- Please add details around your decision for breaking vs non-breaking version upgrade. If this is a breaking change, were backwards-compatible options explored? -->
 
-**Please detail what change(s) this PR introduces and any additional information that should be known during the review of this PR:**
+**Please provide the finalized CHANGELOG entry which details the relevant changes included in this PR:**
+<!--- Copy/paste the CHANGELOG for this version below. -->
 
 ## PR Checklist
 ### Basic Validation
 Please acknowledge that you have successfully performed the following commands locally:
-- [ ] dbt compile
-- [ ] dbt run –full-refresh
-- [ ] dbt run
-- [ ] dbt test
-- [ ] dbt run –vars (if applicable)
+- [ ] dbt run –full-refresh && dbt test
+- [ ] dbt run (if incremental models are present) && dbt test
 
 Before marking this PR as "ready for review" the following have been applied:
-- [ ] The appropriate issue has been linked and tagged
-- [ ] You are assigned to the corresponding issue and this PR
-- [ ] BuildKite integration tests are passing
+- [ ] The appropriate issue has been linked, tagged, and properly assigned.
+- [ ] All necessary documentation and version upgrades have been applied.
+    <!--- Be sure to update the package version in the dbt_project.yml, integration_tests/dbt_project.yml, and README if necessary. -->
+- [ ] docs were regenerated (unless this PR does not include any code or yml updates).
+- [ ] BuildKite integration tests are passing.
+- [ ] Detailed validation steps have been provided below.
 
 ### Detailed Validation
-Please acknowledge that the following validation checks have been performed prior to marking this PR as "ready for review":
-- [ ] You have validated these changes and assure this PR will address the respective Issue/Feature.
-- [ ] You are reasonably confident these changes will not impact any other components of this package or any dependent packages.
-- [ ] You have provided details below around the validation steps performed to gain confidence in these changes.
+Please share any and all of your validation steps:
 <!--- Provide the steps you took to validate your changes below. -->
 
-### Standard Updates
-Please acknowledge that your PR contains the following standard updates:
-- Package versioning has been appropriately indexed in the following locations:
-    - [ ] indexed within dbt_project.yml
-    - [ ] indexed within integration_tests/dbt_project.yml
-- [ ] CHANGELOG has individual entries for each respective change in this PR
-    <!--- If there is a parallel upstream change, remember to reference the corresponding CHANGELOG as an individual entry.  -->
-- [ ] README updates have been applied (if applicable)
-    <!--- Remember to check the following README locations for common updates. →
-        <!--- Suggested install range (needed for breaking changes) →
-        <!--- Dependency matrix is appropriately updated (if applicable) →
-        <!--- New variable documentation (if applicable) -->
-- [ ] DECISIONLOG updates have been updated (if applicable)
-- [ ] Appropriate yml documentation has been added (if applicable)
-
-### dbt Docs
-Please acknowledge that after the above were all completed the below were applied to your branch:
-- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
-
 ### If you had to summarize this PR in an emoji, which would it be?
 <!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
-:dancer:
+:dancer:

.github/workflows/auto-release.yml

@@ -0,0 +1,13 @@
+name: 'auto release'
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  call-workflow-passing-data:
+    if: github.event.pull_request.merged
+    uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main
+    secrets: inherit

CHANGELOG.md

@@ -1,9 +1,18 @@
-# dbt_jira_source v0.UPDATE.UPDATE
-
- ## Under the Hood:
+# dbt_jira_source v0.7.0
+[PR #39](https://github.com/fivetran/dbt_jira_source/pull/39) introduces the following changes: 
+## 🚨 Breaking Changes 🚨
+- To reduce storage, updated default materialization of staging models to views. 
+>  ⚠️ Running a `--full-refresh` will be required if you have previously run these staging models as tables and get the following error: 
+> ```
+> Trying to create view <model path> but it currently exists as a table. Either drop <model path> manually, or run dbt with `--full-refresh` and dbt will drop it for you.
+> ```
 
+## Under the Hood:
+- Added integration testing pipeline for Databricks SQL Warehouse.
+- Included auto-releaser GitHub Actions workflow to automate future releases.
 - Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job.
-- Updated the pull request [templates](/.github).
+- Updated the maintainer pull request [template](https://github.com/fivetran/dbt_jira_source/tree/main/.github/PULL_REQUEST_TEMPLATE).
+
 # dbt_jira_source v0.6.1
 ## 🎉 Feature Updates 🎉
 - Databricks compatibility 🧱 ([#35](https://github.com/fivetran/dbt_jira_source/pull/35))

README.md

@@ -44,7 +44,7 @@ Include the following jira_source package version in your `packages.yml` file.
 ```yaml
 packages:
   - package: fivetran/jira_source
-    version: [">=0.6.0", "<0.7.0"]
+    version: [">=0.7.0", "<0.8.0"]
 ```
 ## Step 3: Define database and schema variables
 By default, this package runs using your destination and the `jira` schema. If this is not where your Jira data is (for example, if your Jira schema is named `jira_fivetran`), add the following configuration to your root `dbt_project.yml` file:

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'jira_source'
-version: '0.6.1'
+version: '0.7.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:
@@ -23,7 +23,5 @@ vars:
     version: "{{ source('jira', 'version') }}"
 models:
   jira_source:
-    tmp:
-      +materialized: view
-    +materialized: table
+    +materialized: view
     +schema: jira_source

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: jira_source_integrations_tests
+      schema: jira_source_integration_tests_02
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: jira_source_integrations_tests
+      schema: jira_source_integration_tests_02
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: jira_source_integrations_tests
+      schema: jira_source_integration_tests_02
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,21 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: jira_source_integrations_tests
+      schema: jira_source_integration_tests_02
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: jira_source_integrations_tests
-      threads: 2
+      schema: jira_source_integration_tests_02
+      threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
+      type: databricks
+    databricks-sql:
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
+      host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
+      http_path: "{{ env_var('CI_DATABRICKS_SQL_DBT_HTTP_PATH') }}"
+      schema: jira_source_integrations_tests_sqlw 
+      threads: 8
+      token: "{{ env_var('CI_DATABRICKS_SQL_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,11 +1,11 @@
 name: 'jira_source_integration_tests'
-version: '0.6.1'
+version: '0.7.0'
 config-version: 2
 profile: 'integration_tests'
 
 
 vars:
-  jira_schema: jira_source_integrations_tests
+  jira_schema: jira_source_integration_tests_02
   jira_source:
     jira_comment_identifier: "comment"
     jira_component_identifier: "component"
@@ -29,6 +29,10 @@ vars:
     jira_user_identifier: "user"
     jira_version_identifier: "version"
 
+models:
+  jira_source:
+    +schema: "{{ 'jira_source_integrations_tests_sqlw' if target.name == 'databricks-sql' else 'jira_source' }}"
+
 seeds:
   jira_source_integration_tests:
     +column_types:
@@ -55,17 +59,17 @@ seeds:
         created: timestamp
         work_ratio: float
         resolved: timestamp
-        assignee: "{{ 'string' if target.name in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+        assignee: "{{ 'string' if target.name in ('bigquery', 'spark', 'databricks', 'databricks-sql') else 'varchar' }}"
     issue_multiselect_history:
       +column_types:
         time: timestamp
         issue_id: "{{ 'int64' if target.name == 'bigquery' else 'bigint' }}"
-        value: "{{ 'string' if target.name in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+        value: "{{ 'string' if target.name in ('bigquery', 'spark', 'databricks', 'databricks-sql') else 'varchar' }}"
     issue_field_history:
       +column_types:
         time: timestamp
         issue_id: "{{ 'int64' if target.name == 'bigquery' else 'bigint' }}"
-        value: "{{ 'string' if target.name in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+        value: "{{ 'string' if target.name in ('bigquery', 'spark', 'databricks', 'databricks-sql') else 'varchar' }}"
     sprint: 
       +column_types:
         id: "{{ 'int64' if target.name == 'bigquery' else 'bigint' }}"
@@ -74,7 +78,7 @@ seeds:
         start_date: timestamp
     field: 
       +column_types:
-        id: "{{ 'string' if target.name in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+        id: "{{ 'string' if target.name in ('bigquery', 'spark', 'databricks', 'databricks-sql') else 'varchar' }}"
 
 dispatch:
   - macro_namespace: dbt_utils

integration_tests/requirements.txt

@@ -1,6 +1,6 @@
 dbt-snowflake>=1.3.0,<2.0.0
 dbt-bigquery>=1.3.0,<2.0.0
-dbt-redshift>=1.3.0,<1.5.0
+dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0