Merge pull request #35 from fivetran/feature/databricks-compatibility

Feature: Databricks compatibility
fivetran · Feb 8, 2023 · b973cfa · b973cfa
2 parents 27ba1ee + 7767922
commit b973cfa
Show file tree

Hide file tree

Showing 9 changed files with 44 additions and 7 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -57,3 +57,17 @@ steps:
             - "CI_REDSHIFT_DBT_USER"
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
+
+  - label: ":databricks: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+# dbt_jira_source v0.6.1
+## 🎉 Feature Updates 🎉
+- Databricks compatibility 🧱 ([#35](https://github.com/fivetran/dbt_jira_source/pull/35))
+
 # dbt_jira_source v0.6.0
 
 ## 🚨 Breaking Changes 🚨:

diff --git a/README.md b/README.md
@@ -23,7 +23,15 @@
 ## Step 1: Prerequisites
 To use this dbt package, you must have the following:
 - At least one Fivetran Jira connector syncing data into your destination. 
-- A **BigQuery**, **Snowflake**, **Redshift**, or **PostgreSQL** destination.
+- A **BigQuery**, **Snowflake**, **Redshift**, **PostgreSQL**, or **Databricks** destination.
+
+### Databricks Dispatch Configuration
+If you are using a Databricks destination with this package you will need to add the below (or a variation of the below) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively.
+```yml
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']
+```
 
 ## Step 2: Install the package
 Include the following jira_source package version in your `packages.yml` file.
@@ -92,6 +100,9 @@ packages:
 
     - package: dbt-labs/dbt_utils
       version: [">=1.0.0", "<2.0.0"]
+
+    - package: dbt-labs/spark_utils
+      version: [">=0.3.0", "<0.4.0"]
 ```
 
 # 🙌 How is this package maintained and can I contribute?

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'jira_source'
-version: '0.6.0'
+version: '0.6.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,10 +1,11 @@
 name: 'jira_source_integration_tests'
-version: '0.6.0'
+version: '0.6.1'
 config-version: 2
 profile: 'integration_tests'
 
 
 vars:
+  jira_schema: jira_source_integrations_tests
   jira_source:
     jira_comment_identifier: "comment"
     jira_component_identifier: "component"
@@ -75,4 +76,8 @@ seeds:
         start_date: timestamp
     field: 
       +column_types:
-        id: "{{ 'string' if target.name in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+        id: "{{ 'string' if target.name in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']
diff --git a/packages.yml b/packages.yml
@@ -1,3 +1,6 @@
 packages:
 - package: fivetran/fivetran_utils
   version: [">=0.4.0", "<0.5.0"]
+
+- package: dbt-labs/spark_utils
+  version: [">=0.3.0", "<0.4.0"]