From dc3e255a9850bcb56745f6a87d370bdb53462f71 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 22:03:25 +0100 Subject: [PATCH 01/13] Updating package dependendcies --- packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages.yml b/packages.yml index 10a32bb0..e5651462 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - - package: fivetran/stripe_source - version: [">=0.3.0", "<0.4.0"] +- package: fivetran/stripe_source + version: [">=0.4.0", "<0.5.0"] From 3f8df1a1597598970cb606fe19e3e66bafd19059 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 22:03:26 +0100 Subject: [PATCH 02/13] Updating require-dbt-version --- dbt_project.yml | 63 ++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 0bef92c8..e49be7af 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,39 +1,32 @@ config-version: 2 - name: 'stripe' -version: '0.4.2' - -require-dbt-version: [">=0.18.0", "<0.20.0"] - +version: '0.5.0' +require-dbt-version: [">=0.20.0"] models: - stripe: - +schema: stripe - +materialized: table - intermediate: - +materialized: ephemeral - + stripe: + +schema: stripe + +materialized: table + intermediate: + +materialized: ephemeral vars: - stripe: - balance_transaction: "{{ ref('stg_stripe__balance_transaction') }}" - card: "{{ ref('stg_stripe__card') }}" - charge: "{{ ref('stg_stripe__charge') }}" - customer: "{{ ref('stg_stripe__customer') }}" - fee: "{{ ref('stg_stripe__fee') }}" - invoice: "{{ ref('stg_stripe__invoice') }}" - invoice_line_item: "{{ ref('stg_stripe__invoice_line_item') }}" - payment_intent: "{{ ref('stg_stripe__payment_intent') }}" - payment_method_card: "{{ ref('stg_stripe__payment_method_card') }}" - payment_method: "{{ ref('stg_stripe__payment_method') }}" - payout: "{{ ref('stg_stripe__payout') }}" - plan: "{{ ref('stg_stripe__plan') }}" - refund: "{{ ref('stg_stripe__refund') }}" - subscription: "{{ ref('stg_stripe__subscription') }}" - - #Variables to enable or disable models if you do not have the respective table. - using_invoices: True - using_payment_method: True - using_subscriptions: True - - dbt_utils_dispatch_list: - - fivetran_utils - - dbt_utils \ No newline at end of file + stripe: + balance_transaction: "{{ ref('stg_stripe__balance_transaction') }}" + card: "{{ ref('stg_stripe__card') }}" + charge: "{{ ref('stg_stripe__charge') }}" + customer: "{{ ref('stg_stripe__customer') }}" + fee: "{{ ref('stg_stripe__fee') }}" + invoice: "{{ ref('stg_stripe__invoice') }}" + invoice_line_item: "{{ ref('stg_stripe__invoice_line_item') }}" + payment_intent: "{{ ref('stg_stripe__payment_intent') }}" + payment_method_card: "{{ ref('stg_stripe__payment_method_card') }}" + payment_method: "{{ ref('stg_stripe__payment_method') }}" + payout: "{{ ref('stg_stripe__payout') }}" + plan: "{{ ref('stg_stripe__plan') }}" + refund: "{{ ref('stg_stripe__refund') }}" + subscription: "{{ ref('stg_stripe__subscription') }}" + using_invoices: true + using_payment_method: true + using_subscriptions: true + dbt_utils_dispatch_list: + - fivetran_utils + - dbt_utils From b724d05743b348bd22c53b16587fb00e48b7ed2a Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 22:03:26 +0100 Subject: [PATCH 03/13] Updating dbt version in requirements.txt --- integration_tests/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 38e8ae6c..b06ff520 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1 +1,3 @@ -dbt==0.19.0 \ No newline at end of file +dbt==0.20.0 +dbt-spark==0.20.0 +dbt-spark[PyHive]==0.20.0 From ec538f7f6df8dc65381ec5bf8369eb563a546fc3 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 22:04:46 +0100 Subject: [PATCH 04/13] Update config.yml --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index db26bab5..43ef9f4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,6 +15,7 @@ jobs: - run: name: "Setup dbt" command: | + sudo apt install libsasl2-dev python3 -m venv venv . venv/bin/activate pip install --upgrade pip setuptools @@ -60,4 +61,4 @@ jobs: - save_cache: key: deps2-{{ .Branch }} paths: - - "venv" \ No newline at end of file + - "venv" From 1ccc55028731569c6adbbe489ad225cdbc724167 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 22:04:59 +0100 Subject: [PATCH 05/13] Update dbt_project.yml --- integration_tests/dbt_project.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index ae06f45d..60efd358 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -2,8 +2,6 @@ config-version: 2 name: 'stripe_integration_tests' version: '0.4.2' - -require-dbt-version: [">=0.18.0", "<0.20.0"] profile: 'integration_tests' vars: @@ -68,4 +66,4 @@ seeds: payout_data: plan_data: refund_data: - subscription_data: \ No newline at end of file + subscription_data: From 1b463ee3a4d093d0b684c8f7ea3530fdb64b6502 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Tue, 27 Jul 2021 10:46:27 -0500 Subject: [PATCH 06/13] dbt 20 readme and minor updates --- README.md | 11 ++++++++++- dbt_project.yml | 3 --- integration_tests/dbt_project.yml | 13 ++----------- packages.yml | 4 ++-- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 612f2894..8c24af7a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![dbt logo and version](https://img.shields.io/static/v1?logo=dbt&label=dbt-version&message=0.20.x&color=orange) # Stripe This package models Stripe data from [Fivetran's connector](https://fivetran.com/docs/applications/stripe). It uses data in the format described by [this ERD](https://fivetran.com/docs/applications/stripe#schemainformation). @@ -22,7 +23,15 @@ This package contains transformation models, designed to work simultaneously wit ## Installation Instructions -Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions, or [read the docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. +Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions, or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. + +Include in your `packages.yml` + +```yaml +packages: + - package: fivetran/stripe + version: [">=0.4.0", "<0.5.0"] +``` ## Configuration diff --git a/dbt_project.yml b/dbt_project.yml index e49be7af..0e111262 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -27,6 +27,3 @@ vars: using_invoices: true using_payment_method: true using_subscriptions: true - dbt_utils_dispatch_list: - - fivetran_utils - - dbt_utils diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 60efd358..d12e34b7 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'stripe_integration_tests' -version: '0.4.2' +version: '0.5.0' profile: 'integration_tests' vars: @@ -57,13 +57,4 @@ seeds: customer_data: +column_types: description: "{{ 'varchar(500)' if target.type == 'redshift' else 'string'}}" - fee_data: - invoice_data: - invoice_line_item_data: - payment_intent_data: - payment_method_card_data: - payment_method_data: - payout_data: - plan_data: - refund_data: - subscription_data: + diff --git a/packages.yml b/packages.yml index e5651462..45574616 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: -- package: fivetran/stripe_source - version: [">=0.4.0", "<0.5.0"] + - package: fivetran/stripe_source + version: [">=0.4.0", "<0.5.0"] From 8006c5838a4f364f80bd9295e7d9d0ce8ce667b2 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Tue, 27 Jul 2021 10:53:46 -0500 Subject: [PATCH 07/13] integration testing quote columns fix --- integration_tests/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index d12e34b7..ec113576 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -40,8 +40,8 @@ vars: subscription: "{{ ref('subscription_data') }}" seeds: - +quote_columns: "{{ true if target.type == 'redshift' else false }}" stripe_integration_tests: + +quote_columns: "{{ true if target.type == 'redshift' else false }}" +column_types: _fivetran_synced: timestamp group_data: From 8256d9a5d8b95e0adea8d2c1594496a1f5369f99 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Tue, 27 Jul 2021 11:01:40 -0500 Subject: [PATCH 08/13] circleCi updates --- dbt_project.yml | 4 ++++ integration_tests/dbt_project.yml | 6 ++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 0e111262..771690a1 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -27,3 +27,7 @@ vars: using_invoices: true using_payment_method: true using_subscriptions: true + + dbt_utils_dispatch_list: + - fivetran_utils + - dbt_utils diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index ec113576..ba9cc909 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -44,17 +44,15 @@ seeds: +quote_columns: "{{ true if target.type == 'redshift' else false }}" +column_types: _fivetran_synced: timestamp - group_data: balance_transaction_data: +column_types: created: timestamp available_on: timestamp - card_data: charge_data: +column_types: - balance_transaction_id: "{{ 'varchar(250)' if target.type == 'redshift' else 'string'}}" + balance_transaction_id: "{{ 'varchar(250)' if target.type == 'redshift' else 'string'}}" created: timestamp customer_data: +column_types: - description: "{{ 'varchar(500)' if target.type == 'redshift' else 'string'}}" + description: "{{ 'varchar(500)' if target.type == 'redshift' else 'string'}}" From ab8da034aff92d13c75dd268eb9d380a483c0f97 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Tue, 27 Jul 2021 11:08:34 -0500 Subject: [PATCH 09/13] seed removal for testing --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 43ef9f4e..157325ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,6 @@ jobs: echo `pwd` cd integration_tests dbt deps - dbt seed --target redshift --full-refresh dbt run --target redshift --full-refresh dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York"}' --target redshift dbt test --target redshift From 70a6b51e608c3677dd2c132c333f24151ee0752a Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 27 Jul 2021 16:00:39 -0500 Subject: [PATCH 10/13] Apply suggestions from code review --- dbt_project.yml | 2 +- integration_tests/requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 771690a1..4999a80b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'stripe' version: '0.5.0' -require-dbt-version: [">=0.20.0"] +require-dbt-version: [">=0.19.0"] models: stripe: +schema: stripe diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index b06ff520..2a8cfad6 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt==0.20.0 -dbt-spark==0.20.0 -dbt-spark[PyHive]==0.20.0 +dbt==0.19.0 +dbt-spark==0.19.0 +dbt-spark[PyHive]==0.19.0 From 5120379066fe1b1d946dc328f124ae2ac3473a8f Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Wed, 28 Jul 2021 11:09:31 +0100 Subject: [PATCH 11/13] fix version and dispatch error --- .circleci/config.yml | 1 + integration_tests/requirements.txt | 6 +++--- macros/date_timezone.sql | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 157325ea..43ef9f4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,6 +29,7 @@ jobs: echo `pwd` cd integration_tests dbt deps + dbt seed --target redshift --full-refresh dbt run --target redshift --full-refresh dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York"}' --target redshift dbt test --target redshift diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 2a8cfad6..b06ff520 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt==0.19.0 -dbt-spark==0.19.0 -dbt-spark[PyHive]==0.19.0 +dbt==0.20.0 +dbt-spark==0.20.0 +dbt-spark[PyHive]==0.20.0 diff --git a/macros/date_timezone.sql b/macros/date_timezone.sql index d8b14617..e9d23ed3 100644 --- a/macros/date_timezone.sql +++ b/macros/date_timezone.sql @@ -1,6 +1,6 @@ {% macro date_timezone(column) -%} -{{ adapter.dispatch('date_timezone', packages=fivetran_utils._get_utils_namespaces()+['stripe'])(column) }} +{{ adapter.dispatch('date_timezone', 'stripe')(column) }} {%- endmacro %} From f3ac57b964a7f75181638226cbc9bb09f59b5d09 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Wed, 28 Jul 2021 09:11:07 -0500 Subject: [PATCH 12/13] minor version upgrade --- README.md | 2 +- dbt_project.yml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index 8c24af7a..0341cdb2 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Include in your `packages.yml` ```yaml packages: - package: fivetran/stripe - version: [">=0.4.0", "<0.5.0"] + version: [">=0.5.0", "<0.6.0"] ``` ## Configuration diff --git a/dbt_project.yml b/dbt_project.yml index 4999a80b..d6afe4c7 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -27,7 +27,3 @@ vars: using_invoices: true using_payment_method: true using_subscriptions: true - - dbt_utils_dispatch_list: - - fivetran_utils - - dbt_utils From 10998c100619b5e12e5bf74b559a0956a01865f8 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Wed, 28 Jul 2021 09:26:05 -0500 Subject: [PATCH 13/13] Apply suggestions from code review --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index d6afe4c7..0e111262 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'stripe' version: '0.5.0' -require-dbt-version: [">=0.19.0"] +require-dbt-version: [">=0.20.0"] models: stripe: +schema: stripe