Merge pull request #18 from fivetran/feature/date-timezones

Incorporate Stripe timezones in reporting
fivetran · Jun 11, 2021 · cb6edfe · cb6edfe
2 parents 7339973 + 56f6d0a
commit cb6edfe
Show file tree

Hide file tree

Showing 7 changed files with 78 additions and 28 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -30,7 +30,7 @@ jobs:
             dbt deps
             dbt seed --target redshift --full-refresh
             dbt run --target redshift --full-refresh
-            dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false}' --target redshift
+            dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York"}' --target redshift
             dbt test --target redshift
       - run:
           name: "Run Tests - Snowflake"
@@ -41,7 +41,7 @@ jobs:
             dbt deps
             dbt seed --target snowflake --full-refresh
             dbt run --target snowflake --full-refresh
-            dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false}' --target snowflake
+            dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York"}' --target snowflake
             dbt test --target snowflake
       - run:
           name: "Run Tests - BigQuery"
@@ -55,7 +55,7 @@ jobs:
             dbt deps
             dbt seed --target bigquery --full-refresh
             dbt run --target bigquery --full-refresh
-            dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false}' --target bigquery
+            dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York"}' --target bigquery
             dbt test --target bigquery
       - save_cache:
           key: deps2-{{ .Branch }}

diff --git a/README.md b/README.md
@@ -70,6 +70,20 @@ models:
 
 *Read more about using custom schemas in dbt [here](https://docs.getdbt.com/docs/building-a-dbt-project/building-models/using-custom-schemas).*
 
+### Setting your timezone
+
+This packages leaves all timestamp columns in the UTC timezone. However, there are certain instances, such in the daily overview model, that timestamps have to be converted to dates. As a result, the timezone used for the timestamp becomes relevant. 
+
+By default, this package will use the UTC timezone when converting to date, but if you want to set the timezone to the time in your Stripe reports, add the following configuration to your `dbt_project.yml`:
+
+```yml
+# dbt_project.yml
+
+...
+vars:
+  stripe_timezone: "America/New_York" # Replace with your timezone
+```
+
 ### Contributions
 
 Additional contributions to this package are very welcome! Please create issues

diff --git a/integration_tests/data/balance_transaction_data.csv b/integration_tests/data/balance_transaction_data.csv
@@ -1,10 +1,10 @@
 id,_fivetran_synced,amount,available_on,connected_account_id,created,currency,description,exchange_rate,fee,net,source,status,type,payout_id
-txn_0IV4rIig6peNXzgDK3XAFbaN,2021-03-15 02:21:18,-18759,2021-03-16 0:00:00,,2021-03-15 01:04:36,usd,f39657a1847db73d9a2abb0bad3d31ae,,0,-18759,po_0IV4rIig6peNXzgD51VFKdaB,pending,payout,po_0IV4rIig6peNXzgD51VFKdaB
-txn_0IDYgiig6peNXzgDuNvLeoLR,2021-01-31 02:21:08,2040000,2021-02-01 0:00:00,,2021-01-25 17:17:15,usd,d267c9d09a2d1e50f12635bd2b56ed29,,59190,1980810,ch_0IDYghig6peNXzgDhuOFKdTS,pending,charge,po_0IFUKBig6peNXzgDZQM7WPAo
-txn_0IG5fvig6peNXzgD8jemrxAp,2021-02-07 02:21:12,117524,2021-02-08 0:00:00,,2021-02-01 16:54:54,usd,0662d6089789e1fabd56c6d55e91ef09,,3438,114086,ch_0IG5fuig6peNXzgDzZkrO949,pending,charge,po_0II1gGig6peNXzgD4BxKEJJJ
-txn_0ILGCbig6peNXzgDqjp7xaaX,2021-02-21 02:21:08,7239375,2021-02-22 0:00:00,,2021-02-15 23:09:59,usd,612942eb1230077a0b1d066de5aba0db,,209972,7029403,ch_0ILGCZig6peNXzgDw4eVbq0v,pending,charge,po_0IN6L6ig6peNXzgDu7m6fhRI
-txn_0IO6DWig6peNXzgDF8qNjZNX,2021-03-01 02:21:13,255360,2021-03-02 0:00:00,,2021-02-23 19:06:41,usd,b8de27e00022f92981c965125c15c22d,,7435,247925,ch_0IO6DVig6peNXzgDgTOGtdeD,pending,charge,po_0IQ08Sig6peNXzgDC6UG0Oxc
-txn_0IROZoig6peNXzgDRsBjph8K,2021-03-10 02:21:13,438750,2021-03-11 0:00:00,,2021-03-04 21:19:18,usd,d3c18c425369d3c6b26126b7f791cf3f,,12754,425996,ch_0IROZmig6peNXzgDwohSYaPj,pending,charge,po_0ITGUHig6peNXzgD78pmjvio
-txn_0IT7uhig6peNXzgDolTgmiUJ,2021-03-15 02:21:18,19350,2021-03-16 0:00:00,,2021-03-09 15:56:02,usd,a29cfd524867f29bd7eddcb21fdf6e3b,,591,18759,ch_0IT7ugig6peNXzgDh9iGZed7,pending,charge,po_0IV4rIig6peNXzgD51VFKdaB
-txn_0IUBwDig6peNXzgDCYCHC7gd,2021-03-13 14:21:36,3519960,2021-03-19 0:00:00,,2021-03-12 14:26:00,usd,e8be2b872cd9d14ddf3b5cd89860d551,,137308,3382652,ch_0IUBwCig6peNXzgDTOTEw24B,pending,charge,
-txn_0IUIaVig6peNXzgDoGvsNC17,2021-03-13 14:21:36,7499999,2021-03-19 0:00:00,,2021-03-12 21:32:01,usd,9b8159d008204266ff7f2df19843d5b4,,217530,7282469,ch_0IUIaTig6peNXzgDhyoZBxWv,pending,charge,
+txn_0IV4rIig6peNXzgDK3XAFbaN,2021-03-15 02:21:18,-18759,2021-03-16 00:00:00,,2021-03-15 01:04:36,usd,f39657a1847db73d9a2abb0bad3d31ae,,0,-18759,po_0IV4rIig6peNXzgD51VFKdaB,pending,payout,po_0IV4rIig6peNXzgD51VFKdaB
+txn_0IDYgiig6peNXzgDuNvLeoLR,2021-01-31 02:21:08,2040000,2021-02-01 00:00:00,,2021-01-25 17:17:15,usd,d267c9d09a2d1e50f12635bd2b56ed29,,59190,1980810,ch_0IDYghig6peNXzgDhuOFKdTS,pending,charge,po_0IFUKBig6peNXzgDZQM7WPAo
+txn_0IG5fvig6peNXzgD8jemrxAp,2021-02-07 02:21:12,117524,2021-02-08 00:00:00,,2021-02-01 16:54:54,usd,0662d6089789e1fabd56c6d55e91ef09,,3438,114086,ch_0IG5fuig6peNXzgDzZkrO949,pending,charge,po_0II1gGig6peNXzgD4BxKEJJJ
+txn_0ILGCbig6peNXzgDqjp7xaaX,2021-02-21 02:21:08,7239375,2021-02-22 00:00:00,,2021-02-15 23:09:59,usd,612942eb1230077a0b1d066de5aba0db,,209972,7029403,ch_0ILGCZig6peNXzgDw4eVbq0v,pending,charge,po_0IN6L6ig6peNXzgDu7m6fhRI
+txn_0IO6DWig6peNXzgDF8qNjZNX,2021-03-01 02:21:13,255360,2021-03-02 00:00:00,,2021-02-23 19:06:41,usd,b8de27e00022f92981c965125c15c22d,,7435,247925,ch_0IO6DVig6peNXzgDgTOGtdeD,pending,charge,po_0IQ08Sig6peNXzgDC6UG0Oxc
+txn_0IROZoig6peNXzgDRsBjph8K,2021-03-10 02:21:13,438750,2021-03-11 00:00:00,,2021-03-04 21:19:18,usd,d3c18c425369d3c6b26126b7f791cf3f,,12754,425996,ch_0IROZmig6peNXzgDwohSYaPj,pending,charge,po_0ITGUHig6peNXzgD78pmjvio
+txn_0IT7uhig6peNXzgDolTgmiUJ,2021-03-15 02:21:18,19350,2021-03-16 00:00:00,,2021-03-09 15:56:02,usd,a29cfd524867f29bd7eddcb21fdf6e3b,,591,18759,ch_0IT7ugig6peNXzgDh9iGZed7,pending,charge,po_0IV4rIig6peNXzgD51VFKdaB
+txn_0IUBwDig6peNXzgDCYCHC7gd,2021-03-13 14:21:36,3519960,2021-03-19 00:00:00,,2021-03-12 14:26:00,usd,e8be2b872cd9d14ddf3b5cd89860d551,,137308,3382652,ch_0IUBwCig6peNXzgDTOTEw24B,pending,charge,
+txn_0IUIaVig6peNXzgDoGvsNC17,2021-03-13 14:21:36,7499999,2021-03-19 00:00:00,,2021-03-12 21:32:01,usd,9b8159d008204266ff7f2df19843d5b4,,217530,7282469,ch_0IUIaTig6peNXzgDhyoZBxWv,pending,charge,
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'stripe_integration_tests'
-version: '0.4.2' 
+version: '0.4.2'
 
 require-dbt-version: [">=0.18.0", "<0.20.0"]
 profile: 'integration_tests'
@@ -48,10 +48,14 @@ seeds:
       _fivetran_synced: timestamp
     group_data:
     balance_transaction_data:
+      +column_types:
+        created: timestamp
+        available_on: timestamp
     card_data:
     charge_data:
       +column_types:
         balance_transaction_id: "{{ 'varchar(250)'  if target.type == 'redshift' else 'string'}}"
+        created: timestamp
     customer_data:
       +column_types:
         description: "{{ 'varchar(500)'  if target.type == 'redshift' else 'string'}}"

diff --git a/macros/date_timezone.sql b/macros/date_timezone.sql
@@ -0,0 +1,30 @@
+{% macro date_timezone(column) -%}
+
+{{ adapter.dispatch('date_timezone', packages=fivetran_utils._get_utils_namespaces()+['stripe'])(column)  }}
+
+{%- endmacro %}
+
+{% macro bigquery__date_timezone(column) -%}
+
+date(
+    {{ column }}
+    {% if var('stripe_timezone', none) %} , "{{ var('stripe_timezone') }}" {% endif %}
+    )
+
+{%- endmacro %}
+
+{% macro default__date_timezone(column) -%}
+
+{% set converted_date %}
+
+{% if var('stripe_timezone', none) %}
+    convert_timezone('{{ var("stripe_timezone") }}', {{ column }})
+{% else %}
+    {{ column }}
+{% endif %}
+
+{% endset %}
+
+{{ dbt_utils.date_trunc('day',converted_date) }}
+
+{%- endmacro %}
diff --git a/models/stripe__customer_overview.sql b/models/stripe__customer_overview.sql
@@ -36,40 +36,40 @@ with balance_transaction_joined as (
         then 1
         else 0 
             end) as total_refund_count,   
-      sum(case when type in ('charge', 'payment') and {{ dbt_utils.date_trunc('month', 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_utils.current_timestamp()) }}
+      sum(case when type in ('charge', 'payment') and {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }}
         then amount 
         else 0 
             end) as sales_this_month,
-      sum(case when type in ('payment_refund', 'refund') and {{ dbt_utils.date_trunc('month', 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_utils.current_timestamp()) }}
+      sum(case when type in ('payment_refund', 'refund') and {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }}
         then amount 
         else 0 
             end) as refunds_this_month,
-      sum(case when {{ dbt_utils.date_trunc('month', 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_utils.current_timestamp()) }}
+      sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }}
         then amount 
         else 0 
             end) as gross_transaction_amount_this_month,
-      sum(case when {{ dbt_utils.date_trunc('month', 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_utils.current_timestamp()) }}
+      sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }}
         then fee 
         else 0 
             end) as fees_this_month,
-      sum(case when {{ dbt_utils.date_trunc('month', 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_utils.current_timestamp()) }}
+      sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }}
         then net 
         else 0 
             end) as net_transaction_amount_this_month,
-      sum(case when type in ('charge', 'payment') and {{ dbt_utils.date_trunc('month', 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_utils.current_timestamp()) }}
+      sum(case when type in ('charge', 'payment') and {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }}
         then 1 
         else 0 
             end) as sales_count_this_month,
-      sum(case when type in ('payment_refund', 'refund') and {{ dbt_utils.date_trunc('month', 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_utils.current_timestamp()) }}
+      sum(case when type in ('payment_refund', 'refund') and {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }}
         then 1 
         else 0 
             end) as refund_count_this_month,
       min(case when type in ('charge', 'payment') 
-        then {{ dbt_utils.date_trunc('day', 'created_at') }}
+        then {{ date_timezone('created_at') }}
         else null 
             end) as first_sale_date,
       min(case when type in ('charge', 'payment') 
-        then {{ dbt_utils.date_trunc('day', 'created_at') }}
+        then {{ date_timezone('created_at') }}
         else null 
             end) as most_recent_sale_date
     from balance_transaction_joined
@@ -82,11 +82,11 @@ with balance_transaction_joined as (
       customer_id,
       count(*) as total_failed_charge_count,
       sum(amount) as total_failed_charge_amount,
-      sum(case when {{ dbt_utils.date_trunc('month', 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_utils.current_timestamp()) }}
+      sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }}
         then 1
         else 0 
             end) as failed_charge_count_this_month,
-      sum(case when {{ dbt_utils.date_trunc('month', 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_utils.current_timestamp()) }}
+      sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }}
         then amount
         else 0 
             end) as failed_charge_amount_this_month

diff --git a/models/stripe__daily_overview.sql b/models/stripe__daily_overview.sql
@@ -11,9 +11,11 @@ with balance_transaction_joined as (
 ), daily_balance_transactions as (
 
   select
-      date(case when type = 'payout' 
-          then available_on 
-          else created_at end) as date,
+      case 
+            when type = 'payout' 
+            then {{ date_timezone('available_on') }}  
+            else {{ date_timezone('created_at') }} 
+      end as date,
     sum(case when type in ('charge', 'payment') 
           then amount 
           else 0 end) as total_sales,
@@ -56,7 +58,7 @@ with balance_transaction_joined as (
 ), daily_failed_charges as (
 
     select
-      date(created_at) as date,
+      {{ date_timezone('created_at') }} as date,
       count(*) as total_failed_charge_count,
       sum(amount) as total_failed_charge_amount
     from incomplete_charges