Skip to content

Commit

Permalink
Remove all references to dynamic incremental load strategy (Velir#240)
Browse files Browse the repository at this point in the history 
* Remove all references to dynamic incremental load strategy

* remove default value

* fix unit test

* unit tests: set static_incremental_days at the project level
  • Loading branch information
@adamribaudo-velir
adamribaudo-velir authored Jun 23, 2023
1 parent fd26c3c commit b126eb5
Show file tree
Hide file tree
Showing 9 changed files with 119 additions and 231 deletions.
64 changes: 19 additions & 45 deletions models/marts/core/dim_ga4__sessions_daily.sql
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,21 @@
{% if var('static_incremental_days', false ) %}
{% set partitions_to_replace = ['current_date'] %}
{% for i in range(var('static_incremental_days')) %}
{% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
{% endfor %}
{{
config(
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
tags = ["incremental"],
partition_by={
"field": "session_partition_date",
"data_type": "date",
"granularity": "day"
},
partitions = partitions_to_replace
)
}}
{% else %}
{{
config(
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
tags = ["incremental"],
partition_by={
"field": "session_partition_date",
"data_type": "date",
"granularity": "day"
}
)
}}
{% endif %}
{% set partitions_to_replace = ['current_date'] %}
{% for i in range(var('static_incremental_days')) %}
{% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
{% endfor %}
{{
config(
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
tags = ["incremental"],
partition_by={
"field": "session_partition_date",
"data_type": "date",
"granularity": "day"
},
partitions = partitions_to_replace
)
}}


with event_dimensions as
(
Expand Down Expand Up @@ -74,11 +60,7 @@ with event_dimensions as
where event_name != 'first_visit'
and event_name != 'session_start'
{% if is_incremental() %}
{% if var('static_incremental_days', false ) %}
and event_date_dt in ({{ partitions_to_replace | join(',') }})
{% else %}
and event_date_dt >= _dbt_max_partition
{% endif %}
{% endif %}
)
,traffic_sources as (
Expand All @@ -102,11 +84,7 @@ with event_dimensions as
from {{ref('stg_ga4__sessions_traffic_sources_last_non_direct_daily')}}
where 1=1
{% if is_incremental() %}
{% if var('static_incremental_days', false ) %}
and session_partition_date in ({{ partitions_to_replace | join(',') }})
{% else %}
and session_partition_date >= _dbt_max_partition
{% endif %}
{% endif %}
)
{% if var('derived_session_properties', false) %}
Expand All @@ -116,11 +94,7 @@ with event_dimensions as
from {{ref('stg_ga4__derived_session_properties_daily')}}
where 1=1
{% if is_incremental() %}
{% if var('static_incremental_days', false ) %}
and session_partition_date in ({{ partitions_to_replace | join(',') }})
{% else %}
and session_partition_date >= _dbt_max_partition
{% endif %}
and session_partition_date in ({{ partitions_to_replace | join(',') }})
{% endif %}
)
{% endif %}
Expand Down
16 changes: 3 additions & 13 deletions models/marts/core/fct_ga4__pages.sql
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
{% set partitions_to_replace = ['current_date'] %}
{% if var('static_incremental_days', false)%}
{% for i in range(var('static_incremental_days')) %}
{% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
{% endfor %}
{% endif %}
{% for i in range(var('static_incremental_days')) %}
{% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
{% endfor %}
{{
config(
materialized = 'incremental',
Expand Down Expand Up @@ -33,11 +31,7 @@ with page_view as (
sum(entrances) as entrances,
from {{ref('stg_ga4__event_page_view')}}
{% if is_incremental() %}
{% if var('static_incremental_days', false) %}
where event_date_dt in ({{ partitions_to_replace | join(',') }})
{% else %}
where event_date_dt >= _dbt_max_partition
{% endif %}
{% endif %}
group by 1,2,3,4,5,6,7
), page_engagement as (
Expand All @@ -56,11 +50,7 @@ from {{ref('stg_ga4__event_page_view')}}
count(event_name) as scroll_events
from {{ref('stg_ga4__event_scroll')}}
{% if is_incremental() %}
{% if var('static_incremental_days', false) %}
where event_date_dt in ({{ partitions_to_replace | join(',') }})
{% else %}
where event_date_dt >= _dbt_max_partition
{% endif %}
{% endif %}
group by 1,2,3
)
Expand Down
58 changes: 18 additions & 40 deletions models/marts/core/fct_ga4__sessions_daily.sql
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,21 @@
{% if var('static_incremental_days', false)%}
{% set partitions_to_replace = ['current_date'] %}
{% for i in range(var('static_incremental_days')) %}
{% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
{% endfor %}
{{
config(
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
tags = ["incremental"],
partition_by={
"field": "session_partition_date",
"data_type": "date",
"granularity": "day"
},
partitions = partitions_to_replace
)
}}
{% else %}
{{
config(
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
tags = ["incremental"],
partition_by={
"field": "session_partition_date",
"data_type": "date",
"granularity": "day"
}
)
}}
{% endif %}
{% set partitions_to_replace = ['current_date'] %}
{% for i in range(var('static_incremental_days')) %}
{% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
{% endfor %}
{{
config(
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
tags = ["incremental"],
partition_by={
"field": "session_partition_date",
"data_type": "date",
"granularity": "day"
},
partitions = partitions_to_replace
)
}}


with session_metrics as (
select
Expand All @@ -48,11 +34,7 @@ with session_metrics as (
from {{ref('stg_ga4__events')}}
where session_key is not null
{% if is_incremental() %}
{% if var('static_incremental_days', false) %}
and event_date_dt in ({{ partitions_to_replace | join(',') }})
{% else %}
and event_date_dt >= _dbt_max_partition
{% endif %}
{% endif %}
group by 1,2,3,4
)
Expand All @@ -64,11 +46,7 @@ with session_metrics as (
select * from {{ref('stg_ga4__session_conversions_daily')}}
where 1=1
{% if is_incremental() %}
{% if var('static_incremental_days', false) %}
and session_partition_date in ({{ partitions_to_replace | join(',') }})
{% else %}
and session_partition_date >= _dbt_max_partition
{% endif %}
{% endif %}
),
join_metrics_and_conversions as (
Expand Down
57 changes: 19 additions & 38 deletions models/staging/stg_ga4__derived_session_properties_daily.sql
View file
Original file line number Diff line number Diff line change
@@ -1,37 +1,22 @@
{% if var('static_incremental_days', false ) %}
{% set partitions_to_replace = ['current_date'] %}
{% for i in range(var('static_incremental_days')) %}
{% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
{% endfor %}
{{
config(
enabled = true if var('derived_session_properties', false) else false,
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
tags = ["incremental"],
partition_by={
"field": "session_partition_date",
"data_type": "date",
"granularity": "day"
},
partitions = partitions_to_replace
)
}}
{% else %}
{{
config(
enabled = true if var('derived_session_properties', false) else false,
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
tags = ["incremental"],
partition_by={
"field": "session_partition_date",
"data_type": "date",
"granularity": "day"
}
)
}}
{% endif %}
{% set partitions_to_replace = ['current_date'] %}
{% for i in range(var('static_incremental_days')) %}
{% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
{% endfor %}
{{
config(
enabled = true if var('derived_session_properties', false) else false,
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
tags = ["incremental"],
partition_by={
"field": "session_partition_date",
"data_type": "date",
"granularity": "day"
},
partitions = partitions_to_replace
)
}}


with unnest_event_params as
(
Expand All @@ -49,11 +34,7 @@ with unnest_event_params as
from {{ref('stg_ga4__events')}}
where event_key is not null
{% if is_incremental() %}
{% if var('static_incremental_days', false ) %}
and event_date_dt in ({{ partitions_to_replace | join(',') }})
{% else %}
and event_date_dt >= _dbt_max_partition
{% endif %}
{% endif %}

)
Expand Down
57 changes: 19 additions & 38 deletions models/staging/stg_ga4__session_conversions_daily.sql
View file
Original file line number Diff line number Diff line change
@@ -1,37 +1,22 @@
{% if var('static_incremental_days', false ) %}
{% set partitions_to_replace = ['current_date'] %}
{% for i in range(var('static_incremental_days')) %}
{% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
{% endfor %}
{{
config(
enabled= var('conversion_events', false) != false,
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
tags = ["incremental"],
partition_by={
"field": "session_partition_date",
"data_type": "date",
"granularity": "day"
},
partitions = partitions_to_replace
)
}}
{% else %}
{{
config(
enabled= var('conversion_events', false) != false,
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
tags = ["incremental"],
partition_by={
"field": "session_partition_date",
"data_type": "date",
"granularity": "day"
}
)
}}
{% endif %}
{% set partitions_to_replace = ['current_date'] %}
{% for i in range(var('static_incremental_days')) %}
{% set partitions_to_replace = partitions_to_replace.append('date_sub(current_date, interval ' + (i+1)|string + ' day)') %}
{% endfor %}
{{
config(
enabled= var('conversion_events', false) != false,
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
tags = ["incremental"],
partition_by={
"field": "session_partition_date",
"data_type": "date",
"granularity": "day"
},
partitions = partitions_to_replace
)
}}



with event_counts as (
Expand All @@ -45,11 +30,7 @@ with event_counts as (
from {{ref('stg_ga4__events')}}
where 1=1
{% if is_incremental() %}
{% if var('static_incremental_days', false ) %}
and event_date_dt in ({{ partitions_to_replace | join(',') }})
{% else %}
and event_date_dt >= _dbt_max_partition
{% endif %}
{% endif %}
group by 1,2
)
Expand Down
Loading

0 comments on commit b126eb5

Please sign in to comment.