Skip to content

Feature/redshift model large #129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
fivetran-catfritz wants to merge 2 commits into
base: main
Choose a base branch
from
+59 −9
Draft

Feature/redshift model large #129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cb5a4be
feature/redshift-model-large
fivetran-catfritz Feb 4, 2026
52edb8b
add seed
fivetran-catfritz Feb 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion integration_tests/dbt_project.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,19 @@ seeds:
salesforce_formula_integration_tests:
+quote_columns: "{{ true if target.type == 'redshift' else false }}"
opportunity:
+columns_types:
+column_types:
amount: float
probability: float
fivetran_formula:
+column_types:
view_sql: "{{ 'varchar(256)' if target.type in ('redshift','postgres') else 'string'}}"
fivetran_formula_model:
+enabled: "{{ false if target.type == 'redshift' else true }}"
fivetran_formula_model_redshift:
+enabled: "{{ true if target.type == 'redshift' else false }}"
+alias: fivetran_formula_model
+column_types:
model_large: super

dispatch:
- macro_namespace: dbt_utils
Expand Down
5 changes: 5 additions & 0 deletions integration_tests/seeds/fivetran_formula_model_redshift.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
object,model_large
opportunity,"SELECT \n mt.\""stage_name\"",\n mt.\""amount\""\nFROM \""salesforce_formula_integrations_tests\"".\""opportunity\"" AS mt"
account,"SELECT \n mt.*,\n (mt.\""number_of_employees\"" * 2) AS \""account_test_formula\""\nFROM \""salesforce_formula_integrations_tests\"".\""account\"" AS mt"
user,"SELECT \n mt.*,\n CASE WHEN mt.\""is_active\"" = true THEN 'active' ELSE 'not active' END AS \""is_active_user\""\nFROM \""salesforce_formula_integrations_tests\"".\""user\"" AS mt\nWHERE NOT mt._fivetran_deleted"
user_role,"SELECT \n mt.*,\n CONCAT(mt.\""developer_name\"", mt.\""id\"") AS \""developer_and_id\""\nFROM \""salesforce_formula_integrations_tests\"".\""user_role\"" AS mt\nWHERE NOT mt._fivetran_deleted"
54 changes: 46 additions & 8 deletions macros/sfdc_formula_view.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,61 @@
)
}}

-- Raise a warning if users are trying to use full_statement_version=false. We are keeping the variable in the macro, however, since we don't want errors if they previously set it to true.
{# Raise a warning if users are trying to use full_statement_version=false. We are keeping the variable in the macro, however, since we don't want errors if they previously set it to true. #}
{% if not full_statement_version %}
{{ exceptions.warn("\nERROR: The full_statement_version=false, reserved_table_name, and fields_to_include parameters are no longer supported. Please update your " ~ this.identifier|upper ~ " model to remove these parameters.\n") }}
See_full_model_error_in_log

{% else %}
{% if target.type == 'redshift' %}
{% set columns = adapter.get_columns_in_relation(source(source_name, 'fivetran_formula_model')) %}
{% set column_names = columns | map(attribute='name') | list %}
{%- set using_model_large = 'model_large' in column_names -%}
{%- set model_column_name = 'model_large' if using_model_large else 'model' -%}
{% else %}
{%- set using_model_large = false -%}
{%- set model_column_name = 'MODEL' if target.type == 'snowflake' else 'model' -%}
{% endif %}

{% if using_quoted_identifiers %}
{%- set table_results = dbt_utils.get_column_values(table=source(source_name, 'fivetran_formula_model'),
column='"MODEL"' if target.type in ('snowflake') else '"model"' if target.type in ('postgres', 'redshift', 'snowflake') else '`model`',
where=("\"OBJECT\" = '" if target.type in ('snowflake') else "\"object\" = '" if target.type in ('postgres', 'redshift') else "`object` = '") ~ source_table ~ "'") -%}
{%- set table_results = dbt_utils.get_column_values(table=source(source_name, 'fivetran_formula_model'), column=adapter.quote(model_column_name) if using_quoted_identifiers else model_column_name, where="object = '" ~ source_table ~ "'") -%}

{% else %}
{%- set table_results = dbt_utils.get_column_values(table=source(source_name, 'fivetran_formula_model'), column='model', where="object = '" ~ source_table ~ "'") -%}
{% if using_model_large %}
{# Extract SQL from model_large super column and remove escape characters #}
{% set raw_sql = table_results[0] %}
{% if raw_sql %}
{{ log("DEBUG: Raw SQL from super column: " ~ raw_sql, info=True) }}

{# Remove outer quotes if present (JSON string format) #}
{% if raw_sql.startswith('"') and raw_sql.endswith('"') %}
{% set clean_sql = raw_sql[1:-1] %}
{% else %}
{% set clean_sql = raw_sql %}
{% endif %}

{{ log("DEBUG: After removing outer quotes: " ~ clean_sql, info=True) }}

{# Process double-escaped characters from super column #}
{% set clean_sql = clean_sql.replace('\\\\n', '\n') %}
{% set clean_sql = clean_sql.replace('\\\\t', '\t') %}
{% set clean_sql = clean_sql.replace('\\\\\"', '"') %}
{% set clean_sql = clean_sql.replace('\\\\_', '_') %}

{# Clean up any remaining single-escaped characters #}
{% set clean_sql = clean_sql.replace('\\n', '\n') %}
{% set clean_sql = clean_sql.replace('\\t', '\t') %}
{% set clean_sql = clean_sql.replace('\\"', '"') %}
{% set clean_sql = clean_sql.replace('\\_', '_') %}

{# Remove any trailing empty quotes #}
{% set clean_sql = clean_sql.replace('""', '') %}
{% set clean_sql = clean_sql.rstrip('"') %}

{{ log("DEBUG: Final cleaned SQL: " ~ clean_sql, info=True) }}
{{ clean_sql }}
{% endif %}
{% else %}
{{ table_results[0] }}
{% endif %}

{{ table_results[0] }}
{% endif %}
{%- endmacro -%}