T 12432 performance testing optimize and partition by on dbt models n…

…ew (#1096)

This PR results from a discussion we had w/ @aalan3, with the aim of optimizing/reducing build time for incremental tables.
This resulted in 2 changes:
- Running OPTIMIZE on incremental tables on-run-end
- Replacing the incremental filter by a static filter,  namely  `NOW() - interval 2 days`

I've checked that:

* [*] I tested the query on dune.com after compiling the model with dbt compile (compiled queries are written to the target directory)
* [*] I used "refs" to reference other models in this repo and "sources" to reference raw or decoded tables 
* [*] the directory tree matches the pattern /sector/blockchain/ e.g. /tokens/ethereum
* [*] if adding a new model, I added a test
* [*] the filename is unique and ends with .sql
* [*] each file has only one view, table or function defined  
* [*] column names are `lowercase_snake_cased`

spellbook/dbt_project.yml

@@ -127,4 +127,10 @@ on-run-end:
   - "{{ alter_tblproperties_opensea_trades() }}"
   - "{{ alter_tblproperties_opensea_volume_day() }}"
   - "{{ alter_tblproperties_opensea_txns_day() }}"
-  - "{{ alter_tblproperties_opensea_active_traders_day() }}"
+  - "{{ alter_tblproperties_opensea_active_traders_day() }}"
+
+  - "OPTIMIZE transfers_ethereum.erc20_agg_hour"
+  - "OPTIMIZE transfers_ethereum.erc20_agg_day"
+  - "OPTIMIZE opensea_ethereum.trades"
+  - "OPTIMIZE opensea_solana.trades"
+  - "OPTIMIZE magiceden_solana.trades"

spellbook/macros/alter_tblproperties/alter_tblproperties_nft_trades.sql

@@ -1,7 +1,7 @@
 {% macro alter_tblproperties_nft_trades() -%}
 {%- if target.name == 'prod'-%}
 ALTER VIEW nft.trades SET TBLPROPERTIES('dune.public'='true',
-                                                    'dune.data_explorer.blockchains'='["solana"]',
+                                                    'dune.data_explorer.blockchains'='["ethereum","solana"]',
                                                     'dune.data_explorer.category'='abstraction',
                                                     'dune.data_explorer.abstraction.type'='sector',
                                                     'dune.data_explorer.abstraction.name'='nft',

spellbook/models/magiceden/magiceden_trades.sql

@@ -7,8 +7,4 @@ SELECT blockchain, 'magiceden' as project, '' as version, tx_hash, block_time, a
 (
         SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id 
         FROM {{ ref('magiceden_solana_trades') }}
-) 
-{% if is_incremental() %}
--- this filter will only be applied on an incremental run
-where block_time > (select max(block_time) from {{ this }})
-{% endif %} 
+) 

spellbook/models/magiceden/solana/magiceden_solana_trades.sql

@@ -29,5 +29,5 @@ AND ARRAY_CONTAINS(log_messages, 'Program log: Instruction: ExecuteSale')
 AND block_time > '2021-09-01'
 {% if is_incremental() %}
 -- this filter will only be applied on an incremental run
-AND block_time > (select max(block_time) from {{ this }})
+AND block_time > now() - interval 2 days
 {% endif %} 

spellbook/models/opensea/ethereum/opensea_ethereum_trades.sql

@@ -38,5 +38,5 @@ FROM
   )
   {% if is_incremental() %}
   -- this filter will only be applied on an incremental run
-  AND evt_block_time > (select max(block_time) from {{ this }})
+  AND evt_block_time > now() - interval 2 days
   {% endif %} 

spellbook/models/opensea/opensea_trades.sql

@@ -6,8 +6,4 @@
 SELECT blockchain, 'opensea' as project, 'v1' as version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM 
 (SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM {{ ref('opensea_ethereum_trades') }} 
 UNION ALL
-SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM {{ ref('opensea_solana_trades') }}) 
-{% if is_incremental() %}
--- this filter will only be applied on an incremental run
-where block_time > (select max(block_time) from {{ this }})
-{% endif %} 
+SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM {{ ref('opensea_solana_trades') }}) 

spellbook/models/opensea/solana/opensea_solana_trades.sql

@@ -29,5 +29,5 @@ AND block_time > '2022-04-06'
 AND ARRAY_CONTAINS(log_messages, 'Program log: Instruction: ExecuteSale')
 {% if is_incremental() %}
 -- this filter will only be applied on an incremental run
-AND block_time > (select max(block_time) from {{ this }})
+AND block_time > now() - interval 2 days
 {% endif %} 

spellbook/models/transfers/ethereum/transfers_ethereum_erc20_agg_day.sql

@@ -18,7 +18,7 @@ from {{ ref('transfers_ethereum_erc20') }} tr
 left join {{ ref('tokens_ethereum_erc20') }} t on t.contract_address = tr.token_address
 {% if is_incremental() %}
 -- this filter will only be applied on an incremental run
-where date_trunc('day', tr.evt_block_time) > (select max(day) from {{ this }})
+where tr.evt_block_time > now() - interval 2 days
 {% endif %}
 group by
     date_trunc('day', tr.evt_block_time), tr.wallet_address, tr.token_address, t.symbol

spellbook/models/transfers/ethereum/transfers_ethereum_erc20_agg_hour.sql

@@ -18,7 +18,7 @@ from {{ ref('transfers_ethereum_erc20') }} tr
 left join {{ ref('tokens_ethereum_erc20') }} t on t.contract_address = tr.token_address
 {% if is_incremental() %}
 -- this filter will only be applied on an incremental run
-where date_trunc('hour', tr.evt_block_time) > (select max(hour) from {{ this }})
+where tr.evt_block_time > now() - interval 2 days
 {% endif %}
 group by
     date_trunc('hour', tr.evt_block_time), tr.wallet_address, tr.token_address, t.symbol