From c427eb8ff93903f91feabafb6fd2f0e28e90e15f Mon Sep 17 00:00:00 2001 From: harisang Date: Wed, 31 Jan 2024 19:08:11 +0200 Subject: [PATCH 01/12] update queries --- src/sql/orderbook/batch_rewards.sql | 12 +++--------- src/sql/orderbook/order_rewards.sql | 13 ++++--------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/src/sql/orderbook/batch_rewards.sql b/src/sql/orderbook/batch_rewards.sql index 078fbd31..ef7d8637 100644 --- a/src/sql/orderbook/batch_rewards.sql +++ b/src/sql/orderbook/batch_rewards.sql @@ -7,17 +7,13 @@ WITH observed_settlements AS (SELECT effective_gas_price * gas_used AS execution_cost, surplus, fee, - -- auction_transaction - at.auction_id + s.auction_id FROM settlement_observations so JOIN settlements s ON s.block_number = so.block_number AND s.log_index = so.log_index - JOIN auction_transaction at - ON s.tx_from = at.tx_from - AND s.tx_nonce = at.tx_nonce JOIN settlement_scores ss - ON at.auction_id = ss.auction_id + ON s.auction_id = ss.auction_id WHERE ss.block_deadline > {{start_block}} AND ss.block_deadline <= {{end_block}}), @@ -35,7 +31,7 @@ WITH observed_settlements AS (SELECT order_surplus AS ( SELECT ss.winner as solver, - at.auction_id, + s.auction_id, s.tx_hash, t.order_uid, o.sell_token, @@ -57,8 +53,6 @@ order_surplus AS ( THEN o.sell_token END AS surplus_token FROM settlements s -- links block_number and log_index to tx_from and tx_nonce - JOIN auction_transaction at -- links auction_id to tx_from and tx_nonce - ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce JOIN settlement_scores ss -- contains block_deadline ON at.auction_id = ss.auction_id JOIN trades t -- contains traded amounts diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index fb6c1a47..e05077bd 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -6,22 +6,19 @@ with trade_hashes as (SELECT settlement.solver, auction_id FROM trades t LEFT OUTER JOIN LATERAL ( - SELECT tx_hash, solver, tx_nonce, tx_from + SELECT tx_hash, solver, tx_nonce, tx_from, auction_id, block_number, log_index FROM settlements s WHERE s.block_number = t.block_number AND s.log_index > t.log_index ORDER BY s.log_index ASC LIMIT 1 ) AS settlement ON true - join auction_transaction - -- This join also eliminates overlapping - -- trades & settlements between barn and prod DB - on settlement.tx_from = auction_transaction.tx_from - and settlement.tx_nonce = auction_transaction.tx_nonce + join settlement_observations so on + settlement.block_number = so.block_number and settlement.log_index = so.log_index where block_number > {{start_block}} and block_number <= {{end_block}}), order_surplus AS ( SELECT - at.auction_id, + s.auction_id, t.order_uid, o.sell_token, o.buy_token, @@ -42,8 +39,6 @@ order_surplus AS ( THEN o.sell_token END AS surplus_token FROM settlements s -- links block_number and log_index to tx_from and tx_nonce - JOIN auction_transaction at -- links auction_id to tx_from and tx_nonce - ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce JOIN settlement_scores ss -- contains block_deadline ON at.auction_id = ss.auction_id JOIN trades t -- contains traded amounts From 3ed1bbbadc8ec6f8b151dd6c61417fa9317107c6 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 1 Feb 2024 11:16:16 +0200 Subject: [PATCH 02/12] addressed reviewer comments --- src/sql/orderbook/batch_rewards.sql | 6 +++--- src/sql/orderbook/order_rewards.sql | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sql/orderbook/batch_rewards.sql b/src/sql/orderbook/batch_rewards.sql index ef7d8637..d2e1f3e9 100644 --- a/src/sql/orderbook/batch_rewards.sql +++ b/src/sql/orderbook/batch_rewards.sql @@ -52,15 +52,15 @@ order_surplus AS ( WHEN o.kind = 'buy' THEN o.sell_token END AS surplus_token - FROM settlements s -- links block_number and log_index to tx_from and tx_nonce + FROM settlements s JOIN settlement_scores ss -- contains block_deadline - ON at.auction_id = ss.auction_id + ON s.auction_id = ss.auction_id JOIN trades t -- contains traded amounts ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block JOIN orders o -- contains tokens and limit amounts ON t.order_uid = o.uid JOIN order_execution oe -- contains surplus fee - ON t.order_uid = oe.order_uid AND at.auction_id = oe.auction_id + ON t.order_uid = oe.order_uid AND s.auction_id = oe.auction_id WHERE ss.block_deadline > {{start_block}} AND ss.block_deadline <= {{end_block}} ) diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index e05077bd..83090791 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -40,13 +40,13 @@ order_surplus AS ( END AS surplus_token FROM settlements s -- links block_number and log_index to tx_from and tx_nonce JOIN settlement_scores ss -- contains block_deadline - ON at.auction_id = ss.auction_id + ON s.auction_id = ss.auction_id JOIN trades t -- contains traded amounts ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block JOIN orders o -- contains tokens and limit amounts ON t.order_uid = o.uid JOIN order_execution oe -- contains surplus fee - ON t.order_uid = oe.order_uid AND at.auction_id = oe.auction_id + ON t.order_uid = oe.order_uid AND s.auction_id = oe.auction_id WHERE s.block_number > {{start_block}} AND s.block_number <= {{end_block}} ) From 06f9fd24d61eedfc685b1dff03eab66470572498 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 1 Feb 2024 11:48:07 +0200 Subject: [PATCH 03/12] minor fix --- src/sql/orderbook/order_rewards.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index 83090791..fa1419f2 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -1,5 +1,5 @@ with trade_hashes as (SELECT settlement.solver, - block_number, + t.block_number, order_uid, fee_amount, settlement.tx_hash, From 406af004a8d0cab81335e84fd41a0e55d9ba5b48 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 1 Feb 2024 11:51:33 +0200 Subject: [PATCH 04/12] make block_number explicit to address failing test --- src/sql/orderbook/order_rewards.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index fa1419f2..0f720cd1 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -15,7 +15,7 @@ with trade_hashes as (SELECT settlement.solver, ) AS settlement ON true join settlement_observations so on settlement.block_number = so.block_number and settlement.log_index = so.log_index - where block_number > {{start_block}} and block_number <= {{end_block}}), + where t.block_number > {{start_block}} and t.block_number <= {{end_block}}), order_surplus AS ( SELECT s.auction_id, From 1c9b67f9d9095995495d87aeab670466a668cac3 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 1 Feb 2024 11:56:05 +0200 Subject: [PATCH 05/12] continued fixing of block_number ambiguity --- src/sql/orderbook/order_rewards.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index 0f720cd1..10c0de08 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -114,8 +114,8 @@ order_surplus AS ( OR (o.kind = 'sell' AND (oq.sell_amount - oq.gas_amount * oq.gas_price / oq.sell_token_price) * oq.buy_amount >= o.buy_amount * oq.sell_amount) OR (o.kind='buy' AND o.sell_amount >= oq.sell_amount + oq.gas_amount * oq.gas_price / oq.sell_token_price)) AND o.partially_fillable='f' -- the code above might fail for partially fillable orders - AND block_number > {{start_block}} - AND block_number <= {{end_block}} + AND t.block_number > {{start_block}} + AND t.block_number <= {{end_block}} AND oq.solver != '\x0000000000000000000000000000000000000000') -- Most efficient column order for sorting would be having tx_hash or order_uid first select block_number, From 38e43a56b8bdc17b36b1da057054f758f99c299c Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 1 Feb 2024 12:00:14 +0200 Subject: [PATCH 06/12] minor fix --- src/sql/orderbook/order_rewards.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index 10c0de08..e4b61ecd 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -1,5 +1,5 @@ with trade_hashes as (SELECT settlement.solver, - t.block_number, + t.block_number as block_number, order_uid, fee_amount, settlement.tx_hash, @@ -118,7 +118,7 @@ order_surplus AS ( AND t.block_number <= {{end_block}} AND oq.solver != '\x0000000000000000000000000000000000000000') -- Most efficient column order for sorting would be having tx_hash or order_uid first -select block_number, +select trade_hashes.block_number as block_number, concat('0x', encode(trade_hashes.order_uid, 'hex')) as order_uid, concat('0x', encode(solver, 'hex')) as solver, quote_solver, From 80c39ce1db56cc3813c485b179e3d77e7abb132c Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 1 Feb 2024 12:44:15 +0200 Subject: [PATCH 07/12] change failing test --- src/sql/orderbook/order_rewards.sql | 25 ++++++++++++----------- tests/integration/test_fetch_orderbook.py | 18 ++++++++-------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index e4b61ecd..a5aeee5d 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -105,18 +105,19 @@ order_surplus AS ( JOIN auction_prices ap-- contains price: protocol fee token ON opf.auction_id = ap.auction_id AND opf.protocol_fee_token = ap.token ), - winning_quotes as (SELECT concat('0x', encode(oq.solver, 'hex')) as quote_solver, - oq.order_uid - FROM trades t - INNER JOIN orders o ON order_uid = uid - JOIN order_quotes oq ON t.order_uid = oq.order_uid - WHERE (o.class = 'market' - OR (o.kind = 'sell' AND (oq.sell_amount - oq.gas_amount * oq.gas_price / oq.sell_token_price) * oq.buy_amount >= o.buy_amount * oq.sell_amount) - OR (o.kind='buy' AND o.sell_amount >= oq.sell_amount + oq.gas_amount * oq.gas_price / oq.sell_token_price)) - AND o.partially_fillable='f' -- the code above might fail for partially fillable orders - AND t.block_number > {{start_block}} - AND t.block_number <= {{end_block}} - AND oq.solver != '\x0000000000000000000000000000000000000000') +winning_quotes as ( + SELECT concat('0x', encode(oq.solver, 'hex')) as quote_solver, + oq.order_uid + FROM trades t + INNER JOIN orders o ON order_uid = uid + JOIN order_quotes oq ON t.order_uid = oq.order_uid + WHERE (o.class = 'market' + OR (o.kind = 'sell' AND (oq.sell_amount - oq.gas_amount * oq.gas_price / oq.sell_token_price) * oq.buy_amount >= o.buy_amount * oq.sell_amount) + OR (o.kind='buy' AND o.sell_amount >= oq.sell_amount + oq.gas_amount * oq.gas_price / oq.sell_token_price)) + AND o.partially_fillable='f' -- the code above might fail for partially fillable orders + AND t.block_number > {{start_block}} + AND t.block_number <= {{end_block}} + AND oq.solver != '\x0000000000000000000000000000000000000000') -- Most efficient column order for sorting would be having tx_hash or order_uid first select trade_hashes.block_number as block_number, concat('0x', encode(trade_hashes.order_uid, 'hex')) as order_uid, diff --git a/tests/integration/test_fetch_orderbook.py b/tests/integration/test_fetch_orderbook.py index 2502b388..4febd5d9 100644 --- a/tests/integration/test_fetch_orderbook.py +++ b/tests/integration/test_fetch_orderbook.py @@ -15,27 +15,27 @@ def test_latest_block_increasing(self): self.assertGreaterEqual(OrderbookFetcher.get_latest_block(), latest_block) def test_get_order_rewards(self): - block_number = 16000000 + block_number = 17500000 block_range = BlockRange(block_number, block_number + 50) rewards_df = OrderbookFetcher.get_order_rewards(block_range) expected = pd.DataFrame( { - "block_number": [16000018, 16000050], + "block_number": [17500006, 17500022], "order_uid": [ - "0xb52fecfe3df73f0e93f1f9b27c92e3def50322960f9c62d0b97bc2ceee36c07a0639dda84198dc06f5bc91bddbb62cd2e38c2f9a6378140f", - "0xf61cba0b42ed3e956f9db049c0523e123967723c5bcf76ccac0b179a66305b2a7fee439ed7a6bb1b8e7ca1ffdb0a5ca8d993c030637815ad", + "0x115bb7fd4ec0c8cec1bed470b599cd187173825bfd32eec84903faf5eb7038546d335b1f889a82a6d9a7fb6db86cab15685e3c3e648dc436", + "0x49f9cf94a59301548bbfaa75c06b1126cb3194d0b18a9e516ebeffabfb7e2d659cf8986e78d6215d1491c66d64409a54cf6268e4648dd550", ], "solver": [ - "0x3cee8c7d9b5c8f225a8c36e7d3514e1860309651", - "0xc9ec550bea1c64d779124b23a26292cc223327b6", + "0x31A9Ec3A6E29039C74723E387DE42b79E6856FD8", + "0x31A9Ec3A6E29039C74723E387DE42b79E6856FD8", ], "quote_solver": [None, None], "tx_hash": [ - "0xb6f7df8a1114129f7b61f2863b3f81b3620e95f73e5b769a62bb7a87ab6983f4", - "0x2ce77009e78c291cdf39eb6f8ddf7e2c3401b4f962ef1240bdac47e632f8eb7f", + "0xb3d59ac16f874a739ce93a58139ba5848d6ce521fa70fad226f3c3d2d47b2aa6", + "0x22916b61ad5207a493d66b3662a27f5af1c2ac76e66672cfd9ae57041a1ca2a2", ], "surplus_fee": ["0", "0"], - "amount": [40.70410, 39.00522], + "amount": [0, 0], "protocol_fee": ["0", "0"], "protocol_fee_token": [None, None], "protocol_fee_native_price": [0.0, 0.0], From 82732c90c8d1085b10bad981f729e716b64c4a3d Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 1 Feb 2024 12:47:08 +0200 Subject: [PATCH 08/12] fix blockrange in test --- tests/integration/test_fetch_orderbook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_fetch_orderbook.py b/tests/integration/test_fetch_orderbook.py index 4febd5d9..4eca3538 100644 --- a/tests/integration/test_fetch_orderbook.py +++ b/tests/integration/test_fetch_orderbook.py @@ -16,7 +16,7 @@ def test_latest_block_increasing(self): def test_get_order_rewards(self): block_number = 17500000 - block_range = BlockRange(block_number, block_number + 50) + block_range = BlockRange(block_number, block_number + 30) rewards_df = OrderbookFetcher.get_order_rewards(block_range) expected = pd.DataFrame( { From 154f682ea6f3c3ff46a2e3cf9037c9a6a4034e30 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 1 Feb 2024 12:50:53 +0200 Subject: [PATCH 09/12] fix capitalization in solver address in test --- tests/integration/test_fetch_orderbook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_fetch_orderbook.py b/tests/integration/test_fetch_orderbook.py index 4eca3538..277b0d78 100644 --- a/tests/integration/test_fetch_orderbook.py +++ b/tests/integration/test_fetch_orderbook.py @@ -26,8 +26,8 @@ def test_get_order_rewards(self): "0x49f9cf94a59301548bbfaa75c06b1126cb3194d0b18a9e516ebeffabfb7e2d659cf8986e78d6215d1491c66d64409a54cf6268e4648dd550", ], "solver": [ - "0x31A9Ec3A6E29039C74723E387DE42b79E6856FD8", - "0x31A9Ec3A6E29039C74723E387DE42b79E6856FD8", + "0x31a9ec3a6e29039c74723e387de42b79e6856fd8", + "0x31a9ec3a6e29039c74723e387de42b79e6856fd8", ], "quote_solver": [None, None], "tx_hash": [ From a376f38cf21147850a845f2e1dbc16f73fe53070 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 1 Feb 2024 12:54:11 +0200 Subject: [PATCH 10/12] convert int to float in failing test --- tests/integration/test_fetch_orderbook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_fetch_orderbook.py b/tests/integration/test_fetch_orderbook.py index 277b0d78..6aa4168c 100644 --- a/tests/integration/test_fetch_orderbook.py +++ b/tests/integration/test_fetch_orderbook.py @@ -35,7 +35,7 @@ def test_get_order_rewards(self): "0x22916b61ad5207a493d66b3662a27f5af1c2ac76e66672cfd9ae57041a1ca2a2", ], "surplus_fee": ["0", "0"], - "amount": [0, 0], + "amount": [0.0, 0.0], "protocol_fee": ["0", "0"], "protocol_fee_token": [None, None], "protocol_fee_native_price": [0.0, 0.0], From a284cd9b60b37536c7e28793706f1e28c2ee7a29 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 1 Feb 2024 13:25:14 +0200 Subject: [PATCH 11/12] revert minor change --- src/sql/orderbook/order_rewards.sql | 40 +++++++++++++++-------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index a5aeee5d..cf41022e 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -1,21 +1,23 @@ -with trade_hashes as (SELECT settlement.solver, - t.block_number as block_number, - order_uid, - fee_amount, - settlement.tx_hash, - auction_id - FROM trades t - LEFT OUTER JOIN LATERAL ( - SELECT tx_hash, solver, tx_nonce, tx_from, auction_id, block_number, log_index - FROM settlements s - WHERE s.block_number = t.block_number - AND s.log_index > t.log_index - ORDER BY s.log_index ASC - LIMIT 1 - ) AS settlement ON true - join settlement_observations so on - settlement.block_number = so.block_number and settlement.log_index = so.log_index - where t.block_number > {{start_block}} and t.block_number <= {{end_block}}), +with trade_hashes as ( + SELECT settlement.solver, + t.block_number as block_number, + order_uid, + fee_amount, + settlement.tx_hash, + auction_id + FROM trades t + LEFT OUTER JOIN LATERAL ( + SELECT tx_hash, solver, tx_nonce, tx_from, auction_id, block_number, log_index + FROM settlements s + WHERE s.block_number = t.block_number + AND s.log_index > t.log_index + ORDER BY s.log_index ASC + LIMIT 1 + ) AS settlement ON true + join settlement_observations so on + settlement.block_number = so.block_number and settlement.log_index = so.log_index + where settlement.block_number > {{start_block}} and settlement.block_number <= {{end_block}} + ), order_surplus AS ( SELECT s.auction_id, @@ -119,7 +121,7 @@ winning_quotes as ( AND t.block_number <= {{end_block}} AND oq.solver != '\x0000000000000000000000000000000000000000') -- Most efficient column order for sorting would be having tx_hash or order_uid first -select trade_hashes.block_number as block_number, +select block_number, concat('0x', encode(trade_hashes.order_uid, 'hex')) as order_uid, concat('0x', encode(solver, 'hex')) as solver, quote_solver, From cf28b4372615e7279e894538479fcaf7a78d007a Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 1 Feb 2024 13:27:51 +0200 Subject: [PATCH 12/12] commit to explicit block number reference --- src/sql/orderbook/order_rewards.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index cf41022e..e5785f2b 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -121,7 +121,7 @@ winning_quotes as ( AND t.block_number <= {{end_block}} AND oq.solver != '\x0000000000000000000000000000000000000000') -- Most efficient column order for sorting would be having tx_hash or order_uid first -select block_number, +select trade_hashes.block_number as block_number, concat('0x', encode(trade_hashes.order_uid, 'hex')) as order_uid, concat('0x', encode(solver, 'hex')) as solver, quote_solver,