From 00f19da2f3bf57d2c8206caafd8acbc1c97e9b45 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 11 Jan 2024 16:12:24 +0100 Subject: [PATCH 01/33] add new column simulation_block to settlement_scores table --- populate_cip_20.sql | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/populate_cip_20.sql b/populate_cip_20.sql index 2b309ed0..0c8995db 100644 --- a/populate_cip_20.sql +++ b/populate_cip_20.sql @@ -34,11 +34,12 @@ CREATE TABLE IF NOT EXISTS auction_participants CREATE TABLE IF NOT EXISTS settlement_scores ( - auction_id bigint PRIMARY KEY, - block_deadline bigint NOT NULL, - winner bytea NOT NULL, - winning_score numeric(78, 0) NOT NULL, - reference_score numeric(78, 0) NOT NULL + auction_id bigint PRIMARY KEY, + winner bytea NOT NULL, + winning_score numeric(78, 0) NOT NULL, + reference_score numeric(78, 0) NOT NULL, + block_deadline bigint NOT NULL, + simulation_block bigint NOT NULL ); -- Populated after block finalization via transactionReceipt. @@ -104,16 +105,16 @@ VALUES (1, '\x5222222222222222222222222222222222222222'::bytea), (10, '\x5444444444444444444444444444444444444444'::bytea), (10, '\x5333333333333333333333333333333333333333'::bytea); -INSERT INTO settlement_scores (auction_id, block_deadline, winning_score, reference_score, winner) -VALUES (1, 10, 5000000000000000000, 4000000000000000000, '\x5111111111111111111111111111111111111111'::bytea), - (2, 11, 6000000000000000000, 3000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), - (3, 12, 21000000000000000000, 3000000000000000000, '\x5111111111111111111111111111111111111111'::bytea), - (5, 14, 5000000000000000000, 3000000000000000000, '\x5111111111111111111111111111111111111111'::bytea), -- jump in auction id - (6, 15, 10000000000000000000, 9000000000000000000, '\x5111111111111111111111111111111111111111'::bytea), -- settled too late - (7, 30, 5000000000000000000, 0, '\x5111111111111111111111111111111111111111'::bytea), -- no competition - (8, 35, 5000000000000000000, 0, '\x5111111111111111111111111111111111111111'::bytea), -- no competition, failed transaction - (9, 36, 5000000000000000000, 1000000000000000000, '\x5111111111111111111111111111111111111111'::bytea), -- score larger than quality - (10, 37, 5000000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea); -- participant with net negative payment +INSERT INTO settlement_scores (auction_id, winning_score, reference_score, winner, block_deadline, simulation_block) +VALUES (1, 5000000000000000000, 4000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 10, 0), + (2, 6000000000000000000, 3000000000000000000, '\x5222222222222222222222222222222222222222'::bytea, 11, 1), + (3, 21000000000000000000, 3000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 12, 2), + (5, 5000000000000000000, 3000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 14, 4), -- jump in auction id + (6, 10000000000000000000, 9000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 15, 5), -- settled too late + (7, 5000000000000000000, 0, '\x5111111111111111111111111111111111111111'::bytea, 30, 6), -- no competition + (8, 5000000000000000000, 0, '\x5111111111111111111111111111111111111111'::bytea, 35, 7), -- no competition, failed transaction + (9, 5000000000000000000, 1000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 36, 8), -- score larger than quality + (10, 5000000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea, 37, 9); -- participant with net negative payment INSERT INTO settlement_observations (block_number, log_index, gas_used, effective_gas_price, surplus, fee) VALUES (1, 0, 100000, 2000000000, 6000000000000000000, 200000000000000), From ef1bb80194cdb4bf696f56cd9eea70afc12502a4 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 11 Jan 2024 16:14:03 +0100 Subject: [PATCH 02/33] rename test db script --- populate_cip_20.sql => populate_test_db.sql | 0 tests/queries/test_batch_rewards.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename populate_cip_20.sql => populate_test_db.sql (100%) diff --git a/populate_cip_20.sql b/populate_test_db.sql similarity index 100% rename from populate_cip_20.sql rename to populate_test_db.sql diff --git a/tests/queries/test_batch_rewards.py b/tests/queries/test_batch_rewards.py index a5feb01f..f5c0d211 100644 --- a/tests/queries/test_batch_rewards.py +++ b/tests/queries/test_batch_rewards.py @@ -10,7 +10,7 @@ class TestBatchRewards(unittest.TestCase): def setUp(self) -> None: db_url = "postgres:postgres@localhost:5432/postgres" self.fetcher = MultiInstanceDBFetcher([db_url]) - with open("./populate_cip_20.sql", "r", encoding="utf-8") as file: + with open("./populate_test_db.sql", "r", encoding="utf-8") as file: self.fetcher.connections[0].execute(file.read()) def test_get_batch_rewards(self): From 41aba0be5fbcdca9ea3cdb531c218ce016f152c2 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 11 Jan 2024 17:44:18 +0100 Subject: [PATCH 03/33] add test for order quotes this adds the tables orders, order_quotes, trades to the test database. a new test using this data was added. it covers - market orders (normal reward) - partially fillable orders (no reward) - in market limit orders (no reward) --- populate_test_db.sql | 118 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 106 insertions(+), 12 deletions(-) diff --git a/populate_test_db.sql b/populate_test_db.sql index 0c8995db..9e8c7f42 100644 --- a/populate_test_db.sql +++ b/populate_test_db.sql @@ -3,6 +3,13 @@ DROP TABLE IF EXISTS auction_transaction; DROP TABLE IF EXISTS auction_participants; DROP TABLE IF EXISTS settlement_scores; DROP TABLE IF EXISTS settlement_observations; +DROP TABLE IF EXISTS orders; +DROP TYPE IF EXISTS OrderKind; +DROP TYPE IF EXISTS SigningScheme; +DROP TYPE IF EXISTS TokenBalance; +DROP TYPE IF EXISTS OrderClass; +DROP TABLE IF EXISTS order_quotes; +DROP TABLE IF EXISTS trades; CREATE TABLE IF NOT EXISTS settlements ( @@ -55,21 +62,78 @@ CREATE TABLE IF NOT EXISTS settlement_observations PRIMARY KEY (block_number, log_index) ); +-- orders table +CREATE TYPE OrderKind AS ENUM ('buy', 'sell'); +CREATE TYPE SigningScheme AS ENUM ('presign', 'eip712', 'ethsign'); +CREATE TYPE TokenBalance AS ENUM ('erc20', 'external'); +CREATE TYPE OrderClass AS ENUM ('market', 'limit'); + +CREATE TABLE orders ( + uid bytea PRIMARY KEY, + owner bytea NOT NULL, + creation_timestamp timestamptz NOT NULL, + sell_token bytea NOT NULL, + buy_token bytea NOT NULL, + sell_amount numeric(78,0) NOT NULL, + buy_amount numeric(78,0) NOT NULL, + valid_to bigint NOT NULL, + fee_amount numeric(78,0) NOT NULL, + kind OrderKind NOT NULL, + partially_fillable boolean NOT NULL, + signature bytea NOT NULL, + cancellation_timestamp timestamptz, + receiver bytea NOT NULL, + app_data bytea NOT NULL, + signing_scheme SigningScheme NOT NULL, + settlement_contract bytea NOT NULL, + sell_token_balance TokenBalance NOT NULL, + buy_token_balance TokenBalance NOT NULL, + full_fee_amount numeric(78,0) NOT NULL, + class OrderClass NOT NULL +); + +CREATE TABLE IF NOT EXISTS order_quotes +( + order_uid bytea PRIMARY KEY, + gas_amount double precision NOT NULL, + gas_price double precision NOT NULL, + sell_token_price double precision NOT NULL, + sell_amount numeric(78, 0) NOT NULL, + buy_amount numeric(78, 0) NOT NULL, + solver bytea NOT NULL +); + +CREATE TABLE IF NOT EXISTS trades +( + block_number bigint NOT NULL, + log_index bigint NOT NULL, + order_uid bytea NOT NULL, + sell_amount numeric(78, 0) NOT NULL, + buy_amount numeric(78, 0) NOT NULL, + fee_amount numeric(78, 0) NOT NULL, + + PRIMARY KEY (block_number, log_index) +); + TRUNCATE settlements; TRUNCATE auction_transaction; TRUNCATE auction_participants; TRUNCATE settlement_scores; TRUNCATE settlement_observations; +TRUNCATE orders; +TRUNCATE order_quotes; +TRUNCATE trades; + INSERT INTO settlements (block_number, log_index, solver, tx_hash, tx_from, tx_nonce) -VALUES (1, 0, '\x5111111111111111111111111111111111111111'::bytea, '\x7111'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 1), - (2, 0, '\x5222222222222222222222222222222222222222'::bytea, '\x7222'::bytea, '\x5222222222222222222222222222222222222222'::bytea, 1), - (5, 0, '\x5111111111111111111111111111111111111111'::bytea, '\x7333'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 2), +VALUES (1, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7111'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 1), + (2, 10, '\x5222222222222222222222222222222222222222'::bytea, '\x7222'::bytea, '\x5222222222222222222222222222222222222222'::bytea, 1), + (5, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7333'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 2), -- would the following entry be in the data base? (submitted too late) -- YES - (20, 0, '\x5111111111111111111111111111111111111111'::bytea, '\x7444'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 3), - (25, 0, '\x5111111111111111111111111111111111111111'::bytea, '\x7555'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 4), - (26, 0, '\x5111111111111111111111111111111111111111'::bytea, '\x7666'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 6); + (20, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7444'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 3), + (25, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7555'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 4), + (26, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7666'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 6); INSERT INTO auction_transaction (auction_id, tx_from, tx_nonce) VALUES (1, '\x5111111111111111111111111111111111111111'::bytea, 1), @@ -117,11 +181,41 @@ VALUES (1, 5000000000000000000, 4000000000000000000, '\x511111111111111111111111 (10, 5000000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea, 37, 9); -- participant with net negative payment INSERT INTO settlement_observations (block_number, log_index, gas_used, effective_gas_price, surplus, fee) -VALUES (1, 0, 100000, 2000000000, 6000000000000000000, 200000000000000), - (2, 0, 150000, 3000000000, 12000000000000000000, 400000000000000), - (5, 0, 100000, 2000000000, 5000000000000000000, 250000000000000), +VALUES (1, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), + (2, 10, 150000, 3000000000, 12000000000000000000, 400000000000000), + (5, 10, 100000, 2000000000, 5000000000000000000, 250000000000000), -- Depends on backend (setting surplus and fee to zero). -- I would prefer to use the real numbers. What is backend gonna do. - (20, 0, 100000, 2000000000, 0, 0), -- would that entry be in the data base? (submitted too late) - (25, 0, 100000, 2000000000, 6000000000000000000, 200000000000000), - (26, 0, 100000, 2000000000, 0, 400000000000000); + (20, 10, 100000, 2000000000, 0, 0), -- would that entry be in the data base? (submitted too late) + (25, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), + (26, 10, 100000, 2000000000, 0, 400000000000000); + +INSERT INTO orders (uid, owner, creation_timestamp, sell_token, buy_token, sell_amount, buy_amount, valid_to, fee_amount, kind, partially_fillable, signature, cancellation_timestamp, receiver, app_data, signing_scheme, settlement_contract, sell_token_balance, buy_token_balance, full_fee_amount, class) +VALUES ('\x1111'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 100000000000000000, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'market'), -- normal sell market order +('\x2222'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 100000000000000000, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'market'), -- normal buy market order +('\x3333'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 100000000000000000, 'sell', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'limit'), -- partially fillable sell limit order +('\x4444'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 0, 'buy', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- partially fillable buy limit order +('\x5555'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 0, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- in market sell limit order +('\x6666'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 0, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'); -- in market buy limit order + +INSERT INTO order_quotes (order_uid, gas_amount, gas_price, sell_token_price, sell_amount, buy_amount, solver) +VALUES ('\x1111'::bytea, 200000, 110000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5111111111111111111111111111111111111111'::bytea), +('\x2222'::bytea, 200000, 110000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea), +('\x3333'::bytea, 200000, 110000000000, 0.123, 6000000000000000000, 3000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), +('\x4444'::bytea, 200000, 110000000000, 0.123, 7000000000000000000, 4000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), +('\x5555'::bytea, 200000, 110000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5333333333333333333333333333333333333333'::bytea), +('\x6666'::bytea, 200000, 110000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5444444444444444444444444444444444444444'::bytea); + +INSERT INTO trades (block_number, log_index, order_uid, sell_amount, buy_amount, fee_amount) +VALUES (1, 0, '\x1111'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), +(2, 0, '\x2222'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000), +(2, 1, '\x3333'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), +(5, 0, '\x4444'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000), +(5, 1, '\x5555'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), +(20, 0, '\x6666'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000); From cc6dfc5a8f0221ef3c2374a53bd287230d24b28e Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 11 Jan 2024 17:51:53 +0100 Subject: [PATCH 04/33] forgot to add actual test --- tests/queries/test_quote_rewards.py | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/queries/test_quote_rewards.py diff --git a/tests/queries/test_quote_rewards.py b/tests/queries/test_quote_rewards.py new file mode 100644 index 00000000..98a15bf9 --- /dev/null +++ b/tests/queries/test_quote_rewards.py @@ -0,0 +1,37 @@ +import unittest + +import pandas.testing +from pandas import DataFrame + +from src.pg_client import MultiInstanceDBFetcher + + +class TestBatchRewards(unittest.TestCase): + def setUp(self) -> None: + db_url = "postgres:postgres@localhost:5432/postgres" + self.fetcher = MultiInstanceDBFetcher([db_url]) + with open("./populate_test_db.sql", "r", encoding="utf-8") as file: + self.fetcher.connections[0].execute(file.read()) + + def test_get_batch_rewards(self): + start_block, end_block = "0", "100" + batch_rewards = self.fetcher.get_quote_rewards(start_block, end_block) + expected = DataFrame( + { + "solver": [ + "0x5111111111111111111111111111111111111111", + "0x5333333333333333333333333333333333333333", + "0x5444444444444444444444444444444444444444", + ], + "num_quotes": [ + 1, + 2, + 1, + ], + } + ) + self.assertIsNone(pandas.testing.assert_frame_equal(expected, batch_rewards)) + + +if __name__ == "__main__": + unittest.main() From fe9e5aaf1b8c5cce1cbcf69196f53200accc1634 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 12 Jan 2024 10:29:43 +0100 Subject: [PATCH 05/33] add protocol fee data to test database - new table fee_policy - changes old order data to give nicer numbers (quote test does not change) --- populate_test_db.sql | 58 ++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/populate_test_db.sql b/populate_test_db.sql index 9e8c7f42..32455aac 100644 --- a/populate_test_db.sql +++ b/populate_test_db.sql @@ -10,6 +10,8 @@ DROP TYPE IF EXISTS TokenBalance; DROP TYPE IF EXISTS OrderClass; DROP TABLE IF EXISTS order_quotes; DROP TABLE IF EXISTS trades; +DROP TABLE IF EXISTS fee_policies; +DROP TYPE IF EXISTS PolicyKind; CREATE TABLE IF NOT EXISTS settlements ( @@ -115,6 +117,23 @@ CREATE TABLE IF NOT EXISTS trades PRIMARY KEY (block_number, log_index) ); +CREATE TYPE PolicyKind AS ENUM ('priceimprovement', 'volume'); + +CREATE TABLE fee_policies ( + auction_id bigint NOT NULL, + order_uid bytea NOT NULL, + -- The order in which the fee policies are inserted and applied. + application_order SERIAL NOT NULL, + -- The type of the fee policy. + kind PolicyKind NOT NULL, + -- The fee should be taken as a percentage of the price improvement. The value is between 0 and 1. + price_improvement_factor double precision, + -- Cap the fee at a certain percentage of the order volume. The value is between 0 and 1. + max_volume_factor double precision, + -- The fee should be taken as a percentage of the order volume. The value is between 0 and 1. + volume_factor double precision, + PRIMARY KEY (auction_id, order_uid, application_order) +); TRUNCATE settlements; TRUNCATE auction_transaction; @@ -124,6 +143,7 @@ TRUNCATE settlement_observations; TRUNCATE orders; TRUNCATE order_quotes; TRUNCATE trades; +TRUNCATE fee_policies; INSERT INTO settlements (block_number, log_index, solver, tx_hash, tx_from, tx_nonce) @@ -192,30 +212,36 @@ VALUES (1, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), INSERT INTO orders (uid, owner, creation_timestamp, sell_token, buy_token, sell_amount, buy_amount, valid_to, fee_amount, kind, partially_fillable, signature, cancellation_timestamp, receiver, app_data, signing_scheme, settlement_contract, sell_token_balance, buy_token_balance, full_fee_amount, class) VALUES ('\x1111'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 100000000000000000, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'market'), -- normal sell market order +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1700000000, 200000000000000000, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'market'), -- sell market order ('\x2222'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 100000000000000000, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'market'), -- normal buy market order +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1700000000, 200000000000000000, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'market'), -- buy market order ('\x3333'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 100000000000000000, 'sell', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'limit'), -- partially fillable sell limit order +', '\x11'::bytea, '\x22'::bytea, 6200000000000000000, 4000000000000000000, 1700000000, 0, 'sell', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- partially fillable sell limit order ('\x4444'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 0, 'buy', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- partially fillable buy limit order +', '\x11'::bytea, '\x22'::bytea, 6200000000000000000, 4000000000000000000, 1700000000, 0, 'buy', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- partially fillable buy limit order ('\x5555'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 0, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- in market sell limit order +', '\x11'::bytea, '\x22'::bytea, 6200000000000000000, 4000000000000000000, 1700000000, 0, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- in market sell limit order ('\x6666'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 0, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'); -- in market buy limit order +', '\x11'::bytea, '\x22'::bytea, 6200000000000000000, 4000000000000000000, 1700000000, 0, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'); -- in market buy limit order INSERT INTO order_quotes (order_uid, gas_amount, gas_price, sell_token_price, sell_amount, buy_amount, solver) -VALUES ('\x1111'::bytea, 200000, 110000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5111111111111111111111111111111111111111'::bytea), -('\x2222'::bytea, 200000, 110000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea), -('\x3333'::bytea, 200000, 110000000000, 0.123, 6000000000000000000, 3000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), -('\x4444'::bytea, 200000, 110000000000, 0.123, 7000000000000000000, 4000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), -('\x5555'::bytea, 200000, 110000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5333333333333333333333333333333333333333'::bytea), -('\x6666'::bytea, 200000, 110000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5444444444444444444444444444444444444444'::bytea); +VALUES ('\x1111'::bytea, 200000, 100000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5111111111111111111111111111111111111111'::bytea), +('\x2222'::bytea, 200000, 100000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea), +('\x3333'::bytea, 200000, 100000000000, 0.123, 6000000000000000000, 3000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), +('\x4444'::bytea, 200000, 100000000000, 0.123, 7000000000000000000, 4000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), +('\x5555'::bytea, 200000, 100000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5333333333333333333333333333333333333333'::bytea), +('\x6666'::bytea, 200000, 100000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5444444444444444444444444444444444444444'::bytea); INSERT INTO trades (block_number, log_index, order_uid, sell_amount, buy_amount, fee_amount) VALUES (1, 0, '\x1111'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), (2, 0, '\x2222'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000), -(2, 1, '\x3333'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), -(5, 0, '\x4444'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000), -(5, 1, '\x5555'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), -(20, 0, '\x6666'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000); +(2, 1, '\x3333'::bytea, 6200000000000000000, 4600000000000000000, 0), +(5, 0, '\x4444'::bytea, 5600000000000000000, 4000000000000000000, 0), +(5, 1, '\x5555'::bytea, 6200000000000000000, 4600000000000000000, 0), +(20, 0, '\x6666'::bytea, 5600000000000000000, 4000000000000000000, 0); + +INSERT INTO fee_policies (auction_id, order_uid, application_order, kind, price_improvement_factor, max_volume_factor, volume_factor) +VALUES (2, '\x3333'::bytea, 3, 'priceimprovement', 0.5, 0.01, NULL), +(5, '\x4444'::bytea, 4, 'priceimprovement', 0.2, 0.1, NULL), +(5, '\x5555'::bytea, 5, 'volume', NULL, NULL, 0.0015), +(6, '\x6666'::bytea, 6, 'priceimprovement', 0.2, 0.1, NULL); From 6cea067cd577f6539755a5bb2492f94c719a7f12 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 15 Jan 2024 08:57:21 +0100 Subject: [PATCH 06/33] update database --- populate_test_db.sql | 60 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/populate_test_db.sql b/populate_test_db.sql index 32455aac..8ebb02a8 100644 --- a/populate_test_db.sql +++ b/populate_test_db.sql @@ -3,6 +3,7 @@ DROP TABLE IF EXISTS auction_transaction; DROP TABLE IF EXISTS auction_participants; DROP TABLE IF EXISTS settlement_scores; DROP TABLE IF EXISTS settlement_observations; +DROP TABLE IF EXISTS auction_prices; DROP TABLE IF EXISTS orders; DROP TYPE IF EXISTS OrderKind; DROP TYPE IF EXISTS SigningScheme; @@ -10,6 +11,7 @@ DROP TYPE IF EXISTS TokenBalance; DROP TYPE IF EXISTS OrderClass; DROP TABLE IF EXISTS order_quotes; DROP TABLE IF EXISTS trades; +DROP TABLE IF EXISTS order_executions; DROP TABLE IF EXISTS fee_policies; DROP TYPE IF EXISTS PolicyKind; @@ -64,6 +66,15 @@ CREATE TABLE IF NOT EXISTS settlement_observations PRIMARY KEY (block_number, log_index) ); +CREATE TABLE IF NOT EXISTS auction_prices +( + auction_id bigint NOT NULL, + token bytea NOT NULL, + price numeric(78, 0) NOT NULL, + + PRIMARY KEY (auction_id, token) +); + -- orders table CREATE TYPE OrderKind AS ENUM ('buy', 'sell'); CREATE TYPE SigningScheme AS ENUM ('presign', 'eip712', 'ethsign'); @@ -117,6 +128,17 @@ CREATE TABLE IF NOT EXISTS trades PRIMARY KEY (block_number, log_index) ); +CREATE TABLE IF NOT EXISTS order_executions +( + order_uid bytea NOT NULL, + auction_id bigint NOT NULL, + reward double precision NOT NULL, + surplus_fee numeric(78, 0) NOT NULL, + solver_fee numeric(78, 0), + + PRIMARY KEY (order_uid, auction_id) +); + CREATE TYPE PolicyKind AS ENUM ('priceimprovement', 'volume'); CREATE TABLE fee_policies ( @@ -140,6 +162,7 @@ TRUNCATE auction_transaction; TRUNCATE auction_participants; TRUNCATE settlement_scores; TRUNCATE settlement_observations; +TRUNCATE auction_prices; TRUNCATE orders; TRUNCATE order_quotes; TRUNCATE trades; @@ -210,19 +233,30 @@ VALUES (1, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), (25, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), (26, 10, 100000, 2000000000, 0, 400000000000000); + +INSERT INTO auction_prices (auction_id, token, price) +VALUES (1, '\x11', 1000000000000000000), +(1, '\x22', 1200000000000000000), +(2, '\x11', 1000000000000000000), +(2, '\x22', 1200000000000000000), +(5, '\x11', 1000000000000000000), +(5, '\x22', 1200000000000000000), +(6, '\x11', 1000000000000000000), +(6, '\x22', 1200000000000000000); + INSERT INTO orders (uid, owner, creation_timestamp, sell_token, buy_token, sell_amount, buy_amount, valid_to, fee_amount, kind, partially_fillable, signature, cancellation_timestamp, receiver, app_data, signing_scheme, settlement_contract, sell_token_balance, buy_token_balance, full_fee_amount, class) VALUES ('\x1111'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1700000000, 200000000000000000, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'market'), -- sell market order +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1700000000, 20000000000000000, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 20000000000000000, 'market'), -- sell market order ('\x2222'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1700000000, 200000000000000000, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'market'), -- buy market order +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1700000000, 20000000000000000, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 20000000000000000, 'market'), -- buy market order ('\x3333'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6200000000000000000, 4000000000000000000, 1700000000, 0, 'sell', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- partially fillable sell limit order +', '\x11'::bytea, '\x22'::bytea, 6020000000000000000, 4000000000000000000, 1700000000, 0, 'sell', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- partially fillable sell limit order ('\x4444'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6200000000000000000, 4000000000000000000, 1700000000, 0, 'buy', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- partially fillable buy limit order +', '\x11'::bytea, '\x22'::bytea, 6020000000000000000, 4000000000000000000, 1700000000, 0, 'buy', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- partially fillable buy limit order ('\x5555'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6200000000000000000, 4000000000000000000, 1700000000, 0, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- in market sell limit order +', '\x11'::bytea, '\x22'::bytea, 6020000000000000000, 4000000000000000000, 1700000000, 0, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- in market sell limit order ('\x6666'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6200000000000000000, 4000000000000000000, 1700000000, 0, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'); -- in market buy limit order +', '\x11'::bytea, '\x22'::bytea, 6020000000000000000, 4000000000000000000, 1700000000, 0, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'); -- in market buy limit order INSERT INTO order_quotes (order_uid, gas_amount, gas_price, sell_token_price, sell_amount, buy_amount, solver) VALUES ('\x1111'::bytea, 200000, 100000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5111111111111111111111111111111111111111'::bytea), @@ -233,13 +267,19 @@ VALUES ('\x1111'::bytea, 200000, 100000000000, 0.123, 6000000000000000000, 45000 ('\x6666'::bytea, 200000, 100000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5444444444444444444444444444444444444444'::bytea); INSERT INTO trades (block_number, log_index, order_uid, sell_amount, buy_amount, fee_amount) -VALUES (1, 0, '\x1111'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), -(2, 0, '\x2222'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000), -(2, 1, '\x3333'::bytea, 6200000000000000000, 4600000000000000000, 0), +VALUES (1, 0, '\x1111'::bytea, 6020000000000000000, 4600000000000000000, 20000000000000000), +(2, 0, '\x2222'::bytea, 5600000000000000000, 4000000000000000000, 20000000000000000), +(2, 1, '\x3333'::bytea, 6020000000000000000, 4600000000000000000, 0), (5, 0, '\x4444'::bytea, 5600000000000000000, 4000000000000000000, 0), -(5, 1, '\x5555'::bytea, 6200000000000000000, 4600000000000000000, 0), +(5, 1, '\x5555'::bytea, 6020000000000000000, 4600000000000000000, 0), (20, 0, '\x6666'::bytea, 5600000000000000000, 4000000000000000000, 0); +INSERT INTO order_executions (order_uid, auction_id, reward, surplus_fee, solver_fee) +VALUES ('\x3333'::bytea, 2, 0, 66464646464646470, NULL), +('\x4444'::bytea, 5, 0, 125000000000000000, NULL), +('\x5555'::bytea, 5, 0, 29043565348022034, NULL), +('\x6666'::bytea, 6, 0, 20000000000000000, NULL); + INSERT INTO fee_policies (auction_id, order_uid, application_order, kind, price_improvement_factor, max_volume_factor, volume_factor) VALUES (2, '\x3333'::bytea, 3, 'priceimprovement', 0.5, 0.01, NULL), (5, '\x4444'::bytea, 4, 'priceimprovement', 0.2, 0.1, NULL), From 274c804666e91bd447999550a08fc67a1bcf27ce Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 15 Jan 2024 08:57:33 +0100 Subject: [PATCH 07/33] add protocol fee query --- queries/orderbook/protocol_fees.sql | 105 ++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 queries/orderbook/protocol_fees.sql diff --git a/queries/orderbook/protocol_fees.sql b/queries/orderbook/protocol_fees.sql new file mode 100644 index 00000000..3f568990 --- /dev/null +++ b/queries/orderbook/protocol_fees.sql @@ -0,0 +1,105 @@ +-- Active: 1704993022452@@127.0.0.1@5432@postgres@public +-- Active: 1704193756200@@cow-protocol-db-read-replica.c5bze0gkehvb.eu-central-1.rds.amazonaws.com@5432@mainnet@public +WITH order_surplus AS ( + SELECT + at.auction_id, + s.tx_hash, + t.order_uid, + o.sell_token, + o.buy_token, + t.sell_amount, + t.buy_amount, + oe.surplus_fee, + o.kind, + CASE + WHEN o.kind = 'sell' + THEN t.buy_amount - o.buy_amount / (o.sell_amount + o.fee_amount) * t.sell_amount + WHEN o.kind = 'buy' + THEN (o.sell_amount + o.fee_amount) / o.buy_amount * t.buy_amount - t.sell_amount + END AS surplus, + CASE + WHEN o.kind = 'sell' + THEN o.buy_token + WHEN o.kind = 'buy' + THEN o.sell_token + END AS surplus_token, + CASE + WHEN o.kind = 'sell' + THEN + t.buy_amount - oq.buy_amount / (o.sell_amount + o.fee_amount) * t.sell_amount + WHEN o.kind = 'buy' + THEN + (oq.sell_amount + o.fee_amount) / o.buy_amount * t.buy_amount - t.sell_amount + END AS price_improvement + FROM settlements s -- links block_number and log_index to tx_from and tx_nonce + JOIN auction_transaction at -- links auction_id to tx_from and tx_nonce + ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce + JOIN settlement_scores ss -- contains block_deadline + ON at.auction_id = ss.auction_id + JOIN trades t -- contains traded amounts + ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block + JOIN orders o -- contains tokens and limit amounts + ON t.order_uid = o.uid + JOIN order_executions oe -- contains surplus fee + ON t.order_uid = oe.order_uid AND at.auction_id = oe.auction_id + JOIN order_quotes oq -- contains quote + ON t.order_uid = oq.order_uid + WHERE ss.block_deadline > 0 + AND ss.block_deadline <= 100 +) +,protocol_fees_raw AS ( + SELECT + os.*, -- TODO: select data + CASE + WHEN fp.kind = 'priceimprovement' + THEN + CASE + WHEN os.kind = 'sell' + THEN + GREATEST( + 0, -- never charge a negative amount + LEAST( + fp.max_volume_factor / (1 - fp.max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume + fp.price_improvement_factor * LEAST(price_improvement, surplus) -- charge a fraction of price improvement + ) + ) + WHEN os.kind = 'buy' + THEN + GREATEST( + 0, -- never charge a negative amount + LEAST( + fp.max_volume_factor / (1 - fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume + fp.price_improvement_factor / (1 - fp.price_improvement_factor) * LEAST(price_improvement, surplus) -- charge a fraction of price improvement + ) + ) + END + WHEN fp.kind = 'volume' + THEN fp.volume_factor / (1 - fp.volume_factor) * os.sell_amount + END AS protocol_fee, + CASE + WHEN fp.kind = 'priceimprovement' + THEN os.surplus_token + WHEN fp.kind = 'volume' + THEN os.sell_token + END AS protocol_fee_token + FROM order_surplus os + JOIN fee_policies fp -- contains protocol fee policy + ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid +) +,order_fees AS ( + SELECT + pfr.*, -- TODO: select data + ap_sell.price as sell_token_price, + ap_protocol.price as protocol_fee_token_price, + CASE + WHEN pfr.sell_token != pfr.protocol_fee_token + THEN ap_sell.price / pow(10, 18) * (pfr.surplus_fee - (pfr.sell_amount - pfr.surplus_fee) / pfr.buy_amount * pfr.protocol_fee) + ELSE ap_sell.price / pow(10, 18) * (pfr.surplus_fee - pfr.protocol_fee) + END AS network_fee + FROM protocol_fees_raw pfr + JOIN auction_prices ap_sell -- contains prices + ON pfr.auction_id = ap_sell.auction_id AND pfr.sell_token = ap_sell.token + JOIN auction_prices ap_protocol -- contains prices + ON pfr.auction_id = ap_protocol.auction_id AND pfr.protocol_fee_token = ap_protocol.token +) +select * from order_fees From 93d209edd2b646972cee1bff67ea701eba0458b1 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 15 Jan 2024 10:18:25 +0100 Subject: [PATCH 08/33] fix typo --- tests/queries/test_quote_rewards.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/test_quote_rewards.py b/tests/queries/test_quote_rewards.py index 98a15bf9..5ee8baa1 100644 --- a/tests/queries/test_quote_rewards.py +++ b/tests/queries/test_quote_rewards.py @@ -6,16 +6,16 @@ from src.pg_client import MultiInstanceDBFetcher -class TestBatchRewards(unittest.TestCase): +class TestQuoteRewards(unittest.TestCase): def setUp(self) -> None: db_url = "postgres:postgres@localhost:5432/postgres" self.fetcher = MultiInstanceDBFetcher([db_url]) with open("./populate_test_db.sql", "r", encoding="utf-8") as file: self.fetcher.connections[0].execute(file.read()) - def test_get_batch_rewards(self): + def test_get_quote_rewards(self): start_block, end_block = "0", "100" - batch_rewards = self.fetcher.get_quote_rewards(start_block, end_block) + quote_rewards = self.fetcher.get_quote_rewards(start_block, end_block) expected = DataFrame( { "solver": [ @@ -30,7 +30,7 @@ def test_get_batch_rewards(self): ], } ) - self.assertIsNone(pandas.testing.assert_frame_equal(expected, batch_rewards)) + self.assertIsNone(pandas.testing.assert_frame_equal(expected, quote_rewards)) if __name__ == "__main__": From 0db278595c46ca2cb85f04ac3f9e2d272e8593f5 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 15 Jan 2024 13:21:55 +0100 Subject: [PATCH 09/33] update database and query - change `priceimprovement` to `surplus` - remove price improvement from query - typo in order_execution table --- populate_test_db.sql | 18 +++++++------- queries/orderbook/protocol_fees.sql | 38 ++++++++--------------------- 2 files changed, 19 insertions(+), 37 deletions(-) diff --git a/populate_test_db.sql b/populate_test_db.sql index 8ebb02a8..38b2baa3 100644 --- a/populate_test_db.sql +++ b/populate_test_db.sql @@ -11,7 +11,7 @@ DROP TYPE IF EXISTS TokenBalance; DROP TYPE IF EXISTS OrderClass; DROP TABLE IF EXISTS order_quotes; DROP TABLE IF EXISTS trades; -DROP TABLE IF EXISTS order_executions; +DROP TABLE IF EXISTS order_execution; DROP TABLE IF EXISTS fee_policies; DROP TYPE IF EXISTS PolicyKind; @@ -128,7 +128,7 @@ CREATE TABLE IF NOT EXISTS trades PRIMARY KEY (block_number, log_index) ); -CREATE TABLE IF NOT EXISTS order_executions +CREATE TABLE IF NOT EXISTS order_execution ( order_uid bytea NOT NULL, auction_id bigint NOT NULL, @@ -139,7 +139,7 @@ CREATE TABLE IF NOT EXISTS order_executions PRIMARY KEY (order_uid, auction_id) ); -CREATE TYPE PolicyKind AS ENUM ('priceimprovement', 'volume'); +CREATE TYPE PolicyKind AS ENUM ('surplus', 'volume'); CREATE TABLE fee_policies ( auction_id bigint NOT NULL, @@ -149,7 +149,7 @@ CREATE TABLE fee_policies ( -- The type of the fee policy. kind PolicyKind NOT NULL, -- The fee should be taken as a percentage of the price improvement. The value is between 0 and 1. - price_improvement_factor double precision, + surplus_factor double precision, -- Cap the fee at a certain percentage of the order volume. The value is between 0 and 1. max_volume_factor double precision, -- The fee should be taken as a percentage of the order volume. The value is between 0 and 1. @@ -274,14 +274,14 @@ VALUES (1, 0, '\x1111'::bytea, 6020000000000000000, 4600000000000000000, 2000000 (5, 1, '\x5555'::bytea, 6020000000000000000, 4600000000000000000, 0), (20, 0, '\x6666'::bytea, 5600000000000000000, 4000000000000000000, 0); -INSERT INTO order_executions (order_uid, auction_id, reward, surplus_fee, solver_fee) +INSERT INTO order_execution (order_uid, auction_id, reward, surplus_fee, solver_fee) VALUES ('\x3333'::bytea, 2, 0, 66464646464646470, NULL), ('\x4444'::bytea, 5, 0, 125000000000000000, NULL), ('\x5555'::bytea, 5, 0, 29043565348022034, NULL), ('\x6666'::bytea, 6, 0, 20000000000000000, NULL); -INSERT INTO fee_policies (auction_id, order_uid, application_order, kind, price_improvement_factor, max_volume_factor, volume_factor) -VALUES (2, '\x3333'::bytea, 3, 'priceimprovement', 0.5, 0.01, NULL), -(5, '\x4444'::bytea, 4, 'priceimprovement', 0.2, 0.1, NULL), +INSERT INTO fee_policies (auction_id, order_uid, application_order, kind, surplus_factor, max_volume_factor, volume_factor) +VALUES (2, '\x3333'::bytea, 3, 'surplus', 0.5, 0.01, NULL), +(5, '\x4444'::bytea, 4, 'surplus', 0.2, 0.1, NULL), (5, '\x5555'::bytea, 5, 'volume', NULL, NULL, 0.0015), -(6, '\x6666'::bytea, 6, 'priceimprovement', 0.2, 0.1, NULL); +(6, '\x6666'::bytea, 6, 'surplus', 0.2, 0.1, NULL); diff --git a/queries/orderbook/protocol_fees.sql b/queries/orderbook/protocol_fees.sql index 3f568990..c97bc415 100644 --- a/queries/orderbook/protocol_fees.sql +++ b/queries/orderbook/protocol_fees.sql @@ -1,5 +1,3 @@ --- Active: 1704993022452@@127.0.0.1@5432@postgres@public --- Active: 1704193756200@@cow-protocol-db-read-replica.c5bze0gkehvb.eu-central-1.rds.amazonaws.com@5432@mainnet@public WITH order_surplus AS ( SELECT at.auction_id, @@ -22,15 +20,7 @@ WITH order_surplus AS ( THEN o.buy_token WHEN o.kind = 'buy' THEN o.sell_token - END AS surplus_token, - CASE - WHEN o.kind = 'sell' - THEN - t.buy_amount - oq.buy_amount / (o.sell_amount + o.fee_amount) * t.sell_amount - WHEN o.kind = 'buy' - THEN - (oq.sell_amount + o.fee_amount) / o.buy_amount * t.buy_amount - t.sell_amount - END AS price_improvement + END AS surplus_token FROM settlements s -- links block_number and log_index to tx_from and tx_nonce JOIN auction_transaction at -- links auction_id to tx_from and tx_nonce ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce @@ -40,10 +30,8 @@ WITH order_surplus AS ( ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block JOIN orders o -- contains tokens and limit amounts ON t.order_uid = o.uid - JOIN order_executions oe -- contains surplus fee + JOIN order_execution oe -- contains surplus fee ON t.order_uid = oe.order_uid AND at.auction_id = oe.auction_id - JOIN order_quotes oq -- contains quote - ON t.order_uid = oq.order_uid WHERE ss.block_deadline > 0 AND ss.block_deadline <= 100 ) @@ -51,33 +39,27 @@ WITH order_surplus AS ( SELECT os.*, -- TODO: select data CASE - WHEN fp.kind = 'priceimprovement' + WHEN fp.kind = 'surplus' THEN CASE WHEN os.kind = 'sell' THEN - GREATEST( - 0, -- never charge a negative amount - LEAST( - fp.max_volume_factor / (1 - fp.max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume - fp.price_improvement_factor * LEAST(price_improvement, surplus) -- charge a fraction of price improvement - ) + LEAST( + fp.max_volume_factor / (1 - fp.max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus ) WHEN os.kind = 'buy' THEN - GREATEST( - 0, -- never charge a negative amount - LEAST( - fp.max_volume_factor / (1 - fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * LEAST(price_improvement, surplus) -- charge a fraction of price improvement - ) + LEAST( + fp.max_volume_factor / (1 - fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus ) END WHEN fp.kind = 'volume' THEN fp.volume_factor / (1 - fp.volume_factor) * os.sell_amount END AS protocol_fee, CASE - WHEN fp.kind = 'priceimprovement' + WHEN fp.kind = 'surplus' THEN os.surplus_token WHEN fp.kind = 'volume' THEN os.sell_token From 41bf6514aad6d6425bb720bdd8322f83a85ca768 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 15 Jan 2024 16:49:31 +0100 Subject: [PATCH 10/33] update query and database - create new test database for protocol fees - add new test for protocol fees --- queries/orderbook/protocol_fees.sql | 60 +++++--- src/pg_client.py | 14 ++ tests/queries/protocol_fee_test_db.sql | 191 +++++++++++++++++++++++++ tests/queries/test_protocol_fees.py | 39 +++++ 4 files changed, 285 insertions(+), 19 deletions(-) create mode 100644 tests/queries/protocol_fee_test_db.sql create mode 100644 tests/queries/test_protocol_fees.py diff --git a/queries/orderbook/protocol_fees.sql b/queries/orderbook/protocol_fees.sql index c97bc415..b27d72b0 100644 --- a/queries/orderbook/protocol_fees.sql +++ b/queries/orderbook/protocol_fees.sql @@ -1,13 +1,13 @@ WITH order_surplus AS ( SELECT + ss.winner as solver, at.auction_id, - s.tx_hash, t.order_uid, - o.sell_token, + o.sell_token, -- the total amount the user sends o.buy_token, t.sell_amount, t.buy_amount, - oe.surplus_fee, + oe.surplus_fee as observed_fee, o.kind, CASE WHEN o.kind = 'sell' @@ -35,9 +35,16 @@ WITH order_surplus AS ( WHERE ss.block_deadline > 0 AND ss.block_deadline <= 100 ) -,protocol_fees_raw AS ( +,order_observation AS ( SELECT - os.*, -- TODO: select data + os.auction_id, + os.solver, + os.sell_amount, -- the total amount the user sends + os.buy_amount, + os.sell_token, + os.observed_fee, + os.surplus, + os.surplus_token, CASE WHEN fp.kind = 'surplus' THEN @@ -68,20 +75,35 @@ WITH order_surplus AS ( JOIN fee_policies fp -- contains protocol fee policy ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid ) -,order_fees AS ( +,order_observations_prices AS ( SELECT - pfr.*, -- TODO: select data - ap_sell.price as sell_token_price, - ap_protocol.price as protocol_fee_token_price, + oo.solver, + oo.surplus, + oo.protocol_fee, CASE - WHEN pfr.sell_token != pfr.protocol_fee_token - THEN ap_sell.price / pow(10, 18) * (pfr.surplus_fee - (pfr.sell_amount - pfr.surplus_fee) / pfr.buy_amount * pfr.protocol_fee) - ELSE ap_sell.price / pow(10, 18) * (pfr.surplus_fee - pfr.protocol_fee) - END AS network_fee - FROM protocol_fees_raw pfr - JOIN auction_prices ap_sell -- contains prices - ON pfr.auction_id = ap_sell.auction_id AND pfr.sell_token = ap_sell.token - JOIN auction_prices ap_protocol -- contains prices - ON pfr.auction_id = ap_protocol.auction_id AND pfr.protocol_fee_token = ap_protocol.token + WHEN oo.sell_token != oo.protocol_fee_token + THEN oo.observed_fee - (oo.sell_amount - oo.observed_fee) / oo.buy_amount * oo.protocol_fee + ELSE oo.observed_fee - oo.protocol_fee + END AS network_fee, + oo.sell_token as network_fee_token, + ap_surplus.price / pow(10, 18) as surplus_token_price, + ap_protocol.price / pow(10, 18) as protocol_fee_token_price, + ap_sell.price / pow(10, 18) as network_fee_token_price + FROM order_observation oo + JOIN auction_prices ap_sell -- contains price: sell token + ON oo.auction_id = ap_sell.auction_id AND oo.sell_token = ap_sell.token + JOIN auction_prices ap_surplus -- contains price: surplus token + ON oo.auction_id = ap_surplus.auction_id AND oo.surplus_token = ap_surplus.token + JOIN auction_prices ap_protocol -- contains price: protocol fee token + ON oo.auction_id = ap_protocol.auction_id AND oo.protocol_fee_token = ap_protocol.token +), +batch_aggregate AS ( + SELECT + concat('0x', encode(solver, 'hex')) as solver, + -- sum(surplus * surplus_token_price) as surplus, + sum(protocol_fee * protocol_fee_token_price) as protocol_fee, + sum(network_fee * network_fee_token_price) as network_fee + FROM order_observations_prices oop + group by solver ) -select * from order_fees +SELECT * FROM batch_aggregate diff --git a/src/pg_client.py b/src/pg_client.py index 6854716d..bddc0815 100644 --- a/src/pg_client.py +++ b/src/pg_client.py @@ -57,6 +57,20 @@ def get_quote_rewards(self, start_block: str, end_block: str) -> DataFrame: return pd.concat(results) + def get_protocol_fees(self, start_block: str, end_block: str) -> DataFrame: + """Returns aggregated solver protocol fees for block range""" + protocol_fee_query = ( + open_query("orderbook/protocol_fees.sql") + .replace("{{start_block}}", start_block) + .replace("{{end_block}}", end_block) + ) + results = [ + self.exec_query(query=protocol_fee_query, engine=engine) + for engine in self.connections + ] + + return pd.concat(results) + def pg_hex2bytea(hex_address: str) -> str: """ diff --git a/tests/queries/protocol_fee_test_db.sql b/tests/queries/protocol_fee_test_db.sql new file mode 100644 index 00000000..caf98ff4 --- /dev/null +++ b/tests/queries/protocol_fee_test_db.sql @@ -0,0 +1,191 @@ +DROP TABLE IF EXISTS settlements; +DROP TABLE IF EXISTS auction_transaction; +DROP TABLE IF EXISTS settlement_scores; +DROP TABLE IF EXISTS auction_prices; +DROP TABLE IF EXISTS orders; +DROP TYPE IF EXISTS OrderKind; +DROP TYPE IF EXISTS OrderClass; +DROP TABLE IF EXISTS trades; +DROP TABLE IF EXISTS order_execution; +DROP TABLE IF EXISTS fee_policies; +DROP TYPE IF EXISTS PolicyKind; + +CREATE TABLE IF NOT EXISTS settlements +( + block_number bigint NOT NULL, + log_index bigint NOT NULL, + solver bytea NOT NULL, + tx_hash bytea NOT NULL, + tx_from bytea NOT NULL, + tx_nonce bigint NOT NULL, + + PRIMARY KEY (block_number, log_index) +); + +CREATE INDEX settlements_tx_from_tx_nonce ON settlements (tx_from, tx_nonce); + +CREATE TABLE IF NOT EXISTS auction_transaction +( + auction_id bigint PRIMARY KEY, + tx_from bytea NOT NULL, + tx_nonce bigint NOT NULL, + UNIQUE (tx_from, tx_nonce) +); + +CREATE TABLE IF NOT EXISTS auction_participants +( + auction_id bigint, + participant bytea +); + +CREATE TABLE IF NOT EXISTS settlement_scores +( + auction_id bigint PRIMARY KEY, + winner bytea NOT NULL, + winning_score numeric(78, 0) NOT NULL, + reference_score numeric(78, 0) NOT NULL, + block_deadline bigint NOT NULL, + simulation_block bigint NOT NULL +); + +CREATE TABLE IF NOT EXISTS auction_prices +( + auction_id bigint NOT NULL, + token bytea NOT NULL, + price numeric(78, 0) NOT NULL, + + PRIMARY KEY (auction_id, token) +); + +-- orders table +CREATE TYPE OrderKind AS ENUM ('buy', 'sell'); +CREATE TYPE OrderClass AS ENUM ('market', 'limit'); + +CREATE TABLE orders ( + uid bytea PRIMARY KEY, + sell_token bytea NOT NULL, + buy_token bytea NOT NULL, + sell_amount numeric(78,0) NOT NULL, + buy_amount numeric(78,0) NOT NULL, + fee_amount numeric(78,0) NOT NULL, + kind OrderKind NOT NULL, + partially_fillable boolean NOT NULL, + full_fee_amount numeric(78,0) NOT NULL, + class OrderClass NOT NULL +); + +CREATE TABLE IF NOT EXISTS trades +( + block_number bigint NOT NULL, + log_index bigint NOT NULL, + order_uid bytea NOT NULL, + sell_amount numeric(78, 0) NOT NULL, + buy_amount numeric(78, 0) NOT NULL, + fee_amount numeric(78, 0) NOT NULL, + + PRIMARY KEY (block_number, log_index) +); + +CREATE TABLE IF NOT EXISTS order_execution +( + order_uid bytea NOT NULL, + auction_id bigint NOT NULL, + reward double precision NOT NULL, + surplus_fee numeric(78, 0) NOT NULL, + solver_fee numeric(78, 0), + + PRIMARY KEY (order_uid, auction_id) +); + +CREATE TYPE PolicyKind AS ENUM ('surplus', 'volume'); + +CREATE TABLE fee_policies ( + auction_id bigint NOT NULL, + order_uid bytea NOT NULL, + -- The order in which the fee policies are inserted and applied. + application_order SERIAL NOT NULL, + -- The type of the fee policy. + kind PolicyKind NOT NULL, + -- The fee should be taken as a percentage of the price improvement. The value is between 0 and 1. + surplus_factor double precision, + -- Cap the fee at a certain percentage of the order volume. The value is between 0 and 1. + max_volume_factor double precision, + -- The fee should be taken as a percentage of the order volume. The value is between 0 and 1. + volume_factor double precision, + PRIMARY KEY (auction_id, order_uid, application_order) +); + +TRUNCATE settlements; +TRUNCATE auction_transaction; +TRUNCATE settlement_scores; +TRUNCATE auction_prices; +TRUNCATE orders; +TRUNCATE trades; +TRUNCATE fee_policies; + + +INSERT INTO settlements (block_number, log_index, solver, tx_hash, tx_from, tx_nonce) +VALUES (1, 10, '\x01'::bytea, '\x01'::bytea, '\x01'::bytea, 1), + (2, 10, '\x02'::bytea, '\x02'::bytea, '\x02'::bytea, 1), + (3, 10, '\x01'::bytea, '\x03'::bytea, '\x01'::bytea, 2), + (4, 10, '\x02'::bytea, '\x04'::bytea, '\x02'::bytea, 2), + (5, 10, '\x01'::bytea, '\x05'::bytea, '\x01'::bytea, 3), + (6, 10, '\x02'::bytea, '\x06'::bytea, '\x02'::bytea, 3); + +INSERT INTO auction_transaction (auction_id, tx_from, tx_nonce) +VALUES (1, '\x01'::bytea, 1), + (2, '\x02'::bytea, 1), + (3, '\x01'::bytea, 2), + (4, '\x02'::bytea, 2), + (5, '\x01'::bytea, 3), + (6, '\x02'::bytea, 3); + +INSERT INTO settlement_scores (auction_id, winning_score, reference_score, winner, block_deadline, simulation_block) +VALUES (1, 5000000000000000000, 4000000000000000000, '\x01'::bytea, 10, 0), + (2, 6000000000000000000, 3000000000000000000, '\x02'::bytea, 11, 1), + (3, 21000000000000000000, 3000000000000000000, '\x01'::bytea, 12, 2), + (4, 21000000000000000000, 3000000000000000000, '\x02'::bytea, 12, 2), + (5, 5000000000000000000, 3000000000000000000, '\x01'::bytea, 14, 4), + (6, 10000000000000000000, 9000000000000000000, '\x02'::bytea, 15, 5); + +INSERT INTO auction_prices (auction_id, token, price) +VALUES (1, '\x01', 500000000000000000000000000), +(1, '\x02', 500000000000000), +(2, '\x01', 500000000000000000000000000), +(2, '\x02', 500000000000000), +(3, '\x01', 500000000000000000000000000), +(3, '\x02', 500000000000000), +(4, '\x01', 500000000000000000000000000), +(4, '\x02', 500000000000000), +(5, '\x01', 500000000000000000000000000), +(5, '\x02', 500000000000000), +(6, '\x01', 500000000000000000000000000), +(6, '\x02', 500000000000000); + +INSERT INTO orders (uid, sell_token, buy_token, sell_amount, buy_amount, fee_amount, kind, partially_fillable, full_fee_amount, class) +VALUES ('\x01'::bytea, '\x01'::bytea, '\x02'::bytea, 95000000, 94000000000000000000, 5000000, 'sell', 'f', 5000000, 'market'), -- sell market order +('\x02'::bytea, '\x01'::bytea, '\x02'::bytea, 101000000, 100000000000000000000, 5000000, 'buy', 'f', 5000000, 'market'), -- buy market order +('\x03'::bytea, '\x01'::bytea, '\x02'::bytea, 100000000, 100000000000000000000, 0, 'sell', 't', 0, 'limit'), -- partially fillable sell limit order +('\x04'::bytea, '\x01'::bytea, '\x02'::bytea, 100000000, 100000000000000000000, 0, 'buy', 't', 0, 'limit'), -- partially fillable buy limit order +('\x05'::bytea, '\x01'::bytea, '\x02'::bytea, 100000000, 94000000000000000000, 0, 'sell', 'f', 0, 'limit'), -- in market sell limit order +('\x06'::bytea, '\x01'::bytea, '\x02'::bytea, 106000000, 100000000000000000000, 0, 'buy', 'f', 0, 'limit'); -- in market buy limit order + +INSERT INTO trades (block_number, log_index, order_uid, sell_amount, buy_amount, fee_amount) +VALUES (1, 0, '\x01'::bytea, 100000000, 95000000000000000000, 5000000), +(2, 0, '\x02'::bytea, 106000000, 100000000000000000000, 5000000), +(3, 0, '\x03'::bytea, 100000000, 101000000000000000000, 0), +(4, 0, '\x04'::bytea, 99000000, 100000000000000000000, 0), +(5, 0, '\x05'::bytea, 100000000, 95000000000000000000, 0), +(6, 0, '\x06'::bytea, 105000000, 100000000000000000000, 0); + +INSERT INTO order_execution (order_uid, auction_id, reward, surplus_fee, solver_fee) +VALUES ('\x03'::bytea, 3, 0, 5931372, NULL), +('\x04'::bytea, 4, 0, 6000000, NULL), +('\x05'::bytea, 5, 0, 6000000, NULL), +('\x06'::bytea, 6, 0, 6000000, NULL); + +INSERT INTO fee_policies (auction_id, order_uid, application_order, kind, surplus_factor, max_volume_factor, volume_factor) +VALUES (3, '\x03'::bytea, 3, 'surplus', 0.5, 0.02, NULL), +(4, '\x04'::bytea, 4, 'surplus', 0.75, 0.1, NULL), +(5, '\x05'::bytea, 5, 'volume', NULL, NULL, 0.0015), +(6, '\x06'::bytea, 6, 'surplus', 0.9, 0.01, NULL); diff --git a/tests/queries/test_protocol_fees.py b/tests/queries/test_protocol_fees.py new file mode 100644 index 00000000..2b17b2e5 --- /dev/null +++ b/tests/queries/test_protocol_fees.py @@ -0,0 +1,39 @@ +import unittest + +import pandas.testing +from pandas import DataFrame + +from src.pg_client import MultiInstanceDBFetcher + + +class TestProtocolFees(unittest.TestCase): + def setUp(self) -> None: + db_url = "postgres:postgres@localhost:5432/postgres" + self.fetcher = MultiInstanceDBFetcher([db_url]) + with open("./tests/queries/protocol_fee_test_db.sql", "r", encoding="utf-8") as file: + self.fetcher.connections[0].execute(file.read()) + + def test_get_protocol_fees(self): + start_block, end_block = "0", "100" + protocol_fees = self.fetcher.get_protocol_fees(start_block, end_block) + expected = DataFrame( + { + "solver": [ + "0x01", + "0x02", + ], + "protocol_fee": [ + 5.751126690035052e14, + 2.0303030303030302e15, + ], + "network_fee": [ + 5424887053768772.0, + 3969696969696969.5, + ], + } + ) + self.assertIsNone(pandas.testing.assert_frame_equal(expected, protocol_fees)) + + +if __name__ == "__main__": + unittest.main() From c64cccfeeb0861f07b3cfdd6a208df75f857f26b Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 15 Jan 2024 16:57:53 +0100 Subject: [PATCH 11/33] revert changes to batch rewards database --- populate_test_db.sql | 102 ++++++++----------------------------------- 1 file changed, 18 insertions(+), 84 deletions(-) diff --git a/populate_test_db.sql b/populate_test_db.sql index 38b2baa3..9e8c7f42 100644 --- a/populate_test_db.sql +++ b/populate_test_db.sql @@ -3,7 +3,6 @@ DROP TABLE IF EXISTS auction_transaction; DROP TABLE IF EXISTS auction_participants; DROP TABLE IF EXISTS settlement_scores; DROP TABLE IF EXISTS settlement_observations; -DROP TABLE IF EXISTS auction_prices; DROP TABLE IF EXISTS orders; DROP TYPE IF EXISTS OrderKind; DROP TYPE IF EXISTS SigningScheme; @@ -11,9 +10,6 @@ DROP TYPE IF EXISTS TokenBalance; DROP TYPE IF EXISTS OrderClass; DROP TABLE IF EXISTS order_quotes; DROP TABLE IF EXISTS trades; -DROP TABLE IF EXISTS order_execution; -DROP TABLE IF EXISTS fee_policies; -DROP TYPE IF EXISTS PolicyKind; CREATE TABLE IF NOT EXISTS settlements ( @@ -66,15 +62,6 @@ CREATE TABLE IF NOT EXISTS settlement_observations PRIMARY KEY (block_number, log_index) ); -CREATE TABLE IF NOT EXISTS auction_prices -( - auction_id bigint NOT NULL, - token bytea NOT NULL, - price numeric(78, 0) NOT NULL, - - PRIMARY KEY (auction_id, token) -); - -- orders table CREATE TYPE OrderKind AS ENUM ('buy', 'sell'); CREATE TYPE SigningScheme AS ENUM ('presign', 'eip712', 'ethsign'); @@ -128,45 +115,15 @@ CREATE TABLE IF NOT EXISTS trades PRIMARY KEY (block_number, log_index) ); -CREATE TABLE IF NOT EXISTS order_execution -( - order_uid bytea NOT NULL, - auction_id bigint NOT NULL, - reward double precision NOT NULL, - surplus_fee numeric(78, 0) NOT NULL, - solver_fee numeric(78, 0), - - PRIMARY KEY (order_uid, auction_id) -); - -CREATE TYPE PolicyKind AS ENUM ('surplus', 'volume'); - -CREATE TABLE fee_policies ( - auction_id bigint NOT NULL, - order_uid bytea NOT NULL, - -- The order in which the fee policies are inserted and applied. - application_order SERIAL NOT NULL, - -- The type of the fee policy. - kind PolicyKind NOT NULL, - -- The fee should be taken as a percentage of the price improvement. The value is between 0 and 1. - surplus_factor double precision, - -- Cap the fee at a certain percentage of the order volume. The value is between 0 and 1. - max_volume_factor double precision, - -- The fee should be taken as a percentage of the order volume. The value is between 0 and 1. - volume_factor double precision, - PRIMARY KEY (auction_id, order_uid, application_order) -); TRUNCATE settlements; TRUNCATE auction_transaction; TRUNCATE auction_participants; TRUNCATE settlement_scores; TRUNCATE settlement_observations; -TRUNCATE auction_prices; TRUNCATE orders; TRUNCATE order_quotes; TRUNCATE trades; -TRUNCATE fee_policies; INSERT INTO settlements (block_number, log_index, solver, tx_hash, tx_from, tx_nonce) @@ -233,55 +190,32 @@ VALUES (1, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), (25, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), (26, 10, 100000, 2000000000, 0, 400000000000000); - -INSERT INTO auction_prices (auction_id, token, price) -VALUES (1, '\x11', 1000000000000000000), -(1, '\x22', 1200000000000000000), -(2, '\x11', 1000000000000000000), -(2, '\x22', 1200000000000000000), -(5, '\x11', 1000000000000000000), -(5, '\x22', 1200000000000000000), -(6, '\x11', 1000000000000000000), -(6, '\x22', 1200000000000000000); - INSERT INTO orders (uid, owner, creation_timestamp, sell_token, buy_token, sell_amount, buy_amount, valid_to, fee_amount, kind, partially_fillable, signature, cancellation_timestamp, receiver, app_data, signing_scheme, settlement_contract, sell_token_balance, buy_token_balance, full_fee_amount, class) VALUES ('\x1111'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1700000000, 20000000000000000, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 20000000000000000, 'market'), -- sell market order +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 100000000000000000, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'market'), -- normal sell market order ('\x2222'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1700000000, 20000000000000000, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 20000000000000000, 'market'), -- buy market order +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 100000000000000000, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'market'), -- normal buy market order ('\x3333'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6020000000000000000, 4000000000000000000, 1700000000, 0, 'sell', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- partially fillable sell limit order +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 100000000000000000, 'sell', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'limit'), -- partially fillable sell limit order ('\x4444'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6020000000000000000, 4000000000000000000, 1700000000, 0, 'buy', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- partially fillable buy limit order +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 0, 'buy', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- partially fillable buy limit order ('\x5555'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6020000000000000000, 4000000000000000000, 1700000000, 0, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- in market sell limit order +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 0, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- in market sell limit order ('\x6666'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6020000000000000000, 4000000000000000000, 1700000000, 0, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'); -- in market buy limit order +', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 0, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'); -- in market buy limit order INSERT INTO order_quotes (order_uid, gas_amount, gas_price, sell_token_price, sell_amount, buy_amount, solver) -VALUES ('\x1111'::bytea, 200000, 100000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5111111111111111111111111111111111111111'::bytea), -('\x2222'::bytea, 200000, 100000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea), -('\x3333'::bytea, 200000, 100000000000, 0.123, 6000000000000000000, 3000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), -('\x4444'::bytea, 200000, 100000000000, 0.123, 7000000000000000000, 4000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), -('\x5555'::bytea, 200000, 100000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5333333333333333333333333333333333333333'::bytea), -('\x6666'::bytea, 200000, 100000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5444444444444444444444444444444444444444'::bytea); +VALUES ('\x1111'::bytea, 200000, 110000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5111111111111111111111111111111111111111'::bytea), +('\x2222'::bytea, 200000, 110000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea), +('\x3333'::bytea, 200000, 110000000000, 0.123, 6000000000000000000, 3000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), +('\x4444'::bytea, 200000, 110000000000, 0.123, 7000000000000000000, 4000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), +('\x5555'::bytea, 200000, 110000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5333333333333333333333333333333333333333'::bytea), +('\x6666'::bytea, 200000, 110000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5444444444444444444444444444444444444444'::bytea); INSERT INTO trades (block_number, log_index, order_uid, sell_amount, buy_amount, fee_amount) -VALUES (1, 0, '\x1111'::bytea, 6020000000000000000, 4600000000000000000, 20000000000000000), -(2, 0, '\x2222'::bytea, 5600000000000000000, 4000000000000000000, 20000000000000000), -(2, 1, '\x3333'::bytea, 6020000000000000000, 4600000000000000000, 0), -(5, 0, '\x4444'::bytea, 5600000000000000000, 4000000000000000000, 0), -(5, 1, '\x5555'::bytea, 6020000000000000000, 4600000000000000000, 0), -(20, 0, '\x6666'::bytea, 5600000000000000000, 4000000000000000000, 0); - -INSERT INTO order_execution (order_uid, auction_id, reward, surplus_fee, solver_fee) -VALUES ('\x3333'::bytea, 2, 0, 66464646464646470, NULL), -('\x4444'::bytea, 5, 0, 125000000000000000, NULL), -('\x5555'::bytea, 5, 0, 29043565348022034, NULL), -('\x6666'::bytea, 6, 0, 20000000000000000, NULL); - -INSERT INTO fee_policies (auction_id, order_uid, application_order, kind, surplus_factor, max_volume_factor, volume_factor) -VALUES (2, '\x3333'::bytea, 3, 'surplus', 0.5, 0.01, NULL), -(5, '\x4444'::bytea, 4, 'surplus', 0.2, 0.1, NULL), -(5, '\x5555'::bytea, 5, 'volume', NULL, NULL, 0.0015), -(6, '\x6666'::bytea, 6, 'surplus', 0.2, 0.1, NULL); +VALUES (1, 0, '\x1111'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), +(2, 0, '\x2222'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000), +(2, 1, '\x3333'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), +(5, 0, '\x4444'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000), +(5, 1, '\x5555'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), +(20, 0, '\x6666'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000); From 49e887d1ed906f0c536a32d932b36097a2744180 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 15 Jan 2024 17:32:07 +0100 Subject: [PATCH 12/33] simplify order quotes test database - use less data - use separate file --- tests/queries/quote_rewards_test_db.sql | 67 +++++++++++++++++++++++++ tests/queries/test_quote_rewards.py | 10 ++-- 2 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 tests/queries/quote_rewards_test_db.sql diff --git a/tests/queries/quote_rewards_test_db.sql b/tests/queries/quote_rewards_test_db.sql new file mode 100644 index 00000000..6121228c --- /dev/null +++ b/tests/queries/quote_rewards_test_db.sql @@ -0,0 +1,67 @@ +DROP TABLE IF EXISTS orders; +DROP TYPE IF EXISTS OrderKind; +DROP TABLE IF EXISTS order_quotes; +DROP TABLE IF EXISTS trades; + + +-- orders table +CREATE TYPE OrderKind AS ENUM ('buy', 'sell'); + +CREATE TABLE IF NOT EXISTS orders +( + uid bytea PRIMARY KEY, + sell_amount numeric(78,0) NOT NULL, + buy_amount numeric(78,0) NOT NULL, + fee_amount numeric(78,0) NOT NULL, + kind OrderKind NOT NULL, + partially_fillable boolean NOT NULL +); + +CREATE TABLE IF NOT EXISTS order_quotes +( + order_uid bytea PRIMARY KEY, + sell_amount numeric(78, 0) NOT NULL, + buy_amount numeric(78, 0) NOT NULL, + solver bytea NOT NULL +); + +CREATE TABLE IF NOT EXISTS trades +( + block_number bigint NOT NULL, + log_index bigint NOT NULL, + order_uid bytea NOT NULL, + sell_amount numeric(78, 0) NOT NULL, + buy_amount numeric(78, 0) NOT NULL, + + PRIMARY KEY (block_number, log_index) +); + + +TRUNCATE orders; +TRUNCATE order_quotes; +TRUNCATE trades; + + +INSERT INTO orders (uid, sell_amount, buy_amount, fee_amount, kind, partially_fillable) +VALUES ('\x01'::bytea, 95000000, 94000000000000000000, 5000000, 'sell', 'f'), -- normal sell market order +('\x02'::bytea, 101000000, 100000000000000000000, 5000000, 'buy', 'f'), -- normal buy market order +('\x03'::bytea, 100000000, 100000000000000000000, 0, 'sell', 't'), -- partially fillable sell limit order +('\x04'::bytea, 100000000, 100000000000000000000, 0, 'buy', 't'), -- partially fillable buy limit order +('\x05'::bytea, 100000000, 94000000000000000000, 0, 'sell', 'f'), -- in market sell limit order +('\x06'::bytea, 106000000, 100000000000000000000, 0, 'buy', 'f'); -- in market buy limit order + +INSERT INTO order_quotes (order_uid, sell_amount, buy_amount, solver) +VALUES ('\x01'::bytea, 95000000, 95000000000000000000, '\x01'::bytea), +('\x02'::bytea, 101000000, 100000000000000000000, '\x02'::bytea), +('\x03'::bytea, 100000000, 95000000000000000000, '\x03'::bytea), +('\x04'::bytea, 105000000, 100000000000000000000, '\x03'::bytea), +('\x05'::bytea, 100000000, 95000000000000000000, '\x03'::bytea), +('\x06'::bytea, 105000000, 100000000000000000000, '\x03'::bytea); + +INSERT INTO trades (block_number, log_index, order_uid, sell_amount, buy_amount) +VALUES (1, 0, '\x01'::bytea, 100000000, 95000000000000000000), +(2, 0, '\x02'::bytea, 106000000, 100000000000000000000), +(3, 0, '\x03'::bytea, 100000000, 101000000000000000000), +(4, 0, '\x04'::bytea, 99000000, 100000000000000000000), +(5, 0, '\x05'::bytea, 100000000, 95000000000000000000), +(6, 0, '\x06'::bytea, 105000000, 100000000000000000000); diff --git a/tests/queries/test_quote_rewards.py b/tests/queries/test_quote_rewards.py index 5ee8baa1..bc069150 100644 --- a/tests/queries/test_quote_rewards.py +++ b/tests/queries/test_quote_rewards.py @@ -10,7 +10,7 @@ class TestQuoteRewards(unittest.TestCase): def setUp(self) -> None: db_url = "postgres:postgres@localhost:5432/postgres" self.fetcher = MultiInstanceDBFetcher([db_url]) - with open("./populate_test_db.sql", "r", encoding="utf-8") as file: + with open("./tests/queries/quote_rewards_test_db.sql", "r", encoding="utf-8") as file: self.fetcher.connections[0].execute(file.read()) def test_get_quote_rewards(self): @@ -19,14 +19,14 @@ def test_get_quote_rewards(self): expected = DataFrame( { "solver": [ - "0x5111111111111111111111111111111111111111", - "0x5333333333333333333333333333333333333333", - "0x5444444444444444444444444444444444444444", + "0x01", + "0x02", + "0x03", ], "num_quotes": [ 1, - 2, 1, + 2, ], } ) From 907f46d582d9af675bf4d24cc1820b29593aeb90 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 15 Jan 2024 17:36:40 +0100 Subject: [PATCH 13/33] move batch rewards test db setup --- tests/queries/batch_rewards_test_db.sql | 128 ++++++++++++++++++++++++ tests/queries/test_batch_rewards.py | 4 +- 2 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 tests/queries/batch_rewards_test_db.sql diff --git a/tests/queries/batch_rewards_test_db.sql b/tests/queries/batch_rewards_test_db.sql new file mode 100644 index 00000000..c984e1d4 --- /dev/null +++ b/tests/queries/batch_rewards_test_db.sql @@ -0,0 +1,128 @@ +DROP TABLE IF EXISTS settlements; +DROP TABLE IF EXISTS auction_transaction; +DROP TABLE IF EXISTS auction_participants; +DROP TABLE IF EXISTS settlement_scores; +DROP TABLE IF EXISTS settlement_observations; + +CREATE TABLE IF NOT EXISTS settlements +( + block_number bigint NOT NULL, + log_index bigint NOT NULL, + solver bytea NOT NULL, + tx_hash bytea NOT NULL, + tx_from bytea NOT NULL, + tx_nonce bigint NOT NULL, + + PRIMARY KEY (block_number, log_index) +); + +CREATE INDEX settlements_tx_from_tx_nonce ON settlements (tx_from, tx_nonce); + +CREATE TABLE IF NOT EXISTS auction_transaction +( + auction_id bigint PRIMARY KEY, + tx_from bytea NOT NULL, + tx_nonce bigint NOT NULL, + UNIQUE (tx_from, tx_nonce) +); + +CREATE TABLE IF NOT EXISTS auction_participants +( + auction_id bigint, + participant bytea +); + +CREATE TABLE IF NOT EXISTS settlement_scores +( + auction_id bigint PRIMARY KEY, + winner bytea NOT NULL, + winning_score numeric(78, 0) NOT NULL, + reference_score numeric(78, 0) NOT NULL, + block_deadline bigint NOT NULL, + simulation_block bigint NOT NULL +); + +-- Populated after block finalization via transactionReceipt. +CREATE TABLE IF NOT EXISTS settlement_observations +( + block_number bigint NOT NULL, + log_index bigint NOT NULL, + gas_used numeric(78, 0) NOT NULL, + effective_gas_price numeric(78, 0) NOT NULL, + surplus numeric(78, 0) NOT NULL, + fee numeric(78, 0) NOT NULL, + + PRIMARY KEY (block_number, log_index) +); + + +TRUNCATE settlements; +TRUNCATE auction_transaction; +TRUNCATE auction_participants; +TRUNCATE settlement_scores; +TRUNCATE settlement_observations; + + +INSERT INTO settlements (block_number, log_index, solver, tx_hash, tx_from, tx_nonce) +VALUES (1, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7111'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 1), + (2, 10, '\x5222222222222222222222222222222222222222'::bytea, '\x7222'::bytea, '\x5222222222222222222222222222222222222222'::bytea, 1), + (5, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7333'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 2), + -- would the following entry be in the data base? (submitted too late) -- YES + (20, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7444'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 3), + (25, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7555'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 4), + (26, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7666'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 6); + +INSERT INTO auction_transaction (auction_id, tx_from, tx_nonce) +VALUES (1, '\x5111111111111111111111111111111111111111'::bytea, 1), + (2, '\x5222222222222222222222222222222222222222'::bytea, 1), + (5, '\x5111111111111111111111111111111111111111'::bytea, 2), + (6, '\x5111111111111111111111111111111111111111'::bytea, 3), -- would that entry be in the data base? (submitted too late) + (7, '\x5111111111111111111111111111111111111111'::bytea, 4), + (8, '\x5111111111111111111111111111111111111111'::bytea, 5), -- would that entry be in the data base? (failed transaction) + (9, '\x5111111111111111111111111111111111111111'::bytea, 6), + (10, '\x5333333333333333333333333333333333333333'::bytea, 1); -- would that entry be in the data base? (failed transaction) + +INSERT INTO auction_participants (auction_id, participant) +VALUES (1, '\x5222222222222222222222222222222222222222'::bytea), + (1, '\x5333333333333333333333333333333333333333'::bytea), + (1, '\x5111111111111111111111111111111111111111'::bytea), + (2, '\x5444444444444444444444444444444444444444'::bytea), + (2, '\x5333333333333333333333333333333333333333'::bytea), + (2, '\x5222222222222222222222222222222222222222'::bytea), + (3, '\x5444444444444444444444444444444444444444'::bytea), + (3, '\x5333333333333333333333333333333333333333'::bytea), + (3, '\x5111111111111111111111111111111111111111'::bytea), + (5, '\x5444444444444444444444444444444444444444'::bytea), + (5, '\x5333333333333333333333333333333333333333'::bytea), + (5, '\x5111111111111111111111111111111111111111'::bytea), + (6, '\x5444444444444444444444444444444444444444'::bytea), + (6, '\x5333333333333333333333333333333333333333'::bytea), + (6, '\x5111111111111111111111111111111111111111'::bytea), + (7, '\x5111111111111111111111111111111111111111'::bytea), + (8, '\x5111111111111111111111111111111111111111'::bytea), + (9, '\x5444444444444444444444444444444444444444'::bytea), + (9, '\x5333333333333333333333333333333333333333'::bytea), + (9, '\x5111111111111111111111111111111111111111'::bytea), + (10, '\x5444444444444444444444444444444444444444'::bytea), + (10, '\x5333333333333333333333333333333333333333'::bytea); + +INSERT INTO settlement_scores (auction_id, winning_score, reference_score, winner, block_deadline, simulation_block) +VALUES (1, 5000000000000000000, 4000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 10, 0), + (2, 6000000000000000000, 3000000000000000000, '\x5222222222222222222222222222222222222222'::bytea, 11, 1), + (3, 21000000000000000000, 3000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 12, 2), + (5, 5000000000000000000, 3000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 14, 4), -- jump in auction id + (6, 10000000000000000000, 9000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 15, 5), -- settled too late + (7, 5000000000000000000, 0, '\x5111111111111111111111111111111111111111'::bytea, 30, 6), -- no competition + (8, 5000000000000000000, 0, '\x5111111111111111111111111111111111111111'::bytea, 35, 7), -- no competition, failed transaction + (9, 5000000000000000000, 1000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 36, 8), -- score larger than quality + (10, 5000000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea, 37, 9); -- participant with net negative payment + +INSERT INTO settlement_observations (block_number, log_index, gas_used, effective_gas_price, surplus, fee) +VALUES (1, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), + (2, 10, 150000, 3000000000, 12000000000000000000, 400000000000000), + (5, 10, 100000, 2000000000, 5000000000000000000, 250000000000000), + -- Depends on backend (setting surplus and fee to zero). + -- I would prefer to use the real numbers. What is backend gonna do. + (20, 10, 100000, 2000000000, 0, 0), -- would that entry be in the data base? (submitted too late) + (25, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), + (26, 10, 100000, 2000000000, 0, 400000000000000); diff --git a/tests/queries/test_batch_rewards.py b/tests/queries/test_batch_rewards.py index f5c0d211..fc6f3fb2 100644 --- a/tests/queries/test_batch_rewards.py +++ b/tests/queries/test_batch_rewards.py @@ -10,7 +10,9 @@ class TestBatchRewards(unittest.TestCase): def setUp(self) -> None: db_url = "postgres:postgres@localhost:5432/postgres" self.fetcher = MultiInstanceDBFetcher([db_url]) - with open("./populate_test_db.sql", "r", encoding="utf-8") as file: + with open( + "./tests/queries/batch_rewards_test_db.sql", "r", encoding="utf-8" + ) as file: self.fetcher.connections[0].execute(file.read()) def test_get_batch_rewards(self): From e3ec33a092818447a15152a46d5bad98bd0790b7 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 15 Jan 2024 17:36:46 +0100 Subject: [PATCH 14/33] black fix --- tests/queries/test_quote_rewards.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/queries/test_quote_rewards.py b/tests/queries/test_quote_rewards.py index bc069150..8bad030b 100644 --- a/tests/queries/test_quote_rewards.py +++ b/tests/queries/test_quote_rewards.py @@ -10,7 +10,9 @@ class TestQuoteRewards(unittest.TestCase): def setUp(self) -> None: db_url = "postgres:postgres@localhost:5432/postgres" self.fetcher = MultiInstanceDBFetcher([db_url]) - with open("./tests/queries/quote_rewards_test_db.sql", "r", encoding="utf-8") as file: + with open( + "./tests/queries/quote_rewards_test_db.sql", "r", encoding="utf-8" + ) as file: self.fetcher.connections[0].execute(file.read()) def test_get_quote_rewards(self): From 35b96ad11ed9b4b234732ba82393112b3df5ab1c Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 15 Jan 2024 17:37:29 +0100 Subject: [PATCH 15/33] remove old test database file --- populate_test_db.sql | 221 ------------------------------------------- 1 file changed, 221 deletions(-) delete mode 100644 populate_test_db.sql diff --git a/populate_test_db.sql b/populate_test_db.sql deleted file mode 100644 index 9e8c7f42..00000000 --- a/populate_test_db.sql +++ /dev/null @@ -1,221 +0,0 @@ -DROP TABLE IF EXISTS settlements; -DROP TABLE IF EXISTS auction_transaction; -DROP TABLE IF EXISTS auction_participants; -DROP TABLE IF EXISTS settlement_scores; -DROP TABLE IF EXISTS settlement_observations; -DROP TABLE IF EXISTS orders; -DROP TYPE IF EXISTS OrderKind; -DROP TYPE IF EXISTS SigningScheme; -DROP TYPE IF EXISTS TokenBalance; -DROP TYPE IF EXISTS OrderClass; -DROP TABLE IF EXISTS order_quotes; -DROP TABLE IF EXISTS trades; - -CREATE TABLE IF NOT EXISTS settlements -( - block_number bigint NOT NULL, - log_index bigint NOT NULL, - solver bytea NOT NULL, - tx_hash bytea NOT NULL, - tx_from bytea NOT NULL, - tx_nonce bigint NOT NULL, - - PRIMARY KEY (block_number, log_index) -); - -CREATE INDEX settlements_tx_from_tx_nonce ON settlements (tx_from, tx_nonce); - -CREATE TABLE IF NOT EXISTS auction_transaction -( - auction_id bigint PRIMARY KEY, - tx_from bytea NOT NULL, - tx_nonce bigint NOT NULL, - UNIQUE (tx_from, tx_nonce) -); - -CREATE TABLE IF NOT EXISTS auction_participants -( - auction_id bigint, - participant bytea -); - -CREATE TABLE IF NOT EXISTS settlement_scores -( - auction_id bigint PRIMARY KEY, - winner bytea NOT NULL, - winning_score numeric(78, 0) NOT NULL, - reference_score numeric(78, 0) NOT NULL, - block_deadline bigint NOT NULL, - simulation_block bigint NOT NULL -); - --- Populated after block finalization via transactionReceipt. -CREATE TABLE IF NOT EXISTS settlement_observations -( - block_number bigint NOT NULL, - log_index bigint NOT NULL, - gas_used numeric(78, 0) NOT NULL, - effective_gas_price numeric(78, 0) NOT NULL, - surplus numeric(78, 0) NOT NULL, - fee numeric(78, 0) NOT NULL, - - PRIMARY KEY (block_number, log_index) -); - --- orders table -CREATE TYPE OrderKind AS ENUM ('buy', 'sell'); -CREATE TYPE SigningScheme AS ENUM ('presign', 'eip712', 'ethsign'); -CREATE TYPE TokenBalance AS ENUM ('erc20', 'external'); -CREATE TYPE OrderClass AS ENUM ('market', 'limit'); - -CREATE TABLE orders ( - uid bytea PRIMARY KEY, - owner bytea NOT NULL, - creation_timestamp timestamptz NOT NULL, - sell_token bytea NOT NULL, - buy_token bytea NOT NULL, - sell_amount numeric(78,0) NOT NULL, - buy_amount numeric(78,0) NOT NULL, - valid_to bigint NOT NULL, - fee_amount numeric(78,0) NOT NULL, - kind OrderKind NOT NULL, - partially_fillable boolean NOT NULL, - signature bytea NOT NULL, - cancellation_timestamp timestamptz, - receiver bytea NOT NULL, - app_data bytea NOT NULL, - signing_scheme SigningScheme NOT NULL, - settlement_contract bytea NOT NULL, - sell_token_balance TokenBalance NOT NULL, - buy_token_balance TokenBalance NOT NULL, - full_fee_amount numeric(78,0) NOT NULL, - class OrderClass NOT NULL -); - -CREATE TABLE IF NOT EXISTS order_quotes -( - order_uid bytea PRIMARY KEY, - gas_amount double precision NOT NULL, - gas_price double precision NOT NULL, - sell_token_price double precision NOT NULL, - sell_amount numeric(78, 0) NOT NULL, - buy_amount numeric(78, 0) NOT NULL, - solver bytea NOT NULL -); - -CREATE TABLE IF NOT EXISTS trades -( - block_number bigint NOT NULL, - log_index bigint NOT NULL, - order_uid bytea NOT NULL, - sell_amount numeric(78, 0) NOT NULL, - buy_amount numeric(78, 0) NOT NULL, - fee_amount numeric(78, 0) NOT NULL, - - PRIMARY KEY (block_number, log_index) -); - - -TRUNCATE settlements; -TRUNCATE auction_transaction; -TRUNCATE auction_participants; -TRUNCATE settlement_scores; -TRUNCATE settlement_observations; -TRUNCATE orders; -TRUNCATE order_quotes; -TRUNCATE trades; - - -INSERT INTO settlements (block_number, log_index, solver, tx_hash, tx_from, tx_nonce) -VALUES (1, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7111'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 1), - (2, 10, '\x5222222222222222222222222222222222222222'::bytea, '\x7222'::bytea, '\x5222222222222222222222222222222222222222'::bytea, 1), - (5, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7333'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 2), - -- would the following entry be in the data base? (submitted too late) -- YES - (20, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7444'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 3), - (25, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7555'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 4), - (26, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7666'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 6); - -INSERT INTO auction_transaction (auction_id, tx_from, tx_nonce) -VALUES (1, '\x5111111111111111111111111111111111111111'::bytea, 1), - (2, '\x5222222222222222222222222222222222222222'::bytea, 1), - (5, '\x5111111111111111111111111111111111111111'::bytea, 2), - (6, '\x5111111111111111111111111111111111111111'::bytea, 3), -- would that entry be in the data base? (submitted too late) - (7, '\x5111111111111111111111111111111111111111'::bytea, 4), - (8, '\x5111111111111111111111111111111111111111'::bytea, 5), -- would that entry be in the data base? (failed transaction) - (9, '\x5111111111111111111111111111111111111111'::bytea, 6), - (10, '\x5333333333333333333333333333333333333333'::bytea, 1); -- would that entry be in the data base? (failed transaction) - -INSERT INTO auction_participants (auction_id, participant) -VALUES (1, '\x5222222222222222222222222222222222222222'::bytea), - (1, '\x5333333333333333333333333333333333333333'::bytea), - (1, '\x5111111111111111111111111111111111111111'::bytea), - (2, '\x5444444444444444444444444444444444444444'::bytea), - (2, '\x5333333333333333333333333333333333333333'::bytea), - (2, '\x5222222222222222222222222222222222222222'::bytea), - (3, '\x5444444444444444444444444444444444444444'::bytea), - (3, '\x5333333333333333333333333333333333333333'::bytea), - (3, '\x5111111111111111111111111111111111111111'::bytea), - (5, '\x5444444444444444444444444444444444444444'::bytea), - (5, '\x5333333333333333333333333333333333333333'::bytea), - (5, '\x5111111111111111111111111111111111111111'::bytea), - (6, '\x5444444444444444444444444444444444444444'::bytea), - (6, '\x5333333333333333333333333333333333333333'::bytea), - (6, '\x5111111111111111111111111111111111111111'::bytea), - (7, '\x5111111111111111111111111111111111111111'::bytea), - (8, '\x5111111111111111111111111111111111111111'::bytea), - (9, '\x5444444444444444444444444444444444444444'::bytea), - (9, '\x5333333333333333333333333333333333333333'::bytea), - (9, '\x5111111111111111111111111111111111111111'::bytea), - (10, '\x5444444444444444444444444444444444444444'::bytea), - (10, '\x5333333333333333333333333333333333333333'::bytea); - -INSERT INTO settlement_scores (auction_id, winning_score, reference_score, winner, block_deadline, simulation_block) -VALUES (1, 5000000000000000000, 4000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 10, 0), - (2, 6000000000000000000, 3000000000000000000, '\x5222222222222222222222222222222222222222'::bytea, 11, 1), - (3, 21000000000000000000, 3000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 12, 2), - (5, 5000000000000000000, 3000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 14, 4), -- jump in auction id - (6, 10000000000000000000, 9000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 15, 5), -- settled too late - (7, 5000000000000000000, 0, '\x5111111111111111111111111111111111111111'::bytea, 30, 6), -- no competition - (8, 5000000000000000000, 0, '\x5111111111111111111111111111111111111111'::bytea, 35, 7), -- no competition, failed transaction - (9, 5000000000000000000, 1000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 36, 8), -- score larger than quality - (10, 5000000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea, 37, 9); -- participant with net negative payment - -INSERT INTO settlement_observations (block_number, log_index, gas_used, effective_gas_price, surplus, fee) -VALUES (1, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), - (2, 10, 150000, 3000000000, 12000000000000000000, 400000000000000), - (5, 10, 100000, 2000000000, 5000000000000000000, 250000000000000), - -- Depends on backend (setting surplus and fee to zero). - -- I would prefer to use the real numbers. What is backend gonna do. - (20, 10, 100000, 2000000000, 0, 0), -- would that entry be in the data base? (submitted too late) - (25, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), - (26, 10, 100000, 2000000000, 0, 400000000000000); - -INSERT INTO orders (uid, owner, creation_timestamp, sell_token, buy_token, sell_amount, buy_amount, valid_to, fee_amount, kind, partially_fillable, signature, cancellation_timestamp, receiver, app_data, signing_scheme, settlement_contract, sell_token_balance, buy_token_balance, full_fee_amount, class) -VALUES ('\x1111'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 100000000000000000, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'market'), -- normal sell market order -('\x2222'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 100000000000000000, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'market'), -- normal buy market order -('\x3333'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 100000000000000000, 'sell', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 200000000000000000, 'limit'), -- partially fillable sell limit order -('\x4444'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 0, 'buy', 't', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- partially fillable buy limit order -('\x5555'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 0, 'sell', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'), -- in market sell limit order -('\x6666'::bytea, '\x1111111111'::bytea, '2024-01-01 00:00:00.000000+00 -', '\x11'::bytea, '\x22'::bytea, 6000000000000000000, 4000000000000000000, 1655195621, 0, 'buy', 'f', '\x987987987987'::bytea, NULL, '\x12341234'::bytea, '\x1234512345'::bytea, 'presign', '\x123456123456'::bytea, 'erc20', 'external', 0, 'limit'); -- in market buy limit order - -INSERT INTO order_quotes (order_uid, gas_amount, gas_price, sell_token_price, sell_amount, buy_amount, solver) -VALUES ('\x1111'::bytea, 200000, 110000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5111111111111111111111111111111111111111'::bytea), -('\x2222'::bytea, 200000, 110000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea), -('\x3333'::bytea, 200000, 110000000000, 0.123, 6000000000000000000, 3000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), -('\x4444'::bytea, 200000, 110000000000, 0.123, 7000000000000000000, 4000000000000000000, '\x5222222222222222222222222222222222222222'::bytea), -('\x5555'::bytea, 200000, 110000000000, 0.123, 6000000000000000000, 4500000000000000000, '\x5333333333333333333333333333333333333333'::bytea), -('\x6666'::bytea, 200000, 110000000000, 0.123, 5500000000000000000, 4000000000000000000, '\x5444444444444444444444444444444444444444'::bytea); - -INSERT INTO trades (block_number, log_index, order_uid, sell_amount, buy_amount, fee_amount) -VALUES (1, 0, '\x1111'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), -(2, 0, '\x2222'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000), -(2, 1, '\x3333'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), -(5, 0, '\x4444'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000), -(5, 1, '\x5555'::bytea, 6200000000000000000, 4600000000000000000, 200000000000000000), -(20, 0, '\x6666'::bytea, 5600000000000000000, 4000000000000000000, 200000000000000000); From da2c4c471540a2f852a128fae0bdebf5eae16b93 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 15 Jan 2024 17:38:19 +0100 Subject: [PATCH 16/33] fix black issue --- tests/queries/test_protocol_fees.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/queries/test_protocol_fees.py b/tests/queries/test_protocol_fees.py index 2b17b2e5..b733612d 100644 --- a/tests/queries/test_protocol_fees.py +++ b/tests/queries/test_protocol_fees.py @@ -10,7 +10,9 @@ class TestProtocolFees(unittest.TestCase): def setUp(self) -> None: db_url = "postgres:postgres@localhost:5432/postgres" self.fetcher = MultiInstanceDBFetcher([db_url]) - with open("./tests/queries/protocol_fee_test_db.sql", "r", encoding="utf-8") as file: + with open( + "./tests/queries/protocol_fee_test_db.sql", "r", encoding="utf-8" + ) as file: self.fetcher.connections[0].execute(file.read()) def test_get_protocol_fees(self): From f4438cdc0bc4046db92a01e4bf686d6189cc4f96 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 18 Jan 2024 12:01:04 +0100 Subject: [PATCH 17/33] modified query - add start and end parameters - avoid division by zero bug - compute network fee correction instead of network fee --- queries/orderbook/protocol_fees.sql | 38 ++++++++++++++++++----------- tests/queries/test_protocol_fees.py | 6 ++--- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/queries/orderbook/protocol_fees.sql b/queries/orderbook/protocol_fees.sql index b27d72b0..8d35f9de 100644 --- a/queries/orderbook/protocol_fees.sql +++ b/queries/orderbook/protocol_fees.sql @@ -32,8 +32,8 @@ WITH order_surplus AS ( ON t.order_uid = o.uid JOIN order_execution oe -- contains surplus fee ON t.order_uid = oe.order_uid AND at.auction_id = oe.auction_id - WHERE ss.block_deadline > 0 - AND ss.block_deadline <= 100 + WHERE ss.block_deadline >= {{start_block}} + AND ss.block_deadline <= {{end_block}} ) ,order_observation AS ( SELECT @@ -51,16 +51,26 @@ WITH order_surplus AS ( CASE WHEN os.kind = 'sell' THEN - LEAST( - fp.max_volume_factor / (1 - fp.max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) + CASE + WHEN fp.max_volume_factor = 1 -- this is done to avoid a division by zero bug + THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus + ELSE + LEAST( + fp.max_volume_factor / (1 - fp.max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) + END WHEN os.kind = 'buy' THEN - LEAST( - fp.max_volume_factor / (1 - fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) + CASE + WHEN fp.max_volume_factor = 1 + THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus + ELSE + LEAST( + fp.max_volume_factor / (1 - fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) + END END WHEN fp.kind = 'volume' THEN fp.volume_factor / (1 - fp.volume_factor) * os.sell_amount @@ -82,9 +92,9 @@ WITH order_surplus AS ( oo.protocol_fee, CASE WHEN oo.sell_token != oo.protocol_fee_token - THEN oo.observed_fee - (oo.sell_amount - oo.observed_fee) / oo.buy_amount * oo.protocol_fee - ELSE oo.observed_fee - oo.protocol_fee - END AS network_fee, + THEN - (oo.sell_amount - oo.observed_fee) / oo.buy_amount * oo.protocol_fee + ELSE - oo.protocol_fee + END AS network_fee_correction, oo.sell_token as network_fee_token, ap_surplus.price / pow(10, 18) as surplus_token_price, ap_protocol.price / pow(10, 18) as protocol_fee_token_price, @@ -102,7 +112,7 @@ batch_aggregate AS ( concat('0x', encode(solver, 'hex')) as solver, -- sum(surplus * surplus_token_price) as surplus, sum(protocol_fee * protocol_fee_token_price) as protocol_fee, - sum(network_fee * network_fee_token_price) as network_fee + sum(network_fee_correction * network_fee_token_price) as network_fee_correction FROM order_observations_prices oop group by solver ) diff --git a/tests/queries/test_protocol_fees.py b/tests/queries/test_protocol_fees.py index b733612d..00bcd5b9 100644 --- a/tests/queries/test_protocol_fees.py +++ b/tests/queries/test_protocol_fees.py @@ -28,9 +28,9 @@ def test_get_protocol_fees(self): 5.751126690035052e14, 2.0303030303030302e15, ], - "network_fee": [ - 5424887053768772.0, - 3969696969696969.5, + "network_fee_correction": [ + -540798946231228.0, + -2030303030303030.5, ], } ) From 1aa5a7760cd2f2af4fc1fc2973a3e543311bb99e Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 18 Jan 2024 13:44:55 +0100 Subject: [PATCH 18/33] add protocol fees to batch rewards query - change batch rewards query - add tables to batch reward database --- queries/orderbook/batch_rewards.sql | 154 +++++++++++++++++++++-- tests/queries/batch_rewards_test_db.sql | 161 +++++++++++++++++++++++- tests/queries/test_batch_rewards.py | 16 +++ 3 files changed, 317 insertions(+), 14 deletions(-) diff --git a/queries/orderbook/batch_rewards.sql b/queries/orderbook/batch_rewards.sql index 31e6dc25..5a5ba1de 100644 --- a/queries/orderbook/batch_rewards.sql +++ b/queries/orderbook/batch_rewards.sql @@ -28,13 +28,136 @@ WITH observed_settlements AS (SELECT WHERE block_deadline >= {{start_block}} AND block_deadline <= {{end_block}} GROUP BY ss.auction_id), +-- protocol fees: +order_surplus AS ( + SELECT + ss.winner as solver, + at.auction_id, + s.tx_hash, + t.order_uid, + o.sell_token, -- the total amount the user sends + o.buy_token, + t.sell_amount, + t.buy_amount, + oe.surplus_fee as observed_fee, + o.kind, + CASE + WHEN o.kind = 'sell' + THEN t.buy_amount - o.buy_amount / (o.sell_amount + o.fee_amount) * t.sell_amount + WHEN o.kind = 'buy' + THEN (o.sell_amount + o.fee_amount) / o.buy_amount * t.buy_amount - t.sell_amount + END AS surplus, + CASE + WHEN o.kind = 'sell' + THEN o.buy_token + WHEN o.kind = 'buy' + THEN o.sell_token + END AS surplus_token + FROM settlements s -- links block_number and log_index to tx_from and tx_nonce + JOIN auction_transaction at -- links auction_id to tx_from and tx_nonce + ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce + JOIN settlement_scores ss -- contains block_deadline + ON at.auction_id = ss.auction_id + JOIN trades t -- contains traded amounts + ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block + JOIN orders o -- contains tokens and limit amounts + ON t.order_uid = o.uid + JOIN order_execution oe -- contains surplus fee + ON t.order_uid = oe.order_uid AND at.auction_id = oe.auction_id + WHERE ss.block_deadline >= {{start_block}} + AND ss.block_deadline <= {{end_block}} +) +,order_observations AS ( + SELECT + os.auction_id, + os.solver, + os.tx_hash, + os.sell_amount, -- the total amount the user sends + os.buy_amount, + os.sell_token, + os.observed_fee, + os.surplus, + os.surplus_token, + CASE + WHEN fp.kind = 'surplus' + THEN + CASE + WHEN os.kind = 'sell' + THEN + CASE + WHEN fp.max_volume_factor = 1 -- this is done to avoid a division by zero bug + THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus + ELSE + LEAST( + fp.max_volume_factor / (1 - fp.max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) + END + WHEN os.kind = 'buy' + THEN + CASE + WHEN fp.max_volume_factor = 1 + THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus + ELSE + LEAST( + fp.max_volume_factor / (1 - fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) + END + END + WHEN fp.kind = 'volume' + THEN fp.volume_factor / (1 - fp.volume_factor) * os.sell_amount + END AS protocol_fee, + CASE + WHEN fp.kind = 'surplus' + THEN os.surplus_token + WHEN fp.kind = 'volume' + THEN os.sell_token + END AS protocol_fee_token + FROM order_surplus os + JOIN fee_policies fp -- contains protocol fee policy + ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid +) +,order_observations_prices AS ( + SELECT + oo.solver, + oo.tx_hash, + oo.surplus, + oo.protocol_fee, + CASE + WHEN oo.sell_token != oo.protocol_fee_token + THEN (oo.sell_amount - oo.observed_fee) / oo.buy_amount * oo.protocol_fee + ELSE oo.protocol_fee + END AS network_fee_correction, + oo.sell_token as network_fee_token, + ap_surplus.price / pow(10, 18) as surplus_token_price, + ap_protocol.price / pow(10, 18) as protocol_fee_token_price, + ap_sell.price / pow(10, 18) as network_fee_token_price + FROM order_observations oo + JOIN auction_prices ap_sell -- contains price: sell token + ON oo.auction_id = ap_sell.auction_id AND oo.sell_token = ap_sell.token + JOIN auction_prices ap_surplus -- contains price: surplus token + ON oo.auction_id = ap_surplus.auction_id AND oo.surplus_token = ap_surplus.token + JOIN auction_prices ap_protocol -- contains price: protocol fee token + ON oo.auction_id = ap_protocol.auction_id AND oo.protocol_fee_token = ap_protocol.token +), +batch_protocol_fees AS ( + SELECT + solver, + tx_hash, + -- sum(surplus * surplus_token_price) as surplus, + sum(protocol_fee * protocol_fee_token_price) as protocol_fee, + sum(network_fee_correction * network_fee_token_price) as network_fee_correction + FROM order_observations_prices oop + group by solver, tx_hash +), reward_data AS (SELECT -- observations - tx_hash, + os.tx_hash, ss.auction_id, -- TODO - Assuming that `solver == winner` when both not null -- We will need to monitor that `solver == winner`! - coalesce(solver, winner) as solver, + coalesce(os.solver, winner) as solver, block_number as settlement_block, block_deadline, case @@ -50,7 +173,10 @@ WITH observed_settlements AS (SELECT winning_score, reference_score, -- auction_participation - participating_solvers + participating_solvers, + -- protocol_fees + protocol_fee, + network_fee_correction FROM settlement_scores ss -- If there are reported scores, -- there will always be a record of auction participants @@ -58,7 +184,9 @@ WITH observed_settlements AS (SELECT ON ss.auction_id = ap.auction_id -- outer joins made in order to capture non-existent settlements. LEFT OUTER JOIN observed_settlements os - ON os.auction_id = ss.auction_id), + ON os.auction_id = ss.auction_id + LEFT OUTER JOIN batch_protocol_fees bpf + ON bpf.tx_hash = os.tx_hash), reward_per_auction as (SELECT tx_hash, auction_id, settlement_block, @@ -66,10 +194,11 @@ WITH observed_settlements AS (SELECT solver, execution_cost, surplus, - fee, - surplus + fee - reference_score as uncapped_reward_eth, + protocol_fee, + fee + network_fee_correction, + surplus + coalesce(protocol_fee - network_fee_correction, 0) + fee - reference_score as uncapped_reward_eth, -- Uncapped Reward = CLAMP_[-E, E + exec_cost](uncapped_reward_eth) - LEAST(GREATEST(-{{EPSILON}}, surplus + fee - reference_score), + LEAST(GREATEST(-{{EPSILON}}, surplus + coalesce(protocol_fee - network_fee_correction, 0) + fee - reference_score), {{EPSILON}} + execution_cost) as capped_payment, winning_score, reference_score, @@ -85,13 +214,20 @@ WITH observed_settlements AS (SELECT SUM(execution_cost) as exececution_cost_wei FROM reward_per_auction rpt GROUP BY solver), + protocol_fees as (SELECT solver, + SUM(protocol_fee) as protocol_fee_wei + FROM reward_per_auction rpt + GROUP BY solver), aggregate_results as (SELECT concat('0x', encode(pc.solver, 'hex')) as solver, coalesce(payment_wei, 0) as payment_eth, coalesce(exececution_cost_wei, 0) as execution_cost_eth, - num_participating_batches + num_participating_batches, + coalesce(protocol_fee_wei, 0) as protocol_fee_eth FROM participation_counts pc LEFT OUTER JOIN primary_rewards pr - ON pr.solver = pc.solver) + ON pr.solver = pc.solver + LEFT OUTER JOIN protocol_fees pf + ON pf.solver = pc.solver) select * from aggregate_results diff --git a/tests/queries/batch_rewards_test_db.sql b/tests/queries/batch_rewards_test_db.sql index c984e1d4..05f92e5c 100644 --- a/tests/queries/batch_rewards_test_db.sql +++ b/tests/queries/batch_rewards_test_db.sql @@ -3,6 +3,14 @@ DROP TABLE IF EXISTS auction_transaction; DROP TABLE IF EXISTS auction_participants; DROP TABLE IF EXISTS settlement_scores; DROP TABLE IF EXISTS settlement_observations; +DROP TABLE IF EXISTS auction_prices; +DROP TABLE IF EXISTS orders; +DROP TYPE IF EXISTS OrderKind; +DROP TYPE IF EXISTS OrderClass; +DROP TABLE IF EXISTS trades; +DROP TABLE IF EXISTS order_execution; +DROP TABLE IF EXISTS fee_policies; +DROP TYPE IF EXISTS PolicyKind; CREATE TABLE IF NOT EXISTS settlements ( @@ -55,12 +63,83 @@ CREATE TABLE IF NOT EXISTS settlement_observations PRIMARY KEY (block_number, log_index) ); +CREATE TABLE IF NOT EXISTS auction_prices +( + auction_id bigint NOT NULL, + token bytea NOT NULL, + price numeric(78, 0) NOT NULL, + + PRIMARY KEY (auction_id, token) +); + +-- orders table +CREATE TYPE OrderKind AS ENUM ('buy', 'sell'); +CREATE TYPE OrderClass AS ENUM ('market', 'limit'); + +CREATE TABLE orders ( + uid bytea PRIMARY KEY, + sell_token bytea NOT NULL, + buy_token bytea NOT NULL, + sell_amount numeric(78,0) NOT NULL, + buy_amount numeric(78,0) NOT NULL, + fee_amount numeric(78,0) NOT NULL, + kind OrderKind NOT NULL, + partially_fillable boolean NOT NULL, + full_fee_amount numeric(78,0) NOT NULL, + class OrderClass NOT NULL +); + +CREATE TABLE IF NOT EXISTS trades +( + block_number bigint NOT NULL, + log_index bigint NOT NULL, + order_uid bytea NOT NULL, + sell_amount numeric(78, 0) NOT NULL, + buy_amount numeric(78, 0) NOT NULL, + fee_amount numeric(78, 0) NOT NULL, + + PRIMARY KEY (block_number, log_index) +); + +CREATE TABLE IF NOT EXISTS order_execution +( + order_uid bytea NOT NULL, + auction_id bigint NOT NULL, + reward double precision NOT NULL, + surplus_fee numeric(78, 0) NOT NULL, + solver_fee numeric(78, 0), + + PRIMARY KEY (order_uid, auction_id) +); + +CREATE TYPE PolicyKind AS ENUM ('surplus', 'volume'); + +CREATE TABLE fee_policies ( + auction_id bigint NOT NULL, + order_uid bytea NOT NULL, + -- The order in which the fee policies are inserted and applied. + application_order SERIAL NOT NULL, + -- The type of the fee policy. + kind PolicyKind NOT NULL, + -- The fee should be taken as a percentage of the price improvement. The value is between 0 and 1. + surplus_factor double precision, + -- Cap the fee at a certain percentage of the order volume. The value is between 0 and 1. + max_volume_factor double precision, + -- The fee should be taken as a percentage of the order volume. The value is between 0 and 1. + volume_factor double precision, + PRIMARY KEY (auction_id, order_uid, application_order) +); + TRUNCATE settlements; TRUNCATE auction_transaction; TRUNCATE auction_participants; TRUNCATE settlement_scores; TRUNCATE settlement_observations; +TRUNCATE auction_prices; +TRUNCATE orders; +TRUNCATE trades; +TRUNCATE fee_policies; INSERT INTO settlements (block_number, log_index, solver, tx_hash, tx_from, tx_nonce) @@ -70,7 +149,13 @@ VALUES (1, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7111'::by -- would the following entry be in the data base? (submitted too late) -- YES (20, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7444'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 3), (25, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7555'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 4), - (26, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7666'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 6); + (26, 10, '\x5111111111111111111111111111111111111111'::bytea, '\x7666'::bytea, '\x5111111111111111111111111111111111111111'::bytea, 6), + (51, 10, '\x01'::bytea, '\x01'::bytea, '\x01'::bytea, 1), + (52, 10, '\x02'::bytea, '\x02'::bytea, '\x02'::bytea, 1), + (53, 10, '\x01'::bytea, '\x03'::bytea, '\x01'::bytea, 2), + (54, 10, '\x02'::bytea, '\x04'::bytea, '\x02'::bytea, 2), + (55, 10, '\x01'::bytea, '\x05'::bytea, '\x01'::bytea, 3), + (56, 10, '\x02'::bytea, '\x06'::bytea, '\x02'::bytea, 3); INSERT INTO auction_transaction (auction_id, tx_from, tx_nonce) VALUES (1, '\x5111111111111111111111111111111111111111'::bytea, 1), @@ -80,7 +165,13 @@ VALUES (1, '\x5111111111111111111111111111111111111111'::bytea, 1), (7, '\x5111111111111111111111111111111111111111'::bytea, 4), (8, '\x5111111111111111111111111111111111111111'::bytea, 5), -- would that entry be in the data base? (failed transaction) (9, '\x5111111111111111111111111111111111111111'::bytea, 6), - (10, '\x5333333333333333333333333333333333333333'::bytea, 1); -- would that entry be in the data base? (failed transaction) + (10, '\x5333333333333333333333333333333333333333'::bytea, 1), -- would that entry be in the data base? (failed transaction) + (51, '\x01'::bytea, 1), + (52, '\x02'::bytea, 1), + (53, '\x01'::bytea, 2), + (54, '\x02'::bytea, 2), + (55, '\x01'::bytea, 3), + (56, '\x02'::bytea, 3); INSERT INTO auction_participants (auction_id, participant) VALUES (1, '\x5222222222222222222222222222222222222222'::bytea), @@ -104,7 +195,13 @@ VALUES (1, '\x5222222222222222222222222222222222222222'::bytea), (9, '\x5333333333333333333333333333333333333333'::bytea), (9, '\x5111111111111111111111111111111111111111'::bytea), (10, '\x5444444444444444444444444444444444444444'::bytea), - (10, '\x5333333333333333333333333333333333333333'::bytea); + (10, '\x5333333333333333333333333333333333333333'::bytea), + (51, '\x01'::bytea), + (52, '\x02'::bytea), + (53, '\x01'::bytea), + (54, '\x02'::bytea), + (55, '\x01'::bytea), + (56, '\x02'::bytea); INSERT INTO settlement_scores (auction_id, winning_score, reference_score, winner, block_deadline, simulation_block) VALUES (1, 5000000000000000000, 4000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 10, 0), @@ -115,7 +212,13 @@ VALUES (1, 5000000000000000000, 4000000000000000000, '\x511111111111111111111111 (7, 5000000000000000000, 0, '\x5111111111111111111111111111111111111111'::bytea, 30, 6), -- no competition (8, 5000000000000000000, 0, '\x5111111111111111111111111111111111111111'::bytea, 35, 7), -- no competition, failed transaction (9, 5000000000000000000, 1000000000000000000, '\x5111111111111111111111111111111111111111'::bytea, 36, 8), -- score larger than quality - (10, 5000000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea, 37, 9); -- participant with net negative payment + (10, 5000000000000000000, 4000000000000000000, '\x5333333333333333333333333333333333333333'::bytea, 37, 9), -- participant with net negative payment + (51, 500000000000000, 0, '\x01'::bytea, 60, 50), + (52, 500000000000000, 0, '\x02'::bytea, 61, 51), + (53, 1000000000000000, 0, '\x01'::bytea, 62, 52), + (54, 2000000000000000, 0, '\x02'::bytea, 62, 52), + (55, 500000000000000, 0, '\x01'::bytea, 64, 54), + (56, 500000000000000, 0, '\x02'::bytea, 65, 55); -- score probably wrong, does not take protocol fee into account INSERT INTO settlement_observations (block_number, log_index, gas_used, effective_gas_price, surplus, fee) VALUES (1, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), @@ -125,4 +228,52 @@ VALUES (1, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), -- I would prefer to use the real numbers. What is backend gonna do. (20, 10, 100000, 2000000000, 0, 0), -- would that entry be in the data base? (submitted too late) (25, 10, 100000, 2000000000, 6000000000000000000, 200000000000000), - (26, 10, 100000, 2000000000, 0, 400000000000000); + (26, 10, 100000, 2000000000, 0, 400000000000000), + (51, 10, 100000, 25000000000, 500000000000000, 2500000000000000), + (52, 10, 100000, 25000000000, 500000000000000, 2500000000000000), + (53, 10, 100000, 25000000000, 500000000000000, 3000000000000000), + (54, 10, 100000, 25000000000, 500000000000000, 4000000000000000), + (55, 10, 100000, 25000000000, 500000000000000, 2500000000000000), + (56, 10, 100000, 25000000000, 500000000000000, 2500000000000000); + +INSERT INTO auction_prices (auction_id, token, price) +VALUES (51, '\x01', 500000000000000000000000000), +(51, '\x02', 500000000000000), +(52, '\x01', 500000000000000000000000000), +(52, '\x02', 500000000000000), +(53, '\x01', 500000000000000000000000000), +(53, '\x02', 500000000000000), +(54, '\x01', 500000000000000000000000000), +(54, '\x02', 500000000000000), +(55, '\x01', 500000000000000000000000000), +(55, '\x02', 500000000000000), +(56, '\x01', 500000000000000000000000000), +(56, '\x02', 500000000000000); + +INSERT INTO orders (uid, sell_token, buy_token, sell_amount, buy_amount, fee_amount, kind, partially_fillable, full_fee_amount, class) +VALUES ('\x01'::bytea, '\x01'::bytea, '\x02'::bytea, 95000000, 94000000000000000000, 5000000, 'sell', 'f', 5000000, 'market'), -- sell market order +('\x02'::bytea, '\x01'::bytea, '\x02'::bytea, 101000000, 100000000000000000000, 5000000, 'buy', 'f', 5000000, 'market'), -- buy market order +('\x03'::bytea, '\x01'::bytea, '\x02'::bytea, 100000000, 100000000000000000000, 0, 'sell', 't', 0, 'limit'), -- partially fillable sell limit order +('\x04'::bytea, '\x01'::bytea, '\x02'::bytea, 100000000, 100000000000000000000, 0, 'buy', 't', 0, 'limit'), -- partially fillable buy limit order +('\x05'::bytea, '\x01'::bytea, '\x02'::bytea, 100000000, 94000000000000000000, 0, 'sell', 'f', 0, 'limit'), -- in market sell limit order +('\x06'::bytea, '\x01'::bytea, '\x02'::bytea, 106000000, 100000000000000000000, 0, 'buy', 'f', 0, 'limit'); -- in market buy limit order + +INSERT INTO trades (block_number, log_index, order_uid, sell_amount, buy_amount, fee_amount) +VALUES (51, 0, '\x01'::bytea, 100000000, 95000000000000000000, 5000000), +(52, 0, '\x02'::bytea, 106000000, 100000000000000000000, 5000000), +(53, 0, '\x03'::bytea, 100000000, 101000000000000000000, 0), +(54, 0, '\x04'::bytea, 99000000, 100000000000000000000, 0), +(55, 0, '\x05'::bytea, 100000000, 95000000000000000000, 0), +(56, 0, '\x06'::bytea, 105000000, 100000000000000000000, 0); + +INSERT INTO order_execution (order_uid, auction_id, reward, surplus_fee, solver_fee) +VALUES ('\x03'::bytea, 53, 0, 5931372, NULL), +('\x04'::bytea, 54, 0, 6000000, NULL), +('\x05'::bytea, 55, 0, 6000000, NULL), +('\x06'::bytea, 56, 0, 6000000, NULL); + +INSERT INTO fee_policies (auction_id, order_uid, application_order, kind, surplus_factor, max_volume_factor, volume_factor) +VALUES (53, '\x03'::bytea, 3, 'surplus', 0.5, 0.02, NULL), +(54, '\x04'::bytea, 4, 'surplus', 0.75, 0.1, NULL), +(55, '\x05'::bytea, 5, 'volume', NULL, NULL, 0.0015), +(56, '\x06'::bytea, 6, 'surplus', 0.9, 0.01, NULL); diff --git a/tests/queries/test_batch_rewards.py b/tests/queries/test_batch_rewards.py index fc6f3fb2..fc790af4 100644 --- a/tests/queries/test_batch_rewards.py +++ b/tests/queries/test_batch_rewards.py @@ -21,29 +21,45 @@ def test_get_batch_rewards(self): expected = DataFrame( { "solver": [ + "0x01", + "0x02", "0x5111111111111111111111111111111111111111", "0x5222222222222222222222222222222222222222", "0x5333333333333333333333333333333333333333", "0x5444444444444444444444444444444444444444", ], "payment_eth": [ + 9.534313722772278e15, + 10.5e15, 600000000000000.00000, 10450000000000000.00000, -10000000000000000.00000, 0.00000, ], "execution_cost_eth": [ + 7500000000000000.0, + 7500000000000000.0, 800000000000000.00000, 450000000000000.00000, 0.00000, 0.00000, ], "num_participating_batches": [ + 3, + 3, 7, 2, 7, 6, ], + "protocol_fee_eth": [ + 5.751126690035052e14, + 2.0303030303030302e15, + 0.0, + 0.0, + 0.0, + 0.0, + ], } ) self.assertIsNone(pandas.testing.assert_frame_equal(expected, batch_rewards)) From 604b4a45545d665dcf871e6ec7016e791cbc7942 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 11:40:11 +0100 Subject: [PATCH 19/33] address review comments - add a few comments - reorder terms - fix typos --- queries/orderbook/batch_rewards.sql | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/queries/orderbook/batch_rewards.sql b/queries/orderbook/batch_rewards.sql index 5a5ba1de..4e95aae8 100644 --- a/queries/orderbook/batch_rewards.sql +++ b/queries/orderbook/batch_rewards.sql @@ -35,17 +35,17 @@ order_surplus AS ( at.auction_id, s.tx_hash, t.order_uid, - o.sell_token, -- the total amount the user sends + o.sell_token, o.buy_token, - t.sell_amount, - t.buy_amount, - oe.surplus_fee as observed_fee, + t.sell_amount, -- the total amount the user sends + t.buy_amount, -- the total amount the user receives + oe.surplus_fee as observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price o.kind, CASE WHEN o.kind = 'sell' - THEN t.buy_amount - o.buy_amount / (o.sell_amount + o.fee_amount) * t.sell_amount + THEN t.buy_amount - t.sell_amount * o.buy_amount / (o.sell_amount + o.fee_amount) WHEN o.kind = 'buy' - THEN (o.sell_amount + o.fee_amount) / o.buy_amount * t.buy_amount - t.sell_amount + THEN t.buy_amount * (o.sell_amount + o.fee_amount) / o.buy_amount - t.sell_amount END AS surplus, CASE WHEN o.kind = 'sell' @@ -72,7 +72,7 @@ order_surplus AS ( os.auction_id, os.solver, os.tx_hash, - os.sell_amount, -- the total amount the user sends + os.sell_amount, os.buy_amount, os.sell_token, os.observed_fee, @@ -85,7 +85,11 @@ order_surplus AS ( WHEN os.kind = 'sell' THEN CASE - WHEN fp.max_volume_factor = 1 -- this is done to avoid a division by zero bug + WHEN fp.max_volume_factor = 1 -- this is done to avoid a division by zero errors + -- We assume that the case surplus_factor != 1 always. In + -- that case reconstructing the protocol fee would be + -- impossible anyways. This query will return a division by + -- zero error in that case. THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus ELSE LEAST( @@ -194,8 +198,8 @@ batch_protocol_fees AS ( solver, execution_cost, surplus, - protocol_fee, - fee + network_fee_correction, + fee - coalesce(network_fee_correction, 0) as fee, -- an estimated network fee + coalesce(protocol_fee, 0) as protocol_fee, -- the protocol fee surplus + coalesce(protocol_fee - network_fee_correction, 0) + fee - reference_score as uncapped_reward_eth, -- Uncapped Reward = CLAMP_[-E, E + exec_cost](uncapped_reward_eth) LEAST(GREATEST(-{{EPSILON}}, surplus + coalesce(protocol_fee - network_fee_correction, 0) + fee - reference_score), From 34462be9cf354822fa6ca1eeae2eb4bee67d6a64 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 12:19:58 +0100 Subject: [PATCH 20/33] add protocol fees to transfers - new protocol_fee_eth field in payouts - subtract protocol fee from eth reimbursement - update tests: add one new test, update old tests (also for quote rewards) --- src/fetch/payouts.py | 10 ++++- tests/unit/test_payouts.py | 78 +++++++++++++++++++++++++++++--------- 2 files changed, 69 insertions(+), 19 deletions(-) diff --git a/src/fetch/payouts.py b/src/fetch/payouts.py index 45998382..93247683 100644 --- a/src/fetch/payouts.py +++ b/src/fetch/payouts.py @@ -33,6 +33,7 @@ "reward_cow", "secondary_reward_cow", "quote_reward_cow", + "protocol_fee_eth", } SLIPPAGE_COLUMNS = { "solver", @@ -79,10 +80,13 @@ def __init__( # pylint: disable=too-many-arguments primary_reward_cow: int, secondary_reward_cow: int, quote_reward_cow: int, + protocol_fee_eth: int, ): assert exec_cost >= 0, f"invalid execution cost {exec_cost}" assert secondary_reward_eth >= 0, "invalid secondary_reward_eth" assert secondary_reward_cow >= 0, "invalid secondary_reward_cow" + assert quote_reward_cow >= 0, "invalid quote reward" + assert protocol_fee_eth >= 0, "invalid protocol_fee_eth" self.solver = solver self.solver_name = solver_name @@ -94,6 +98,7 @@ def __init__( # pylint: disable=too-many-arguments self.secondary_reward_eth = secondary_reward_eth self.secondary_reward_cow = secondary_reward_cow self.quote_reward_cow = quote_reward_cow + self.protocol_fee_eth = protocol_fee_eth @classmethod def from_series(cls, frame: Series) -> RewardAndPenaltyDatum: @@ -120,11 +125,12 @@ def from_series(cls, frame: Series) -> RewardAndPenaltyDatum: secondary_reward_eth=int(frame["secondary_reward_eth"]), secondary_reward_cow=int(frame["secondary_reward_cow"]), quote_reward_cow=int(frame["quote_reward_cow"]), + protocol_fee_eth=int(frame["protocol_fee_eth"]), ) def total_outgoing_eth(self) -> int: """Total outgoing amount (including slippage) for the payout.""" - return self.payment_eth + self.secondary_reward_eth + self.slippage_eth + return self.payment_eth + self.secondary_reward_eth + self.slippage_eth - self.protocol_fee_eth def total_cow_reward(self) -> int: """Total outgoing COW token reward""" @@ -161,7 +167,7 @@ def as_payouts(self) -> list[Transfer]: total_eth_reward = int(self.total_eth_reward()) total_cow_reward = int(self.total_cow_reward()) - reimbursement_eth = int(self.exec_cost + self.slippage_eth) + reimbursement_eth = int(self.exec_cost + self.slippage_eth - self.protocol_fee_eth) # We do not have access to token conversion here, but we do have other converted values # x_eth:x_cow = y_eth:y_cow --> y_cow = y_eth * x_cow / x_eth reimbursement_cow = ( diff --git a/tests/unit/test_payouts.py b/tests/unit/test_payouts.py index fc4ff8e5..efd1c214 100644 --- a/tests/unit/test_payouts.py +++ b/tests/unit/test_payouts.py @@ -67,6 +67,12 @@ def setUp(self) -> None: 7, 6, ] + self.protocol_fees = [ + 300000000000000.0, + 200000000000000.0, + 100000000000000.0, + 0.0, + ] # Mocking TokenConversion! self.mock_converter = TokenConversion( eth_to_token=lambda t: int(t * 1000), token_to_eth=lambda t: t // 1000 @@ -79,6 +85,7 @@ def test_extend_payment_df(self): "payment_eth": self.eth_payments, "execution_cost_eth": self.execution_costs, "num_participating_batches": self.batch_participation, + "protocol_fee_eth": self.protocol_fees } base_payout_df = DataFrame(base_data_dict) result = extend_payment_df(base_payout_df, converter=self.mock_converter) @@ -88,6 +95,7 @@ def test_extend_payment_df(self): "payment_eth": self.eth_payments, "execution_cost_eth": self.execution_costs, "num_participating_batches": self.batch_participation, + "protocol_fee_eth": self.protocol_fees, "reward_eth": [ -200000000000000.00000, 10000000000000000.00000, @@ -142,6 +150,7 @@ def test_validate_df_columns(self): "payment_eth": [], "execution_cost_eth": [], "num_participating_batches": [], + "protocol_fee_eth": [], "reward_eth": [], "reward_cow": [], "secondary_reward_cow": [], @@ -191,6 +200,7 @@ def test_construct_payouts(self): "payment_eth": self.eth_payments, "execution_cost_eth": self.execution_costs, "num_participating_batches": self.batch_participation, + "protocol_fee_eth": self.protocol_fees, } ), converter=self.mock_converter, @@ -219,6 +229,7 @@ def test_construct_payouts(self): "payment_eth": [600000000000000.0, 1.045e16, -1e16, 0.0], "execution_cost_eth": [800000000000000.0, 450000000000000.0, 0.0, 0.0], "num_participating_batches": [7, 2, 7, 6], + "protocol_fee_eth": self.protocol_fees, "reward_eth": [-200000000000000.0, 1e16, -1e16, 0.0], "reward_cow": [ -200000000000000000, @@ -290,6 +301,12 @@ def test_prepare_transfers(self): 90000000000000000000.00000, 180000000000000000000.00000, ], + "protocol_fee_eth": [ + 300000000000000.0, + 200000000000000.0, + 100000000000000.0, + 0.0, + ], "solver_name": ["S_1", "S_2", "S_3", None], "eth_slippage_wei": [1.0, 0.0, -1.0, None], "reward_target": [ @@ -308,12 +325,12 @@ def test_prepare_transfers(self): Transfer( token=None, recipient=Address(self.solvers[0]), - amount_wei=663636363636364, + amount_wei=363636363636364, ), Transfer( token=None, recipient=Address(self.solvers[1]), - amount_wei=450000000000000, + amount_wei=250000000000000, ), Transfer( token=Token(COW_TOKEN_ADDRESS), @@ -344,7 +361,7 @@ def test_prepare_transfers(self): Overdraft( period, account=Address(self.solvers[2]), - wei=9936363636363638, + wei=10036363636363638, name="S_3", ) ], @@ -366,6 +383,7 @@ def sample_record( participation: int, slippage: int, num_quotes: int, + protocol_fee: int, ): """Assumes a conversion rate of ETH:COW <> 1:self.conversion_rate""" return RewardAndPenaltyDatum( @@ -379,32 +397,39 @@ def sample_record( secondary_reward_cow=participation * self.conversion_rate, slippage_eth=slippage, quote_reward_cow=QUOTE_REWARD * num_quotes, + protocol_fee_eth=protocol_fee, ) def test_invalid_input(self): with self.assertRaises(AssertionError): - self.sample_record(0, -1, 0, 0, 0) + self.sample_record(0, -1, 0, 0, 0, 0) + + with self.assertRaises(AssertionError): + self.sample_record(0, 0, -1, 0, 0, 0) + + with self.assertRaises(AssertionError): + self.sample_record(0, 0, 0, 0, -1, 0) with self.assertRaises(AssertionError): - self.sample_record(0, 0, -1, 0, 0) + self.sample_record(0, 0, 0, 0, 0, -1) - def test_reward_datum_0_0_0_0(self): - test_datum = self.sample_record(0, 0, 0, 0, 0) + def test_reward_datum_0_0_0_0_0(self): + test_datum = self.sample_record(0, 0, 0, 0, 0, 0) self.assertFalse(test_datum.is_overdraft()) self.assertEqual(test_datum.as_payouts(), []) - def test_reward_datum_1_1_0_0(self): + def test_reward_datum_1_1_0_0_0_0(self): cost = 1 - test_datum = self.sample_record(1, cost, 0, 0, 0) + test_datum = self.sample_record(1, cost, 0, 0, 0, 0) self.assertFalse(test_datum.is_overdraft()) self.assertEqual( test_datum.as_payouts(), [Transfer(token=None, recipient=self.solver, amount_wei=cost)], ) - def test_reward_datum_3_2_0_minus1(self): + def test_reward_datum_3_2_0_minus1_0_0(self): payment, cost, participation, slippage = 3, 2, 0, -1 - test_datum = self.sample_record(payment, cost, participation, slippage, 0) + test_datum = self.sample_record(payment, cost, participation, slippage, 0, 0) self.assertFalse(test_datum.is_overdraft()) self.assertEqual( test_datum.as_payouts(), @@ -421,11 +446,30 @@ def test_reward_datum_3_2_0_minus1(self): ), ], ) + def test_reward_datum_8_4_0_minus1_0_2(self): + payment, cost, participation, slippage, protocol_fee = 8, 4, 0, -1, 2 + test_datum = self.sample_record(payment, cost, participation, slippage, 0, protocol_fee) + self.assertFalse(test_datum.is_overdraft()) + self.assertEqual( + test_datum.as_payouts(), + [ + Transfer( + token=None, + recipient=self.solver, + amount_wei=cost + slippage - protocol_fee, + ), + Transfer( + token=self.cow_token, + recipient=self.reward_target, + amount_wei=(payment - cost) * self.conversion_rate, + ), + ], + ) def test_reward_datum_cost_exceeds_payment_degenerate(self): # Degenerate Case! payment, cost, participation, slippage = 1, 10, 0, -1 - test_datum = self.sample_record(payment, cost, participation, slippage, 0) + test_datum = self.sample_record(payment, cost, participation, slippage, 0, 0) self.assertFalse(test_datum.is_overdraft()) self.assertEqual( test_datum.as_payouts(), @@ -439,7 +483,7 @@ def test_reward_datum_cost_exceeds_payment_non_degenerate(self): cost = max(sum(x) for x in triplets) + 1 for payment, participation, slippage in triplets: - test_datum = self.sample_record(payment, cost, participation, slippage, 0) + test_datum = self.sample_record(payment, cost, participation, slippage, 0, 0) self.assertFalse(test_datum.is_overdraft()) self.assertLess(test_datum.total_outgoing_eth(), cost) self.assertEqual( @@ -462,12 +506,12 @@ def test_reward_datum_overdraft(self): for payment, participation, slippage in triplets: for cost in [0, 1, 100]: # Doesn't matter their costs, they are in overdraft state! - rec = self.sample_record(payment, cost, participation, slippage, 0) + rec = self.sample_record(payment, cost, participation, slippage, 0, 0) self.assertTrue(rec.is_overdraft()) - def test_reward_datum_1_1_1_1(self): + def test_reward_datum_1_1_1_1_0_0(self): payment, cost, participation, slippage = 1, 1, 1, 1 - test_datum = self.sample_record(payment, cost, participation, slippage, 0) + test_datum = self.sample_record(payment, cost, participation, slippage, 0, 0) self.assertFalse(test_datum.is_overdraft()) self.assertEqual( @@ -491,7 +535,7 @@ def test_reward_datum_1_1_1_1(self): def test_payout_negative_payments(self): payment, cost, participation, slippage = -1, 1, 1, 1 - test_datum = self.sample_record(payment, cost, participation, slippage, 0) + test_datum = self.sample_record(payment, cost, participation, slippage, 0, 0) self.assertEqual( test_datum.as_payouts(), [ From 579b55c49e6acfd67d6eddfa9a111bbd3fdd86cf Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 12:25:11 +0100 Subject: [PATCH 21/33] black fix --- src/fetch/payouts.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/fetch/payouts.py b/src/fetch/payouts.py index 93247683..c7775743 100644 --- a/src/fetch/payouts.py +++ b/src/fetch/payouts.py @@ -130,7 +130,12 @@ def from_series(cls, frame: Series) -> RewardAndPenaltyDatum: def total_outgoing_eth(self) -> int: """Total outgoing amount (including slippage) for the payout.""" - return self.payment_eth + self.secondary_reward_eth + self.slippage_eth - self.protocol_fee_eth + return ( + self.payment_eth + + self.secondary_reward_eth + + self.slippage_eth + - self.protocol_fee_eth + ) def total_cow_reward(self) -> int: """Total outgoing COW token reward""" @@ -167,7 +172,9 @@ def as_payouts(self) -> list[Transfer]: total_eth_reward = int(self.total_eth_reward()) total_cow_reward = int(self.total_cow_reward()) - reimbursement_eth = int(self.exec_cost + self.slippage_eth - self.protocol_fee_eth) + reimbursement_eth = int( + self.exec_cost + self.slippage_eth - self.protocol_fee_eth + ) # We do not have access to token conversion here, but we do have other converted values # x_eth:x_cow = y_eth:y_cow --> y_cow = y_eth * x_cow / x_eth reimbursement_cow = ( From ea970bb705cf4ae99f65367f7e9fdf6d8935548d Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 12:27:34 +0100 Subject: [PATCH 22/33] more black fixes --- tests/unit/test_payouts.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_payouts.py b/tests/unit/test_payouts.py index efd1c214..e3298cbc 100644 --- a/tests/unit/test_payouts.py +++ b/tests/unit/test_payouts.py @@ -85,7 +85,7 @@ def test_extend_payment_df(self): "payment_eth": self.eth_payments, "execution_cost_eth": self.execution_costs, "num_participating_batches": self.batch_participation, - "protocol_fee_eth": self.protocol_fees + "protocol_fee_eth": self.protocol_fees, } base_payout_df = DataFrame(base_data_dict) result = extend_payment_df(base_payout_df, converter=self.mock_converter) @@ -446,9 +446,12 @@ def test_reward_datum_3_2_0_minus1_0_0(self): ), ], ) + def test_reward_datum_8_4_0_minus1_0_2(self): payment, cost, participation, slippage, protocol_fee = 8, 4, 0, -1, 2 - test_datum = self.sample_record(payment, cost, participation, slippage, 0, protocol_fee) + test_datum = self.sample_record( + payment, cost, participation, slippage, 0, protocol_fee + ) self.assertFalse(test_datum.is_overdraft()) self.assertEqual( test_datum.as_payouts(), @@ -483,7 +486,9 @@ def test_reward_datum_cost_exceeds_payment_non_degenerate(self): cost = max(sum(x) for x in triplets) + 1 for payment, participation, slippage in triplets: - test_datum = self.sample_record(payment, cost, participation, slippage, 0, 0) + test_datum = self.sample_record( + payment, cost, participation, slippage, 0, 0 + ) self.assertFalse(test_datum.is_overdraft()) self.assertLess(test_datum.total_outgoing_eth(), cost) self.assertEqual( From 4ffea87b15759a6617a1459aeb1629aabce5e218 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 16:09:22 +0100 Subject: [PATCH 23/33] small fix to query --- queries/orderbook/batch_rewards.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/queries/orderbook/batch_rewards.sql b/queries/orderbook/batch_rewards.sql index 4e95aae8..64905f7f 100644 --- a/queries/orderbook/batch_rewards.sql +++ b/queries/orderbook/batch_rewards.sql @@ -179,8 +179,8 @@ batch_protocol_fees AS ( -- auction_participation participating_solvers, -- protocol_fees - protocol_fee, - network_fee_correction + coalesce(cast(protocol_fee as numeric(78, 0)), 0) as protocol_fee, + coalesce(cast(network_fee_correction as numeric(78, 0)), 0) as network_fee_correction FROM settlement_scores ss -- If there are reported scores, -- there will always be a record of auction participants @@ -198,9 +198,9 @@ batch_protocol_fees AS ( solver, execution_cost, surplus, - fee - coalesce(network_fee_correction, 0) as fee, -- an estimated network fee - coalesce(protocol_fee, 0) as protocol_fee, -- the protocol fee - surplus + coalesce(protocol_fee - network_fee_correction, 0) + fee - reference_score as uncapped_reward_eth, + protocol_fee + fee - network_fee_correction as fee,-- total fee for ranking + protocol_fee, -- the protocol fee + surplus + protocol_fee + fee - network_fee_correction - reference_score as uncapped_reward_eth, -- Uncapped Reward = CLAMP_[-E, E + exec_cost](uncapped_reward_eth) LEAST(GREATEST(-{{EPSILON}}, surplus + coalesce(protocol_fee - network_fee_correction, 0) + fee - reference_score), {{EPSILON}} + execution_cost) as capped_payment, @@ -232,7 +232,7 @@ batch_protocol_fees AS ( ON pr.solver = pc.solver LEFT OUTER JOIN protocol_fees pf ON pf.solver = pc.solver) - +-- select * from aggregate_results order by solver; From b76b3ed21cd2c6ca7dbc2eccdc21cbcc19091b6e Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 16:51:55 +0100 Subject: [PATCH 24/33] remove unused files --- tests/queries/protocol_fee_test_db.sql | 191 ------------------------- tests/queries/test_protocol_fees.py | 41 ------ 2 files changed, 232 deletions(-) delete mode 100644 tests/queries/protocol_fee_test_db.sql delete mode 100644 tests/queries/test_protocol_fees.py diff --git a/tests/queries/protocol_fee_test_db.sql b/tests/queries/protocol_fee_test_db.sql deleted file mode 100644 index caf98ff4..00000000 --- a/tests/queries/protocol_fee_test_db.sql +++ /dev/null @@ -1,191 +0,0 @@ -DROP TABLE IF EXISTS settlements; -DROP TABLE IF EXISTS auction_transaction; -DROP TABLE IF EXISTS settlement_scores; -DROP TABLE IF EXISTS auction_prices; -DROP TABLE IF EXISTS orders; -DROP TYPE IF EXISTS OrderKind; -DROP TYPE IF EXISTS OrderClass; -DROP TABLE IF EXISTS trades; -DROP TABLE IF EXISTS order_execution; -DROP TABLE IF EXISTS fee_policies; -DROP TYPE IF EXISTS PolicyKind; - -CREATE TABLE IF NOT EXISTS settlements -( - block_number bigint NOT NULL, - log_index bigint NOT NULL, - solver bytea NOT NULL, - tx_hash bytea NOT NULL, - tx_from bytea NOT NULL, - tx_nonce bigint NOT NULL, - - PRIMARY KEY (block_number, log_index) -); - -CREATE INDEX settlements_tx_from_tx_nonce ON settlements (tx_from, tx_nonce); - -CREATE TABLE IF NOT EXISTS auction_transaction -( - auction_id bigint PRIMARY KEY, - tx_from bytea NOT NULL, - tx_nonce bigint NOT NULL, - UNIQUE (tx_from, tx_nonce) -); - -CREATE TABLE IF NOT EXISTS auction_participants -( - auction_id bigint, - participant bytea -); - -CREATE TABLE IF NOT EXISTS settlement_scores -( - auction_id bigint PRIMARY KEY, - winner bytea NOT NULL, - winning_score numeric(78, 0) NOT NULL, - reference_score numeric(78, 0) NOT NULL, - block_deadline bigint NOT NULL, - simulation_block bigint NOT NULL -); - -CREATE TABLE IF NOT EXISTS auction_prices -( - auction_id bigint NOT NULL, - token bytea NOT NULL, - price numeric(78, 0) NOT NULL, - - PRIMARY KEY (auction_id, token) -); - --- orders table -CREATE TYPE OrderKind AS ENUM ('buy', 'sell'); -CREATE TYPE OrderClass AS ENUM ('market', 'limit'); - -CREATE TABLE orders ( - uid bytea PRIMARY KEY, - sell_token bytea NOT NULL, - buy_token bytea NOT NULL, - sell_amount numeric(78,0) NOT NULL, - buy_amount numeric(78,0) NOT NULL, - fee_amount numeric(78,0) NOT NULL, - kind OrderKind NOT NULL, - partially_fillable boolean NOT NULL, - full_fee_amount numeric(78,0) NOT NULL, - class OrderClass NOT NULL -); - -CREATE TABLE IF NOT EXISTS trades -( - block_number bigint NOT NULL, - log_index bigint NOT NULL, - order_uid bytea NOT NULL, - sell_amount numeric(78, 0) NOT NULL, - buy_amount numeric(78, 0) NOT NULL, - fee_amount numeric(78, 0) NOT NULL, - - PRIMARY KEY (block_number, log_index) -); - -CREATE TABLE IF NOT EXISTS order_execution -( - order_uid bytea NOT NULL, - auction_id bigint NOT NULL, - reward double precision NOT NULL, - surplus_fee numeric(78, 0) NOT NULL, - solver_fee numeric(78, 0), - - PRIMARY KEY (order_uid, auction_id) -); - -CREATE TYPE PolicyKind AS ENUM ('surplus', 'volume'); - -CREATE TABLE fee_policies ( - auction_id bigint NOT NULL, - order_uid bytea NOT NULL, - -- The order in which the fee policies are inserted and applied. - application_order SERIAL NOT NULL, - -- The type of the fee policy. - kind PolicyKind NOT NULL, - -- The fee should be taken as a percentage of the price improvement. The value is between 0 and 1. - surplus_factor double precision, - -- Cap the fee at a certain percentage of the order volume. The value is between 0 and 1. - max_volume_factor double precision, - -- The fee should be taken as a percentage of the order volume. The value is between 0 and 1. - volume_factor double precision, - PRIMARY KEY (auction_id, order_uid, application_order) -); - -TRUNCATE settlements; -TRUNCATE auction_transaction; -TRUNCATE settlement_scores; -TRUNCATE auction_prices; -TRUNCATE orders; -TRUNCATE trades; -TRUNCATE fee_policies; - - -INSERT INTO settlements (block_number, log_index, solver, tx_hash, tx_from, tx_nonce) -VALUES (1, 10, '\x01'::bytea, '\x01'::bytea, '\x01'::bytea, 1), - (2, 10, '\x02'::bytea, '\x02'::bytea, '\x02'::bytea, 1), - (3, 10, '\x01'::bytea, '\x03'::bytea, '\x01'::bytea, 2), - (4, 10, '\x02'::bytea, '\x04'::bytea, '\x02'::bytea, 2), - (5, 10, '\x01'::bytea, '\x05'::bytea, '\x01'::bytea, 3), - (6, 10, '\x02'::bytea, '\x06'::bytea, '\x02'::bytea, 3); - -INSERT INTO auction_transaction (auction_id, tx_from, tx_nonce) -VALUES (1, '\x01'::bytea, 1), - (2, '\x02'::bytea, 1), - (3, '\x01'::bytea, 2), - (4, '\x02'::bytea, 2), - (5, '\x01'::bytea, 3), - (6, '\x02'::bytea, 3); - -INSERT INTO settlement_scores (auction_id, winning_score, reference_score, winner, block_deadline, simulation_block) -VALUES (1, 5000000000000000000, 4000000000000000000, '\x01'::bytea, 10, 0), - (2, 6000000000000000000, 3000000000000000000, '\x02'::bytea, 11, 1), - (3, 21000000000000000000, 3000000000000000000, '\x01'::bytea, 12, 2), - (4, 21000000000000000000, 3000000000000000000, '\x02'::bytea, 12, 2), - (5, 5000000000000000000, 3000000000000000000, '\x01'::bytea, 14, 4), - (6, 10000000000000000000, 9000000000000000000, '\x02'::bytea, 15, 5); - -INSERT INTO auction_prices (auction_id, token, price) -VALUES (1, '\x01', 500000000000000000000000000), -(1, '\x02', 500000000000000), -(2, '\x01', 500000000000000000000000000), -(2, '\x02', 500000000000000), -(3, '\x01', 500000000000000000000000000), -(3, '\x02', 500000000000000), -(4, '\x01', 500000000000000000000000000), -(4, '\x02', 500000000000000), -(5, '\x01', 500000000000000000000000000), -(5, '\x02', 500000000000000), -(6, '\x01', 500000000000000000000000000), -(6, '\x02', 500000000000000); - -INSERT INTO orders (uid, sell_token, buy_token, sell_amount, buy_amount, fee_amount, kind, partially_fillable, full_fee_amount, class) -VALUES ('\x01'::bytea, '\x01'::bytea, '\x02'::bytea, 95000000, 94000000000000000000, 5000000, 'sell', 'f', 5000000, 'market'), -- sell market order -('\x02'::bytea, '\x01'::bytea, '\x02'::bytea, 101000000, 100000000000000000000, 5000000, 'buy', 'f', 5000000, 'market'), -- buy market order -('\x03'::bytea, '\x01'::bytea, '\x02'::bytea, 100000000, 100000000000000000000, 0, 'sell', 't', 0, 'limit'), -- partially fillable sell limit order -('\x04'::bytea, '\x01'::bytea, '\x02'::bytea, 100000000, 100000000000000000000, 0, 'buy', 't', 0, 'limit'), -- partially fillable buy limit order -('\x05'::bytea, '\x01'::bytea, '\x02'::bytea, 100000000, 94000000000000000000, 0, 'sell', 'f', 0, 'limit'), -- in market sell limit order -('\x06'::bytea, '\x01'::bytea, '\x02'::bytea, 106000000, 100000000000000000000, 0, 'buy', 'f', 0, 'limit'); -- in market buy limit order - -INSERT INTO trades (block_number, log_index, order_uid, sell_amount, buy_amount, fee_amount) -VALUES (1, 0, '\x01'::bytea, 100000000, 95000000000000000000, 5000000), -(2, 0, '\x02'::bytea, 106000000, 100000000000000000000, 5000000), -(3, 0, '\x03'::bytea, 100000000, 101000000000000000000, 0), -(4, 0, '\x04'::bytea, 99000000, 100000000000000000000, 0), -(5, 0, '\x05'::bytea, 100000000, 95000000000000000000, 0), -(6, 0, '\x06'::bytea, 105000000, 100000000000000000000, 0); - -INSERT INTO order_execution (order_uid, auction_id, reward, surplus_fee, solver_fee) -VALUES ('\x03'::bytea, 3, 0, 5931372, NULL), -('\x04'::bytea, 4, 0, 6000000, NULL), -('\x05'::bytea, 5, 0, 6000000, NULL), -('\x06'::bytea, 6, 0, 6000000, NULL); - -INSERT INTO fee_policies (auction_id, order_uid, application_order, kind, surplus_factor, max_volume_factor, volume_factor) -VALUES (3, '\x03'::bytea, 3, 'surplus', 0.5, 0.02, NULL), -(4, '\x04'::bytea, 4, 'surplus', 0.75, 0.1, NULL), -(5, '\x05'::bytea, 5, 'volume', NULL, NULL, 0.0015), -(6, '\x06'::bytea, 6, 'surplus', 0.9, 0.01, NULL); diff --git a/tests/queries/test_protocol_fees.py b/tests/queries/test_protocol_fees.py deleted file mode 100644 index 00bcd5b9..00000000 --- a/tests/queries/test_protocol_fees.py +++ /dev/null @@ -1,41 +0,0 @@ -import unittest - -import pandas.testing -from pandas import DataFrame - -from src.pg_client import MultiInstanceDBFetcher - - -class TestProtocolFees(unittest.TestCase): - def setUp(self) -> None: - db_url = "postgres:postgres@localhost:5432/postgres" - self.fetcher = MultiInstanceDBFetcher([db_url]) - with open( - "./tests/queries/protocol_fee_test_db.sql", "r", encoding="utf-8" - ) as file: - self.fetcher.connections[0].execute(file.read()) - - def test_get_protocol_fees(self): - start_block, end_block = "0", "100" - protocol_fees = self.fetcher.get_protocol_fees(start_block, end_block) - expected = DataFrame( - { - "solver": [ - "0x01", - "0x02", - ], - "protocol_fee": [ - 5.751126690035052e14, - 2.0303030303030302e15, - ], - "network_fee_correction": [ - -540798946231228.0, - -2030303030303030.5, - ], - } - ) - self.assertIsNone(pandas.testing.assert_frame_equal(expected, protocol_fees)) - - -if __name__ == "__main__": - unittest.main() From 9419122b514cbd4530fbaef3f2605adc15dfb843 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Sat, 20 Jan 2024 09:24:03 +0100 Subject: [PATCH 25/33] Revert "add protocol fees to transfers" This reverts commit 34462be9cf354822fa6ca1eeae2eb4bee67d6a64. --- src/fetch/payouts.py | 17 +------- tests/unit/test_payouts.py | 83 ++++++++------------------------------ 2 files changed, 19 insertions(+), 81 deletions(-) diff --git a/src/fetch/payouts.py b/src/fetch/payouts.py index c7775743..45998382 100644 --- a/src/fetch/payouts.py +++ b/src/fetch/payouts.py @@ -33,7 +33,6 @@ "reward_cow", "secondary_reward_cow", "quote_reward_cow", - "protocol_fee_eth", } SLIPPAGE_COLUMNS = { "solver", @@ -80,13 +79,10 @@ def __init__( # pylint: disable=too-many-arguments primary_reward_cow: int, secondary_reward_cow: int, quote_reward_cow: int, - protocol_fee_eth: int, ): assert exec_cost >= 0, f"invalid execution cost {exec_cost}" assert secondary_reward_eth >= 0, "invalid secondary_reward_eth" assert secondary_reward_cow >= 0, "invalid secondary_reward_cow" - assert quote_reward_cow >= 0, "invalid quote reward" - assert protocol_fee_eth >= 0, "invalid protocol_fee_eth" self.solver = solver self.solver_name = solver_name @@ -98,7 +94,6 @@ def __init__( # pylint: disable=too-many-arguments self.secondary_reward_eth = secondary_reward_eth self.secondary_reward_cow = secondary_reward_cow self.quote_reward_cow = quote_reward_cow - self.protocol_fee_eth = protocol_fee_eth @classmethod def from_series(cls, frame: Series) -> RewardAndPenaltyDatum: @@ -125,17 +120,11 @@ def from_series(cls, frame: Series) -> RewardAndPenaltyDatum: secondary_reward_eth=int(frame["secondary_reward_eth"]), secondary_reward_cow=int(frame["secondary_reward_cow"]), quote_reward_cow=int(frame["quote_reward_cow"]), - protocol_fee_eth=int(frame["protocol_fee_eth"]), ) def total_outgoing_eth(self) -> int: """Total outgoing amount (including slippage) for the payout.""" - return ( - self.payment_eth - + self.secondary_reward_eth - + self.slippage_eth - - self.protocol_fee_eth - ) + return self.payment_eth + self.secondary_reward_eth + self.slippage_eth def total_cow_reward(self) -> int: """Total outgoing COW token reward""" @@ -172,9 +161,7 @@ def as_payouts(self) -> list[Transfer]: total_eth_reward = int(self.total_eth_reward()) total_cow_reward = int(self.total_cow_reward()) - reimbursement_eth = int( - self.exec_cost + self.slippage_eth - self.protocol_fee_eth - ) + reimbursement_eth = int(self.exec_cost + self.slippage_eth) # We do not have access to token conversion here, but we do have other converted values # x_eth:x_cow = y_eth:y_cow --> y_cow = y_eth * x_cow / x_eth reimbursement_cow = ( diff --git a/tests/unit/test_payouts.py b/tests/unit/test_payouts.py index e3298cbc..fc4ff8e5 100644 --- a/tests/unit/test_payouts.py +++ b/tests/unit/test_payouts.py @@ -67,12 +67,6 @@ def setUp(self) -> None: 7, 6, ] - self.protocol_fees = [ - 300000000000000.0, - 200000000000000.0, - 100000000000000.0, - 0.0, - ] # Mocking TokenConversion! self.mock_converter = TokenConversion( eth_to_token=lambda t: int(t * 1000), token_to_eth=lambda t: t // 1000 @@ -85,7 +79,6 @@ def test_extend_payment_df(self): "payment_eth": self.eth_payments, "execution_cost_eth": self.execution_costs, "num_participating_batches": self.batch_participation, - "protocol_fee_eth": self.protocol_fees, } base_payout_df = DataFrame(base_data_dict) result = extend_payment_df(base_payout_df, converter=self.mock_converter) @@ -95,7 +88,6 @@ def test_extend_payment_df(self): "payment_eth": self.eth_payments, "execution_cost_eth": self.execution_costs, "num_participating_batches": self.batch_participation, - "protocol_fee_eth": self.protocol_fees, "reward_eth": [ -200000000000000.00000, 10000000000000000.00000, @@ -150,7 +142,6 @@ def test_validate_df_columns(self): "payment_eth": [], "execution_cost_eth": [], "num_participating_batches": [], - "protocol_fee_eth": [], "reward_eth": [], "reward_cow": [], "secondary_reward_cow": [], @@ -200,7 +191,6 @@ def test_construct_payouts(self): "payment_eth": self.eth_payments, "execution_cost_eth": self.execution_costs, "num_participating_batches": self.batch_participation, - "protocol_fee_eth": self.protocol_fees, } ), converter=self.mock_converter, @@ -229,7 +219,6 @@ def test_construct_payouts(self): "payment_eth": [600000000000000.0, 1.045e16, -1e16, 0.0], "execution_cost_eth": [800000000000000.0, 450000000000000.0, 0.0, 0.0], "num_participating_batches": [7, 2, 7, 6], - "protocol_fee_eth": self.protocol_fees, "reward_eth": [-200000000000000.0, 1e16, -1e16, 0.0], "reward_cow": [ -200000000000000000, @@ -301,12 +290,6 @@ def test_prepare_transfers(self): 90000000000000000000.00000, 180000000000000000000.00000, ], - "protocol_fee_eth": [ - 300000000000000.0, - 200000000000000.0, - 100000000000000.0, - 0.0, - ], "solver_name": ["S_1", "S_2", "S_3", None], "eth_slippage_wei": [1.0, 0.0, -1.0, None], "reward_target": [ @@ -325,12 +308,12 @@ def test_prepare_transfers(self): Transfer( token=None, recipient=Address(self.solvers[0]), - amount_wei=363636363636364, + amount_wei=663636363636364, ), Transfer( token=None, recipient=Address(self.solvers[1]), - amount_wei=250000000000000, + amount_wei=450000000000000, ), Transfer( token=Token(COW_TOKEN_ADDRESS), @@ -361,7 +344,7 @@ def test_prepare_transfers(self): Overdraft( period, account=Address(self.solvers[2]), - wei=10036363636363638, + wei=9936363636363638, name="S_3", ) ], @@ -383,7 +366,6 @@ def sample_record( participation: int, slippage: int, num_quotes: int, - protocol_fee: int, ): """Assumes a conversion rate of ETH:COW <> 1:self.conversion_rate""" return RewardAndPenaltyDatum( @@ -397,39 +379,32 @@ def sample_record( secondary_reward_cow=participation * self.conversion_rate, slippage_eth=slippage, quote_reward_cow=QUOTE_REWARD * num_quotes, - protocol_fee_eth=protocol_fee, ) def test_invalid_input(self): with self.assertRaises(AssertionError): - self.sample_record(0, -1, 0, 0, 0, 0) - - with self.assertRaises(AssertionError): - self.sample_record(0, 0, -1, 0, 0, 0) - - with self.assertRaises(AssertionError): - self.sample_record(0, 0, 0, 0, -1, 0) + self.sample_record(0, -1, 0, 0, 0) with self.assertRaises(AssertionError): - self.sample_record(0, 0, 0, 0, 0, -1) + self.sample_record(0, 0, -1, 0, 0) - def test_reward_datum_0_0_0_0_0(self): - test_datum = self.sample_record(0, 0, 0, 0, 0, 0) + def test_reward_datum_0_0_0_0(self): + test_datum = self.sample_record(0, 0, 0, 0, 0) self.assertFalse(test_datum.is_overdraft()) self.assertEqual(test_datum.as_payouts(), []) - def test_reward_datum_1_1_0_0_0_0(self): + def test_reward_datum_1_1_0_0(self): cost = 1 - test_datum = self.sample_record(1, cost, 0, 0, 0, 0) + test_datum = self.sample_record(1, cost, 0, 0, 0) self.assertFalse(test_datum.is_overdraft()) self.assertEqual( test_datum.as_payouts(), [Transfer(token=None, recipient=self.solver, amount_wei=cost)], ) - def test_reward_datum_3_2_0_minus1_0_0(self): + def test_reward_datum_3_2_0_minus1(self): payment, cost, participation, slippage = 3, 2, 0, -1 - test_datum = self.sample_record(payment, cost, participation, slippage, 0, 0) + test_datum = self.sample_record(payment, cost, participation, slippage, 0) self.assertFalse(test_datum.is_overdraft()) self.assertEqual( test_datum.as_payouts(), @@ -447,32 +422,10 @@ def test_reward_datum_3_2_0_minus1_0_0(self): ], ) - def test_reward_datum_8_4_0_minus1_0_2(self): - payment, cost, participation, slippage, protocol_fee = 8, 4, 0, -1, 2 - test_datum = self.sample_record( - payment, cost, participation, slippage, 0, protocol_fee - ) - self.assertFalse(test_datum.is_overdraft()) - self.assertEqual( - test_datum.as_payouts(), - [ - Transfer( - token=None, - recipient=self.solver, - amount_wei=cost + slippage - protocol_fee, - ), - Transfer( - token=self.cow_token, - recipient=self.reward_target, - amount_wei=(payment - cost) * self.conversion_rate, - ), - ], - ) - def test_reward_datum_cost_exceeds_payment_degenerate(self): # Degenerate Case! payment, cost, participation, slippage = 1, 10, 0, -1 - test_datum = self.sample_record(payment, cost, participation, slippage, 0, 0) + test_datum = self.sample_record(payment, cost, participation, slippage, 0) self.assertFalse(test_datum.is_overdraft()) self.assertEqual( test_datum.as_payouts(), @@ -486,9 +439,7 @@ def test_reward_datum_cost_exceeds_payment_non_degenerate(self): cost = max(sum(x) for x in triplets) + 1 for payment, participation, slippage in triplets: - test_datum = self.sample_record( - payment, cost, participation, slippage, 0, 0 - ) + test_datum = self.sample_record(payment, cost, participation, slippage, 0) self.assertFalse(test_datum.is_overdraft()) self.assertLess(test_datum.total_outgoing_eth(), cost) self.assertEqual( @@ -511,12 +462,12 @@ def test_reward_datum_overdraft(self): for payment, participation, slippage in triplets: for cost in [0, 1, 100]: # Doesn't matter their costs, they are in overdraft state! - rec = self.sample_record(payment, cost, participation, slippage, 0, 0) + rec = self.sample_record(payment, cost, participation, slippage, 0) self.assertTrue(rec.is_overdraft()) - def test_reward_datum_1_1_1_1_0_0(self): + def test_reward_datum_1_1_1_1(self): payment, cost, participation, slippage = 1, 1, 1, 1 - test_datum = self.sample_record(payment, cost, participation, slippage, 0, 0) + test_datum = self.sample_record(payment, cost, participation, slippage, 0) self.assertFalse(test_datum.is_overdraft()) self.assertEqual( @@ -540,7 +491,7 @@ def test_reward_datum_1_1_1_1_0_0(self): def test_payout_negative_payments(self): payment, cost, participation, slippage = -1, 1, 1, 1 - test_datum = self.sample_record(payment, cost, participation, slippage, 0, 0) + test_datum = self.sample_record(payment, cost, participation, slippage, 0) self.assertEqual( test_datum.as_payouts(), [ From 8afce0f8ee0294c06888931084c006e7aecbe758 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 22 Jan 2024 12:15:33 +0100 Subject: [PATCH 26/33] rename tables and compute network fee this is consistent with the queries in dune-sync --- queries/orderbook/batch_rewards.sql | 36 ++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/queries/orderbook/batch_rewards.sql b/queries/orderbook/batch_rewards.sql index 64905f7f..a70420f7 100644 --- a/queries/orderbook/batch_rewards.sql +++ b/queries/orderbook/batch_rewards.sql @@ -67,7 +67,7 @@ order_surplus AS ( WHERE ss.block_deadline >= {{start_block}} AND ss.block_deadline <= {{end_block}} ) -,order_observations AS ( +,order_protocol_fee AS ( SELECT os.auction_id, os.solver, @@ -122,28 +122,28 @@ order_surplus AS ( JOIN fee_policies fp -- contains protocol fee policy ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid ) -,order_observations_prices AS ( +,order_protocol_fee_prices AS ( SELECT - oo.solver, - oo.tx_hash, - oo.surplus, - oo.protocol_fee, + opf.solver, + opf.tx_hash, + opf.surplus, + opf.protocol_fee, CASE - WHEN oo.sell_token != oo.protocol_fee_token - THEN (oo.sell_amount - oo.observed_fee) / oo.buy_amount * oo.protocol_fee - ELSE oo.protocol_fee + WHEN opf.sell_token != opf.protocol_fee_token + THEN (opf.sell_amount - opf.observed_fee) / opf.buy_amount * opf.protocol_fee + ELSE opf.protocol_fee END AS network_fee_correction, - oo.sell_token as network_fee_token, + opf.sell_token as network_fee_token, ap_surplus.price / pow(10, 18) as surplus_token_price, ap_protocol.price / pow(10, 18) as protocol_fee_token_price, ap_sell.price / pow(10, 18) as network_fee_token_price - FROM order_observations oo + FROM order_protocol_fee opf JOIN auction_prices ap_sell -- contains price: sell token - ON oo.auction_id = ap_sell.auction_id AND oo.sell_token = ap_sell.token + ON opf.auction_id = ap_sell.auction_id AND opf.sell_token = ap_sell.token JOIN auction_prices ap_surplus -- contains price: surplus token - ON oo.auction_id = ap_surplus.auction_id AND oo.surplus_token = ap_surplus.token + ON opf.auction_id = ap_surplus.auction_id AND opf.surplus_token = ap_surplus.token JOIN auction_prices ap_protocol -- contains price: protocol fee token - ON oo.auction_id = ap_protocol.auction_id AND oo.protocol_fee_token = ap_protocol.token + ON opf.auction_id = ap_protocol.auction_id AND opf.protocol_fee_token = ap_protocol.token ), batch_protocol_fees AS ( SELECT @@ -152,7 +152,7 @@ batch_protocol_fees AS ( -- sum(surplus * surplus_token_price) as surplus, sum(protocol_fee * protocol_fee_token_price) as protocol_fee, sum(network_fee_correction * network_fee_token_price) as network_fee_correction - FROM order_observations_prices oop + FROM order_protocol_fee_prices group by solver, tx_hash ), reward_data AS (SELECT @@ -198,10 +198,10 @@ batch_protocol_fees AS ( solver, execution_cost, surplus, - protocol_fee + fee - network_fee_correction as fee,-- total fee for ranking - protocol_fee, -- the protocol fee + protocol_fee, + fee - network_fee_correction as network_fee, surplus + protocol_fee + fee - network_fee_correction - reference_score as uncapped_reward_eth, - -- Uncapped Reward = CLAMP_[-E, E + exec_cost](uncapped_reward_eth) + -- capped Reward = CLAMP_[-E, E + exec_cost](uncapped_reward_eth) LEAST(GREATEST(-{{EPSILON}}, surplus + coalesce(protocol_fee - network_fee_correction, 0) + fee - reference_score), {{EPSILON}} + execution_cost) as capped_payment, winning_score, From e6d0b95b566860ebb3a111cab65e87e63f9b52b9 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 22 Jan 2024 12:16:06 +0100 Subject: [PATCH 27/33] remove unused protocol fee query --- queries/orderbook/protocol_fees.sql | 119 ---------------------------- src/pg_client.py | 14 ---- 2 files changed, 133 deletions(-) delete mode 100644 queries/orderbook/protocol_fees.sql diff --git a/queries/orderbook/protocol_fees.sql b/queries/orderbook/protocol_fees.sql deleted file mode 100644 index 8d35f9de..00000000 --- a/queries/orderbook/protocol_fees.sql +++ /dev/null @@ -1,119 +0,0 @@ -WITH order_surplus AS ( - SELECT - ss.winner as solver, - at.auction_id, - t.order_uid, - o.sell_token, -- the total amount the user sends - o.buy_token, - t.sell_amount, - t.buy_amount, - oe.surplus_fee as observed_fee, - o.kind, - CASE - WHEN o.kind = 'sell' - THEN t.buy_amount - o.buy_amount / (o.sell_amount + o.fee_amount) * t.sell_amount - WHEN o.kind = 'buy' - THEN (o.sell_amount + o.fee_amount) / o.buy_amount * t.buy_amount - t.sell_amount - END AS surplus, - CASE - WHEN o.kind = 'sell' - THEN o.buy_token - WHEN o.kind = 'buy' - THEN o.sell_token - END AS surplus_token - FROM settlements s -- links block_number and log_index to tx_from and tx_nonce - JOIN auction_transaction at -- links auction_id to tx_from and tx_nonce - ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce - JOIN settlement_scores ss -- contains block_deadline - ON at.auction_id = ss.auction_id - JOIN trades t -- contains traded amounts - ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block - JOIN orders o -- contains tokens and limit amounts - ON t.order_uid = o.uid - JOIN order_execution oe -- contains surplus fee - ON t.order_uid = oe.order_uid AND at.auction_id = oe.auction_id - WHERE ss.block_deadline >= {{start_block}} - AND ss.block_deadline <= {{end_block}} -) -,order_observation AS ( - SELECT - os.auction_id, - os.solver, - os.sell_amount, -- the total amount the user sends - os.buy_amount, - os.sell_token, - os.observed_fee, - os.surplus, - os.surplus_token, - CASE - WHEN fp.kind = 'surplus' - THEN - CASE - WHEN os.kind = 'sell' - THEN - CASE - WHEN fp.max_volume_factor = 1 -- this is done to avoid a division by zero bug - THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus - ELSE - LEAST( - fp.max_volume_factor / (1 - fp.max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) - END - WHEN os.kind = 'buy' - THEN - CASE - WHEN fp.max_volume_factor = 1 - THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus - ELSE - LEAST( - fp.max_volume_factor / (1 - fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) - END - END - WHEN fp.kind = 'volume' - THEN fp.volume_factor / (1 - fp.volume_factor) * os.sell_amount - END AS protocol_fee, - CASE - WHEN fp.kind = 'surplus' - THEN os.surplus_token - WHEN fp.kind = 'volume' - THEN os.sell_token - END AS protocol_fee_token - FROM order_surplus os - JOIN fee_policies fp -- contains protocol fee policy - ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid -) -,order_observations_prices AS ( - SELECT - oo.solver, - oo.surplus, - oo.protocol_fee, - CASE - WHEN oo.sell_token != oo.protocol_fee_token - THEN - (oo.sell_amount - oo.observed_fee) / oo.buy_amount * oo.protocol_fee - ELSE - oo.protocol_fee - END AS network_fee_correction, - oo.sell_token as network_fee_token, - ap_surplus.price / pow(10, 18) as surplus_token_price, - ap_protocol.price / pow(10, 18) as protocol_fee_token_price, - ap_sell.price / pow(10, 18) as network_fee_token_price - FROM order_observation oo - JOIN auction_prices ap_sell -- contains price: sell token - ON oo.auction_id = ap_sell.auction_id AND oo.sell_token = ap_sell.token - JOIN auction_prices ap_surplus -- contains price: surplus token - ON oo.auction_id = ap_surplus.auction_id AND oo.surplus_token = ap_surplus.token - JOIN auction_prices ap_protocol -- contains price: protocol fee token - ON oo.auction_id = ap_protocol.auction_id AND oo.protocol_fee_token = ap_protocol.token -), -batch_aggregate AS ( - SELECT - concat('0x', encode(solver, 'hex')) as solver, - -- sum(surplus * surplus_token_price) as surplus, - sum(protocol_fee * protocol_fee_token_price) as protocol_fee, - sum(network_fee_correction * network_fee_token_price) as network_fee_correction - FROM order_observations_prices oop - group by solver -) -SELECT * FROM batch_aggregate diff --git a/src/pg_client.py b/src/pg_client.py index bddc0815..6854716d 100644 --- a/src/pg_client.py +++ b/src/pg_client.py @@ -57,20 +57,6 @@ def get_quote_rewards(self, start_block: str, end_block: str) -> DataFrame: return pd.concat(results) - def get_protocol_fees(self, start_block: str, end_block: str) -> DataFrame: - """Returns aggregated solver protocol fees for block range""" - protocol_fee_query = ( - open_query("orderbook/protocol_fees.sql") - .replace("{{start_block}}", start_block) - .replace("{{end_block}}", end_block) - ) - results = [ - self.exec_query(query=protocol_fee_query, engine=engine) - for engine in self.connections - ] - - return pd.concat(results) - def pg_hex2bytea(hex_address: str) -> str: """ From 307401e2fb9c25eab91f451edeb746271612e30f Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 25 Jan 2024 15:50:53 +0100 Subject: [PATCH 28/33] add protocol fee transfer to special safe -adds one additional transfer in eth to a safe set up for the collection of protocol fees - protocol fees are computed by multiplying amounts in the protocol fee token with native prices from the auction --- src/fetch/payouts.py | 12 ++++++++++++ tests/unit/test_payouts.py | 23 +++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/fetch/payouts.py b/src/fetch/payouts.py index 45998382..ddd7959b 100644 --- a/src/fetch/payouts.py +++ b/src/fetch/payouts.py @@ -25,6 +25,8 @@ PERIOD_BUDGET_COW = 306646 * 10**18 QUOTE_REWARD = 9 * 10**18 +PROTOCOL_FEE_SAFE = Address("0xB64963f95215FDe6510657e719bd832BB8bb941B") + PAYMENT_COLUMNS = { "solver", "payment_eth", @@ -33,6 +35,7 @@ "reward_cow", "secondary_reward_cow", "quote_reward_cow", + "protocol_fee_eth", } SLIPPAGE_COLUMNS = { "solver", @@ -50,6 +53,7 @@ "secondary_reward_cow", "secondary_reward_eth", "quote_reward_cow", + "protocol_fee_eth", ] @@ -308,6 +312,14 @@ def prepare_transfers(payout_df: DataFrame, period: AccountingPeriod) -> PeriodP overdrafts.append(overdraft) transfers += payout_datum.as_payouts() + transfers.append( + Transfer( + token=None, + recipient=PROTOCOL_FEE_SAFE, + amount_wei=payout_df.protocol_fee_eth.sum(), + ) + ) + return PeriodPayouts(overdrafts, transfers) diff --git a/tests/unit/test_payouts.py b/tests/unit/test_payouts.py index fc4ff8e5..682187f8 100644 --- a/tests/unit/test_payouts.py +++ b/tests/unit/test_payouts.py @@ -14,6 +14,7 @@ prepare_transfers, RewardAndPenaltyDatum, QUOTE_REWARD, + PROTOCOL_FEE_SAFE, ) from src.models.accounting_period import AccountingPeriod from src.models.overdraft import Overdraft @@ -67,6 +68,12 @@ def setUp(self) -> None: 7, 6, ] + self.protocol_fee_eth = [ + 1000000000000000.0, + 2000000000000000.0, + 0.0, + 0.0, + ] # Mocking TokenConversion! self.mock_converter = TokenConversion( eth_to_token=lambda t: int(t * 1000), token_to_eth=lambda t: t // 1000 @@ -79,6 +86,7 @@ def test_extend_payment_df(self): "payment_eth": self.eth_payments, "execution_cost_eth": self.execution_costs, "num_participating_batches": self.batch_participation, + "protocol_fee_eth": self.protocol_fee_eth, } base_payout_df = DataFrame(base_data_dict) result = extend_payment_df(base_payout_df, converter=self.mock_converter) @@ -88,6 +96,7 @@ def test_extend_payment_df(self): "payment_eth": self.eth_payments, "execution_cost_eth": self.execution_costs, "num_participating_batches": self.batch_participation, + "protocol_fee_eth": self.protocol_fee_eth, "reward_eth": [ -200000000000000.00000, 10000000000000000.00000, @@ -142,6 +151,7 @@ def test_validate_df_columns(self): "payment_eth": [], "execution_cost_eth": [], "num_participating_batches": [], + "protocol_fee_eth": [], "reward_eth": [], "reward_cow": [], "secondary_reward_cow": [], @@ -191,6 +201,7 @@ def test_construct_payouts(self): "payment_eth": self.eth_payments, "execution_cost_eth": self.execution_costs, "num_participating_batches": self.batch_participation, + "protocol_fee_eth": self.protocol_fee_eth, } ), converter=self.mock_converter, @@ -219,6 +230,12 @@ def test_construct_payouts(self): "payment_eth": [600000000000000.0, 1.045e16, -1e16, 0.0], "execution_cost_eth": [800000000000000.0, 450000000000000.0, 0.0, 0.0], "num_participating_batches": [7, 2, 7, 6], + "protocol_fee_eth": [ + 1000000000000000.0, + 2000000000000000.0, + 0.0, + 0.0, + ], "reward_eth": [-200000000000000.0, 1e16, -1e16, 0.0], "reward_cow": [ -200000000000000000, @@ -265,6 +282,7 @@ def test_prepare_transfers(self): "num_quotes": self.num_quotes, "payment_eth": [600000000000000.0, 1.045e16, -1e16, 0.0], "execution_cost_eth": [800000000000000.0, 450000000000000.0, 0.0, 0.0], + "protocol_fee_eth": self.protocol_fee_eth, "reward_eth": [-200000000000000.0, 1e16, -1e16, 0.0], "reward_cow": [ -200000000000000000, @@ -335,6 +353,11 @@ def test_prepare_transfers(self): recipient=Address(self.reward_targets[3]), amount_wei=54545454545454544, ), + Transfer( + token=None, + recipient=PROTOCOL_FEE_SAFE, + amount_wei=3000000000000000, + ), ], ) From 9f269019e686a7f85e90d24a69b4d21d756e0a24 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 26 Jan 2024 12:47:55 +0100 Subject: [PATCH 29/33] correct volume cap for protocol fee - use sell amount as baseline for volume cap for sell orders - correctly compute cap for buy orders - remove unnecessary case for 100% volume cap --- queries/orderbook/batch_rewards.sql | 34 ++++++++++------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/queries/orderbook/batch_rewards.sql b/queries/orderbook/batch_rewards.sql index a70420f7..fd175bdc 100644 --- a/queries/orderbook/batch_rewards.sql +++ b/queries/orderbook/batch_rewards.sql @@ -84,30 +84,20 @@ order_surplus AS ( CASE WHEN os.kind = 'sell' THEN - CASE - WHEN fp.max_volume_factor = 1 -- this is done to avoid a division by zero errors - -- We assume that the case surplus_factor != 1 always. In - -- that case reconstructing the protocol fee would be - -- impossible anyways. This query will return a division by - -- zero error in that case. - THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus - ELSE - LEAST( - fp.max_volume_factor / (1 - fp.max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) - END + -- We assume that the case surplus_factor != 1 always. In + -- that case reconstructing the protocol fee would be + -- impossible anyways. This query will return a division by + -- zero error in that case. + LEAST( + fp.max_volume_factor * os.sell_amount * os.buy_amount / (os.sell_amount - os.observed_fee), -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) WHEN os.kind = 'buy' THEN - CASE - WHEN fp.max_volume_factor = 1 - THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus - ELSE - LEAST( - fp.max_volume_factor / (1 - fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) - END + LEAST( + fp.max_volume_factor / (1 + fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) END WHEN fp.kind = 'volume' THEN fp.volume_factor / (1 - fp.volume_factor) * os.sell_amount From 0a8d0eed88a0d6ae87547c1508d81f09d2210a12 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 29 Jan 2024 11:35:58 +0200 Subject: [PATCH 30/33] fix test also added comment on how test values are computed --- tests/queries/test_batch_rewards.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/test_batch_rewards.py b/tests/queries/test_batch_rewards.py index fc790af4..540d565e 100644 --- a/tests/queries/test_batch_rewards.py +++ b/tests/queries/test_batch_rewards.py @@ -53,8 +53,8 @@ def test_get_batch_rewards(self): 6, ], "protocol_fee_eth": [ - 5.751126690035052e14, - 2.0303030303030302e15, + 5.751126690035052e14, # 0.5 / (1 - 0.5) * 1e18 * 5e14 / 1e18 + 0.0015 / (1 - 0.0015) * 1e8 * 5e26 / 1e18 + 2.0198019801980198e15, # 0.75 / (1 - 0.75) * 1e6 * 5e26 / 1e18 + 0.01 / (1 + 0.01) * 105e6 * 5e26 / 1e18 0.0, 0.0, 0.0, From a0bbdcd546daa9ffbeb946635946fac97796fdf5 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 29 Jan 2024 11:38:58 +0200 Subject: [PATCH 31/33] fix error in volume based fee also adapted the test --- queries/orderbook/batch_rewards.sql | 2 +- tests/queries/test_batch_rewards.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/queries/orderbook/batch_rewards.sql b/queries/orderbook/batch_rewards.sql index fd175bdc..664c0f01 100644 --- a/queries/orderbook/batch_rewards.sql +++ b/queries/orderbook/batch_rewards.sql @@ -100,7 +100,7 @@ order_surplus AS ( ) END WHEN fp.kind = 'volume' - THEN fp.volume_factor / (1 - fp.volume_factor) * os.sell_amount + THEN fp.volume_factor / (1 + fp.volume_factor) * os.sell_amount END AS protocol_fee, CASE WHEN fp.kind = 'surplus' diff --git a/tests/queries/test_batch_rewards.py b/tests/queries/test_batch_rewards.py index 540d565e..f0605711 100644 --- a/tests/queries/test_batch_rewards.py +++ b/tests/queries/test_batch_rewards.py @@ -53,8 +53,8 @@ def test_get_batch_rewards(self): 6, ], "protocol_fee_eth": [ - 5.751126690035052e14, # 0.5 / (1 - 0.5) * 1e18 * 5e14 / 1e18 + 0.0015 / (1 - 0.0015) * 1e8 * 5e26 / 1e18 - 2.0198019801980198e15, # 0.75 / (1 - 0.75) * 1e6 * 5e26 / 1e18 + 0.01 / (1 + 0.01) * 105e6 * 5e26 / 1e18 + 5.748876684972541e14, # 0.5 / (1 - 0.5) * 1e18 * 5e14 / 1e18 + 0.0015 / (1 + 0.0015) * 1e8 * 5e26 / 1e18 + 2.0198019801980198e15, # 0.75 / (1 - 0.75) * 1e6 * 5e26 / 1e18 + 0.01 / (1 + 0.01) * 105e6 * 5e26 / 1e18 0.0, 0.0, 0.0, From b726a5265e2717aaa1339bb5e906cbdb3d2182b5 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 29 Jan 2024 15:25:06 +0200 Subject: [PATCH 32/33] add protocol fee info to logs --- src/fetch/payouts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fetch/payouts.py b/src/fetch/payouts.py index ddd7959b..f4237fdd 100644 --- a/src/fetch/payouts.py +++ b/src/fetch/payouts.py @@ -408,10 +408,12 @@ def construct_payouts( performance_reward = complete_payout_df["reward_cow"].sum() participation_reward = complete_payout_df["secondary_reward_cow"].sum() quote_reward = complete_payout_df["quote_reward_cow"].sum() + protocol_fee = complete_payout_df["protocol_fee_eth"].sum() dune.log_saver.print( f"Performance Reward: {performance_reward / 10 ** 18:.4f}\n" f"Participation Reward: {participation_reward / 10 ** 18:.4f}\n" - f"Quote Reward: {quote_reward / 10 ** 18:.4f}\n", + f"Quote Reward: {quote_reward / 10 ** 18:.4f}\n" + f"Protocol Fees: {protocol_fee / 10 ** 18:.4f}\n", category=Category.TOTALS, ) payouts = prepare_transfers(complete_payout_df, dune.period) From 85e07a465a29e49f995620dd4da6ce2ff3260760 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 5 Feb 2024 10:48:51 +0100 Subject: [PATCH 33/33] cast protocol fee to int in transfer otherwise the creation of transfers fails. (for some reason this is not covered in a test) --- src/fetch/payouts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fetch/payouts.py b/src/fetch/payouts.py index f4237fdd..4724f0ee 100644 --- a/src/fetch/payouts.py +++ b/src/fetch/payouts.py @@ -316,7 +316,7 @@ def prepare_transfers(payout_df: DataFrame, period: AccountingPeriod) -> PeriodP Transfer( token=None, recipient=PROTOCOL_FEE_SAFE, - amount_wei=payout_df.protocol_fee_eth.sum(), + amount_wei=int(payout_df.protocol_fee_eth.sum()), ) )