update query and database

- create new test database for protocol fees - add new test for protocol fees
cowprotocol · Jan 15, 2024 · 41bf651 · 41bf651
1 parent 0db2785
commit 41bf651
Show file tree

Hide file tree

Showing 4 changed files with 285 additions and 19 deletions.
diff --git a/queries/orderbook/protocol_fees.sql b/queries/orderbook/protocol_fees.sql
@@ -1,13 +1,13 @@
 WITH order_surplus AS (
     SELECT
+        ss.winner as solver,
         at.auction_id,
-        s.tx_hash,
         t.order_uid,
-        o.sell_token,
+        o.sell_token, -- the total amount the user sends
         o.buy_token,
         t.sell_amount,
         t.buy_amount,
-        oe.surplus_fee,
+        oe.surplus_fee as observed_fee,
         o.kind,
         CASE
             WHEN o.kind = 'sell'
@@ -35,9 +35,16 @@ WITH order_surplus AS (
     WHERE ss.block_deadline > 0
         AND ss.block_deadline <= 100
 )
-,protocol_fees_raw AS (
+,order_observation AS (
     SELECT
-        os.*, -- TODO: select data
+        os.auction_id,
+        os.solver,
+        os.sell_amount, -- the total amount the user sends
+        os.buy_amount,
+        os.sell_token,
+        os.observed_fee,
+        os.surplus,
+        os.surplus_token,
         CASE
             WHEN fp.kind = 'surplus'
                 THEN
@@ -68,20 +75,35 @@ WITH order_surplus AS (
     JOIN fee_policies fp -- contains protocol fee policy
         ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid
 )
-,order_fees AS (
+,order_observations_prices AS (
     SELECT
-        pfr.*, -- TODO: select data
-        ap_sell.price as sell_token_price,
-        ap_protocol.price as protocol_fee_token_price,
+        oo.solver,
+        oo.surplus,
+        oo.protocol_fee,
         CASE
-            WHEN pfr.sell_token != pfr.protocol_fee_token
-                THEN ap_sell.price / pow(10, 18) * (pfr.surplus_fee - (pfr.sell_amount - pfr.surplus_fee) / pfr.buy_amount * pfr.protocol_fee)
-            ELSE ap_sell.price / pow(10, 18) * (pfr.surplus_fee - pfr.protocol_fee)
-        END AS network_fee
-    FROM protocol_fees_raw pfr
-    JOIN auction_prices ap_sell -- contains prices
-        ON pfr.auction_id = ap_sell.auction_id AND pfr.sell_token = ap_sell.token
-    JOIN auction_prices ap_protocol -- contains prices
-        ON pfr.auction_id = ap_protocol.auction_id AND pfr.protocol_fee_token = ap_protocol.token
+            WHEN oo.sell_token != oo.protocol_fee_token
+                THEN oo.observed_fee - (oo.sell_amount - oo.observed_fee) / oo.buy_amount * oo.protocol_fee
+            ELSE oo.observed_fee - oo.protocol_fee
+        END AS network_fee,
+        oo.sell_token as network_fee_token,
+        ap_surplus.price / pow(10, 18) as surplus_token_price,
+        ap_protocol.price / pow(10, 18) as protocol_fee_token_price,
+        ap_sell.price / pow(10, 18) as network_fee_token_price
+    FROM order_observation oo
+    JOIN auction_prices ap_sell -- contains price: sell token
+        ON oo.auction_id = ap_sell.auction_id AND oo.sell_token = ap_sell.token
+    JOIN auction_prices ap_surplus -- contains price: surplus token
+        ON oo.auction_id = ap_surplus.auction_id AND oo.surplus_token = ap_surplus.token
+    JOIN auction_prices ap_protocol -- contains price: protocol fee token
+        ON oo.auction_id = ap_protocol.auction_id AND oo.protocol_fee_token = ap_protocol.token
+),
+batch_aggregate AS (
+    SELECT
+        concat('0x', encode(solver, 'hex')) as solver,
+        -- sum(surplus * surplus_token_price) as surplus,
+        sum(protocol_fee * protocol_fee_token_price) as protocol_fee,
+        sum(network_fee * network_fee_token_price) as network_fee
+    FROM order_observations_prices oop
+    group by solver
 )
-select * from order_fees
+SELECT * FROM batch_aggregate
diff --git a/src/pg_client.py b/src/pg_client.py
@@ -57,6 +57,20 @@ def get_quote_rewards(self, start_block: str, end_block: str) -> DataFrame:
 
         return pd.concat(results)
 
+    def get_protocol_fees(self, start_block: str, end_block: str) -> DataFrame:
+        """Returns aggregated solver protocol fees for block range"""
+        protocol_fee_query = (
+            open_query("orderbook/protocol_fees.sql")
+            .replace("{{start_block}}", start_block)
+            .replace("{{end_block}}", end_block)
+        )
+        results = [
+            self.exec_query(query=protocol_fee_query, engine=engine)
+            for engine in self.connections
+        ]
+
+        return pd.concat(results)
+
 
 def pg_hex2bytea(hex_address: str) -> str:
     """

diff --git a/tests/queries/protocol_fee_test_db.sql b/tests/queries/protocol_fee_test_db.sql
@@ -0,0 +1,191 @@
+DROP TABLE IF EXISTS settlements;
+DROP TABLE IF EXISTS auction_transaction;
+DROP TABLE IF EXISTS settlement_scores;
+DROP TABLE IF EXISTS auction_prices;
+DROP TABLE IF EXISTS orders;
+DROP TYPE IF EXISTS OrderKind;
+DROP TYPE IF EXISTS OrderClass;
+DROP TABLE IF EXISTS trades;
+DROP TABLE IF EXISTS order_execution;
+DROP TABLE IF EXISTS fee_policies;
+DROP TYPE IF EXISTS PolicyKind;
+
+CREATE TABLE IF NOT EXISTS settlements
+(
+  block_number bigint NOT NULL,
+  log_index    bigint NOT NULL,
+  solver       bytea  NOT NULL,
+  tx_hash      bytea  NOT NULL,
+  tx_from      bytea  NOT NULL,
+  tx_nonce     bigint NOT NULL,
+
+  PRIMARY KEY (block_number, log_index)
+);
+
+CREATE INDEX settlements_tx_from_tx_nonce ON settlements (tx_from, tx_nonce);
+
+CREATE TABLE IF NOT EXISTS auction_transaction
+(
+  auction_id bigint PRIMARY KEY,
+  tx_from    bytea  NOT NULL,
+  tx_nonce   bigint NOT NULL,
+  UNIQUE (tx_from, tx_nonce)
+);
+
+CREATE TABLE IF NOT EXISTS auction_participants
+(
+  auction_id  bigint,
+  participant bytea
+);
+
+CREATE TABLE IF NOT EXISTS settlement_scores
+(
+  auction_id       bigint PRIMARY KEY,
+  winner           bytea          NOT NULL,
+  winning_score    numeric(78, 0) NOT NULL,
+  reference_score  numeric(78, 0) NOT NULL,
+  block_deadline   bigint         NOT NULL,
+  simulation_block bigint         NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS auction_prices
+(
+  auction_id bigint NOT NULL,
+  token bytea NOT NULL,
+  price  numeric(78, 0) NOT NULL,
+
+  PRIMARY KEY (auction_id, token)
+);
+
+-- orders table
+CREATE TYPE OrderKind AS ENUM ('buy', 'sell');
+CREATE TYPE OrderClass AS ENUM ('market', 'limit');
+
+CREATE TABLE orders (
+    uid bytea PRIMARY KEY,
+    sell_token bytea NOT NULL,
+    buy_token bytea NOT NULL,
+    sell_amount numeric(78,0) NOT NULL,
+    buy_amount numeric(78,0) NOT NULL,
+    fee_amount numeric(78,0) NOT NULL,
+    kind OrderKind NOT NULL,
+    partially_fillable boolean NOT NULL,
+    full_fee_amount numeric(78,0) NOT NULL,
+    class OrderClass NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS trades
+(
+  block_number bigint         NOT NULL,
+  log_index    bigint         NOT NULL,
+  order_uid    bytea          NOT NULL,
+  sell_amount  numeric(78, 0) NOT NULL,
+  buy_amount   numeric(78, 0) NOT NULL,
+  fee_amount   numeric(78, 0) NOT NULL,
+
+  PRIMARY KEY (block_number, log_index)
+);
+
+CREATE TABLE IF NOT EXISTS order_execution
+(
+  order_uid bytea NOT NULL,
+  auction_id bigint NOT NULL,
+  reward double precision NOT NULL,
+  surplus_fee numeric(78, 0) NOT NULL,
+  solver_fee numeric(78, 0),
+
+  PRIMARY KEY (order_uid, auction_id)
+);
+
+CREATE TYPE PolicyKind AS ENUM ('surplus', 'volume');
+
+CREATE TABLE fee_policies (
+  auction_id bigint NOT NULL,
+  order_uid bytea NOT NULL,
+  -- The order in which the fee policies are inserted and applied.
+  application_order SERIAL NOT NULL,
+  -- The type of the fee policy.
+  kind PolicyKind NOT NULL,
+  -- The fee should be taken as a percentage of the price improvement. The value is between 0 and 1.
+  surplus_factor double precision,
+  -- Cap the fee at a certain percentage of the order volume. The value is between 0 and 1.
+  max_volume_factor double precision,
+  -- The fee should be taken as a percentage of the order volume. The value is between 0 and 1.
+  volume_factor double precision,
+  PRIMARY KEY (auction_id, order_uid, application_order)
+);
+
+TRUNCATE settlements;
+TRUNCATE auction_transaction;
+TRUNCATE settlement_scores;
+TRUNCATE auction_prices;
+TRUNCATE orders;
+TRUNCATE trades;
+TRUNCATE fee_policies;
+
+
+INSERT INTO settlements (block_number, log_index, solver, tx_hash, tx_from, tx_nonce)
+VALUES (1, 10, '\x01'::bytea, '\x01'::bytea, '\x01'::bytea, 1),
+       (2, 10, '\x02'::bytea, '\x02'::bytea, '\x02'::bytea, 1),
+       (3, 10, '\x01'::bytea, '\x03'::bytea, '\x01'::bytea, 2),
+       (4, 10, '\x02'::bytea, '\x04'::bytea, '\x02'::bytea, 2),
+       (5, 10, '\x01'::bytea, '\x05'::bytea, '\x01'::bytea, 3),
+       (6, 10, '\x02'::bytea, '\x06'::bytea, '\x02'::bytea, 3);
+
+INSERT INTO auction_transaction (auction_id, tx_from, tx_nonce)
+VALUES (1, '\x01'::bytea, 1),
+       (2, '\x02'::bytea, 1),
+       (3, '\x01'::bytea, 2),
+       (4, '\x02'::bytea, 2),
+       (5, '\x01'::bytea, 3),
+       (6, '\x02'::bytea, 3);
+
+INSERT INTO settlement_scores (auction_id, winning_score, reference_score, winner, block_deadline, simulation_block)
+VALUES (1, 5000000000000000000, 4000000000000000000, '\x01'::bytea, 10, 0),
+       (2, 6000000000000000000, 3000000000000000000, '\x02'::bytea, 11, 1),
+       (3, 21000000000000000000, 3000000000000000000, '\x01'::bytea, 12, 2),
+       (4, 21000000000000000000, 3000000000000000000, '\x02'::bytea, 12, 2),
+       (5, 5000000000000000000, 3000000000000000000, '\x01'::bytea, 14, 4),
+       (6, 10000000000000000000, 9000000000000000000, '\x02'::bytea, 15, 5);
+
+INSERT INTO auction_prices (auction_id, token, price)
+VALUES (1, '\x01', 500000000000000000000000000),
+(1, '\x02', 500000000000000),
+(2, '\x01', 500000000000000000000000000),
+(2, '\x02', 500000000000000),
+(3, '\x01', 500000000000000000000000000),
+(3, '\x02', 500000000000000),
+(4, '\x01', 500000000000000000000000000),
+(4, '\x02', 500000000000000),
+(5, '\x01', 500000000000000000000000000),
+(5, '\x02', 500000000000000),
+(6, '\x01', 500000000000000000000000000),
+(6, '\x02', 500000000000000);
+
+INSERT INTO orders (uid, sell_token, buy_token, sell_amount, buy_amount, fee_amount, kind, partially_fillable, full_fee_amount, class)
+VALUES ('\x01'::bytea, '\x01'::bytea, '\x02'::bytea, 95000000, 94000000000000000000, 5000000, 'sell', 'f', 5000000, 'market'), -- sell market order
+('\x02'::bytea, '\x01'::bytea, '\x02'::bytea, 101000000, 100000000000000000000, 5000000, 'buy', 'f', 5000000, 'market'), -- buy market order
+('\x03'::bytea, '\x01'::bytea, '\x02'::bytea, 100000000, 100000000000000000000, 0, 'sell', 't', 0, 'limit'), -- partially fillable sell limit order
+('\x04'::bytea, '\x01'::bytea, '\x02'::bytea, 100000000, 100000000000000000000, 0, 'buy', 't', 0, 'limit'), -- partially fillable buy limit order
+('\x05'::bytea, '\x01'::bytea, '\x02'::bytea, 100000000, 94000000000000000000, 0, 'sell', 'f', 0, 'limit'), -- in market sell limit order
+('\x06'::bytea, '\x01'::bytea, '\x02'::bytea, 106000000, 100000000000000000000, 0, 'buy', 'f', 0, 'limit'); -- in market buy limit order
+
+INSERT INTO trades (block_number, log_index, order_uid, sell_amount, buy_amount, fee_amount)
+VALUES (1, 0, '\x01'::bytea, 100000000, 95000000000000000000, 5000000),
+(2, 0, '\x02'::bytea, 106000000, 100000000000000000000, 5000000),
+(3, 0, '\x03'::bytea, 100000000, 101000000000000000000, 0),
+(4, 0, '\x04'::bytea, 99000000, 100000000000000000000, 0),
+(5, 0, '\x05'::bytea, 100000000, 95000000000000000000, 0),
+(6, 0, '\x06'::bytea, 105000000, 100000000000000000000, 0);
+
+INSERT INTO order_execution (order_uid, auction_id, reward, surplus_fee, solver_fee)
+VALUES ('\x03'::bytea, 3, 0, 5931372, NULL),
+('\x04'::bytea, 4, 0, 6000000, NULL),
+('\x05'::bytea, 5, 0, 6000000, NULL),
+('\x06'::bytea, 6, 0, 6000000, NULL);
+
+INSERT INTO fee_policies (auction_id, order_uid, application_order, kind, surplus_factor, max_volume_factor, volume_factor)
+VALUES (3, '\x03'::bytea, 3, 'surplus', 0.5, 0.02, NULL),
+(4, '\x04'::bytea, 4, 'surplus', 0.75, 0.1, NULL),
+(5, '\x05'::bytea, 5, 'volume', NULL, NULL, 0.0015),
+(6, '\x06'::bytea, 6, 'surplus', 0.9, 0.01, NULL);
diff --git a/tests/queries/test_protocol_fees.py b/tests/queries/test_protocol_fees.py
@@ -0,0 +1,39 @@
+import unittest
+
+import pandas.testing
+from pandas import DataFrame
+
+from src.pg_client import MultiInstanceDBFetcher
+
+
+class TestProtocolFees(unittest.TestCase):
+    def setUp(self) -> None:
+        db_url = "postgres:postgres@localhost:5432/postgres"
+        self.fetcher = MultiInstanceDBFetcher([db_url])
+        with open("./tests/queries/protocol_fee_test_db.sql", "r", encoding="utf-8") as file:
+            self.fetcher.connections[0].execute(file.read())
+
+    def test_get_protocol_fees(self):
+        start_block, end_block = "0", "100"
+        protocol_fees = self.fetcher.get_protocol_fees(start_block, end_block)
+        expected = DataFrame(
+            {
+                "solver": [
+                    "0x01",
+                    "0x02",
+                ],
+                "protocol_fee": [
+                    5.751126690035052e14,
+                    2.0303030303030302e15,
+                ],
+                "network_fee": [
+                    5424887053768772.0,
+                    3969696969696969.5,
+                ],
+            }
+        )
+        self.assertIsNone(pandas.testing.assert_frame_equal(expected, protocol_fees))
+
+
+if __name__ == "__main__":
+    unittest.main()