Change surplus based tests to use new data from orderbook

src/apis/orderbookapi.py

@@ -7,6 +7,7 @@
 import json
 import requests
 from src.helper_functions import get_logger
+from src.models import Trade, OrderData, OrderExecution
 from src.constants import (
     header,
     REQUEST_TIMEOUT,
@@ -57,3 +58,66 @@ def get_solver_competition_data(self, tx_hash: str) -> Optional[dict[str, Any]]:
             )
             return None
         return solver_competition_data
+
+    def get_order_data(self, uid: str) -> dict[str, Any] | None:
+        """Get order data from uid.
+        The returned dict follows the schema outlined here:
+        https://api.cow.fi/docs/#/default/get_api_v1_orders__UID_
+        """
+        prod_endpoint_url = f"{PROD_BASE_URL}orders/{uid}"
+        barn_endpoint_url = f"{BARN_BASE_URL}orders/{uid}"
+        order_data: Optional[dict[str, Any]] = None
+        try:
+            json_order_data = requests.get(
+                prod_endpoint_url,
+                headers=header,
+                timeout=REQUEST_TIMEOUT,
+            )
+            if json_order_data.status_code == SUCCESS_CODE:
+                order_data = json_order_data.json()
+            elif json_order_data.status_code == FAIL_CODE:
+                barn_order_data = requests.get(
+                    barn_endpoint_url, headers=header, timeout=REQUEST_TIMEOUT
+                )
+                if barn_order_data.status_code == SUCCESS_CODE:
+                    order_data = barn_order_data.json()
+                else:
+                    return None
+        except requests.RequestException as err:
+            self.logger.warning(
+                f"Connection error while fetching order data. UID: {uid}, error: {err}"
+            )
+            return None
+        return order_data
+
+    def get_trade(
+        self, order_response: dict[str, Any], execution_response: dict[str, Any]
+    ) -> Trade:
+        """Create Trade from order and execution data."""
+        data = OrderData(
+            int(order_response["buyAmount"]),
+            int(order_response["sellAmount"]),
+            int(order_response["feeAmount"]),
+            order_response["buyToken"],
+            order_response["sellToken"],
+            order_response["kind"] == "sell",
+            order_response["partiallyFillable"],
+        )
+        execution = OrderExecution(
+            int(execution_response["buyAmount"]),
+            int(execution_response["sellAmount"]),
+            0,
+        )
+        return Trade(data, execution)
+
+    def get_uid_trades(self, solution: dict[str, Any]) -> dict[str, Trade] | None:
+        """Get a dictionary mapping UIDs to trades in a solution."""
+        trades_dict: dict[str, Trade] = {}
+        for execution in solution["orders"]:
+            uid = execution["id"]
+            order_data = self.get_order_data(uid)
+            if order_data is None:
+                return None
+            trades_dict[uid] = self.get_trade(order_data, execution)
+
+        return trades_dict

src/monitoring_tests/combinatorial_auction_surplus_test.py

@@ -7,9 +7,7 @@
 from typing import Any
 from fractions import Fraction
 from src.monitoring_tests.base_test import BaseTest
-from src.apis.web3api import Web3API
 from src.apis.orderbookapi import OrderbookAPI
-from src.models import Trade
 from src.constants import SURPLUS_ABSOLUTE_DEVIATION_ETH
 
 
@@ -35,7 +33,6 @@ class CombinatorialAuctionSurplusTest(BaseTest):
 
     def __init__(self) -> None:
         super().__init__()
-        self.web3_api = Web3API()
         self.orderbook_api = OrderbookAPI()
 
     def run_combinatorial_auction(self, competition_data: dict[str, Any]) -> bool:
@@ -51,15 +48,15 @@ def run_combinatorial_auction(self, competition_data: dict[str, Any]) -> bool:
         """
 
         solutions = competition_data["solutions"]
-        winning_solution = competition_data["solutions"][-1]
 
-        aggregate_solutions = [
-            self.get_token_pairs_surplus(
+        aggregate_solutions: list[dict[tuple[str, str], Fraction]] = []
+        for solution in solutions:
+            aggregate_solution = self.get_token_pairs_surplus(
                 solution, competition_data["auction"]["prices"]
             )
-            for solution in solutions
-        ]
-        winning_aggregate_solution = aggregate_solutions[-1]
+            if aggregate_solution is None:
+                return False
+            aggregate_solutions.append(aggregate_solution)
 
         baseline_surplus = self.compute_baseline_surplus(aggregate_solutions)
         filter_mask = self.filter_solutions(aggregate_solutions, baseline_surplus)
@@ -80,18 +77,16 @@ def run_combinatorial_auction(self, competition_data: dict[str, Any]) -> bool:
             sum(surplus for _, surplus in token_pair_surplus.items())
             for _, token_pair_surplus in winning_solvers.items()
         )
-        total_surplus = sum(
-            surplus for _, surplus in winning_aggregate_solution.items()
-        )
+        total_surplus = sum(surplus for _, surplus in aggregate_solutions[-1].items())
 
         a_abs_eth = total_combinatorial_surplus - total_surplus
 
         log_output = "\t".join(
             [
                 "Combinatorial auction surplus test:",
                 f"Tx Hash: {competition_data['transactionHash']}",
-                f"Winning Solver: {winning_solution['solver']}",
-                f"Winning surplus: {self.convert_fractions_to_floats(winning_aggregate_solution)}",
+                f"Winning Solver: {competition_data['solutions'][-1]['solver']}",
+                f"Winning surplus: {self.convert_fractions_to_floats(aggregate_solutions[-1])}",
                 f"Baseline surplus: {self.convert_fractions_to_floats(baseline_surplus)}",
                 f"Solutions filtering winner: {filter_mask[-1]}",
                 f"Solvers filtering winner: {solutions_filtering_winner}",
@@ -113,30 +108,16 @@ def run_combinatorial_auction(self, competition_data: dict[str, Any]) -> bool:
 
         return True
 
-    def get_uid_trades(self, solution: dict[str, Any]) -> dict[str, Trade]:
-        """Get a dictionary mapping UIDs to trades in a solution."""
-        calldata = solution["callData"]
-        settlement = self.web3_api.get_settlement_from_calldata(calldata)
-        trades = self.web3_api.get_trades(settlement)
-        trades_dict = {
-            solution["orders"][i]["id"]: trade for (i, trade) in enumerate(trades)
-        }
-        return trades_dict
-
     def get_token_pairs_surplus(
         self, solution: dict[str, Any], prices: dict[str, float]
-    ) -> dict[tuple[str, str], Fraction]:
+    ) -> dict[tuple[str, str], Fraction] | None:
         """Aggregate surplus of a solution on the different token pairs.
         The result is a dict containing directed token pairs and the aggregated surplus on them.
-
-        Instead of surplus we use the minimum of surplus and the objective. This is more
-        conservative than just using objective. If fees are larger than costs, the objective is
-        larger than surplus and surplus is used for the comparison. If fees are larger than costs,
-        the objective is smaller than surplus and the objective is used instead of surplus for
-        filtering. This takes care of the case of solvers providing a lot of surplus but at really
-        large costs.
         """
-        trades_dict = self.get_uid_trades(solution)
+        trades_dict = self.orderbook_api.get_uid_trades(solution)
+        if trades_dict is None:
+            return None
+
         surplus_dict: dict[tuple[str, str], Fraction] = {}
         for uid in trades_dict:
             trade = trades_dict[uid]
@@ -159,14 +140,6 @@ def get_token_pairs_surplus(
                 + surplus_token_to_eth * trade.get_surplus()
             )
 
-        # use the minimum of surplus and objective in case there is only one token pair
-        if len(surplus_dict) == 1:
-            for token_pair in surplus_dict:
-                surplus_dict[token_pair] = min(
-                    surplus_dict[token_pair],
-                    Fraction(solution["objective"]["total"]) / 10**18,
-                )
-
         return surplus_dict
 
     def compute_baseline_surplus(

src/monitoring_tests/solver_competition_surplus_test.py

@@ -6,9 +6,8 @@
 from typing import Any
 from fractions import Fraction
 from src.monitoring_tests.base_test import BaseTest
-from src.apis.web3api import Web3API
 from src.apis.orderbookapi import OrderbookAPI
-from src.models import Trade
+from src.models import Trade, OrderExecution
 from src.constants import SURPLUS_ABSOLUTE_DEVIATION_ETH, SURPLUS_REL_DEVIATION
 
 
@@ -20,7 +19,6 @@ class SolverCompetitionSurplusTest(BaseTest):
 
     def __init__(self) -> None:
         super().__init__()
-        self.web3_api = Web3API()
         self.orderbook_api = OrderbookAPI()
 
     def compare_orders_surplus(self, competition_data: dict[str, Any]) -> bool:
@@ -32,7 +30,9 @@ def compare_orders_surplus(self, competition_data: dict[str, Any]) -> bool:
 
         solution = competition_data["solutions"][-1]
 
-        trades_dict = self.get_uid_trades(solution)
+        trades_dict = self.orderbook_api.get_uid_trades(solution)
+        if trades_dict is None:
+            return False
 
         for uid in trades_dict:
             trade = trades_dict[uid]
@@ -46,10 +46,10 @@ def compare_orders_surplus(self, competition_data: dict[str, Any]) -> bool:
             )
 
             trade_alt_dict = self.get_trade_alternatives(
-                uid, competition_data["solutions"][0:-1]
+                trade, uid, competition_data["solutions"][0:-1]
             )
 
-            for solver_alt, trade_alt in trade_alt_dict.items():
+            for solver_alt, trade_alt in trade_alt_dict:
                 a_abs = trade_alt.compare_surplus(trade)
                 a_abs_eth = a_abs * token_to_eth
                 a_rel = trade_alt.compare_price(trade)
@@ -82,35 +82,38 @@ def compare_orders_surplus(self, competition_data: dict[str, Any]) -> bool:
         return True
 
     def get_trade_alternatives(
-        self, uid: str, solution_alternatives: list[dict[str, Any]]
-    ) -> dict[str, Trade]:
+        self, trade: Trade, uid: str, solution_alternatives: list[dict[str, Any]]
+    ) -> list[tuple[str, Trade]]:
         """Compute surplus and exchange rate for an order with uid as settled in alternative
         solutions."""
-        trade_alt_dict: dict[str, Trade] = {}
+        trade_alt_list: list[tuple[str, Trade]] = []
+        order_data = trade.data
         for solution_alt in solution_alternatives:
-            if (
-                solution_alt["objective"]["fees"]
-                < 0.9 * solution_alt["objective"]["cost"]
-            ):
-                continue
-            trades_dict_alt = self.get_uid_trades(solution_alt)
+            executions_dict_alt = self.get_uid_order_execution(solution_alt)
             try:
-                trade_alt = trades_dict_alt[uid]
+                trade_alt = Trade(order_data, executions_dict_alt[uid])
             except KeyError:
                 continue
-            trade_alt_dict[solution_alt["solver"]] = trade_alt
-
-        return trade_alt_dict
-
-    def get_uid_trades(self, solution: dict[str, Any]) -> dict[str, Trade]:
-        """Get a dictionary mapping UIDs to trades in a solution."""
-        calldata = solution["callData"]
-        settlement = self.web3_api.get_settlement_from_calldata(calldata)
-        trades = self.web3_api.get_trades(settlement)
-        trades_dict = {
-            solution["orders"][i]["id"]: trade for (i, trade) in enumerate(trades)
-        }
-        return trades_dict
+            trade_alt_list.append((solution_alt["solver"], trade_alt))
+
+        return trade_alt_list
+
+    def get_uid_order_execution(
+        self, solution: dict[str, Any]
+    ) -> dict[str, OrderExecution]:
+        """Given a solution from the competition endpoint, compute the executin for all included
+        orders.
+        """
+        result: dict[str, OrderExecution] = {}
+        for order in solution["orders"]:
+            buy_amount = int(order["buyAmount"])
+            sell_amount = int(order["sellAmount"])
+            # fee amount is set to zero for the moment, could be computed from clearing prices
+            # and buy and sell token of the order
+            fee_amount = 0
+            order_execution = OrderExecution(buy_amount, sell_amount, fee_amount)
+            result[order["id"]] = order_execution
+        return result
 
     def run(self, tx_hash: str) -> bool:
         """

tests/e2e/combinatorial_auction_surplus_test.py

@@ -11,23 +11,8 @@
 class TestCombinatorialAuctionSurplus(unittest.TestCase):
     def test_surplus(self) -> None:
         surplus_test = CombinatorialAuctionSurplusTest()
-        # # Baseline EBBO error
-        # tx_hash = "0x4115f6f4abaea17f2ebef3a1e75c589c38cac048ff5116d406038e48ff7aeacd"
-        # # large settlement with bad execution for one of the orders
-        # tx_hash = "0xc22b1e4984b212e679d4af49c1622e7018c83d5e32ece590cf84a3e1950f9f18"
-        # # EBBO violations
-        # #
-        # tx_hash = "0x2ff69424f7bf8951ed5e7dd04b648380b0e73dbf7f0191c800651bc4b16a30c5"
-        # # combinatorial auction worse than current auction
-        # tx_hash = "0xb743b023ad838f04680fd321bf579c35931c4f886f664bd2b6e675c310a9c287"
-        # # combinatorial auction better than current auction
-        # tx_hash = "0x46639ae0e516bcad7b052fb6bfb6227d0aa2707e9882dd8d86bab2ab6aeee155"
-        # tx_hash = "0xe28b92ba73632d6b167fdb9bbfec10744ce208536901dd43379a6778c4408536"
-        # tx_hash = "0xad0ede9fd68481b8ef4722d069598898e01d61427ccb378ca4c82c772c6644e0"
-        # tx_hash = "0xead8f01e8e24fdc306fca8fcac5146edc22c27e49a7aad6134adc2ad50ba8581"
-        # tx_hash = "0x6200e744e5d6f9990271be53840c01044cc19f3a8526190e1eaac0bc5fefed85"
-        # uncovered bug with wrong scaling of objective
-        tx_hash = "0x97b2f8402d239e16b62b7cc2302ed77ac8fa40d63114ab6804041c9d3b9e6b81"
+        # CoW with liquidity order by Naive solver
+        tx_hash = "0x6b728195926e033ab92bbe7db51170c582ff57ba841aaaca3a9319cfe34491ff"
         self.assertTrue(surplus_test.run(tx_hash))
 
 

tests/e2e/surplus_test.py

@@ -9,22 +9,11 @@
 
 
 class TestSurplus(unittest.TestCase):
-    def test_surplus(self):
+    def test_surplus(self) -> None:
         surplus_test = SolverCompetitionSurplusTest()
-        # minor EBBO violation
-        tx_hash = "0xb2189d1a9fe31d15522f0110c0a2907354fbb1edccd1a6186ef0608fe5ad5722"
+        # new competition format: no alert or info
+        tx_hash = "0xc140a3adc9debfc00a45cc713afbac1bbe197ad2dd1d7fa5b4a36de1080a3d66"
         self.assertTrue(surplus_test.run(tx_hash))
-        # hash not in the data base
-        tx_hash = "0x999999999fe31d15522f0110c0a2907354fbb1edccd1a6186ef0608fe5ad5722"
-        self.assertFalse(surplus_test.run(tx_hash))
-        # surplus shift to partial fill
-        tx_hash = "0xf467a6a01f61fa608c1bc116e2f4f4df1b95461827b1e7700c1d36628875feab"
-        self.assertTrue(surplus_test.run(tx_hash))
-        # order combined with partial fill
-        tx_hash = "0x8e9f98cabf9b6ff4e001eda5efacfd70590a60bd03a499d8b02130b67b208eb1"
-        self.assertTrue(surplus_test.run(tx_hash))
-        # partial fill with competition
-        # look for this
 
 
 if __name__ == "__main__":