From 481870e615a141fdf146521fe2f263e3a346dc33 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Wed, 1 May 2024 09:04:49 -0400 Subject: [PATCH] Adding tolerance to the combinatorial auction alert (#108) This PR adds a tolerance to the alert of the combinatorial auction test. This is implemented by adding information on surplus differences to `filter_mask`. That information is then checked later and compared to the new constant `COMBINATORIAL_AUCTION_ABSOLUTE_DEVIATION_ETH`. That constant is for now set to `0.001 ETH`. For logging purposes, the function `convert_fractions_to_floats` was extended to also handle lists. The code was tested on a recent alert for the transaction hash `"0x1541857aaf67c8d06028a92202c8b44a731c51bcd2819d1a0a62faf00f073227"`. There, baseline solver was providing `0.0004 ETH` more surplus. Before the change, the test emits an alert. After the change, the test only logs an info message. This PR closes #107. --- src/constants.py | 3 ++ .../combinatorial_auction_surplus_test.py | 40 +++++++++++++------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/constants.py b/src/constants.py index 9da41d3..e9302ef 100644 --- a/src/constants.py +++ b/src/constants.py @@ -9,6 +9,9 @@ SURPLUS_ABSOLUTE_DEVIATION_ETH = 0.005 SURPLUS_REL_DEVIATION = 0.004 +# combinatorial auctions +COMBINATORIAL_AUCTION_ABSOLUTE_DEVIATION_ETH = 0.001 + # cost coverage test COST_COVERAGE_ABSOLUTE_DEVIATION_ETH = 0.01 COST_COVERAGE_RELATIVE_DEVIATION = 0.50 diff --git a/src/monitoring_tests/combinatorial_auction_surplus_test.py b/src/monitoring_tests/combinatorial_auction_surplus_test.py index 597b8f2..377ec6c 100644 --- a/src/monitoring_tests/combinatorial_auction_surplus_test.py +++ b/src/monitoring_tests/combinatorial_auction_surplus_test.py @@ -1,6 +1,7 @@ """ Comparing order surplus per token pair to a reference solver in the competition. """ + # pylint: disable=logging-fstring-interpolation # pylint: disable=duplicate-code @@ -8,7 +9,10 @@ from fractions import Fraction from src.monitoring_tests.base_test import BaseTest from src.apis.orderbookapi import OrderbookAPI -from src.constants import SURPLUS_ABSOLUTE_DEVIATION_ETH +from src.constants import ( + SURPLUS_ABSOLUTE_DEVIATION_ETH, + COMBINATORIAL_AUCTION_ABSOLUTE_DEVIATION_ETH, +) class CombinatorialAuctionSurplusTest(BaseTest): @@ -70,7 +74,7 @@ def run_combinatorial_auction(self, competition_data: dict[str, Any]) -> bool: solutions_filtering_winner = [ solution["solver"] for i, solution in enumerate(solutions) - if i in filter_mask[-1] + if i in [ind for ind, _ in filter_mask[-1]] ] total_combinatorial_surplus = sum( @@ -88,7 +92,7 @@ def run_combinatorial_auction(self, competition_data: dict[str, Any]) -> bool: f"Winning Solver: {competition_data['solutions'][-1]['solver']}", f"Winning surplus: {self.convert_fractions_to_floats(aggregate_solutions[-1])}", f"Baseline surplus: {self.convert_fractions_to_floats(baseline_surplus)}", - f"Solutions filtering winner: {filter_mask[-1]}", + f"Solutions filtering winner: {self.convert_fractions_to_floats(filter_mask[-1])}", f"Solvers filtering winner: {solutions_filtering_winner}", f"Combinatorial winners: {self.convert_fractions_to_floats(winning_solvers)}", f"Total surplus: {float(total_surplus):.5f} ETH", @@ -96,7 +100,12 @@ def run_combinatorial_auction(self, competition_data: dict[str, Any]) -> bool: f"Absolute difference: {float(a_abs_eth):.5f}ETH", ] ) - if "baseline" in solutions_filtering_winner: + + if any( + solutions[ind]["solver"] == "baseline" + and surplus_difference > COMBINATORIAL_AUCTION_ABSOLUTE_DEVIATION_ETH + for ind, surplus_difference in filter_mask[-1] + ): self.alert(log_output) elif ( a_abs_eth > SURPLUS_ABSOLUTE_DEVIATION_ETH / 10 @@ -167,22 +176,25 @@ def filter_solutions( self, aggregate_solutions: list[dict[tuple[str, str], Fraction]], baseline_surplus: dict[tuple[str, str], tuple[Fraction, int]], - ) -> list[list[int]]: - """Filter out solutions which do not provide enough surplus. + ) -> list[list[tuple[int, Fraction]]]: + """Check which baseline solutions filter out solutions. Only solutions are considered for ranking that supply more surplus than any of the single - aggregate order solutions on all token pairs they touch. - The function returns a list of integers for each solution. The integers in those lists - correspond to those baselines that would have resulted in filtering of the solution. This - information is used e.g. to trigger alerts depending on which solver resulted in filtering. + aggregate order solutions on all directed token pairs they touch. + For each solution a list of tuples is returned. The first entry of each tuple is an integer + corresponding to the index of a baseline solution would have resulted in filtering of the + given solution. The second entry is the difference in surplus of the filtered and filtering + solution. + This information is used e.g. to trigger alerts depending on which solver resulted in + filtering. """ - result: list[list[int]] = [] + result: list[list[tuple[int, Fraction]]] = [] for aggregate_solution in aggregate_solutions: flag = [] for token_pair, surplus in aggregate_solution.items(): if token_pair in baseline_surplus: surplus_ref, solution_index = baseline_surplus[token_pair] if surplus < surplus_ref: - flag.append(solution_index) + flag.append((solution_index, surplus_ref - surplus)) result.append(flag) return result @@ -191,7 +203,7 @@ def determine_winning_solutions( self, aggregate_solutions: list[dict[tuple[str, str], Fraction]], baseline_surplus: dict[tuple[str, str], tuple[Fraction, int]], - filter_mask: list[list[int]], + filter_mask: list[list[tuple[int, Fraction]]], ) -> dict[tuple[str, str], int]: """Determine the winning solutions for the different token pairs. There is one batch winner, and the remaining token pairs are won by single aggregate order @@ -239,6 +251,8 @@ def convert_fractions_to_floats(self, obj: Any, precision: int = 4) -> Any: k: self.convert_fractions_to_floats(v, precision) for k, v in obj.items() } + if isinstance(obj, list): + return [self.convert_fractions_to_floats(v, precision) for v in obj] if isinstance(obj, tuple): return tuple(self.convert_fractions_to_floats(x, precision) for x in obj) if isinstance(obj, Fraction):