backtesting progress

progress

cleanup

neurons/validator.py

@@ -318,7 +318,7 @@ def rc_priority_fn(synapse: template.protocol.ValidatorCheckpoint) -> float:
 
         self.mdd_checker = MDDChecker(self.metagraph, self.position_manager, live_price_fetcher=self.live_price_fetcher,
                                       shutdown_dict=shutdown_dict)
-        self.weight_setter = SubtensorWeightSetter(self.config, self.metagraph, position_manager=self.position_manager)
+        self.weight_setter = SubtensorWeightSetter(self.metagraph, position_manager=self.position_manager)
 
         self.request_core_manager = RequestCoreManager(self.position_manager, self.weight_setter, self.plagiarism_detector)
         self.miner_statistics_manager = MinerStatisticsManager(self.position_manager, self.weight_setter, self.plagiarism_detector)

runnable/generate_request_core.py

@@ -327,8 +327,6 @@ def generate_request_core(self, time_now:int, selected_miner_hotkeys: List[str]
     challengeperiod_manager.position_manager = position_manager
     perf_ledger_manager.position_manager = position_manager
     subtensor_weight_setter = SubtensorWeightSetter(
-        config=None,
-        wallet=None,
         metagraph=None,
         running_unit_tests=False,
         position_manager=position_manager,

runnable/generate_request_minerstatistics.py

@@ -664,7 +664,6 @@ def generate_request_minerstatistics(self, time_now: int, checkpoints: bool = Tr
     challengeperiod_manager.position_manager = position_manager
     perf_ledger_manager.position_manager = position_manager
     subtensor_weight_setter = SubtensorWeightSetter(
-        config=None,
         metagraph=None,
         running_unit_tests=False,
         position_manager=position_manager,

runnable/generate_request_outputs.py

@@ -75,7 +75,6 @@ def repull_data_from_disk(self):
         challengeperiod_manager.position_manager = self.position_manager
         perf_ledger_manager.position_manager = self.position_manager
         self.subtensor_weight_setter = SubtensorWeightSetter(
-            config=None,
             metagraph=None,
             running_unit_tests=False,
             position_manager=self.position_manager,

shared_objects/cache_controller.py

@@ -16,10 +16,11 @@ class CacheController:
     MAX_DAILY_DRAWDOWN = 'MAX_DAILY_DRAWDOWN'
     MAX_TOTAL_DRAWDOWN = 'MAX_TOTAL_DRAWDOWN'
 
-    def __init__(self, metagraph=None, running_unit_tests=False):
+    def __init__(self, metagraph=None, running_unit_tests=False, is_backtesting=False):
         self.running_unit_tests = running_unit_tests
         self.init_cache_files()
         self.metagraph = metagraph  # Refreshes happen on validator
+        self.is_backtesting = is_backtesting
         self._last_update_time_ms = 0
         self.DD_V2_TIME = TimeUtil.millis_to_datetime(1715359820000 + 1000 * 60 * 60 * 2)  # 5/10/24 TODO: Update before mainnet release
 
@@ -62,9 +63,14 @@ def generate_elimination_row(hotkey, dd, reason, t_ms=None, price_info=None, ret
 
     def refresh_allowed(self, refresh_interval_ms):
         self.attempted_start_time_ms = TimeUtil.now_in_millis()
+
+        if self.is_backtesting:
+            return True
+
         return self.running_unit_tests or \
                     self.attempted_start_time_ms - self.get_last_update_time_ms() > refresh_interval_ms
 
+
     def init_cache_files(self) -> None:
         ValiBkpUtils.make_dir(ValiBkpUtils.get_vali_dir(running_unit_tests=self.running_unit_tests))
         # Check if the get_miner_dir directory exists. If not, create it

time_util/time_util.py

@@ -1,5 +1,6 @@
 # developer: Taoshidev
 # Copyright © 2024 Taoshi Inc
+import functools
 import re
 import time
 from datetime import datetime, timedelta, timezone
@@ -195,14 +196,20 @@ def my_function():
                  pass
 """
 def timeme(func):
-    def wrapper(*args, **kwargs):
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):  # Explicitly declare self
+        if isinstance(self, object) and hasattr(self, "is_backtesting") and self.is_backtesting:
+            #print(f"Skipping timing for {func.__name__} because is_backtesting is True")
+            return func(self, *args, **kwargs)  # Call function without timing
+
+        # Time the function execution
         start = time.time()
-        result = func(*args, **kwargs)
+        result = func(self, *args, **kwargs)
         end = time.time()
-        print(f"{func.__name__} took {end - start} s to run")
+        print(f"{func.__name__} took {end - start:.6f} s to run")
         return result
 
-    return wrapper
+    return functools.update_wrapper(wrapper, func)
 
 class UnifiedMarketCalendar:
     def __init__(self):

vali_objects/utils/challengeperiod_manager.py

@@ -22,26 +22,31 @@ class FailedChallengeReason(Enum):
 
 class ChallengePeriodManager(CacheController):
     def __init__(self, metagraph, perf_ledger_manager : PerfLedgerManager =None, running_unit_tests=False,
-                 position_manager: PositionManager =None, ipc_manager=None):
-        super().__init__(metagraph, running_unit_tests=running_unit_tests)
+                 position_manager: PositionManager =None, ipc_manager=None, is_backtesting=False):
+        super().__init__(metagraph, running_unit_tests=running_unit_tests, is_backtesting=is_backtesting)
         self.perf_ledger_manager = perf_ledger_manager if perf_ledger_manager else \
             PerfLedgerManager(metagraph, running_unit_tests=running_unit_tests)
         self.position_manager = position_manager
         self.elimination_manager = self.position_manager.elimination_manager
-        disk_challenegeperiod_testing = self.get_challengeperiod_testing(from_disk=True)
-        disk_challenegeperiod_success = self.get_challengeperiod_success(from_disk=True)
+
+        if self.is_backtesting:
+            initial_challenegeperiod_testing = {}
+            initial_challenegeperiod_success = {}
+        else:
+            initial_challenegeperiod_testing = self.get_challengeperiod_testing(from_disk=True)
+            initial_challenegeperiod_success = self.get_challengeperiod_success(from_disk=True)
         self.using_ipc = bool(ipc_manager)
         if ipc_manager:
             self.challengeperiod_testing = ipc_manager.dict()
             self.challengeperiod_success = ipc_manager.dict()
-            for k, v in disk_challenegeperiod_testing.items():
+            for k, v in initial_challenegeperiod_testing.items():
                 self.challengeperiod_testing[k] = v
-            for k, v in disk_challenegeperiod_success.items():
+            for k, v in initial_challenegeperiod_success.items():
                 self.challengeperiod_success[k] = v
         else:
-            self.challengeperiod_testing = disk_challenegeperiod_testing
-            self.challengeperiod_success = disk_challenegeperiod_success
-        if len(self.get_challengeperiod_testing()) == 0 and len(self.get_challengeperiod_success()) == 0:
+            self.challengeperiod_testing = initial_challenegeperiod_testing
+            self.challengeperiod_success = initial_challenegeperiod_success
+        if not self.is_backtesting and len(self.get_challengeperiod_testing()) == 0 and len(self.get_challengeperiod_success()) == 0:
             ValiBkpUtils.write_file(
                 ValiBkpUtils.get_challengeperiod_file_location(running_unit_tests=self.running_unit_tests),
                 {"testing": {}, "success": {}}
@@ -530,6 +535,8 @@ def _demote_challengeperiod_in_memory(self, eliminations_with_reasons: dict[str,
             self.elimination_manager.append_elimination_row(hotkey=hotkey,current_dd=elim_mdd,mdd_failure=elim_reason)
 
     def _write_challengeperiod_from_memory_to_disk(self):
+        if self.is_backtesting:
+            return
         challengeperiod_data = {
             "testing": self.get_challengeperiod_testing(),
             "success": self.get_challengeperiod_success()

vali_objects/utils/elimination_manager.py

@@ -23,8 +23,8 @@ class EliminationManager(CacheController):
     """
 
     def __init__(self, metagraph, position_manager, challengeperiod_manager,
-                 running_unit_tests=False, shutdown_dict=None, ipc_manager=None):
-        super().__init__(metagraph=metagraph)
+                 running_unit_tests=False, shutdown_dict=None, ipc_manager=None, is_backtesting=False):
+        super().__init__(metagraph=metagraph, is_backtesting=is_backtesting)
         self.position_manager = position_manager
         self.shutdown_dict = shutdown_dict
         self.challengeperiod_manager = challengeperiod_manager
@@ -184,7 +184,8 @@ def _delete_eliminated_expired_miners(self):
                     bt.logging.info(f"Zombie miner dir not found. Already deleted. [{miner_dir}]")
 
     def save_eliminations(self):
-        self.write_eliminations_to_disk(self.eliminations)
+        if not self.is_backtesting:
+            self.write_eliminations_to_disk(self.eliminations)
 
     def write_eliminations_to_disk(self, eliminations):
         if not isinstance(eliminations, list):

vali_objects/utils/position_manager.py

@@ -37,9 +37,10 @@ def __init__(self, metagraph=None, running_unit_tests=False,
                  elimination_manager=None,
                  secrets=None,
                  ipc_manager=None,
-                 live_price_fetcher=None):
+                 live_price_fetcher=None,
+                 is_backtesting=False):
 
-        super().__init__(metagraph=metagraph, running_unit_tests=running_unit_tests)
+        super().__init__(metagraph=metagraph, running_unit_tests=running_unit_tests, is_backtesting=is_backtesting)
         # Populate memory with positions
 
         self.perf_ledger_manager = perf_ledger_manager
@@ -56,13 +57,16 @@ def __init__(self, metagraph=None, running_unit_tests=False,
         else:
             self.hotkey_to_positions = {}
         self.secrets = secrets
-        self.populate_memory_positions_for_first_time()
+        self._populate_memory_positions_for_first_time()
         self.live_price_fetcher = live_price_fetcher
 
     @timeme
-    def populate_memory_positions_for_first_time(self):
-        temp = self.get_positions_for_all_miners(from_disk=True)
-        for hk, positions in temp.items():
+    def _populate_memory_positions_for_first_time(self):
+        if self.is_backtesting:
+            return
+
+        initial_hk_to_positions = self.get_positions_for_all_miners(from_disk=True)
+        for hk, positions in initial_hk_to_positions.items():
             if positions:  # Only populate if there are no positions in the miner dir
                 self.hotkey_to_positions[hk] = positions
 
@@ -815,17 +819,18 @@ def _save_miner_position_to_memory(self, position: Position):
 
 
     def save_miner_position(self, position: Position, delete_open_position_if_exists=True) -> None:
-        miner_dir = ValiBkpUtils.get_partitioned_miner_positions_dir(position.miner_hotkey,
-                                                                     position.trade_pair.trade_pair_id,
-                                                                     order_status=OrderStatus.OPEN if position.is_open_position else OrderStatus.CLOSED,
-                                                                     running_unit_tests=self.running_unit_tests)
-        if position.is_closed_position and delete_open_position_if_exists:
-            self.delete_open_position_if_exists(position)
-        elif position.is_open_position:
-            self.verify_open_position_write(miner_dir, position)
-
-        #print(f'Saving position {position.position_uuid} for miner {position.miner_hotkey} and trade pair {position.trade_pair.trade_pair_id} is_open {position.is_open_position}')
-        ValiBkpUtils.write_file(miner_dir + position.position_uuid, position)
+        if not self.is_backtesting:
+            miner_dir = ValiBkpUtils.get_partitioned_miner_positions_dir(position.miner_hotkey,
+                                                                         position.trade_pair.trade_pair_id,
+                                                                         order_status=OrderStatus.OPEN if position.is_open_position else OrderStatus.CLOSED,
+                                                                         running_unit_tests=self.running_unit_tests)
+            if position.is_closed_position and delete_open_position_if_exists:
+                self.delete_open_position_if_exists(position)
+            elif position.is_open_position:
+                self.verify_open_position_write(miner_dir, position)
+
+            #print(f'Saving position {position.position_uuid} for miner {position.miner_hotkey} and trade pair {position.trade_pair.trade_pair_id} is_open {position.is_open_position}')
+            ValiBkpUtils.write_file(miner_dir + position.position_uuid, position)
         self._save_miner_position_to_memory(position)
 
     def overwrite_position_on_disk(self, position: Position) -> None:
@@ -905,9 +910,10 @@ def delete_position(self, p: Position, check_open_and_closed_dirs=False):
         else:
             file_paths = [self.get_filepath_for_position(hotkey, trade_pair_id, position_uuid, is_open)]
         for fp in file_paths:
-            if os.path.exists(fp):
-                os.remove(fp)
-                bt.logging.info(f"Deleted position from disk: {fp}")
+            if not self.is_backtesting:
+                if os.path.exists(fp):
+                    os.remove(fp)
+                    bt.logging.info(f"Deleted position from disk: {fp}")
             self._delete_position_from_memory(hotkey, position_uuid)
 
     def _delete_position_from_memory(self, hotkey, position_uuid):