Challenge MDD Elimination Fix

runnable/run_challenge_review.py

@@ -25,7 +25,6 @@
     challengeperiod_manager.position_manager = position_manager
     perf_ledger_manager.position_manager = position_manager
     subtensor_weight_setter = SubtensorWeightSetter(
-        config=None,
         metagraph=None,
         running_unit_tests=False,
         position_manager=position_manager,
@@ -39,11 +38,13 @@
     inspection_hotkeys_dict = challengeperiod_manager.challengeperiod_testing
 
     ## filter the ledger for the miners that passed the challenge period
-    success_hotkeys = list(inspection_hotkeys_dict.keys())
-    filtered_ledger = perf_ledger_manager.filtered_ledger_for_scoring(hotkeys=success_hotkeys)
-
+    testing_hotkeys = list(inspection_hotkeys_dict.keys())
+    filtered_ledger = perf_ledger_manager.filtered_ledger_for_scoring(hotkeys=testing_hotkeys)
+    filtered_positions, _ = position_manager.filtered_positions_for_scoring(hotkeys=testing_hotkeys)
     # Get all possible positions, even beyond the lookback range
     success, eliminations = challengeperiod_manager.inspect(
+        positions=filtered_positions,
+        success_hotkeys=list(challengeperiod_manager.challengeperiod_success.keys()),
         ledger=filtered_ledger,
         inspection_hotkeys=inspection_hotkeys_dict,
         current_time=current_time,

tests/shared_objects/mock_classes.py

@@ -58,14 +58,14 @@ def __init__(self, metagraph, position_manager):
 class MockLivePriceFetcher(LivePriceFetcher):
     def __init__(self, secrets, disable_ws):
         super().__init__(secrets=secrets, disable_ws=disable_ws)
-        self.polygon_data_service = MockPolygonDataService(api_key=secrets["polygon_apikey"])
+        self.polygon_data_service = MockPolygonDataService(api_key=secrets["polygon_apikey"], disable_ws=disable_ws)
 
     def get_sorted_price_sources_for_trade_pair(self, trade_pair, processed_ms):
         return [PriceSource(price=1, open=1, high=1, close=1, low=1, bid=1, ask=1)]
 
 class MockPolygonDataService(PolygonDataService):
-    def __init__(self, api_key):
-        super().__init__(api_key)
+    def __init__(self, api_key, disable_ws):
+        super().__init__(api_key, disable_ws)
         self.trade_pair_to_recent_events_realtime = defaultdict()
 
     def get_last_quote(self, trade_pair: TradePair, processed_ms: int) -> (float, float):

vali_objects/utils/challengeperiod_manager.py

@@ -307,17 +307,29 @@ def inspect(
             # We want to know if the miner still has time, as we know the criteria to pass is not met
             time_criteria = current_time - inspection_time <= ValiConfig.CHALLENGE_PERIOD_MS
 
+            # Get hotkey to ledger dict that only includes the inspection miner
+            has_minimum_ledger, inspection_ledger = ChallengePeriodManager.screen_minimum_ledger(ledger=ledger,
+                                                                                                 inspection_hotkey=hotkey)
+            if not has_minimum_ledger:
+                passing_criteria = False
+
+            # This step we want to check their drawdown. If they fail, we can move on.
+            failing_criteria, recorded_drawdown_percentage = LedgerUtils.is_beyond_max_drawdown(
+                ledger_element=inspection_ledger.get(hotkey))
+
+            if failing_criteria:
+                bt.logging.info(
+                    f'Hotkey {hotkey} has failed the challenge period due to drawdown {recorded_drawdown_percentage}. cp_failed')
+                failing_miners[hotkey] = (
+                EliminationReason.FAILED_CHALLENGE_PERIOD_DRAWDOWN.value, recorded_drawdown_percentage)
+                continue
+
             # Get hotkey to positions dict that only includes the inspection miner
             has_minimum_positions, inspection_positions = ChallengePeriodManager.screen_minimum_positions(positions=positions, inspection_hotkey=hotkey)
             if not has_minimum_positions:
                 miners_not_enough_positions.append((hotkey, positions.get(hotkey, [])))
                 passing_criteria = False
 
-            # Get hotkey to ledger dict that only includes the inspection miner
-            has_minimum_ledger, inspection_ledger = ChallengePeriodManager.screen_minimum_ledger(ledger=ledger, inspection_hotkey=hotkey)
-            if not has_minimum_ledger:
-                passing_criteria = False
-
             # This step is meant to ensure no positions or ledgers reference missing hotkeys, we need them to evaluate
             if not passing_criteria:
                 if not time_criteria:
@@ -326,14 +338,6 @@ def inspect(
                     failing_miners[hotkey] = (EliminationReason.FAILED_CHALLENGE_PERIOD_TIME.value, -1)
 
                 continue  # Moving on, as the miner is already failing
-            # This step we want to check their drawdown. If they fail, we can move on.
-            failing_criteria, recorded_drawdown_percentage = LedgerUtils.is_beyond_max_drawdown(ledger_element=ledger[hotkey])
-
-            if failing_criteria:
-                bt.logging.info(f'Hotkey {hotkey} has failed the challenge period due to drawdown {recorded_drawdown_percentage}. cp_failed')
-                failing_miners[hotkey] = (EliminationReason.FAILED_CHALLENGE_PERIOD_DRAWDOWN.value, recorded_drawdown_percentage)
-                continue
-
 
             # The main logic loop. They are in the competition but haven't passed yet, need to check the time after.
             passing_criteria = ChallengePeriodManager.screen_passing_criteria(

vali_objects/utils/position_filtering.py

@@ -94,8 +94,6 @@ def filter_positions_for_duration(positions: list[Position]):
         """
         filtered_positions = []
         for position in positions:
-            if not position.is_closed_position:
-                continue
 
             if position.close_ms - position.open_ms < ValiConfig.MINIMUM_POSITION_DURATION_MS:
                 continue