[https://nvbugs/5480289][fix] release slot manager in mtp MTPHiddenStatesManager (#7340)

Signed-off-by: Yue Weng <[email protected]>

Author: yweng0828

tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py

@@ -304,6 +304,7 @@ def clear(self):
         self.static_inputs.clear()
         self.graph_outputs.clear()
         self.graph_metadata.clear()
+        self.padding_dummy_request = None
         del self.memory_pool
         self.memory_pool = None
         torch.cuda.empty_cache()

tensorrt_llm/_torch/pyexecutor/resource_manager.py

@@ -1042,6 +1042,13 @@ def remove_slot(self, request_id: int):
             slot = self.slot_mapping.pop(request_id)
             self.free_slots.add(slot)
 
+    def shutdown(self):
+        req_ids_list = list(self.slot_mapping.keys())
+        for rid in req_ids_list:
+            self.remove_slot(rid)
+        assert len(self.slot_mapping) == 0 and len(
+            self.free_slots) == self.max_num_requests
+
 
 class ResourceManager:
 

tensorrt_llm/_torch/speculative/mtp.py

@@ -85,7 +85,7 @@ def add_dummy_requests(self, request_ids: List[int]):
             self.slot_manager.add_slot(rid)
 
     def shutdown(self):
-        pass
+        self.slot_manager.shutdown()
 
     def get_max_resource_count(self) -> int:
         return self.max_num_requests