Re-add the API serving timing metrics.

python/sglang/srt/entrypoints/openai/serving_base.py

@@ -88,6 +88,9 @@ async def handle_request(
         """Handle the specific request type with common pattern
         If you want to override this method, you should be careful to record the validation time.
         """
+        received_time = time.time()
+        received_time_perf = time.perf_counter()
+
         try:
             # Validate request
             validation_start = time.perf_counter()
@@ -103,6 +106,12 @@ async def handle_request(
             if hasattr(adapted_request, "validation_time"):
                 adapted_request.validation_time = validation_time
 
+            if hasattr(adapted_request, "received_time"):
+                adapted_request.received_time = received_time
+
+            if hasattr(adapted_request, "received_time_perf"):
+                adapted_request.received_time_perf = received_time_perf
+
             # Note(Xinyuan): raw_request below is only used for detecting the connection of the client
             if hasattr(request, "stream") and request.stream:
                 return await self._handle_streaming_request(

python/sglang/srt/managers/io_struct.py

@@ -110,6 +110,18 @@ class SpeculativeDecodingMetricsMixin:
     spec_accepted_tokens: List[int]
 
 
+@dataclass
+class APIServingTimingMixin:
+    # Validation step duration
+    validation_time: Optional[float] = None
+
+    # For metrics
+    received_time: Optional[float] = None
+
+    # Perf_counter equivalents for accurate time calculations
+    received_time_perf: Optional[float] = None
+
+
 # Parameters for a session
 @dataclass
 class SessionParams:
@@ -138,7 +150,7 @@ class SessionParams:
 
 
 @dataclass
-class GenerateReqInput(BaseReq):
+class GenerateReqInput(BaseReq, APIServingTimingMixin):
     # The input prompt. It can be a single prompt or a batch of prompts.
     text: Optional[Union[List[str], str]] = None
     # The token ids for text; one can specify either text or input_ids
@@ -201,9 +213,6 @@ class GenerateReqInput(BaseReq):
     # For reasoning
     reasoning: bool = False
 
-    # Validation step duration
-    validation_time: Optional[float] = None
-
     # For data parallel rank routing
     data_parallel_rank: Optional[int] = None
 
@@ -573,6 +582,7 @@ def _validate_session_params(self):
                 raise ValueError("Session params must be a dict or a list of dicts.")
 
     def __getitem__(self, i):
+        timing_keys = APIServingTimingMixin.__dataclass_fields__.keys()
         return GenerateReqInput(
             text=self.text[i] if self.text is not None else None,
             input_ids=self.input_ids[i] if self.input_ids is not None else None,
@@ -623,7 +633,6 @@ def __getitem__(self, i):
             decode_tp_size=(
                 self.decode_tp_size[i] if self.decode_tp_size is not None else None
             ),
-            validation_time=self.validation_time,
             data_parallel_rank=(
                 self.data_parallel_rank if self.data_parallel_rank is not None else None
             ),
@@ -635,6 +644,7 @@ def __getitem__(self, i):
             return_bytes=self.return_bytes,
             return_entropy=self.return_entropy,
             http_worker_ipc=self.http_worker_ipc,
+            **{key: getattr(self, key) for key in timing_keys},
         )
 
 
@@ -724,7 +734,7 @@ def __iter__(self):
 
 
 @dataclass
-class EmbeddingReqInput(BaseReq):
+class EmbeddingReqInput(BaseReq, APIServingTimingMixin):
     # The input prompt. It can be a single prompt or a batch of prompts.
     text: Optional[Union[List[List[str]], List[str], str]] = None
     # The image input. It can be an image instance, file name, URL, or base64 encoded string.
@@ -832,6 +842,8 @@ def __getitem__(self, i):
                 http_worker_ipc=self.http_worker_ipc,
             )
 
+        timing_keys = APIServingTimingMixin.__dataclass_fields__.keys()
+
         return EmbeddingReqInput(
             text=self.text[i] if self.text is not None else None,
             input_ids=self.input_ids[i] if self.input_ids is not None else None,
@@ -840,9 +852,9 @@ def __getitem__(self, i):
             video_data=self.video_data[i] if self.video_data is not None else None,
             sampling_params=self.sampling_params[i],
             rid=self.rid[i],
-            validation_time=self.validation_time,
             dimensions=self.dimensions,
             http_worker_ipc=self.http_worker_ipc,
+            **{key: getattr(self, key) for key in timing_keys},
         )
 
 

python/sglang/srt/managers/tokenizer_manager.py

@@ -431,7 +431,7 @@ async def generate_request(
         obj: Union[GenerateReqInput, EmbeddingReqInput],
         request: Optional[fastapi.Request] = None,
     ):
-        created_time = time.time()
+        created_time = obj.received_time if obj.received_time else time.time()
         self.auto_create_handle_loop()
         obj.normalize_batch_and_arguments()