add sample_param logs and env: HEALTH_TIMEOUT (#698)

Co-authored-by: shihaobai <[email protected]>
ModelTC · Jan 6, 2025 · 777fc04 · 777fc04
1 parent 7243b60
commit 777fc04
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 22 deletions.
diff --git a/lightllm/server/httpserver/manager.py b/lightllm/server/httpserver/manager.py
@@ -358,27 +358,29 @@ async def _wait_to_token_package(
                         prompt_cache_len = metadata.pop("prompt_cache_len", 0)
                         prompt_cache_ratio = prompt_cache_len / prompt_tokens
                         format_start_time = datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%d %H:%M:%S")
-                        logger.info(
-                            f"X-Request-Id:{x_request_id} "
-                            f"X-Session-Id:{x_session_id} start_time:{format_start_time} "
-                            f"lightllm_req_id:{group_request_id} first_token_cost:{first_token_cost_ms}ms "
-                            f"total_cost_time:{total_cost_time_ms}ms,out_token_counter:{out_token_counter} "
-                            f"mean_per_token_cost_time: {mean_per_token_cost_time_ms}ms "
-                            f"prompt_token_num:{prompt_tokens} "
-                            f"prompt_cache_len:{prompt_cache_len} "
-                            f"prompt_cache_ratio:{prompt_cache_ratio} "
-                        )
-                        self.metric_client.histogram_observe(
-                            "lightllm_request_inference_duration", total_cost_time_ms / 1000.0
-                        )
-                        self.metric_client.histogram_observe(
-                            "lightllm_request_mean_time_per_token_duration", mean_per_token_cost_time_ms / 1000.0
-                        )
-                        self.metric_client.histogram_observe(
-                            "lightllm_request_first_token_duration", first_token_cost_ms / 1000.0
-                        )
-                        self.metric_client.histogram_observe("lightllm_request_generated_tokens", out_token_counter)
-                        self.metric_client.counter_inc("lightllm_request_success")
+                        if request is not None:
+                            logger.info(
+                                f"X-Request-Id:{x_request_id} "
+                                f"X-Session-Id:{x_session_id} start_time:{format_start_time} "
+                                f"lightllm_req_id:{group_request_id} first_token_cost:{first_token_cost_ms}ms "
+                                f"total_cost_time:{total_cost_time_ms}ms,out_token_counter:{out_token_counter} "
+                                f"mean_per_token_cost_time: {mean_per_token_cost_time_ms}ms "
+                                f"prompt_token_num:{prompt_tokens} "
+                                f"prompt_cache_len:{prompt_cache_len} "
+                                f"prompt_cache_ratio:{prompt_cache_ratio} "
+                                f"sampling_params: {{{sampling_params.to_string()}}}"
+                            )
+                            self.metric_client.histogram_observe(
+                                "lightllm_request_inference_duration", total_cost_time_ms / 1000.0
+                            )
+                            self.metric_client.histogram_observe(
+                                "lightllm_request_mean_time_per_token_duration", mean_per_token_cost_time_ms / 1000.0
+                            )
+                            self.metric_client.histogram_observe(
+                                "lightllm_request_first_token_duration", first_token_cost_ms / 1000.0
+                            )
+                            self.metric_client.histogram_observe("lightllm_request_generated_tokens", out_token_counter)
+                            self.metric_client.counter_inc("lightllm_request_success")
 
                         return
                 req_status.out_token_info_list.clear()

diff --git a/lightllm/server/sampling_params.py b/lightllm/server/sampling_params.py
@@ -260,3 +260,9 @@ def to_origin_dict(self):
         ret["group_request_id"] = self.group_request_id
         ret["suggested_dp_index"] = self.suggested_dp_index
         return ret
+
+    def to_string(self):
+        output_str = ""
+        for name, value in vars(self).items():
+            output_str += f"{name}: {value} "
+        return output_str
diff --git a/lightllm/utils/health_check.py b/lightllm/utils/health_check.py
@@ -1,3 +1,4 @@
+import os
 import asyncio
 import numpy as np
 from dataclasses import dataclass
@@ -19,6 +20,7 @@
 class HealthObj:
     _is_health: bool = True
     _is_health_checking: bool = False
+    timeout: int = int(os.getenv("HEALTH_TIMEOUT", 100))
 
     def begin_check(self):
         self._is_health_checking = True
@@ -65,7 +67,7 @@ async def check_timeout(results_generator):
                 pass
 
         try:
-            await asyncio.wait_for(check_timeout(results_generator), timeout=88)
+            await asyncio.wait_for(check_timeout(results_generator), timeout=health_obj.timeout)
             health_obj.set_health()
         except asyncio.TimeoutError:
             health_obj.set_unhealth()