fix: Heavy logging by TensorRT-LLM monolith worker

examples/trtllm/common/base_engine.py

@@ -15,6 +15,7 @@
 
 
 import asyncio
+import os
 import threading
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
@@ -32,7 +33,7 @@
 
 from .kv_cache_event_publisher import KVCacheEventPublisher
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class ChatProcessorMixin:

examples/trtllm/common/generators.py

@@ -15,6 +15,7 @@
 
 
 import json
+import os
 import signal
 import uuid
 
@@ -23,7 +24,7 @@
 from tensorrt_llm.executor import CppExecutorError
 from tensorrt_llm.logger import logger
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 # TODO: support disaggregated as well

examples/trtllm/common/kv_cache_event_publisher.py

@@ -14,11 +14,12 @@
 # limitations under the License.
 
 import ctypes
+import os
 from ctypes import c_char_p, c_int64, c_uint32
 
 from tensorrt_llm.logger import logger
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class DynamoResult:

examples/trtllm/common/kv_router.py

@@ -15,6 +15,7 @@
 
 
 import enum
+import os
 import random
 import traceback
 from typing import AsyncIterator
@@ -25,7 +26,7 @@
 from dynamo.llm import AggregatedMetrics, KvIndexer, KvMetricsAggregator, OverlapScores
 from dynamo.runtime import dynamo_endpoint
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class RoutingStrategy(enum.Enum):

examples/trtllm/common/processor.py

@@ -15,6 +15,7 @@
 
 import asyncio
 import json
+import os
 import time
 from typing import (
     Any,
@@ -51,7 +52,7 @@
 )
 from transformers import AutoTokenizer
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class ConversationMessage(TypedDict):

examples/trtllm/common/utils.py

@@ -15,6 +15,7 @@
 
 
 import asyncio
+import os
 import threading
 import traceback
 import weakref
@@ -23,7 +24,7 @@
 
 from tensorrt_llm.logger import logger
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class ManagedThread(threading.Thread):

examples/trtllm/disaggregated/server.py

@@ -16,6 +16,7 @@
 import asyncio
 import copy
 import json
+import os
 
 import uvloop
 from common.base_engine import ChatProcessorMixin
@@ -32,7 +33,7 @@
 from dynamo.llm import KvIndexer, KvMetricsAggregator
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class DisaggServer(ChatProcessorMixin):

examples/trtllm/disaggregated/worker.py

@@ -49,7 +49,7 @@
 from dynamo.llm import KvMetricsPublisher
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 def update_args_from_disagg_config(

examples/trtllm/monolith/dynamo_engine.py

@@ -19,6 +19,7 @@
 - This is only supposed to be used by dynamo-run launcher.
 - It is part of bring-your-own-engine python feature in dynamo-run.
 """
+import os
 import sys
 from pathlib import Path
 
@@ -40,7 +41,7 @@
 from common.parser import parse_dynamo_run_args  # noqa: E402
 from common.protocol import AdaptedChatCompletionRequest  # noqa: E402
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class DynamoTRTLLMEngine(BaseTensorrtLLMEngine):

examples/trtllm/monolith/router.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import asyncio
+import os
 
 import uvloop
 from common.base_engine import ChatProcessorMixin
@@ -31,7 +32,7 @@
 from dynamo.llm import KvIndexer, KvMetricsAggregator
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class Router(ChatProcessorMixin):

examples/trtllm/monolith/worker.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import asyncio
+import os
 
 import uvloop
 from common.base_engine import BaseTensorrtLLMEngine, TensorrtLLMEngineConfig
@@ -29,7 +30,7 @@
 from dynamo.llm import KvMetricsPublisher
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level("debug")
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class TensorrtLLMEngine(BaseTensorrtLLMEngine):