From b7cff829b209d4d2b2e19f073bdb6d6a27c6bb9d Mon Sep 17 00:00:00 2001
From: Jacky <18255193+kthui@users.noreply.github.com>
Date: Fri, 21 Mar 2025 19:32:19 -0700
Subject: [PATCH 1/3] fix: Avoid heavy logging by TensorRT-LLM monolith worker

---
 examples/trtllm/monolith/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/trtllm/monolith/worker.py b/examples/trtllm/monolith/worker.py
index 0103a3b929d6..f51c0003466b 100644
--- a/examples/trtllm/monolith/worker.py
+++ b/examples/trtllm/monolith/worker.py
@@ -29,7 +29,7 @@
 from dynamo.llm import KvMetricsPublisher
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level("debug")
+logger.set_level("info")
 
 
 class TensorrtLLMEngine(BaseTensorrtLLMEngine):

From 17d6491b0053b9ea1ccc04e9f0b4cac27c19b930 Mon Sep 17 00:00:00 2001
From: Jacky <18255193+kthui@users.noreply.github.com>
Date: Tue, 25 Mar 2025 09:33:49 -0700
Subject: [PATCH 2/3] Update TRT-LLM example to read log level from env var if
 set

---
 examples/trtllm/common/base_engine.py              | 3 ++-
 examples/trtllm/common/generators.py               | 3 ++-
 examples/trtllm/common/kv_cache_event_publisher.py | 3 ++-
 examples/trtllm/common/kv_router.py                | 3 ++-
 examples/trtllm/common/processor.py                | 3 ++-
 examples/trtllm/common/utils.py                    | 3 ++-
 examples/trtllm/disaggregated/server.py            | 3 ++-
 examples/trtllm/disaggregated/worker.py            | 2 +-
 examples/trtllm/monolith/dynamo_engine.py          | 3 ++-
 examples/trtllm/monolith/router.py                 | 3 ++-
 examples/trtllm/monolith/worker.py                 | 3 ++-
 11 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/examples/trtllm/common/base_engine.py b/examples/trtllm/common/base_engine.py
index b530cb5f9975..a94a0aa51b64 100644
--- a/examples/trtllm/common/base_engine.py
+++ b/examples/trtllm/common/base_engine.py
@@ -15,6 +15,7 @@
 
 
 import asyncio
+import os
 import threading
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
@@ -32,7 +33,7 @@
 
 from .kv_cache_event_publisher import KVCacheEventPublisher
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
 
 
 class ChatProcessorMixin:
diff --git a/examples/trtllm/common/generators.py b/examples/trtllm/common/generators.py
index fbb478793acb..dd7c44a6020b 100644
--- a/examples/trtllm/common/generators.py
+++ b/examples/trtllm/common/generators.py
@@ -15,6 +15,7 @@
 
 
 import json
+import os
 import signal
 import uuid
 
@@ -23,7 +24,7 @@
 from tensorrt_llm.executor import CppExecutorError
 from tensorrt_llm.logger import logger
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
 
 
 # TODO: support disaggregated as well
diff --git a/examples/trtllm/common/kv_cache_event_publisher.py b/examples/trtllm/common/kv_cache_event_publisher.py
index 27d175d5477b..771de7921a1a 100644
--- a/examples/trtllm/common/kv_cache_event_publisher.py
+++ b/examples/trtllm/common/kv_cache_event_publisher.py
@@ -14,11 +14,12 @@
 # limitations under the License.
 
 import ctypes
+import os
 from ctypes import c_char_p, c_int64, c_uint32
 
 from tensorrt_llm.logger import logger
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
 
 
 class DynamoResult:
diff --git a/examples/trtllm/common/kv_router.py b/examples/trtllm/common/kv_router.py
index f94c751f38bf..d9ddb243a163 100644
--- a/examples/trtllm/common/kv_router.py
+++ b/examples/trtllm/common/kv_router.py
@@ -15,6 +15,7 @@
 
 
 import enum
+import os
 import random
 import traceback
 from typing import AsyncIterator
@@ -25,7 +26,7 @@
 from dynamo.llm import AggregatedMetrics, KvIndexer, KvMetricsAggregator, OverlapScores
 from dynamo.runtime import dynamo_endpoint
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
 
 
 class RoutingStrategy(enum.Enum):
diff --git a/examples/trtllm/common/processor.py b/examples/trtllm/common/processor.py
index a6b7d863d4d6..69a272f3969e 100644
--- a/examples/trtllm/common/processor.py
+++ b/examples/trtllm/common/processor.py
@@ -15,6 +15,7 @@
 
 import asyncio
 import json
+import os
 import time
 from typing import (
     Any,
@@ -51,7 +52,7 @@
 )
 from transformers import AutoTokenizer
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
 
 
 class ConversationMessage(TypedDict):
diff --git a/examples/trtllm/common/utils.py b/examples/trtllm/common/utils.py
index b0d0168cbfc9..a03a1d0128bb 100644
--- a/examples/trtllm/common/utils.py
+++ b/examples/trtllm/common/utils.py
@@ -15,6 +15,7 @@
 
 
 import asyncio
+import os
 import threading
 import traceback
 import weakref
@@ -23,7 +24,7 @@
 
 from tensorrt_llm.logger import logger
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
 
 
 class ManagedThread(threading.Thread):
diff --git a/examples/trtllm/disaggregated/server.py b/examples/trtllm/disaggregated/server.py
index 14189985de21..66b00050dceb 100644
--- a/examples/trtllm/disaggregated/server.py
+++ b/examples/trtllm/disaggregated/server.py
@@ -16,6 +16,7 @@
 import asyncio
 import copy
 import json
+import os
 
 import uvloop
 from common.base_engine import ChatProcessorMixin
@@ -32,7 +33,7 @@
 from dynamo.llm import KvIndexer, KvMetricsAggregator
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
 
 
 class DisaggServer(ChatProcessorMixin):
diff --git a/examples/trtllm/disaggregated/worker.py b/examples/trtllm/disaggregated/worker.py
index 355b72ce77b7..459d0c14e772 100644
--- a/examples/trtllm/disaggregated/worker.py
+++ b/examples/trtllm/disaggregated/worker.py
@@ -49,7 +49,7 @@
 from dynamo.llm import KvMetricsPublisher
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
 
 
 def update_args_from_disagg_config(
diff --git a/examples/trtllm/monolith/dynamo_engine.py b/examples/trtllm/monolith/dynamo_engine.py
index 273ae6482dad..911c8748291d 100644
--- a/examples/trtllm/monolith/dynamo_engine.py
+++ b/examples/trtllm/monolith/dynamo_engine.py
@@ -19,6 +19,7 @@
 - This is only supposed to be used by dynamo-run launcher.
 - It is part of bring-your-own-engine python feature in dynamo-run.
 """
+import os
 import sys
 from pathlib import Path
 
@@ -40,7 +41,7 @@
 from common.parser import parse_dynamo_run_args  # noqa: E402
 from common.protocol import AdaptedChatCompletionRequest  # noqa: E402
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
 
 
 class DynamoTRTLLMEngine(BaseTensorrtLLMEngine):
diff --git a/examples/trtllm/monolith/router.py b/examples/trtllm/monolith/router.py
index 7f31d1d021ae..dcbaa33a086e 100644
--- a/examples/trtllm/monolith/router.py
+++ b/examples/trtllm/monolith/router.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import asyncio
+import os
 
 import uvloop
 from common.base_engine import ChatProcessorMixin
@@ -31,7 +32,7 @@
 from dynamo.llm import KvIndexer, KvMetricsAggregator
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
 
 
 class Router(ChatProcessorMixin):
diff --git a/examples/trtllm/monolith/worker.py b/examples/trtllm/monolith/worker.py
index f51c0003466b..aed1f31b0683 100644
--- a/examples/trtllm/monolith/worker.py
+++ b/examples/trtllm/monolith/worker.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import asyncio
+import os
 
 import uvloop
 from common.base_engine import BaseTensorrtLLMEngine, TensorrtLLMEngineConfig
@@ -29,7 +30,7 @@
 from dynamo.llm import KvMetricsPublisher
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level("info")
+logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
 
 
 class TensorrtLLMEngine(BaseTensorrtLLMEngine):

From ea74f83ba0dfe2b2bf82ba8e0ec5eaab8470f22a Mon Sep 17 00:00:00 2001
From: Jacky <18255193+kthui@users.noreply.github.com>
Date: Tue, 25 Mar 2025 09:57:45 -0700
Subject: [PATCH 3/3] Align env var prefix with others

---
 examples/trtllm/common/base_engine.py              | 2 +-
 examples/trtllm/common/generators.py               | 2 +-
 examples/trtllm/common/kv_cache_event_publisher.py | 2 +-
 examples/trtllm/common/kv_router.py                | 2 +-
 examples/trtllm/common/processor.py                | 2 +-
 examples/trtllm/common/utils.py                    | 2 +-
 examples/trtllm/disaggregated/server.py            | 2 +-
 examples/trtllm/disaggregated/worker.py            | 2 +-
 examples/trtllm/monolith/dynamo_engine.py          | 2 +-
 examples/trtllm/monolith/router.py                 | 2 +-
 examples/trtllm/monolith/worker.py                 | 2 +-
 11 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/examples/trtllm/common/base_engine.py b/examples/trtllm/common/base_engine.py
index a94a0aa51b64..55a5c40b551d 100644
--- a/examples/trtllm/common/base_engine.py
+++ b/examples/trtllm/common/base_engine.py
@@ -33,7 +33,7 @@
 
 from .kv_cache_event_publisher import KVCacheEventPublisher
 
-logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class ChatProcessorMixin:
diff --git a/examples/trtllm/common/generators.py b/examples/trtllm/common/generators.py
index dd7c44a6020b..141da02581a5 100644
--- a/examples/trtllm/common/generators.py
+++ b/examples/trtllm/common/generators.py
@@ -24,7 +24,7 @@
 from tensorrt_llm.executor import CppExecutorError
 from tensorrt_llm.logger import logger
 
-logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 # TODO: support disaggregated as well
diff --git a/examples/trtllm/common/kv_cache_event_publisher.py b/examples/trtllm/common/kv_cache_event_publisher.py
index 771de7921a1a..3e184cf42c24 100644
--- a/examples/trtllm/common/kv_cache_event_publisher.py
+++ b/examples/trtllm/common/kv_cache_event_publisher.py
@@ -19,7 +19,7 @@
 
 from tensorrt_llm.logger import logger
 
-logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class DynamoResult:
diff --git a/examples/trtllm/common/kv_router.py b/examples/trtllm/common/kv_router.py
index d9ddb243a163..adab960de03d 100644
--- a/examples/trtllm/common/kv_router.py
+++ b/examples/trtllm/common/kv_router.py
@@ -26,7 +26,7 @@
 from dynamo.llm import AggregatedMetrics, KvIndexer, KvMetricsAggregator, OverlapScores
 from dynamo.runtime import dynamo_endpoint
 
-logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class RoutingStrategy(enum.Enum):
diff --git a/examples/trtllm/common/processor.py b/examples/trtllm/common/processor.py
index 69a272f3969e..682209095829 100644
--- a/examples/trtllm/common/processor.py
+++ b/examples/trtllm/common/processor.py
@@ -52,7 +52,7 @@
 )
 from transformers import AutoTokenizer
 
-logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class ConversationMessage(TypedDict):
diff --git a/examples/trtllm/common/utils.py b/examples/trtllm/common/utils.py
index a03a1d0128bb..7ba6f268c2c7 100644
--- a/examples/trtllm/common/utils.py
+++ b/examples/trtllm/common/utils.py
@@ -24,7 +24,7 @@
 
 from tensorrt_llm.logger import logger
 
-logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class ManagedThread(threading.Thread):
diff --git a/examples/trtllm/disaggregated/server.py b/examples/trtllm/disaggregated/server.py
index 66b00050dceb..ceeb09add61c 100644
--- a/examples/trtllm/disaggregated/server.py
+++ b/examples/trtllm/disaggregated/server.py
@@ -33,7 +33,7 @@
 from dynamo.llm import KvIndexer, KvMetricsAggregator
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class DisaggServer(ChatProcessorMixin):
diff --git a/examples/trtllm/disaggregated/worker.py b/examples/trtllm/disaggregated/worker.py
index 459d0c14e772..7e6d6375df13 100644
--- a/examples/trtllm/disaggregated/worker.py
+++ b/examples/trtllm/disaggregated/worker.py
@@ -49,7 +49,7 @@
 from dynamo.llm import KvMetricsPublisher
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 def update_args_from_disagg_config(
diff --git a/examples/trtllm/monolith/dynamo_engine.py b/examples/trtllm/monolith/dynamo_engine.py
index 911c8748291d..3b36dbfdca49 100644
--- a/examples/trtllm/monolith/dynamo_engine.py
+++ b/examples/trtllm/monolith/dynamo_engine.py
@@ -41,7 +41,7 @@
 from common.parser import parse_dynamo_run_args  # noqa: E402
 from common.protocol import AdaptedChatCompletionRequest  # noqa: E402
 
-logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class DynamoTRTLLMEngine(BaseTensorrtLLMEngine):
diff --git a/examples/trtllm/monolith/router.py b/examples/trtllm/monolith/router.py
index dcbaa33a086e..26fa1f939278 100644
--- a/examples/trtllm/monolith/router.py
+++ b/examples/trtllm/monolith/router.py
@@ -32,7 +32,7 @@
 from dynamo.llm import KvIndexer, KvMetricsAggregator
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class Router(ChatProcessorMixin):
diff --git a/examples/trtllm/monolith/worker.py b/examples/trtllm/monolith/worker.py
index aed1f31b0683..87d8d3235e33 100644
--- a/examples/trtllm/monolith/worker.py
+++ b/examples/trtllm/monolith/worker.py
@@ -30,7 +30,7 @@
 from dynamo.llm import KvMetricsPublisher
 from dynamo.runtime import DistributedRuntime, dynamo_endpoint, dynamo_worker
 
-logger.set_level(os.getenv("DYNAMO_TRTLLM_LOG_LEVEL", "info"))
+logger.set_level(os.getenv("DYN_TRTLLM_LOG_LEVEL", "info"))
 
 
 class TensorrtLLMEngine(BaseTensorrtLLMEngine):