From 63c0714835284f320b6aeff36449c41143a82024 Mon Sep 17 00:00:00 2001 From: Amit Zuker <203509407+amitz-nv@users.noreply.github.com> Date: Mon, 15 Sep 2025 15:31:28 +0000 Subject: [PATCH 1/3] Change examples/llm-api/llm_multilora.py to be able to receive the LoRA adapters dir paths through cmd line, change test_llmapi_example_multilora to pass these LoRA adapters path as cmd line args to avoid downloading from HF Signed-off-by: Amit Zuker <203509407+amitz-nv@users.noreply.github.com> --- examples/llm-api/llm_multilora.py | 40 ++++++++++++++----- .../defs/llmapi/test_llm_examples.py | 11 ++++- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/examples/llm-api/llm_multilora.py b/examples/llm-api/llm_multilora.py index 0c6fa4f5417..a9b56319ec6 100644 --- a/examples/llm-api/llm_multilora.py +++ b/examples/llm-api/llm_multilora.py @@ -1,6 +1,7 @@ ### :section Customization ### :title Generate text with multiple LoRA adapters ### :order 5 +import click from huggingface_hub import snapshot_download from tensorrt_llm import LLM @@ -8,17 +9,37 @@ from tensorrt_llm.lora_helper import LoraConfig -def main(): +@click.command() +@click.option("--chatbot_lora_dir", + type=str, + default=None, + help="Path to the chatbot LoRA directory") +@click.option("--mental_health_lora_dir", + type=str, + default=None, + help="Path to the mental health LoRA directory") +@click.option("--tarot_lora_dir", + type=str, + default=None, + help="Path to the tarot LoRA directory") +def main(chatbot_lora_dir: str | None, mental_health_lora_dir: str | None, + tarot_lora_dir: str | None): # Download the LoRA adapters from huggingface hub. - lora_dir1 = snapshot_download(repo_id="snshrivas10/sft-tiny-chatbot") - lora_dir2 = snapshot_download( - repo_id="givyboy/TinyLlama-1.1B-Chat-v1.0-mental-health-conversational") - lora_dir3 = snapshot_download(repo_id="barissglc/tinyllama-tarot-v1") + if chatbot_lora_dir is None: + chatbot_lora_dir = snapshot_download( + repo_id="snshrivas10/sft-tiny-chatbot") + if mental_health_lora_dir is None: + mental_health_lora_dir = snapshot_download( + repo_id= + "givyboy/TinyLlama-1.1B-Chat-v1.0-mental-health-conversational") + if tarot_lora_dir is None: + tarot_lora_dir = snapshot_download( + repo_id="barissglc/tinyllama-tarot-v1") # Currently, we need to pass at least one lora_dir to LLM constructor via build_config.lora_config. # This is necessary because it requires some configuration in the lora_dir to build the engine with LoRA support. - lora_config = LoraConfig(lora_dir=[lora_dir1], + lora_config = LoraConfig(lora_dir=[chatbot_lora_dir], max_lora_rank=64, max_loras=3, max_cpu_loras=3) @@ -39,10 +60,11 @@ def main(): for output in llm.generate(prompts, lora_request=[ None, - LoRARequest("chatbot", 1, lora_dir1), None, - LoRARequest("mental-health", 2, lora_dir2), + LoRARequest("chatbot", 1, chatbot_lora_dir), None, - LoRARequest("tarot", 3, lora_dir3) + LoRARequest("mental-health", 2, + mental_health_lora_dir), None, + LoRARequest("tarot", 3, tarot_lora_dir) ]): prompt = output.prompt generated_text = output.outputs[0].text diff --git a/tests/integration/defs/llmapi/test_llm_examples.py b/tests/integration/defs/llmapi/test_llm_examples.py index 1935acda092..f06c153b3b6 100644 --- a/tests/integration/defs/llmapi/test_llm_examples.py +++ b/tests/integration/defs/llmapi/test_llm_examples.py @@ -110,7 +110,16 @@ def test_llmapi_example_inference_async_streaming(llm_root, engine_dir, def test_llmapi_example_multilora(llm_root, engine_dir, llm_venv): - _run_llmapi_example(llm_root, engine_dir, llm_venv, "llm_multilora.py") + cmd_line_args = [ + "--chatbot_lora_dir", + f"{llm_models_root()}/llama-models-v2/sft-tiny-chatbot", + "--mental_health_lora_dir", + f"{llm_models_root()}/llama-models-v2/TinyLlama-1.1B-Chat-v1.0-mental-health-conversational", + "--tarot_lora_dir", + f"{llm_models_root()}/llama-models-v2/tinyllama-tarot-v1" + ] + _run_llmapi_example(llm_root, engine_dir, llm_venv, "llm_multilora.py", + *cmd_line_args) def test_llmapi_example_guided_decoding(llm_root, engine_dir, llm_venv): From 7524a048448b73a217517de9a9eb850673f10880 Mon Sep 17 00:00:00 2001 From: Amit Zuker <203509407+amitz-nv@users.noreply.github.com> Date: Mon, 15 Sep 2025 15:31:28 +0000 Subject: [PATCH 2/3] Change type hint to be compatible with python 3.8 - use optional instead of | Signed-off-by: Amit Zuker <203509407+amitz-nv@users.noreply.github.com> --- examples/llm-api/llm_multilora.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/llm-api/llm_multilora.py b/examples/llm-api/llm_multilora.py index a9b56319ec6..8ed63e27314 100644 --- a/examples/llm-api/llm_multilora.py +++ b/examples/llm-api/llm_multilora.py @@ -1,6 +1,8 @@ ### :section Customization ### :title Generate text with multiple LoRA adapters ### :order 5 +from typing import Optional + import click from huggingface_hub import snapshot_download @@ -22,8 +24,8 @@ type=str, default=None, help="Path to the tarot LoRA directory") -def main(chatbot_lora_dir: str | None, mental_health_lora_dir: str | None, - tarot_lora_dir: str | None): +def main(chatbot_lora_dir: Optional[str], mental_health_lora_dir: Optional[str], + tarot_lora_dir: Optional[str]): # Download the LoRA adapters from huggingface hub. if chatbot_lora_dir is None: From 88620a1ab62677e891f7e0c2d2b217c4141e8d72 Mon Sep 17 00:00:00 2001 From: Amit Zuker <203509407+amitz-nv@users.noreply.github.com> Date: Mon, 15 Sep 2025 15:31:29 +0000 Subject: [PATCH 3/3] Use argparse instead of click in examples/llm-api/llm_multilora.py Signed-off-by: Amit Zuker <203509407+amitz-nv@users.noreply.github.com> --- examples/llm-api/llm_multilora.py | 36 +++++++++++++++++-------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/examples/llm-api/llm_multilora.py b/examples/llm-api/llm_multilora.py index 8ed63e27314..525f839d79d 100644 --- a/examples/llm-api/llm_multilora.py +++ b/examples/llm-api/llm_multilora.py @@ -1,9 +1,10 @@ ### :section Customization ### :title Generate text with multiple LoRA adapters ### :order 5 + +import argparse from typing import Optional -import click from huggingface_hub import snapshot_download from tensorrt_llm import LLM @@ -11,23 +12,10 @@ from tensorrt_llm.lora_helper import LoraConfig -@click.command() -@click.option("--chatbot_lora_dir", - type=str, - default=None, - help="Path to the chatbot LoRA directory") -@click.option("--mental_health_lora_dir", - type=str, - default=None, - help="Path to the mental health LoRA directory") -@click.option("--tarot_lora_dir", - type=str, - default=None, - help="Path to the tarot LoRA directory") def main(chatbot_lora_dir: Optional[str], mental_health_lora_dir: Optional[str], tarot_lora_dir: Optional[str]): - # Download the LoRA adapters from huggingface hub. + # Download the LoRA adapters from huggingface hub, if not provided via command line args. if chatbot_lora_dir is None: chatbot_lora_dir = snapshot_download( repo_id="snshrivas10/sft-tiny-chatbot") @@ -82,4 +70,20 @@ def main(chatbot_lora_dir: Optional[str], mental_health_lora_dir: Optional[str], if __name__ == '__main__': - main() + parser = argparse.ArgumentParser( + description="Generate text with multiple LoRA adapters") + parser.add_argument('--chatbot_lora_dir', + type=str, + default=None, + help='Path to the chatbot LoRA directory') + parser.add_argument('--mental_health_lora_dir', + type=str, + default=None, + help='Path to the mental health LoRA directory') + parser.add_argument('--tarot_lora_dir', + type=str, + default=None, + help='Path to the tarot LoRA directory') + args = parser.parse_args() + main(args.chatbot_lora_dir, args.mental_health_lora_dir, + args.tarot_lora_dir)