From 70e326f8687fedcffa405adddb828342536a801f Mon Sep 17 00:00:00 2001 From: Um Changyong Date: Wed, 6 Nov 2024 08:56:29 +0900 Subject: [PATCH] update: modify parameter for the LoRA adapter on the vllm page --- docs/docs/integrations/llms/vllm.ipynb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/docs/integrations/llms/vllm.ipynb b/docs/docs/integrations/llms/vllm.ipynb index 1e1baff963161..e68ca9525ad73 100644 --- a/docs/docs/integrations/llms/vllm.ipynb +++ b/docs/docs/integrations/llms/vllm.ipynb @@ -266,8 +266,18 @@ "from langchain_community.llms import VLLM\n", "from vllm.lora.request import LoRARequest\n", "\n", - "llm = VLLM(model=\"meta-llama/Llama-2-7b-hf\", enable_lora=True)\n", - "\n", + "llm = VLLM(\n", + " model=\"meta-llama/Llama-3.2-3B-Instruct\",\n", + " max_new_tokens=300,\n", + " top_k=1,\n", + " top_p=0.90,\n", + " temperature=0.1,\n", + " vllm_kwargs={\n", + " \"gpu_memory_utilization\": 0.5,\n", + " \"enable_lora\": True,\n", + " \"max_model_len\": 350,\n", + " },\n", + ")\n", "LoRA_ADAPTER_PATH = \"path/to/adapter\"\n", "lora_adapter = LoRARequest(\"lora_adapter\", 1, LoRA_ADAPTER_PATH)\n", "\n",