[LoRA] Remove tensorizer

docs/models/extensions/tensorizer.md

@@ -32,25 +32,15 @@ python examples/features/tensorize_vllm_model.py \
    --suffix v1
 ```
 
-This saves the model tensors at `s3://my-bucket/vllm/facebook/opt-125m/v1`. If you intend on applying a LoRA adapter to your tensorized model, you can pass the HF id of the LoRA adapter in the above command, and the artifacts will be saved there too:
-
-```bash
-python examples/features/tensorize_vllm_model.py \
-   --model facebook/opt-125m \
-   --lora-path <lora_id> \
-   serialize \
-   --serialized-directory s3://my-bucket \
-   --suffix v1
-```
+This saves the model tensors at `s3://my-bucket/vllm/facebook/opt-125m/v1`.
 
 ## Serving the model using Tensorizer
 
-Once the model is serialized where you want it, you can load the model using `vllm serve` or the `LLM` entrypoint. You can pass the directory where you saved the model to the `model` argument for `LLM()` and `vllm serve`. For example, to serve the tensorized model saved previously with the LoRA adapter, you'd do:
+Once the model is serialized where you want it, you can load the model using `vllm serve` or the `LLM` entrypoint. You can pass the directory where you saved the model to the `model` argument for `LLM()` and `vllm serve`. For example, to serve the tensorized model saved previously, you'd do:
 
 ```bash
 vllm serve s3://my-bucket/vllm/facebook/opt-125m/v1 \
-    --load-format tensorizer \
-    --enable-lora 
+    --load-format tensorizer
 ```
 
 Or, with `LLM()`:
@@ -60,7 +50,6 @@ from vllm import LLM
 llm = LLM(
     "s3://my-bucket/vllm/facebook/opt-125m/v1", 
     load_format="tensorizer",
-    enable_lora=True,
 )
 ```
 
@@ -73,7 +62,6 @@ As an example, CPU concurrency can be limited when serializing with `tensorizer`
 ```bash
 python examples/features/tensorize_vllm_model.py \
    --model facebook/opt-125m \
-   --lora-path <lora_id> \
    serialize \
    --serialized-directory s3://my-bucket \
    --serialization-kwargs '{"limit_cpu_concurrency": 2}' \
@@ -85,7 +73,6 @@ As an example when customizing the loading process via `TensorDeserializer`, you
 ```bash
 vllm serve s3://my-bucket/vllm/facebook/opt-125m/v1 \
     --load-format tensorizer \
-    --enable-lora \
     --model-loader-extra-config '{"deserialization_kwargs": {"num_readers": 2}}'
 ```
 
@@ -96,7 +83,6 @@ from vllm import LLM
 llm = LLM(
     "s3://my-bucket/vllm/facebook/opt-125m/v1", 
     load_format="tensorizer",
-    enable_lora=True,
     model_loader_extra_config={"deserialization_kwargs": {"num_readers": 2}},
 )
 ```

examples/features/tensorize_vllm_model.py

@@ -6,13 +6,11 @@
 import os
 import uuid
 
-from vllm import LLM, SamplingParams
+from vllm import LLM
 from vllm.engine.arg_utils import EngineArgs
-from vllm.lora.request import LoRARequest
 from vllm.model_executor.model_loader.tensorizer import (
     TensorizerArgs,
     TensorizerConfig,
-    tensorize_lora_adapter,
     tensorize_vllm_model,
     tensorizer_kwargs_arg,
 )
@@ -105,21 +103,6 @@
 under the `tensorizer options` section. These can also be used for
 deserialization in this example script, although `--tensorizer-uri` and
 `--path-to-tensors` are functionally the same in this case.
-
-Tensorizer can also be used to save and load LoRA adapters. A LoRA adapter
-can be serialized directly with the path to the LoRA adapter on HF Hub and
-a TensorizerConfig object. In this script, passing a HF id to a LoRA adapter
-will serialize the LoRA adapter artifacts to `--serialized-directory`.
-
-You can then use the LoRA adapter with `vllm serve`, for instance, by ensuring 
-the LoRA artifacts are in your model artifacts directory and specifying 
-`--enable-lora`. For instance:
-
-```
-vllm serve s3://my-bucket/vllm/facebook/opt-125m/v1 \
-    --load-format tensorizer \
-    --enable-lora 
-```
 """
 
 
@@ -134,18 +117,6 @@ def get_parser():
     )
     parser = EngineArgs.add_cli_args(parser)
 
-    parser.add_argument(
-        "--lora-path",
-        type=str,
-        required=False,
-        help="Path to a LoRA adapter to "
-        "serialize along with model tensors. This can then be deserialized "
-        "along with the model by instantiating a TensorizerConfig object, "
-        "creating a dict from it with TensorizerConfig.to_serializable(), "
-        "and passing it to LoRARequest's initializer with the kwarg "
-        "tensorizer_config_dict.",
-    )
-
     subparsers = parser.add_subparsers(dest="command", required=True)
 
     serialize_parser = subparsers.add_parser(
@@ -263,42 +234,12 @@ def merge_extra_config_with_tensorizer_config(extra_cfg: dict, cfg: TensorizerCo
 
 
 def deserialize(args, tensorizer_config):
-    if args.lora_path:
-        tensorizer_config.lora_dir = tensorizer_config.tensorizer_dir
-        llm = LLM(
-            model=args.model,
-            load_format="tensorizer",
-            tensor_parallel_size=args.tensor_parallel_size,
-            model_loader_extra_config=tensorizer_config,
-            enable_lora=True,
-        )
-        sampling_params = SamplingParams(
-            temperature=0, max_tokens=256, stop=["[/assistant]"]
-        )
-
-        # Truncating this as the extra text isn't necessary
-        prompts = ["[user] Write a SQL query to answer the question based on ..."]
-
-        # Test LoRA load
-        print(
-            llm.generate(
-                prompts,
-                sampling_params,
-                lora_request=LoRARequest(
-                    "sql-lora",
-                    1,
-                    args.lora_path,
-                    tensorizer_config_dict=tensorizer_config.to_serializable(),
-                ),
-            )
-        )
-    else:
-        llm = LLM(
-            model=args.model,
-            load_format="tensorizer",
-            tensor_parallel_size=args.tensor_parallel_size,
-            model_loader_extra_config=tensorizer_config,
-        )
+    llm = LLM(
+        model=args.model,
+        load_format="tensorizer",
+        tensor_parallel_size=args.tensor_parallel_size,
+        model_loader_extra_config=tensorizer_config,
+    )
     return llm
 
 
@@ -366,10 +307,6 @@ def main():
             **credentials,
         )
 
-        if args.lora_path:
-            tensorizer_config.lora_dir = tensorizer_config.tensorizer_dir
-            tensorize_lora_adapter(args.lora_path, tensorizer_config)
-
         merge_extra_config_with_tensorizer_config(extra_config, tensorizer_config)
         tensorize_vllm_model(engine_args, tensorizer_config)
 

tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py

@@ -13,13 +13,11 @@
 from vllm.engine.arg_utils import EngineArgs
 from vllm.model_executor.model_loader.tensorizer import (
     TensorizerConfig,
-    tensorize_lora_adapter,
     tensorize_vllm_model,
 )
 from vllm.platforms import current_platform
 
 MODEL_NAME = "unsloth/llama-3.2-1b-Instruct"
-LORA_PATH = "davzoku/finqa_adapter_1b"
 
 
 def _cleanup():
@@ -44,11 +42,10 @@ def model_uri(tmp_dir):
 
 
 @pytest.fixture(scope="module")
-def tensorize_model_and_lora(tmp_dir, model_uri):
-    tensorizer_config = TensorizerConfig(tensorizer_uri=model_uri, lora_dir=tmp_dir)
+def tensorize_model(model_uri):
+    tensorizer_config = TensorizerConfig(tensorizer_uri=model_uri)
     args = EngineArgs(model=MODEL_NAME)
 
-    tensorize_lora_adapter(LORA_PATH, tensorizer_config)
     tensorize_vllm_model(args, tensorizer_config)
 
     # Manually invoke a _cleanup() here, as the cleanup()
@@ -59,7 +56,7 @@ def tensorize_model_and_lora(tmp_dir, model_uri):
 
 
 @pytest.fixture(scope="module")
-def server(model_uri, tensorize_model_and_lora):
+def server(model_uri, tensorize_model):
     # In this case, model_uri is a directory with a model.tensors
     # file and all necessary model artifacts, particularly a
     # HF `config.json` file. In this case, Tensorizer can infer the
@@ -72,7 +69,6 @@ def server(model_uri, tensorize_model_and_lora):
         "tensorizer",
         "--served-model-name",
         MODEL_NAME,
-        "--enable-lora",
     ]
     if current_platform.is_rocm():
         args += ["--attention-backend", "TRITON_ATTN"]

tests/lora/test_llama_tp.py

@@ -1,18 +1,14 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-import subprocess
-import sys
 
 import pytest
 
 import vllm
 import vllm.config
-from vllm import LLM
 from vllm.lora.request import LoRARequest
-from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
 from vllm.platforms import current_platform
 
-from ..utils import VLLM_PATH, create_new_process_for_each_test, multi_gpu_test
+from ..utils import create_new_process_for_each_test, multi_gpu_test
 
 PROMPT_TEMPLATE = """<|eot_id|><|start_header_id|>user<|end_header_id|>
 I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.
@@ -40,7 +36,6 @@ def do_sample(
     llm: vllm.LLM,
     lora_path: str,
     lora_id: int,
-    tensorizer_config_dict: dict | None = None,
 ) -> list[str]:
     prompts = [
         PROMPT_TEMPLATE.format(context="How many candidates are there?"),
@@ -56,27 +51,11 @@ def do_sample(
     sampling_params = vllm.SamplingParams(
         temperature=0, max_tokens=64, stop=["<|im_end|>"]
     )
-    if tensorizer_config_dict is not None:
-        outputs = llm.generate(
-            prompts,
-            sampling_params,
-            lora_request=LoRARequest(
-                str(lora_id),
-                lora_id,
-                lora_path,
-                tensorizer_config_dict=tensorizer_config_dict,
-            )
-            if lora_id
-            else None,
-        )
-    else:
-        outputs = llm.generate(
-            prompts,
-            sampling_params,
-            lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
-            if lora_id
-            else None,
-        )
+    outputs = llm.generate(
+        prompts,
+        sampling_params,
+        lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
+    )
     lora_request = LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None
     generated_texts: list[str] = []
     for output in outputs:
@@ -94,31 +73,13 @@ def do_sample(
     return generated_texts
 
 
-def generate_and_test(
-    llm, llama32_lora_files, tensorizer_config_dict: dict | None = None
-):
+def generate_and_test(llm, llama32_lora_files):
     print("lora adapter created")
     print("lora 1")
-    assert (
-        do_sample(
-            llm,
-            llama32_lora_files,
-            tensorizer_config_dict=tensorizer_config_dict,
-            lora_id=1,
-        )
-        == EXPECTED_LORA_OUTPUT
-    )
+    assert do_sample(llm, llama32_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
 
     print("lora 2")
-    assert (
-        do_sample(
-            llm,
-            llama32_lora_files,
-            tensorizer_config_dict=tensorizer_config_dict,
-            lora_id=2,
-        )
-        == EXPECTED_LORA_OUTPUT
-    )
+    assert do_sample(llm, llama32_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
 
     print("removing lora")
 
@@ -168,79 +129,3 @@ def test_llama_lora_tp4_fully_sharded_loras(llama32_lora_files):
         fully_sharded_loras=True,
     )
     generate_and_test(llm, llama32_lora_files)
-
-
-@multi_gpu_test(num_gpus=2)
-def test_tp2_serialize_and_deserialize_lora(
-    tmp_path,
-    llama32_lora_files,
-):
-    # Run the tensorizing of the LoRA adapter and the model in a subprocess
-    # to guarantee cleanup
-
-    tp_size = 2
-    model_name = "model-rank-%03d.tensors"
-
-    model_ref = MODEL_PATH
-    lora_path = llama32_lora_files
-    suffix = "test"
-    try:
-        result = subprocess.run(
-            [
-                sys.executable,
-                f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py",
-                "--model",
-                MODEL_PATH,
-                "--lora-path",
-                lora_path,
-                "--tensor-parallel-size",
-                str(tp_size),
-                "serialize",
-                "--serialized-directory",
-                str(tmp_path),
-                "--suffix",
-                suffix,
-                "--serialization-kwargs",
-                '{"limit_cpu_concurrency": 4}',
-            ],
-            check=True,
-            capture_output=True,
-            text=True,
-        )
-    except subprocess.CalledProcessError as e:
-        print("Tensorizing failed.")
-        print("STDOUT:\n", e.stdout)
-        print("STDERR:\n", e.stderr)
-        raise
-
-    print("STDOUT:\n", result.stdout)
-
-    model_uri = tmp_path / "vllm" / model_ref / suffix / model_name
-    tensorizer_config = TensorizerConfig(tensorizer_uri=str(model_uri))
-
-    loaded_llm = LLM(
-        model=model_ref,
-        load_format="tensorizer",
-        enable_lora=True,
-        enforce_eager=True,
-        model_loader_extra_config=tensorizer_config,
-        max_num_seqs=7,
-        max_model_len=1024,
-        tensor_parallel_size=2,
-        max_loras=2,
-        # Leave headroom for LoRA adapter loading and Triton JIT
-        # compilation, which can allocate GPU memory concurrently
-        # during the first inference step.
-        gpu_memory_utilization=0.85,
-    )
-
-    tc_as_dict = tensorizer_config.to_serializable()
-
-    print("lora adapter created")
-    print("lora 1")
-    assert (
-        do_sample(
-            loaded_llm, llama32_lora_files, tensorizer_config_dict=tc_as_dict, lora_id=1
-        )
-        == EXPECTED_LORA_OUTPUT
-    )