Mini testbench for various cases (service, model, num_device)

Author: opus24

examples/lpu_inference.py

@@ -0,0 +1,23 @@
+from vllm import LLM, SamplingParams
+
+# Sample prompts.
+prompts = [
+    "Hello, my name is"
+]
+# Create a sampling params object.
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95, top_k=1, min_tokens=30, max_tokens=30)
+
+# Create an LLM.
+#llm = LLM(model="facebook/opt-1.3b", device="fpga", pipeline_parallel_size=2)
+llm = LLM(model="meta-llama/Meta-Llama-3-8B", device="fpga", tensor_parallel_size=1)
+#llm = LLM(model="TinyLlama/TinyLlama-1.1B-Chat-v1.0", device="fpga", tensor_parallel_size=1)
+
+# Generate texts from the prompts. The output is a list of RequestOutput objects
+# that contain the prompt, generated text, and other information.
+outputs = llm.generate(prompts, sampling_params)
+
+# Print the outputs.
+for output in outputs:
+    prompt = output.prompt
+    generated_text = output.outputs[0].text
+    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")

examples/lpu_inference_arg.py

@@ -0,0 +1,24 @@
+from vllm import LLM, SamplingParams
+import argparse
+
+# Get arguments
+parser = argparse.ArgumentParser(description='vLLM Inference Test Script')
+parser.add_argument("-m", "--model", default="facebook/opt-1.3b", type=str, help="name of the language model")
+parser.add_argument("-n", "--ncore", default=1, type=int, help="the number of the LPU")
+parser.add_argument("-i", "--i_token", default="Hello, my name is", type=str, help="input prompt")
+parser.add_argument("-o", "--o_token", default=32, type=int, help="the number of output")
+args = parser.parse_args()
+
+# Sample prompts.
+prompts = [args.i_token]
+
+# Create a sampling params object and LLM
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95, top_k=1, max_tokens=args.o_token)
+llm = LLM(model=args.model, device="fpga", tensor_parallel_size=args.ncore)
+
+# Run and print the outputs.
+outputs = llm.generate(prompts, sampling_params)
+for output in outputs:
+    prompt = output.prompt
+    generated_text = output.outputs[0].text
+    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")

examples/mini_testbench.sh

@@ -0,0 +1,69 @@
+
+log_sum="log/service_model_device.txt"
+
+model_ids=("TinyLlama/TinyLlama-1.1B-Chat-v1.0") # "facebook/opt-1.3b" "huggyllama/llama-7b")
+num_devices=(1 2 4) 
+
+current_datetime=$(date "+%Y-%m-%d %H:%M:%S")
+echo "$current_datetime"
+echo "$current_datetime" >> ${log_sum}
+
+"""
+for model_id in "${model_ids[@]}"; do
+  for num_device in "${num_devices[@]}"; do
+    #IFS='\' read -ra parts <<< "$model_id"
+    #model_name="${parts[-1]}"
+    model_name=$(echo "$model_id" | awk -F'/' '{print $NF}')
+    echo "*********************************"
+    echo "**** Start inference_${model_name}_${num_device}"
+    echo "*********************************"
+    python lpu_inference_arg.py -m ${model_id} -n ${num_device} > log/inference_${model_name}_${num_device}.txt
+    echo "*********************************" >> ${log_sum}
+    echo "The Result of log/inference_${model_name}_${num_device}.txt" >> ${log_sum}
+    tail -n 1 "log/inference_${model_name}_${num_device}.txt" >> ${log_sum}
+    echo "" >> ${log_sum}
+  done
+done
+"""
+
+for model_id in "${model_ids[@]}"; do
+  for num_device in "${num_devices[@]}"; do
+    model_name=$(echo "$model_id" | awk -F'/' '{print $NF}')
+    echo "*********************************"
+    echo "**** Start serving_${model_name}_${num_device}"
+    echo "*********************************"
+    python -m vllm.entrypoints.api_server --model ${model_id} --device fpga --tensor-parallel-size ${num_device} &
+
+    # Waiting for server
+    while ! nc -z localhost "8000"; do  
+        echo "Waiting for server..."
+        sleep 3 
+    done
+    echo "The server is ready!"
+
+    python lpu_client.py > log/vllm_serve_${model_name}_${num_device}.txt
+
+    # Waiting for process kill
+    PID=$(jobs -p | tail -n 1)
+    if [ -n "$PID" ]; then
+        kill -SIGINT "$PID"
+        while true; do
+            if ps -p "$PID" > /dev/null; then
+                echo "Kill the process..."
+                sleep 3
+            else
+                echo "Process (PID: $PID) is killed."
+                break
+            fi
+        done
+    fi
+
+    # Write log in text file
+    echo "*********************************" >> ${log_sum}
+    echo "The Result of log/vllm_serve_${model_name}_${num_device}.txt" >> ${log_sum}
+    tail -n 1 "log/vllm_serve_${model_name}_${num_device}.txt" >> ${log_sum}
+    echo "" >> ${log_sum}
+  done
+done
+
+

scripts/install_script.sh

@@ -1,6 +1,7 @@
 
 site_packages=$(python3 -c "import site; print(site.getsitepackages()[0])")
 
+pip install numpy==1.26.0
 pip install vllm==0.5.5
 pip install mistral_common
 echo "Start move vllm to ${site_packages}"