## usage

### start

```bash
export VLLM_WORKER_MULTIPROC_METHOD=spawn
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python3 -m fastchat.serve.vllm_worker \
    --model-path /data/models/Qwen/Qwen2-72B-Instruct \
    --tokenizer /data/dpo/lora/b15s1/saves/Qwen2-72B-Instruct/v7.9/v7.3 \
    --enable-lora \
    --lora-modules m1=/data/modules/lora/adapter/m1 m2=/data/modules/lora/adapter/m2 m3=/data/modules/lora/adapter/m3 \
    --model-names qwen2-72b-instruct,m1,m2,m3\
    --controller http://localhost:21001 \
    --host 0.0.0.0 \
    --num-gpus 8 \
    --port 31034 \
    --limit-worker-concurrency 100 \
    --worker-address http://localhost:31034
```

### post

- example1

```bash
curl --location --request POST 'http://llm-gw.sunlinecloud.cn/v1/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-xxx' \
--data-raw '{
    "model": "m1",
    "stream": false,
    "temperature": 0.7,
    "top_p": 0.1,
    "max_tokens": 4096,
    "messages": [
      {
        "role": "user",
        "content": "Hi?"
      }
    ]
  }'
```
- example2

```bash
curl --location --request POST 'http://llm-gw.sunlinecloud.cn/v1/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-xxx' \
--data-raw '{
    "model": "qwen2-72b-instruct",
    "stream": false,
    "temperature": 0.7,
    "top_p": 0.1,
    "max_tokens": 4096,
    "messages": [
      {
        "role": "user",
        "content": "Hi?"
      }
    ]
  }'
```

## usage

### start

```bash
export VLLM_WORKER_MULTIPROC_METHOD=spawn
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python3 -m fastchat.serve.vllm_worker \
    --model-path /data/models/Qwen/Qwen2-72B-Instruct \
    --tokenizer /data/dpo/lora/b15s1/saves/Qwen2-72B-Instruct/v7.9/v7.3 \
    --enable-lora \
    --lora-modules m1=/data/modules/lora/adapter/m1 m2=/data/modules/lora/adapter/m2 m3=/data/modules/lora/adapter/m3 \
    --model-names qwen2-72b-instruct,m1,m2,m3\
    --controller http://localhost:21001 \
    --host 0.0.0.0 \
    --num-gpus 8 \
    --port 31034 \
    --limit-worker-concurrency 100 \
    --worker-address http://localhost:31034
```

### post

- example1

```bash
curl --location --request POST 'http://llm-gw.sunlinecloud.cn/v1/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-xxx' \
--data-raw '{
    "model": "m1",
    "stream": false,
    "temperature": 0.7,
    "top_p": 0.1,
    "max_tokens": 4096,
    "messages": [
      {
        "role": "user",
        "content": "Hi?"
      }
    ]
  }'
```
- example2

```bash
curl --location --request POST 'http://llm-gw.sunlinecloud.cn/v1/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-xxx' \
--data-raw '{
    "model": "qwen2-72b-instruct",
    "stream": false,
    "temperature": 0.7,
    "top_p": 0.1,
    "max_tokens": 4096,
    "messages": [
      {
        "role": "user",
        "content": "Hi?"
      }
    ]
  }'
```