Skip to content

Commit 27fcccc

Browse files
CatherineSuesimon-mokey4ngupfixerchaunceyjiang
authored
Cherry-pick conflicts: features-based-on-v0.9.2 → features-based-on-v0.10.0 (#8)
* Revert "[V0 deprecation] Remove V0 CPU/XPU/TPU backends (#20412)" This reverts commit e202dd2. * Merge pull request #3 from moirai-internal/dev_container Add dev container Dockerfile and devcontainer.json * pick [Bugfix] Improve JSON extraction in LlamaToolParser * pick opc-request-id middleware * cherry pick the fix for non english token in logprobs (#7) * [Bugfix]: Fix messy code when using logprobs (#19209) Signed-off-by: chaunceyjiang <[email protected]> * [Bugfix]: Fix messy code when using logprobs (#20910) Signed-off-by: chaunceyjiang <[email protected]> * fix transformers compatible issue vllm-project/vllm-ascend#2046 --------- Signed-off-by: chaunceyjiang <[email protected]> Co-authored-by: Chauncey <[email protected]> * Merge pull request #3 from moirai-internal/dev_container Add dev container Dockerfile and devcontainer.json * pick [Bugfix] Improve JSON extraction in LlamaToolParser * pick opc-request-id middleware * ✅ Resolved: Cherry-pick from features-based-on-v0.9.2 to features-based-on-v0.10.0 * Revert "Revert "[V0 deprecation] Remove V0 CPU/XPU/TPU backends (#20412)"" This reverts commit a5dd03c. * remove fstring * Delete torch_sdpa.py --------- Signed-off-by: chaunceyjiang <[email protected]> Co-authored-by: simon-mo <[email protected]> Co-authored-by: Keyang Ru <[email protected]> Co-authored-by: Chao Yang <[email protected]> Co-authored-by: Chauncey <[email protected]> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Varun Shenoy <[email protected]> Co-authored-by: Tejesh Anand <[email protected]>
1 parent 6d8d0a2 commit 27fcccc

File tree

5 files changed

+468
-40
lines changed

5 files changed

+468
-40
lines changed

‎.devcontainer/devcontainer.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"name": "vllm",
3+
"build": {
4+
"dockerfile": "Dockerfile"
5+
},
6+
"remoteUser": "devuser",
7+
"customizations": {
8+
"vscode": {
9+
"extensions": [
10+
// Python development
11+
"ms-python.python",
12+
"charliermarsh.ruff",
13+
// Rust development
14+
"rust-lang.rust-analyzer",
15+
"tamasfe.even-better-toml"
16+
]
17+
}
18+
},
19+
"forwardPorts": [],
20+
"runArgs": [
21+
"--gpus",
22+
"all"
23+
],
24+
// The two lines below ensures that your local changes in the sglang
25+
// repo is automatically synced to the sglang pip package installed
26+
// in the dev docker container. You can remove / comment out these
27+
// two lines if you prefer to sync code changes manually.
28+
"workspaceMount": "source=${localWorkspaceFolder},target=/vllm-workspace/vllm,type=bind",
29+
"workspaceFolder": "/vllm-workspace/vllm"
30+
}

‎docker/Dockerfile.dev‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
# Extend from the base vllm image
2+
FROM vllm/vllm-openai:latest
3+
4+
# Override the base image's entrypoint with a shell
5+
ENTRYPOINT ["/bin/bash"]
6+
7+
# Install development tools and utilities
8+
RUN apt-get update && apt-get install -y \
9+
gdb \
10+
ninja-build \
11+
vim \
12+
tmux \
13+
htop \
14+
wget \
15+
curl \
16+
locales \
17+
lsof \
18+
git \
19+
git-lfs \
20+
zsh \
21+
tree \
22+
silversearcher-ag \
23+
cloc \
24+
unzip \
25+
pkg-config \
26+
libssl-dev \
27+
bear \
28+
ccache \
29+
&& apt install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \
30+
&& rm -rf /var/lib/apt/lists/* \
31+
&& apt-get clean
32+
33+
RUN apt update -y \
34+
&& apt install -y --no-install-recommends gnupg \
35+
&& echo "deb http://developer.download.nvidia.com/devtools/repos/ubuntu2004/amd64 /" | tee /etc/apt/sources.list.d/nvidia-devtools.list \
36+
&& apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub \
37+
&& apt update -y \
38+
&& apt install nsight-systems-cli -y
39+
40+
# Set up locale
41+
RUN locale-gen en_US.UTF-8
42+
ENV LANG en_US.UTF-8
43+
ENV LANGUAGE en_US:en
44+
ENV LC_ALL en_US.UTF-8
45+
46+
# Install minimal Python packages
47+
RUN python3 -m pip install --no-cache-dir \
48+
pytest \
49+
black \
50+
isort \
51+
icdiff \
52+
scikit_build_core \
53+
uv \
54+
pre-commit
55+
56+
# Install diff-so-fancy
57+
RUN curl -LSso /usr/local/bin/diff-so-fancy https://github.com/so-fancy/diff-so-fancy/releases/download/v1.4.4/diff-so-fancy \
58+
&& chmod +x /usr/local/bin/diff-so-fancy
59+
60+
# Install clang-format
61+
RUN curl -LSso /usr/local/bin/clang-format https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-32d3ac78/clang-format-16_linux-amd64 \
62+
&& chmod +x /usr/local/bin/clang-format
63+
64+
# Install clangd
65+
RUN curl -L https://github.com/clangd/clangd/releases/download/18.1.3/clangd-linux-18.1.3.zip -o clangd.zip \
66+
&& unzip clangd.zip \
67+
&& cp -r clangd_18.1.3/bin/* /usr/local/bin/ \
68+
&& cp -r clangd_18.1.3/lib/* /usr/local/lib/ \
69+
&& rm -rf clangd_18.1.3 clangd.zip
70+
71+
# Install CMake
72+
RUN wget https://github.com/Kitware/CMake/releases/download/v3.31.1/cmake-3.31.1-linux-x86_64.tar.gz \
73+
&& tar -xzf cmake-3.31.1-linux-x86_64.tar.gz \
74+
&& cp -r cmake-3.31.1-linux-x86_64/bin/* /usr/local/bin/ \
75+
&& cp -r cmake-3.31.1-linux-x86_64/share/* /usr/local/share/ \
76+
&& rm -rf cmake-3.31.1-linux-x86_64 cmake-3.31.1-linux-x86_64.tar.gz
77+
78+
# Add yank script
79+
RUN echo '#!/bin/bash' > /usr/local/bin/yank && \
80+
echo 'put() {' >> /usr/local/bin/yank && \
81+
echo ' esc=$1' >> /usr/local/bin/yank && \
82+
echo ' test -n "$TMUX" -o -z "${TERM##screen*}" && esc="\033Ptmux;\033$esc\033\\"' >> /usr/local/bin/yank && \
83+
echo ' printf "$esc"' >> /usr/local/bin/yank && \
84+
echo '}' >> /usr/local/bin/yank && \
85+
echo 'put "\033]52;c;!\a"' >> /usr/local/bin/yank && \
86+
echo 'buf=$( cat "$@" )' >> /usr/local/bin/yank && \
87+
echo 'len=$( printf %s "$buf" | wc -c ) max=74994' >> /usr/local/bin/yank && \
88+
echo 'test $len -gt $max && echo "$0: input is $(( len - max )) bytes too long" >&2' >> /usr/local/bin/yank && \
89+
echo 'put "\033]52;c;$( printf %s "$buf" | head -c $max | base64 | tr -d '\''\r\n'\'' )\a"' >> /usr/local/bin/yank && \
90+
echo 'test -n "$TMUX" && tmux set-buffer "$buf" ||:' >> /usr/local/bin/yank && \
91+
chmod +x /usr/local/bin/yank
92+
93+
# Install oh-my-zsh and plugins
94+
RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended \
95+
&& git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-autosuggestions \
96+
&& git clone https://github.com/zsh-users/zsh-syntax-highlighting.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-syntax-highlighting
97+
98+
# Configure Vim
99+
RUN echo 'function! Yank(text) abort' > /root/.vimrc && \
100+
echo ' let escape = system("yank", a:text)' >> /root/.vimrc && \
101+
echo ' if v:shell_error' >> /root/.vimrc && \
102+
echo ' echoerr escape' >> /root/.vimrc && \
103+
echo ' else' >> /root/.vimrc && \
104+
echo ' call writefile([escape], "/dev/tty", "b")' >> /root/.vimrc && \
105+
echo ' endif' >> /root/.vimrc && \
106+
echo 'endfunction' >> /root/.vimrc && \
107+
echo '' >> /root/.vimrc && \
108+
echo 'noremap <silent> <Leader>y y:<C-U>call Yank(@0)<CR>' >> /root/.vimrc && \
109+
echo '' >> /root/.vimrc && \
110+
echo '" automatically run yank(1) whenever yanking in Vim' >> /root/.vimrc && \
111+
echo 'function! CopyYank() abort' >> /root/.vimrc && \
112+
echo ' call Yank(join(v:event.regcontents, "\n"))' >> /root/.vimrc && \
113+
echo 'endfunction' >> /root/.vimrc && \
114+
echo '' >> /root/.vimrc && \
115+
echo 'autocmd TextYankPost * call CopyYank()' >> /root/.vimrc && \
116+
echo '' >> /root/.vimrc && \
117+
echo '" Basic settings' >> /root/.vimrc && \
118+
echo 'set number' >> /root/.vimrc && \
119+
echo 'syntax on' >> /root/.vimrc && \
120+
echo 'set mouse=a' >> /root/.vimrc && \
121+
echo 'filetype indent on' >> /root/.vimrc && \
122+
echo '' >> /root/.vimrc && \
123+
echo '" Indentation' >> /root/.vimrc && \
124+
echo 'set autoindent nosmartindent' >> /root/.vimrc && \
125+
echo 'set smarttab' >> /root/.vimrc && \
126+
echo 'set expandtab' >> /root/.vimrc && \
127+
echo 'set shiftwidth=4' >> /root/.vimrc && \
128+
echo 'set softtabstop=4' >> /root/.vimrc && \
129+
echo '' >> /root/.vimrc && \
130+
echo '" Visual guides' >> /root/.vimrc && \
131+
echo 'set colorcolumn=120' >> /root/.vimrc && \
132+
echo 'highlight ColorColumn ctermbg=5' >> /root/.vimrc && \
133+
echo '' >> /root/.vimrc && \
134+
echo '" Status line' >> /root/.vimrc && \
135+
echo 'set laststatus=2' >> /root/.vimrc && \
136+
echo 'set statusline=%<%f\ %h%m%r%=%{\"[\".(&fenc==\"\"?&enc:&fenc).((exists(\"+bomb\")\ &&\ &bomb)?\",B\":\"\").\"]\ \"}%k\ %-14.(%l,%c%V%)\ %P' >> /root/.vimrc && \
137+
echo '' >> /root/.vimrc && \
138+
echo '" Backspace behavior' >> /root/.vimrc && \
139+
echo 'set backspace=2' >> /root/.vimrc && \
140+
echo '' >> /root/.vimrc && \
141+
echo '" Encoding' >> /root/.vimrc && \
142+
echo 'set encoding=utf-8' >> /root/.vimrc && \
143+
echo 'set fileencoding=utf-8' >> /root/.vimrc
144+
145+
# Configure tmux
146+
RUN echo '# Pane border styling' > /root/.tmux.conf && \
147+
echo 'set -g pane-border-style fg=#742727,bg=black' >> /root/.tmux.conf && \
148+
echo 'set -g pane-active-border-style fg=red,bg=black' >> /root/.tmux.conf && \
149+
echo '' >> /root/.tmux.conf && \
150+
echo '# Status bar styling' >> /root/.tmux.conf && \
151+
echo 'set -g status-style bg=#0C8A92,fg=black' >> /root/.tmux.conf && \
152+
echo '' >> /root/.tmux.conf && \
153+
echo '# Change prefix key to backtick' >> /root/.tmux.conf && \
154+
echo 'set-option -g prefix `' >> /root/.tmux.conf && \
155+
echo 'unbind C-b' >> /root/.tmux.conf && \
156+
echo 'bind-key ` send-prefix' >> /root/.tmux.conf && \
157+
echo '' >> /root/.tmux.conf && \
158+
echo '# Split panes using - and = with current path' >> /root/.tmux.conf && \
159+
echo 'unbind \"' >> /root/.tmux.conf && \
160+
echo 'bind - splitw -v -c \"#{pane_current_path}\"' >> /root/.tmux.conf && \
161+
echo 'unbind %' >> /root/.tmux.conf && \
162+
echo 'bind = splitw -h -c \"#{pane_current_path}\"' >> /root/.tmux.conf && \
163+
echo '' >> /root/.tmux.conf && \
164+
echo '# Vi mode settings' >> /root/.tmux.conf && \
165+
echo 'bind-key -T copy-mode-vi Y send-keys -X copy-pipe \"yank > #{pane_tty}\"' >> /root/.tmux.conf && \
166+
echo 'set-window-option -g mode-keys vi' >> /root/.tmux.conf && \
167+
echo '' >> /root/.tmux.conf && \
168+
echo '# Other settings' >> /root/.tmux.conf && \
169+
echo 'set-option -g escape-time 0' >> /root/.tmux.conf && \
170+
echo 'set-option -g base-index 1' >> /root/.tmux.conf && \
171+
echo 'set-window-option -g mouse on' >> /root/.tmux.conf
172+
173+
# Configure Git
174+
RUN git config --global core.editor "vim" \
175+
&& git config --global core.whitespace "fix,-indent-with-non-tab,trailing-space,cr-at-eol" \
176+
&& git config --global core.pager "diff-so-fancy | less --tabs=4 -RFX" \
177+
&& git config --global color.ui true \
178+
&& git config --global color."diff-highlight".oldNormal "red bold" \
179+
&& git config --global color."diff-highlight".oldHighlight "red bold 52" \
180+
&& git config --global color."diff-highlight".newNormal "green bold" \
181+
&& git config --global color."diff-highlight".newHighlight "green bold 22" \
182+
&& git config --global color.diff.meta "11" \
183+
&& git config --global color.diff.frag "magenta bold" \
184+
&& git config --global color.diff.commit "yellow bold" \
185+
&& git config --global color.diff.old "red bold" \
186+
&& git config --global color.diff.new "green bold" \
187+
&& git config --global color.diff.whitespace "red reverse" \
188+
&& git config --global alias.lg "log --color --graph --pretty=format:'%Cred%h%Creset - %s %Cgreen(%cr) %C(bold blue)<%an>%Creset%C(auto)%d%Creset' --abbrev-commit --" \
189+
&& git config --global http.sslVerify false \
190+
&& git config --global pull.rebase true
191+
192+
# Configure zsh
193+
COPY --chown=root:root <<-"EOF" /root/.zshrc
194+
export ZSH="/root/.oh-my-zsh"
195+
196+
# Theme
197+
ZSH_THEME="robbyrussell"
198+
199+
# Plugins
200+
plugins=(
201+
git
202+
z
203+
zsh-autosuggestions
204+
zsh-syntax-highlighting
205+
)
206+
207+
source $ZSH/oh-my-zsh.sh
208+
209+
# Aliases
210+
alias ll='ls -alF'
211+
alias la='ls -A'
212+
alias l='ls -CF'
213+
alias vi='vim'
214+
215+
# Enhanced history
216+
HISTSIZE=10000
217+
SAVEHIST=10000
218+
setopt HIST_IGNORE_ALL_DUPS
219+
setopt HIST_FIND_NO_DUPS
220+
setopt INC_APPEND_HISTORY
221+
EOF
222+
223+
# Set workspace directory
224+
WORKDIR /vllm-workspace
225+
226+
RUN git clone --depth=1 https://github.com/vllm-project/vllm.git
227+
228+
# Create .devcontainer and .vscode directories
229+
RUN mkdir -p /vllm-workspace/vllm/.devcontainer /vllm-workspace/vllm/.vscode
230+
231+
# Copy .devcontainer and .vscode from host to container
232+
COPY .devcontainer /vllm-workspace/vllm/.devcontainer/
233+
234+
ENV PYTHONPATH=/vllm-workspace/vllm

‎tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3+
4+
import pytest
5+
from transformers import AutoTokenizer
6+
7+
from vllm.entrypoints.openai.protocol import ExtractedToolCallInformation
8+
from vllm.entrypoints.openai.tool_parsers.llama_tool_parser import (
9+
Llama3JsonToolParser)
10+
11+
12+
@pytest.fixture
13+
def parser():
14+
# Use a small tokenizer for testing
15+
tokenizer = AutoTokenizer.from_pretrained("gpt2")
16+
return Llama3JsonToolParser(tokenizer)
17+
18+
19+
def test_extract_tool_calls_simple(parser):
20+
# Test with a simple tool call
21+
model_output = ('Here is the result: {"name": "getOpenIncidentsTool", '
22+
'"parameters": {}} Would you like to know more?')
23+
result = parser.extract_tool_calls(model_output, None)
24+
25+
assert isinstance(result, ExtractedToolCallInformation)
26+
assert result.tools_called is True
27+
assert len(result.tool_calls) == 1
28+
assert result.tool_calls[0].type == "function"
29+
assert result.tool_calls[0].function.name == "getOpenIncidentsTool"
30+
assert result.tool_calls[0].function.arguments == "{}"
31+
assert result.content is None
32+
33+
34+
def test_extract_tool_calls_with_arguments(parser):
35+
# Test with a tool call that has arguments
36+
model_output = (
37+
'{"name": "searchTool", "parameters": {"query": "test query", '
38+
'"limit": 10}}')
39+
result = parser.extract_tool_calls(model_output, None)
40+
41+
assert result.tools_called is True
42+
assert len(result.tool_calls) == 1
43+
assert result.tool_calls[0].function.name == "searchTool"
44+
assert '"query": "test query"' in result.tool_calls[0].function.arguments
45+
assert '"limit": 10' in result.tool_calls[0].function.arguments
46+
47+
48+
def test_extract_tool_calls_no_json(parser):
49+
# Test with text that doesn't contain a JSON object
50+
model_output = "This is just some text without any tool calls"
51+
result = parser.extract_tool_calls(model_output, None)
52+
53+
assert result.tools_called is False
54+
assert len(result.tool_calls) == 0
55+
assert result.content == model_output
56+
57+
58+
def test_extract_tool_calls_invalid_json(parser):
59+
# Test with invalid JSON
60+
model_output = '{"name": "invalidTool", "parameters": {invalid json}'
61+
result = parser.extract_tool_calls(model_output, None)
62+
63+
assert result.tools_called is False
64+
assert len(result.tool_calls) == 0
65+
assert result.content == model_output
66+
67+
68+
def test_extract_tool_calls_with_arguments_key(parser):
69+
# Test with a tool call that uses "arguments" instead of "parameters"
70+
model_output = '{"name": "searchTool", "arguments": {"query": "test"}}'
71+
result = parser.extract_tool_calls(model_output, None)
72+
73+
assert result.tools_called is True
74+
assert len(result.tool_calls) == 1
75+
assert result.tool_calls[0].function.name == "searchTool"
76+
assert '"query": "test"' in result.tool_calls[0].function.arguments
77+
78+
79+
def test_extract_tool_calls_multiple_json(parser):
80+
# Test with multiple JSONs separated by semicolons
81+
model_output = (
82+
'{"name": "searchTool", "parameters": {"query": "test1"}}; '
83+
'{"name": "getOpenIncidentsTool", "parameters": {}}; '
84+
'{"name": "searchTool", "parameters": {"query": "test2"}}')
85+
result = parser.extract_tool_calls(model_output, None)
86+
87+
assert result.tools_called is True
88+
assert len(result.tool_calls) == 3
89+
90+
# Check first tool call
91+
assert result.tool_calls[0].function.name == "searchTool"
92+
assert '"query": "test1"' in result.tool_calls[0].function.arguments
93+
94+
# Check second tool call
95+
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
96+
assert result.tool_calls[1].function.arguments == "{}"
97+
98+
# Check third tool call
99+
assert result.tool_calls[2].function.name == "searchTool"
100+
assert '"query": "test2"' in result.tool_calls[2].function.arguments
101+
102+
103+
def test_extract_tool_calls_multiple_json_with_whitespace(parser):
104+
# Test with multiple JSONs separated by semicolons and extra whitespace
105+
model_output = (
106+
'{"name": "searchTool", "parameters": {"query": "test1"}} ; '
107+
'{"name": "getOpenIncidentsTool", "parameters": {}} ; '
108+
'{"name": "searchTool", "parameters": {"query": "test2"}}')
109+
result = parser.extract_tool_calls(model_output, None)
110+
111+
assert result.tools_called is True
112+
assert len(result.tool_calls) == 3
113+
assert result.tool_calls[0].function.name == "searchTool"
114+
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
115+
assert result.tool_calls[2].function.name == "searchTool"
116+
117+
118+
def test_extract_tool_calls_multiple_json_with_surrounding_text(parser):
119+
# Test with multiple JSONs and surrounding text
120+
model_output = (
121+
'Here are the results: '
122+
'{"name": "searchTool", "parameters": {"query": "test1"}}; '
123+
'{"name": "getOpenIncidentsTool", "parameters": {}}; '
124+
'{"name": "searchTool", "parameters": {"query": "test2"}} '
125+
'Would you like to know more?')
126+
result = parser.extract_tool_calls(model_output, None)
127+
128+
assert result.tools_called is True
129+
assert len(result.tool_calls) == 3
130+
assert result.tool_calls[0].function.name == "searchTool"
131+
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
132+
assert result.tool_calls[2].function.name == "searchTool"

0 commit comments

Comments
 (0)