Skip to content

Cherry-pick for Profiler bug #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed

Cherry-pick for Profiler bug #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
52d2820
pick opc-request-id middleware
upfixer Apr 15, 2025
6e1e9c5
pick [Bugfix] Improve JSON extraction in LlamaToolParser
key4ng Jun 5, 2025
eb0c356
Merge pull request #3 from moirai-internal/dev_container
key4ng Jun 17, 2025
0d831bd
[Bugfix] Correct max tokens for non-contiguous embeds (#21798)
milesial Jul 30, 2025
2f12e79
[Tests] Fixing bug inside MultiModalProfiler. (#21842)
shenoyvvarun Jul 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions .devcontainer/devcontainer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
{
"name": "vllm",
"build": {
"dockerfile": "Dockerfile"
},
"remoteUser": "devuser",
"customizations": {
"vscode": {
"extensions": [
// Python development
"ms-python.python",
"charliermarsh.ruff",
// Rust development
"rust-lang.rust-analyzer",
"tamasfe.even-better-toml"
]
}
},
"forwardPorts": [],
"runArgs": [
"--gpus",
"all"
],
// The two lines below ensures that your local changes in the sglang
// repo is automatically synced to the sglang pip package installed
// in the dev docker container. You can remove / comment out these
// two lines if you prefer to sync code changes manually.
"workspaceMount": "source=${localWorkspaceFolder},target=/vllm-workspace/vllm,type=bind",
"workspaceFolder": "/vllm-workspace/vllm"
}
234 changes: 234 additions & 0 deletions docker/Dockerfile.dev
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,234 @@
# Extend from the base vllm image
FROM vllm/vllm-openai:latest

# Override the base image's entrypoint with a shell
ENTRYPOINT ["/bin/bash"]

# Install development tools and utilities
RUN apt-get update && apt-get install -y \
gdb \
ninja-build \
vim \
tmux \
htop \
wget \
curl \
locales \
lsof \
git \
git-lfs \
zsh \
tree \
silversearcher-ag \
cloc \
unzip \
pkg-config \
libssl-dev \
bear \
ccache \
&& apt install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get clean

RUN apt update -y \
&& apt install -y --no-install-recommends gnupg \
&& echo "deb http://developer.download.nvidia.com/devtools/repos/ubuntu2004/amd64 /" | tee /etc/apt/sources.list.d/nvidia-devtools.list \
&& apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub \
Copy link

@gemini-code-assist gemini-code-assist bot Jul 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

Fetching GPG keys over an unencrypted http connection is insecure and vulnerable to Man-in-the-Middle attacks. Please use https instead. Additionally, apt-key is deprecated and its use should be avoided in favor of storing keys in /usr/share/keyrings/.

    && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub \

&& apt update -y \
&& apt install nsight-systems-cli -y

# Set up locale
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8

# Install minimal Python packages
RUN python3 -m pip install --no-cache-dir \
pytest \
black \
isort \
icdiff \
scikit_build_core \
uv \
pre-commit

# Install diff-so-fancy
RUN curl -LSso /usr/local/bin/diff-so-fancy https://github.com/so-fancy/diff-so-fancy/releases/download/v1.4.4/diff-so-fancy \
&& chmod +x /usr/local/bin/diff-so-fancy

# Install clang-format
RUN curl -LSso /usr/local/bin/clang-format https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-32d3ac78/clang-format-16_linux-amd64 \
&& chmod +x /usr/local/bin/clang-format

# Install clangd
RUN curl -L https://github.com/clangd/clangd/releases/download/18.1.3/clangd-linux-18.1.3.zip -o clangd.zip \
&& unzip clangd.zip \
&& cp -r clangd_18.1.3/bin/* /usr/local/bin/ \
&& cp -r clangd_18.1.3/lib/* /usr/local/lib/ \
&& rm -rf clangd_18.1.3 clangd.zip

# Install CMake
RUN wget https://github.com/Kitware/CMake/releases/download/v3.31.1/cmake-3.31.1-linux-x86_64.tar.gz \
&& tar -xzf cmake-3.31.1-linux-x86_64.tar.gz \
&& cp -r cmake-3.31.1-linux-x86_64/bin/* /usr/local/bin/ \
&& cp -r cmake-3.31.1-linux-x86_64/share/* /usr/local/share/ \
&& rm -rf cmake-3.31.1-linux-x86_64 cmake-3.31.1-linux-x86_64.tar.gz

# Add yank script
RUN echo '#!/bin/bash' > /usr/local/bin/yank && \
echo 'put() {' >> /usr/local/bin/yank && \
echo ' esc=$1' >> /usr/local/bin/yank && \
echo ' test -n "$TMUX" -o -z "${TERM##screen*}" && esc="\033Ptmux;\033$esc\033\\"' >> /usr/local/bin/yank && \
echo ' printf "$esc"' >> /usr/local/bin/yank && \
echo '}' >> /usr/local/bin/yank && \
echo 'put "\033]52;c;!\a"' >> /usr/local/bin/yank && \
echo 'buf=$( cat "$@" )' >> /usr/local/bin/yank && \
echo 'len=$( printf %s "$buf" | wc -c ) max=74994' >> /usr/local/bin/yank && \
echo 'test $len -gt $max && echo "$0: input is $(( len - max )) bytes too long" >&2' >> /usr/local/bin/yank && \
echo 'put "\033]52;c;$( printf %s "$buf" | head -c $max | base64 | tr -d '\''\r\n'\'' )\a"' >> /usr/local/bin/yank && \
echo 'test -n "$TMUX" && tmux set-buffer "$buf" ||:' >> /usr/local/bin/yank && \
chmod +x /usr/local/bin/yank

# Install oh-my-zsh and plugins
RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended \
&& git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-autosuggestions \
&& git clone https://github.com/zsh-users/zsh-syntax-highlighting.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-syntax-highlighting

# Configure Vim
RUN echo 'function! Yank(text) abort' > /root/.vimrc && \
echo ' let escape = system("yank", a:text)' >> /root/.vimrc && \
echo ' if v:shell_error' >> /root/.vimrc && \
echo ' echoerr escape' >> /root/.vimrc && \
echo ' else' >> /root/.vimrc && \
echo ' call writefile([escape], "/dev/tty", "b")' >> /root/.vimrc && \
echo ' endif' >> /root/.vimrc && \
echo 'endfunction' >> /root/.vimrc && \
echo '' >> /root/.vimrc && \
echo 'noremap <silent> <Leader>y y:<C-U>call Yank(@0)<CR>' >> /root/.vimrc && \
echo '' >> /root/.vimrc && \
echo '" automatically run yank(1) whenever yanking in Vim' >> /root/.vimrc && \
echo 'function! CopyYank() abort' >> /root/.vimrc && \
echo ' call Yank(join(v:event.regcontents, "\n"))' >> /root/.vimrc && \
echo 'endfunction' >> /root/.vimrc && \
echo '' >> /root/.vimrc && \
echo 'autocmd TextYankPost * call CopyYank()' >> /root/.vimrc && \
echo '' >> /root/.vimrc && \
echo '" Basic settings' >> /root/.vimrc && \
echo 'set number' >> /root/.vimrc && \
echo 'syntax on' >> /root/.vimrc && \
echo 'set mouse=a' >> /root/.vimrc && \
echo 'filetype indent on' >> /root/.vimrc && \
echo '' >> /root/.vimrc && \
echo '" Indentation' >> /root/.vimrc && \
echo 'set autoindent nosmartindent' >> /root/.vimrc && \
echo 'set smarttab' >> /root/.vimrc && \
echo 'set expandtab' >> /root/.vimrc && \
echo 'set shiftwidth=4' >> /root/.vimrc && \
echo 'set softtabstop=4' >> /root/.vimrc && \
echo '' >> /root/.vimrc && \
echo '" Visual guides' >> /root/.vimrc && \
echo 'set colorcolumn=120' >> /root/.vimrc && \
echo 'highlight ColorColumn ctermbg=5' >> /root/.vimrc && \
echo '' >> /root/.vimrc && \
echo '" Status line' >> /root/.vimrc && \
echo 'set laststatus=2' >> /root/.vimrc && \
echo 'set statusline=%<%f\ %h%m%r%=%{\"[\".(&fenc==\"\"?&enc:&fenc).((exists(\"+bomb\")\ &&\ &bomb)?\",B\":\"\").\"]\ \"}%k\ %-14.(%l,%c%V%)\ %P' >> /root/.vimrc && \
echo '' >> /root/.vimrc && \
echo '" Backspace behavior' >> /root/.vimrc && \
echo 'set backspace=2' >> /root/.vimrc && \
echo '' >> /root/.vimrc && \
echo '" Encoding' >> /root/.vimrc && \
echo 'set encoding=utf-8' >> /root/.vimrc && \
echo 'set fileencoding=utf-8' >> /root/.vimrc

# Configure tmux
RUN echo '# Pane border styling' > /root/.tmux.conf && \
echo 'set -g pane-border-style fg=#742727,bg=black' >> /root/.tmux.conf && \
echo 'set -g pane-active-border-style fg=red,bg=black' >> /root/.tmux.conf && \
echo '' >> /root/.tmux.conf && \
echo '# Status bar styling' >> /root/.tmux.conf && \
echo 'set -g status-style bg=#0C8A92,fg=black' >> /root/.tmux.conf && \
echo '' >> /root/.tmux.conf && \
echo '# Change prefix key to backtick' >> /root/.tmux.conf && \
echo 'set-option -g prefix `' >> /root/.tmux.conf && \
echo 'unbind C-b' >> /root/.tmux.conf && \
echo 'bind-key ` send-prefix' >> /root/.tmux.conf && \
echo '' >> /root/.tmux.conf && \
echo '# Split panes using - and = with current path' >> /root/.tmux.conf && \
echo 'unbind \"' >> /root/.tmux.conf && \
echo 'bind - splitw -v -c \"#{pane_current_path}\"' >> /root/.tmux.conf && \
echo 'unbind %' >> /root/.tmux.conf && \
echo 'bind = splitw -h -c \"#{pane_current_path}\"' >> /root/.tmux.conf && \
echo '' >> /root/.tmux.conf && \
echo '# Vi mode settings' >> /root/.tmux.conf && \
echo 'bind-key -T copy-mode-vi Y send-keys -X copy-pipe \"yank > #{pane_tty}\"' >> /root/.tmux.conf && \
echo 'set-window-option -g mode-keys vi' >> /root/.tmux.conf && \
echo '' >> /root/.tmux.conf && \
echo '# Other settings' >> /root/.tmux.conf && \
echo 'set-option -g escape-time 0' >> /root/.tmux.conf && \
echo 'set-option -g base-index 1' >> /root/.tmux.conf && \
echo 'set-window-option -g mouse on' >> /root/.tmux.conf

# Configure Git
RUN git config --global core.editor "vim" \
&& git config --global core.whitespace "fix,-indent-with-non-tab,trailing-space,cr-at-eol" \
&& git config --global core.pager "diff-so-fancy | less --tabs=4 -RFX" \
&& git config --global color.ui true \
&& git config --global color."diff-highlight".oldNormal "red bold" \
&& git config --global color."diff-highlight".oldHighlight "red bold 52" \
&& git config --global color."diff-highlight".newNormal "green bold" \
&& git config --global color."diff-highlight".newHighlight "green bold 22" \
&& git config --global color.diff.meta "11" \
&& git config --global color.diff.frag "magenta bold" \
&& git config --global color.diff.commit "yellow bold" \
&& git config --global color.diff.old "red bold" \
&& git config --global color.diff.new "green bold" \
&& git config --global color.diff.whitespace "red reverse" \
&& git config --global alias.lg "log --color --graph --pretty=format:'%Cred%h%Creset - %s %Cgreen(%cr) %C(bold blue)<%an>%Creset%C(auto)%d%Creset' --abbrev-commit --" \
&& git config --global http.sslVerify false \
Copy link

@gemini-code-assist gemini-code-assist bot Jul 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

critical

Disabling SSL verification for git globally with http.sslVerify false is a significant security vulnerability. It exposes the container to Man-in-the-Middle (MITM) attacks during git operations, as it will trust any SSL certificate.

This setting should be removed. If there's a specific need to connect to a git repository with a self-signed certificate, it should be configured on a per-repository basis, not globally.

&& git config --global pull.rebase true

# Configure zsh
COPY --chown=root:root <<-"EOF" /root/.zshrc
export ZSH="/root/.oh-my-zsh"

# Theme
ZSH_THEME="robbyrussell"

# Plugins
plugins=(
git
z
zsh-autosuggestions
zsh-syntax-highlighting
)

source $ZSH/oh-my-zsh.sh

# Aliases
alias ll='ls -alF'
alias la='ls -A'
alias l='ls -CF'
alias vi='vim'

# Enhanced history
HISTSIZE=10000
SAVEHIST=10000
setopt HIST_IGNORE_ALL_DUPS
setopt HIST_FIND_NO_DUPS
setopt INC_APPEND_HISTORY
EOF

# Set workspace directory
WORKDIR /vllm-workspace

RUN git clone --depth=1 https://github.com/vllm-project/vllm.git

# Create .devcontainer and .vscode directories
RUN mkdir -p /vllm-workspace/vllm/.devcontainer /vllm-workspace/vllm/.vscode

# Copy .devcontainer and .vscode from host to container
COPY .devcontainer /vllm-workspace/vllm/.devcontainer/

ENV PYTHONPATH=/vllm-workspace/vllm
131 changes: 131 additions & 0 deletions tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
# SPDX-License-Identifier: Apache-2.0

import pytest
from transformers import AutoTokenizer

from vllm.entrypoints.openai.protocol import ExtractedToolCallInformation
from vllm.entrypoints.openai.tool_parsers.llama_tool_parser import (
Llama3JsonToolParser)


@pytest.fixture
def parser():
# Use a small tokenizer for testing
tokenizer = AutoTokenizer.from_pretrained("gpt2")
return Llama3JsonToolParser(tokenizer)


def test_extract_tool_calls_simple(parser):
# Test with a simple tool call
model_output = ('Here is the result: {"name": "getOpenIncidentsTool", '
'"parameters": {}} Would you like to know more?')
result = parser.extract_tool_calls(model_output, None)

assert isinstance(result, ExtractedToolCallInformation)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].type == "function"
assert result.tool_calls[0].function.name == "getOpenIncidentsTool"
assert result.tool_calls[0].function.arguments == "{}"
assert result.content is None


def test_extract_tool_calls_with_arguments(parser):
# Test with a tool call that has arguments
model_output = (
'{"name": "searchTool", "parameters": {"query": "test query", '
'"limit": 10}}')
result = parser.extract_tool_calls(model_output, None)

assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "searchTool"
assert '"query": "test query"' in result.tool_calls[0].function.arguments
assert '"limit": 10' in result.tool_calls[0].function.arguments


def test_extract_tool_calls_no_json(parser):
# Test with text that doesn't contain a JSON object
model_output = "This is just some text without any tool calls"
result = parser.extract_tool_calls(model_output, None)

assert result.tools_called is False
assert len(result.tool_calls) == 0
assert result.content == model_output


def test_extract_tool_calls_invalid_json(parser):
# Test with invalid JSON
model_output = '{"name": "invalidTool", "parameters": {invalid json}'
result = parser.extract_tool_calls(model_output, None)

assert result.tools_called is False
assert len(result.tool_calls) == 0
assert result.content == model_output


def test_extract_tool_calls_with_arguments_key(parser):
# Test with a tool call that uses "arguments" instead of "parameters"
model_output = '{"name": "searchTool", "arguments": {"query": "test"}}'
result = parser.extract_tool_calls(model_output, None)

assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "searchTool"
assert '"query": "test"' in result.tool_calls[0].function.arguments


def test_extract_tool_calls_multiple_json(parser):
# Test with multiple JSONs separated by semicolons
model_output = (
'{"name": "searchTool", "parameters": {"query": "test1"}}; '
'{"name": "getOpenIncidentsTool", "parameters": {}}; '
'{"name": "searchTool", "parameters": {"query": "test2"}}')
result = parser.extract_tool_calls(model_output, None)

assert result.tools_called is True
assert len(result.tool_calls) == 3

# Check first tool call
assert result.tool_calls[0].function.name == "searchTool"
assert '"query": "test1"' in result.tool_calls[0].function.arguments

# Check second tool call
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
assert result.tool_calls[1].function.arguments == "{}"

# Check third tool call
assert result.tool_calls[2].function.name == "searchTool"
assert '"query": "test2"' in result.tool_calls[2].function.arguments


def test_extract_tool_calls_multiple_json_with_whitespace(parser):
# Test with multiple JSONs separated by semicolons and extra whitespace
model_output = (
'{"name": "searchTool", "parameters": {"query": "test1"}} ; '
'{"name": "getOpenIncidentsTool", "parameters": {}} ; '
'{"name": "searchTool", "parameters": {"query": "test2"}}')
result = parser.extract_tool_calls(model_output, None)

assert result.tools_called is True
assert len(result.tool_calls) == 3
assert result.tool_calls[0].function.name == "searchTool"
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
assert result.tool_calls[2].function.name == "searchTool"


def test_extract_tool_calls_multiple_json_with_surrounding_text(parser):
# Test with multiple JSONs and surrounding text
model_output = (
'Here are the results: '
'{"name": "searchTool", "parameters": {"query": "test1"}}; '
'{"name": "getOpenIncidentsTool", "parameters": {}}; '
'{"name": "searchTool", "parameters": {"query": "test2"}} '
'Would you like to know more?')
result = parser.extract_tool_calls(model_output, None)

assert result.tools_called is True
assert len(result.tool_calls) == 3
assert result.tool_calls[0].function.name == "searchTool"
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
assert result.tool_calls[2].function.name == "searchTool"
Loading
Loading