[Feature][Frontend] Expose effective attention block size for DCP

.buildkite/intel_jobs/entrypoints_intel.yaml

@@ -134,7 +134,8 @@ steps:
     'cd tests &&
     pytest -v -s tool_use
     --deselect="tests/tool_use/mistral/test_mistral_tool_calls.py::test_tool_call_none_with_tools[ministral-3b]"
-    --deselect="tests/tool_use/test_chat_completions.py::test_chat_completion_with_tools[granite-3.0-8b]" &&
+    --deselect="tests/tool_use/test_chat_completions.py::test_chat_completion_with_tools[granite-3.0-8b]"
+    --deselect="tests/tool_use/test_chat_completions.py::test_chat_completion_with_tools[llama3.2]" &&
     pytest -v -s entrypoints/tool_parsers &&
     pytest -v -s entrypoints/generate &&
     pytest -v -s entrypoints/anthropic &&

.buildkite/intel_jobs/lm_eval_intel.yaml

@@ -23,5 +23,6 @@ steps:
   commands:
     - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'cd tests &&
+      'uv pip install "gpt-oss[eval]==0.0.5" &&
+      cd tests &&
       pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt'

.buildkite/intel_jobs/test-intel.yaml

@@ -97,7 +97,7 @@ steps:
     commands:
       - >-
         bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-        'cd tests &&
+        'cd tests && pip install dpctl &&
         pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
         pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
         pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&

.buildkite/scripts/ci-bake-rocm.sh

@@ -540,6 +540,26 @@ prepare_ci_build_context() {
     echo "Using canonical CI Docker context: ${ROCM_BUILD_CONTEXT_ROOT}"
 }
 
+configure_custom_rocm_stages() {
+    using_custom_rocm_dockerfiles || return 0
+    local context_root="${ROCM_BUILD_CONTEXT_ROOT:-.}"
+    local dockerfile="${context_root}/${CI_BASE_DOCKERFILE}"
+    local stages=""
+
+    (cd "${context_root}" && \
+        validate_rocm_dockerfile "${CI_BASE_DOCKERFILE}" \
+            ci_base test export_vllm export_test_smoke csrc-build rust-build) \
+        || return $?
+    # Hash all stages for custom layouts, including additional helper stages.
+    stages=$(rocm_dockerfile_stages "${dockerfile}" | paste -sd ' ')
+    CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${stages}}"
+    ROCM_CSRC_DOCKERFILE_STAGES="${ROCM_CSRC_DOCKERFILE_STAGES:-${stages}}"
+    ROCM_RUST_DOCKERFILE_STAGES="${ROCM_RUST_DOCKERFILE_STAGES:-${stages}}"
+    if [[ " ${stages} " != *" build_rocshmem "* ]]; then
+        unset ROCSHMEM_BRANCH ROCSHMEM_CACHE_KEY DEEPEP_CACHE_KEY
+    fi
+}
+
 compose_dependency_cache_key() {
     local prefix="$1"
     local material="$2"
@@ -569,11 +589,11 @@ hash_dockerfile_stages() {
             }
             emit = 0
         }
-        $1 == "FROM" {
+        toupper($1) == "FROM" {
             stage = ""
             for (idx = 1; idx <= NF; idx++) {
                 if (tolower($idx) == "as" && idx < NF) {
-                    stage = $(idx + 1)
+                    stage = tolower($(idx + 1))
                 }
             }
             emit = (stage in wanted)
@@ -613,11 +633,11 @@ discover_dockerfile_stage_args() {
         }
         {
             line = $0
-            if ($1 == "FROM") {
+            if (toupper($1) == "FROM") {
                 stage = ""
                 for (idx = 1; idx <= NF; idx++) {
                     if (tolower($idx) == "as" && idx < NF) {
-                        stage = $(idx + 1)
+                        stage = tolower($(idx + 1))
                     }
                 }
                 emit = (stage in wanted)
@@ -630,7 +650,7 @@ discover_dockerfile_stage_args() {
             for (idx = 1; idx <= line_count; idx++) {
                 line = lines[idx]
                 arg_name = line
-                sub(/^[[:space:]]*ARG[[:space:]]+/, "", arg_name)
+                sub(/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+/, "", arg_name)
                 if (arg_name != line) {
                     sub(/[=[:space:]].*/, "", arg_name)
                     if (arg_name ~ /^[A-Za-z_][A-Za-z0-9_]*$/) {
@@ -716,7 +736,7 @@ extract_dockerfile_arg_default() {
     if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then
         physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}"
     fi
-    sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
+    sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
         "${physical_dockerfile}" | head -1
 }
 
@@ -1000,7 +1020,21 @@ create_and_bootstrap_builder() {
 }
 
 init_config() {
+    # shellcheck source=.buildkite/scripts/rocm/build-config.sh
+    source "$(dirname "${BASH_SOURCE[0]}")/rocm/build-config.sh"
+    configure_rocm_build
+
     TARGET="${1:-test-ci}"
+    if using_custom_rocm_dockerfiles; then
+        if [[ -z "${BASE_IMAGE:-}" ]]; then
+            echo "Custom ROCm builds require BASE_IMAGE from the selected base handoff" >&2
+            return 1
+        fi
+        if ! is_ci_base_target && [[ -z "${CI_BASE_IMAGE:-}" ]]; then
+            echo "Custom ROCm builds require CI_BASE_IMAGE from the selected ci_base handoff" >&2
+            return 1
+        fi
+    fi
     BAKE_TARGETS=("${TARGET}")
     DEPENDENCY_CACHE_TARGETS=()
     CI_HCL_SOURCE="${CI_HCL_SOURCE:-${CI_HCL_FILE:-${DEFAULT_CI_HCL_SOURCE}}}"
@@ -1010,10 +1044,12 @@ init_config() {
     PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH:-gfx90a;gfx942;gfx950}"
     CI_BASE_CONTENT_FILES="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
     CI_BASE_DOCKERFILE="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
-    CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
+    if ! using_custom_rocm_dockerfiles; then
+        CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
+    fi
     CI_BASE_METADATA_VERSION="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
     CI_BASE_IMAGE_TAG="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}"
-    export PYTORCH_ROCM_ARCH
+    export PYTORCH_ROCM_ARCH CI_BASE_DOCKERFILE
 
     SCRIPT_TMP_DIR=$(mktemp -d -t ci-bake-rocm.XXXXXX)
     CI_HCL_PATH="${SCRIPT_TMP_DIR}/ci.hcl"
@@ -1042,6 +1078,11 @@ print_header() {
 }
 
 validate_inputs() {
+    if using_custom_rocm_dockerfiles; then
+        validate_rocm_dockerfile "${ROCM_BASE_DOCKERFILE}" || return $?
+        validate_rocm_dockerfile "${CI_BASE_DOCKERFILE}" \
+            ci_base test export_vllm export_test_smoke csrc-build rust-build || return $?
+    fi
     if [[ ! -f "${VLLM_BAKE_FILE}" ]]; then
         echo "Error: vLLM bake file not found at ${VLLM_BAKE_FILE}"
         echo "Make sure you're running from the vLLM repository root"
@@ -1069,6 +1110,11 @@ load_ci_hcl() {
 
 init_bake_files() {
     BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
+    if using_custom_rocm_dockerfiles; then
+        # Custom stacks reuse local cache without replacing the
+        # standard ROCm registry caches for the same commit or branch.
+        BAKE_FILES+=(--set '*.cache-to=')
+    fi
 }
 
 compute_ci_base_hash_if_needed() {
@@ -1804,7 +1850,7 @@ compute_rocm_csrc_content_hash() {
     local -a content_args=()
 
     bake_dir=$(dirname "${VLLM_BAKE_FILE}")
-    dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
+    dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
     read -r -a content_paths <<< "${content_files}"
     mapfile -t content_args < <(
         get_content_arg_names "${dockerfile_rocm}" "${stages}" "${ROCM_CSRC_CONTENT_ARGS:-}"
@@ -1853,7 +1899,7 @@ compute_rocm_rust_content_hash() {
     local -a content_args=()
 
     bake_dir=$(dirname "${VLLM_BAKE_FILE}")
-    dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
+    dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
     read -r -a content_paths <<< "${content_files}"
     mapfile -t content_args < <(
         get_content_arg_names \
@@ -1943,7 +1989,7 @@ write_rocm_build_arg_override() {
     local arg_value=""
 
     bake_dir=$(dirname "${VLLM_BAKE_FILE}")
-    dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
+    dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
     mapfile -t arg_names < <(
         {
             get_content_arg_names \
@@ -2259,7 +2305,7 @@ extract_dependency_pins() {
     local val=""
 
     bake_dir=$(dirname "${VLLM_BAKE_FILE}")
-    dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
+    dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
     physical_dockerfile="${dockerfile_rocm}"
     if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile_rocm}" != /* ]]; then
         physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile_rocm}"
@@ -2275,12 +2321,12 @@ extract_dependency_pins() {
         fi
 
         val=$(
-            sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
+            sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
                 "${physical_dockerfile}" | head -1
         )
         if [[ -n "${val}" ]]; then
             export "${var}=${val}"
-            echo "Extracted ${var}=${val} from Dockerfile.rocm"
+            echo "Extracted ${var}=${val} from ${dockerfile_rocm}"
         fi
     done
 }
@@ -2297,7 +2343,7 @@ compute_dependency_cache_keys() {
     local deepep_material=""
 
     bake_dir=$(dirname "${VLLM_BAKE_FILE}")
-    dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
+    dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
     nixl_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "NIXL_BRANCH")
     ucx_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "UCX_BRANCH")
     rocshmem_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "ROCSHMEM_BRANCH")
@@ -2687,6 +2733,7 @@ main() {
     init_bake_files
     if is_ci_base_target; then
         prepare_ci_build_context
+        configure_custom_rocm_stages
     fi
     compute_ci_base_hash_if_needed
     configure_ci_base_image_refs
@@ -2697,6 +2744,7 @@ main() {
     # version metadata only after that lookup sees the available tag history.
     if ! is_ci_base_target; then
         prepare_ci_build_context
+        configure_custom_rocm_stages
     fi
     extract_dependency_pins
     write_rocm_build_arg_override

.buildkite/scripts/ci-otel/ci_otel.sh

@@ -65,6 +65,39 @@ _ci_otel_python() {
   fi
 }
 
+# The Docker test commands run as root, while /workdir is owned by the
+# Buildkite checkout user. Run the GPU sampler as that unprivileged identity so
+# any future project-code import cannot leave root-owned files in the checkout.
+_ci_otel_prepare_gpu_user() {
+  _CI_INFRA_GPU_UID=""
+  _CI_INFRA_GPU_GID=""
+  [ "$(id -u 2>/dev/null || :)" = "0" ] || return 0
+  command -v setpriv >/dev/null 2>&1 || return 1
+
+  _CI_INFRA_OTEL_WORKSPACE_DIR="${CI_INFRA_OTEL_WORKSPACE_DIR:-/workdir}"
+  [ -d "${_CI_INFRA_OTEL_WORKSPACE_DIR}" ] || return 1
+  # The owner fields are numeric and whitespace-free on GNU stat.
+  # shellcheck disable=SC2046
+  set -- $(stat -c '%u %g' -- "${_CI_INFRA_OTEL_WORKSPACE_DIR}" 2>/dev/null || :)
+  [ "$#" -eq 2 ] || return 1
+  case "$1:$2" in
+    *[!0-9:]* | 0:* | *:0) return 1 ;;
+  esac
+
+  _CI_INFRA_GPU_UID="$1"
+  _CI_INFRA_GPU_GID="$2"
+  chown "${_CI_INFRA_GPU_UID}:${_CI_INFRA_GPU_GID}" \
+    "${CI_INFRA_OTEL_SPOOL_DIR}" || return 1
+  chmod 0700 "${CI_INFRA_OTEL_SPOOL_DIR}" || return 1
+  if [ "${_CI_INFRA_OTEL_OWNS_RUNTIME}" = "1" ]; then
+    # Permit traversal to the private child spool, but not directory listing.
+    chmod 0711 "${CI_INFRA_OTEL_RUNTIME_DIR}" || return 1
+  fi
+  setpriv --reuid="${_CI_INFRA_GPU_UID}" \
+    --regid="${_CI_INFRA_GPU_GID}" --clear-groups --no-new-privs \
+    -- test -w "${CI_INFRA_OTEL_SPOOL_DIR}"
+}
+
 ci_otel_start() {
   _CI_INFRA_OTEL_COMMAND_INDEX="$1"
   _CI_INFRA_OTEL_COMMAND_LABEL="${2:-command ${_CI_INFRA_OTEL_COMMAND_INDEX}}"
@@ -87,9 +120,23 @@ ci_otel_start() {
   if [ "${CI_INFRA_GPU_SAMPLING:-1}" != "0" ] &&
     [ -f "${CI_INFRA_OTEL_DIR}/ci_gpu.py" ] &&
     command -v nvidia-smi >/dev/null 2>&1; then
-    PYTHONDONTWRITEBYTECODE=1 \
-      "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_gpu.py" "$$" </dev/null &
-    _CI_INFRA_GPU_PID=$!
+    if _ci_otel_prepare_gpu_user; then
+      if [ -n "${_CI_INFRA_GPU_UID}" ]; then
+        PYTHONDONTWRITEBYTECODE=1 setpriv \
+          --reuid="${_CI_INFRA_GPU_UID}" --regid="${_CI_INFRA_GPU_GID}" \
+          --clear-groups --no-new-privs -- \
+          "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_gpu.py" "$$" \
+          </dev/null &
+      else
+        PYTHONDONTWRITEBYTECODE=1 \
+          "${_CI_INFRA_OTEL_PYTHON}" "${CI_INFRA_OTEL_DIR}/ci_gpu.py" "$$" \
+          </dev/null &
+      fi
+      _CI_INFRA_GPU_PID=$!
+    else
+      echo "vLLM CI OTel: non-root GPU sampler unavailable; sampling disabled" \
+        >&2 || :
+    fi
   fi
 }
 

.buildkite/scripts/ci-otel/tests/test_ci_otel.py

@@ -987,6 +987,85 @@ def upload(spans, timeout_seconds):
     assert [len(batch[0].events) for batch in batches] == [1, 1, 1]
 
 
+def _write_executable(path: Path, body: str) -> None:
+    path.write_text(f"#!/bin/sh\n{body}")
+    path.chmod(0o755)
+
+
+@pytest.mark.parametrize(
+    "workspace_owner,expect_sampler",
+    [("997 996", True), ("0 0", False)],
+)
+def test_root_shell_runs_gpu_sampler_as_checkout_owner(
+    tmp_path, workspace_owner, expect_sampler
+):
+    bin_dir = tmp_path / "bin"
+    workspace = tmp_path / "workdir"
+    runtime = tmp_path / "runtime"
+    log = tmp_path / "calls.log"
+    bin_dir.mkdir()
+    workspace.mkdir()
+
+    _write_executable(
+        bin_dir / "id",
+        '[ "$1" = "-u" ] && { echo 0; exit 0; }; exec /usr/bin/id "$@"\n',
+    )
+    _write_executable(
+        bin_dir / "stat",
+        f"printf '%s\\n' '{workspace_owner}'\n",
+    )
+    for command in ("chown", "chmod"):
+        _write_executable(bin_dir / command, "exit 0\n")
+    _write_executable(
+        bin_dir / "setpriv",
+        'printf "setpriv %s\\n" "$*" >> "$CI_OTEL_TEST_LOG"\n'
+        'while [ "$#" -gt 0 ] && [ "$1" != "--" ]; do shift; done\n'
+        '[ "$#" -gt 0 ] && shift\n'
+        'exec "$@"\n',
+    )
+    _write_executable(
+        bin_dir / "python3",
+        '[ "$1" = "-c" ] && exit 0\n'
+        '[ "$2" = "new-context" ] && {\n'
+        '  echo "01010101010101010101010101010101 0202020202020202 - 1"\n'
+        "  exit 0\n"
+        "}\n"
+        'case "$1" in *ci_gpu.py) echo gpu >> "$CI_OTEL_TEST_LOG";; esac\n'
+        "exit 0\n",
+    )
+    _write_executable(bin_dir / "nvidia-smi", "exit 0\n")
+    shell = (
+        f'. "{SCRIPTS_DIR / "ci_otel.sh"}"; '
+        f"ci_otel_start 1 {_quoted('true')}; ci_otel_finish 0"
+    )
+
+    result = subprocess.run(
+        ["/bin/sh", "-c", shell],
+        check=False,
+        capture_output=True,
+        text=True,
+        env={
+            **os.environ,
+            "PATH": f"{bin_dir}:{os.environ['PATH']}",
+            "CI_INFRA_OTEL_DIR": str(SCRIPTS_DIR),
+            "CI_INFRA_OTEL_RUNTIME_DIR": str(runtime),
+            "CI_INFRA_OTEL_WORKSPACE_DIR": str(workspace),
+            "CI_OTEL_TEST_LOG": str(log),
+        },
+    )
+
+    assert result.returncode == 0, result.stderr
+    calls = log.read_text() if log.exists() else ""
+    if expect_sampler:
+        expected = "--reuid=997 --regid=996 --clear-groups --no-new-privs --"
+        assert calls.count("setpriv") == 2
+        assert calls.count(expected) == 2
+        assert "gpu" in calls
+    else:
+        assert calls == ""
+        assert "non-root GPU sampler unavailable" in result.stderr
+
+
 @pytest.mark.parametrize("sampling", ["1", "0"])
 def test_gpu_sampler_stops_with_command_and_preserves_failure_status(
     tmp_path, sampling