[CI] Bake hash-locked KV connectors into the CI image and re-enable their jobs

.buildkite/scripts/check-kv-connectors.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# KV connector packages are baked into the CI test images from hash-locked
+# requirements (requirements/test/kv_connectors_cuda.txt on CUDA, the ROCm
+# test lock on ROCm). This script only verifies them; jobs must not install
+# packages at runtime.
+
+set -euo pipefail
+
+python3 - <<'PY'
+import importlib.metadata as metadata
+import sys
+
+import torch
+
+if torch.version.hip is not None:
+    required = ("tblib", "lm-eval")
+else:
+    cuda_major = torch.version.cuda.split(".", 1)[0]
+    if cuda_major != "13":
+        sys.exit(
+            f"KV connectors are only baked into the CUDA 13 CI image, got CUDA {cuda_major}"
+        )
+    required = (
+        "lmcache",
+        "nixl",
+        "nixl-cu13",
+        "mooncake-transfer-engine-cuda13",
+        "cupy-cuda13x",
+    )
+
+missing = []
+for package_name in required:
+    try:
+        print(f"{package_name}: {metadata.version(package_name)}")
+    except metadata.PackageNotFoundError:
+        missing.append(package_name)
+
+for package_name in ("nixl-cu12", "mooncake-transfer-engine"):
+    try:
+        metadata.version(package_name)
+        missing.append(f"{package_name} must not be installed")
+    except metadata.PackageNotFoundError:
+        pass
+
+if missing:
+    sys.exit(
+        "KV connector packages are not baked into this image: "
+        + ", ".join(missing)
+        + ". Rebuild the CI image instead of installing them at runtime."
+    )
+PY

.buildkite/test-amd.yaml

@@ -5544,71 +5544,69 @@ steps:
   - pytest -v -s v1/simple_kv_offload
   optional: true
 
-# Disabled until the KV connectors installation is locked.
-# - label: ":amd: (MI355 DPX) V1 KV Connectors Shard %N"
-#   timeout_in_minutes: 60
-#   parallelism: 4
-#   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
-#   dind: false
-#   agent_pool: mi355_dpx
-#   working_dir: /vllm-workspace/tests
-#   source_file_dependencies:
-#   - vllm/
-#   - tests/v1/core
-#   - tests/v1/executor
-#   - tests/v1/kv_offload
-#   - tests/v1/simple_kv_offload
-#   - tests/v1/worker
-#   - tests/v1/streaming_input
-#   - tests/v1/kv_connector/unit
-#   - tests/v1/ec_connector/unit
-#   - tests/v1/metrics
-#   - tests/entrypoints/openai/correctness/test_lmeval.py
-#   - vllm/config/
-#   - vllm/distributed/
-#   - vllm/engine/
-#   - vllm/entrypoints/pooling/
-#   - vllm/inputs/
-#   - vllm/lora/
-#   - vllm/model_executor/
-#   - vllm/multimodal/
-#   - vllm/outputs.py
-#   - vllm/platforms/
-#   - vllm/pooling_params.py
-#   - vllm/profiler/
-#   - vllm/sampling_params.py
-#   - vllm/tokenizers/
-#   - vllm/transformers_utils/
-#   - vllm/utils/
-#   - vllm/v1/
-#   commands:
-#   - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
-#   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#   - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
-#   - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
-#   optional: true
+- label: ":amd: (MI355 DPX) V1 KV Connectors Shard %N"
+  timeout_in_minutes: 60
+  parallelism: 4
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_dpx
+  working_dir: /vllm-workspace/tests
+  source_file_dependencies:
+  - vllm/
+  - tests/v1/core
+  - tests/v1/executor
+  - tests/v1/kv_offload
+  - tests/v1/simple_kv_offload
+  - tests/v1/worker
+  - tests/v1/streaming_input
+  - tests/v1/kv_connector/unit
+  - tests/v1/ec_connector/unit
+  - tests/v1/metrics
+  - tests/entrypoints/openai/correctness/test_lmeval.py
+  - vllm/config/
+  - vllm/distributed/
+  - vllm/engine/
+  - vllm/entrypoints/pooling/
+  - vllm/inputs/
+  - vllm/lora/
+  - vllm/model_executor/
+  - vllm/multimodal/
+  - vllm/outputs.py
+  - vllm/platforms/
+  - vllm/pooling_params.py
+  - vllm/profiler/
+  - vllm/sampling_params.py
+  - vllm/tokenizers/
+  - vllm/transformers_utils/
+  - vllm/utils/
+  - vllm/v1/
+  commands:
+  - bash /vllm-workspace/.buildkite/scripts/check-kv-connectors.sh
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
+  - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
+  optional: true
 
-# Disabled until the KV connectors installation is locked.
-# - label: ":amd: (MI355 DPX) MoRIIO Connector Unit Tests"
-#   timeout_in_minutes: 30
-#   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
-#   dind: false
-#   agent_pool: mi355_dpx
-#   num_gpus: 1
-#   optional: true
-#   working_dir: "/vllm-workspace/tests"
-#   source_file_dependencies:
-#   - vllm/distributed/kv_transfer/kv_connector/v1/moriio/
-#   - vllm/distributed/kv_transfer/kv_connector/v1/base.py
-#   - vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py
-#   - examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py
-#   - tests/v1/kv_connector/unit/test_moriio_
-#   - tests/v1/kv_connector/unit/utils.py
-#   - requirements/kv_connectors_rocm.txt
-#   commands:
-#   - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
-#   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#   - pytest -v -s -rs v1/kv_connector/unit/test_moriio_*.py
+- label: ":amd: (MI355 DPX) MoRIIO Connector Unit Tests"
+  timeout_in_minutes: 30
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_dpx
+  num_gpus: 1
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - vllm/distributed/kv_transfer/kv_connector/v1/moriio/
+  - vllm/distributed/kv_transfer/kv_connector/v1/base.py
+  - vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py
+  - examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py
+  - tests/v1/kv_connector/unit/test_moriio_
+  - tests/v1/kv_connector/unit/utils.py
+  - requirements/test/rocm.txt
+  commands:
+  - bash /vllm-workspace/.buildkite/scripts/check-kv-connectors.sh
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -v -s -rs v1/kv_connector/unit/test_moriio_*.py
 
 - label: ":amd: (MI355 DPX) UMBP Connector Unit Tests"
   timeout_in_minutes: 30
@@ -5633,52 +5631,50 @@ steps:
   - python3 -c "import mori"
   - pytest -v -s -rs v1/kv_connector/unit/test_umbp_*.py
 
-# Disabled until the KV connectors installation is locked.
-# - label: ":amd: (MI355) MoRIIO WRITE P/D Dummy-Weight E2E Shard %N"
-#   timeout_in_minutes: 45
-#   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
-#   dind: false
-#   agent_pool: mi355_2
-#   num_gpus: 2
-#   parallelism: 6
-#   optional: true
-#   working_dir: "/vllm-workspace/tests"
-#   source_file_dependencies:
-#   - vllm/distributed/kv_transfer/kv_connector/v1/moriio/
-#   - vllm/distributed/kv_transfer/kv_connector/v1/base.py
-#   - vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py
-#   - vllm/model_executor/model_loader/dummy_loader.py
-#   - examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py
-#   - tests/v1/kv_connector/moriio_integration/
-#   - requirements/kv_connectors_rocm.txt
-#   commands:
-#   - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
-#   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#   - pytest -v -s -rs v1/kv_connector/moriio_integration/test_write_dummy_e2e_rocm.py -k "not heterogeneous_tp" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
+- label: ":amd: (MI355) MoRIIO WRITE P/D Dummy-Weight E2E Shard %N"
+  timeout_in_minutes: 45
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_2
+  num_gpus: 2
+  parallelism: 6
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - vllm/distributed/kv_transfer/kv_connector/v1/moriio/
+  - vllm/distributed/kv_transfer/kv_connector/v1/base.py
+  - vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py
+  - vllm/model_executor/model_loader/dummy_loader.py
+  - examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py
+  - tests/v1/kv_connector/moriio_integration/
+  - requirements/test/rocm.txt
+  commands:
+  - bash /vllm-workspace/.buildkite/scripts/check-kv-connectors.sh
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -v -s -rs v1/kv_connector/moriio_integration/test_write_dummy_e2e_rocm.py -k "not heterogeneous_tp" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
 
 # The same P/D between different TP sizes (WRITE TP2 -> TP1, READ of MLA
 # TP1 <-> TP2), which needs 3 GPUs: 8 cases, ~36 min on MI355.
-# Disabled until the KV connectors installation is locked.
-# - label: ":amd: (MI355) MoRIIO WRITE P/D Dummy-Weight Heterogeneous TP"
-#   timeout_in_minutes: 45
-#   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
-#   dind: false
-#   agent_pool: mi355_4
-#   num_gpus: 4
-#   optional: true
-#   working_dir: "/vllm-workspace/tests"
-#   source_file_dependencies:
-#   - vllm/distributed/kv_transfer/kv_connector/v1/moriio/
-#   - vllm/distributed/kv_transfer/kv_connector/v1/base.py
-#   - vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py
-#   - vllm/model_executor/model_loader/dummy_loader.py
-#   - examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py
-#   - tests/v1/kv_connector/moriio_integration/
-#   - requirements/kv_connectors_rocm.txt
-#   commands:
-#   - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
-#   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#   - pytest -v -s -rs v1/kv_connector/moriio_integration/test_write_dummy_e2e_rocm.py -k heterogeneous_tp
+- label: ":amd: (MI355) MoRIIO WRITE P/D Dummy-Weight Heterogeneous TP"
+  timeout_in_minutes: 45
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_4
+  num_gpus: 4
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - vllm/distributed/kv_transfer/kv_connector/v1/moriio/
+  - vllm/distributed/kv_transfer/kv_connector/v1/base.py
+  - vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py
+  - vllm/model_executor/model_loader/dummy_loader.py
+  - examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py
+  - tests/v1/kv_connector/moriio_integration/
+  - requirements/test/rocm.txt
+  commands:
+  - bash /vllm-workspace/.buildkite/scripts/check-kv-connectors.sh
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -v -s -rs v1/kv_connector/moriio_integration/test_write_dummy_e2e_rocm.py -k heterogeneous_tp
 
 - label: ":amd: (MI355 DPX) V1 Metrics + LM Eval"
   timeout_in_minutes: 65