Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ futures-util = { version = "0.3.32" }
hf-hub = { version = "0.4.2", default-features = false }

# ModelExpress for model downloading
modelexpress-client = { version = "0.3.0" }
modelexpress-common = { version = "0.3.0" }
modelexpress-client = { version = "0.5.0" }
modelexpress-common = { version = "0.5.0" }
Comment thread
pvijayakrish marked this conversation as resolved.
Resolved
Comment thread
pvijayakrish marked this conversation as resolved.
Resolved

humantime = { version = "2.2.0" }
indexmap = { version = "2" }
Expand Down
4 changes: 2 additions & 2 deletions container/context.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ vllm:
enable_gpu_memory_service: "true"
enable_kvbm: "true"
enable_modelexpress: "true"
modelexpress_version: "0.4.0"
modelexpress_version: "0.5.0"
Comment thread
pvijayakrish marked this conversation as resolved.
Resolved
Comment thread
pvijayakrish marked this conversation as resolved.
Resolved
# aws-sdk-cpp tag for the NIXL OBJ / S3 backend (built in wheel_builder).
aws_sdk_cpp_version: "1.11.760"

Expand Down Expand Up @@ -133,7 +133,7 @@ sglang:
enable_gpu_memory_service: "true"
enable_kvbm: "false"
enable_modelexpress: "true"
modelexpress_version: "0.4.0"
modelexpress_version: "0.5.0"

trtllm:
cuda13.1:
Expand Down
8 changes: 6 additions & 2 deletions container/templates/sglang_runtime.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -132,12 +132,16 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \

{% if context.sglang.enable_modelexpress == "true" %}
# Install only the ModelExpress client package. --no-deps preserves the upstream
# SGLang runtime dependency stack.
# SGLang runtime dependency stack. google-crc32c is imported eagerly by the MX
# sglang loader (>=0.5.0) and is not in the SGLang base image, so install it
# alongside; the import check below fails the build on any future --no-deps gap.
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
set -eux; \
export PIP_CACHE_DIR=/root/.cache/pip; \
pip install --break-system-packages --no-deps \
"modelexpress==${MODELEXPRESS_VERSION}"
"modelexpress==${MODELEXPRESS_VERSION}"; \
pip install --break-system-packages "google-crc32c>=1.5.0"; \
python3 -c "import modelexpress.engines.sglang"
{% endif %}
{% endif %}
{% endif %}
Expand Down
6 changes: 1 addition & 5 deletions lib/llm/src/hub.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,11 +160,7 @@ pub async fn from_hf(name: impl AsRef<Path>, ignore_weights: bool) -> anyhow::Re
Ok(mut client) => {
tracing::info!("Successfully connected to ModelExpress server");
match client
.request_model_with_provider_and_fallback(
&model_name,
MxModelProvider::HuggingFace,
ignore_weights,
)
.request_model(&model_name, MxModelProvider::HuggingFace, ignore_weights)
.await
{
Ok(()) => {
Expand Down
Loading