Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion components/backends/trtllm/multinode/multinode-examples.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ export MOUNTS="${PWD}/../:/mnt"

# NOTE: In general, Deepseek R1 is very large, so it is recommended to
# pre-download the model weights and save them in some shared location,
# NFS storage, HF_CACHE, etc. and modify the `--model-path` below
# NFS storage, HF_HOME, etc. and modify the `--model-path` below
# to reuse the pre-downloaded weights instead.
#
# On Blackwell systems (ex: GB200), it is recommended to use the FP4 weights:
Expand Down
2 changes: 1 addition & 1 deletion components/backends/trtllm/performance_sweeps/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ export IMAGE="<dynamo_trtllm_image>"

# NOTE: In general, Deepseek R1 is very large, so it is recommended to
# pre-download the model weights and save them in some shared location,
# NFS storage, HF_CACHE, etc. and modify the `--model-path` below
# NFS storage, HF_HOME, etc. and modify the `--model-path` below
# to reuse the pre-downloaded weights instead.
#
# On Blackwell systems (ex: GB200), it is recommended to use the FP4 weights:
Expand Down
28 changes: 14 additions & 14 deletions container/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ DEFAULT_FRAMEWORK=VLLM
SOURCE_DIR=$(dirname "$(readlink -f "$0")")

IMAGE=
HF_CACHE=
DEFAULT_HF_CACHE=${SOURCE_DIR}/.cache/huggingface
HF_HOME=${HF_HOME:-}
DEFAULT_HF_HOME=${SOURCE_DIR}/.cache/huggingface
GPUS="all"
PRIVILEGED=
VOLUME_MOUNTS=
Expand Down Expand Up @@ -85,9 +85,9 @@ get_options() {
missing_requirement "$1"
fi
;;
--hf-cache)
--hf-cache|--hf-home)
if [ "$2" ]; then
HF_CACHE=$2
HF_HOME=$2
shift
else
missing_requirement "$1"
Expand Down Expand Up @@ -250,8 +250,8 @@ get_options() {
VOLUME_MOUNTS+=" -v /tmp:/tmp "
VOLUME_MOUNTS+=" -v /mnt/:/mnt "

if [ -z "$HF_CACHE" ]; then
HF_CACHE=$DEFAULT_HF_CACHE
if [ -z "$HF_HOME" ]; then
HF_HOME=$DEFAULT_HF_HOME
fi

if [ -z "${PRIVILEGED}" ]; then
Expand All @@ -263,19 +263,19 @@ get_options() {
INTERACTIVE=" -it "
fi

if [[ ${HF_CACHE^^} == "NONE" ]]; then
HF_CACHE=
if [[ ${HF_HOME^^} == "NONE" ]]; then
HF_HOME=
fi

if [ -n "$HF_CACHE" ]; then
mkdir -p "$HF_CACHE"
if [ -n "$HF_HOME" ]; then
mkdir -p "$HF_HOME"
# Use /home/ubuntu for local-dev target, /root for dev target.
if [ "$TARGET" = "local-dev" ] || [[ "$IMAGE" == *"local-dev"* ]]; then
HF_CACHE_TARGET="/home/ubuntu/.cache/huggingface"
HF_HOME_TARGET="/home/ubuntu/.cache/huggingface"
else
HF_CACHE_TARGET="/root/.cache/huggingface"
HF_HOME_TARGET="/root/.cache/huggingface"
fi
VOLUME_MOUNTS+=" -v $HF_CACHE:$HF_CACHE_TARGET"
VOLUME_MOUNTS+=" -v $HF_HOME:$HF_HOME_TARGET"
fi

if [ -z "${PRIVILEGED}" ]; then
Expand Down Expand Up @@ -325,7 +325,7 @@ show_help() {
echo " [--name name for launched container, default NONE]"
echo " [--privileged whether to launch in privileged mode, default FALSE unless mounting workspace]"
echo " [--dry-run print docker commands without running]"
echo " [--hf-cache directory to volume mount as the hf cache, default is NONE unless mounting workspace]"
echo " [--hf-home|--hf-cache directory to volume mount as the hf home, default is NONE unless mounting workspace]"
echo " [--gpus gpus to enable, default is 'all', 'none' disables gpu support]"
echo " [--use-nixl-gds add volume mounts and capabilities needed for NVIDIA GPUDirect Storage]"
echo " [--network network mode for container, default is 'host']"
Expand Down
Loading