ai-dynamo · keivenchang · Sep 5, 2025
@@ -280,23 +280,63 @@ RUN --mount=type=bind,source=./container/launch_message.txt,target=/workspace/la
 ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
 CMD []
 
-#######################################
-########## Local Development #######
-#######################################
+#######################################################################
+########## DEVELOPMENT TARGETS FEATURE MATRIX #########################
+#######################################################################
+# Feature              │ local-dev Target    │ dev Target
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Purpose              │ Dev Container       │ Command-line with
+#                      │ plugin use only     │ run.sh script
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Default User         │ ubuntu user         │ root user
+# ─────────────────────┼─────────────────────┼─────────────────────
+# User Setup           │ Full ubuntu user    │ No user setup
+#                      │ with UID/GID        │
+#                      │ mapping             │
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Permissions          │ ubuntu user with    │ Root-level
+#                      │ sudo privileges     │ permissions
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Home Directory       │ /home/ubuntu        │ /root
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Working Directory    │ /home/ubuntu/dynamo │ /workspace
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Rust Toolchain       │ User's home         │ System locations
+#                      │ (~/.rustup,         │ (/usr/local/rustup,
+#                      │  ~/.cargo)          │  /usr/local/cargo)
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Python Environment   │ User-owned venv     │ System location
+#                      │                     │ (/opt/dynamo/venv)
+# ─────────────────────┼─────────────────────┼─────────────────────
+# File Permissions     │ User-level with     │ Root-level
+#                      │ proper ownership    │ permissions
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Compatibility        │ MS Plug-in: Dev     │ Backward compatibility
+#                      │ Container workflow  │ with existing
+#                      │                     │ workflows
 #
-# PURPOSE: Local development
-#
-# This stage adds development tools, utilities, and dependencies specifically
-# needed for:
-# - Local development and debugging
-# - vscode/cursor development
+# USAGE GUIDELINES:
+# • Use local-dev: VS Code/Cursor Dev Container plugin only
+# • Use dev: run.sh script for command-line development
+
+
+#######################################################################
+########## Development (Dev Container only) ###########################
+#######################################################################
 #
-# Use this stage when you need a full development environment with additional
-# tooling beyond the base runtime image.
+# This stage is for Dev Container plug-in use only.
+# It provides a local development environment with extra tools and dependencies
+# not present in the base runtime image.
 
-FROM runtime AS dev
+FROM runtime AS local-dev
 
-# Install utilities
+# Don't want ubuntu to be editable, just change uid and gid.
+ENV USERNAME=ubuntu
+ARG USER_UID
+ARG USER_GID
+ARG WORKSPACE_DIR=/workspace
+
+# Install utilities as root
 RUN apt-get update -y && \
     apt-get install -y --no-install-recommends  \
     # Install utilities
@@ -325,46 +365,59 @@ RUN apt-get update -y && \
     protobuf-compiler && \
     rm -rf /var/lib/apt/lists/*
 
-# Rust environment setup
-ENV RUSTUP_HOME=/usr/local/rustup \
-    CARGO_HOME=/usr/local/cargo \
-    CARGO_TARGET_DIR=/opt/dynamo/target \
-    PATH=/usr/local/cargo/bin:$PATH
-
-COPY --from=dynamo_base $RUSTUP_HOME $RUSTUP_HOME
-COPY --from=dynamo_base $CARGO_HOME $CARGO_HOME
+COPY --from=runtime /usr/local/bin /usr/local/bin
 
 # https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user
 # Will use the default ubuntu user, but give sudo access
 # Needed so files permissions aren't set to root ownership when writing from inside container
-
-# Don't want ubuntu to be editable, just change uid and gid. User ubuntu is hardcoded in .devcontainer
-ENV USERNAME=ubuntu
-ARG USER_UID=1000
-ARG USER_GID=1000
-
 RUN apt-get update && apt-get install -y sudo gnupg2 gnupg1 \
     && echo "$USERNAME ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/$USERNAME \
     && chmod 0440 /etc/sudoers.d/$USERNAME \
     && mkdir -p /home/$USERNAME \
+    && groupmod -g $USER_GID $USERNAME \
+    && usermod -u $USER_UID -g $USER_GID $USERNAME \
     && chown -R $USERNAME:$USERNAME /home/$USERNAME \
     && rm -rf /var/lib/apt/lists/* \
     && chsh -s /bin/bash $USERNAME
 
+# At this point, we are executing as the ubuntu user
+USER $USERNAME
+ENV HOME=/home/$USERNAME
+WORKDIR $HOME
+
+# Set workspace directory variable
+ENV WORKSPACE_DIR=${WORKSPACE_DIR}
+
+# Development environment variables for the dev target
+# Path configuration notes:
+# - DYNAMO_HOME: Main project directory (workspace mount point)
+# - CARGO_TARGET_DIR: Build artifacts in workspace/target for persistence
+# - CARGO_HOME: Must be in $HOME/.cargo (not workspace) because:
+#               * Workspace gets mounted to different paths where cargo binaries may not exist
+#               * Contains critical cargo binaries and registry that need consistent paths
+# - RUSTUP_HOME: Must be in $HOME/.rustup (not workspace) because:
+#                * Contains rust toolchain binaries that must be at expected system paths
+#                * Workspace mount point would break rustup's toolchain resolution
+# - PATH: Includes cargo binaries for rust tool access
+ENV DYNAMO_HOME=${WORKSPACE_DIR}
+ENV CARGO_TARGET_DIR=${WORKSPACE_DIR}/target
+ENV CARGO_HOME=${HOME}/.cargo
+ENV RUSTUP_HOME=${HOME}/.rustup
+ENV PATH=${CARGO_HOME}/bin:$PATH
+
+COPY --from=dynamo_base --chown=$USER_UID:$USER_GID /usr/local/rustup $RUSTUP_HOME
+COPY --from=dynamo_base --chown=$USER_UID:$USER_GID /usr/local/cargo $CARGO_HOME
+
 # This is a slow operation (~40s on my cpu)
 # Much better than chown -R $USERNAME:$USERNAME /opt/dynamo/venv (~10min on my cpu)
 COPY --from=runtime --chown=$USER_UID:$USER_GID ${VIRTUAL_ENV} ${VIRTUAL_ENV}
-RUN chown $USERNAME:$USERNAME ${VIRTUAL_ENV}
-COPY --from=runtime --chown=$USERNAME:$USERNAME /usr/local/bin /usr/local/bin
 
 # so we can use maturin develop
 RUN uv pip install maturin[patchelf]
 
-USER $USERNAME
-ENV HOME=/home/$USERNAME
-ENV PYTHONPATH=$PYTHONPATH:$HOME/dynamo/components/planner/src
-ENV CARGO_TARGET_DIR=$HOME/dynamo/.build/target
-WORKDIR $HOME
+# Make sure to sync this with the one specified on README.md.
+# This is a generic PYTHONPATH which works for all the frameworks, so some paths may not be relevant for this particular framework.
+ENV PYTHONPATH=${WORKSPACE_DIR}/components/metrics/src:${WORKSPACE_DIR}/components/frontend/src:${WORKSPACE_DIR}/components/planner/src:${WORKSPACE_DIR}/components/backends/mocker/src:${WORKSPACE_DIR}/components/backends/trtllm/src:${WORKSPACE_DIR}/components/backends/vllm/src:${WORKSPACE_DIR}/components/backends/sglang/src:${WORKSPACE_DIR}/components/backends/llama_cpp/src
 
 # https://code.visualstudio.com/remote/advancedcontainers/persist-bash-history
 RUN SNIPPET="export PROMPT_COMMAND='history -a' && export HISTFILE=$HOME/.commandhistory/.bash_history" \
@@ -376,3 +429,79 @@ RUN mkdir -p /home/$USERNAME/.cache/
 
 ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
 CMD []
+
+
+###########################################################
+########## Development (run.sh, runs as root user) ########
+###########################################################
+#
+# PURPOSE: Local development environment for use with run.sh (not Dev Container plug-in)
+#
+# This stage runs as root and provides:
+# - Development tools and utilities for local debugging
+# - Support for vscode/cursor development outside the Dev Container plug-in
+#
+# Use this stage if you need a full-featured development environment with extra tools,
+# but do not use it with the Dev Container plug-in.
+
+FROM runtime AS dev
+
+# Don't want ubuntu to be editable, just change uid and gid.
+ARG WORKSPACE_DIR=/workspace
+
+# Install utilities as root
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends  \
+    # Install utilities
+    nvtop \
+    wget \
+    tmux \
+    vim \
+    git \
+    openssh-client \
+    iproute2 \
+    rsync \
+    zip \
+    unzip \
+    htop \
+    # Build Dependencies
+    autoconf \
+    automake \
+    cmake \
+    libtool \
+    meson \
+    net-tools \
+    pybind11-dev \
+    # Rust build dependencies
+    clang \
+    libclang-dev \
+    protobuf-compiler && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY --from=runtime /usr/local/bin /usr/local/bin
+
+# Set workspace directory variable
+ENV WORKSPACE_DIR=${WORKSPACE_DIR} \
+    DYNAMO_HOME=${WORKSPACE_DIR} \
+    RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    CARGO_TARGET_DIR=/workspace/target \
+    VIRTUAL_ENV=/opt/dynamo/venv \
+    PATH=/usr/local/cargo/bin:$PATH
+
+COPY --from=dynamo_base /usr/local/rustup /usr/local/rustup
+COPY --from=dynamo_base /usr/local/cargo /usr/local/cargo
+
+# This is a slow operation (~40s on my cpu)
+# Much better than chown -R $USERNAME:$USERNAME /opt/dynamo/venv (~10min on my cpu)
+COPY --from=runtime ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+
+# so we can use maturin develop
+RUN uv pip install maturin[patchelf]
+
+# Make sure to sync this with the one specified on README.md.
+# This is a generic PYTHONPATH which works for all the frameworks, so some paths may not be relevant for this particular framework.
+ENV PYTHONPATH=${WORKSPACE_DIR}/components/metrics/src:${WORKSPACE_DIR}/components/frontend/src:${WORKSPACE_DIR}/components/planner/src:${WORKSPACE_DIR}/components/backends/mocker/src:${WORKSPACE_DIR}/components/backends/trtllm/src:${WORKSPACE_DIR}/components/backends/vllm/src:${WORKSPACE_DIR}/components/backends/sglang/src:${WORKSPACE_DIR}/components/backends/llama_cpp/src
+
+ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
+CMD []
@@ -443,7 +443,6 @@ error() {
 
 get_options "$@"
 
-
 # Automatically set ARCH and ARCH_ALT if PLATFORM is linux/arm64
 ARCH="amd64"
 if [[ "$PLATFORM" == *"linux/arm64"* ]]; then
@@ -465,7 +464,7 @@ fi
 # Add NIXL_REF as a build argument
 BUILD_ARGS+=" --build-arg NIXL_REF=${NIXL_REF} "
 
-if [[ $TARGET == "dev" ]]; then
+if [[ $TARGET == "local-dev" ]]; then
     BUILD_ARGS+=" --build-arg USER_UID=$(id -u) --build-arg USER_GID=$(id -g) "
 fi
 
@@ -599,8 +598,6 @@ if [ "$USE_SCCACHE" = true ]; then
     BUILD_ARGS+=" --build-arg USE_SCCACHE=true"
     BUILD_ARGS+=" --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}"
     BUILD_ARGS+=" --build-arg SCCACHE_REGION=${SCCACHE_REGION}"
-
-
 fi
 
 LATEST_TAG="--tag dynamo:latest-${FRAMEWORK,,}"

@@ -43,7 +43,6 @@ INTERACTIVE=
 USE_NIXL_GDS=
 RUNTIME=nvidia
 WORKDIR=/workspace
-USER=
 
 get_options() {
     while :; do
@@ -125,14 +124,6 @@ get_options() {
                 missing_requirement "$1"
             fi
             ;;
-        --user)
-            if [ "$2" ]; then
-                USER="$2"
-                shift
-            else
-                missing_requirement "$1"
-            fi
-            ;;
         --privileged)
             if [ "$2" ]; then
                 PRIVILEGED=$2
@@ -283,12 +274,6 @@ get_options() {
         RM_STRING=" --rm "
     fi
 
-    if [[ ${USER} == "" ]]; then
-        USER_STRING=""
-    else
-        USER_STRING="--user ${USER}"
-    fi
-
     if [ -n "$USE_NIXL_GDS" ]; then
         VOLUME_MOUNTS+=" -v /run/udev:/run/udev:ro "
         NIXL_GDS_CAPS="--cap-add=IPC_LOCK"
@@ -306,14 +291,15 @@ get_options() {
     if [[ "$GPUS" == "none" || "$GPUS" == "NONE" ]]; then
             RUNTIME=""
     fi
+
     REMAINING_ARGS=("$@")
 }
 
 show_help() {
     echo "usage: run.sh"
     echo "  [--image image]"
     echo "  [--framework framework one of ${!FRAMEWORKS[*]}]"
-    echo "  [--name name for launched container, default NONE] "
+    echo "  [--name name for launched container, default NONE]"
     echo "  [--privileged whether to launch in privileged mode, default FALSE unless mounting workspace]"
     echo "  [--dry-run print docker commands without running]"
     echo "  [--hf-cache directory to volume mount as the hf cache, default is NONE unless mounting workspace]"
@@ -325,7 +311,8 @@ show_help() {
     echo "  [-- stop processing and pass remaining args as command to docker run]"
     echo "  [--workdir set the working directory inside the container]"
     echo "  [--runtime add runtime variables]"
-    echo "  [--user override the user for running the container]"
+    echo "  [--entrypoint override container entrypoint]"
+    echo "  [-h, --help show this help]"
     exit 0
 }
 
@@ -341,7 +328,6 @@ error() {
 get_options "$@"
 
 # RUN the image
-
 if [ -z "$RUN_PREFIX" ]; then
     set -x
 fi
@@ -363,7 +349,6 @@ ${RUN_PREFIX} docker run \
     ${NIXL_GDS_CAPS} \
     --ipc host \
     ${PRIVILEGED_STRING} \
-    ${USER_STRING} \
     ${NAME_STRING} \
     ${ENTRYPOINT_STRING} \
     ${IMAGE} \