feat(sidecar): add e2e CI testing for sidecar launch scripts

.github/workflows/post-merge-ci.yml

@@ -214,6 +214,10 @@ jobs:
       extra_tags: |
         ${{ github.ref_name == 'main' && 'main-vllm-runtime' || '' }}
         ${{ github.ref_name == 'main' && format('main-vllm-runtime-{0}', github.sha) || '' }}
+      # Lets sidecar-vllm-test-image (pr.yaml) layer onto the latest main -test
+      # image instead of rebuilding vllm-runtime on every lib/sidecar/**-only PR.
+      test_extra_tags: |
+        ${{ github.ref_name == 'main' && 'main-vllm-runtime-test' || '' }}
       archive_sources: false
     secrets: inherit
 
@@ -279,6 +283,10 @@ jobs:
       extra_tags: |
         ${{ github.ref_name == 'main' && 'main-sglang-runtime' || '' }}
         ${{ github.ref_name == 'main' && format('main-sglang-runtime-{0}', github.sha) || '' }}
+      # Lets sidecar-sglang-test-image (pr.yaml) layer onto the latest main -test
+      # image instead of rebuilding sglang-runtime on every lib/sidecar/**-only PR.
+      test_extra_tags: |
+        ${{ github.ref_name == 'main' && 'main-sglang-runtime-test' || '' }}
       archive_sources: false
     secrets: inherit
 
@@ -344,6 +352,10 @@ jobs:
       extra_tags: |
         ${{ github.ref_name == 'main' && 'main-trtllm-runtime' || '' }}
         ${{ github.ref_name == 'main' && format('main-trtllm-runtime-{0}', github.sha) || '' }}
+      # Lets sidecar-trtllm-test-image (pr.yaml) layer onto the latest main -test
+      # image instead of rebuilding trtllm-runtime on every lib/sidecar/**-only PR.
+      test_extra_tags: |
+        ${{ github.ref_name == 'main' && 'main-trtllm-runtime-test' || '' }}
       archive_sources: false
     secrets: inherit
 

.github/workflows/pr.yaml

@@ -133,6 +133,12 @@
       - snapshot-placeholder-trtllm
       - power-agent
       - sidecar-build
+      - sidecar-vllm-test-image
+      - sidecar-sglang-test-image
+      - sidecar-trtllm-test-image
+      - sidecar-vllm-test
+      - sidecar-sglang-test
+      - sidecar-trtllm-test
       - vllm-build
       - vllm-local-dev-build
       - sglang-local-dev-build
@@ -358,14 +364,257 @@
       diff_base_branch: ${{ needs.changed-files.outputs.base_ref }}
       diff_merge_base_sha: ${{ needs.changed-files.outputs.merge_base_sha }}
 
+  # --------------------------------------------------------------------------
+  # Sidecar E2E: lib/sidecar/*/launch/agg.sh, run against a CI-only image that
+  # layers the dynamo-*-sidecar binary onto the backend's already-built -test
+  # image (neither the sidecar image nor the backend runtime images bundle
+  # both today — see lib/sidecar/README.md and lib/sidecar/ci/sidecar-test-image.Dockerfile).
+  # Gated on `sidecar` only (lib/sidecar/** changes), not on `core`/vllm/sglang/
+  # trtllm — a lib/sidecar/**-only PR does not trigger vllm-build/sglang-build/
+  # trtllm-build, so the "Calculate tags" step in each *-test-image job below
+  # falls back to the main-{framework}-runtime-test floating tag (kept fresh by
+  # post-merge-ci.yml's test_extra_tags) instead of rebuilding the runtime image
+  # just to layer a sidecar binary on top of it.
+  # gpu_1 fan-out is a deliberate exception to the narrow-scope TODO on the
+  # `sidecar` filter in .github/filters.yaml; revisit if CI cost becomes an issue.
+  # --------------------------------------------------------------------------
+
+  sidecar-vllm-test-image:
+    name: dynamo-sidecar (vllm test image)
+    needs: [changed-files, vllm-build, sidecar-build]
+    # vllm-build and sidecar-build both also trigger on `sidecar` (see their
+    # own if: conditions), so both are guaranteed to have run whenever this does.
+    if: needs.changed-files.outputs.sidecar == 'true'
+    runs-on: prod-builder-v3
+    permissions:
+      contents: read
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+    - name: Docker Login
+      uses: $/.github/actions/docker-login
+      with:
+        aws_default_region: ${{ vars.AWS_DEFAULT_REGION }}
+        ecr_registry: ${{ vars.ECR_REGISTRY }}
+    - name: Initialize Dynamo Builder
+      uses: $/.github/actions/init-dynamo-builder
+      with:
+        builder_name: ${{ needs.changed-files.outputs.builder_name }}
+        flavor: general
+        arch: linux/amd64
+        namespace: ${{ vars.BUILDKIT_NAMESPACE }}
+    - name: Calculate tags
+      id: tags
+      env:
+        ECR_REGISTRY: ${{ vars.ECR_REGISTRY }}
+        ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }}
+      run: |
+        set -euo pipefail
+        # This job runs on any lib/sidecar/** change alone, which does not
+        # trigger vllm-build. When vllm-build didn't run (or failed) this PR,
+        # reuse the latest main-vllm-runtime-test floating tag (kept fresh by
+        # post-merge-ci.yml) instead of rebuilding vllm-runtime just to layer
+        # a sidecar binary on top of it.
+        if [ "${{ needs.vllm-build.result }}" = "success" ]; then
+          echo "base_image=${ECR_REGISTRY}/${ECR_REPOSITORY}:${GITHUB_SHA}-${{ needs.vllm-build.outputs.target_tag_plain }}-test" >> "${GITHUB_OUTPUT}"
+        else
+          echo "base_image=${ECR_REGISTRY}/${ECR_REPOSITORY}:main-vllm-runtime-test" >> "${GITHUB_OUTPUT}"
+        fi
+        # Must stay in sync with shared-build-sidecar.yml's calculate-target-tag step.
+        echo "sidecar_image=${ECR_REGISTRY}/${ECR_REPOSITORY}:${GITHUB_SHA}-dynamo-sidecar" >> "${GITHUB_OUTPUT}"
+        echo "image_tag=${ECR_REGISTRY}/${ECR_REPOSITORY}:${GITHUB_SHA}-vllm-sidecar-test" >> "${GITHUB_OUTPUT}"
+    - name: Build and Push sidecar test image
+      uses: $/.github/actions/docker-remote-build
+      with:
+        image_tag: ${{ steps.tags.outputs.image_tag }}
+        framework: dynamo-sidecar-test
+        platform: linux/amd64
+        cuda_version: ''
+        dockerfile: lib/sidecar/ci/sidecar-test-image.Dockerfile
+        aws_default_region: ${{ vars.AWS_DEFAULT_REGION }}
+        extra_build_args: |
+          BASE_IMAGE=${{ steps.tags.outputs.base_image }}
+          SIDECAR_IMAGE=${{ steps.tags.outputs.sidecar_image }}
+          BACKEND=vllm
+        push_image: true
+        no_load: true
+        use_sccache: false
+
+  sidecar-vllm-test:
+    name: sidecar-runtime # This name overlaps with other sidecar jobs to group them in the UI
+    needs: [changed-files, sidecar-vllm-test-image]
+    uses: $/.github/workflows/shared-test.yml
+    with:
+      test_suite_name: vllm
+      test_type: Sidecar E2E Test
+      amd_runner: prod-tester-amd-gpu-v2
+      target_tag_plain: vllm-sidecar
+      cuda_version: '["13.0"]'
+      platform: '["amd64"]'
+      run_sanity_check: false
+      gpu_test_markers: pre_merge and sidecar and vllm and gpu_1
+      gpu_test_timeout_minutes: 30
+    secrets: inherit
+
+  sidecar-sglang-test-image:
+    name: dynamo-sidecar (sglang test image)
+    needs: [changed-files, sglang-build, sidecar-build]
+    # sglang-build and sidecar-build both also trigger on `sidecar` (see their
+    # own if: conditions), so both are guaranteed to have run whenever this does.
+    if: needs.changed-files.outputs.sidecar == 'true'
+    runs-on: prod-builder-v3
+    permissions:
+      contents: read
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+    - name: Docker Login
+      uses: $/.github/actions/docker-login
+      with:
+        aws_default_region: ${{ vars.AWS_DEFAULT_REGION }}
+        ecr_registry: ${{ vars.ECR_REGISTRY }}
+    - name: Initialize Dynamo Builder
+      uses: $/.github/actions/init-dynamo-builder
+      with:
+        builder_name: ${{ needs.changed-files.outputs.builder_name }}
+        flavor: general
+        arch: linux/amd64
+        namespace: ${{ vars.BUILDKIT_NAMESPACE }}
+    - name: Calculate tags
+      id: tags
+      env:
+        ECR_REGISTRY: ${{ vars.ECR_REGISTRY }}
+        ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }}
+      run: |
+        set -euo pipefail
+        # This job runs on any lib/sidecar/** change alone, which does not
+        # trigger sglang-build. When sglang-build didn't run (or failed) this
+        # PR, reuse the latest main-sglang-runtime-test floating tag (kept
+        # fresh by post-merge-ci.yml) instead of rebuilding sglang-runtime
+        # just to layer a sidecar binary on top of it.
+        if [ "${{ needs.sglang-build.result }}" = "success" ]; then
+          echo "base_image=${ECR_REGISTRY}/${ECR_REPOSITORY}:${GITHUB_SHA}-${{ needs.sglang-build.outputs.target_tag_plain }}-test" >> "${GITHUB_OUTPUT}"
+        else
+          echo "base_image=${ECR_REGISTRY}/${ECR_REPOSITORY}:main-sglang-runtime-test" >> "${GITHUB_OUTPUT}"
+        fi
+        # Must stay in sync with shared-build-sidecar.yml's calculate-target-tag step.
+        echo "sidecar_image=${ECR_REGISTRY}/${ECR_REPOSITORY}:${GITHUB_SHA}-dynamo-sidecar" >> "${GITHUB_OUTPUT}"
+        echo "image_tag=${ECR_REGISTRY}/${ECR_REPOSITORY}:${GITHUB_SHA}-sglang-sidecar-test" >> "${GITHUB_OUTPUT}"
+    - name: Build and Push sidecar test image
+      uses: $/.github/actions/docker-remote-build
+      with:
+        image_tag: ${{ steps.tags.outputs.image_tag }}
+        framework: dynamo-sidecar-test
+        platform: linux/amd64
+        cuda_version: ''
+        dockerfile: lib/sidecar/ci/sidecar-test-image.Dockerfile
+        aws_default_region: ${{ vars.AWS_DEFAULT_REGION }}
+        extra_build_args: |
+          BASE_IMAGE=${{ steps.tags.outputs.base_image }}
+          SIDECAR_IMAGE=${{ steps.tags.outputs.sidecar_image }}
+          BACKEND=sglang
+        push_image: true
+        no_load: true
+        use_sccache: false
+
+  sidecar-sglang-test:
+    name: sidecar-runtime # This name overlaps with other sidecar jobs to group them in the UI
+    needs: [changed-files, sidecar-sglang-test-image]
+    uses: $/.github/workflows/shared-test.yml
+    with:
+      test_suite_name: sglang
+      test_type: Sidecar E2E Test
+      amd_runner: prod-tester-amd-gpu-v2
+      target_tag_plain: sglang-sidecar
+      cuda_version: '["13.0"]'
+      platform: '["amd64"]'
+      run_sanity_check: false
+      gpu_test_markers: pre_merge and sidecar and sglang and gpu_1
+      gpu_test_timeout_minutes: 30
+    secrets: inherit
+
+  sidecar-trtllm-test-image:
+    name: dynamo-sidecar (trtllm test image)
+    needs: [changed-files, trtllm-build, sidecar-build]
+    # trtllm-build and sidecar-build both also trigger on `sidecar` (see their
+    # own if: conditions), so both are guaranteed to have run whenever this does.
+    if: needs.changed-files.outputs.sidecar == 'true'
+    runs-on: prod-builder-v3
+    permissions:
+      contents: read
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+    - name: Docker Login
+      uses: $/.github/actions/docker-login
+      with:
+        aws_default_region: ${{ vars.AWS_DEFAULT_REGION }}
+        ecr_registry: ${{ vars.ECR_REGISTRY }}
+    - name: Initialize Dynamo Builder
+      uses: $/.github/actions/init-dynamo-builder
+      with:
+        builder_name: ${{ needs.changed-files.outputs.builder_name }}
+        flavor: general
+        arch: linux/amd64
+        namespace: ${{ vars.BUILDKIT_NAMESPACE }}
+    - name: Calculate tags
+      id: tags
+      env:
+        ECR_REGISTRY: ${{ vars.ECR_REGISTRY }}
+        ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }}
+      run: |
+        set -euo pipefail
+        # This job runs on any lib/sidecar/** change alone, which does not
+        # trigger trtllm-build. When trtllm-build didn't run (or failed) this
+        # PR, reuse the latest main-trtllm-runtime-test floating tag (kept
+        # fresh by post-merge-ci.yml) instead of rebuilding trtllm-runtime
+        # just to layer a sidecar binary on top of it.
+        if [ "${{ needs.trtllm-build.result }}" = "success" ]; then
+          echo "base_image=${ECR_REGISTRY}/${ECR_REPOSITORY}:${GITHUB_SHA}-${{ needs.trtllm-build.outputs.target_tag_plain }}-test" >> "${GITHUB_OUTPUT}"
+        else
+          echo "base_image=${ECR_REGISTRY}/${ECR_REPOSITORY}:main-trtllm-runtime-test" >> "${GITHUB_OUTPUT}"
+        fi
+        # Must stay in sync with shared-build-sidecar.yml's calculate-target-tag step.
+        echo "sidecar_image=${ECR_REGISTRY}/${ECR_REPOSITORY}:${GITHUB_SHA}-dynamo-sidecar" >> "${GITHUB_OUTPUT}"
+        echo "image_tag=${ECR_REGISTRY}/${ECR_REPOSITORY}:${GITHUB_SHA}-trtllm-sidecar-test" >> "${GITHUB_OUTPUT}"
+    - name: Build and Push sidecar test image
+      uses: $/.github/actions/docker-remote-build
+      with:
+        image_tag: ${{ steps.tags.outputs.image_tag }}
+        framework: dynamo-sidecar-test
+        platform: linux/amd64
+        cuda_version: ''
+        dockerfile: lib/sidecar/ci/sidecar-test-image.Dockerfile
+        aws_default_region: ${{ vars.AWS_DEFAULT_REGION }}
+        extra_build_args: |
+          BASE_IMAGE=${{ steps.tags.outputs.base_image }}
+          SIDECAR_IMAGE=${{ steps.tags.outputs.sidecar_image }}
+          BACKEND=trtllm
+        push_image: true
+        no_load: true
+        use_sccache: false
+
+  sidecar-trtllm-test:
+    name: sidecar-runtime # This name overlaps with other sidecar jobs to group them in the UI
+    needs: [changed-files, sidecar-trtllm-test-image]
+    uses: $/.github/workflows/shared-test.yml
+    with:
+      test_suite_name: trtllm
+      test_type: Sidecar E2E Test
+      amd_runner: prod-tester-amd-gpu-v2
+      target_tag_plain: trtllm-sidecar
+      cuda_version: '["13.0"]'
+      platform: '["amd64"]'
+      run_sanity_check: false
+      gpu_test_markers: pre_merge and sidecar and trtllm and gpu_1
+      gpu_test_timeout_minutes: 30
+    secrets: inherit
+
 # ============================================================================
 # FRAMEWORK PIPELINES (Build → Test → Copy)
 # ============================================================================
   # ============================================================================
   # BUILD PIPELINES
   # ============================================================================
 
   vllm-build:
     name: vllm-runtime # This name overlaps with other vllm jobs to group them in the UI
     needs: [changed-files]
     # `sample` is here because sample tests piggyback on this image and run as part of vllm-test.

.github/workflows/shared-build-image.yml

@@ -37,6 +37,11 @@
         required: false
         type: string
         default: ''
+      test_extra_tags:
+        description: 'Additional tags for the -test image only (newline-separated plain tag names, no ECR prefix). Unlike extra_tags (runtime image), the -test image otherwise gets no floating tag.'
+        required: false
+        type: string
+        default: ''
       no_cache:
         description: 'Disable Docker build cache'
         required: false
@@ -603,6 +608,10 @@
       - name: Build and Push Test Image
         if: ${{ !inputs.compliance_only && inputs.target != 'dev' && inputs.target != 'local-dev' && inputs.target != 'frontend' }}
         shell: bash
+        env:
+          ECR_REGISTRY: ${{ vars.ECR_REGISTRY }}
+          ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }}
+          TEST_EXTRA_TAGS: ${{ inputs.test_extra_tags }}
         run: |
           PUSH_ARGS=""
           if [ "${{ inputs.push_image }}" == "true" ]; then
@@ -630,6 +639,15 @@
             PYPI_SECRET_ARGS+=(--secret "id=pypi-netrc,src=${NETRC}")
           fi
 
+          TEST_TAG_ARGS=(-t "${{ steps.calculate-target-tag.outputs.test_image_uri }}")
+          if [ -n "$TEST_EXTRA_TAGS" ]; then
+            while IFS= read -r tag; do
+              if [ -n "$tag" ]; then
+                TEST_TAG_ARGS+=(-t "${ECR_REGISTRY}/${ECR_REPOSITORY}:${tag}")
+              fi
+            done <<< "$TEST_EXTRA_TAGS"
+          fi
+
           docker buildx build \
             --progress=plain \
             ${PUSH_ARGS} \
@@ -639,7 +657,7 @@
             --target ${TEST_TARGET} \
             -f container/Dockerfile.test \
             --build-arg BASE_IMAGE=${{ steps.calculate-target-tag.outputs.image_uri }} \
-            -t ${{ steps.calculate-target-tag.outputs.test_image_uri }} .
+            "${TEST_TAG_ARGS[@]}" .
       - name: Show summary
         shell: bash
         if: ${{ inputs.push_image == 'true' && inputs.show_summary == 'true' }}

lib/sidecar/ci/sidecar-test-image.Dockerfile

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# CI-only: layers a dynamo-<BACKEND>-sidecar binary onto an already-built
+# backend runtime test image, so tests/serve/test_{vllm,sglang,trtllm}_sidecar.py
+# can run lib/sidecar/<backend>/launch/agg.sh end-to-end. Not used for any
+# shipped image — see .github/workflows/pr.yaml sidecar-*-test-image jobs.
+
+ARG BASE_IMAGE
+ARG SIDECAR_IMAGE
+
+FROM ${SIDECAR_IMAGE} AS sidecar
+
+FROM ${BASE_IMAGE}
+ARG BACKEND
+COPY --from=sidecar /usr/local/bin/dynamo-${BACKEND}-sidecar /usr/local/bin/dynamo-${BACKEND}-sidecar
+
+# vllm-rs ships inside the vllm wheel, not on PATH — lib/sidecar/vllm/launch/agg.sh
+# expects it on PATH, but lib/sidecar/vllm/deploy/agg.yaml resolves it the same
+# way this does. A no-op for sglang/trtllm; each backend gets its own
+# conditional RUN block below rather than a shared one, so this stays one
+# Dockerfile for all three instead of three near-duplicate ones.
+RUN if [ "$BACKEND" = "vllm" ]; then \
+      ln -sf "$(python3 -c 'import os,vllm; print(os.path.join(os.path.dirname(vllm.__file__), "vllm-rs"))')" /usr/local/bin/vllm-rs; \
+    fi
+
+# smg-grpc-proto is TRT-LLM's optional `grpc-smg` extra; the 1.3.0rc25 runtime
+# image does not bundle it. lib/sidecar/trtllm/launch/agg.sh pip-installs it at
+# runtime if missing, which would otherwise be a live network fetch on every CI
+# run. Pre-baking it here with the same version constraint agg.sh uses makes
+# that check a no-op and keeps the test hermetic.
+RUN if [ "$BACKEND" = "trtllm" ]; then \
+      python3 -m pip install --no-cache-dir "smg-grpc-proto>=0.4.2"; \
+    fi

pyproject.toml

@@ -343,6 +343,7 @@ markers = [
     "mocker: marks tests for the mocker component",
     "router: marks tests for router component",
     "planner: marks tests for planner component",
+    "sidecar: marks tests that launch a Dynamo dynamo-*-sidecar binary against its native-gRPC engine via lib/sidecar/*/launch/*.sh",
     "kvbm: marks tests for KV behavior and model determinism",
     "kvbm_consolidator: marks KVBM consolidator/router integration tests",
     "kvbm_determinism: marks KVBM model determinism tests",