tanmayv25 · Sep 18, 2026 · Sep 9, 2026 · Sep 9, 2026 · Sep 9, 2026 · Sep 9, 2026
@@ -305,6 +305,32 @@ runs:
 
         exit ${TEST_EXIT_CODE}
 
+    # conftest.py's autouse `logger` fixture writes every test's full root-logger
+    # output (ManagedProcess/EngineProcess health-check status, retry reasons,
+    # subprocess log tails, ...) to /tmp/dynamo_tests/<test-name>/test.log.txt --
+    # a much richer trace than the JUnit XML's truncated captured-output, but
+    # nothing ever uploaded it. failure()-gated (not always()) to keep this
+    # opt-in-by-outcome: pytest-local is shared by every framework's CI, and
+    # most runs pass.
+    - name: Collect per-test log files
+      if: failure()
+      shell: bash
+      run: |
+        set -euo pipefail
+        mkdir -p test-results/dynamo-tests-logs
+        if [ -d /tmp/dynamo_tests ]; then
+          cp -r /tmp/dynamo_tests/. test-results/dynamo-tests-logs/
+        fi
+
+    - name: Upload per-test log files
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+      if: failure()
+      with:
+        name: dynamo-tests-logs-${{ inputs.test_suite_name }}-${{ inputs.test_type }}-${{ inputs.platform_arch }}-${{ github.run_id }}-${{ job.check_run_id }}
+        path: test-results/dynamo-tests-logs/
+        retention-days: 7
+        if-no-files-found: ignore
+
     # Allure results and coverage data are uploaded per stage, under the same
     # artifact names as before the step collapse, because an external CI-metrics
     # pipeline consumes them by name. `always()` because the step above exits

@@ -352,6 +352,7 @@ sidecar:
   # acceptable CI fan-out is agreed. Keep the remaining scope narrow for now.
   - 'lib/sidecar/**'
   - '.github/workflows/shared-build-sidecar.yml'
+  - 'tests/serve/test_sidecar.py'
   - 'Cargo.toml'
   - 'Cargo.lock'
   # Compliance policy and baseline changes can fail the sidecar image build.

@@ -816,7 +816,13 @@ jobs:
       enable_coverage: true
       run_cpu_only_tests: true
       cpu_only_test_markers: vllm and gpu_0
-      gpu_test_markers: vllm and gpu_1
+      # not sidecar: sidecar-marked tests need the dynamo-vllm-sidecar binary
+      # installed (see pr.yaml's sidecar-vllm-test), which this plain
+      # vllm-runtime image doesn't have.
+      # TODO: this ad-hoc exclusion is duplicated across pr.yaml, post-merge-ci.yml,
+      # and nightly-ci.yml's gpu_test_markers; consolidate into shared-test.yml
+      # once there's a second marker that needs the same treatment.
+      gpu_test_markers: vllm and gpu_1 and not sidecar
       gpu_test_timeout_minutes: 240
       # Profiled tests run in the parallel stage; unprofiled fall through to sequential.
       # 24 GiB admits all currently profiled vLLM tests (max is ~20.4 GiB) on a 48 GiB GPU.
@@ -891,7 +897,7 @@ jobs:
       platform: '["amd64"]'
       enable_coverage: true
       run_sanity_check: false
-      gpu_test_markers: vllm and (gpu_1 or gpu_2)
+      gpu_test_markers: vllm and (gpu_1 or gpu_2) and not sidecar
       gpu_test_timeout_minutes: 240
       # 80 GiB H100 admits more of the profiled pool per slot than the 24 GiB lanes.
       run_gpu_parallel_tests: true
@@ -919,7 +925,7 @@ jobs:
       run_cpu_only_tests: true
       cpu_only_test_markers: sglang and gpu_0
       # Exclude h100 tests: they need >24 GiB and run on the sglang-h100-test lane below.
-      gpu_test_markers: sglang and gpu_1 and not h100
+      gpu_test_markers: sglang and gpu_1 and not h100 and not sidecar
       gpu_test_timeout_minutes: 240
       # Profiled tests run in the VRAM-aware GPU stage; unprofiled fall through to sequential.
       # Current single-GPU runners are 24 GiB, so this cap admits the profiled SGLang pool
@@ -994,7 +1000,7 @@ jobs:
       platform: '["amd64"]'
       enable_coverage: true
       run_sanity_check: false
-      gpu_test_markers: sglang and (gpu_1 or gpu_2)
+      gpu_test_markers: sglang and (gpu_1 or gpu_2) and not sidecar
       gpu_test_timeout_minutes: 240
       # 80 GiB H100 admits more of the profiled pool per slot than the 24 GiB lanes.
       run_gpu_parallel_tests: true
@@ -1021,7 +1027,7 @@ jobs:
       enable_coverage: true
       run_cpu_only_tests: true
       cpu_only_test_markers: trtllm and gpu_0
-      gpu_test_markers: trtllm and gpu_1
+      gpu_test_markers: trtllm and gpu_1 and not sidecar
       gpu_test_timeout_minutes: 240
       # Profiled tests run in the VRAM-aware GPU stage; unprofiled fall through to sequential.
       # Current single-GPU runners are 24 GiB, so this cap admits the profiled TRT-LLM pool
@@ -1073,7 +1079,7 @@ jobs:
       platform: '["amd64"]'
       enable_coverage: true
       run_sanity_check: false
-      gpu_test_markers: trtllm and (gpu_1 or gpu_2)
+      gpu_test_markers: trtllm and (gpu_1 or gpu_2) and not sidecar
       gpu_test_timeout_minutes: 240
       # 80 GiB H100 admits more of the profiled pool per slot than the 24 GiB lanes.
       run_gpu_parallel_tests: true

@@ -215,6 +215,11 @@ jobs:
       extra_tags: |
         ${{ github.ref_name == 'main' && 'main-vllm-runtime' || '' }}
         ${{ github.ref_name == 'main' && format('main-vllm-runtime-{0}', github.sha) || '' }}
+      # Lets sidecar-vllm-test (pr.yaml) fall back to this floating tag via its
+      # source_ref input instead of rebuilding vllm-runtime on every
+      # lib/sidecar/**-only PR.
+      test_extra_tags: |
+        ${{ github.ref_name == 'main' && 'main-vllm-runtime-test' || '' }}
       archive_sources: false
     secrets: inherit
 
@@ -280,6 +285,11 @@ jobs:
       extra_tags: |
         ${{ github.ref_name == 'main' && 'main-sglang-runtime' || '' }}
         ${{ github.ref_name == 'main' && format('main-sglang-runtime-{0}', github.sha) || '' }}
+      # Lets sidecar-sglang-test (pr.yaml) fall back to this floating tag via
+      # its source_ref input instead of rebuilding sglang-runtime on every
+      # lib/sidecar/**-only PR.
+      test_extra_tags: |
+        ${{ github.ref_name == 'main' && 'main-sglang-runtime-test' || '' }}
       archive_sources: false
     secrets: inherit
 
@@ -345,6 +355,11 @@ jobs:
       extra_tags: |
         ${{ github.ref_name == 'main' && 'main-trtllm-runtime' || '' }}
         ${{ github.ref_name == 'main' && format('main-trtllm-runtime-{0}', github.sha) || '' }}
+      # Lets sidecar-trtllm-test (pr.yaml) fall back to this floating tag via
+      # its source_ref input instead of rebuilding trtllm-runtime on every
+      # lib/sidecar/**-only PR.
+      test_extra_tags: |
+        ${{ github.ref_name == 'main' && 'main-trtllm-runtime-test' || '' }}
       archive_sources: false
     secrets: inherit
 
@@ -471,7 +486,10 @@ jobs:
       platform: '["amd64", "arm64"]' # arm64 for CPU tests, single GPU tests are skipped
       run_cpu_only_tests: true
       cpu_only_test_markers: '(pre_merge or post_merge) and vllm and gpu_0'
-      gpu_test_markers: '(pre_merge or post_merge) and vllm and gpu_1'
+      # not sidecar: sidecar-marked tests need dynamo-vllm-sidecar layered onto
+      # the image (see pr.yaml's sidecar-vllm-test), which this plain
+      # vllm-runtime image doesn't have.
+      gpu_test_markers: '(pre_merge or post_merge) and vllm and gpu_1 and not sidecar'
       gpu_test_timeout_minutes: 120
       # Profiled tests (those carrying @pytest.mark.profiled_vram_gib(N)) run concurrently in
       # the parallel stage; the rest fall through to the sequential stage. 24 GiB admits all
@@ -549,7 +567,10 @@ jobs:
       platform: '["amd64", "arm64"]' # arm64 for CPU tests, single GPU tests are skipped
       run_cpu_only_tests: true
       cpu_only_test_markers: '(pre_merge or post_merge) and sglang and gpu_0'
-      gpu_test_markers: '(pre_merge or post_merge) and sglang and gpu_1'
+      # not sidecar: sidecar-marked tests need dynamo-sglang-sidecar layered
+      # onto the image (see pr.yaml's sidecar-sglang-test), which this plain
+      # sglang-runtime image doesn't have.
+      gpu_test_markers: '(pre_merge or post_merge) and sglang and gpu_1 and not sidecar'
       gpu_test_timeout_minutes: 120
       # Profiled tests run in the VRAM-aware GPU stage; unprofiled fall through to sequential.
       # Current single-GPU runners are 24 GiB, so this cap admits the profiled SGLang pool
@@ -591,7 +612,10 @@ jobs:
       platform: '["amd64", "arm64"]' # arm64 for CPU tests, single GPU tests are skipped
       run_cpu_only_tests: true
       cpu_only_test_markers: '(pre_merge or post_merge) and trtllm and gpu_0'
-      gpu_test_markers: '(pre_merge or post_merge) and trtllm and gpu_1'
+      # not sidecar: sidecar-marked tests need dynamo-trtllm-sidecar layered
+      # onto the image (see pr.yaml's sidecar-trtllm-test), which this plain
+      # trtllm-runtime image doesn't have.
+      gpu_test_markers: '(pre_merge or post_merge) and trtllm and gpu_1 and not sidecar'
       gpu_test_timeout_minutes: 120
       # Profiled tests run in the VRAM-aware GPU stage; unprofiled fall through to sequential.
       # Current single-GPU runners are 24 GiB, so this cap admits the profiled TRT-LLM pool

@@ -134,6 +134,9 @@ jobs:
       - snapshot-placeholder-trtllm
       - power-agent
       - sidecar-build
+      - sidecar-vllm-test
+      - sidecar-sglang-test
+      - sidecar-trtllm-test
       - vllm-build
       - vllm-local-dev-build
       - sglang-local-dev-build
@@ -355,12 +358,99 @@ jobs:
     uses: $/.github/workflows/shared-build-sidecar.yml
     with:
       builder_name: ${{ needs.changed-files.outputs.builder_name }}
+      # Default (60) is tight for a cold cache: multi-arch cargo build of 3
+      # sidecar crates + dynamo-llm/dynamo-runtime, plus the compliance and
+      # binary-extraction steps that follow it. Observed a legitimate cold
+      # build exceed 60m with room to spare in run 34547963529.
+      build_timeout_minutes: 90
       artifact_retention_days: '7'
       diff_event_context: pr
       diff_current_branch: ${{ github.ref_name }}
       diff_base_branch: ${{ needs.changed-files.outputs.base_ref }}
       diff_merge_base_sha: ${{ needs.changed-files.outputs.merge_base_sha }}
 
+  # Reuse the backend test image and install the sidecar artifact before pytest.
+
+  sidecar-vllm-test:
+    name: sidecar-runtime # This name overlaps with other sidecar jobs to group them in the UI
+    needs: [changed-files, vllm-build, sidecar-build]
+    # vllm-build must have succeeded or been intentionally skipped (a
+    # lib/sidecar-only PR) -- not failed or cancelled -- or the fallback below
+    # would silently test this PR's sidecar binary against main's vllm image
+    # and report green while this PR's own vllm-runtime build is red.
+    if: ${{ !cancelled() && needs.changed-files.outputs.sidecar == 'true' && needs.sidecar-build.result == 'success' && (needs.vllm-build.result == 'success' || needs.vllm-build.result == 'skipped') }}
+    permissions:
+      actions: read
+      contents: read
+    uses: $/.github/workflows/shared-test.yml
+    with:
+      test_suite_name: vllm
+      framework: vllm  # matches the framework passed to vllm-build
+      test_type: Sidecar E2E Test
+      amd_runner: prod-tester-amd-gpu-v2
+      target_tag_plain: ${{ needs.vllm-build.result == 'success' && needs.vllm-build.outputs.target_tag_plain || 'vllm-runtime' }}
+      source_ref: ${{ needs.vllm-build.result == 'success' && github.sha || 'main' }}
+      cuda_version: '["13.0"]'
+      platform: '["amd64"]'
+      run_sanity_check: false
+      gpu_test_markers: pre_merge and sidecar and vllm and gpu_1
+      gpu_test_timeout_minutes: 45
+      sidecar_backend: vllm
+      sidecar_binary_artifact: ${{ needs.sidecar-build.outputs.binary_artifact }}
+    secrets: inherit
+
+  sidecar-sglang-test:
+    name: sidecar-runtime # This name overlaps with other sidecar jobs to group them in the UI
+    needs: [changed-files, sglang-build, sidecar-build]
+    # See sidecar-vllm-test above: sglang-build must have succeeded or been
+    # intentionally skipped, not failed or cancelled.
+    if: ${{ !cancelled() && needs.changed-files.outputs.sidecar == 'true' && needs.sidecar-build.result == 'success' && (needs.sglang-build.result == 'success' || needs.sglang-build.result == 'skipped') }}
+    permissions:
+      actions: read
+      contents: read
+    uses: $/.github/workflows/shared-test.yml
+    with:
+      test_suite_name: sglang
+      framework: sglang  # matches the framework passed to sglang-build
+      test_type: Sidecar E2E Test
+      amd_runner: prod-tester-amd-gpu-v2
+      target_tag_plain: ${{ needs.sglang-build.result == 'success' && needs.sglang-build.outputs.target_tag_plain || 'sglang-runtime' }}
+      source_ref: ${{ needs.sglang-build.result == 'success' && github.sha || 'main' }}
+      cuda_version: '["13.0"]'
+      platform: '["amd64"]'
+      run_sanity_check: false
+      gpu_test_markers: pre_merge and sidecar and sglang and gpu_1
+      gpu_test_timeout_minutes: 45
+      sidecar_backend: sglang
+      sidecar_binary_artifact: ${{ needs.sidecar-build.outputs.binary_artifact }}
+    secrets: inherit
+
+  sidecar-trtllm-test:
+    name: sidecar-runtime # This name overlaps with other sidecar jobs to group them in the UI
+    needs: [changed-files, trtllm-build, sidecar-build]
+    # See sidecar-vllm-test above: trtllm-build must have succeeded or been
+    # intentionally skipped, not failed or cancelled.
+    if: ${{ !cancelled() && needs.changed-files.outputs.sidecar == 'true' && needs.sidecar-build.result == 'success' && (needs.trtllm-build.result == 'success' || needs.trtllm-build.result == 'skipped') }}
+    permissions:
+      actions: read
+      contents: read
+    uses: $/.github/workflows/shared-test.yml
+    with:
+      test_suite_name: trtllm
+      framework: trtllm  # matches the framework passed to trtllm-build
+      test_type: Sidecar E2E Test
+      amd_runner: prod-tester-amd-gpu-v2
+      target_tag_plain: ${{ needs.trtllm-build.result == 'success' && needs.trtllm-build.outputs.target_tag_plain || 'trtllm-runtime' }}
+      source_ref: ${{ needs.trtllm-build.result == 'success' && github.sha || 'main' }}
+      cuda_version: '["13.1"]'
+      platform: '["amd64"]'
+      run_sanity_check: false
+      gpu_test_markers: pre_merge and sidecar and trtllm and gpu_1
+      gpu_test_timeout_minutes: 45
+      sidecar_backend: trtllm
+      sidecar_binary_artifact: ${{ needs.sidecar-build.outputs.binary_artifact }}
+    secrets: inherit
+
 # ============================================================================
 # FRAMEWORK PIPELINES (Build → Test → Copy)
 # ============================================================================
@@ -813,7 +903,10 @@ jobs:
       # Keep the outer step budget above the test's 450s timeout so pytest can
       # finish teardown and upload diagnostics instead of being killed first.
       cpu_e2e_test_timeout_minutes: 15
-      gpu_test_markers: pre_merge and vllm and gpu_1
+      # not sidecar: sidecar-marked tests need dynamo-vllm-sidecar layered onto
+      # the image (see sidecar-vllm-test below), which this plain vllm-runtime
+      # image doesn't have.
+      gpu_test_markers: pre_merge and vllm and gpu_1 and not sidecar
       gpu_test_timeout_minutes: 45
       # Profiled tests run in the parallel stage; unprofiled fall through to sequential.
       # 24 GiB admits all currently profiled vLLM tests (max is ~20.4 GiB) on a 48 GiB GPU.
@@ -858,7 +951,10 @@ jobs:
       platform: '["amd64", "arm64"]' # arm64 for CPU tests, single GPU tests are skipped
       run_cpu_only_tests: true
       cpu_only_test_markers: pre_merge and sglang and gpu_0
-      gpu_test_markers: pre_merge and sglang and gpu_1
+      # not sidecar: sidecar-marked tests need dynamo-sglang-sidecar layered
+      # onto the image (see sidecar-sglang-test below), which this plain
+      # sglang-runtime image doesn't have.
+      gpu_test_markers: pre_merge and sglang and gpu_1 and not sidecar
       # Profiled tests run in the VRAM-aware GPU stage; unprofiled fall through to sequential.
       # Current single-GPU runners are 24 GiB, so this cap admits the profiled SGLang pool
       # while yielding one auto slot today. Larger runners will get more slots from the same markers.
@@ -903,7 +999,10 @@ jobs:
       platform: '["amd64", "arm64"]' # arm64 for CPU tests, single GPU tests are skipped
       run_cpu_only_tests: true
       cpu_only_test_markers: pre_merge and trtllm and gpu_0
-      gpu_test_markers: pre_merge and trtllm and gpu_1
+      # not sidecar: sidecar-marked tests need dynamo-trtllm-sidecar layered
+      # onto the image (see sidecar-trtllm-test below), which this plain
+      # trtllm-runtime image doesn't have.
+      gpu_test_markers: pre_merge and trtllm and gpu_1 and not sidecar
       # Profiled tests run in the VRAM-aware GPU stage; unprofiled fall through to sequential.
       # Current single-GPU runners are 24 GiB, so this cap admits the profiled TRT-LLM pool
       # while yielding one auto slot today. Larger runners will get more slots from the same markers.

@@ -37,6 +37,11 @@ on:
         required: false
         type: string
         default: ''
+      test_extra_tags:
+        description: 'Additional tags for the -test image only (newline-separated plain tag names, no ECR prefix). Unlike extra_tags (runtime image), the -test image otherwise gets no floating tag.'
+        required: false
+        type: string
+        default: ''
       no_cache:
         description: 'Disable Docker build cache'
         required: false
@@ -603,6 +608,14 @@ jobs:
       - name: Build and Push Test Image
         if: ${{ !inputs.compliance_only && inputs.target != 'dev' && inputs.target != 'local-dev' && inputs.target != 'frontend' }}
         shell: bash
+        env:
+          ECR_REGISTRY: ${{ vars.ECR_REGISTRY }}
+          ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }}
+          TEST_EXTRA_TAGS: ${{ inputs.test_extra_tags }}
+          # Routed through env (not inlined via ${{ }}) so a value containing
+          # shell metacharacters can't be interpreted as script rather than data.
+          IMAGE_URI: ${{ steps.calculate-target-tag.outputs.image_uri }}
+          TEST_IMAGE_URI: ${{ steps.calculate-target-tag.outputs.test_image_uri }}
         run: |
           PUSH_ARGS=""
           if [ "${{ inputs.push_image }}" == "true" ]; then
@@ -630,6 +643,15 @@ jobs:
             PYPI_SECRET_ARGS+=(--secret "id=pypi-netrc,src=${NETRC}")
           fi
 
+          TEST_TAG_ARGS=(-t "${TEST_IMAGE_URI}")
+          if [ -n "$TEST_EXTRA_TAGS" ]; then
+            while IFS= read -r tag; do
+              if [ -n "$tag" ]; then
+                TEST_TAG_ARGS+=(-t "${ECR_REGISTRY}/${ECR_REPOSITORY}:${tag}")
+              fi
+            done <<< "$TEST_EXTRA_TAGS"
+          fi
+
           docker buildx build \
             --progress=plain \
             ${PUSH_ARGS} \
@@ -638,8 +660,8 @@ jobs:
             --platform ${{ inputs.platform }} \
             --target ${TEST_TARGET} \
             -f container/Dockerfile.test \
-            --build-arg BASE_IMAGE=${{ steps.calculate-target-tag.outputs.image_uri }} \
-            -t ${{ steps.calculate-target-tag.outputs.test_image_uri }} .
+            --build-arg BASE_IMAGE="${IMAGE_URI}" \
+            "${TEST_TAG_ARGS[@]}" .
       - name: Show summary
         shell: bash
         if: ${{ inputs.push_image == 'true' && inputs.show_summary == 'true' }}