From 957c940dab15d724f4dde2a65030bed365f4f328 Mon Sep 17 00:00:00 2001 From: XinyueZhang369 Date: Thu, 11 Jun 2026 23:01:04 -0700 Subject: [PATCH 1/3] test(ci): run e2e-2gpu-pd on hostNetwork RDMA runner (2-gpu-h100-test) Signed-off-by: XinyueZhang369 --- .github/workflows/pr-test-rust.yml | 2 +- .../runner-values-2-gpu-h100.yaml | 73 ++++++++++++++++--- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index 867ea59def..5c8391e1e8 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -660,7 +660,7 @@ jobs: with: engine: ${{ matrix.engine }} gpu_tier: "2" - runner: 2-gpu-h100 + runner: 2-gpu-h100-test timeout: ${{ matrix.timeout }} test_dirs: e2e_test/router vllm_kv_backend: ${{ matrix.kv_backend || 'nixl' }} diff --git a/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml b/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml index 739e772ae2..ac8ad56b71 100644 --- a/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml +++ b/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml @@ -6,11 +6,11 @@ githubConfigUrl: "https://github.com/lightseekorg/smg" githubConfigSecret: github-arc-secret ## Runner scaling configuration -minRunners: 0 -maxRunners: 6 +minRunners: 1 +maxRunners: 1 ## name of the runner scale set to create. Defaults to the helm release name -runnerScaleSetName: "2-gpu-h100" +runnerScaleSetName: "2-gpu-h100-test" resourceMeta: {} keyVault: {} @@ -20,6 +20,14 @@ template: spec: terminationGracePeriodSeconds: 600 + # hostNetwork puts the runner in the host netns so PD disaggregation KV + # transfer (NIXL/Mooncake) sees the host's populated RoCE GID tables. An + # overlay-CNI pod sees the mlx5 NICs as PORT_ACTIVE but with an empty GID + # table, so RDMA init dies at worker startup. ClusterFirstWithHostNet keeps + # cluster DNS working while on the host network. + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: nvidia.com/gpu: "true" node.kubernetes.io/instance-type: BM.GPU.H100.8 @@ -30,8 +38,11 @@ template: value: "true" effect: "NoSchedule" + # With hostNetwork the pod shares the host port space, so spread 2-gpu PD + # runners one-per-node to avoid host-port collisions between concurrent PD + # jobs (replaces the previous GPU bin-packing podAffinity). affinity: - podAffinity: + podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: @@ -40,8 +51,7 @@ template: - key: actions.github.com/scale-set-name operator: In values: - - 1-gpu-h100 - - 2-gpu-h100 + - 2-gpu-h100-test topologyKey: kubernetes.io/hostname volumes: @@ -63,18 +73,49 @@ template: containers: - name: runner - image: fra.ocir.io/idqj093njucb/action-runner:v0.0.2 + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.3 command: - /home/runner/run.sh + securityContext: + capabilities: + # RDMA verbs pin and register memory for queue pairs; without + # IPC_LOCK registration fails even though the GID table resolves. + add: ["IPC_LOCK"] env: - name: DOCKER_HOST value: unix:///var/run/docker.sock - name: RUNNER_WAIT_FOR_DOCKER_IN_SECONDS value: '120' + - name: HF_HOME + value: /models + - name: HF_TOKEN # required: ci_download_model.sh runs `hf download` for the gated Llama model + valueFrom: + secretKeyRef: + key: HUGGINGFACE_API_KEY + name: huggingface-secret + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + key: OPENAI_API_KEY + name: openai-api-key + - name: ANTHROPIC_API_KEY + valueFrom: + secretKeyRef: + key: ANTHROPIC_API_KEY + name: anthropic-api-key + - name: XAI_API_KEY + valueFrom: + secretKeyRef: + key: XAI_API_KEY + name: xai-api-key + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP resources: requests: - cpu: "16" - memory: "64Gi" + cpu: '32' + memory: 128Gi limits: nvidia.com/gpu: 2 volumeMounts: @@ -97,7 +138,7 @@ template: - /home/runner/tmpDir/ command: - cp - image: fra.ocir.io/idqj093njucb/action-runner:v0.0.2 + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.3 volumeMounts: - mountPath: /home/runner/tmpDir name: dind-externals @@ -109,10 +150,22 @@ template: env: - name: DOCKER_GROUP_GID value: '123' + # Suppress dind's default TLS TCP listener (:2376). dockerd only needs + # the unix socket here; under hostNetwork a TCP listener would bind a + # node port unnecessarily. + - name: DOCKER_TLS_CERTDIR + value: "" image: fra.ocir.io/idqj093njucb/docker:dind restartPolicy: Always securityContext: privileged: true + resources: + limits: + cpu: '2' + memory: 4Gi + requests: + cpu: '1' + memory: 2Gi startupProbe: exec: command: From 6aeda237b3b0fdcc0cd83463ce48d6c675b98076 Mon Sep 17 00:00:00 2001 From: XinyueZhang369 Date: Fri, 12 Jun 2026 12:34:16 -0700 Subject: [PATCH 2/3] fix rdma on runner Signed-off-by: XinyueZhang369 --- .../k8s-runner-resources/runner-values-2-gpu-h100.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml b/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml index ac8ad56b71..563d74152f 100644 --- a/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml +++ b/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml @@ -77,10 +77,12 @@ template: command: - /home/runner/run.sh securityContext: - capabilities: - # RDMA verbs pin and register memory for queue pairs; without - # IPC_LOCK registration fails even though the GID table resolves. - add: ["IPC_LOCK"] + # The runner runs as non-root (uid 1001), so capabilities.add: + # [IPC_LOCK] only reaches the bounding set (not effective) and the + # default 64KiB RLIMIT_MEMLOCK blocks RDMA completion-queue + # allocation (ibv_create_cq -> ENOMEM, e.g. sglang/Mooncake). + # privileged makes IPC_LOCK effective and lifts the memlock cap. + privileged: true env: - name: DOCKER_HOST value: unix:///var/run/docker.sock From 8b2b1b0e4d0e5140ee7c6ea0ec8e498e3dbbe8d2 Mon Sep 17 00:00:00 2001 From: XinyueZhang369 Date: Fri, 12 Jun 2026 14:58:32 -0700 Subject: [PATCH 3/3] clean up test code Signed-off-by: XinyueZhang369 --- .github/workflows/pr-test-rust.yml | 2 +- scripts/k8s-runner-resources/arc-runner-autoscaler.yaml | 4 ++-- .../k8s-runner-resources/runner-values-2-gpu-h100.yaml | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index 5c8391e1e8..867ea59def 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -660,7 +660,7 @@ jobs: with: engine: ${{ matrix.engine }} gpu_tier: "2" - runner: 2-gpu-h100-test + runner: 2-gpu-h100 timeout: ${{ matrix.timeout }} test_dirs: e2e_test/router vllm_kv_backend: ${{ matrix.kv_backend || 'nixl' }} diff --git a/scripts/k8s-runner-resources/arc-runner-autoscaler.yaml b/scripts/k8s-runner-resources/arc-runner-autoscaler.yaml index 06b9bd54b3..d72c16c140 100644 --- a/scripts/k8s-runner-resources/arc-runner-autoscaler.yaml +++ b/scripts/k8s-runner-resources/arc-runner-autoscaler.yaml @@ -80,8 +80,8 @@ spec: kind: RunnerDeployment name: arc-runner-2-gpu-h100 - minReplicas: 2 - maxReplicas: 10 + minReplicas: 0 + maxReplicas: 0 metrics: - type: TotalNumberOfQueuedAndInProgressWorkflowRuns diff --git a/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml b/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml index 563d74152f..0777f7b04b 100644 --- a/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml +++ b/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml @@ -6,11 +6,11 @@ githubConfigUrl: "https://github.com/lightseekorg/smg" githubConfigSecret: github-arc-secret ## Runner scaling configuration -minRunners: 1 -maxRunners: 1 +minRunners: 2 +maxRunners: 8 ## name of the runner scale set to create. Defaults to the helm release name -runnerScaleSetName: "2-gpu-h100-test" +runnerScaleSetName: "2-gpu-h100" resourceMeta: {} keyVault: {} @@ -51,7 +51,7 @@ template: - key: actions.github.com/scale-set-name operator: In values: - - 2-gpu-h100-test + - 2-gpu-h100 topologyKey: kubernetes.io/hostname volumes: