diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index 22e3176250..9e7b45a2e1 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -224,6 +224,7 @@ jobs: upload_benchmarks: true parallel_opts: "" # No parallel for benchmarks (performance measurement) ignore_opts: "--ignore=e2e_test/benchmarks/test_go_bindings_perf.py --ignore=e2e_test/benchmarks/test_nightly_perf.py" # Go and nightly benchmarks run in dedicated jobs + runner: 4-gpu-h100 - name: agentic-apis timeout: 30 test_dirs: "e2e_test/responses e2e_test/messages" @@ -284,7 +285,8 @@ jobs: test_filter: "" setup_trtllm: true ignore_opts: "" - runs-on: k8s-runner-gpu + runner: 4-gpu-h100 + runs-on: ${{ matrix.runner || 'k8s-runner-gpu' }} timeout-minutes: ${{ matrix.timeout }} steps: - name: Checkout code @@ -305,7 +307,7 @@ jobs: - name: Setup SGLang backend if: ${{ !matrix.setup_vllm && !matrix.setup_trtllm }} uses: ./.github/actions/setup-sglang - + - name: Download wheel artifact uses: actions/download-artifact@v7 with: @@ -557,7 +559,7 @@ jobs: if [[ "${{ needs.check-ci.outputs.should_run }}" != "true" ]]; then echo "CI was skipped (external contributor without run-ci label)" exit 0 - elif [[ "${{ needs.python-lint.result }}" == "failure" || "${{ needs.build-wheel.result }}" == "failure" || "${{ needs.unit-tests.result }}" == "failure" || "${{ needs.gateway-e2e.result }}" == "failure" || "${{ needs.go-unit-tests.result }}" == "failure" || "${{ needs.go-bindings-e2e.result }}" == "failure" ]]; then + elif [[ "${{ needs.python-lint.result }}" == "failure" || "${{ needs.build-wheel.result }}" == "failure" || "${{ needs.python-unit-tests.result }}" == "failure" || "${{ needs.unit-tests.result }}" == "failure" || "${{ needs.gateway-e2e.result }}" == "failure" || "${{ needs.go-unit-tests.result }}" == "failure" || "${{ needs.go-bindings-e2e.result }}" == "failure" ]]; then echo "One or more jobs failed" exit 1 else diff --git a/e2e_test/benchmarks/test_pd_perf.py b/e2e_test/benchmarks/test_pd_perf.py index ca883fc70b..e124ad088a 100644 --- a/e2e_test/benchmarks/test_pd_perf.py +++ b/e2e_test/benchmarks/test_pd_perf.py @@ -20,7 +20,7 @@ def test_pd_perf(self, setup_backend, genai_bench_runner): # accurate GPU utilization sampling (at least 30+ seconds) max_requests_per_run=200, thresholds={ - "ttft_mean_max": 13, + "ttft_mean_max": 5, "e2e_latency_mean_max": 16, "input_throughput_mean_min": 350, "output_throughput_mean_min": 18, diff --git a/e2e_test/benchmarks/test_regular_perf.py b/e2e_test/benchmarks/test_regular_perf.py index 9fd7518a5d..6cb21bfb54 100644 --- a/e2e_test/benchmarks/test_regular_perf.py +++ b/e2e_test/benchmarks/test_regular_perf.py @@ -21,7 +21,7 @@ def test_regular_perf(self, setup_backend, genai_bench_runner): # accurate GPU utilization sampling (at least 30+ seconds) max_requests_per_run=200, thresholds={ - "ttft_mean_max": 6, + "ttft_mean_max": 0.8, "e2e_latency_mean_max": 14, "input_throughput_mean_min": 800, "output_throughput_mean_min": 12, diff --git a/scripts/k8s-runner-resources/arc-runner-autoscaler.yaml b/scripts/k8s-runner-resources/arc-runner-autoscaler.yaml new file mode 100644 index 0000000000..50e64ad066 --- /dev/null +++ b/scripts/k8s-runner-resources/arc-runner-autoscaler.yaml @@ -0,0 +1,39 @@ +apiVersion: actions.summerwind.dev/v1alpha1 +kind: HorizontalRunnerAutoscaler +metadata: + name: arc-runner-h100-autoscaler + namespace: actions-runner-system +spec: + scaleTargetRef: + kind: RunnerDeployment + name: arc-runner-gpu-h100 + + minReplicas: 10 + maxReplicas: 20 + + metrics: + - type: PercentageRunnersBusy + scaleUpThreshold: "0.95" + scaleDownThreshold: "0.25" + scaleUpFactor: "0.5" + scaleDownFactor: "0.5" +--- +apiVersion: actions.summerwind.dev/v1alpha1 +kind: HorizontalRunnerAutoscaler +metadata: + name: arc-cpu-runner-autoscaler + namespace: actions-runner-system +spec: + scaleTargetRef: + kind: RunnerDeployment + name: arc-runner-cpu + + minReplicas: 4 + maxReplicas: 8 + + metrics: + - type: PercentageRunnersBusy + scaleUpThreshold: "0.95" + scaleDownThreshold: "0.25" + scaleUpFactor: "0.5" + scaleDownFactor: "0.5" diff --git a/scripts/k8s-runner-resources/arc-runner-cpu.yaml b/scripts/k8s-runner-resources/arc-runner-cpu.yaml new file mode 100644 index 0000000000..ad463404cd --- /dev/null +++ b/scripts/k8s-runner-resources/arc-runner-cpu.yaml @@ -0,0 +1,26 @@ +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: arc-runner-cpu + namespace: actions-runner-system +spec: + replicas: 4 + template: + spec: + repository: lightseekorg/smg + labels: + - k8s-runner-cpu + serviceAccountName: arc-runner-sa + + containers: + - name: runner + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.1 + resources: + requests: + cpu: "8" + memory: "16Gi" + limits: + cpu: "8" + memory: "16Gi" + - name: docker + image: fra.ocir.io/idqj093njucb/docker:dind diff --git a/scripts/k8s-runner-resources/arc-runner-gpu.yaml b/scripts/k8s-runner-resources/arc-runner-gpu.yaml new file mode 100644 index 0000000000..bde6d6ba95 --- /dev/null +++ b/scripts/k8s-runner-resources/arc-runner-gpu.yaml @@ -0,0 +1,159 @@ +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: arc-runner-gpu-h100 + namespace: actions-runner-system +spec: + replicas: 10 + template: + spec: + repository: lightseekorg/smg + labels: + - 4-gpu-h100 + - k8s-runner-gpu + serviceAccountName: arc-runner-sa + + nodeSelector: + nvidia.com/gpu: "true" + beta.kubernetes.io/instance-type: BM.GPU.H100.8 + + tolerations: + - key: "nvidia.com/gpu" + operator: "Equal" + value: "true" + effect: "NoSchedule" + + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: runner-deployment-name + operator: In + values: + - arc-runner-gpu-h100 + topologyKey: kubernetes.io/hostname + + volumes: + - name: model-cache + persistentVolumeClaim: + claimName: model-cache + - name: docker-sock + emptyDir: {} + - name: docker-storage + emptyDir: {} + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 16Gi + + containers: + - name: runner + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.1 + resources: + limits: + nvidia.com/gpu: 4 + volumeMounts: + - name: model-cache + mountPath: /models + - name: docker-sock + mountPath: /var/run + - name: dshm + mountPath: /dev/shm + env: + - name: DOCKER_HOST + value: unix:///var/run/docker.sock + - name: docker + image: fra.ocir.io/idqj093njucb/docker:dind + securityContext: + privileged: true # Required for DinD + env: + - name: DOCKER_TLS_CERTDIR + value: "" # Disables TLS for shared socket use + volumeMounts: + - name: docker-sock + mountPath: /var/run + - name: docker-storage + mountPath: /var/lib/docker +--- +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: arc-runner-gpu-a10 + namespace: actions-runner-system +spec: + replicas: 2 + template: + spec: + repository: lightseekorg/smg + labels: + - 4-gpu-a10 + - k8s-runner-gpu + serviceAccountName: arc-runner-sa + + nodeSelector: + nvidia.com/gpu: "true" + beta.kubernetes.io/instance-type: BM.GPU.A10.4 + + tolerations: + - key: "nvidia.com/gpu" + operator: "Equal" + value: "true" + effect: "NoSchedule" + + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: runner-deployment-name + operator: In + values: + - arc-runner-gpu-a10 + topologyKey: kubernetes.io/hostname + + volumes: + - name: model-cache + persistentVolumeClaim: + claimName: model-cache + - name: docker-sock + emptyDir: {} + - name: docker-storage + emptyDir: {} + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 16Gi + + containers: + - name: runner + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.1 + resources: + limits: + nvidia.com/gpu: 4 + volumeMounts: + - name: model-cache + mountPath: /models + - name: docker-sock + mountPath: /var/run + - name: dshm + mountPath: /dev/shm + env: + - name: DOCKER_HOST + value: unix:///var/run/docker.sock + - name: docker + image: fra.ocir.io/idqj093njucb/docker:dind + securityContext: + privileged: true # Required for DinD + env: + - name: DOCKER_TLS_CERTDIR + value: "" # Disables TLS for shared socket use + volumeMounts: + - name: docker-sock + mountPath: /var/run + - name: docker-storage + mountPath: /var/lib/docker diff --git a/scripts/k8s-runner-resources/arc-runner-rbac.yaml b/scripts/k8s-runner-resources/arc-runner-rbac.yaml new file mode 100644 index 0000000000..c317d45716 --- /dev/null +++ b/scripts/k8s-runner-resources/arc-runner-rbac.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: arc-runner-sa + namespace: actions-runner-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: arc-runner + namespace: actions-runner-system +rules: + # Argo Workflows + - apiGroups: [""] + resources: + - secrets + verbs: + - get + - list + - watch + + # Pods + - apiGroups: [""] + resources: + - pods + - pods/log + - pods/exec + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: arc-runner-rb + namespace: actions-runner-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: arc-runner +subjects: + - kind: ServiceAccount + name: arc-runner-sa + namespace: actions-runner-system