From faacd1fb5f177625b11fed0d1b5db18d333d060f Mon Sep 17 00:00:00 2001 From: XinyueZhang369 Date: Fri, 6 Mar 2026 00:56:52 -0800 Subject: [PATCH 1/4] Change to use actions.github.com/v1alpha1 runner set for arc runners Signed-off-by: XinyueZhang369 --- .github/workflows/nightly-benchmark.yml | 4 +- scripts/k8s-runner-resources/README.md | 135 ++++++++++++++++++ .../arc-secret-template.yaml | 10 ++ .../runner-values-1-gpu-h100.yaml | 130 +++++++++++++++++ .../runner-values-2-gpu-h100.yaml | 127 ++++++++++++++++ .../runner-values-4-gpu-general.yaml | 135 ++++++++++++++++++ .../runner-values-4-gpu-h100.yaml | 126 ++++++++++++++++ .../runner-values-cpu.yaml | 90 ++++++++++++ 8 files changed, 755 insertions(+), 2 deletions(-) create mode 100644 scripts/k8s-runner-resources/README.md create mode 100644 scripts/k8s-runner-resources/arc-secret-template.yaml create mode 100644 scripts/k8s-runner-resources/runner-values-1-gpu-h100.yaml create mode 100644 scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml create mode 100644 scripts/k8s-runner-resources/runner-values-4-gpu-general.yaml create mode 100644 scripts/k8s-runner-resources/runner-values-4-gpu-h100.yaml create mode 100644 scripts/k8s-runner-resources/runner-values-cpu.yaml diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index 140baab555..8982c047ce 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -81,7 +81,7 @@ jobs: name: "${{ matrix.model.id }} / single-${{ matrix.variant.id }}" needs: build-wheel if: ${{ !cancelled() }} - runs-on: ["k8s-runner-gpu", "4-gpu-h100"] + runs-on: 4-gpu-h100 timeout-minutes: 1440 strategy: fail-fast: false @@ -188,7 +188,7 @@ jobs: name: "${{ matrix.model.id }} / multi-${{ matrix.variant.id }}" needs: build-wheel if: ${{ !cancelled() }} - runs-on: ["k8s-runner-gpu", "4-gpu-h100"] + runs-on: 4-gpu-h100" timeout-minutes: 1440 strategy: fail-fast: false diff --git a/scripts/k8s-runner-resources/README.md b/scripts/k8s-runner-resources/README.md new file mode 100644 index 0000000000..bed884c6ba --- /dev/null +++ b/scripts/k8s-runner-resources/README.md @@ -0,0 +1,135 @@ +# ARC (Actions Runner Controller) Deployment Guide + +## Prerequisites + +- Kubernetes cluster (v1.28+) +- `kubectl` configured with cluster access +- `helm` v3 installed + +## 1. Create a GitHub App + +1. Go to **GitHub Organization Settings > Developer settings > GitHub Apps > New GitHub App** +2. Set the following: + - **GitHub App name**: e.g. `smg-k8s-arc-app` + - **Homepage URL**: any valid URL + - **Webhook**: uncheck "Active" (not needed) +3. Grant **Repository permissions**: + - **Actions**: Read and Write + - **Administration**: Read and Write + - **Checks**: Read and Write + - **Metadata**: Read-only +4. Click **Create GitHub App** +5. Note the **App ID** from the app's settings page +6. Generate a **Private Key** (PEM file) and save it + +## 2. Install the GitHub App + +1. From the GitHub App settings, click **Install App** +2. Select the organization and choose the target repository (e.g. `lightseekorg/smg`) +3. Click **Install** +4. Note the **Installation ID** from the URL: `https://github.com/organizations//settings/installations/` + +## 3. Create the Kubernetes Secret + +Edit `arc-secret-template.yaml` with the values collected above: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: github-arc-secret + namespace: actions-runner-system +type: Opaque +stringData: + github_app_id: "" + github_app_installation_id: "" + github_app_private_key: "" +``` + +Create the namespace and secret: + +```bash +kubectl create namespace actions-runner-system +kubectl apply -f scripts/k8s-runner-resources/arc-secret-template.yaml +``` + +## 4. Install the ARC Controller + +```bash +helm install arc \ + --namespace actions-runner-system \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller +``` + +Verify the controller is running: + +```bash +kubectl get pods -n actions-runner-system -l app.kubernetes.io/name=gha-runner-scale-set-controller +``` + +## 5. Install Runner Scale Sets + +Install each runner set using its values file: + +```bash +# CPU runners +helm install k8s-runner-cpu \ + --namespace actions-runner-system \ + -f scripts/k8s-runner-resources/runner-values-cpu.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set + +# 4x GPU H100/A10 runners +helm install k8s-runner-gpu \ + --namespace actions-runner-system \ + -f scripts/k8s-runner-resources/runner-values-4-gpu-general.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set + +# 4x GPU H100 runners +helm install 4-gpu-h100 \ + --namespace actions-runner-system \ + -f scripts/k8s-runner-resources/runner-values-4-gpu-h100.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set + +# 2x GPU H100 runners +helm install 2-gpu-h100 \ + --namespace actions-runner-system \ + -f scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set + +# 1x GPU H100 runners +helm install 1-gpu-h100 \ + --namespace actions-runner-system \ + -f scripts/k8s-runner-resources/runner-values-1-gpu-h100.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set +``` + +## 6. Verify + +Check that listener pods are running for each runner set: + +```bash +kubectl get pods -n actions-runner-system +``` + +Each runner set should have a listener pod in `Running` state. Runner pods will be created on-demand when workflows target the corresponding `runs-on` label (matching `runnerScaleSetName` in each values file). + +## Upgrading + +To update a runner set (e.g. after changing values): + +```bash +helm upgrade \ + --namespace actions-runner-system \ + -f scripts/k8s-runner-resources/.yaml \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set +``` + +## Uninstalling + +```bash +# Remove a runner set +helm uninstall -n actions-runner-system + +# Remove the controller (after all runner sets are removed) +helm uninstall arc -n actions-runner-system +``` diff --git a/scripts/k8s-runner-resources/arc-secret-template.yaml b/scripts/k8s-runner-resources/arc-secret-template.yaml new file mode 100644 index 0000000000..eb180a9373 --- /dev/null +++ b/scripts/k8s-runner-resources/arc-secret-template.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: github-arc-secret + namespace: actions-runner-systemapp +type: Opaque +stringData: + github_app_id: "2703284" + github_app_installation_id: "105440663" + github_app_private_key: "github-app-private-key" diff --git a/scripts/k8s-runner-resources/runner-values-1-gpu-h100.yaml b/scripts/k8s-runner-resources/runner-values-1-gpu-h100.yaml new file mode 100644 index 0000000000..5b506c122b --- /dev/null +++ b/scripts/k8s-runner-resources/runner-values-1-gpu-h100.yaml @@ -0,0 +1,130 @@ +## GitHub config URL for the runner scale set +githubConfigUrl: "https://github.com/lightseekorg/smg" + +## Name of the Kubernetes secret containing GitHub credentials +## The secret should contain either a PAT (github_token) or GitHub App credentials +githubConfigSecret: github-arc-secret + +## Runner scaling configuration +minRunners: 0 +maxRunners: 6 + +## name of the runner scale set to create. Defaults to the helm release name +runnerScaleSetName: "1-gpu-h100" + +## name of runner group +runnerGroup: "k8s-runner-gpu" + +resourceMeta: {} +keyVault: {} + +## Runner pod template +template: + spec: + nodeSelector: + nvidia.com/gpu: "true" + node.kubernetes.io/instance-type: BM.GPU.H100.8 + + tolerations: + - key: "nvidia.com/gpu" + operator: "Equal" + value: "true" + effect: "NoSchedule" + + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: actions.github.com/scale-set-name + operator: In + values: + - 1-gpu-h100 + - 2-gpu-h100 + topologyKey: kubernetes.io/hostname + + volumes: + - name: work + emptyDir: {} + - name: model-cache + persistentVolumeClaim: + claimName: model-cache + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 16Gi + - name: dind-sock + emptyDir: {} + - name: dind-externals + emptyDir: {} + - name: docker-storage + emptyDir: {} + + containers: + - name: runner + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.2 + command: + - /home/runner/run.sh + env: + - name: DOCKER_HOST + value: unix:///var/run/docker.sock + - name: RUNNER_WAIT_FOR_DOCKER_IN_SECONDS + value: '120' + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: model-cache + mountPath: /models + - name: dshm + mountPath: /dev/shm + - name: work + mountPath: /home/runner/_work + - name: dind-sock + mountPath: /var/run + - name: docker-storage + mountPath: /var/lib/docker + + initContainers: + - name: init-dind-externals + args: + - '-r' + - /home/runner/externals/. + - /home/runner/tmpDir/ + command: + - cp + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.2 + volumeMounts: + - mountPath: /home/runner/tmpDir + name: dind-externals + - name: dind + args: + - dockerd + - '--host=unix:///var/run/docker.sock' + - '--group=$(DOCKER_GROUP_GID)' + env: + - name: DOCKER_GROUP_GID + value: '123' + image: fra.ocir.io/idqj093njucb/docker:dind + restartPolicy: Always + securityContext: + privileged: true + startupProbe: + exec: + command: + - docker + - info + failureThreshold: 24 + initialDelaySeconds: 0 + periodSeconds: 5 + volumeMounts: + - mountPath: /home/runner/_work + name: work + - mountPath: /var/run + name: dind-sock + - mountPath: /home/runner/externals + name: dind-externals + - name: docker-storage + mountPath: /var/lib/docker diff --git a/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml b/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml new file mode 100644 index 0000000000..89251840d8 --- /dev/null +++ b/scripts/k8s-runner-resources/runner-values-2-gpu-h100.yaml @@ -0,0 +1,127 @@ +## GitHub config URL for the runner scale set +githubConfigUrl: "https://github.com/lightseekorg/smg" + +## Name of the Kubernetes secret containing GitHub credentials +## The secret should contain either a PAT (github_token) or GitHub App credentials +githubConfigSecret: github-arc-secret + +## Runner scaling configuration +minRunners: 0 +maxRunners: 6 + +## name of the runner scale set to create. Defaults to the helm release name +runnerScaleSetName: "2-gpu-h100" + +resourceMeta: {} +keyVault: {} + +## Runner pod template +template: + spec: + nodeSelector: + nvidia.com/gpu: "true" + node.kubernetes.io/instance-type: BM.GPU.H100.8 + + tolerations: + - key: "nvidia.com/gpu" + operator: "Equal" + value: "true" + effect: "NoSchedule" + + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: actions.github.com/scale-set-name + operator: In + values: + - 1-gpu-h100 + - 2-gpu-h100 + topologyKey: kubernetes.io/hostname + + volumes: + - name: work + emptyDir: {} + - name: model-cache + persistentVolumeClaim: + claimName: model-cache + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 16Gi + - name: dind-sock + emptyDir: {} + - name: dind-externals + emptyDir: {} + - name: docker-storage + emptyDir: {} + + containers: + - name: runner + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.2 + command: + - /home/runner/run.sh + env: + - name: DOCKER_HOST + value: unix:///var/run/docker.sock + - name: RUNNER_WAIT_FOR_DOCKER_IN_SECONDS + value: '120' + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: model-cache + mountPath: /models + - name: dshm + mountPath: /dev/shm + - name: work + mountPath: /home/runner/_work + - name: dind-sock + mountPath: /var/run + - name: docker-storage + mountPath: /var/lib/docker + + initContainers: + - name: init-dind-externals + args: + - '-r' + - /home/runner/externals/. + - /home/runner/tmpDir/ + command: + - cp + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.2 + volumeMounts: + - mountPath: /home/runner/tmpDir + name: dind-externals + - name: dind + args: + - dockerd + - '--host=unix:///var/run/docker.sock' + - '--group=$(DOCKER_GROUP_GID)' + env: + - name: DOCKER_GROUP_GID + value: '123' + image: fra.ocir.io/idqj093njucb/docker:dind + restartPolicy: Always + securityContext: + privileged: true + startupProbe: + exec: + command: + - docker + - info + failureThreshold: 24 + initialDelaySeconds: 0 + periodSeconds: 5 + volumeMounts: + - mountPath: /home/runner/_work + name: work + - mountPath: /var/run + name: dind-sock + - mountPath: /home/runner/externals + name: dind-externals + - name: docker-storage + mountPath: /var/lib/docker diff --git a/scripts/k8s-runner-resources/runner-values-4-gpu-general.yaml b/scripts/k8s-runner-resources/runner-values-4-gpu-general.yaml new file mode 100644 index 0000000000..16fd285234 --- /dev/null +++ b/scripts/k8s-runner-resources/runner-values-4-gpu-general.yaml @@ -0,0 +1,135 @@ +## GitHub config URL for the runner scale set +githubConfigUrl: "https://github.com/lightseekorg/smg" + +## Name of the Kubernetes secret containing GitHub credentials +## The secret should contain either a PAT (github_token) or GitHub App credentials +githubConfigSecret: github-arc-secret + +## Runner scaling configuration +minRunners: 10 +maxRunners: 20 + +## name of the runner scale set to create. Defaults to the helm release name +runnerScaleSetName: "k8s-runner-gpu" + +resourceMeta: {} +keyVault: {} + +## Runner pod template +template: + spec: + nodeSelector: + nvidia.com/gpu: "true" + + tolerations: + - key: "nvidia.com/gpu" + operator: "Equal" + value: "true" + effect: "NoSchedule" + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node.kubernetes.io/instance-type + operator: In + values: + - BM.GPU.H100.8 + - BM.GPU.A10.4 + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: actions.github.com/scale-set-name + operator: In + values: + - k8s-runner-gpu + - 4-gpu-h100 + topologyKey: kubernetes.io/hostname + + volumes: + - name: work + emptyDir: {} + - name: model-cache + persistentVolumeClaim: + claimName: model-cache + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 16Gi + - name: dind-sock + emptyDir: {} + - name: dind-externals + emptyDir: {} + - name: docker-storage + emptyDir: {} + + containers: + - name: runner + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.2 + command: + - /home/runner/run.sh + env: + - name: DOCKER_HOST + value: unix:///var/run/docker.sock + - name: RUNNER_WAIT_FOR_DOCKER_IN_SECONDS + value: '120' + resources: + limits: + nvidia.com/gpu: 4 + volumeMounts: + - name: model-cache + mountPath: /models + - name: dshm + mountPath: /dev/shm + - name: work + mountPath: /home/runner/_work + - name: dind-sock + mountPath: /var/run + - name: docker-storage + mountPath: /var/lib/docker + + initContainers: + - name: init-dind-externals + args: + - '-r' + - /home/runner/externals/. + - /home/runner/tmpDir/ + command: + - cp + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.2 + volumeMounts: + - mountPath: /home/runner/tmpDir + name: dind-externals + - name: dind + args: + - dockerd + - '--host=unix:///var/run/docker.sock' + - '--group=$(DOCKER_GROUP_GID)' + env: + - name: DOCKER_GROUP_GID + value: '123' + image: fra.ocir.io/idqj093njucb/docker:dind + restartPolicy: Always + securityContext: + privileged: true + startupProbe: + exec: + command: + - docker + - info + failureThreshold: 24 + initialDelaySeconds: 0 + periodSeconds: 5 + volumeMounts: + - mountPath: /home/runner/_work + name: work + - mountPath: /var/run + name: dind-sock + - mountPath: /home/runner/externals + name: dind-externals + - name: docker-storage + mountPath: /var/lib/docker diff --git a/scripts/k8s-runner-resources/runner-values-4-gpu-h100.yaml b/scripts/k8s-runner-resources/runner-values-4-gpu-h100.yaml new file mode 100644 index 0000000000..6b2656d339 --- /dev/null +++ b/scripts/k8s-runner-resources/runner-values-4-gpu-h100.yaml @@ -0,0 +1,126 @@ +## GitHub config URL for the runner scale set +githubConfigUrl: "https://github.com/lightseekorg/smg" + +## Name of the Kubernetes secret containing GitHub credentials +## The secret should contain either a PAT (github_token) or GitHub App credentials +githubConfigSecret: github-arc-secret + +## Runner scaling configuration +minRunners: 4 +maxRunners: 8 + +## name of the runner scale set to create. Defaults to the helm release name +runnerScaleSetName: "4-gpu-h100" + +resourceMeta: {} +keyVault: {} + +## Runner pod template +template: + spec: + nodeSelector: + nvidia.com/gpu: "true" + node.kubernetes.io/instance-type: BM.GPU.H100.8 + + tolerations: + - key: "nvidia.com/gpu" + operator: "Equal" + value: "true" + effect: "NoSchedule" + + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: actions.github.com/scale-set-name + operator: In + values: + - 4-gpu-h100 + topologyKey: kubernetes.io/hostname + + volumes: + - name: work + emptyDir: {} + - name: model-cache + persistentVolumeClaim: + claimName: model-cache + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 16Gi + - name: dind-sock + emptyDir: {} + - name: dind-externals + emptyDir: {} + - name: docker-storage + emptyDir: {} + + containers: + - name: runner + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.2 + command: + - /home/runner/run.sh + env: + - name: DOCKER_HOST + value: unix:///var/run/docker.sock + - name: RUNNER_WAIT_FOR_DOCKER_IN_SECONDS + value: '120' + resources: + limits: + nvidia.com/gpu: 4 + volumeMounts: + - name: model-cache + mountPath: /models + - name: dshm + mountPath: /dev/shm + - name: work + mountPath: /home/runner/_work + - name: dind-sock + mountPath: /var/run + - name: docker-storage + mountPath: /var/lib/docker + + initContainers: + - name: init-dind-externals + args: + - '-r' + - /home/runner/externals/. + - /home/runner/tmpDir/ + command: + - cp + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.2 + volumeMounts: + - mountPath: /home/runner/tmpDir + name: dind-externals + - name: dind + args: + - dockerd + - '--host=unix:///var/run/docker.sock' + - '--group=$(DOCKER_GROUP_GID)' + env: + - name: DOCKER_GROUP_GID + value: '123' + image: fra.ocir.io/idqj093njucb/docker:dind + restartPolicy: Always + securityContext: + privileged: true + startupProbe: + exec: + command: + - docker + - info + failureThreshold: 24 + initialDelaySeconds: 0 + periodSeconds: 5 + volumeMounts: + - mountPath: /home/runner/_work + name: work + - mountPath: /var/run + name: dind-sock + - mountPath: /home/runner/externals + name: dind-externals + - name: docker-storage + mountPath: /var/lib/docker diff --git a/scripts/k8s-runner-resources/runner-values-cpu.yaml b/scripts/k8s-runner-resources/runner-values-cpu.yaml new file mode 100644 index 0000000000..37cd151244 --- /dev/null +++ b/scripts/k8s-runner-resources/runner-values-cpu.yaml @@ -0,0 +1,90 @@ +## GitHub config URL for the runner scale set +githubConfigUrl: "https://github.com/lightseekorg/smg" + +## Name of the Kubernetes secret containing GitHub credentials +## The secret should contain either a PAT (github_token) or GitHub App credentials +githubConfigSecret: github-arc-secret + +## Runner scaling configuration +minRunners: 4 +maxRunners: 8 + +## name of the runner scale set to create. Defaults to the helm release name +runnerScaleSetName: "k8s-runner-cpu" + +resourceMeta: {} +keyVault: {} + +## Runner pod template +template: + spec: + volumes: + - name: work + emptyDir: {} + - name: dind-sock + emptyDir: {} + - name: dind-externals + emptyDir: {} + + containers: + - name: runner + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.2 + command: + - /home/runner/run.sh + env: + - name: DOCKER_HOST + value: unix:///var/run/docker.sock + - name: RUNNER_WAIT_FOR_DOCKER_IN_SECONDS + value: '120' + resources: + requests: + cpu: "8" + memory: "16Gi" + limits: + cpu: "8" + memory: "16Gi" + volumeMounts: + - name: work + mountPath: /home/runner/_work + - name: dind-sock + mountPath: /var/run + + initContainers: + - name: init-dind-externals + args: + - '-r' + - /home/runner/externals/. + - /home/runner/tmpDir/ + command: + - cp + image: fra.ocir.io/idqj093njucb/action-runner:v0.0.2 + volumeMounts: + - mountPath: /home/runner/tmpDir + name: dind-externals + - name: dind + args: + - dockerd + - '--host=unix:///var/run/docker.sock' + - '--group=$(DOCKER_GROUP_GID)' + env: + - name: DOCKER_GROUP_GID + value: '123' + image: fra.ocir.io/idqj093njucb/docker:dind + restartPolicy: Always + securityContext: + privileged: true + startupProbe: + exec: + command: + - docker + - info + failureThreshold: 24 + initialDelaySeconds: 0 + periodSeconds: 5 + volumeMounts: + - mountPath: /home/runner/_work + name: work + - mountPath: /var/run + name: dind-sock + - mountPath: /home/runner/externals + name: dind-externals From 7cda2c73f92abd7ed0cad38888dd8658922d523a Mon Sep 17 00:00:00 2001 From: XinyueZhang369 Date: Fri, 6 Mar 2026 10:06:02 -0800 Subject: [PATCH 2/4] Remove old resource files Signed-off-by: XinyueZhang369 --- .../arc-runner-autoscaler.yaml | 57 ------ .../k8s-runner-resources/arc-runner-cpu.yaml | 27 --- .../k8s-runner-resources/arc-runner-gpu.yaml | 172 ------------------ .../k8s-runner-resources/arc-runner-rbac.yaml | 45 ----- 4 files changed, 301 deletions(-) delete mode 100644 scripts/k8s-runner-resources/arc-runner-autoscaler.yaml delete mode 100644 scripts/k8s-runner-resources/arc-runner-cpu.yaml delete mode 100644 scripts/k8s-runner-resources/arc-runner-gpu.yaml delete mode 100644 scripts/k8s-runner-resources/arc-runner-rbac.yaml diff --git a/scripts/k8s-runner-resources/arc-runner-autoscaler.yaml b/scripts/k8s-runner-resources/arc-runner-autoscaler.yaml deleted file mode 100644 index fb568f0ba9..0000000000 --- a/scripts/k8s-runner-resources/arc-runner-autoscaler.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: actions.summerwind.dev/v1alpha1 -kind: HorizontalRunnerAutoscaler -metadata: - name: arc-runner-h100-autoscaler - namespace: actions-runner-system -spec: - scaleTargetRef: - kind: RunnerDeployment - name: arc-runner-gpu-h100 - - minReplicas: 11 - maxReplicas: 20 - - metrics: - - type: TotalNumberOfQueuedAndInProgressWorkflowRuns - repositoryNames: - - lightseekorg/smg ---- -apiVersion: actions.summerwind.dev/v1alpha1 -kind: HorizontalRunnerAutoscaler -metadata: - name: arc-runner-a10-autoscaler - namespace: actions-runner-system -spec: - scaleTargetRef: - kind: RunnerDeployment - name: arc-runner-gpu-a10 - - minReplicas: 2 - maxReplicas: 4 - - metrics: - - type: PercentageRunnersBusy - scaleUpThreshold: "0.95" - scaleDownThreshold: "0.25" - scaleUpFactor: "0.5" - scaleDownFactor: "0.5" ---- -apiVersion: actions.summerwind.dev/v1alpha1 -kind: HorizontalRunnerAutoscaler -metadata: - name: arc-cpu-runner-autoscaler - namespace: actions-runner-system -spec: - scaleTargetRef: - kind: RunnerDeployment - name: arc-runner-cpu - - minReplicas: 4 - maxReplicas: 8 - - metrics: - - type: PercentageRunnersBusy - scaleUpThreshold: "0.95" - scaleDownThreshold: "0.25" - scaleUpFactor: "0.5" - scaleDownFactor: "0.5" diff --git a/scripts/k8s-runner-resources/arc-runner-cpu.yaml b/scripts/k8s-runner-resources/arc-runner-cpu.yaml deleted file mode 100644 index 664d412618..0000000000 --- a/scripts/k8s-runner-resources/arc-runner-cpu.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: actions.summerwind.dev/v1alpha1 -kind: RunnerDeployment -metadata: - name: arc-runner-cpu - namespace: actions-runner-system -spec: - replicas: 4 - template: - spec: - ephemeral: true - repository: lightseekorg/smg - labels: - - k8s-runner-cpu - serviceAccountName: arc-runner-sa - - containers: - - name: runner - image: fra.ocir.io/idqj093njucb/action-runner:v0.0.1 - resources: - requests: - cpu: "8" - memory: "16Gi" - limits: - cpu: "8" - memory: "16Gi" - - name: docker - image: fra.ocir.io/idqj093njucb/docker:dind diff --git a/scripts/k8s-runner-resources/arc-runner-gpu.yaml b/scripts/k8s-runner-resources/arc-runner-gpu.yaml deleted file mode 100644 index 3a62d6f3d9..0000000000 --- a/scripts/k8s-runner-resources/arc-runner-gpu.yaml +++ /dev/null @@ -1,172 +0,0 @@ -apiVersion: actions.summerwind.dev/v1alpha1 -kind: RunnerDeployment -metadata: - name: arc-runner-gpu-h100 - namespace: actions-runner-system -spec: - replicas: 11 - template: - spec: - ephemeral: true - repository: lightseekorg/smg - labels: - - 4-gpu-h100 - - k8s-runner-gpu - serviceAccountName: arc-runner-sa - - nodeSelector: - nvidia.com/gpu: "true" - node.kubernetes.io/instance-type: BM.GPU.H100.8 - - tolerations: - - key: "nvidia.com/gpu" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - affinity: - podAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: runner-deployment-name - operator: In - values: - - arc-runner-gpu-h100 - topologyKey: kubernetes.io/hostname - - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: docker-sock - emptyDir: {} - - name: docker-storage - emptyDir: - medium: Memory - sizeLimit: 4Gi - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 16Gi - - containers: - - name: runner - image: fra.ocir.io/idqj093njucb/action-runner:v0.0.1 - resources: - limits: - nvidia.com/gpu: 4 - volumeMounts: - - name: model-cache - mountPath: /models - - name: docker-sock - mountPath: /var/run - - name: dshm - mountPath: /dev/shm - env: - - name: DOCKER_HOST - value: unix:///var/run/docker.sock - - name: docker - image: fra.ocir.io/idqj093njucb/docker:dind - securityContext: - privileged: true # Required for DinD - resources: - requests: - cpu: "1" - memory: "2Gi" - limits: - cpu: "2" - memory: "4Gi" - env: - - name: DOCKER_TLS_CERTDIR - value: "" # Disables TLS for shared socket use - - name: DOCKER_DRIVER - value: overlay2 - volumeMounts: - - name: docker-sock - mountPath: /var/run - - name: docker-storage - mountPath: /var/lib/docker ---- -apiVersion: actions.summerwind.dev/v1alpha1 -kind: RunnerDeployment -metadata: - name: arc-runner-gpu-a10 - namespace: actions-runner-system -spec: - replicas: 2 - template: - spec: - ephemeral: true - repository: lightseekorg/smg - labels: - - 4-gpu-a10 - - k8s-runner-gpu - serviceAccountName: arc-runner-sa - - nodeSelector: - nvidia.com/gpu: "true" - node.kubernetes.io/instance-type: BM.GPU.A10.4 - - tolerations: - - key: "nvidia.com/gpu" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - affinity: - podAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: runner-deployment-name - operator: In - values: - - arc-runner-gpu-a10 - topologyKey: kubernetes.io/hostname - - volumes: - - name: model-cache - persistentVolumeClaim: - claimName: model-cache - - name: docker-sock - emptyDir: {} - - name: docker-storage - emptyDir: {} - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 16Gi - - containers: - - name: runner - image: fra.ocir.io/idqj093njucb/action-runner:v0.0.1 - resources: - limits: - nvidia.com/gpu: 4 - volumeMounts: - - name: model-cache - mountPath: /models - - name: docker-sock - mountPath: /var/run - - name: dshm - mountPath: /dev/shm - env: - - name: DOCKER_HOST - value: unix:///var/run/docker.sock - - name: docker - image: fra.ocir.io/idqj093njucb/docker:dind - securityContext: - privileged: true # Required for DinD - env: - - name: DOCKER_TLS_CERTDIR - value: "" # Disables TLS for shared socket use - volumeMounts: - - name: docker-sock - mountPath: /var/run - - name: docker-storage - mountPath: /var/lib/docker diff --git a/scripts/k8s-runner-resources/arc-runner-rbac.yaml b/scripts/k8s-runner-resources/arc-runner-rbac.yaml deleted file mode 100644 index c317d45716..0000000000 --- a/scripts/k8s-runner-resources/arc-runner-rbac.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: arc-runner-sa - namespace: actions-runner-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: arc-runner - namespace: actions-runner-system -rules: - # Argo Workflows - - apiGroups: [""] - resources: - - secrets - verbs: - - get - - list - - watch - - # Pods - - apiGroups: [""] - resources: - - pods - - pods/log - - pods/exec - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: arc-runner-rb - namespace: actions-runner-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: arc-runner -subjects: - - kind: ServiceAccount - name: arc-runner-sa - namespace: actions-runner-system From 8ec269eb81840db00847b18a3189df90a370a22b Mon Sep 17 00:00:00 2001 From: XinyueZhang369 Date: Fri, 6 Mar 2026 10:13:41 -0800 Subject: [PATCH 3/4] Fix typo Signed-off-by: XinyueZhang369 --- .github/workflows/nightly-benchmark.yml | 2 +- scripts/k8s-runner-resources/arc-secret-template.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index 8982c047ce..e30c47e406 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -188,7 +188,7 @@ jobs: name: "${{ matrix.model.id }} / multi-${{ matrix.variant.id }}" needs: build-wheel if: ${{ !cancelled() }} - runs-on: 4-gpu-h100" + runs-on: 4-gpu-h100 timeout-minutes: 1440 strategy: fail-fast: false diff --git a/scripts/k8s-runner-resources/arc-secret-template.yaml b/scripts/k8s-runner-resources/arc-secret-template.yaml index eb180a9373..1be4443b90 100644 --- a/scripts/k8s-runner-resources/arc-secret-template.yaml +++ b/scripts/k8s-runner-resources/arc-secret-template.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: name: github-arc-secret - namespace: actions-runner-systemapp + namespace: actions-runner-system type: Opaque stringData: github_app_id: "2703284" From 159bbd0a06fb096042f2d1b3fd34c19db380ad61 Mon Sep 17 00:00:00 2001 From: XinyueZhang369 Date: Fri, 6 Mar 2026 10:22:49 -0800 Subject: [PATCH 4/4] remove runner group Signed-off-by: XinyueZhang369 --- scripts/k8s-runner-resources/runner-values-1-gpu-h100.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/k8s-runner-resources/runner-values-1-gpu-h100.yaml b/scripts/k8s-runner-resources/runner-values-1-gpu-h100.yaml index 5b506c122b..f0217002a3 100644 --- a/scripts/k8s-runner-resources/runner-values-1-gpu-h100.yaml +++ b/scripts/k8s-runner-resources/runner-values-1-gpu-h100.yaml @@ -12,9 +12,6 @@ maxRunners: 6 ## name of the runner scale set to create. Defaults to the helm release name runnerScaleSetName: "1-gpu-h100" -## name of runner group -runnerGroup: "k8s-runner-gpu" - resourceMeta: {} keyVault: {}