From c18680c906ca916050a1cbcaed2ba12bb74932f7 Mon Sep 17 00:00:00 2001 From: limengxuan <391013634@qq.com> Date: Fri, 15 Nov 2024 09:31:50 +0800 Subject: [PATCH 01/10] update ci and initdevices logic Signed-off-by: limengxuan <391013634@qq.com> --- README.md | 10 +- README_cn.md | 10 +- .../device-plugin/daemonsetnvidia.yaml | 12 +- .../hami/templates/scheduler/configmap.yaml | 4 +- .../templates/scheduler/configmapnew.yaml | 6 +- .../hami/templates/scheduler/deployment.yaml | 30 +-- .../templates/scheduler/device-configmap.yaml | 40 ++- .../scheduler/job-patch/job-createSecret.yaml | 2 +- .../scheduler/job-patch/job-patchWebhook.yaml | 2 +- charts/hami/values.yaml | 17 +- cmd/device-plugin/nvidia/main.go | 3 +- cmd/device-plugin/nvidia/plugin-manager.go | 4 +- cmd/device-plugin/nvidia/vgpucfg.go | 65 +---- docs/ascend910b-support.md | 4 +- docs/ascend910b-support_cn.md | 4 +- examples/nvidia/default_use.yaml | 2 +- .../nvinternal/plugin/manager/factory.go | 4 +- .../nvinternal/plugin/manager/options.go | 4 +- .../nvinternal/plugin/register.go | 10 +- .../nvidiadevice/nvinternal/plugin/server.go | 54 +++- .../nvinternal/plugin/server_test.go | 4 +- .../nvidiadevice/nvinternal/rm/device_map.go | 8 +- .../nvidiadevice/nvinternal/rm/devices.go | 6 +- .../nvinternal/rm/nvml_manager.go | 6 +- .../nvidiadevice/nvinternal/rm/rm.go | 8 +- .../nvinternal/rm/tegra_devices.go | 4 +- .../nvinternal/rm/tegra_manager.go | 4 +- pkg/device/ascend/device.go | 9 +- pkg/device/ascend/vnpu.go | 49 +--- pkg/device/cambricon/device.go | 11 +- pkg/device/devices.go | 59 ++++- pkg/device/devices_test.go | 249 ++++++++++++++++++ pkg/device/hygon/device.go | 11 +- pkg/device/iluvatar/device.go | 11 +- pkg/device/iluvatar/device_test.go | 2 +- pkg/device/metax/device.go | 7 +- pkg/device/mthreads/device.go | 11 +- pkg/device/nvidia/device.go | 129 ++++++--- pkg/device/nvidia/device_test.go | 175 +++++++++++- pkg/k8sutil/pod_test.go | 13 +- pkg/scheduler/config/config.go | 3 +- pkg/scheduler/scheduler_test.go | 12 +- pkg/scheduler/score_test.go | 9 +- pkg/scheduler/webhook_test.go | 13 +- pkg/util/types.go | 48 +--- pkg/util/util.go | 29 -- pkg/util/util_test.go | 139 ---------- version.mk | 2 +- 48 files changed, 821 insertions(+), 497 deletions(-) create mode 100644 pkg/device/devices_test.go diff --git a/README.md b/README.md index f0636a0656..765a9b24ab 100644 --- a/README.md +++ b/README.md @@ -108,16 +108,10 @@ Add our repo in helm helm repo add hami-charts https://project-hami.github.io/HAMi/ ``` -Check your Kubernetes version by using the following command: +Use the following command for deployment: ``` -kubectl version -``` - -During installation, set the Kubernetes scheduler image version to match your Kubernetes server version. For instance, if your cluster server version is 1.16.8, use the following command for deployment: - -``` -helm install hami hami-charts/hami --set scheduler.kubeScheduler.imageTag=v1.16.8 -n kube-system +helm install hami hami-charts/hami -n kube-system ``` Customize your installation by adjusting the [configs](docs/config.md). diff --git a/README_cn.md b/README_cn.md index bbace227aa..0d0e36473d 100644 --- a/README_cn.md +++ b/README_cn.md @@ -104,16 +104,10 @@ helm repo add hami-charts https://project-hami.github.io/HAMi/ kubectl label nodes {nodeid} gpu=on ``` -使用下列指令获取集群服务端版本 +使用如下指令进行安装 ```bash -kubectl version -``` - -在安装过程中须根据集群服务端版本（上一条指令的结果）指定调度器镜像版本，例如集群服务端版本为1.16.8，则可以使用如下指令进行安装 - -```bash -helm install hami hami-charts/hami --set scheduler.kubeScheduler.imageTag=v1.16.8 -n kube-system +helm install hami hami-charts/hami -n kube-system ``` 你可以修改这里的[配置](docs/config_cn.md)来定制安装 diff --git a/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/charts/hami/templates/device-plugin/daemonsetnvidia.yaml index ae7cf694fe..8874aee412 100644 --- a/charts/hami/templates/device-plugin/daemonsetnvidia.yaml +++ b/charts/hami/templates/device-plugin/daemonsetnvidia.yaml @@ -30,7 +30,6 @@ spec: runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }} {{- end }} {{- include "hami-vgpu.imagePullSecrets" . | nindent 6}} - # serviceAccountName: serviceAccountName: {{ include "hami-vgpu.device-plugin" . }} priorityClassName: system-node-critical hostPID: true @@ -45,11 +44,8 @@ spec: command: ["/bin/sh","-c", {{ printf "cp -f /k8s-vgpu/lib/nvidia/* %s/vgpu/" .Values.global.gpuHookPath | quote }}] command: - nvidia-device-plugin - - --resource-name={{ .Values.resourceName }} + - --config-file=/device-config.yaml - --mig-strategy={{ .Values.devicePlugin.migStrategy }} - - --device-memory-scaling={{ .Values.devicePlugin.deviceMemoryScaling }} - - --device-cores-scaling={{ .Values.devicePlugin.deviceCoreScaling }} - - --device-split-count={{ .Values.devicePlugin.deviceSplitCount }} - --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }} {{- range .Values.devicePlugin.extraArgs }} - {{ . }} @@ -79,6 +75,9 @@ spec: mountPath: /config - name: hosttmp mountPath: /tmp + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml - name: vgpu-monitor image: {{ .Values.devicePlugin.image }}:{{ .Values.version }} imagePullPolicy: {{ .Values.devicePlugin.imagePullPolicy | quote }} @@ -137,6 +136,9 @@ spec: - name: deviceconfig configMap: name: {{ template "hami-vgpu.device-plugin" . }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device {{- if .Values.devicePlugin.nvidianodeSelector }} nodeSelector: {{ toYaml .Values.devicePlugin.nvidianodeSelector | nindent 8 }} {{- end }} diff --git a/charts/hami/templates/scheduler/configmap.yaml b/charts/hami/templates/scheduler/configmap.yaml index c843bb9e3c..b69ee15e36 100644 --- a/charts/hami/templates/scheduler/configmap.yaml +++ b/charts/hami/templates/scheduler/configmap.yaml @@ -25,7 +25,7 @@ data: }, "managedResources": [ {{- if .Values.devices.ascend.enabled }} - {{- range .Values.devices.ascend.resources }} + {{- range .Values.devices.ascend.customresources }} { "name": "{{ . }}", "ignoredByScheduler": true @@ -33,7 +33,7 @@ data: {{- end }} {{- end }} {{- if .Values.devices.mthreads.enabled }} - {{- range .Values.devices.mthreads.resources }} + {{- range .Values.devices.mthreads.customresources }} { "name": "{{ . }}", "ignoredByScheduler": true diff --git a/charts/hami/templates/scheduler/configmapnew.yaml b/charts/hami/templates/scheduler/configmapnew.yaml index a2b3263786..acc5076902 100644 --- a/charts/hami/templates/scheduler/configmapnew.yaml +++ b/charts/hami/templates/scheduler/configmapnew.yaml @@ -8,7 +8,7 @@ metadata: {{- include "hami-vgpu.labels" . | nindent 4 }} data: config.yaml: | - {{- if gt (.Values.scheduler.kubeScheduler.imageTag | substr 3 5| atoi) 25}} + {{- if gt (.Capabilities.KubeVersion.Minor | int) 25}} apiVersion: kubescheduler.config.k8s.io/v1 {{- else }} apiVersion: kubescheduler.config.k8s.io/v1beta2 @@ -50,13 +50,13 @@ data: - name: {{ .Values.iluvatarResourceName }} ignoredByScheduler: true {{- if .Values.devices.ascend.enabled }} - {{- range .Values.devices.ascend.resources }} + {{- range .Values.devices.ascend.customresources }} - name: {{ . }} ignoredByScheduler: true {{- end }} {{- end }} {{- if .Values.devices.mthreads.enabled }} - {{- range .Values.devices.mthreads.resources }} + {{- range .Values.devices.mthreads.customresources }} - name: {{ . }} ignoredByScheduler: true {{- end }} diff --git a/charts/hami/templates/scheduler/deployment.yaml b/charts/hami/templates/scheduler/deployment.yaml index 92b924debb..0e0246d682 100644 --- a/charts/hami/templates/scheduler/deployment.yaml +++ b/charts/hami/templates/scheduler/deployment.yaml @@ -33,11 +33,11 @@ spec: containers: {{- if .Values.scheduler.kubeScheduler.enabled }} - name: kube-scheduler - image: {{ .Values.scheduler.kubeScheduler.image }}:{{ .Values.scheduler.kubeScheduler.imageTag }} + image: {{ .Values.scheduler.kubeScheduler.image }}:{{ .Capabilities.KubeVersion.Version }} imagePullPolicy: {{ .Values.scheduler.kubeScheduler.imagePullPolicy | quote }} command: - kube-scheduler - {{- if ge (.Values.scheduler.kubeScheduler.imageTag | substr 3 5| atoi) 22}} + {{- if ge (.Capabilities.KubeVersion.Minor | int) 22 }} {{- range .Values.scheduler.kubeScheduler.extraNewArgs }} - {{ . }} {{- end }} @@ -78,29 +78,15 @@ spec: {{- end }} command: - scheduler - - --resource-name={{ .Values.resourceName }} - - --resource-mem={{ .Values.resourceMem }} - - --resource-cores={{ .Values.resourceCores }} - - --resource-mem-percentage={{ .Values.resourceMemPercentage }} - - --resource-priority={{ .Values.resourcePriority }} - --http_bind=0.0.0.0:443 - --cert_file=/tls/tls.crt - --key_file=/tls/tls.key - --scheduler-name={{ .Values.schedulerName }} - --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }} - - --default-mem={{ .Values.scheduler.defaultMem }} - - --default-gpu={{ .Values.scheduler.defaultGPUNum }} - - --default-cores={{ .Values.scheduler.defaultCores }} - - --iluvatar-memory={{ .Values.iluvatarResourceMem }} - - --iluvatar-cores={{ .Values.iluvatarResourceCore }} - - --cambricon-mlu-name={{ .Values.mluResourceName }} - - --cambricon-mlu-memory={{ .Values.mluResourceMem }} - - --cambricon-mlu-cores={{ .Values.mluResourceCores }} - - --overwrite-env={{ .Values.scheduler.overwriteEnv }} - --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }} - --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }} + - --device-config-file=/device-config.yaml {{- if .Values.devices.ascend.enabled }} - - --ascend-config-file=/ascend-config.yaml - --enable-ascend=true {{- end }} {{- if .Values.scheduler.nodeLabelSelector }} @@ -121,11 +107,9 @@ spec: volumeMounts: - name: tls-config mountPath: /tls - {{- if .Values.devices.ascend.enabled }} - name: device-config - mountPath: /ascend-config.yaml - subPath: ascend-config.yaml - {{- end }} + mountPath: /device-config.yaml + subPath: device-config.yaml {{- if .Values.scheduler.livenessProbe }} livenessProbe: httpGet: @@ -144,17 +128,15 @@ spec: {{- if .Values.scheduler.kubeScheduler.enabled }} - name: scheduler-config configMap: - {{- if ge (.Values.scheduler.kubeScheduler.imageTag | substr 3 5| atoi) 22 }} + {{- if ge (.Capabilities.KubeVersion.Minor | int) 22 }} name: {{ template "hami-vgpu.scheduler" . }}-newversion {{- else }} name: {{ template "hami-vgpu.scheduler" . }} {{- end }} {{- end }} - {{- if .Values.devices.ascend.enabled }} - name: device-config configMap: name: {{ include "hami-vgpu.scheduler" . }}-device - {{- end }} {{- if .Values.scheduler.nodeSelector }} nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }} {{- end }} diff --git a/charts/hami/templates/scheduler/device-configmap.yaml b/charts/hami/templates/scheduler/device-configmap.yaml index 85afedea56..7e132e1528 100644 --- a/charts/hami/templates/scheduler/device-configmap.yaml +++ b/charts/hami/templates/scheduler/device-configmap.yaml @@ -1,4 +1,3 @@ -{{- if .Values.devices.ascend.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -7,10 +6,41 @@ metadata: app.kubernetes.io/component: hami-scheduler {{- include "hami-vgpu.labels" . | nindent 4 }} data: - ascend-config.yaml: |- - {{- if .Files.Glob "files/ascend-config.yaml" }} - {{- .Files.Get "files/ascend-config.yaml" | nindent 4}} + device-config.yaml: |- + {{- if .Files.Glob "files/device-config.yaml" }} + {{- .Files.Get "files/device-config.yaml" | nindent 4}} {{- else }} + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + deviceSplitCount: 10 + deviceMemoryScaling: 1 + deviceCoreScaling: 1 + cambricon: + resourceCountName: {{ .Values.mluResourceName }} + resourceMemoryName: {{ .Values.mluResourceMem }} + resourceCoreName: {{ .Values.mluResourceCores }} + hygon: + resourceCountName: {{ .Values.dcuResourceName }} + resourceMemoryName: {{ .Values.dcuResourceMem }} + resourceCoreName: {{ .Values.dcuResourceCores }} + metax: + resourceCountName: "metax-tech.com/gpu" + mthreads: + resourceCountName: "mthreads.com/vgpu" + resourceMemoryName: "mthreads.com/sgpu-memory" + resourceCoreName: "mthreads.com/sgpu-core" + iluvatar: + resourceCountName: {{ .Values.iluvatarResourceName }} + resourceMemoryName: {{ .Values.iluvatarResourceMem }} + resourceCoreName: {{ .Values.iluvatarResourceCore }} vnpus: - chipName: 910B commonWord: Ascend910A @@ -71,5 +101,3 @@ data: aiCore: 4 aiCPU: 4 {{ end }} - -{{- end }} diff --git a/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml index e8fe3afd1d..a066f566c2 100644 --- a/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml +++ b/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml @@ -30,7 +30,7 @@ spec: {{- end }} containers: - name: create - {{- if ge (.Values.scheduler.kubeScheduler.imageTag | substr 3 5| atoi) 22 }} + {{- if ge (.Capabilities.KubeVersion.Minor | int) 22 }} image: {{ .Values.scheduler.patch.imageNew }} {{- else }} image: {{ .Values.scheduler.patch.image }} diff --git a/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml index a26fd42613..e6fde7f85f 100644 --- a/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml +++ b/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml @@ -30,7 +30,7 @@ spec: {{- end }} containers: - name: patch - {{- if ge (.Values.scheduler.kubeScheduler.imageTag | substr 3 5| atoi) 22 }} + {{- if ge (.Capabilities.KubeVersion.Minor | int) 22 }} image: {{ .Values.scheduler.patch.imageNew }} {{- else }} image: {{ .Values.scheduler.patch.image }} diff --git a/charts/hami/values.yaml b/charts/hami/values.yaml index 97d6e1e020..86bff27bdb 100644 --- a/charts/hami/values.yaml +++ b/charts/hami/values.yaml @@ -3,7 +3,7 @@ nameOverride: "" fullnameOverride: "" imagePullSecrets: [ ] -version: "v2.4.0" +version: "latest" #Nvidia GPU Parameters resourceName: "nvidia.com/gpu" @@ -46,11 +46,8 @@ scheduler: # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default # scheduler pod from the cluster first, we must specify node name to skip the schedule workflow. nodeName: "" -# nodeLabelSelector: -# "gpu": "on" - defaultMem: 0 - defaultCores: 0 - defaultGPUNum: 1 + #nodeLabelSelector: + # "gpu": "on" overwriteEnv: "false" defaultSchedulerPolicy: nodeSchedulerPolicy: binpack @@ -61,7 +58,6 @@ scheduler: kubeScheduler: # @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default. enabled: true - imageTag: "v1.20.0" image: registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler imagePullPolicy: IfNotPresent extraNewArgs: @@ -115,9 +111,6 @@ devicePlugin: monitorimage: "projecthami/hami" monitorctrPath: /usr/local/vgpu/containers imagePullPolicy: IfNotPresent - deviceSplitCount: 10 - deviceMemoryScaling: 1 - deviceCoreScaling: 1 runtimeClassName: "" migStrategy: "none" disablecorelimit: "false" @@ -138,7 +131,7 @@ devicePlugin: devices: mthreads: enabled: false - resources: + customresources: - mthreads.com/vgpu ascend: enabled: false @@ -148,7 +141,7 @@ devices: nodeSelector: ascend: "on" tolerations: [] - resources: + customresources: - huawei.com/Ascend910A - huawei.com/Ascend910A-memory - huawei.com/Ascend910B diff --git a/cmd/device-plugin/nvidia/main.go b/cmd/device-plugin/nvidia/main.go index 342b5b8fb3..c7fe2cf720 100644 --- a/cmd/device-plugin/nvidia/main.go +++ b/cmd/device-plugin/nvidia/main.go @@ -23,7 +23,6 @@ import ( "syscall" "time" - "github.com/Project-HAMi/HAMi/pkg/device" "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/info" "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin" "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/rm" @@ -162,7 +161,7 @@ func start(c *cli.Context, flags []cli.Flag) error { return fmt.Errorf("failed to create FS watcher: %v", err) } defer watcher.Close() - device.InitDevices() + //device.InitDevices() /*Loading config files*/ klog.Infof("Start working on node %s", util.NodeName) diff --git a/cmd/device-plugin/nvidia/plugin-manager.go b/cmd/device-plugin/nvidia/plugin-manager.go index d22fecbd27..237f6b759f 100644 --- a/cmd/device-plugin/nvidia/plugin-manager.go +++ b/cmd/device-plugin/nvidia/plugin-manager.go @@ -21,14 +21,14 @@ import ( "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/cdi" "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager" - "github.com/Project-HAMi/HAMi/pkg/util" + "github.com/Project-HAMi/HAMi/pkg/device/nvidia" "github.com/NVIDIA/go-nvlib/pkg/nvml" spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" ) // NewPluginManager creates an NVML-based plugin manager. -func NewPluginManager(config *util.DeviceConfig) (manager.Interface, error) { +func NewPluginManager(config *nvidia.DeviceConfig) (manager.Interface, error) { var err error switch *config.Flags.MigStrategy { case spec.MigStrategyNone: diff --git a/cmd/device-plugin/nvidia/vgpucfg.go b/cmd/device-plugin/nvidia/vgpucfg.go index 91b4f309fa..a6d2a83078 100644 --- a/cmd/device-plugin/nvidia/vgpucfg.go +++ b/cmd/device-plugin/nvidia/vgpucfg.go @@ -17,11 +17,13 @@ limitations under the License. package main import ( - "encoding/json" "fmt" "os" "strings" + "github.com/Project-HAMi/HAMi/pkg/device" + "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin" + "github.com/Project-HAMi/HAMi/pkg/device/nvidia" "github.com/Project-HAMi/HAMi/pkg/util" spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" @@ -95,66 +97,25 @@ func updateFromCLIFlag[T any](pflag **T, c *cli.Context, flagName string) { } } -func readFromConfigFile() error { - jsonbyte, err := os.ReadFile("/config/config.json") - if err != nil { - return err - } - var deviceConfigs util.DevicePluginConfigs - err = json.Unmarshal(jsonbyte, &deviceConfigs) - if err != nil { - return err - } - klog.Infof("Device Plugin Configs: %v", fmt.Sprintf("%v", deviceConfigs)) - for _, val := range deviceConfigs.Nodeconfig { - if os.Getenv(util.NodeNameEnvName) == val.Name { - klog.Infof("Reading config from file %s", val.Name) - if val.Devicememoryscaling > 0 { - *util.DeviceMemoryScaling = val.Devicememoryscaling - } - if val.Devicecorescaling > 0 { - *util.DeviceCoresScaling = val.Devicecorescaling - } - if val.Devicesplitcount > 0 { - *util.DeviceSplitCount = val.Devicesplitcount - } - if val.FilterDevice != nil && (len(val.FilterDevice.UUID) > 0 || len(val.FilterDevice.Index) > 0) { - util.DevicePluginFilterDevice = val.FilterDevice - } - klog.Infof("FilterDevice: %v", val.FilterDevice) - } - } - return nil -} - -func generateDeviceConfigFromNvidia(cfg *spec.Config, c *cli.Context, flags []cli.Flag) (util.DeviceConfig, error) { - devcfg := util.DeviceConfig{} +func generateDeviceConfigFromNvidia(cfg *spec.Config, c *cli.Context, flags []cli.Flag) (nvidia.DeviceConfig, error) { + devcfg := nvidia.DeviceConfig{} devcfg.Config = cfg klog.Infoln("flags=", flags) for _, flag := range flags { for _, n := range flag.Names() { // Common flags - if strings.Compare(n, "device-split-count") == 0 { - updateFromCLIFlag(&util.DeviceSplitCount, c, n) - } - if strings.Compare(n, "device-memory-scaling") == 0 { - updateFromCLIFlag(&util.DeviceMemoryScaling, c, n) - klog.Infoln("DeviceMemoryScaling", *util.DeviceMemoryScaling) - } - if strings.Compare(n, "device-cores-scaling") == 0 { - updateFromCLIFlag(&util.DeviceCoresScaling, c, n) - } - if strings.Compare(n, "disable-core-limit") == 0 { - updateFromCLIFlag(&util.DisableCoreLimit, c, n) - } - if strings.Compare(n, "resource-name") == 0 { - updateFromCLIFlag(&devcfg.ResourceName, c, n) + if strings.Compare(n, "config-file") == 0 { + updateFromCLIFlag(&plugin.ConfigFile, c, n) } } } - if err := readFromConfigFile(); err != nil { - return devcfg, err + + config, err := device.LoadConfig(*plugin.ConfigFile) + if err != nil { + klog.Fatalf("failed to load ascend vnpu config file %s: %v", *plugin.ConfigFile, err) } + devcfg.ResourceName = &config.NvidiaConfig.ResourceCountName + klog.Infoln("reading config=", config.NvidiaConfig.ResourceCountName, "devcfg", *devcfg.ResourceName, "configfile=", *plugin.ConfigFile) return devcfg, nil } diff --git a/docs/ascend910b-support.md b/docs/ascend910b-support.md index 7b6d229a95..6c970ff5df 100644 --- a/docs/ascend910b-support.md +++ b/docs/ascend910b-support.md @@ -11,7 +11,7 @@ ## Prerequisites -* Ascend device type: 910B(300T A2) +* Ascend device type: 910B,910B3,310P * driver version >= 24.1.rc1 * Ascend docker runtime @@ -52,7 +52,7 @@ You can customize the ascend share configuration by following the steps below: └── values.yaml ``` - ### Create the ascend-config.yaml file, the content is as follows + ### Create the device-config.yaml file, the content is as follows ```yaml vnpus: diff --git a/docs/ascend910b-support_cn.md b/docs/ascend910b-support_cn.md index c8f1a1bb13..291b36ac7a 100644 --- a/docs/ascend910b-support_cn.md +++ b/docs/ascend910b-support_cn.md @@ -12,7 +12,7 @@ * Ascend docker runtime * driver version > 24.1.rc1 -* Ascend device type: 910B(300T A2) +* Ascend device type: 910B,910B3,310P ## 开启NPU复用 @@ -50,7 +50,7 @@ HAMi 目前有一个 NPU 内置[虚拟化配置文件](https://github.com/Projec └── values.yaml ``` - ### 在 files 目录下创建 Create the ascend-config.yaml 文件，配置文件如下所示, 可以按需调整 + ### 在 files 目录下创建 Create the device-config.yaml 文件，配置文件如下所示, 可以按需调整 ```yaml vnpus: diff --git a/examples/nvidia/default_use.yaml b/examples/nvidia/default_use.yaml index 999cb9d43b..ab484967ba 100644 --- a/examples/nvidia/default_use.yaml +++ b/examples/nvidia/default_use.yaml @@ -5,7 +5,7 @@ metadata: spec: containers: - name: ubuntu-container - image: ubuntu:18.04 + image: ubuntu:22.04 command: ["bash", "-c", "sleep 86400"] resources: limits: diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/factory.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/factory.go index 2f9fe19228..10ab6d0668 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/factory.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/factory.go @@ -24,7 +24,7 @@ import ( "k8s.io/klog/v2" "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/cdi" - "github.com/Project-HAMi/HAMi/pkg/util" + "github.com/Project-HAMi/HAMi/pkg/device/nvidia" ) type manager struct { @@ -34,7 +34,7 @@ type manager struct { cdiHandler cdi.Interface cdiEnabled bool - config *util.DeviceConfig + config *nvidia.DeviceConfig infolib info.Interface } diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/options.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/options.go index 4f1997969c..256cc92282 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/options.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/options.go @@ -19,7 +19,7 @@ package manager import ( "github.com/NVIDIA/go-nvlib/pkg/nvml" "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/cdi" - "github.com/Project-HAMi/HAMi/pkg/util" + "github.com/Project-HAMi/HAMi/pkg/device/nvidia" ) // Option is a function that configures a manager @@ -61,7 +61,7 @@ func WithMigStrategy(migStrategy string) Option { } // WithConfig sets the config reference for the manager -func WithConfig(config *util.DeviceConfig) Option { +func WithConfig(config *nvidia.DeviceConfig) Option { return func(m *manager) { m.config = config } diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go index 5f14523b0d..01f536ae17 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go @@ -126,10 +126,10 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*api.DeviceInfo { } registeredmem := int32(memoryTotal / 1024 / 1024) - if *util.DeviceMemoryScaling != 1 { - registeredmem = int32(float64(registeredmem) * *util.DeviceMemoryScaling) + if plugin.schedulerConfig.DeviceMemoryScaling != 1 { + registeredmem = int32(float64(registeredmem) * plugin.schedulerConfig.DeviceMemoryScaling) } - klog.Infoln("MemoryScaling=", *util.DeviceMemoryScaling, "registeredmem=", registeredmem) + klog.Infoln("MemoryScaling=", plugin.schedulerConfig.DeviceMemoryScaling, "registeredmem=", registeredmem) health := true for _, val := range devs { if strings.Compare(val.ID, UUID) == 0 { @@ -149,9 +149,9 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*api.DeviceInfo { } res = append(res, &api.DeviceInfo{ ID: UUID, - Count: int32(*util.DeviceSplitCount), + Count: int32(plugin.schedulerConfig.DeviceSplitCount), Devmem: registeredmem, - Devcore: int32(*util.DeviceCoresScaling * 100), + Devcore: int32(plugin.schedulerConfig.DeviceCoreScaling * 100), Type: fmt.Sprintf("%v-%v", "NVIDIA", Model), Numa: numa, Health: health, diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go index 44b3f28cf6..818ad5db1e 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go @@ -17,6 +17,7 @@ package plugin import ( + "encoding/json" "errors" "fmt" "net" @@ -54,6 +55,7 @@ const ( var ( hostHookPath string + ConfigFile *string ) func init() { @@ -63,10 +65,11 @@ func init() { // NvidiaDevicePlugin implements the Kubernetes device plugin API type NvidiaDevicePlugin struct { rm rm.ResourceManager - config *util.DeviceConfig + config *nvidia.DeviceConfig deviceListEnvvar string deviceListStrategies spec.DeviceListStrategies socket string + schedulerConfig nvidia.NvidiaConfig cdiHandler cdi.Interface cdiEnabled bool @@ -77,12 +80,52 @@ type NvidiaDevicePlugin struct { stop chan interface{} } +func readFromConfigFile(sConfig *nvidia.NvidiaConfig) error { + jsonbyte, err := os.ReadFile("/config/config.json") + if err != nil { + return err + } + var deviceConfigs nvidia.DevicePluginConfigs + err = json.Unmarshal(jsonbyte, &deviceConfigs) + if err != nil { + return err + } + klog.Infof("Device Plugin Configs: %v", fmt.Sprintf("%v", deviceConfigs)) + for _, val := range deviceConfigs.Nodeconfig { + if os.Getenv(util.NodeNameEnvName) == val.Name { + klog.Infof("Reading config from file %s", val.Name) + if val.Devicememoryscaling > 0 { + sConfig.DeviceMemoryScaling = val.Devicememoryscaling + } + if val.Devicecorescaling > 0 { + sConfig.DeviceCoreScaling = val.Devicecorescaling + } + if val.Devicesplitcount > 0 { + sConfig.DeviceSplitCount = val.Devicesplitcount + } + if val.FilterDevice != nil && (len(val.FilterDevice.UUID) > 0 || len(val.FilterDevice.Index) > 0) { + nvidia.DevicePluginFilterDevice = val.FilterDevice + } + klog.Infof("FilterDevice: %v", val.FilterDevice) + } + } + return nil +} + // NewNvidiaDevicePlugin returns an initialized NvidiaDevicePlugin -func NewNvidiaDevicePlugin(config *util.DeviceConfig, resourceManager rm.ResourceManager, cdiHandler cdi.Interface, cdiEnabled bool) *NvidiaDevicePlugin { +func NewNvidiaDevicePlugin(config *nvidia.DeviceConfig, resourceManager rm.ResourceManager, cdiHandler cdi.Interface, cdiEnabled bool) *NvidiaDevicePlugin { _, name := resourceManager.Resource().Split() deviceListStrategies, _ := spec.NewDeviceListStrategies(*config.Flags.Plugin.DeviceListStrategy) + sConfig, err := device.LoadConfig(*ConfigFile) + klog.Infoln("reading config=", config, "resourceName", config.ResourceName, "configfile=", *ConfigFile) + if err != nil { + klog.Fatalf(`failed to load device config file %s: %v`, *ConfigFile, err) + } + readFromConfigFile(&sConfig.NvidiaConfig) + device.InitDevicesWithConfig(sConfig) + return &NvidiaDevicePlugin{ rm: resourceManager, config: config, @@ -92,6 +135,7 @@ func NewNvidiaDevicePlugin(config *util.DeviceConfig, resourceManager rm.Resourc cdiHandler: cdiHandler, cdiEnabled: cdiEnabled, cdiAnnotationPrefix: *config.Flags.Plugin.CDIAnnotationPrefix, + schedulerConfig: sConfig.NvidiaConfig, // These will be reinitialized every // time the plugin server is restarted. @@ -356,10 +400,10 @@ func (plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *kubeletdev } response.Envs["CUDA_DEVICE_SM_LIMIT"] = fmt.Sprint(devreq[0].Usedcores) response.Envs["CUDA_DEVICE_MEMORY_SHARED_CACHE"] = fmt.Sprintf("%s/vgpu/%v.cache", hostHookPath, uuid.New().String()) - if *util.DeviceMemoryScaling > 1 { + if plugin.schedulerConfig.DeviceMemoryScaling > 1 { response.Envs["CUDA_OVERSUBSCRIBE"] = "true" } - if *util.DisableCoreLimit { + if plugin.schedulerConfig.DisableCoreLimit { response.Envs[api.CoreLimitSwitch] = "disable" } cacheFileHostDirectory := fmt.Sprintf("%s/vgpu/containers/%s_%s", hostHookPath, current.UID, currentCtr.Name) @@ -540,7 +584,7 @@ func (plugin *NvidiaDevicePlugin) deviceIDsFromAnnotatedDeviceIDs(ids []string) } func (plugin *NvidiaDevicePlugin) apiDevices() []*kubeletdevicepluginv1beta1.Device { - return plugin.rm.Devices().GetPluginDevices() + return plugin.rm.Devices().GetPluginDevices(plugin.schedulerConfig.DeviceSplitCount) } func (plugin *NvidiaDevicePlugin) apiEnvs(envvar string, deviceIDs []string) map[string]string { diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server_test.go index 152690fc5f..d9d06ab1c8 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server_test.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server_test.go @@ -22,7 +22,7 @@ import ( v1 "github.com/NVIDIA/k8s-device-plugin/api/config/v1" "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/cdi" - "github.com/Project-HAMi/HAMi/pkg/util" + "github.com/Project-HAMi/HAMi/pkg/device/nvidia" "github.com/stretchr/testify/require" kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" ) @@ -143,7 +143,7 @@ func TestCDIAllocateResponse(t *testing.T) { t.Run(tc.description, func(t *testing.T) { deviceListStrategies, _ := v1.NewDeviceListStrategies(tc.deviceListStrategies) plugin := NvidiaDevicePlugin{ - config: &util.DeviceConfig{ + config: &nvidia.DeviceConfig{ Config: &v1.Config{ Flags: v1.Flags{ CommandLineFlags: v1.CommandLineFlags{ diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/device_map.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/device_map.go index f33d80a8f0..38a64713da 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/rm/device_map.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/device_map.go @@ -19,7 +19,7 @@ package rm import ( "fmt" - "github.com/Project-HAMi/HAMi/pkg/util" + "github.com/Project-HAMi/HAMi/pkg/device/nvidia" "github.com/NVIDIA/go-nvlib/pkg/nvlib/device" "github.com/NVIDIA/go-nvlib/pkg/nvml" @@ -28,14 +28,14 @@ import ( type deviceMapBuilder struct { device.Interface - config *util.DeviceConfig + config *nvidia.DeviceConfig } // DeviceMap stores a set of devices per resource name. type DeviceMap map[spec.ResourceName]Devices // NewDeviceMap creates a device map for the specified NVML library and config. -func NewDeviceMap(nvmllib nvml.Interface, config *util.DeviceConfig) (DeviceMap, error) { +func NewDeviceMap(nvmllib nvml.Interface, config *nvidia.DeviceConfig) (DeviceMap, error) { b := deviceMapBuilder{ Interface: device.New(device.WithNvml(nvmllib)), config: config, @@ -265,7 +265,7 @@ func (d DeviceMap) getIDsOfDevicesToReplicate(r *spec.ReplicatedResource) ([]str } // updateDeviceMapWithReplicas returns an updated map of resource names to devices with replica information from spec.Config.Sharing.TimeSlicing.Resources -func updateDeviceMapWithReplicas(config *util.DeviceConfig, oDevices DeviceMap) (DeviceMap, error) { +func updateDeviceMapWithReplicas(config *nvidia.DeviceConfig, oDevices DeviceMap) (DeviceMap, error) { devices := make(DeviceMap) // Begin by walking config.Sharing.TimeSlicing.Resources and building a map of just the resource names. diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices.go index 108b9b8b04..c9fe31f781 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices.go @@ -21,8 +21,6 @@ import ( "strconv" "strings" - "github.com/Project-HAMi/HAMi/pkg/util" - kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" ) @@ -142,12 +140,12 @@ func (ds Devices) GetIDs() []string { } // GetPluginDevices returns the plugin Devices from all devices in the Devices -func (ds Devices) GetPluginDevices() []*kubeletdevicepluginv1beta1.Device { +func (ds Devices) GetPluginDevices(count uint) []*kubeletdevicepluginv1beta1.Device { var res []*kubeletdevicepluginv1beta1.Device if !strings.Contains(ds.GetIDs()[0], "MIG") { for _, dev := range ds { - for i := uint(0); i < *util.DeviceSplitCount; i++ { + for i := uint(0); i < count; i++ { id := fmt.Sprintf("%v-%v", dev.ID, i) res = append(res, &kubeletdevicepluginv1beta1.Device{ ID: id, diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_manager.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_manager.go index 1e1fb659fa..c0abe579a4 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_manager.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_manager.go @@ -19,7 +19,7 @@ package rm import ( "fmt" - "github.com/Project-HAMi/HAMi/pkg/util" + "github.com/Project-HAMi/HAMi/pkg/device/nvidia" "github.com/NVIDIA/go-nvlib/pkg/nvml" "k8s.io/klog/v2" @@ -33,7 +33,7 @@ type nvmlResourceManager struct { var _ ResourceManager = (*nvmlResourceManager)(nil) // NewNVMLResourceManagers returns a set of ResourceManagers, one for each NVML resource in 'config'. -func NewNVMLResourceManagers(nvmllib nvml.Interface, config *util.DeviceConfig) ([]ResourceManager, error) { +func NewNVMLResourceManagers(nvmllib nvml.Interface, config *nvidia.DeviceConfig) ([]ResourceManager, error) { ret := nvmllib.Init() if ret != nvml.SUCCESS { return nil, fmt.Errorf("failed to initialize NVML: %v", ret) @@ -56,7 +56,7 @@ func NewNVMLResourceManagers(nvmllib nvml.Interface, config *util.DeviceConfig) continue } for key, value := range devices { - if util.FilterDeviceToRegister(value.ID, value.Index) { + if nvidia.FilterDeviceToRegister(value.ID, value.Index) { klog.V(5).InfoS("Filtering device", "device", value.ID) delete(devices, key) continue diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/rm.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/rm.go index afc907d68a..cc9d43c151 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/rm/rm.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/rm.go @@ -24,13 +24,13 @@ import ( "github.com/NVIDIA/go-nvlib/pkg/nvlib/info" "github.com/NVIDIA/go-nvlib/pkg/nvml" spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" - "github.com/Project-HAMi/HAMi/pkg/util" + "github.com/Project-HAMi/HAMi/pkg/device/nvidia" "k8s.io/klog/v2" ) // resourceManager forms the base type for specific resource manager implementations type resourceManager struct { - config *util.DeviceConfig + config *nvidia.DeviceConfig resource spec.ResourceName devices Devices } @@ -45,7 +45,7 @@ type ResourceManager interface { } // NewResourceManagers returns a []ResourceManager, one for each resource in 'config'. -func NewResourceManagers(nvmllib nvml.Interface, config *util.DeviceConfig) ([]ResourceManager, error) { +func NewResourceManagers(nvmllib nvml.Interface, config *nvidia.DeviceConfig) ([]ResourceManager, error) { // logWithReason logs the output of the has* / is* checks from the info.Interface logWithReason := func(f func() (bool, string), tag string) bool { is, reason := f() @@ -111,7 +111,7 @@ func (r *resourceManager) Devices() Devices { } // AddDefaultResourcesToConfig adds default resource matching rules to config.Resources -func AddDefaultResourcesToConfig(config *util.DeviceConfig) error { +func AddDefaultResourcesToConfig(config *nvidia.DeviceConfig) error { //config.Resources.AddGPUResource("*", "gpu") config.Resources.GPUs = append(config.Resources.GPUs, spec.Resource{ Pattern: "*", diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_devices.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_devices.go index d7df377f77..607cb7009e 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_devices.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_devices.go @@ -19,7 +19,7 @@ package rm import ( "fmt" - "github.com/Project-HAMi/HAMi/pkg/util" + "github.com/Project-HAMi/HAMi/pkg/device/nvidia" ) const ( @@ -28,7 +28,7 @@ const ( // buildTegraDeviceMap creates a DeviceMap for the tegra devices in the sytesm. // NOTE: At present only a single tegra device is expected. -func buildTegraDeviceMap(config *util.DeviceConfig) (DeviceMap, error) { +func buildTegraDeviceMap(config *nvidia.DeviceConfig) (DeviceMap, error) { devices := make(DeviceMap) name := tegraDeviceName diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_manager.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_manager.go index 053a5a22bc..ade00f40f8 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_manager.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_manager.go @@ -19,7 +19,7 @@ package rm import ( "fmt" - "github.com/Project-HAMi/HAMi/pkg/util" + "github.com/Project-HAMi/HAMi/pkg/device/nvidia" ) type tegraResourceManager struct { @@ -29,7 +29,7 @@ type tegraResourceManager struct { var _ ResourceManager = (*tegraResourceManager)(nil) // NewTegraResourceManagers returns a set of ResourceManagers for tegra resources -func NewTegraResourceManagers(config *util.DeviceConfig) ([]ResourceManager, error) { +func NewTegraResourceManagers(config *nvidia.DeviceConfig) ([]ResourceManager, error) { deviceMap, err := buildTegraDeviceMap(config) if err != nil { return nil, fmt.Errorf("error building Tegra device map: %v", err) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 8980e61787..a367e8a068 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -69,16 +69,12 @@ func (dev *Devices) trimMemory(m int64) (int64, string) { return 0, "" } -func InitDevices() []*Devices { +func InitDevices(config []VNPUConfig) []*Devices { var devs []*Devices if !enableAscend { return devs } - config, err := LoadConfig(configFile) - if err != nil { - klog.Fatalf("failed to load ascend vnpu config file %s: %v", configFile, err) - } - for _, vnpu := range config.VNPUs { + for _, vnpu := range config { commonWord := vnpu.CommonWord dev := &Devices{ config: vnpu, @@ -101,7 +97,6 @@ func InitDevices() []*Devices { func ParseConfig(fs *flag.FlagSet) { fs.BoolVar(&enableAscend, "enable-ascend", false, "enable ascend device") - fs.StringVar(&configFile, "ascend-config-file", "", "ascend vnpu config file") } func (dev *Devices) CommonWord() string { diff --git a/pkg/device/ascend/vnpu.go b/pkg/device/ascend/vnpu.go index 4641f0b601..57321f259f 100644 --- a/pkg/device/ascend/vnpu.go +++ b/pkg/device/ascend/vnpu.go @@ -16,44 +16,21 @@ limitations under the License. package ascend -import ( - "os" - - "k8s.io/apimachinery/pkg/util/yaml" -) - type Template struct { - Name string `json:"name"` - Memory int64 `json:"memory"` - AICore int32 `json:"aiCore,omitempty"` - AICPU int32 `json:"aiCPU,omitempty"` + Name string `yaml:"name"` + Memory int64 `yaml:"memory"` + AICore int32 `yaml:"aiCore,omitempty"` + AICPU int32 `yaml:"aiCPU,omitempty"` } type VNPUConfig struct { - CommonWord string `json:"commonWord"` - ChipName string `json:"chipName"` - ResourceName string `json:"resourceName"` - ResourceMemoryName string `json:"resourceMemoryName"` - MemoryAllocatable int64 `json:"memoryAllocatable"` - MemoryCapacity int64 `json:"memoryCapacity"` - AICore int32 `json:"aiCore"` - AICPU int32 `json:"aiCPU"` - Templates []Template `json:"templates"` -} - -type Config struct { - VNPUs []VNPUConfig `json:"vnpus"` -} - -func LoadConfig(path string) (*Config, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var yamlData Config - err = yaml.Unmarshal(data, &yamlData) - if err != nil { - return nil, err - } - return &yamlData, nil + CommonWord string `yaml:"commonWord"` + ChipName string `yaml:"chipName"` + ResourceName string `yaml:"resourceName"` + ResourceMemoryName string `yaml:"resourceMemoryName"` + MemoryAllocatable int64 `yaml:"memoryAllocatable"` + MemoryCapacity int64 `yaml:"memoryCapacity"` + AICore int32 `yaml:"aiCore"` + AICPU int32 `yaml:"aiCPU"` + Templates []Template `yaml:"templates"` } diff --git a/pkg/device/cambricon/device.go b/pkg/device/cambricon/device.go index 6b1530c4e6..da96640ad2 100644 --- a/pkg/device/cambricon/device.go +++ b/pkg/device/cambricon/device.go @@ -60,6 +60,12 @@ var ( MLUResourceCores string ) +type CambriconConfig struct { + ResourceCountName string `yaml:"resourceCountName"` + ResourceMemoryName string `yaml:"resourceMemoryName"` + ResourceCoreName string `yaml:"resourceCoreName"` +} + type CambriconDevices struct { } @@ -69,7 +75,10 @@ func ParseConfig(fs *flag.FlagSet) { fs.StringVar(&MLUResourceCores, "cambricon-mlu-cores", "cambricon.com/mlu.smlu.vcore", "cambricon mlu core resource") } -func InitMLUDevice() *CambriconDevices { +func InitMLUDevice(config CambriconConfig) *CambriconDevices { + MLUResourceCount = config.ResourceCountName + MLUResourceMemory = config.ResourceMemoryName + MLUResourceCores = config.ResourceCoreName util.InRequestDevices[CambriconMLUDevice] = "hami.io/cambricon-mlu-devices-to-allocate" util.SupportDevices[CambriconMLUDevice] = "hami.io/cambricon-mlu-devices-allocated" return &CambriconDevices{} diff --git a/pkg/device/devices.go b/pkg/device/devices.go index 25f4e21bf6..f6c6d33cb7 100644 --- a/pkg/device/devices.go +++ b/pkg/device/devices.go @@ -34,6 +34,7 @@ import ( "github.com/Project-HAMi/HAMi/pkg/util/client" "github.com/Project-HAMi/HAMi/pkg/util/nodelock" + "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" @@ -58,28 +59,38 @@ type Devices interface { //ParseConfig(fs *flag.FlagSet) } +type Config struct { + NvidiaConfig nvidia.NvidiaConfig `yaml:"nvidia"` + MetaxConfig metax.MetaxConfig `yaml:"metax"` + HygonConfig hygon.HygonConfig `yaml:"hygon"` + CambriconConfig cambricon.CambriconConfig `yaml:"cambricon"` + MthreadsConfig mthreads.MthreadsConfig `yaml:"mthreads"` + IluvatarConfig iluvatar.IluvatarConfig `yaml:"iluvatar"` + VNPUs []ascend.VNPUConfig `yaml:"vnpus"` +} + var ( HandshakeAnnos = map[string]string{} RegisterAnnos = map[string]string{} + devices map[string]Devices DevicesToHandle []string + configFile string + DebugMode bool ) -var devices map[string]Devices -var DebugMode bool - func GetDevices() map[string]Devices { return devices } -func InitDevices() { +func InitDevicesWithConfig(config *Config) { devices = make(map[string]Devices) DevicesToHandle = []string{} - devices[cambricon.CambriconMLUDevice] = cambricon.InitMLUDevice() - devices[nvidia.NvidiaGPUDevice] = nvidia.InitNvidiaDevice() - devices[hygon.HygonDCUDevice] = hygon.InitDCUDevice() - devices[iluvatar.IluvatarGPUDevice] = iluvatar.InitIluvatarDevice() - devices[mthreads.MthreadsGPUDevice] = mthreads.InitMthreadsDevice() - devices[metax.MetaxGPUDevice] = metax.InitMetaxDevice() + devices[nvidia.NvidiaGPUDevice] = nvidia.InitNvidiaDevice(config.NvidiaConfig) + devices[cambricon.CambriconMLUDevice] = cambricon.InitMLUDevice(config.CambriconConfig) + devices[hygon.HygonDCUDevice] = hygon.InitDCUDevice(config.HygonConfig) + devices[iluvatar.IluvatarGPUDevice] = iluvatar.InitIluvatarDevice(config.IluvatarConfig) + devices[mthreads.MthreadsGPUDevice] = mthreads.InitMthreadsDevice(config.MthreadsConfig) + devices[metax.MetaxGPUDevice] = metax.InitMetaxDevice(config.MetaxConfig) DevicesToHandle = append(DevicesToHandle, nvidia.NvidiaGPUCommonWord) DevicesToHandle = append(DevicesToHandle, cambricon.CambriconMLUCommonWord) @@ -87,12 +98,24 @@ func InitDevices() { DevicesToHandle = append(DevicesToHandle, iluvatar.IluvatarGPUCommonWord) DevicesToHandle = append(DevicesToHandle, mthreads.MthreadsGPUCommonWord) DevicesToHandle = append(DevicesToHandle, metax.MetaxGPUCommonWord) - for _, dev := range ascend.InitDevices() { + for _, dev := range ascend.InitDevices(config.VNPUs) { devices[dev.CommonWord()] = dev DevicesToHandle = append(DevicesToHandle, dev.CommonWord()) } } +func InitDevices() { + if len(devices) > 0 { + return + } + config, err := LoadConfig(configFile) + klog.Infoln("reading config=", config, "configfile=", configFile) + if err != nil { + klog.Fatalf("failed to load device config file %s: %v", configFile, err) + } + InitDevicesWithConfig(config) +} + func PodAllocationTrySuccess(nodeName string, devName string, lockName string, pod *corev1.Pod) { refreshed, err := client.GetClient().CoreV1().Pods(pod.Namespace).Get(context.Background(), pod.Name, metav1.GetOptions{}) if err != nil { @@ -146,6 +169,20 @@ func GlobalFlagSet() *flag.FlagSet { mthreads.ParseConfig(fs) metax.ParseConfig(fs) fs.BoolVar(&DebugMode, "debug", false, "debug mode") + fs.StringVar(&configFile, "device-config-file", "", "device config file") klog.InitFlags(fs) return fs } + +func LoadConfig(path string) (*Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var yamlData Config + err = yaml.Unmarshal(data, &yamlData) + if err != nil { + return nil, err + } + return &yamlData, nil +} diff --git a/pkg/device/devices_test.go b/pkg/device/devices_test.go new file mode 100644 index 0000000000..695186655a --- /dev/null +++ b/pkg/device/devices_test.go @@ -0,0 +1,249 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package device + +import ( + "testing" + + "gopkg.in/yaml.v2" + "gotest.tools/v3/assert" + + "github.com/Project-HAMi/HAMi/pkg/device/ascend" + "github.com/Project-HAMi/HAMi/pkg/device/cambricon" + "github.com/Project-HAMi/HAMi/pkg/device/hygon" + "github.com/Project-HAMi/HAMi/pkg/device/iluvatar" + "github.com/Project-HAMi/HAMi/pkg/device/metax" + "github.com/Project-HAMi/HAMi/pkg/device/mthreads" +) + +func Test_LoadConfig(t *testing.T) { + configMapdata := ` +nvidia: + resourceCountName: nvidia.com/gpu + resourceMemoryName: nvidia.com/gpumem + resourceMemoryPercentageName: nvidia.com/gpumem-percentage + resourceCoreName: nvidia.com/gpucores + resourcePriorityName: nvidia.com/priority + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 +cambricon: + resourceCountName: cambricon.com/vmlu + resourceMemoryName: cambricon.com/mlu.smlu.vmemory + resourceCoreName: cambricon.com/mlu.smlu.vcore +hygon: + resourceCountName: hygon.com/dcunum + resourceMemoryName: hygon.com/dcumem + resourceCoreName: hygon.com/dcucores +metax: + resourceCountName: "metax-tech.com/gpu" +mthreads: + resourceCountName: "mthreads.com/vgpu" + resourceMemoryName: "mthreads.com/sgpu-memory" + resourceCoreName: "mthreads.com/sgpu-core" +iluvatar: + resourceCountName: iluvatar.ai/vgpu + resourceMemoryName: iluvatar.ai/vcuda-memory + resourceCoreName: iluvatar.ai/vcuda-core +vnpus: +- chipName: 910B + commonWord: Ascend910A + resourceName: huawei.com/Ascend910A + resourceMemoryName: huawei.com/Ascend910A-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + aiCore: 30 + templates: + - name: vir02 + memory: 2184 + aiCore: 2 + - name: vir04 + memory: 4369 + aiCore: 4 + - name: vir08 + memory: 8738 + aiCore: 8 + - name: vir16 + memory: 17476 + aiCore: 16 +- chipName: 910B3 + commonWord: Ascend910B + resourceName: huawei.com/Ascend910B + resourceMemoryName: huawei.com/Ascend910B-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 +- chipName: 310P3 + commonWord: Ascend310P + resourceName: huawei.com/Ascend310P + resourceMemoryName: huawei.com/Ascend310P-memory + memoryAllocatable: 21527 + memoryCapacity: 24576 + aiCore: 8 + aiCPU: 7 + templates: + - name: vir01 + memory: 3072 + aiCore: 1 + aiCPU: 1 + - name: vir02 + memory: 6144 + aiCore: 2 + aiCPU: 2 + - name: vir04 + memory: 12288 + aiCore: 4 + aiCPU: 4` + var yamlData Config + err := yaml.Unmarshal([]byte(configMapdata), &yamlData) + assert.NilError(t, err) + assert.Equal(t, yamlData.NvidiaConfig.ResourceCountName, "nvidia.com/gpu") + assert.Equal(t, yamlData.NvidiaConfig.ResourceMemoryName, "nvidia.com/gpumem") + assert.Equal(t, yamlData.NvidiaConfig.ResourceMemoryPercentageName, "nvidia.com/gpumem-percentage") + assert.Equal(t, yamlData.NvidiaConfig.ResourceCoreName, "nvidia.com/gpucores") + assert.Equal(t, yamlData.NvidiaConfig.ResourcePriority, "nvidia.com/priority") + assert.Equal(t, yamlData.NvidiaConfig.OverwriteEnv, false) + assert.Equal(t, yamlData.NvidiaConfig.DefaultMemory, int32(0)) + assert.Equal(t, yamlData.NvidiaConfig.DefaultCores, int32(0)) + assert.Equal(t, yamlData.NvidiaConfig.DefaultGPUNum, int32(1)) + cambriconConfig := cambricon.CambriconConfig{ + ResourceCountName: "cambricon.com/vmlu", + ResourceMemoryName: "cambricon.com/mlu.smlu.vmemory", + ResourceCoreName: "cambricon.com/mlu.smlu.vcore", + } + assert.DeepEqual(t, yamlData.CambriconConfig, cambriconConfig) + hygonConfig := hygon.HygonConfig{ + ResourceCountName: "hygon.com/dcunum", + ResourceMemoryName: "hygon.com/dcumem", + ResourceCoreName: "hygon.com/dcucores", + } + assert.DeepEqual(t, yamlData.HygonConfig, hygonConfig) + iluvatarConfig := iluvatar.IluvatarConfig{ + ResourceCountName: "iluvatar.ai/vgpu", + ResourceMemoryName: "iluvatar.ai/vcuda-memory", + ResourceCoreName: "iluvatar.ai/vcuda-core", + } + assert.DeepEqual(t, yamlData.IluvatarConfig, iluvatarConfig) + methreadsConfig := mthreads.MthreadsConfig{ + ResourceCountName: "mthreads.com/vgpu", + ResourceMemoryName: "mthreads.com/sgpu-memory", + ResourceCoreName: "mthreads.com/sgpu-core", + } + assert.DeepEqual(t, yamlData.MthreadsConfig, methreadsConfig) + metaxConfig := metax.MetaxConfig{ + ResourceCountName: "metax-tech.com/gpu", + } + assert.DeepEqual(t, yamlData.MetaxConfig, metaxConfig) + ascendConfig := []ascend.VNPUConfig{} + ascendConfig = append(ascendConfig, ascend.VNPUConfig{ + ChipName: "910B", + CommonWord: "Ascend910A", + ResourceName: "huawei.com/Ascend910A", + ResourceMemoryName: "huawei.com/Ascend910A-memory", + MemoryAllocatable: 32768, + MemoryCapacity: 32768, + AICore: 30, + Templates: []ascend.Template{ + { + Name: "vir02", + Memory: 2184, + AICore: 2, + }, + { + Name: "vir04", + Memory: 4369, + AICore: 4, + }, + { + Name: "vir08", + Memory: 8738, + AICore: 8, + }, + { + Name: "vir16", + Memory: 17476, + AICore: 16, + }, + }, + }) + ascendConfig = append(ascendConfig, ascend.VNPUConfig{ + ChipName: "910B3", + CommonWord: "Ascend910B", + ResourceName: "huawei.com/Ascend910B", + ResourceMemoryName: "huawei.com/Ascend910B-memory", + MemoryAllocatable: 65536, + MemoryCapacity: 65536, + AICore: 20, + AICPU: 7, + Templates: []ascend.Template{ + { + Name: "vir05_1c_16g", + Memory: 16384, + AICore: 5, + AICPU: 1, + }, + { + Name: "vir10_3c_32g", + Memory: 32768, + AICore: 10, + AICPU: 3, + }, + }, + }) + ascendConfig = append(ascendConfig, ascend.VNPUConfig{ + ChipName: "310P3", + CommonWord: "Ascend310P", + ResourceName: "huawei.com/Ascend310P", + ResourceMemoryName: "huawei.com/Ascend310P-memory", + MemoryAllocatable: 21527, + MemoryCapacity: 24576, + AICore: 8, + AICPU: 7, + Templates: []ascend.Template{ + { + Name: "vir01", + Memory: 3072, + AICore: 1, + AICPU: 1, + }, + { + Name: "vir02", + Memory: 6144, + AICore: 2, + AICPU: 2, + }, + { + Name: "vir04", + Memory: 12288, + AICore: 4, + AICPU: 4, + }, + }, + }) + assert.DeepEqual(t, yamlData.VNPUs, ascendConfig) +} diff --git a/pkg/device/hygon/device.go b/pkg/device/hygon/device.go index e82afd13d3..0e42883968 100644 --- a/pkg/device/hygon/device.go +++ b/pkg/device/hygon/device.go @@ -50,7 +50,16 @@ var ( HygonResourceCores string ) -func InitDCUDevice() *DCUDevices { +type HygonConfig struct { + ResourceCountName string `yaml:"resourceCountName"` + ResourceMemoryName string `yaml:"resourceMemoryName"` + ResourceCoreName string `yaml:"resourceCoreName"` +} + +func InitDCUDevice(config HygonConfig) *DCUDevices { + HygonResourceCount = config.ResourceCountName + HygonResourceMemory = config.ResourceMemoryName + HygonResourceCores = config.ResourceCoreName util.InRequestDevices[HygonDCUDevice] = "hami.io/dcu-devices-to-allocate" util.SupportDevices[HygonDCUDevice] = "hami.io/dcu-devices-allocated" util.HandshakeAnnos[HygonDCUDevice] = HandshakeAnnos diff --git a/pkg/device/iluvatar/device.go b/pkg/device/iluvatar/device.go index 6b344db0ec..bc0af06d6d 100644 --- a/pkg/device/iluvatar/device.go +++ b/pkg/device/iluvatar/device.go @@ -50,7 +50,16 @@ var ( IluvatarResourceCores string ) -func InitIluvatarDevice() *IluvatarDevices { +type IluvatarConfig struct { + ResourceCountName string `yaml:"resourceCountName"` + ResourceMemoryName string `yaml:"resourceMemoryName"` + ResourceCoreName string `yaml:"resourceCoreName"` +} + +func InitIluvatarDevice(config IluvatarConfig) *IluvatarDevices { + IluvatarResourceCount = config.ResourceCountName + IluvatarResourceMemory = config.ResourceMemoryName + IluvatarResourceCores = config.ResourceCoreName util.InRequestDevices[IluvatarGPUDevice] = "hami.io/iluvatar-vgpu-devices-to-allocate" util.SupportDevices[IluvatarGPUDevice] = "hami.io/iluvatar-vgpu-devices-allocated" return &IluvatarDevices{} diff --git a/pkg/device/iluvatar/device_test.go b/pkg/device/iluvatar/device_test.go index 630c504983..6eac81949d 100644 --- a/pkg/device/iluvatar/device_test.go +++ b/pkg/device/iluvatar/device_test.go @@ -101,7 +101,7 @@ func TestGetNodeDevices(t *testing.T) { } func TestPatchAnnotations(t *testing.T) { - InitIluvatarDevice() + InitIluvatarDevice(IluvatarConfig{}) tests := []struct { name string diff --git a/pkg/device/metax/device.go b/pkg/device/metax/device.go index 1f90ca3541..6df0d81f93 100644 --- a/pkg/device/metax/device.go +++ b/pkg/device/metax/device.go @@ -44,7 +44,12 @@ var ( MetaxResourceCount string ) -func InitMetaxDevice() *MetaxDevices { +type MetaxConfig struct { + ResourceCountName string `yaml:"resourceCountName"` +} + +func InitMetaxDevice(config MetaxConfig) *MetaxDevices { + MetaxResourceCount = config.ResourceCountName util.InRequestDevices[MetaxGPUDevice] = "hami.io/metax-gpu-devices-to-allocate" util.SupportDevices[MetaxGPUDevice] = "hami.io/metax-gpu-devices-allocated" return &MetaxDevices{} diff --git a/pkg/device/mthreads/device.go b/pkg/device/mthreads/device.go index 20f677892e..5951041a3e 100644 --- a/pkg/device/mthreads/device.go +++ b/pkg/device/mthreads/device.go @@ -57,7 +57,16 @@ var ( legalMemoryslices = []int64{2, 4, 8, 16, 32, 64, 96} ) -func InitMthreadsDevice() *MthreadsDevices { +type MthreadsConfig struct { + ResourceCountName string `yaml:"resourceCountName"` + ResourceMemoryName string `yaml:"resourceMemoryName"` + ResourceCoreName string `yaml:"resourceCoreName"` +} + +func InitMthreadsDevice(config MthreadsConfig) *MthreadsDevices { + MthreadsResourceCount = config.ResourceCountName + MthreadsResourceCores = config.ResourceCoreName + MthreadsResourceMemory = config.ResourceMemoryName util.InRequestDevices[MthreadsGPUDevice] = "hami.io/mthreads-vgpu-devices-to-allocate" util.SupportDevices[MthreadsGPUDevice] = "hami.io/mthreads-vgpu-devices-allocated" return &MthreadsDevices{} diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index 140cfb300b..147b85b39e 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -24,10 +24,10 @@ import ( "strings" "github.com/Project-HAMi/HAMi/pkg/api" - "github.com/Project-HAMi/HAMi/pkg/scheduler/config" "github.com/Project-HAMi/HAMi/pkg/util" "github.com/Project-HAMi/HAMi/pkg/util/nodelock" + spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/klog/v2" @@ -49,23 +49,67 @@ const ( ) var ( - ResourceName string - ResourceMem string - ResourceCores string - ResourceMemPercentage string - ResourcePriority string - DebugMode bool - OverwriteEnv bool + NodeName string + RuntimeSocketFlag string + DisableCoreLimit *bool + + // DevicePluginFilterDevice need device-plugin filter this device, don't register this device. + DevicePluginFilterDevice *FilterDevice ) +type NvidiaConfig struct { + ResourceCountName string `yaml:"resourceCountName"` + ResourceMemoryName string `yaml:"resourceMemoryName"` + ResourceCoreName string `yaml:"resourceCoreName"` + ResourceMemoryPercentageName string `yaml:"resourceMemoryPercentageName"` + ResourcePriority string `yaml:"resourcePriorityName"` + OverwriteEnv bool `yaml:"overwriteEnv"` + DefaultMemory int32 `yaml:"defaultMemory"` + DefaultCores int32 `yaml:"defaultCores"` + DefaultGPUNum int32 `yaml:"defaultGPUNum"` + DeviceSplitCount uint `yaml:"deviceSplitCount"` + DeviceMemoryScaling float64 `yaml:"deviceMemoryScaling"` + DeviceCoreScaling float64 `yaml:"deviceCoreScaling"` + DisableCoreLimit bool `yaml:"disableCoreLimit"` +} + +type FilterDevice struct { + // UUID is the device ID. + UUID []string `json:"uuid"` + // Index is the device index. + Index []uint `json:"index"` +} + +type DevicePluginConfigs struct { + Nodeconfig []struct { + Name string `json:"name"` + Devicememoryscaling float64 `json:"devicememoryscaling"` + Devicecorescaling float64 `json:"devicecorescaling"` + Devicesplitcount uint `json:"devicesplitcount"` + Migstrategy string `json:"migstrategy"` + FilterDevice *FilterDevice `json:"filterdevices"` + } `json:"nodeconfig"` +} + +type DeviceConfig struct { + *spec.Config + + ResourceName *string + DebugMode *bool +} + type NvidiaGPUDevices struct { + config NvidiaConfig } -func InitNvidiaDevice() *NvidiaGPUDevices { +func InitNvidiaDevice(nvconfig NvidiaConfig) *NvidiaGPUDevices { + klog.InfoS("initializing nvidia device", "resourceName", nvconfig.ResourceCountName, "resourceMem", nvconfig.ResourceMemoryName, "DefaultGPUNum", nvconfig.DefaultGPUNum) util.InRequestDevices[NvidiaGPUDevice] = "hami.io/vgpu-devices-to-allocate" util.SupportDevices[NvidiaGPUDevice] = "hami.io/vgpu-devices-allocated" util.HandshakeAnnos[NvidiaGPUDevice] = HandshakeAnnos - return &NvidiaGPUDevices{} + return &NvidiaGPUDevices{ + config: nvconfig, + } } func (dev *NvidiaGPUDevices) CommonWord() string { @@ -73,12 +117,35 @@ func (dev *NvidiaGPUDevices) CommonWord() string { } func ParseConfig(fs *flag.FlagSet) { - fs.StringVar(&ResourceName, "resource-name", "nvidia.com/gpu", "resource name") - fs.StringVar(&ResourceMem, "resource-mem", "nvidia.com/gpumem", "gpu memory to allocate") - fs.StringVar(&ResourceMemPercentage, "resource-mem-percentage", "nvidia.com/gpumem-percentage", "gpu memory fraction to allocate") - fs.StringVar(&ResourceCores, "resource-cores", "nvidia.com/gpucores", "cores percentage to use") - fs.StringVar(&ResourcePriority, "resource-priority", "vgputaskpriority", "vgpu task priority 0 for high and 1 for low") - fs.BoolVar(&OverwriteEnv, "overwrite-env", false, "If set NVIDIA_VISIBLE_DEVICES=none to pods with no-gpu allocation") +} + +func FilterDeviceToRegister(uuid, indexStr string) bool { + if DevicePluginFilterDevice == nil || (len(DevicePluginFilterDevice.UUID) == 0 && len(DevicePluginFilterDevice.Index) == 0) { + return false + } + uuidMap, indexMap := make(map[string]struct{}), make(map[uint]struct{}) + for _, u := range DevicePluginFilterDevice.UUID { + uuidMap[u] = struct{}{} + } + for _, index := range DevicePluginFilterDevice.Index { + indexMap[index] = struct{}{} + } + if uuid != "" { + if _, ok := uuidMap[uuid]; ok { + return true + } + } + if indexStr != "" { + index, err := strconv.Atoi(indexStr) + if err != nil { + klog.Errorf("Error converting index to int: %v", err) + return false + } + if _, ok := indexMap[uint(index)]; ok { + return true + } + } + return false } func (dev *NvidiaGPUDevices) NodeCleanUp(nn string) error { @@ -138,7 +205,7 @@ func (dev *NvidiaGPUDevices) GetNodeDevices(n corev1.Node) ([]*api.DeviceInfo, e func (dev *NvidiaGPUDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) { /*gpu related */ - priority, ok := ctr.Resources.Limits[corev1.ResourceName(ResourcePriority)] + priority, ok := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourcePriority)] if ok { ctr.Env = append(ctr.Env, corev1.EnvVar{ Name: api.TaskPriority, @@ -146,23 +213,23 @@ func (dev *NvidiaGPUDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Po }) } - _, resourceNameOK := ctr.Resources.Limits[corev1.ResourceName(ResourceName)] + _, resourceNameOK := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceCountName)] if resourceNameOK { return resourceNameOK, nil } - _, resourceCoresOK := ctr.Resources.Limits[corev1.ResourceName(ResourceCores)] - _, resourceMemOK := ctr.Resources.Limits[corev1.ResourceName(ResourceMem)] - _, resourceMemPercentageOK := ctr.Resources.Limits[corev1.ResourceName(ResourceMemPercentage)] + _, resourceCoresOK := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceCoreName)] + _, resourceMemOK := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceMemoryName)] + _, resourceMemPercentageOK := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceMemoryPercentageName)] if resourceCoresOK || resourceMemOK || resourceMemPercentageOK { - if config.DefaultResourceNum > 0 { - ctr.Resources.Limits[corev1.ResourceName(ResourceName)] = *resource.NewQuantity(int64(config.DefaultResourceNum), resource.BinarySI) + if dev.config.DefaultGPUNum > 0 { + ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceCountName)] = *resource.NewQuantity(int64(dev.config.DefaultGPUNum), resource.BinarySI) resourceNameOK = true } } - if !resourceNameOK && OverwriteEnv { + if !resourceNameOK && dev.config.OverwriteEnv { ctr.Env = append(ctr.Env, corev1.EnvVar{ Name: "NVIDIA_VISIBLE_DEVICES", Value: "none", @@ -262,10 +329,10 @@ func (dev *NvidiaGPUDevices) PatchAnnotations(annoinput *map[string]string, pd u } func (dev *NvidiaGPUDevices) GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest { - resourceName := corev1.ResourceName(ResourceName) - resourceMem := corev1.ResourceName(ResourceMem) - resourceMemPercentage := corev1.ResourceName(ResourceMemPercentage) - resourceCores := corev1.ResourceName(ResourceCores) + resourceName := corev1.ResourceName(dev.config.ResourceCountName) + resourceMem := corev1.ResourceName(dev.config.ResourceMemoryName) + resourceMemPercentage := corev1.ResourceName(dev.config.ResourceMemoryPercentageName) + resourceCores := corev1.ResourceName(dev.config.ResourceCoreName) v, ok := ctr.Resources.Limits[resourceName] if !ok { v, ok = ctr.Resources.Requests[resourceName] @@ -295,13 +362,13 @@ func (dev *NvidiaGPUDevices) GenerateResourceRequests(ctr *corev1.Container) uti } } if mempnum == 101 && memnum == 0 { - if config.DefaultMem != 0 { - memnum = int(config.DefaultMem) + if dev.config.DefaultMemory != 0 { + memnum = int(dev.config.DefaultMemory) } else { mempnum = 100 } } - corenum := config.DefaultCores + corenum := dev.config.DefaultCores core, ok := ctr.Resources.Limits[resourceCores] if !ok { core, ok = ctr.Resources.Requests[resourceCores] diff --git a/pkg/device/nvidia/device_test.go b/pkg/device/nvidia/device_test.go index c6451c0434..5716e6ffb3 100644 --- a/pkg/device/nvidia/device_test.go +++ b/pkg/device/nvidia/device_test.go @@ -19,7 +19,6 @@ package nvidia import ( "testing" - "github.com/Project-HAMi/HAMi/pkg/scheduler/config" "github.com/Project-HAMi/HAMi/pkg/util" "gotest.tools/v3/assert" @@ -35,11 +34,6 @@ func Test_DefaultResourceNum(t *testing.T) { } func Test_MutateAdmission(t *testing.T) { - ResourceName = "nvidia.com/gpu" - ResourceMem = "nvidia.com/gpumem" - ResourceMemPercentage = "nvidia.com/gpumem-percentage" - ResourceCores = "nvidia.com/gpucores" - config.DefaultResourceNum = 1 tests := []struct { name string args *corev1.Container @@ -105,7 +99,15 @@ func Test_MutateAdmission(t *testing.T) { }, } - gpuDevices := &NvidiaGPUDevices{} + gpuDevices := &NvidiaGPUDevices{ + config: NvidiaConfig{ + ResourceCountName: "nvidia.com/gpu", + ResourceMemoryName: "nvidia.com/gpumem", + ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage", + ResourceCoreName: "nvidia.com/gpucores", + DefaultGPUNum: int32(1), + }, + } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got, _ := gpuDevices.MutateAdmission(test.args, &corev1.Pod{}) @@ -117,7 +119,15 @@ func Test_MutateAdmission(t *testing.T) { } func Test_CheckUUID(t *testing.T) { - gpuDevices := &NvidiaGPUDevices{} + gpuDevices := &NvidiaGPUDevices{ + config: NvidiaConfig{ + ResourceCountName: "nvidia.com/gpu", + ResourceMemoryName: "nvidia.com/gpumem", + ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage", + ResourceCoreName: "nvidia.com/gpucores", + DefaultGPUNum: int32(1), + }, + } tests := []struct { name string args struct { @@ -208,7 +218,15 @@ func Test_CheckUUID(t *testing.T) { } func Test_CheckType(t *testing.T) { - gpuDevices := &NvidiaGPUDevices{} + gpuDevices := &NvidiaGPUDevices{ + config: NvidiaConfig{ + ResourceCountName: "nvidia.com/gpu", + ResourceMemoryName: "nvidia.com/gpumem", + ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage", + ResourceCoreName: "nvidia.com/gpucores", + DefaultGPUNum: int32(1), + }, + } tests := []struct { name string args struct { @@ -275,3 +293,142 @@ func Test_CheckType(t *testing.T) { }) } } + +func Test_FilterDeviceToRegister(t *testing.T) { + tests := []struct { + name string + args struct { + uuid string + idx string + *FilterDevice + } + want bool + }{ + { + name: "filter is nil", + args: struct { + uuid string + idx string + *FilterDevice + }{ + uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", + idx: "0", + FilterDevice: nil, + }, + want: false, + }, + { + name: "uuid is empty", + args: struct { + uuid string + idx string + *FilterDevice + }{ + uuid: "", + idx: "0", + FilterDevice: &FilterDevice{ + UUID: []string{"GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76"}, + }, + }, + want: false, + }, + { + name: "uuid is not in filter", + args: struct { + uuid string + idx string + *FilterDevice + }{ + uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", + idx: "0", + FilterDevice: &FilterDevice{ + UUID: []string{"GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b77"}, + }, + }, + want: false, + }, + { + name: "uuid is in filter", + args: struct { + uuid string + idx string + *FilterDevice + }{ + uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", + idx: "0", + FilterDevice: &FilterDevice{ + UUID: []string{"GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76"}, + }, + }, + want: true, + }, + { + name: "idx is empty", + args: struct { + uuid string + idx string + *FilterDevice + }{ + uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", + idx: "", + FilterDevice: &FilterDevice{ + Index: []uint{0}, + }, + }, + want: false, + }, + { + name: "idx is not in filter", + args: struct { + uuid string + idx string + *FilterDevice + }{ + uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", + idx: "0", + FilterDevice: &FilterDevice{ + Index: []uint{1}, + }, + }, + want: false, + }, + { + name: "idx is in filter", + args: struct { + uuid string + idx string + *FilterDevice + }{ + uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", + idx: "0", + FilterDevice: &FilterDevice{ + Index: []uint{0}, + }, + }, + want: true, + }, + { + name: "idx is invalid", + args: struct { + uuid string + idx string + *FilterDevice + }{ + uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", + idx: "a", + FilterDevice: &FilterDevice{ + Index: []uint{0}, + }, + }, + want: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + DevicePluginFilterDevice = test.args.FilterDevice + got := FilterDeviceToRegister(test.args.uuid, test.args.idx) + assert.DeepEqual(t, test.want, got) + }) + } +} diff --git a/pkg/k8sutil/pod_test.go b/pkg/k8sutil/pod_test.go index 2a0c97ac3c..b342a6d4d4 100644 --- a/pkg/k8sutil/pod_test.go +++ b/pkg/k8sutil/pod_test.go @@ -29,11 +29,14 @@ import ( ) func Test_Resourcereqs(t *testing.T) { - nvidia.ResourceName = "hami.io/gpu" - nvidia.ResourceMem = "hami.io/gpumem" - nvidia.ResourceMemPercentage = "hami.io/gpumem-percentage" - nvidia.ResourceCores = "hami.io/gpucores" - device.InitDevices() + device.InitDevicesWithConfig(&device.Config{ + NvidiaConfig: nvidia.NvidiaConfig{ + ResourceCountName: "hami.io/gpu", + ResourceMemoryName: "hami.io/gpumem", + ResourceMemoryPercentageName: "hami.io/gpumem-percentage", + ResourceCoreName: "hami.io/gpucores", + }, + }) tests := []struct { name string diff --git a/pkg/scheduler/config/config.go b/pkg/scheduler/config/config.go index a2461c25ba..e32774edc5 100644 --- a/pkg/scheduler/config/config.go +++ b/pkg/scheduler/config/config.go @@ -21,10 +21,11 @@ import "github.com/Project-HAMi/HAMi/pkg/util" var ( HTTPBind string SchedulerName string + MetricsBindAddress string + DefaultMem int32 DefaultCores int32 DefaultResourceNum int32 - MetricsBindAddress string // NodeSchedulerPolicy is config this scheduler node to use `binpack` or `spread`. default value is binpack. NodeSchedulerPolicy = util.NodeSchedulerPolicyBinpack.String() diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 7efb8b2cdc..d5cdf58534 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -135,7 +135,14 @@ func Test_Filter(t *testing.T) { informerFactory.Start(s.stopCh) informerFactory.WaitForCacheSync(s.stopCh) s.addAllEventHandlers() - device.InitDevices() + device.InitDevicesWithConfig(&device.Config{ + NvidiaConfig: nvidia.NvidiaConfig{ + ResourceCountName: "hami.io/gpu", + ResourceMemoryName: "hami.io/gpumem", + ResourceMemoryPercentageName: "hami.io/gpumem-percentage", + ResourceCoreName: "hami.io/gpucores", + }, + }) pod1 := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -469,9 +476,6 @@ func Test_Filter(t *testing.T) { }, } - nvidia.ResourceName = "hami.io/gpu" - nvidia.ResourceMem = "hami.io/gpumem" - nvidia.ResourceCores = "hami.io/gpucores" for _, test := range tests { t.Run(test.name, func(t *testing.T) { initNode() diff --git a/pkg/scheduler/score_test.go b/pkg/scheduler/score_test.go index 8d6bcedf54..b96559b4a3 100644 --- a/pkg/scheduler/score_test.go +++ b/pkg/scheduler/score_test.go @@ -1352,7 +1352,14 @@ func Test_calcScore(t *testing.T) { s := NewScheduler() for _, test := range tests { t.Run(test.name, func(t *testing.T) { - device.InitDevices() + device.InitDevicesWithConfig(&device.Config{ + NvidiaConfig: nvidia.NvidiaConfig{ + ResourceCountName: "hami.io/gpu", + ResourceMemoryName: "hami.io/gpumem", + ResourceMemoryPercentageName: "hami.io/gpumem-percentage", + ResourceCoreName: "hami.io/gpucores", + }, + }) got, gotErr := s.calcScore(test.args.nodes, test.args.nums, test.args.annos, test.args.task) assert.DeepEqual(t, test.wants.err, gotErr) wantMap := make(map[string]*policy.NodeScore) diff --git a/pkg/scheduler/webhook_test.go b/pkg/scheduler/webhook_test.go index 47f78c74ec..b53ecf17ba 100644 --- a/pkg/scheduler/webhook_test.go +++ b/pkg/scheduler/webhook_test.go @@ -94,11 +94,14 @@ func TestHandle(t *testing.T) { func TestPodHasNodeName(t *testing.T) { config.SchedulerName = "hami-scheduler" - nvidia.ResourceName = "hami.io/gpu" - nvidia.ResourceMem = "hami.io/gpumem" - nvidia.ResourceMemPercentage = "hami.io/gpumem-percentage" - nvidia.ResourceCores = "hami.io/gpucores" - device.InitDevices() + device.InitDevicesWithConfig(&device.Config{ + NvidiaConfig: nvidia.NvidiaConfig{ + ResourceCountName: "hami.io/gpu", + ResourceMemoryName: "hami.io/gpumem", + ResourceMemoryPercentageName: "hami.io/gpumem-percentage", + ResourceCoreName: "hami.io/gpucores", + }, + }) // create a Pod object pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/util/types.go b/pkg/util/types.go index c13b9f08ff..89e14ef9a5 100644 --- a/pkg/util/types.go +++ b/pkg/util/types.go @@ -17,7 +17,6 @@ limitations under the License. package util import ( - spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" corev1 "k8s.io/api/core/v1" ) @@ -33,10 +32,6 @@ const ( DeviceBindFailed = "failed" DeviceBindSuccess = "success" - //Set default mem to 5000m - //DefaultMem = 5000 - //DefaultCores = 0. - DeviceLimit = 100 //TimeLayout = "ANSIC" //DefaultTimeout = time.Second * 60. @@ -49,52 +44,13 @@ const ( NodeNameEnvName = "NODE_NAME" ) -type FilterDevice struct { - // UUID is the device ID. - UUID []string `json:"uuid"` - // Index is the device index. - Index []uint `json:"index"` -} - -type DevicePluginConfigs struct { - Nodeconfig []struct { - Name string `json:"name"` - Devicememoryscaling float64 `json:"devicememoryscaling"` - Devicecorescaling float64 `json:"devicecorescaling"` - Devicesplitcount uint `json:"devicesplitcount"` - Migstrategy string `json:"migstrategy"` - FilterDevice *FilterDevice `json:"filterdevices"` - } `json:"nodeconfig"` -} - -type DeviceConfig struct { - *spec.Config - - ResourceName *string - DebugMode *bool -} - var ( DebugMode bool - DeviceSplitCount *uint - DeviceMemoryScaling *float64 - DeviceCoresScaling *float64 - NodeName string - RuntimeSocketFlag string - DisableCoreLimit *bool - - // DevicePluginFilterDevice need device-plugin filter this device, don't register this device. - DevicePluginFilterDevice *FilterDevice + NodeName string + RuntimeSocketFlag string ) -// type ContainerDevices struct { -// Devices []string `json:"devices,omitempty"` -// } -// -// type PodDevices struct { -// Containers []ContainerDevices `json:"containers,omitempty"` -// } type ContainerDevice struct { // TODO current Idx cannot use, because EncodeContainerDevices method not encode this filed. Idx int diff --git a/pkg/util/util.go b/pkg/util/util.go index 025b825a15..1ed565c55d 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -415,32 +415,3 @@ func MarkAnnotationsToDelete(devType string, nn string) error { } return PatchNodeAnnotations(n, tmppat) } - -func FilterDeviceToRegister(uuid, indexStr string) bool { - if DevicePluginFilterDevice == nil || (len(DevicePluginFilterDevice.UUID) == 0 && len(DevicePluginFilterDevice.Index) == 0) { - return false - } - uuidMap, indexMap := make(map[string]struct{}), make(map[uint]struct{}) - for _, u := range DevicePluginFilterDevice.UUID { - uuidMap[u] = struct{}{} - } - for _, index := range DevicePluginFilterDevice.Index { - indexMap[index] = struct{}{} - } - if uuid != "" { - if _, ok := uuidMap[uuid]; ok { - return true - } - } - if indexStr != "" { - index, err := strconv.Atoi(indexStr) - if err != nil { - klog.Errorf("Error converting index to int: %v", err) - return false - } - if _, ok := indexMap[uint(index)]; ok { - return true - } - } - return false -} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 08aec4f0c4..0c4b680905 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -282,142 +282,3 @@ func TestUnMarshalNodeDevices(t *testing.T) { }) } } - -func Test_FilterDeviceToRegister(t *testing.T) { - tests := []struct { - name string - args struct { - uuid string - idx string - *FilterDevice - } - want bool - }{ - { - name: "filter is nil", - args: struct { - uuid string - idx string - *FilterDevice - }{ - uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", - idx: "0", - FilterDevice: nil, - }, - want: false, - }, - { - name: "uuid is empty", - args: struct { - uuid string - idx string - *FilterDevice - }{ - uuid: "", - idx: "0", - FilterDevice: &FilterDevice{ - UUID: []string{"GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76"}, - }, - }, - want: false, - }, - { - name: "uuid is not in filter", - args: struct { - uuid string - idx string - *FilterDevice - }{ - uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", - idx: "0", - FilterDevice: &FilterDevice{ - UUID: []string{"GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b77"}, - }, - }, - want: false, - }, - { - name: "uuid is in filter", - args: struct { - uuid string - idx string - *FilterDevice - }{ - uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", - idx: "0", - FilterDevice: &FilterDevice{ - UUID: []string{"GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76"}, - }, - }, - want: true, - }, - { - name: "idx is empty", - args: struct { - uuid string - idx string - *FilterDevice - }{ - uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", - idx: "", - FilterDevice: &FilterDevice{ - Index: []uint{0}, - }, - }, - want: false, - }, - { - name: "idx is not in filter", - args: struct { - uuid string - idx string - *FilterDevice - }{ - uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", - idx: "0", - FilterDevice: &FilterDevice{ - Index: []uint{1}, - }, - }, - want: false, - }, - { - name: "idx is in filter", - args: struct { - uuid string - idx string - *FilterDevice - }{ - uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", - idx: "0", - FilterDevice: &FilterDevice{ - Index: []uint{0}, - }, - }, - want: true, - }, - { - name: "idx is invalid", - args: struct { - uuid string - idx string - *FilterDevice - }{ - uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", - idx: "a", - FilterDevice: &FilterDevice{ - Index: []uint{0}, - }, - }, - want: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - DevicePluginFilterDevice = test.args.FilterDevice - got := FilterDeviceToRegister(test.args.uuid, test.args.idx) - assert.DeepEqual(t, test.want, got) - }) - } -} diff --git a/version.mk b/version.mk index 3f7ce0d016..e303f22512 100644 --- a/version.mk +++ b/version.mk @@ -5,7 +5,7 @@ DEVICES=nvidia OUTPUT_DIR=bin TARGET_ARCH=amd64 GOLANG_IMAGE=golang:1.22.5-bullseye -NVIDIA_IMAGE=nvidia/cuda:12.3.2-devel-ubuntu22.04 +NVIDIA_IMAGE=nvidia/cuda:12.3.2-devel-ubuntu20.04 DEST_DIR=/usr/local/vgpu/ VERSION = v0.0.1 From 577f93fee29d054b5b58922071052a395f7dc64d Mon Sep 17 00:00:00 2001 From: limengxuan <391013634@qq.com> Date: Fri, 15 Nov 2024 14:23:25 +0800 Subject: [PATCH 02/10] prepare to update v2.4.1 Signed-off-by: limengxuan <391013634@qq.com> --- VERSION | 2 +- charts/hami/Chart.yaml | 4 ++-- charts/hami/values.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 8721bbc46a..a372120910 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v2.4.0 +v2.4.1 diff --git a/charts/hami/Chart.yaml b/charts/hami/Chart.yaml index 624b722da3..39388849cd 100644 --- a/charts/hami/Chart.yaml +++ b/charts/hami/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: hami -version: 2.4.0 +version: 2.4.1 kubeVersion: ">= 1.16.0" description: Heterogeneous AI Computing Virtualization Middleware keywords: @@ -12,5 +12,5 @@ maintainers: email: limengxuan@4paradigm.com - name: zhangxiao email: xiaozhang0210@hotmail.com -appVersion: "2.4.0" +appVersion: "2.4.1" diff --git a/charts/hami/values.yaml b/charts/hami/values.yaml index 86bff27bdb..d8636d32ec 100644 --- a/charts/hami/values.yaml +++ b/charts/hami/values.yaml @@ -3,7 +3,7 @@ nameOverride: "" fullnameOverride: "" imagePullSecrets: [ ] -version: "latest" +version: "v2.4.1" #Nvidia GPU Parameters resourceName: "nvidia.com/gpu" From a453160fddcd22fc5b88c7697d1c8dfc35228328 Mon Sep 17 00:00:00 2001 From: limengxuan <391013634@qq.com> Date: Mon, 18 Nov 2024 17:58:14 +0800 Subject: [PATCH 03/10] update vGPUmonitor to add dynamic adjustment on core and memory limit Signed-off-by: limengxuan <391013634@qq.com> --- pkg/monitor/nvidia/cudevshr.go | 2 ++ pkg/monitor/nvidia/v0/spec.go | 16 ++++++++++++++++ pkg/monitor/nvidia/v1/spec.go | 16 ++++++++++++++++ pkg/scheduler/pods.go | 2 ++ 4 files changed, 36 insertions(+) diff --git a/pkg/monitor/nvidia/cudevshr.go b/pkg/monitor/nvidia/cudevshr.go index 0198c66267..09536ed355 100644 --- a/pkg/monitor/nvidia/cudevshr.go +++ b/pkg/monitor/nvidia/cudevshr.go @@ -55,9 +55,11 @@ type UsageInfo interface { DeviceMemoryOffset(idx int) uint64 DeviceMemoryTotal(idx int) uint64 DeviceSmUtil(idx int) uint64 + SetDeviceSmLimit(l uint64) IsValidUUID(idx int) bool DeviceUUID(idx int) string DeviceMemoryLimit(idx int) uint64 + SetDeviceMemoryLimit(l uint64) LastKernelTime() int64 //UsedMemory(idx int) (uint64, error) GetPriority() int diff --git a/pkg/monitor/nvidia/v0/spec.go b/pkg/monitor/nvidia/v0/spec.go index f29839cf27..9163e76279 100644 --- a/pkg/monitor/nvidia/v0/spec.go +++ b/pkg/monitor/nvidia/v0/spec.go @@ -129,6 +129,14 @@ func (s Spec) DeviceSmUtil(idx int) uint64 { return v } +func (s Spec) SetDeviceSmLimit(l uint64) { + idx := uint64(0) + for idx < s.sr.num { + s.sr.smLimit[idx] = l + idx += 1 + } +} + func (s Spec) IsValidUUID(idx int) bool { return s.sr.uuids[idx].uuid[0] != 0 } @@ -141,6 +149,14 @@ func (s Spec) DeviceMemoryLimit(idx int) uint64 { return s.sr.limit[idx] } +func (s Spec) SetDeviceMemoryLimit(l uint64) { + idx := uint64(0) + for idx < s.sr.num { + s.sr.limit[idx] = l + idx += 1 + } +} + func (s Spec) LastKernelTime() int64 { return 0 } diff --git a/pkg/monitor/nvidia/v1/spec.go b/pkg/monitor/nvidia/v1/spec.go index 079507d5e1..5bfcd9363d 100644 --- a/pkg/monitor/nvidia/v1/spec.go +++ b/pkg/monitor/nvidia/v1/spec.go @@ -136,6 +136,14 @@ func (s Spec) DeviceSmUtil(idx int) uint64 { return v } +func (s Spec) SetDeviceSmLimit(l uint64) { + idx := uint64(0) + for idx < s.sr.num { + s.sr.smLimit[idx] = l + idx += 1 + } +} + func (s Spec) IsValidUUID(idx int) bool { return s.sr.uuids[idx].uuid[0] != 0 } @@ -148,6 +156,14 @@ func (s Spec) DeviceMemoryLimit(idx int) uint64 { return s.sr.limit[idx] } +func (s Spec) SetDeviceMemoryLimit(l uint64) { + idx := uint64(0) + for idx < s.sr.num { + s.sr.limit[idx] = l + idx += 1 + } +} + func (s Spec) LastKernelTime() int64 { return s.sr.lastKernelTime } diff --git a/pkg/scheduler/pods.go b/pkg/scheduler/pods.go index 1bd026f0d4..d4fb2d399e 100644 --- a/pkg/scheduler/pods.go +++ b/pkg/scheduler/pods.go @@ -61,6 +61,8 @@ func (m *podManager) addPod(pod *corev1.Pod, nodeID string, devices util.PodDevi pi := &podInfo{Name: pod.Name, UID: pod.UID, Namespace: pod.Namespace, NodeID: nodeID, Devices: devices} m.pods[pod.UID] = pi klog.Infof("Pod added: Name: %s, UID: %s, Namespace: %s, NodeID: %s", pod.Name, pod.UID, pod.Namespace, nodeID) + } else { + m.pods[pod.UID].Devices = devices } } From 33113b263efbfe049cd87ab00c2ef26cb9dc7eaf Mon Sep 17 00:00:00 2001 From: limengxuan <391013634@qq.com> Date: Fri, 29 Nov 2024 17:32:38 +0800 Subject: [PATCH 04/10] update Signed-off-by: limengxuan <391013634@qq.com> --- .github/workflows/test-self-hosted.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/test-self-hosted.yaml diff --git a/.github/workflows/test-self-hosted.yaml b/.github/workflows/test-self-hosted.yaml new file mode 100644 index 0000000000..cec1c35e73 --- /dev/null +++ b/.github/workflows/test-self-hosted.yaml @@ -0,0 +1,21 @@ +name: Test self-hosted-runner + +on: + push: + # Exclude branches created by Dependabot to avoid triggering current workflow + # for PRs initiated by Dependabot. + branches-ignore: + - 'dependabot/**' + pull_request: + paths: + - "charts/**" + +jobs: + e2e: + runs-on: self-hosted + steps: + - name: e2e test + # https://github.com/actions/virtual-environments/issues/709 + run: | + echo "Need to add e2e test" + From 2ce717dc6e5aecde60d243aa6e72b4519581855d Mon Sep 17 00:00:00 2001 From: limengxuan <391013634@qq.com> Date: Mon, 16 Dec 2024 17:55:59 +0800 Subject: [PATCH 05/10] fix chart can't be deployed properly Signed-off-by: limengxuan <391013634@qq.com> --- .vscode/settings.json | 3 +++ charts/hami/templates/device-plugin/daemonsetnvidia.yaml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..249c88a192 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "cmake.sourceDirectory": "/root/k8s-vgpu-scheduler/libvgpu" +} \ No newline at end of file diff --git a/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/charts/hami/templates/device-plugin/daemonsetnvidia.yaml index 43ad0828cc..93eeae475e 100644 --- a/charts/hami/templates/device-plugin/daemonsetnvidia.yaml +++ b/charts/hami/templates/device-plugin/daemonsetnvidia.yaml @@ -65,7 +65,7 @@ spec: value: {{ .Values.global.gpuHookPath }} securityContext: privileged: true - allowPrivilegeEscalation: false + allowPrivilegeEscalation: true capabilities: drop: ["ALL"] add: ["SYS_ADMIN"] From f5b2591dffc937d160976c4c09eb0ae4c80ebd17 Mon Sep 17 00:00:00 2001 From: limengxuan <391013634@qq.com> Date: Mon, 16 Dec 2024 17:56:19 +0800 Subject: [PATCH 06/10] fix chart can't be deployed properly Signed-off-by: limengxuan <391013634@qq.com> --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 249c88a192..0000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "cmake.sourceDirectory": "/root/k8s-vgpu-scheduler/libvgpu" -} \ No newline at end of file From 0ff2c42678d36cbeabac30021d77f025856ed59b Mon Sep 17 00:00:00 2001 From: limengxuan <391013634@qq.com> Date: Tue, 17 Dec 2024 11:55:13 +0800 Subject: [PATCH 07/10] fix nodelock issues Signed-off-by: limengxuan <391013634@qq.com> --- .../nvidiadevice/nvinternal/plugin/server.go | 3 +-- pkg/device/ascend/device.go | 2 +- pkg/device/devices.go | 4 ++-- pkg/device/nvidia/device.go | 2 +- pkg/scheduler/scheduler.go | 3 +-- pkg/scheduler/score.go | 4 ++-- pkg/util/nodelock/nodelock.go | 23 ++++++++++++++----- 7 files changed, 25 insertions(+), 16 deletions(-) diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go index 96cffc4a83..88b28e96e2 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go @@ -52,7 +52,6 @@ import ( "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/rm" "github.com/Project-HAMi/HAMi/pkg/device/nvidia" "github.com/Project-HAMi/HAMi/pkg/util" - "github.com/Project-HAMi/HAMi/pkg/util/nodelock" cdiapi "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" "github.com/google/uuid" @@ -388,7 +387,7 @@ func (plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *kubeletdev nodename := os.Getenv(util.NodeNameEnvName) current, err := util.GetPendingPod(ctx, nodename) if err != nil { - nodelock.ReleaseNodeLock(nodename, NodeLockNvidia) + //nodelock.ReleaseNodeLock(nodename, NodeLockNvidia, current) return &kubeletdevicepluginv1beta1.AllocateResponse{}, err } klog.V(5).Infof("allocate pod name is %s/%s, annotation is %+v", current.Namespace, current.Name, current.Annotations) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 4f872519b5..8ceb42bf4d 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -196,7 +196,7 @@ func (dev *Devices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error { return nil } - return nodelock.ReleaseNodeLock(n.Name, NodeLockAscend) + return nodelock.ReleaseNodeLock(n.Name, NodeLockAscend, p, false) } func (dev *Devices) NodeCleanUp(nn string) error { diff --git a/pkg/device/devices.go b/pkg/device/devices.go index 1b11df8b24..241179a41f 100644 --- a/pkg/device/devices.go +++ b/pkg/device/devices.go @@ -236,7 +236,7 @@ func PodAllocationSuccess(nodeName string, pod *corev1.Pod, lockname string) { if err != nil { klog.Errorf("patchPodAnnotations failed:%v", err.Error()) } - err = nodelock.ReleaseNodeLock(nodeName, lockname) + err = nodelock.ReleaseNodeLock(nodeName, lockname, pod, false) if err != nil { klog.Errorf("release lock failed:%v", err.Error()) } @@ -249,7 +249,7 @@ func PodAllocationFailed(nodeName string, pod *corev1.Pod, lockname string) { if err != nil { klog.Errorf("patchPodAnnotations failed:%v", err.Error()) } - err = nodelock.ReleaseNodeLock(nodeName, lockname) + err = nodelock.ReleaseNodeLock(nodeName, lockname, pod, false) if err != nil { klog.Errorf("release lock failed:%v", err.Error()) } diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index cca4cfc08e..2c9727ee99 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -203,7 +203,7 @@ func (dev *NvidiaGPUDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) erro if !found { return nil } - return nodelock.ReleaseNodeLock(n.Name, NodeLockNvidia) + return nodelock.ReleaseNodeLock(n.Name, NodeLockNvidia, p, false) } func (dev *NvidiaGPUDevices) GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error) { diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 7c1601c2d1..ecb8b48d39 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -319,8 +319,7 @@ func (s *Scheduler) getNodesUsage(nodes *[]string, task *corev1.Pod) (*map[strin } } } - //klog.V(5).Infof("usage: pod %v assigned %v %v", p.Name, p.NodeID, p.Devices) - klog.Infof("usage: pod %v assigned %v %v", p.Name, p.NodeID, p.Devices) + klog.V(5).Infof("usage: pod %v assigned %v %v", p.Name, p.NodeID, p.Devices) } s.overviewstatus = overallnodeMap for _, nodeID := range *nodes { diff --git a/pkg/scheduler/score.go b/pkg/scheduler/score.go index 78c288328c..e351c72fa6 100644 --- a/pkg/scheduler/score.go +++ b/pkg/scheduler/score.go @@ -29,9 +29,9 @@ import ( ) func viewStatus(usage NodeUsage) { - klog.Info("devices status") + klog.V(5).Info("devices status") for _, val := range usage.Devices.DeviceLists { - klog.InfoS("device status", "device id", val.Device.ID, "device detail", val) + klog.V(5).InfoS("device status", "device id", val.Device.ID, "device detail", val) } } diff --git a/pkg/util/nodelock/nodelock.go b/pkg/util/nodelock/nodelock.go index 7e01804ea2..45644271db 100644 --- a/pkg/util/nodelock/nodelock.go +++ b/pkg/util/nodelock/nodelock.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/Project-HAMi/HAMi/pkg/util/client" @@ -35,7 +36,11 @@ const ( NodeLockSep = "," ) +var lock sync.Mutex + func SetNodeLock(nodeName string, lockname string, pods *corev1.Pod) error { + lock.Lock() + defer lock.Unlock() ctx := context.Background() node, err := client.GetClient().CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) if err != nil { @@ -62,19 +67,25 @@ func SetNodeLock(nodeName string, lockname string, pods *corev1.Pod) error { if err != nil { return fmt.Errorf("setNodeLock exceeds retry count %d", MaxLockRetry) } - klog.InfoS("Node lock set", "node", nodeName) + klog.InfoS("Node lock set", "node", nodeName, "podName", pods.Name) return nil } -func ReleaseNodeLock(nodeName string, lockname string) error { +func ReleaseNodeLock(nodeName string, lockname string, pod *corev1.Pod, timeout bool) error { + lock.Lock() + defer lock.Unlock() ctx := context.Background() node, err := client.GetClient().CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) if err != nil { return err } - if _, ok := node.ObjectMeta.Annotations[NodeLockKey]; !ok { - klog.InfoS("Node lock not set", "node", nodeName) + if lockStr, ok := node.ObjectMeta.Annotations[NodeLockKey]; !ok { return nil + } else { + if !strings.Contains(lockStr, pod.Name) && !timeout { + klog.InfoS("NodeLock is not set by this pod", lockStr, "pod", pod.Name) + return nil + } } newNode := node.DeepCopy() delete(newNode.ObjectMeta.Annotations, NodeLockKey) @@ -94,7 +105,7 @@ func ReleaseNodeLock(nodeName string, lockname string) error { if err != nil { return fmt.Errorf("releaseNodeLock exceeds retry count %d", MaxLockRetry) } - klog.InfoS("Node lock released", "node", nodeName) + klog.InfoS("Node lock released", "node", nodeName, "podName", pod.Name) return nil } @@ -113,7 +124,7 @@ func LockNode(nodeName string, lockname string, pods *corev1.Pod) error { } if time.Since(lockTime) > time.Minute*5 { klog.InfoS("Node lock expired", "node", nodeName, "lockTime", lockTime) - err = ReleaseNodeLock(nodeName, lockname) + err = ReleaseNodeLock(nodeName, lockname, pods, true) if err != nil { klog.ErrorS(err, "Failed to release node lock", "node", nodeName) return err From 566ceca80c272fecd8eb3815ed5204ed50d11424 Mon Sep 17 00:00:00 2001 From: limengxuan <391013634@qq.com> Date: Thu, 19 Dec 2024 10:27:54 +0800 Subject: [PATCH 08/10] update dynamic-mig document and examples Signed-off-by: limengxuan <391013634@qq.com> --- docs/config.md | 35 ++++- docs/config_cn.md | 31 ++++- docs/dynamic-mig-support.md | 163 +++++++++++++++++++++++ docs/dynamic-mig-support_cn.md | 163 +++++++++++++++++++++++ examples/nvidia/dynamic_mig_example.yaml | 18 +++ pkg/device/nvidia/device.go | 2 +- 6 files changed, 400 insertions(+), 12 deletions(-) create mode 100644 docs/dynamic-mig-support.md create mode 100644 docs/dynamic-mig-support_cn.md create mode 100644 examples/nvidia/dynamic_mig_example.yaml diff --git a/docs/config.md b/docs/config.md index 80db448c00..b29c8054c5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -45,7 +45,34 @@ helm install hami hami-charts/hami --set devicePlugin.deviceMemoryScaling=5 ... * `resourcePriority:` String type, vgpu task priority name, default: "nvidia.com/priority" -# Container config envs +# Pod configs: annotations + +* `nvidia.com/use-gpuuuid:` + String type, ie: "GPU-AAA,GPU-BBB" + If set, devices allocated by this pod must be one of UUIDs defined in this string. +* `nvidia.com/nouse-gpuuuid` + String type, ie: "GPU-AAA,GPU-BBB" + If set, devices allocated by this pod will NOT in UUIDs defined in this string. +* `nvidia.com/nouse-gputype:` + String type, ie: "Tesla V100-PCIE-32GB, NVIDIA A10" + If set, devices allocated by this pod will NOT in types defined in this string. +* `nvidia.com/use-gputype` + String type, ie: "Tesla V100-PCIE-32GB, NVIDIA A10" + If set, devices allocated by this pod MUST be one of types defined in this string. +* `hami.io/node-scheduler-policy` + String type, "binpack" or "spread" + binpack: the scheduler will try to allocate the pod to used GPU nodes for execution. + spread: the scheduler will try to allocate the pod to different GPU nodes for execution. +* `hami.io/gpu-scheduler-policy` + String type, "binpack" or "spread" + binpack: the scheduler will try to allocate the pod to the same GPU card for execution. + spread:the scheduler will try to allocate the pod to different GPU card for execution. +* `nvidia.com/vgpu-mode` + String type, "hami-core" or "mig" + Which type of vgpu instance this pod wish to use + + +# Container configs: env * `GPU_CORE_UTILIZATION_POLICY:` String type, "default", "force", "disable" @@ -53,12 +80,6 @@ helm install hami hami-charts/hami --set devicePlugin.deviceMemoryScaling=5 ... "default" means the dafault utilization policy "force" means the container will always limit the core utilization below "nvidia.com/gpucores" "disable" means the container will ignore the utilization limitation set by "nvidia.com/gpucores" during task execution - -* `ACTIVE_OOM_KILLER:` - Bool type, "true","false" - default: false - "true" means there will be a daemon process which monitors all running tasks inside this container, and instantly kill any process which exceeds the limitation set by "nvidia.com/gpumem" or "nvidia.com/gpumemory" - * `CUDA_DISABLE_CONTROL` Bool type, "true","false" default: false diff --git a/docs/config_cn.md b/docs/config_cn.md index a60d4cc789..9d548cb207 100644 --- a/docs/config_cn.md +++ b/docs/config_cn.md @@ -33,16 +33,39 @@ helm install vgpu vgpu-charts/vgpu --set devicePlugin.deviceMemoryScaling=5 ... * `resourcePriority:` 字符串类型，表示申请任务的任务优先级，默认: "nvidia.com/priority" +# Pod配置（在注解中指定） + +* `nvidia.com/use-gpuuuid:` + 字符串类型, 如: "GPU-AAA,GPU-BBB" + 如果设置, 该任务申请的设备只能是字符串中定义的设备之一。 +* `nvidia.com/nouse-gpuuuid` + 字符串类型, 如: "GPU-AAA,GPU-BBB" + 如果设置, 该任务不能使用字符串中定义的任何设备 +* `nvidia.com/nouse-gputype:` + 字符串类型, 如: "Tesla V100-PCIE-32GB, NVIDIA A10" + 如果设置, 该任务不能使用字符串中定义的任何设备型号 +* `nvidia.com/use-gputype` + 字符串类型, 如: "Tesla V100-PCIE-32GB, NVIDIA A10" + 如果设置, 该任务申请的设备只能使用字符串中定义的设备型号。 +* `hami.io/gpu-scheduler-policy` + 字符串类型, "binpack" 或 "spread" + spread:, 调度器会尽量将任务均匀地分配在不同GPU中 + binpack: 调度器会尽量将任务分配在已分配的GPU中，从而减少碎片 +* `hami.io/node-scheduler-policy` + 字符串类型, "binpack" 或 "spread" + spread: 调度器会尽量将任务均匀地分配到不同节点上 + binpack: 调度器会尽量将任务分配在已分配任务的节点上，从而减少碎片 +* `nvidia.com/vgpu-mode` + 字符串类型, "hami-core" 或 "mig" + 该任务希望使用的vgpu类型 + + # 容器配置（在容器的环境变量中指定） * `GPU_CORE_UTILIZATION_POLICY:` 字符串类型，"default", "force", "disable" 默认为"default" 代表容器算力限制策略， "default"为默认，"force"为强制限制算力，一般用于测试算力限制的功能，"disable"为忽略算力限制 -* `ACTIVE_OOM_KILLER:` - 布尔类型，"true", "false" - 默认为false - 若设置为true，则代表监控系统将会持续监控进程的显存使用量，并主动kill掉任何用超配额的进行。 * `CUDA_DISABLE_CONTROL` 布尔类型，"true", "false" 默认为false diff --git a/docs/dynamic-mig-support.md b/docs/dynamic-mig-support.md new file mode 100644 index 0000000000..c531aa69fc --- /dev/null +++ b/docs/dynamic-mig-support.md @@ -0,0 +1,163 @@ +## Introduction + +**We now support dynamic-mig by using mig-parted to adjust mig-devices dynamically**, including: + +***Dynamic MIG instance management***: User don't need to operate on GPU node, using 'nvidia-smi -i 0 -mig 1' or other command to manage MIG instance, all will be done by HAMi-device-plugin. + +***Dynamic MIG Adjustment***: Each MIG device managed by HAMi will dyamically adjust their MIG template according to tasks submitted when necessary. + +***Device MIG Observation***: Each MIG instance generated by HAMi will be shown in scheduler-monitor, including task information. user can get a clear overview of MIG nodes. + +***Compatable with HAMi-core nodes***: HAMi can manage a unified GPU pool of `HAMi-core node` and `mig node`. A task can be scheduled to either node if not appointed manually by using `nvidia.com/vgpu-mode` annotation. + +***Unified API with HAMi-core***: Zero work needs to be done to make the job compatible with dynamic-mig feature. + +## Prerequisites + +* NVIDIA Blackwell and Hopper™ and Ampere Devices +* HAMi > v2.5.0 +* Nvidia-container-toolkit + +## Enabling Dynamic-mig Support + +* Install the chart using helm, See 'enabling vGPU support in kubernetes' section [here](https://github.com/Project-HAMi/HAMi#enabling-vgpu-support-in-kubernetes) + +* Configure `mode` in device-plugin configMap to `mig` for MIG nodes +``` +kubectl describe cm hami-device-plugin -n kube-system +``` + +```json +{ + "nodeconfig": [ + { + "name": "MIG-NODE-A", + "operatingmode": "mig", + "filterdevices": { + "uuid": [], + "index": [] + } + } + ] +} +``` + +* Restart the following pods for the change to take effect: + * hami-scheduler + * hami-device-plugin on 'MIG-NODE-A' + +## Custom mig configuration (Optional) +HAMi currently has a [built-in mig configuration](https://github.com/Project-HAMi/HAMi/blob/master/charts/hami/templates/scheduler/device-configmap.yaml) for MIG. + +You can customize the mig configuration by following the steps below: + + ### Change the content of 'device-configmap.yaml' in charts/hami/templates/scheduler, the as follows + + ```yaml + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} + deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} + deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }} + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + memory: 6144 + count: 4 + - + - name: 2g.12gb + memory: 12288 + count: 2 + - + - name: 4g.24gb + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB", "A100-SXM4-40GB" ] + allowedGeometries: + - + - name: 1g.5gb + memory: 5120 + count: 7 + - + - name: 2g.10gb + memory: 10240 + count: 3 + - name: 1g.5gb + memory: 5120 + count: 1 + - + - name: 3g.20gb + memory: 20480 + count: 2 + - + - name: 7g.40gb + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + memory: 10240 + count: 7 + - + - name: 2g.20gb + memory: 20480 + count: 3 + - name: 1g.10gb + memory: 10240 + count: 1 + - + - name: 3g.40gb + memory: 40960 + count: 2 + - + - name: 7g.79gb + memory: 80896 + count: 1 + ``` + ### Helm installation and updates will be based on the configuration in this file, overwriting the built-in configuration of Helm + + ### Be aware HAMi will find and use the first MIG template suitable to the task in the order of this configMap + +## Running MIG jobs + +MIG instance can now be requested by a container the same way as using `hami-core` +simply by specifying the `nvidia.com/gpu` and `nvidia.com/gpumem` resource type. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod + annotations: + nvidia.com/vgpu-mode: "mig" #(Optional), if not set, this pod can be assigned to a MIG instance or a hami-core instance +spec: + containers: + - name: ubuntu-container + image: ubuntu:18.04 + command: ["bash", "-c", "sleep 86400"] + resources: + limits: + nvidia.com/gpu: 2 + nvidia.com/gpumem: 8000 +``` + +In this example above, the task allocates two mig instances, each with at least 8G device memory. + +## Notes + +1. You don't need to do anything on MIG node, all are managed by mig-parted in hami-device-plugin. + +2. Nvidia devices before Ampere architect can't use 'mig' mode + +3. You won't see any mig resources(ie, `nvidia.com/mig-1g.10gb`) on node, hami uses a unified resource name for both 'mig' and 'hami-core' node \ No newline at end of file diff --git a/docs/dynamic-mig-support_cn.md b/docs/dynamic-mig-support_cn.md new file mode 100644 index 0000000000..2a78989450 --- /dev/null +++ b/docs/dynamic-mig-support_cn.md @@ -0,0 +1,163 @@ +## 动态MIG功能简介 + +**HAMi将在2.5版本后支持动态MIG切分模式**, 其功能包括: + +***动态MIG实例管理***: 用户不需要在节点上事先生成MIG实例，HAMi会根据任务需要自动创建 + +***动态切换MIG切分方案***: HAMi会根据设备上的任务情况和新任务的需求，动态的切换MIG模版 + +***MIG实例监控***: 每个由HAMi管理的MIG实例都可以在调度器监控中找到，用户可以通过该监控清晰地获取整个集群的MIG视图 + +***可与使用hami-core的节点进行统一的资源池化***: HAMi将MIG与hami-core这两种切分方案进行了统一的池化处理，若任务未指定切分模式的话，分配给hami-core或者mig都是有可能的 + +***统一的API***: 使用动态MIG功能完全不需要进行任务层的适配工作 + +## 需求 + +* NVIDIA Blackwell and Hopper™ and Ampere Devices +* HAMi > v2.5.0 +* Nvidia-container-toolkit + +## 开启动态MIG功能 + +* 通过[这里](https://github.com/Project-HAMi/HAMi#enabling-vgpu-support-in-kubernetes)的文档部署HAMi + +* 通过以下指令修改configMap，并将节点的工作模式修改为`mig` +``` +kubectl describe cm hami-device-plugin -n kube-system +``` + +```json +{ + "nodeconfig": [ + { + "name": "MIG-NODE-A", + "operatingmode": "mig", + "filterdevices": { + "uuid": [], + "index": [] + } + } + ] +} +``` + +* 重启以下2个pod使修改后的配置生效: + * hami-scheduler + * 在'MIG-NODE-A'上的hami-device-plugin + +## 修改MIG模版列表 (可选) + +HAMi目前包含[MIG配置模版](https://github.com/Project-HAMi/HAMi/blob/master/charts/hami/templates/scheduler/device-configmap.yaml) + +你可以根据自己的集群环境，通过以下的方式去进行修改: + + ### 修改`charts/hami/templates/scheduler`路径下的`device-configmap.yaml` + + ```yaml + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} + deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} + deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }} + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + memory: 6144 + count: 4 + - + - name: 2g.12gb + memory: 12288 + count: 2 + - + - name: 4g.24gb + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB", "A100-SXM4-40GB" ] + allowedGeometries: + - + - name: 1g.5gb + memory: 5120 + count: 7 + - + - name: 2g.10gb + memory: 10240 + count: 3 + - name: 1g.5gb + memory: 5120 + count: 1 + - + - name: 3g.20gb + memory: 20480 + count: 2 + - + - name: 7g.40gb + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + memory: 10240 + count: 7 + - + - name: 2g.20gb + memory: 20480 + count: 3 + - name: 1g.10gb + memory: 10240 + count: 1 + - + - name: 3g.40gb + memory: 40960 + count: 2 + - + - name: 7g.79gb + memory: 80896 + count: 1 + ``` + ### 修改后可以通过更新或重新部署chart来生效 + + ### 在收到任务请求后，HAMi会在上述定义的MIG模版中的依次查找，直到找到一个可以运行任务的模版 + +## 使用MIG模式运行任务 + +MIG实例子可以通过和使用hami-core相同的方式进行申请，只需要指定`nvidia.com/gpu`和`nvidia.com/gpumem`即可 + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod + annotations: + nvidia.com/vgpu-mode: "mig" #(Optional), if not set, this pod can be assigned to a MIG instance or a hami-core instance +spec: + containers: + - name: ubuntu-container + image: ubuntu:18.04 + command: ["bash", "-c", "sleep 86400"] + resources: + limits: + nvidia.com/gpu: 2 + nvidia.com/gpumem: 8000 +``` + +在上面的例子中，该任务申请了2个MIG实例，每个实例至少需要8G显存 + +## 备注 + +1. 你不需要在MIG节点上进行任何操作，所有MIG实例的创建和维护都是由hami-vgpu-device-plugin进行的 + +2. 安培架构之前的NVIDIA设备无法使用`MIG`模式 + +3. 你不会在节点上看到MIG资源名(例如, `nvidia.com/mig-1g.10gb`)，HAMi对于hami-core和mig使用统一的资源名进行管理 \ No newline at end of file diff --git a/examples/nvidia/dynamic_mig_example.yaml b/examples/nvidia/dynamic_mig_example.yaml new file mode 100644 index 0000000000..360c9e886a --- /dev/null +++ b/examples/nvidia/dynamic_mig_example.yaml @@ -0,0 +1,18 @@ +## This example will allocate 2g.10gb * 2 for A100-40GB-PCIE device +## or 1g.10gb * 2 for A100-80GB-XSM device. +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod + annotations: + nvidia.com/vgpu-mode: "mig" +spec: + containers: + - name: ubuntu-container + image: ubuntu:18.04 + command: ["bash", "-c", "sleep 86400"] + resources: + limits: + nvidia.com/gpu: 2 + nvidia.com/gpumem: 8000 + diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index 2c9727ee99..2bb7686916 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -45,7 +45,7 @@ const ( GPUUseUUID = "nvidia.com/use-gpuuuid" // GPUNoUseUUID is user can not use specify GPU device for set GPU UUID. GPUNoUseUUID = "nvidia.com/nouse-gpuuuid" - AllocateMode = "nvidia.com/mode" + AllocateMode = "nvidia.com/vgpu-mode" MigMode = "mig" HamiCoreMode = "hami-core" From 98ecaa9d4c0d484eafca39a7e50baab4ce5e930f Mon Sep 17 00:00:00 2001 From: limengxuan <391013634@qq.com> Date: Thu, 19 Dec 2024 10:35:33 +0800 Subject: [PATCH 09/10] update dynamic-mig document and examples Signed-off-by: limengxuan <391013634@qq.com> --- docs/dynamic-mig-support.md | 4 ++-- docs/dynamic-mig-support_cn.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/dynamic-mig-support.md b/docs/dynamic-mig-support.md index c531aa69fc..97cf3a60e1 100644 --- a/docs/dynamic-mig-support.md +++ b/docs/dynamic-mig-support.md @@ -125,9 +125,9 @@ You can customize the mig configuration by following the steps below: memory: 80896 count: 1 ``` - ### Helm installation and updates will be based on the configuration in this file, overwriting the built-in configuration of Helm - ### Be aware HAMi will find and use the first MIG template suitable to the task in the order of this configMap + > **Note** Helm installation and updates will be based on the configuration in this file, overwriting the built-in configuration of Helm + > **Note** Be aware HAMi will find and use the first MIG template suitable to the task in the order of this configMap ## Running MIG jobs diff --git a/docs/dynamic-mig-support_cn.md b/docs/dynamic-mig-support_cn.md index 2a78989450..646b389ab5 100644 --- a/docs/dynamic-mig-support_cn.md +++ b/docs/dynamic-mig-support_cn.md @@ -126,9 +126,9 @@ HAMi目前包含[MIG配置模版](https://github.com/Project-HAMi/HAMi/blob/mast memory: 80896 count: 1 ``` - ### 修改后可以通过更新或重新部署chart来生效 + > **Note** 修改后可以通过更新或重新部署chart来生效 - ### 在收到任务请求后，HAMi会在上述定义的MIG模版中的依次查找，直到找到一个可以运行任务的模版 + > **Note** 在收到任务请求后，HAMi会在上述定义的MIG模版中的依次查找，直到找到一个可以运行任务的模版 ## 使用MIG模式运行任务 From a66fad036eb9199a12ce4099e419860471368fd8 Mon Sep 17 00:00:00 2001 From: limengxuan <391013634@qq.com> Date: Thu, 19 Dec 2024 10:36:32 +0800 Subject: [PATCH 10/10] update dynamic-mig document and examples Signed-off-by: limengxuan <391013634@qq.com> --- docs/dynamic-mig-support.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dynamic-mig-support.md b/docs/dynamic-mig-support.md index 97cf3a60e1..9d718170af 100644 --- a/docs/dynamic-mig-support.md +++ b/docs/dynamic-mig-support.md @@ -127,6 +127,7 @@ You can customize the mig configuration by following the steps below: ``` > **Note** Helm installation and updates will be based on the configuration in this file, overwriting the built-in configuration of Helm + > **Note** Be aware HAMi will find and use the first MIG template suitable to the task in the order of this configMap ## Running MIG jobs