diff --git a/docs/config.md b/docs/config.md index 80db448c00..b29c8054c5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -45,7 +45,34 @@ helm install hami hami-charts/hami --set devicePlugin.deviceMemoryScaling=5 ... * `resourcePriority:` String type, vgpu task priority name, default: "nvidia.com/priority" -# Container config envs +# Pod configs: annotations + +* `nvidia.com/use-gpuuuid:` + String type, ie: "GPU-AAA,GPU-BBB" + If set, devices allocated by this pod must be one of UUIDs defined in this string. +* `nvidia.com/nouse-gpuuuid` + String type, ie: "GPU-AAA,GPU-BBB" + If set, devices allocated by this pod will NOT in UUIDs defined in this string. +* `nvidia.com/nouse-gputype:` + String type, ie: "Tesla V100-PCIE-32GB, NVIDIA A10" + If set, devices allocated by this pod will NOT in types defined in this string. +* `nvidia.com/use-gputype` + String type, ie: "Tesla V100-PCIE-32GB, NVIDIA A10" + If set, devices allocated by this pod MUST be one of types defined in this string. +* `hami.io/node-scheduler-policy` + String type, "binpack" or "spread" + binpack: the scheduler will try to allocate the pod to used GPU nodes for execution. + spread: the scheduler will try to allocate the pod to different GPU nodes for execution. +* `hami.io/gpu-scheduler-policy` + String type, "binpack" or "spread" + binpack: the scheduler will try to allocate the pod to the same GPU card for execution. + spread:the scheduler will try to allocate the pod to different GPU card for execution. +* `nvidia.com/vgpu-mode` + String type, "hami-core" or "mig" + Which type of vgpu instance this pod wish to use + + +# Container configs: env * `GPU_CORE_UTILIZATION_POLICY:` String type, "default", "force", "disable" @@ -53,12 +80,6 @@ helm install hami hami-charts/hami --set devicePlugin.deviceMemoryScaling=5 ... "default" means the dafault utilization policy "force" means the container will always limit the core utilization below "nvidia.com/gpucores" "disable" means the container will ignore the utilization limitation set by "nvidia.com/gpucores" during task execution - -* `ACTIVE_OOM_KILLER:` - Bool type, "true","false" - default: false - "true" means there will be a daemon process which monitors all running tasks inside this container, and instantly kill any process which exceeds the limitation set by "nvidia.com/gpumem" or "nvidia.com/gpumemory" - * `CUDA_DISABLE_CONTROL` Bool type, "true","false" default: false diff --git a/docs/config_cn.md b/docs/config_cn.md index a60d4cc789..9d548cb207 100644 --- a/docs/config_cn.md +++ b/docs/config_cn.md @@ -33,16 +33,39 @@ helm install vgpu vgpu-charts/vgpu --set devicePlugin.deviceMemoryScaling=5 ... * `resourcePriority:` 字符串类型，表示申请任务的任务优先级，默认: "nvidia.com/priority" +# Pod配置（在注解中指定） + +* `nvidia.com/use-gpuuuid:` + 字符串类型, 如: "GPU-AAA,GPU-BBB" + 如果设置, 该任务申请的设备只能是字符串中定义的设备之一。 +* `nvidia.com/nouse-gpuuuid` + 字符串类型, 如: "GPU-AAA,GPU-BBB" + 如果设置, 该任务不能使用字符串中定义的任何设备 +* `nvidia.com/nouse-gputype:` + 字符串类型, 如: "Tesla V100-PCIE-32GB, NVIDIA A10" + 如果设置, 该任务不能使用字符串中定义的任何设备型号 +* `nvidia.com/use-gputype` + 字符串类型, 如: "Tesla V100-PCIE-32GB, NVIDIA A10" + 如果设置, 该任务申请的设备只能使用字符串中定义的设备型号。 +* `hami.io/gpu-scheduler-policy` + 字符串类型, "binpack" 或 "spread" + spread:, 调度器会尽量将任务均匀地分配在不同GPU中 + binpack: 调度器会尽量将任务分配在已分配的GPU中，从而减少碎片 +* `hami.io/node-scheduler-policy` + 字符串类型, "binpack" 或 "spread" + spread: 调度器会尽量将任务均匀地分配到不同节点上 + binpack: 调度器会尽量将任务分配在已分配任务的节点上，从而减少碎片 +* `nvidia.com/vgpu-mode` + 字符串类型, "hami-core" 或 "mig" + 该任务希望使用的vgpu类型 + + # 容器配置（在容器的环境变量中指定） * `GPU_CORE_UTILIZATION_POLICY:` 字符串类型，"default", "force", "disable" 默认为"default" 代表容器算力限制策略， "default"为默认，"force"为强制限制算力，一般用于测试算力限制的功能，"disable"为忽略算力限制 -* `ACTIVE_OOM_KILLER:` - 布尔类型，"true", "false" - 默认为false - 若设置为true，则代表监控系统将会持续监控进程的显存使用量，并主动kill掉任何用超配额的进行。 * `CUDA_DISABLE_CONTROL` 布尔类型，"true", "false" 默认为false diff --git a/docs/dynamic-mig-support.md b/docs/dynamic-mig-support.md new file mode 100644 index 0000000000..9d718170af --- /dev/null +++ b/docs/dynamic-mig-support.md @@ -0,0 +1,164 @@ +## Introduction + +**We now support dynamic-mig by using mig-parted to adjust mig-devices dynamically**, including: + +***Dynamic MIG instance management***: User don't need to operate on GPU node, using 'nvidia-smi -i 0 -mig 1' or other command to manage MIG instance, all will be done by HAMi-device-plugin. + +***Dynamic MIG Adjustment***: Each MIG device managed by HAMi will dyamically adjust their MIG template according to tasks submitted when necessary. + +***Device MIG Observation***: Each MIG instance generated by HAMi will be shown in scheduler-monitor, including task information. user can get a clear overview of MIG nodes. + +***Compatable with HAMi-core nodes***: HAMi can manage a unified GPU pool of `HAMi-core node` and `mig node`. A task can be scheduled to either node if not appointed manually by using `nvidia.com/vgpu-mode` annotation. + +***Unified API with HAMi-core***: Zero work needs to be done to make the job compatible with dynamic-mig feature. + +## Prerequisites + +* NVIDIA Blackwell and Hopper™ and Ampere Devices +* HAMi > v2.5.0 +* Nvidia-container-toolkit + +## Enabling Dynamic-mig Support + +* Install the chart using helm, See 'enabling vGPU support in kubernetes' section [here](https://github.com/Project-HAMi/HAMi#enabling-vgpu-support-in-kubernetes) + +* Configure `mode` in device-plugin configMap to `mig` for MIG nodes +``` +kubectl describe cm hami-device-plugin -n kube-system +``` + +```json +{ + "nodeconfig": [ + { + "name": "MIG-NODE-A", + "operatingmode": "mig", + "filterdevices": { + "uuid": [], + "index": [] + } + } + ] +} +``` + +* Restart the following pods for the change to take effect: + * hami-scheduler + * hami-device-plugin on 'MIG-NODE-A' + +## Custom mig configuration (Optional) +HAMi currently has a [built-in mig configuration](https://github.com/Project-HAMi/HAMi/blob/master/charts/hami/templates/scheduler/device-configmap.yaml) for MIG. + +You can customize the mig configuration by following the steps below: + + ### Change the content of 'device-configmap.yaml' in charts/hami/templates/scheduler, the as follows + + ```yaml + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} + deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} + deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }} + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + memory: 6144 + count: 4 + - + - name: 2g.12gb + memory: 12288 + count: 2 + - + - name: 4g.24gb + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB", "A100-SXM4-40GB" ] + allowedGeometries: + - + - name: 1g.5gb + memory: 5120 + count: 7 + - + - name: 2g.10gb + memory: 10240 + count: 3 + - name: 1g.5gb + memory: 5120 + count: 1 + - + - name: 3g.20gb + memory: 20480 + count: 2 + - + - name: 7g.40gb + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + memory: 10240 + count: 7 + - + - name: 2g.20gb + memory: 20480 + count: 3 + - name: 1g.10gb + memory: 10240 + count: 1 + - + - name: 3g.40gb + memory: 40960 + count: 2 + - + - name: 7g.79gb + memory: 80896 + count: 1 + ``` + + > **Note** Helm installation and updates will be based on the configuration in this file, overwriting the built-in configuration of Helm + + > **Note** Be aware HAMi will find and use the first MIG template suitable to the task in the order of this configMap + +## Running MIG jobs + +MIG instance can now be requested by a container the same way as using `hami-core` +simply by specifying the `nvidia.com/gpu` and `nvidia.com/gpumem` resource type. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod + annotations: + nvidia.com/vgpu-mode: "mig" #(Optional), if not set, this pod can be assigned to a MIG instance or a hami-core instance +spec: + containers: + - name: ubuntu-container + image: ubuntu:18.04 + command: ["bash", "-c", "sleep 86400"] + resources: + limits: + nvidia.com/gpu: 2 + nvidia.com/gpumem: 8000 +``` + +In this example above, the task allocates two mig instances, each with at least 8G device memory. + +## Notes + +1. You don't need to do anything on MIG node, all are managed by mig-parted in hami-device-plugin. + +2. Nvidia devices before Ampere architect can't use 'mig' mode + +3. You won't see any mig resources(ie, `nvidia.com/mig-1g.10gb`) on node, hami uses a unified resource name for both 'mig' and 'hami-core' node \ No newline at end of file diff --git a/docs/dynamic-mig-support_cn.md b/docs/dynamic-mig-support_cn.md new file mode 100644 index 0000000000..646b389ab5 --- /dev/null +++ b/docs/dynamic-mig-support_cn.md @@ -0,0 +1,163 @@ +## 动态MIG功能简介 + +**HAMi将在2.5版本后支持动态MIG切分模式**, 其功能包括: + +***动态MIG实例管理***: 用户不需要在节点上事先生成MIG实例，HAMi会根据任务需要自动创建 + +***动态切换MIG切分方案***: HAMi会根据设备上的任务情况和新任务的需求，动态的切换MIG模版 + +***MIG实例监控***: 每个由HAMi管理的MIG实例都可以在调度器监控中找到，用户可以通过该监控清晰地获取整个集群的MIG视图 + +***可与使用hami-core的节点进行统一的资源池化***: HAMi将MIG与hami-core这两种切分方案进行了统一的池化处理，若任务未指定切分模式的话，分配给hami-core或者mig都是有可能的 + +***统一的API***: 使用动态MIG功能完全不需要进行任务层的适配工作 + +## 需求 + +* NVIDIA Blackwell and Hopper™ and Ampere Devices +* HAMi > v2.5.0 +* Nvidia-container-toolkit + +## 开启动态MIG功能 + +* 通过[这里](https://github.com/Project-HAMi/HAMi#enabling-vgpu-support-in-kubernetes)的文档部署HAMi + +* 通过以下指令修改configMap，并将节点的工作模式修改为`mig` +``` +kubectl describe cm hami-device-plugin -n kube-system +``` + +```json +{ + "nodeconfig": [ + { + "name": "MIG-NODE-A", + "operatingmode": "mig", + "filterdevices": { + "uuid": [], + "index": [] + } + } + ] +} +``` + +* 重启以下2个pod使修改后的配置生效: + * hami-scheduler + * 在'MIG-NODE-A'上的hami-device-plugin + +## 修改MIG模版列表 (可选) + +HAMi目前包含[MIG配置模版](https://github.com/Project-HAMi/HAMi/blob/master/charts/hami/templates/scheduler/device-configmap.yaml) + +你可以根据自己的集群环境，通过以下的方式去进行修改: + + ### 修改`charts/hami/templates/scheduler`路径下的`device-configmap.yaml` + + ```yaml + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} + deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} + deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }} + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + memory: 6144 + count: 4 + - + - name: 2g.12gb + memory: 12288 + count: 2 + - + - name: 4g.24gb + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB", "A100-SXM4-40GB" ] + allowedGeometries: + - + - name: 1g.5gb + memory: 5120 + count: 7 + - + - name: 2g.10gb + memory: 10240 + count: 3 + - name: 1g.5gb + memory: 5120 + count: 1 + - + - name: 3g.20gb + memory: 20480 + count: 2 + - + - name: 7g.40gb + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + memory: 10240 + count: 7 + - + - name: 2g.20gb + memory: 20480 + count: 3 + - name: 1g.10gb + memory: 10240 + count: 1 + - + - name: 3g.40gb + memory: 40960 + count: 2 + - + - name: 7g.79gb + memory: 80896 + count: 1 + ``` + > **Note** 修改后可以通过更新或重新部署chart来生效 + + > **Note** 在收到任务请求后，HAMi会在上述定义的MIG模版中的依次查找，直到找到一个可以运行任务的模版 + +## 使用MIG模式运行任务 + +MIG实例子可以通过和使用hami-core相同的方式进行申请，只需要指定`nvidia.com/gpu`和`nvidia.com/gpumem`即可 + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod + annotations: + nvidia.com/vgpu-mode: "mig" #(Optional), if not set, this pod can be assigned to a MIG instance or a hami-core instance +spec: + containers: + - name: ubuntu-container + image: ubuntu:18.04 + command: ["bash", "-c", "sleep 86400"] + resources: + limits: + nvidia.com/gpu: 2 + nvidia.com/gpumem: 8000 +``` + +在上面的例子中，该任务申请了2个MIG实例，每个实例至少需要8G显存 + +## 备注 + +1. 你不需要在MIG节点上进行任何操作，所有MIG实例的创建和维护都是由hami-vgpu-device-plugin进行的 + +2. 安培架构之前的NVIDIA设备无法使用`MIG`模式 + +3. 你不会在节点上看到MIG资源名(例如, `nvidia.com/mig-1g.10gb`)，HAMi对于hami-core和mig使用统一的资源名进行管理 \ No newline at end of file diff --git a/examples/nvidia/dynamic_mig_example.yaml b/examples/nvidia/dynamic_mig_example.yaml new file mode 100644 index 0000000000..360c9e886a --- /dev/null +++ b/examples/nvidia/dynamic_mig_example.yaml @@ -0,0 +1,18 @@ +## This example will allocate 2g.10gb * 2 for A100-40GB-PCIE device +## or 1g.10gb * 2 for A100-80GB-XSM device. +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod + annotations: + nvidia.com/vgpu-mode: "mig" +spec: + containers: + - name: ubuntu-container + image: ubuntu:18.04 + command: ["bash", "-c", "sleep 86400"] + resources: + limits: + nvidia.com/gpu: 2 + nvidia.com/gpumem: 8000 + diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index 2c9727ee99..2bb7686916 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -45,7 +45,7 @@ const ( GPUUseUUID = "nvidia.com/use-gpuuuid" // GPUNoUseUUID is user can not use specify GPU device for set GPU UUID. GPUNoUseUUID = "nvidia.com/nouse-gpuuuid" - AllocateMode = "nvidia.com/mode" + AllocateMode = "nvidia.com/vgpu-mode" MigMode = "mig" HamiCoreMode = "hami-core"