Enable Dynamic-mig feature for HAMi

charts/hami/templates/device-plugin/configmap.yaml

@@ -11,6 +11,7 @@ data:
         "nodeconfig": [
             {
                 "name": "m5-cloudinfra-online02",
+                "operatingmode": "hami-core",
                 "devicememoryscaling": 1.8,
                 "devicesplitcount": 10,
                 "migstrategy":"none",

charts/hami/templates/device-plugin/daemonsetnvidia.yaml

@@ -64,6 +64,7 @@ spec:
             - name: HOOK_PATH
               value: {{ .Values.global.gpuHookPath }}
           securityContext:
+            privileged: true
             allowPrivilegeEscalation: false
             capabilities:
               drop: ["ALL"]
@@ -113,6 +114,8 @@ spec:
               mountPath: /sysinfo
             - name: hostvar
               mountPath: /hostvar
+            - name: hosttmp
+              mountPath: /tmp
       volumes:
         - name: ctrs
           hostPath:

charts/hami/templates/scheduler/device-configmap.yaml

@@ -23,6 +23,63 @@ data:
       deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }}
       deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }}
       deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }}
+      knownMigGeometries:
+      - models: [ "A30" ]
+        allowedGeometries:
+          - 
+            - name: 1g.6gb
+              memory: 6144
+              count: 4
+          - 
+            - name: 2g.12gb
+              memory: 12288
+              count: 2
+          - 
+            - name: 4g.24gb
+              memory: 24576
+              count: 1
+      - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB", "A100-SXM4-40GB" ]
+        allowedGeometries:
+          - 
+            - name: 1g.5gb
+              memory: 5120
+              count: 7
+          - 
+            - name: 2g.10gb
+              memory: 10240
+              count: 3
+            - name: 1g.5gb
+              memory: 5120
+              count: 1
+          - 
+            - name: 3g.20gb
+              memory: 20480
+              count: 2
+          - 
+            - name: 7g.40gb
+              memory: 40960
+              count: 1
+      - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
+        allowedGeometries:
+          - 
+            - name: 1g.10gb
+              memory: 10240
+              count: 7
+          - 
+            - name: 2g.20gb
+              memory: 20480
+              count: 3
+            - name: 1g.10gb
+              memory: 10240
+              count: 1
+          - 
+            - name: 3g.40gb
+              memory: 40960
+              count: 2
+          - 
+            - name: 7g.79gb
+              memory: 80896
+              count: 1
     cambricon:
       resourceCountName: {{ .Values.mluResourceName }}
       resourceMemoryName: {{ .Values.mluResourceMem }}

cmd/scheduler/metrics.go

@@ -99,9 +99,30 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
 		"GPU Memory Allocated Percentage on a certain GPU",
 		[]string{"nodeid", "deviceuuid", "deviceidx"}, nil,
 	)
+	nodeGPUMigInstance := prometheus.NewDesc(
+		"nodeGPUMigInstance",
+		"GPU Sharing mode. 0 for hami-core, 1 for mig, 2 for mps",
+		[]string{"nodeid", "deviceuuid", "deviceidx", "migname"}, nil,
+	)
 	nu := sher.InspectAllNodesUsage()
 	for nodeID, val := range *nu {
 		for _, devs := range val.Devices.DeviceLists {
+			if devs.Device.Mode == "mig" {
+				for idx, migs := range devs.Device.MigUsage.UsageList {
+					klog.Infoln("mig instances=", devs.Device.MigUsage)
+					inuse := 0
+					if migs.InUse {
+						inuse = 1
+					}
+					ch <- prometheus.MustNewConstMetric(
+						nodeGPUMigInstance,
+						prometheus.GaugeValue,
+						float64(inuse),
+						nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), migs.Name+"-"+fmt.Sprint(idx),
+					)
+				}
+			}
+
 			ch <- prometheus.MustNewConstMetric(
 				nodevGPUMemoryLimitDesc,
 				prometheus.GaugeValue,

device-config.yaml

@@ -0,0 +1,135 @@
+nvidia:
+  resourceCountName: nvidia.com/vgpu
+  resourceMemoryName: nvidia.com/gpumem
+  resourceMemoryPercentageName: nvidia.com/gpumem-percentage
+  resourceCoreName: nvidia.com/gpucores
+  resourcePriorityName: nvidia.com/priority
+  overwriteEnv: false
+  defaultMemory: 0
+  defaultCores: 0
+  defaultGPUNum: 1
+  deviceSplitCount: 10
+  deviceMemoryScaling: 1
+  deviceCoreScaling: 1
+  knownMigGeometries:
+  - models: [ "A30" ]
+    allowedGeometries:
+      - 
+        - name: 1g.6gb
+          memory: 6144
+          count: 4
+      - 
+        - name: 1g.6gb
+          memory: 6144
+          count: 2
+        - name: 2g.12gb
+          memory: 12288
+          count: 1
+      - 
+        - name: 2g.12gb
+          memory: 12288
+          count: 2
+      - 
+        - name: 4g.24gb
+          memory: 24576
+          count: 1  
+  - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB", "A100-SXM4-40GB" ]
+    allowedGeometries:
+      - 
+        - name: 1g.5gb
+          memory: 5120
+          count: 7
+      - 
+        - name: 2g.10gb
+          memory: 10240
+          count: 3
+        - name: 1g.5gb
+          memory: 12288
+          count: 1
+      - 
+        - name: 3g.20gb
+          memory: 20480
+          count: 2
+      - 
+        - name: 7g.40gb
+          memory: 40960
+          count: 1
+  - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
+    allowedGeometries:
+cambricon:
+  resourceCountName: cambricon.com/vmlu
+  resourceMemoryName: cambricon.com/mlu.smlu.vmemory
+  resourceCoreName: cambricon.com/mlu.smlu.vcore
+hygon:
+  resourceCountName: hygon.com/dcunum
+  resourceMemoryName: hygon.com/dcumem
+  resourceCoreName: hygon.com/dcucores
+metax:
+  resourceCountName: "metax-tech.com/gpu"
+mthreads:
+  resourceCountName: "mthreads.com/vgpu"
+  resourceMemoryName: "mthreads.com/sgpu-memory"
+  resourceCoreName: "mthreads.com/sgpu-core"
+iluvatar: 
+  resourceCountName: iluvatar.ai/vgpu
+  resourceMemoryName: iluvatar.ai/vcuda-memory
+  resourceCoreName: iluvatar.ai/vcuda-core
+vnpus:
+- chipName: 910B
+  commonWord: Ascend910A
+  resourceName: huawei.com/Ascend910A
+  resourceMemoryName: huawei.com/Ascend910A-memory
+  memoryAllocatable: 32768
+  memoryCapacity: 32768
+  aiCore: 30
+  templates:
+    - name: vir02
+      memory: 2184
+      aiCore: 2
+    - name: vir04
+      memory: 4369
+      aiCore: 4
+    - name: vir08
+      memory: 8738
+      aiCore: 8
+    - name: vir16
+      memory: 17476
+      aiCore: 16
+- chipName: 910B3
+  commonWord: Ascend910B
+  resourceName: huawei.com/Ascend910B
+  resourceMemoryName: huawei.com/Ascend910B-memory
+  memoryAllocatable: 65536
+  memoryCapacity: 65536
+  aiCore: 20
+  aiCPU: 7
+  templates:
+    - name: vir05_1c_16g
+      memory: 16384
+      aiCore: 5
+      aiCPU: 1
+    - name: vir10_3c_32g
+      memory: 32768
+      aiCore: 10
+      aiCPU: 3
+- chipName: 310P3
+  commonWord: Ascend310P
+  resourceName: huawei.com/Ascend310P
+  resourceMemoryName: huawei.com/Ascend310P-memory
+  memoryAllocatable: 21527
+  memoryCapacity: 24576
+  aiCore: 8
+  aiCPU: 7
+  templates:
+    - name: vir01
+      memory: 3072
+      aiCore: 1
+      aiCPU: 1
+    - name: vir02
+      memory: 6144
+      aiCore: 2
+      aiCPU: 2
+    - name: vir04
+      memory: 12288
+      aiCore: 4
+      aiCPU: 4

docker/Dockerfile

@@ -8,6 +8,7 @@ ADD . /k8s-vgpu
 #RUN --mount=type=cache,target=/go/pkg/mod \
 #    cd /k8s-vgpu && make all
 RUN cd /k8s-vgpu && make all
+RUN go install github.com/NVIDIA/mig-parted/cmd/nvidia-mig-parted@latest
 
 FROM $NVIDIA_IMAGE AS nvbuild
 COPY ./libvgpu /libvgpu
@@ -26,6 +27,7 @@ LABEL version="$VERSION"
 LABEL maintainer="opensource@4paradigm.com"
 COPY ./LICENSE /k8s-vgpu/LICENSE
 COPY --from=gobuild /k8s-vgpu/bin /k8s-vgpu/bin
+COPY --from=gobuild /go/bin/nvidia-mig-parted /k8s-vgpu/bin/
 COPY ./docker/entrypoint.sh /k8s-vgpu/bin/entrypoint.sh
 COPY ./lib /k8s-vgpu/lib
 COPY --from=nvbuild /libvgpu/build/libvgpu.so /k8s-vgpu/lib/nvidia/

docker/Dockerfile.withlib

@@ -5,9 +5,12 @@ FROM $GOLANG_IMAGE AS build
 FROM $GOLANG_IMAGE AS GOBUILD
 ADD . /k8s-vgpu
 ARG GOPROXY=https://goproxy.cn,direct
+RUN go env -w GO111MODULE=on
 RUN cd /k8s-vgpu && make all
+RUN go install github.com/NVIDIA/mig-parted/cmd/nvidia-mig-parted@latest
 
-FROM ubuntu:24.04
+#FROM ubuntu:24.04
+FROM harbor.4pd.io/sagegpt-aio/pk_platform/ubuntu:22.04
 ENV NVIDIA_DISABLE_REQUIRE="true"
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=utility
@@ -17,6 +20,7 @@ LABEL version="$VERSION"
 LABEL maintainer="opensource@4paradigm.com"
 COPY ./LICENSE /k8s-vgpu/LICENSE
 COPY --from=GOBUILD /k8s-vgpu/bin /k8s-vgpu/bin
+COPY --from=GOBUILD /go/bin/nvidia-mig-parted /k8s-vgpu/bin/
 COPY ./docker/entrypoint.sh /k8s-vgpu/bin/entrypoint.sh
 COPY ./lib /k8s-vgpu/lib
 COPY ./libvgpu.so /k8s-vgpu/lib/nvidia/

pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go

@@ -42,7 +42,6 @@ import (
 	"github.com/NVIDIA/go-nvml/pkg/nvml"
 	"k8s.io/klog/v2"
 
-	"github.com/Project-HAMi/HAMi/pkg/api"
 	"github.com/Project-HAMi/HAMi/pkg/device/nvidia"
 	"github.com/Project-HAMi/HAMi/pkg/util"
 )
@@ -108,11 +107,11 @@ func parseNvidiaNumaInfo(idx int, nvidiaTopoStr string) (int, error) {
 	return result, nil
 }
 
-func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*api.DeviceInfo {
+func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*util.DeviceInfo {
 	devs := plugin.Devices()
 	klog.V(5).InfoS("getAPIDevices", "devices", devs)
 	nvml.Init()
-	res := make([]*api.DeviceInfo, 0, len(devs))
+	res := make([]*util.DeviceInfo, 0, len(devs))
 	for UUID := range devs {
 		ndev, ret := nvml.DeviceGetHandleByUUID(UUID)
 		if ret != nvml.SUCCESS {
@@ -160,14 +159,15 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*api.DeviceInfo {
 		if err != nil {
 			klog.ErrorS(err, "failed to get numa information", "idx", idx)
 		}
-		res = append(res, &api.DeviceInfo{
+		res = append(res, &util.DeviceInfo{
 			ID:      UUID,
-			Index:   idx,
+			Index:   uint(idx),
 			Count:   int32(plugin.schedulerConfig.DeviceSplitCount),
 			Devmem:  registeredmem,
 			Devcore: int32(plugin.schedulerConfig.DeviceCoreScaling * 100),
 			Type:    fmt.Sprintf("%v-%v", "NVIDIA", Model),
 			Numa:    numa,
+			Mode:    plugin.operatingMode,
 			Health:  health,
 		})
 		klog.Infof("nvml registered device id=%v, memory=%v, type=%v, numa=%v", idx, registeredmem, Model, numa)