Skip to content

refactor(MIG): support MIG instance dynamic allocation/deallocation #2378

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
archlitchi merged 11 commits into from
Aug 10, 2026
Merged

refactor(MIG): support MIG instance dynamic allocation/deallocation #2378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
4d86341
feat(nvidia): support topology-aware dynamic MIG allocation
FouoF Aug 5, 2026
bf7b35b
clean pod-resource & fix test
FouoF Aug 5, 2026
175a5d3
update design docs and simplify node report annotation
FouoF Aug 5, 2026
76e046a
update metrics to match dcgm
FouoF Aug 5, 2026
72278af
add license
FouoF Aug 5, 2026
dde32fc
add import
FouoF Aug 5, 2026
c7fcb65
clean up mig-parted
FouoF Aug 6, 2026
9542ac7
resolve commnet
FouoF Aug 7, 2026
cf45c6b
fix(nvidia): serialize MIG startup and restore reservations
FouoF Aug 7, 2026
079cb7c
clenup migmgr
FouoF Aug 7, 2026
9c5464e
update MIG configuration and documentation
FouoF Aug 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
218 changes: 10 additions & 208 deletions charts/hami/templates/scheduler/device-configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,223 +32,25 @@ data:
gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }}
libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }}
runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}"
knownMigGeometries:
migProfileAllowlist:
- models: [ "A30" ]
allowedGeometries:
-
- name: 1g.6gb
core: 25
memory: 6144
count: 4
-
- name: 2g.12gb
core: 50
memory: 12288
count: 2
-
- name: 4g.24gb
core: 100
memory: 24576
count: 1
profiles: [ "1g.6gb", "2g.12gb", "4g.24gb" ]
- models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"]
allowedGeometries:
-
- name: 1g.5gb
core: 14
memory: 5120
count: 7
-
- name: 1g.5gb
core: 14
memory: 5120
count: 1
- name: 2g.10gb
core: 28
memory: 10240
count: 3
-
- name: 3g.20gb
core: 42
memory: 20480
count: 2
-
- name: 7g.40gb
core: 100
memory: 40960
count: 1
profiles: [ "1g.5gb", "2g.10gb", "3g.20gb", "7g.40gb" ]
- models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
allowedGeometries:
-
- name: 1g.10gb
core: 14
memory: 9728
count: 7
-
- name: 1g.10gb
core: 14
memory: 9728
count: 1
- name: 2g.20gb
core: 28
memory: 19968
count: 3
-
- name: 3g.40gb
core: 42
memory: 40192
count: 2
-
- name: 7g.80gb
core: 98
memory: 80896
count: 1
profiles: [ "1g.10gb", "2g.20gb", "3g.40gb", "7g.79gb" ]
- models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"]
allowedGeometries:
-
- name: 1g.10gb
core: 14
memory: 10240
count: 7
-
- name: 1g.10gb
core: 14
memory: 10240
count: 1
- name: 2g.20gb
core: 28
memory: 20480
count: 3
-
- name: 3g.40gb
core: 42
memory: 40960
count: 2
-
- name: 7g.80gb
core: 100
memory: 81920
count: 1
profiles: [ "1g.10gb", "2g.20gb", "3g.40gb", "7g.80gb" ]
- models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"]
allowedGeometries:
-
- name: 1g.12gb
core: 14
memory: 12288
count: 7
-
- name: 1g.12gb
core: 14
memory: 12288
count: 1
- name: 2g.24gb
core: 28
memory: 24576
count: 3
-
- name: 3g.47gb
core: 42
memory: 48128
count: 2
-
- name: 7g.94gb
core: 100
memory: 96256
count: 1
profiles: [ "1g.12gb", "2g.24gb", "3g.47gb", "7g.94gb" ]
- models: [ "H20", "H100 on GH200"]
allowedGeometries:
-
- name: 1g.12gb
core: 14
memory: 12288
count: 7
-
- name: 1g.12gb
core: 14
memory: 12288
count: 1
- name: 2g.24gb
core: 28
memory: 24576
count: 3
-
- name: 3g.48gb
core: 42
memory: 49152
count: 2
-
- name: 7g.96gb
core: 100
memory: 98304
count: 1
profiles: [ "1g.12gb", "2g.24gb", "3g.48gb", "7g.96gb" ]
- models: [ "H200 NVL", "H200-SXM5"]
allowedGeometries:
-
- name: 1g.18gb
core: 14
memory: 18432
count: 7
-
- name: 1g.18gb
core: 14
memory: 18432
count: 1
- name: 2g.35gb
core: 28
memory: 35840
count: 3
-
- name: 3g.71gb
core: 42
memory: 72704
count: 2
-
- name: 7g.141gb
core: 100
memory: 144384
count: 1
profiles: [ "1g.18gb", "2g.35gb", "3g.71gb", "7g.141gb" ]
- models: [ "B200" ]
allowedGeometries:
-
- name: 1g.23gb
core: 14
memory: 23552
count: 7
-
- name: 1g.23gb
core: 14
memory: 23552
count: 1
- name: 2g.45gb
core: 28
memory: 46080
count: 3
-
- name: 3g.90gb
core: 42
memory: 92160
count: 2
-
- name: 7g.180gb
core: 100
memory: 184320
count: 1
profiles: [ "1g.23gb", "2g.45gb", "3g.90gb", "7g.180gb" ]
- models: [ "RTX PRO 6000 Blackwell Server Edition" ]
allowedGeometries:
-
- name: 1g.24gb
core: 25
memory: 24576
count: 4
-
- name: 2g.48gb
core: 50
memory: 49152
count: 2
-
- name: 4g.96gb
core: 100
memory: 98304
count: 1
profiles: [ "1g.24gb", "2g.48gb", "4g.96gb" ]
cambricon:
resourceCountName: {{ .Values.mluResourceName }}
resourceMemoryName: {{ .Values.mluResourceMem }}
Expand Down
8 changes: 2 additions & 6 deletions charts/hami/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -550,12 +550,8 @@ prometheus:
# device-config:
# content: |
# nvidia:
# knownMigGeometries:
# migProfileAllowlist:
# - models: [ "RTX 6000 Ada" ]
# allowedGeometries:
# - - name: "1g.6gb"
# core: 25
# memory: 6144
# count: 4
# profiles: [ "1g.6gb", "2g.12gb", "4g.24gb" ]
device-config:
content: ""
41 changes: 32 additions & 9 deletions cmd/scheduler/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,8 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric
)
nodeGPUMigInstance := prometheus.NewDesc(
"hami_node_gpu_mig_instance_info",
"GPU Sharing mode. 0 for hami-core, 1 for mig, 2 for mps",
[]string{"node", "device_uuid", "device_index", "mig_name"}, nil,
"Realized MIG instance identity and scheduler placement",
[]string{"node", "device_uuid", "device_index", "mig_uuid", "profile", "gpu_instance_id", "compute_instance_id", "placement_start", "placement_size"}, nil,
)

// Legacy metric descriptors (only created when legacy mode is enabled)
Expand Down Expand Up @@ -203,17 +203,40 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric
for _, devs := range val.Devices.DeviceLists {
coreLimit, coreAllocated := normalizeAMDCoreMetrics(devs.Device.Type, devs.Device.Totalcore, devs.Device.Usedcores)
if devs.Device.Mode == "mig" {
for idx, migs := range devs.Device.MigUsage.UsageList {
klog.V(3).Infoln("mig instances=", devs.Device.MigUsage)
inuse := 0
if migs.InUse {
inuse = 1
for _, allocation := range devs.Device.MigAllocationsInUse {
if !allocation.RuntimeReady {
continue
}
if err := sendMetric(ch, nodeGPUMigInstance, prometheus.GaugeValue, float64(inuse), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), migs.Name+"-"+fmt.Sprint(idx)); err != nil {
klog.V(3).InfoS("MIG instance allocation",
"profile", allocation.Profile,
"gpuInstanceID", allocation.GPUInstanceID,
"computeInstanceID", allocation.ComputeInstanceID,
"migUUID", allocation.MigUUID)
if err := sendMetric(
ch,
nodeGPUMigInstance,
prometheus.GaugeValue,
1,
nodeID,
devs.Device.ID,
fmt.Sprint(devs.Device.Index),
allocation.MigUUID,
allocation.Profile,
fmt.Sprint(allocation.GPUInstanceID),
fmt.Sprint(allocation.ComputeInstanceID),
fmt.Sprint(allocation.Placement.Start),
fmt.Sprint(allocation.Placement.Size),
); err != nil {
klog.V(4).Infof("Failed to send nodeGPUMigInstance metric: %v", err)
}
if legacy {
sendLegacyMetric(ch, legacyMigInstance, prometheus.GaugeValue, float64(inuse), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), migs.Name+"-"+fmt.Sprint(idx))
sendLegacyMetric(
ch,
legacyMigInstance,
prometheus.GaugeValue,
1,
nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), allocation.Profile+"-"+fmt.Sprint(allocation.GPUInstanceID),
)
}
}
}
Expand Down
31 changes: 31 additions & 0 deletions cmd/scheduler/metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,3 +377,34 @@ nodeGPUMemoryPercentage{deviceidx="2",deviceuuid="normal-memory",nodeid="node-1"
t.Fatalf("unexpected collecting result:\n%s", err)
}
}

func TestClusterManagerCollectorExposesDynamicMigRuntimeIdentity(t *testing.T) {
nodeUsage := map[string]*schedulerpkg.NodeUsage{
"node-1": {
Devices: policy.DeviceUsageList{DeviceLists: []*policy.DeviceListsScore{{
Device: &device.DeviceUsage{
ID: "GPU-parent", Index: 0, Mode: "mig", Type: "NVIDIA",
MigAllocationsInUse: []device.MigAllocation{{
Profile: "2g.10gb", Placement: device.MigPlacement{Start: 0, Size: 2},
MigUUID: "MIG-runtime", GPUInstanceID: 4, ComputeInstanceID: 0, RuntimeReady: true,
}},
},
}}},
},
}
collector := ClusterManagerCollector{
ClusterManager: &ClusterManager{},
metricsProvider: &fakeMetricsProvider{
nodeUsage: nodeUsage, quotaManager: device.NewQuotaManager(), podManager: device.NewPodManager(),
},
}

want := `
# HELP hami_node_gpu_mig_instance_info Realized MIG instance identity and scheduler placement
# TYPE hami_node_gpu_mig_instance_info gauge
hami_node_gpu_mig_instance_info{compute_instance_id="0",device_index="0",device_uuid="GPU-parent",gpu_instance_id="4",mig_uuid="MIG-runtime",node="node-1",placement_size="2",placement_start="0",profile="2g.10gb"} 1
`
if err := promtestutil.CollectAndCompare(collector, strings.NewReader(want), "hami_node_gpu_mig_instance_info"); err != nil {
t.Fatalf("unexpected collecting result:\n%s", err)
}
}
Loading
Loading