From 8cb94f6da65576b06a04e6ed9d14832da6b5fa99 Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Thu, 16 Apr 2026 03:49:22 +0000 Subject: [PATCH 01/13] feat(ascend): add ResourceCoreName to support hami-core compute partitioning Signed-off-by: ashergaga <1214443299@qq.com> --- .../templates/scheduler/device-configmap.yaml | 2 + charts/hami/values.yaml | 4 + pkg/device/ascend/device.go | 123 ++++++++++++++---- pkg/device/ascend/vnpu.go | 1 + 4 files changed, 107 insertions(+), 23 deletions(-) diff --git a/charts/hami/templates/scheduler/device-configmap.yaml b/charts/hami/templates/scheduler/device-configmap.yaml index aaab6b664b..23db1d21ac 100644 --- a/charts/hami/templates/scheduler/device-configmap.yaml +++ b/charts/hami/templates/scheduler/device-configmap.yaml @@ -333,6 +333,7 @@ data: commonWord: Ascend910B3 resourceName: huawei.com/Ascend910B3 resourceMemoryName: huawei.com/Ascend910B3-memory + resourceCoreName: huawei.com/Ascend910B3-core memoryAllocatable: 65536 memoryCapacity: 65536 memoryFactor: 1 @@ -413,6 +414,7 @@ data: commonWord: Ascend910C resourceName: huawei.com/Ascend910C resourceMemoryName: huawei.com/Ascend910C-memory + resourceCoreName: huawei.com/Ascend910C-core memoryAllocatable: 65536 memoryCapacity: 65536 aiCore: 20 diff --git a/charts/hami/values.yaml b/charts/hami/values.yaml index 4f821ac2ee..35c97a9da8 100644 --- a/charts/hami/values.yaml +++ b/charts/hami/values.yaml @@ -486,12 +486,16 @@ devices: - huawei.com/Ascend910B2-memory - huawei.com/Ascend910B3 - huawei.com/Ascend910B3-memory + - huawei.com/Ascend910B3-core - huawei.com/Ascend910B4 - huawei.com/Ascend910B4-memory - huawei.com/Ascend910B4-1 - huawei.com/Ascend910B4-1-memory - huawei.com/Ascend310P - huawei.com/Ascend310P-memory + - huawei.com/Ascend910C + - huawei.com/Ascend910C-memory + - huawei.com/Ascend910C-core iluvatar: enabled: false customresources: diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 5ac32325cd..021f912deb 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -41,6 +41,8 @@ const ( Ascend910Prefix = "Ascend910" Ascend910CType = "Ascend910C" Ascend910NetworkWeight = 10 + VNPUModeAnnotation = "huawei.com/vnpu-mode" + VNPUModeHamiCore = "hami-core" ) type Devices struct { @@ -52,8 +54,10 @@ type Devices struct { } type RuntimeInfo struct { - UUID string `json:"UUID,omitempty"` - Temp string `json:"temp,omitempty"` + UUID string `json:"UUID,omitempty"` + Temp string `json:"temp,omitempty"` + Memory int64 `json:"memory,omitempty"` + Core int32 `json:"core,omitempty"` } var ( @@ -111,11 +115,11 @@ func (dev *Devices) CommonWord() string { } func (dev *Devices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) { - count, ok := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceName)] + count, ok := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceName)] if !ok { return false, nil } - + reqNum := count.Value() if dev.config.CommonWord == Ascend910CType { if reqNum == 1 { @@ -134,12 +138,41 @@ func (dev *Devices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, } } + // Check if hami-core is declared + vnpuMode := p.Annotations[VNPUModeAnnotation] + isHAMiCore := (vnpuMode == VNPUModeHamiCore) + + if isHAMiCore { + klog.V(3).Infof("Ascend core resource detected, injecting postStart lifecycle for container %s", ctr.Name) + + if ctr.Lifecycle == nil { + ctr.Lifecycle = &corev1.Lifecycle{} + } + + // Inject PostStart hook to start the limiter process + if ctr.Lifecycle.PostStart == nil { + ctr.Lifecycle.PostStart = &corev1.LifecycleHandler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "bash", + "-c", + "export RUST_LOG=info\n/hami-vnpu-core/target/debug/limiter > /tmp/inst1_manager.log 2>&1 &", + }, + }, + } + } + } + trimMem := dev.config.MemoryAllocatable memory, ok := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceMemoryName)] if ok { - trimMem, _ = dev.trimMemory(memory.Value()) - if trimMem <= 0 { - return false, fmt.Errorf("%s %d is invalid", dev.config.ResourceMemoryName, memory.Value()) + if isHAMiCore { + trimMem = memory.Value() + } else { + trimMem, _ = dev.trimMemory(memory.Value()) + if trimMem <= 0 { + return false, fmt.Errorf("%s %d is invalid", dev.config.ResourceMemoryName, memory.Value()) + } } } if count.Value() > 1 { @@ -179,15 +212,27 @@ func (dev *Devices) PatchAnnotations(pod *corev1.Pod, annoInput *map[string]stri (*annoInput)[device.InRequestDevices[commonWord]] = device.EncodePodSingleDevice(devList) (*annoInput)[device.SupportDevices[commonWord]] = device.EncodePodSingleDevice(devList) (*annoInput)["predicate-time"] = strconv.FormatInt(time.Now().Unix(), 10) + + // Check if hami-core is declared + vnpuMode := pod.Annotations[VNPUModeAnnotation] + klog.V(4).Infof("Patching pod %s with vnpu mode: %s", pod.Name, vnpuMode) + allocateStr := fmt.Sprintf("huawei.com/%s", dev.CommonWord()) var rtInfo []RuntimeInfo for _, dp := range devList { for _, val := range dp { - _, temp := dev.trimMemory(int64(val.Usedmem)) - rtInfo = append(rtInfo, RuntimeInfo{ - UUID: val.UUID, - Temp: temp, - }) + info := RuntimeInfo{UUID: val.UUID} + + // If is hami core, populate Memory and Core directly without using Temp + if vnpuMode == VNPUModeHamiCore { + info.Memory = int64(val.Usedmem) + info.Core = val.Usedcores + } else { + _, temp := dev.trimMemory(int64(val.Usedmem)) + info.Temp = temp + } + + rtInfo = append(rtInfo, info) } } s, err := json.Marshal(rtInfo) @@ -247,6 +292,8 @@ func (dev *Devices) CheckHealth(devType string, n *corev1.Node) (bool, bool) { func (dev *Devices) GenerateResourceRequests(ctr *corev1.Container) device.ContainerDeviceRequest { ascendResourceCount := corev1.ResourceName(dev.config.ResourceName) ascendResourceMem := corev1.ResourceName(dev.config.ResourceMemoryName) + ascendResourceCore := corev1.ResourceName(dev.config.ResourceCoreName) + v, ok := ctr.Resources.Limits[ascendResourceCount] if !ok { v, ok = ctr.Resources.Requests[ascendResourceCount] @@ -261,18 +308,35 @@ func (dev *Devices) GenerateResourceRequests(ctr *corev1.Container) device.Conta mem, ok = ctr.Resources.Requests[ascendResourceMem] } if ok { - memnums, ok := mem.AsInt64() - if ok { - if dev.config.MemoryFactor > 1 { - rawMemnums := memnums - memnums = memnums * int64(dev.config.MemoryFactor) - klog.V(4).Infof("Update Ascend memory request. before %d, after %d, factor %d", rawMemnums, memnums, dev.config.MemoryFactor) + memnums, _ := mem.AsInt64() + + // If "core" is requested, it explicitly indicates the use of soft-partitioning. + isCoreRequested := false + if ascendResourceCore != "" { + _, isCoreRequested = ctr.Resources.Limits[ascendResourceCore] + if !isCoreRequested { + _, isCoreRequested = ctr.Resources.Requests[ascendResourceCore] } + } + + if isCoreRequested { + // Soft-partitioning: Use the raw value directly. + memnum = int(memnums) + } else { m, _ := dev.trimMemory(memnums) memnum = int(m) } } + + // Process Core Resources corenum := int32(0) + if ascendResourceCore != "" { + if cv, ok := ctr.Resources.Limits[ascendResourceCore]; ok { + corenum = int32(cv.Value()) + } else if cv, ok := ctr.Resources.Requests[ascendResourceCore]; ok { + corenum = int32(cv.Value()) + } + } mempnum := 0 if memnum == 0 { @@ -340,7 +404,7 @@ func (dev *Devices) GetResourceNames() device.ResourceNames { return device.ResourceNames{ ResourceCountName: dev.config.ResourceName, ResourceMemoryName: dev.config.ResourceMemoryName, - ResourceCoreName: "", + ResourceCoreName: dev.config.ResourceCoreName, } } @@ -349,6 +413,13 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD originReq := k.Nums prevnuma := -1 klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k) + + vnpuMode := "" + if pod != nil && pod.Annotations != nil { + vnpuMode = pod.Annotations[VNPUModeAnnotation] + } + klog.V(4).InfoS("Fit: vnpu-mode annotation", "pod", pod.Name, "vnpuMode", vnpuMode) + var tmpDevs map[string]device.ContainerDevices tmpDevs = make(map[string]device.ContainerDevices) reason := make(map[string]int) @@ -405,19 +476,25 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) continue } - if dev.Totalcore-dev.Usedcores < k.Coresreq { + // Set dev.Totalcore to 100 if vnpuMode is hami-core + effectiveTotalCore := dev.Totalcore + if vnpuMode == VNPUModeHamiCore { + effectiveTotalCore = 100 + } + + if effectiveTotalCore-dev.Usedcores < k.Coresreq { reason[common.CardInsufficientCore]++ - klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", dev.Totalcore, "device used core", dev.Usedcores, "request cores", k.Coresreq) + klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", effectiveTotalCore, "device used core", dev.Usedcores, "request cores", k.Coresreq) continue } // Coresreq=100 indicates it want this card exclusively - if dev.Totalcore == 100 && k.Coresreq == 100 && dev.Used > 0 { + if effectiveTotalCore == 100 && k.Coresreq == 100 && dev.Used > 0 { reason[common.ExclusiveDeviceAllocateConflict]++ klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) continue } // You can't allocate core=0 job to an already full GPU - if dev.Totalcore != 0 && dev.Usedcores == dev.Totalcore && k.Coresreq == 0 { + if effectiveTotalCore != 0 && dev.Usedcores == effectiveTotalCore && k.Coresreq == 0 { reason[common.CardComputeUnitsExhausted]++ klog.V(5).InfoS(common.CardComputeUnitsExhausted, "pod", klog.KObj(pod), "device", dev.ID, "device index", i) continue diff --git a/pkg/device/ascend/vnpu.go b/pkg/device/ascend/vnpu.go index 26783e376d..4c558780dc 100644 --- a/pkg/device/ascend/vnpu.go +++ b/pkg/device/ascend/vnpu.go @@ -28,6 +28,7 @@ type VNPUConfig struct { ChipName string `yaml:"chipName"` ResourceName string `yaml:"resourceName"` ResourceMemoryName string `yaml:"resourceMemoryName"` + ResourceCoreName string `yaml:"resourceCoreName"` MemoryAllocatable int64 `yaml:"memoryAllocatable"` MemoryCapacity int64 `yaml:"memoryCapacity"` MemoryFactor int32 `yaml:"memoryFactor"` From 8072c6e1ff453dd8ccf8c3faffe82672e5263136 Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Fri, 17 Apr 2026 08:06:16 +0000 Subject: [PATCH 02/13] feat: add docs ascend910-hami-vnpu-support Signed-off-by: ashergaga <1214443299@qq.com> --- docs/ascend910-hami-vnpu-core-support.md | 104 +++++++++++++++++++ docs/ascend910-hami-vnpu-core-support_cn.md | 108 ++++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 docs/ascend910-hami-vnpu-core-support.md create mode 100644 docs/ascend910-hami-vnpu-core-support_cn.md diff --git a/docs/ascend910-hami-vnpu-core-support.md b/docs/ascend910-hami-vnpu-core-support.md new file mode 100644 index 0000000000..4b7755fe9f --- /dev/null +++ b/docs/ascend910-hami-vnpu-core-support.md @@ -0,0 +1,104 @@ +# Introduction to Ascend910B Series and Ascend910C Support + +Implements a soft slicing mechanism based on `libvnpu.so` interception and `limiter` token scheduling, enabling fine-grained resource sharing. For detailed information, check [hami-vnpu-core](https://github.com/Project-HAMi/hami-vnpu-core). + +* **_NPU sharing_**: Each task can allocate a portion of Ascend NPU instead of a whole NLU card, thus NPU can be shared among multiple tasks. +* **_Device Memory Control_**: Ascend NPUs can be allocated with certain device memory size and guarantee it that it does not exceed the boundary. +* **_Device Core Control_**: Ascend NPUs can be allocated with certain compute cores and guarantee it that it does not exceed the boundary. + +## Prerequisites + +* Ascend docker runtime +* driver version > 25.5 +* Ascend device type: 910B3、910C + +## Enabling NPU Sharing + +* [Ascend docker runtime](https://gitcode.com/Ascend/mind-cluster/tree/master/component/ascend-docker-runtime) + +* [ascend-device-plugin](https://github.com/Project-HAMi/ascend-device-plugin) +* **Chip Mode**: enable `device-share` mode on Ascend chips for virtualization + +### Host Environment Preparation + +Before launching any containers, the **Global Shared Memory (SHM) Region** must be initialized on the host to allow inter-Pod coordination. + +#### 1. Create the Shared Directory + +```bash +sudo mkdir -p /tmp/hami-shared-region +sudo chmod 777 /tmp/hami-shared-region +``` + +#### 2. Deploy hami-vnpu-core Components + +Place the following files in a fixed host path (`/usr/local/hami-vnpu-core/`) for mounting into containers: + +``` +/usr/local/hami-vnpu-core/ +├── limiter # Manager daemon binary (compiled from hami-vnpu-core) +├── libvnpu.so # Interception library for LD_PRELOAD +└── ld.so.preload # Global preload config +``` + + ### Create device-config.yaml + +The content is as follows: + + ```yaml +vnpus: +- chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + resourceCoreName: huawei.com/Ascend910B3-core + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 +- chipName: Ascend910 + commonWord: Ascend910C + resourceName: huawei.com/Ascend910C + resourceMemoryName: huawei.com/Ascend910C-memory + resourceCoreName: huawei.com/Ascend910C-core + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + ``` + + ### Install and update with Helm + + Helm installation and updates will be based on the configuration in this file, overwriting the built-in configuration of Helm. + +## Running NPU Workloads + +You can request Ascend 910B series resources using the `huawei.com/ascend910Bx` ,`huawei.com/ascend910Bx-memory` and `huawei.com/ascend910Bx-core` resource types: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: ascend-soft-slice-pod + annotations: + huawei.com/vnpu-mode: 'hami-core' # Enables hami-vnpu-core soft-segmentation for this pod +spec: + containers: + - name: npu_pod + ... + resources: + limits: + huawei.com/Ascend910B3: "1" # Request 1 physical NPU + huawei.com/Ascend910B3-memory: "28672" # Request 28Gi memory + huawei.com/Ascend910B3-core: "40" # Request 40% core +``` + diff --git a/docs/ascend910-hami-vnpu-core-support_cn.md b/docs/ascend910-hami-vnpu-core-support_cn.md new file mode 100644 index 0000000000..3e867d0f14 --- /dev/null +++ b/docs/ascend910-hami-vnpu-core-support_cn.md @@ -0,0 +1,108 @@ +# huawei.com/Ascend910B、huawei.com/Ascend910C系列软切分支持简介 + +实现了基于 `libvnpu.so` 拦截和limiter令牌调度的软切分机制,能够实现精细化的资源共享。详细信息请参阅 [hami-vnpu-core](https://www.google.com/search?q=链接)。 + +* **_NPU 共享_**: 每个任务可以只占用一部分显卡,多个任务可以共享一张显卡; + +* **_可限制分配的显存大小_**: 你可以用显存值(例如 3000M)来分配 NPU,本组件会确保任务使用的显存不会超过分配数值; + +* **_可限制分配的算力大小_**: 你可以用固定数量来分配 NPU 的 AI 核心和 AI CPU 核心,本组件会确保任务使用的算力不会超过分配数值。 + +## 节点需求 + +* Ascend docker runtime +* driver version > 25.5 +* Ascend device type: 910B3、910C + +## 开启 NPU 复用 + +* 部署 [Ascend docker runtime](https://gitcode.com/Ascend/mind-cluster/tree/master/component/ascend-docker-runtime) + +* 部署 [ascend-device-plugin](https://github.com/Project-HAMi/ascend-device-plugin) +* 芯片模式:在昇腾芯片上开启 `device-share` 模式以支持虚拟化。 + +## 宿主机环境准备 + +在启动任何容器之前,必须在宿主机上初始化 **全局共享内存 (SHM) 区域**,以便进行 Pod 间的协同。 + +1. **创建共享目录** + + ``` + sudo mkdir -p /tmp/hami-shared-region + sudo chmod 777 /tmp/hami-shared-region + ``` + +2. **部署 hami-vnpu-core 组件** + + 将以下文件放置在固定的宿主机路径(`/usr/local/hami-vnpu-core/`)中,以便挂载到容器内: + + ``` + /usr/local/hami-vnpu-core/ + ├── limiter # Manager daemon binary (compiled from hami-vnpu-core) + ├── libvnpu.so # Interception library for LD_PRELOAD + └── ld.so.preload # Global preload config + ``` + + ### 在 files 目录下创建 device-config.yaml + + 配置文件如下所示,可以按需调整: + + ```yaml + vnpus: + - chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + resourceCoreName: huawei.com/Ascend910B3-core + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: Ascend910 + commonWord: Ascend910C + resourceName: huawei.com/Ascend910C + resourceMemoryName: huawei.com/Ascend910C-memory + resourceCoreName: huawei.com/Ascend910C-core + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + ``` + + ### Helm 安装和更新 + + Helm 安装、更新将基于该配置文件,覆盖默认的配置文件 + + + +## 运行 NPU 任务 + +可通过使用 `huawei.com/ascend910Bx` 、 `huawei.com/ascend910Bx-memory` 和 `huawei.com/ascend910Bx-core`资源类型,来请求 Ascend 910B 系列设备: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: ascend-soft-slice-pod + annotations: + huawei.com/vnpu-mode: 'hami-core' # 添加该注解的走hami-vnpu-core软切分 +spec: + containers: + - name: npu_pod + ... + resources: + limits: + huawei.com/Ascend910B3: "1" # 请求 1 块物理 NPU + huawei.com/Ascend910B3-memory: "28672" # 请求 28Gi 显存 + huawei.com/Ascend910B3-core: "40" # 请求 40% 的算力 +``` + From 14d132d5e492dd5eb871721fba33ff4e0b5a186d Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Fri, 17 Apr 2026 08:32:15 +0000 Subject: [PATCH 03/13] feat: fix ai suggestion Signed-off-by: ashergaga <1214443299@qq.com> --- pkg/device/ascend/device.go | 39 ++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 021f912deb..7bcf78149b 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -42,7 +42,7 @@ const ( Ascend910CType = "Ascend910C" Ascend910NetworkWeight = 10 VNPUModeAnnotation = "huawei.com/vnpu-mode" - VNPUModeHamiCore = "hami-core" + VNPUModeHamiCore = "hami-core" ) type Devices struct { @@ -115,7 +115,7 @@ func (dev *Devices) CommonWord() string { } func (dev *Devices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) { - count, ok := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceName)] + count, ok := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceName)] if !ok { return false, nil } @@ -156,7 +156,7 @@ func (dev *Devices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, Command: []string{ "bash", "-c", - "export RUST_LOG=info\n/hami-vnpu-core/target/debug/limiter > /tmp/inst1_manager.log 2>&1 &", + "export RUST_LOG=info\n/hami-vnpu-core/limiter > /tmp/limiter_manager.log 2>&1 &", }, }, } @@ -309,21 +309,28 @@ func (dev *Devices) GenerateResourceRequests(ctr *corev1.Container) device.Conta } if ok { memnums, _ := mem.AsInt64() - - // If "core" is requested, it explicitly indicates the use of soft-partitioning. - isCoreRequested := false - if ascendResourceCore != "" { - _, isCoreRequested = ctr.Resources.Limits[ascendResourceCore] - if !isCoreRequested { - _, isCoreRequested = ctr.Resources.Requests[ascendResourceCore] + if ok { + if dev.config.MemoryFactor > 1 { + rawMemnums := memnums + memnums = memnums * int64(dev.config.MemoryFactor) + klog.V(4).Infof("Update Ascend memory request. before %d, after %d, factor %d", rawMemnums, memnums, dev.config.MemoryFactor) + } + // If "core" is requested, it explicitly indicates the use of soft-partitioning. + isCoreRequested := false + if ascendResourceCore != "" { + _, isCoreRequested = ctr.Resources.Limits[ascendResourceCore] + if !isCoreRequested { + _, isCoreRequested = ctr.Resources.Requests[ascendResourceCore] + } } - } - if isCoreRequested { - // Soft-partitioning: Use the raw value directly. - memnum = int(memnums) - } else { - m, _ := dev.trimMemory(memnums) + if isCoreRequested { + // Soft-partitioning: Use the raw value directly. + memnum = int(memnums) + } else { + m, _ := dev.trimMemory(memnums) + memnum = int(m) + } memnum = int(m) } } From 0663b976eb70150b171f04afa29eab770c061bec Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Mon, 20 Apr 2026 06:13:17 +0000 Subject: [PATCH 04/13] feat: fix golangci lint Signed-off-by: ashergaga <1214443299@qq.com> --- pkg/device/ascend/device.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 7bcf78149b..61373f8e78 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -331,7 +331,6 @@ func (dev *Devices) GenerateResourceRequests(ctr *corev1.Container) device.Conta m, _ := dev.trimMemory(memnums) memnum = int(m) } - memnum = int(m) } } From b1cce71960edd154512160f82f1d7e8444dc6e56 Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Mon, 20 Apr 2026 06:35:25 +0000 Subject: [PATCH 05/13] feat: fix format Signed-off-by: ashergaga <1214443299@qq.com> --- pkg/device/ascend/device.go | 60 ++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 61373f8e78..9e33010586 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -54,10 +54,10 @@ type Devices struct { } type RuntimeInfo struct { - UUID string `json:"UUID,omitempty"` - Temp string `json:"temp,omitempty"` - Memory int64 `json:"memory,omitempty"` - Core int32 `json:"core,omitempty"` + UUID string `json:"UUID,omitempty"` + Temp string `json:"temp,omitempty"` + Memory int64 `json:"memory,omitempty"` + Core int32 `json:"core,omitempty"` } var ( @@ -119,7 +119,7 @@ func (dev *Devices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, if !ok { return false, nil } - + reqNum := count.Value() if dev.config.CommonWord == Ascend910CType { if reqNum == 1 { @@ -140,29 +140,29 @@ func (dev *Devices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, // Check if hami-core is declared vnpuMode := p.Annotations[VNPUModeAnnotation] - isHAMiCore := (vnpuMode == VNPUModeHamiCore) + isHAMiCore := (vnpuMode == VNPUModeHamiCore) if isHAMiCore { - klog.V(3).Infof("Ascend core resource detected, injecting postStart lifecycle for container %s", ctr.Name) - - if ctr.Lifecycle == nil { - ctr.Lifecycle = &corev1.Lifecycle{} - } - - // Inject PostStart hook to start the limiter process - if ctr.Lifecycle.PostStart == nil { - ctr.Lifecycle.PostStart = &corev1.LifecycleHandler{ - Exec: &corev1.ExecAction{ - Command: []string{ - "bash", - "-c", - "export RUST_LOG=info\n/hami-vnpu-core/limiter > /tmp/limiter_manager.log 2>&1 &", - }, - }, - } - } - } - + klog.V(3).Infof("Ascend core resource detected, injecting postStart lifecycle for container %s", ctr.Name) + + if ctr.Lifecycle == nil { + ctr.Lifecycle = &corev1.Lifecycle{} + } + + // Inject PostStart hook to start the limiter process + if ctr.Lifecycle.PostStart == nil { + ctr.Lifecycle.PostStart = &corev1.LifecycleHandler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "bash", + "-c", + "export RUST_LOG=info\n/hami-vnpu-core/limiter > /tmp/limiter_manager.log 2>&1 &", + }, + }, + } + } + } + trimMem := dev.config.MemoryAllocatable memory, ok := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceMemoryName)] if ok { @@ -231,7 +231,7 @@ func (dev *Devices) PatchAnnotations(pod *corev1.Pod, annoInput *map[string]stri _, temp := dev.trimMemory(int64(val.Usedmem)) info.Temp = temp } - + rtInfo = append(rtInfo, info) } } @@ -308,7 +308,7 @@ func (dev *Devices) GenerateResourceRequests(ctr *corev1.Container) device.Conta mem, ok = ctr.Resources.Requests[ascendResourceMem] } if ok { - memnums, _ := mem.AsInt64() + memnums, ok := mem.AsInt64() if ok { if dev.config.MemoryFactor > 1 { rawMemnums := memnums @@ -333,7 +333,7 @@ func (dev *Devices) GenerateResourceRequests(ctr *corev1.Container) device.Conta } } } - + // Process Core Resources corenum := int32(0) if ascendResourceCore != "" { @@ -419,7 +419,7 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD originReq := k.Nums prevnuma := -1 klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k) - + vnpuMode := "" if pod != nil && pod.Annotations != nil { vnpuMode = pod.Annotations[VNPUModeAnnotation] From b8f8ff8db7d7c775fe39af0efd4420b82ef1115b Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Mon, 20 Apr 2026 08:00:21 +0000 Subject: [PATCH 06/13] feat: add unit test & proposal Signed-off-by: ashergaga <1214443299@qq.com> --- docs/ascend910-hami-vnpu-core-support.md | 4 +- docs/ascend910-hami-vnpu-core-support_cn.md | 4 +- ...ng_hami_vnpu_core_for_virtualization_cn.md | 174 +++++++++++++++++ pkg/device/ascend/device_test.go | 184 ++++++++++++++++++ 4 files changed, 362 insertions(+), 4 deletions(-) create mode 100644 docs/proposals/integrating_hami_vnpu_core_for_virtualization_cn.md diff --git a/docs/ascend910-hami-vnpu-core-support.md b/docs/ascend910-hami-vnpu-core-support.md index 4b7755fe9f..7ce3ebe250 100644 --- a/docs/ascend910-hami-vnpu-core-support.md +++ b/docs/ascend910-hami-vnpu-core-support.md @@ -26,8 +26,8 @@ Before launching any containers, the **Global Shared Memory (SHM) Region** must #### 1. Create the Shared Directory ```bash -sudo mkdir -p /tmp/hami-shared-region -sudo chmod 777 /tmp/hami-shared-region +sudo mkdir -p /usr/local/hami-shared-region +sudo chmod 777 /usr/local/hami-shared-region ``` #### 2. Deploy hami-vnpu-core Components diff --git a/docs/ascend910-hami-vnpu-core-support_cn.md b/docs/ascend910-hami-vnpu-core-support_cn.md index 3e867d0f14..435cdd407a 100644 --- a/docs/ascend910-hami-vnpu-core-support_cn.md +++ b/docs/ascend910-hami-vnpu-core-support_cn.md @@ -28,8 +28,8 @@ 1. **创建共享目录** ``` - sudo mkdir -p /tmp/hami-shared-region - sudo chmod 777 /tmp/hami-shared-region + sudo mkdir -p /usr/local/hami-shared-region + sudo chmod 777 /usr/local/hami-shared-region ``` 2. **部署 hami-vnpu-core 组件** diff --git a/docs/proposals/integrating_hami_vnpu_core_for_virtualization_cn.md b/docs/proposals/integrating_hami_vnpu_core_for_virtualization_cn.md new file mode 100644 index 0000000000..4432fecbb8 --- /dev/null +++ b/docs/proposals/integrating_hami_vnpu_core_for_virtualization_cn.md @@ -0,0 +1,174 @@ +HAMi-vnpu-core 是一个Rust语言编写的昇腾 NPU 容器内资源控制器,通过 libvnpu.so(拦截器) + Limiter(管理器)实现用户态拦截,通过NPU_MEM_QUOTA 和 NPU_PRIORITY 两个环境变量分别声明显存限制和调度优先级。本方案在 HAMi 调度中集成该能力,以支持昇腾 NPU 的显存虚拟化与算力时间片软切分。 + +## 一、前置条件 + +昇腾驱动版本25.5以上,芯片开启 device-share 模式: + +#### 功能说明 + +**npu-smi set -t device-share -i** *id* **-d** value 用于设置指定设备的所有芯片的容器共享模式。 + +#### 参数说明 + +| 类型 | 描述 | +| ------- | ----------------------------------------------------------- | +| *id* | 设备ID。通过**npu-smi info -l**命令查出的NPU ID即为设备ID。 | +| *value* | 容器使能状态:分为禁用、使能。默认禁用。0:禁用 1:使能 | + +## 二、HAMi-scheduler改动点 + +### 2\.1 扩展资源名 + +复用 huawei.com/Ascend910B3-memory :显存分配逻辑可复用原逻辑; + +新增 huawei.com/Ascend910B3-core:声明了该资源名的请求走vnpu软切分,否则走原硬切分逻辑。 +| 资源名称 | 单位 | 含义 | 示例 | +|--------------|----------|----------|----------| +| huawei.com/Ascend910B3 | 整数 | NPU卡数 | 1 | +| huawei.com/Ascend910B3-memory | Mi | 显存配额 | 28672 (28Gi) | +| huawei.com/Ascend910B3-core | 整数 | 百分比 | 20, 40 | + + + +### 2\.2 Filter阶段 + +**Filter阶段,**修改 **Fit** 函数核心逻辑,确保单张卡总算力不超过 100;同时修改 PatchAnnotation 注入的注解。 + +**Pod配额预期分配结果格式(Pod Annotation):** + +```shell {"data-theme":"githubLight"} +{ + "huawei.com/Ascend910B3": "[ + { + "UUID":"xxx", + "memory": 28672, + "core": 20, + } + ]" +} +``` + +**PatchAnnotation**判断逻辑 vnpu软切分新增显存(memory)和核心(core)字段 + +```go {"data-title":"pkg/device/ascend/device.go","data-theme":"githubLight"} +func (dev *Devices) PatchAnnotations(pod *corev1.Pod, annoInput *map[string]string, pd device.PodDevices) map[string]string { + commonWord := dev.CommonWord() + devList, ok := pd[commonWord] + if ok && len(devList) > 0 { + ... + for _, dp := range devList { + for _, val := range dp { + ... + + // 解析 Pod 的memQuota、 priority 资源声明 构建 RuntimeInfo + rtInfo = append(rtInfo, RuntimeInfo{ + UUID: val.UUID, + Temp: tempName, + MemQuota: memory, + Priority: core, + }) + } + } + ... + } + return *annoInput +} +``` + +### 2\.3 容器limiter进程启动 + +通过K8S生命周期钩子 PostStart 执行自定义动作,启动limiter进程。pkg\\device\\ascend\\device.go **MutateAdmission**注入 postStart 钩子 + +```ruby {"data-theme":"githubLight"} +lifecycle: + postStart: + exec: + command: + - "bash" + - "-c" + - | + export RUST_LOG=info + /usr/local/hami-vnpu-core/limiter > /usr/local/hami-vnpu-core/inst1_manager.log 2>&1 & +``` + +**libvnpu.so改动点:**因为 postStart 无法保证在entrypoint之前完成。在业务真正开启分配内存或启动算力之前,**循环检测确保 lilmiter已经准备就绪**。 + +```ruby {"data-theme":"githubLight"} +impl SchedulerClient { + pub fn new() -> Self { + let pid = std::process::id(); + let shmem_name = local_shmem_name(); + + // ================== 新增:自动化等待逻辑 ================== + let shm_path = format!("/dev/shm/{}", shmem_name); + let mut retry_count = 0; + + info!("[Worker PID:{}] Checking for limiter at {}...", pid, shm_path); + + while !std::path::Path::new(&shm_path).exists() { + if retry_count % 50 == 0 { // 每 5 秒提醒一次 + warn!("[Worker PID:{}] Still waiting for limiter process to create shared memory...", pid); + } + + std::thread::sleep(std::time::Duration::from_millis(100)); + retry_count += 1; + + // 如果 1 分钟还没起来就崩溃 + if retry_count > 600 { + panic!("[Scheduler] FATAL: Limiter not found after 60 seconds."); + } + } + info!("[Worker PID:{}] Limiter detected. Connecting to shared memory.", pid); + // ======================================================== + + // 此时文件一定存在了,安全打开 + let shmem = shmem::shm_setup::open_shmem::(shmem_name.as_str()); + } +} +``` + +## 三、hami-ascend-device-plugin改动点 + +### 3\.1 宿主机固定路径存放limiter和libvnpu.so + +宿主机固定路径存放limiter和libvnpu.so,为后续映射limiter和libvnpu.so到容器内准备: + +```shell {"data-theme":"githubLight"} +/usr/local/hami-vnpu-core/ +├── limiter +└── libvnpu.so +└── ld.so.preload // 动态链接预加载配置文件 +``` + +**ld.so.preload文件内容** + +```plain-text {"data-theme":"githubLight"} +/hami-vnpu-core/target/debug/libvnpu.so +``` + +### 3\.2 宿主机创建共享内存区域 + +```shell {"data-theme":"githubLight"} +sudo mkdir -p /usr/local/hami-shared-region +sudo chmod 777 /usr/local/hami-shared-region +``` + +### 3\.3 Allocate函数增强 + +```go {"data-theme":"githubLight"} +func (ps *PluginServer) Allocate(ctx context.Context, reqs *v1beta1.AllocateRequest) + /* + 1. (可省略)处理设备注入 可见设备环境变量设置后 设备文件会自动注入) + 2. 处理存储卷注入 (-v) + A. 华为驱动与 SMI 工具链 + B. vnpu-core代码路径 : /usr/local/hami-vnpu-core + C. 注入 HAMi 劫持库路径 (对应脚本中的 LIBRARY_PATH),通过/etc/ld.so.preload 挂载 + D. HAMi 算力切分专用共享目录 : /usr/local/hami-shared-region:/hami-shared-region + 3. 环境变量注入 + A. 可见设备IDs + B. 共享内存通信路径:NPU_GLOBAL_SHM_PATH = /hami-shared-region/{ID}_global_registry + C. 显存配额NPU_MEM_QUOTA:从Annotation读取 memory = 28672 + D. 优先级NPU_PRIORITY:从Annotation读取 core = 20 + */ +} +``` \ No newline at end of file diff --git a/pkg/device/ascend/device_test.go b/pkg/device/ascend/device_test.go index fc9e59da84..013c1d6bdf 100644 --- a/pkg/device/ascend/device_test.go +++ b/pkg/device/ascend/device_test.go @@ -345,6 +345,62 @@ func Test_PatchAnnotations(t *testing.T) { } } +func Test_PatchAnnotations_VNPUCoreMode(t *testing.T) { + dev := Devices{ + config: VNPUConfig{ + CommonWord: "Ascend910B3", + ResourceName: "huawei.com/Ascend910B3", + }, + } + + tests := []struct { + name string + pod *corev1.Pod + pd device.PodDevices + want map[string]string + }{ + { + name: "vNPU-mode patch: check json contains cores and memory instead of template", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + }, + pd: device.PodDevices{ + "Ascend910B3": device.PodSingleDevice{ + []device.ContainerDevice{ + { + Idx: 0, + UUID: "ascend-uuid-1", + Type: "Ascend", + Usedcores: 5, + Usedmem: 8192, + }, + }, + }, + }, + want: map[string]string{ + "huawei.com/Ascend910B3": "[{\"UUID\":\"ascend-uuid-1\",\"core\":5,\"memory\":8192}]", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + annoInput := make(map[string]string) + result := dev.PatchAnnotations(test.pod, &annoInput, test.pd) + + val, ok := result["huawei.com/Ascend910B3"] + assert.Assert(t, ok) + assert.Assert(t, strings.Contains(val, "\"core\":5")) + assert.Assert(t, strings.Contains(val, "\"memory\":8192")) + assert.Assert(t, !strings.Contains(val, "\"temp\"")) + }) + } +} + func Test_checkType(t *testing.T) { dev := Devices{ config: VNPUConfig{ @@ -739,6 +795,88 @@ func Test_MutateAdmission910C(t *testing.T) { } } +func Test_MutateAdmission_VNPUCoreMode(t *testing.T) { + const VNPUModeAnnotation = "huawei.com/vnpu-mode" + const VNPUModeHamiCore = "hami-core" + + tests := []struct { + name string + args struct { + ctr corev1.Container + pod corev1.Pod + } + wantPostStart bool + wantMem int64 + wantCore int64 + }{ + { + name: "vNPU-mode hami-core: inject postStart and keep raw memory", + args: struct { + ctr corev1.Container + pod corev1.Pod + }{ + ctr: corev1.Container{ + Name: "test-container", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "huawei.com/Ascend910B3": resource.MustParse("1"), + "huawei.com/Ascend910B3-memory": resource.MustParse("15360"), + "huawei.com/Ascend910B3-core": resource.MustParse("20"), + }, + Requests: corev1.ResourceList{ + "huawei.com/Ascend910B3": resource.MustParse("1"), + "huawei.com/Ascend910B3-memory": resource.MustParse("15360"), + "huawei.com/Ascend910B3-core": resource.MustParse("20"), + }, + }, + }, + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + VNPUModeAnnotation: VNPUModeHamiCore, + }, + }, + }, + }, + wantPostStart: true, + wantMem: 15360, + wantCore: 20, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dev := Devices{ + config: VNPUConfig{ + CommonWord: "Ascend910B3", + ResourceName: "huawei.com/Ascend910B3", + ResourceMemoryName: "huawei.com/Ascend910B3-memory", + ResourceCoreName: "huawei.com/Ascend910B3-core", + MemoryAllocatable: 32768, + }, + } + + ok, err := dev.MutateAdmission(&test.args.ctr, &test.args.pod) + assert.NilError(t, err) + assert.Equal(t, ok, true) + + if test.wantPostStart { + assert.Assert(t, test.args.ctr.Lifecycle != nil, "Lifecycle should not be nil") + assert.Assert(t, test.args.ctr.Lifecycle.PostStart != nil, "PostStart should not be nil") + assert.Assert(t, test.args.ctr.Lifecycle.PostStart.Exec != nil) + commandStr := strings.Join(test.args.ctr.Lifecycle.PostStart.Exec.Command, " ") + assert.Assert(t, strings.Contains(commandStr, "/hami-vnpu-core/limiter")) + } + + memLimit := test.args.ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceMemoryName)] + assert.Equal(t, memLimit.Value(), test.wantMem) + + coreLimit := test.args.ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceCoreName)] + assert.Equal(t, coreLimit.Value(), test.wantCore) + }) + } +} + func Test_GenerateResourceRequests(t *testing.T) { tests := []struct { name string @@ -835,6 +973,52 @@ func Test_GenerateResourceRequests(t *testing.T) { } } +func Test_GenerateResourceRequests_VNPUCoreMode(t *testing.T) { + tests := []struct { + name string + args corev1.Container + want device.ContainerDeviceRequest + }{ + { + name: "parse custom vNPU core and memory resources", + args: corev1.Container{ + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "huawei.com/Ascend910B3": resource.MustParse("1"), + "huawei.com/Ascend910B3-core": resource.MustParse("10"), + "huawei.com/Ascend910B3-memory": resource.MustParse("15360"), + }, + }, + }, + want: device.ContainerDeviceRequest{ + Nums: int32(1), + Type: "Ascend910B3", + Memreq: int32(15360), + Coresreq: int32(10), + MemPercentagereq: int32(0), + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dev := Devices{ + config: VNPUConfig{ + CommonWord: "Ascend910B3", + ResourceName: "huawei.com/Ascend910B3", + ResourceCoreName: "huawei.com/Ascend910B3-core", + ResourceMemoryName: "huawei.com/Ascend910B3-memory", + }, + } + result := dev.GenerateResourceRequests(&test.args) + + assert.Equal(t, result.Memreq, test.want.Memreq) + assert.Equal(t, result.Coresreq, test.want.Coresreq) + assert.Equal(t, result.Type, test.want.Type) + }) + } +} + func Test_GenerateResourceRequestsFactor(t *testing.T) { req := corev1.Container{ Resources: corev1.ResourceRequirements{ From 1be08037b55e764e425d5311e60cc6fdc72e87fd Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Tue, 21 Apr 2026 02:46:27 +0000 Subject: [PATCH 07/13] feat: hami-vnpu-core ReadMe modify Signed-off-by: ashergaga <1214443299@qq.com> --- docs/ascend910-hami-vnpu-core-support.md | 22 --------------------- docs/ascend910-hami-vnpu-core-support_cn.md | 22 --------------------- 2 files changed, 44 deletions(-) diff --git a/docs/ascend910-hami-vnpu-core-support.md b/docs/ascend910-hami-vnpu-core-support.md index 7ce3ebe250..bdd344e5da 100644 --- a/docs/ascend910-hami-vnpu-core-support.md +++ b/docs/ascend910-hami-vnpu-core-support.md @@ -19,28 +19,6 @@ Implements a soft slicing mechanism based on `libvnpu.so` interception and `lim * [ascend-device-plugin](https://github.com/Project-HAMi/ascend-device-plugin) * **Chip Mode**: enable `device-share` mode on Ascend chips for virtualization -### Host Environment Preparation - -Before launching any containers, the **Global Shared Memory (SHM) Region** must be initialized on the host to allow inter-Pod coordination. - -#### 1. Create the Shared Directory - -```bash -sudo mkdir -p /usr/local/hami-shared-region -sudo chmod 777 /usr/local/hami-shared-region -``` - -#### 2. Deploy hami-vnpu-core Components - -Place the following files in a fixed host path (`/usr/local/hami-vnpu-core/`) for mounting into containers: - -``` -/usr/local/hami-vnpu-core/ -├── limiter # Manager daemon binary (compiled from hami-vnpu-core) -├── libvnpu.so # Interception library for LD_PRELOAD -└── ld.so.preload # Global preload config -``` - ### Create device-config.yaml The content is as follows: diff --git a/docs/ascend910-hami-vnpu-core-support_cn.md b/docs/ascend910-hami-vnpu-core-support_cn.md index 435cdd407a..7271c99201 100644 --- a/docs/ascend910-hami-vnpu-core-support_cn.md +++ b/docs/ascend910-hami-vnpu-core-support_cn.md @@ -21,28 +21,6 @@ * 部署 [ascend-device-plugin](https://github.com/Project-HAMi/ascend-device-plugin) * 芯片模式:在昇腾芯片上开启 `device-share` 模式以支持虚拟化。 -## 宿主机环境准备 - -在启动任何容器之前,必须在宿主机上初始化 **全局共享内存 (SHM) 区域**,以便进行 Pod 间的协同。 - -1. **创建共享目录** - - ``` - sudo mkdir -p /usr/local/hami-shared-region - sudo chmod 777 /usr/local/hami-shared-region - ``` - -2. **部署 hami-vnpu-core 组件** - - 将以下文件放置在固定的宿主机路径(`/usr/local/hami-vnpu-core/`)中,以便挂载到容器内: - - ``` - /usr/local/hami-vnpu-core/ - ├── limiter # Manager daemon binary (compiled from hami-vnpu-core) - ├── libvnpu.so # Interception library for LD_PRELOAD - └── ld.so.preload # Global preload config - ``` - ### 在 files 目录下创建 device-config.yaml 配置文件如下所示,可以按需调整: From 3b26978a42ebefa3c2ff008694016c9d810e4d6b Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Mon, 27 Apr 2026 08:42:32 +0000 Subject: [PATCH 08/13] feat: add failureReason ModeNotFit Signed-off-by: ashergaga <1214443299@qq.com> --- pkg/device/ascend/device.go | 37 +++++++++++++++++++++++++++++++++---- pkg/device/common/common.go | 1 + 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 7e0765e0da..10d61a6d91 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -43,6 +43,7 @@ const ( Ascend910NetworkWeight = 10 VNPUModeAnnotation = "huawei.com/vnpu-mode" VNPUModeHamiCore = "hami-core" + VNPUNodeSelectorAnnotation = "hami-vnpu-core" ) type Devices struct { @@ -430,16 +431,44 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD originReq := k.Nums prevnuma := -1 klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k) + var tmpDevs map[string]device.ContainerDevices + tmpDevs = make(map[string]device.ContainerDevices) + reason := make(map[string]int) vnpuMode := "" if pod != nil && pod.Annotations != nil { vnpuMode = pod.Annotations[VNPUModeAnnotation] } + + + isHAMiCore := (vnpuMode == VNPUModeHamiCore) + + // Werify whether the Node supports hami vnpu core + nodeSupportHamiCore := false + if nodeInfo.Node.Annotations != nil { + if val, ok := nodeInfo.Node.Annotations[VNPUNodeSelectorAnnotation]; ok && val == "true" { + nodeSupportHamiCore = true + } + } + + var totalMemPerCard int32 = 0 + if len(devices) > 0 { + totalMemPerCard = devices[0].Totalmem + } + + if request.Memreq > 0 && request.Memreq < totalMemPerCard && request.Nums > 0 { + if !nodeSupportHamiCore && isHAMiCore { + reason[common.ModeNotFit]++ + klog.V(4).InfoS("Node filtered: Node does not support hami-core mode", "node", nodeInfo.Node.Name, "pod", pod.Name) + return false, nil, common.GenReason(reason, len(devices)) + } else if nodeSupportHamiCore && !isHAMiCore { + reason[common.ModeNotFit]++ + klog.V(4).InfoS("Node filtered: Reserved for hami-core but pod is legacy vNPU", "node", nodeInfo.Node.Name, "pod", pod.Name) + return false, nil, common.GenReason(reason, len(devices)) + } + } klog.V(4).InfoS("Fit: vnpu-mode annotation", "pod", pod.Name, "vnpuMode", vnpuMode) - var tmpDevs map[string]device.ContainerDevices - tmpDevs = make(map[string]device.ContainerDevices) - reason := make(map[string]int) needTopology := false if strings.HasPrefix(npu.CommonWord(), Ascend910Prefix) && hasNetworkID(devices) { klog.V(4).Infof("all devices have NetworkID. device CommonWord %s", npu.CommonWord()) @@ -495,7 +524,7 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD } // Set dev.Totalcore to 100 if vnpuMode is hami-core effectiveTotalCore := dev.Totalcore - if vnpuMode == VNPUModeHamiCore { + if isHAMiCore { effectiveTotalCore = 100 } diff --git a/pkg/device/common/common.go b/pkg/device/common/common.go index 65410143b0..c14ec66778 100644 --- a/pkg/device/common/common.go +++ b/pkg/device/common/common.go @@ -37,6 +37,7 @@ const ( NodeUnfitPod = "NodeUnfitPod" NodeFitPod = "NodeFitPod" ResourceQuotaNotFit = "ResourceQuotaNotFit" + ModeNotFit = "ModeNotFit" ) func GenReason(reasons map[string]int, cards int) string { From dd90ef315dc4d31968df702610b5fffacfcee647 Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Tue, 28 Apr 2026 08:02:50 +0000 Subject: [PATCH 09/13] feat: fix ai suggestion Signed-off-by: ashergaga <1214443299@qq.com> --- pkg/device/ascend/device.go | 19 ++++++++----------- pkg/device/common/common.go | 2 +- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 10d61a6d91..0b148a8aec 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -439,16 +439,13 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD if pod != nil && pod.Annotations != nil { vnpuMode = pod.Annotations[VNPUModeAnnotation] } - isHAMiCore := (vnpuMode == VNPUModeHamiCore) - // Werify whether the Node supports hami vnpu core + // Verify whether the Node supports hami vnpu core nodeSupportHamiCore := false if nodeInfo.Node.Annotations != nil { - if val, ok := nodeInfo.Node.Annotations[VNPUNodeSelectorAnnotation]; ok && val == "true" { - nodeSupportHamiCore = true - } + nodeSupportHamiCore = nodeInfo.Node.Annotations[VNPUNodeSelectorAnnotation] == "true" } var totalMemPerCard int32 = 0 @@ -457,13 +454,13 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD } if request.Memreq > 0 && request.Memreq < totalMemPerCard && request.Nums > 0 { - if !nodeSupportHamiCore && isHAMiCore { + if nodeSupportHamiCore != isHAMiCore { reason[common.ModeNotFit]++ - klog.V(4).InfoS("Node filtered: Node does not support hami-core mode", "node", nodeInfo.Node.Name, "pod", pod.Name) - return false, nil, common.GenReason(reason, len(devices)) - } else if nodeSupportHamiCore && !isHAMiCore { - reason[common.ModeNotFit]++ - klog.V(4).InfoS("Node filtered: Reserved for hami-core but pod is legacy vNPU", "node", nodeInfo.Node.Name, "pod", pod.Name) + logMsg := "Node filtered: Reserved for hami-core but pod is legacy vNPU" + if isHAMiCore { + logMsg = "Node filtered: Node does not support hami-core mode" + } + klog.V(4).InfoS(logMsg, "node", nodeInfo.Node.Name, "pod", pod.Name) return false, nil, common.GenReason(reason, len(devices)) } } diff --git a/pkg/device/common/common.go b/pkg/device/common/common.go index c14ec66778..6405dc8dc7 100644 --- a/pkg/device/common/common.go +++ b/pkg/device/common/common.go @@ -37,7 +37,7 @@ const ( NodeUnfitPod = "NodeUnfitPod" NodeFitPod = "NodeFitPod" ResourceQuotaNotFit = "ResourceQuotaNotFit" - ModeNotFit = "ModeNotFit" + ModeNotFit = "ModeNotFit" ) func GenReason(reasons map[string]int, cards int) string { From a6c1449a785d113565d6f97a27b6fb83e3076668 Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Tue, 28 Apr 2026 08:10:06 +0000 Subject: [PATCH 10/13] feat: fix cilint Signed-off-by: ashergaga <1214443299@qq.com> --- pkg/device/ascend/device.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 0b148a8aec..c612bdf607 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -37,12 +37,12 @@ import ( ) const ( - NodeLockAscend = "hami.io/mutex.lock" - Ascend910Prefix = "Ascend910" - Ascend910CType = "Ascend910C" - Ascend910NetworkWeight = 10 - VNPUModeAnnotation = "huawei.com/vnpu-mode" - VNPUModeHamiCore = "hami-core" + NodeLockAscend = "hami.io/mutex.lock" + Ascend910Prefix = "Ascend910" + Ascend910CType = "Ascend910C" + Ascend910NetworkWeight = 10 + VNPUModeAnnotation = "huawei.com/vnpu-mode" + VNPUModeHamiCore = "hami-core" VNPUNodeSelectorAnnotation = "hami-vnpu-core" ) @@ -439,10 +439,10 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD if pod != nil && pod.Annotations != nil { vnpuMode = pod.Annotations[VNPUModeAnnotation] } - + isHAMiCore := (vnpuMode == VNPUModeHamiCore) - // Verify whether the Node supports hami vnpu core + // Verify whether the Node supports hami vnpu core nodeSupportHamiCore := false if nodeInfo.Node.Annotations != nil { nodeSupportHamiCore = nodeInfo.Node.Annotations[VNPUNodeSelectorAnnotation] == "true" @@ -454,7 +454,7 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD } if request.Memreq > 0 && request.Memreq < totalMemPerCard && request.Nums > 0 { - if nodeSupportHamiCore != isHAMiCore { + if nodeSupportHamiCore != isHAMiCore { reason[common.ModeNotFit]++ logMsg := "Node filtered: Reserved for hami-core but pod is legacy vNPU" if isHAMiCore { From 2936d0060164c8787efae572c68910ea37aecc59 Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Tue, 28 Apr 2026 08:41:00 +0000 Subject: [PATCH 11/13] feat: fix unit test error Signed-off-by: ashergaga <1214443299@qq.com> --- pkg/device/ascend/device.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index c612bdf607..c4d03ce380 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -444,7 +444,8 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD // Verify whether the Node supports hami vnpu core nodeSupportHamiCore := false - if nodeInfo.Node.Annotations != nil { + + if nodeInfo != nil && nodeInfo.Node != nil && nodeInfo.Node.Annotations != nil { nodeSupportHamiCore = nodeInfo.Node.Annotations[VNPUNodeSelectorAnnotation] == "true" } @@ -454,13 +455,13 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD } if request.Memreq > 0 && request.Memreq < totalMemPerCard && request.Nums > 0 { - if nodeSupportHamiCore != isHAMiCore { + if !nodeSupportHamiCore && isHAMiCore { reason[common.ModeNotFit]++ - logMsg := "Node filtered: Reserved for hami-core but pod is legacy vNPU" - if isHAMiCore { - logMsg = "Node filtered: Node does not support hami-core mode" - } - klog.V(4).InfoS(logMsg, "node", nodeInfo.Node.Name, "pod", pod.Name) + klog.V(4).InfoS("Node filtered: Node does not support hami-core mode", "node", nodeInfo.Node.Name, "pod", pod.Name) + return false, nil, common.GenReason(reason, len(devices)) + } else if nodeSupportHamiCore && !isHAMiCore { + reason[common.ModeNotFit]++ + klog.V(4).InfoS("Node filtered: Reserved for hami-core but pod is legacy vNPU", "node", nodeInfo.Node.Name, "pod", pod.Name) return false, nil, common.GenReason(reason, len(devices)) } } From 27bef0df9f5601f863a33fa2e876fce0e7341122 Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Tue, 28 Apr 2026 09:25:18 +0000 Subject: [PATCH 12/13] feat: add unit test Signed-off-by: ashergaga <1214443299@qq.com> --- pkg/device/ascend/device_test.go | 58 +++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/pkg/device/ascend/device_test.go b/pkg/device/ascend/device_test.go index 013c1d6bdf..ac2616710a 100644 --- a/pkg/device/ascend/device_test.go +++ b/pkg/device/ascend/device_test.go @@ -1385,14 +1385,15 @@ func TestDevices_Fit(t *testing.T) { devs := InitDevices(config) tests := []struct { - name string - devices []*device.DeviceUsage - request device.ContainerDeviceRequest - annos map[string]string - wantFit bool - wantLen int - wantDevIDs []string - wantReason string + name string + devices []*device.DeviceUsage + request device.ContainerDeviceRequest + annos map[string]string + nodeAnnotation map[string]string + wantFit bool + wantLen int + wantDevIDs []string + wantReason string }{ { name: "fit success", @@ -1776,6 +1777,42 @@ func TestDevices_Fit(t *testing.T) { wantDevIDs: []string{"dev-2", "dev-1"}, wantReason: "", }, + { + name: "fit fail: hami-core pod on legacy node (ModeNotFit)", + devices: []*device.DeviceUsage{{ + ID: "dev-0", Index: 0, Used: 0, Count: 100, + Usedmem: 0, Totalmem: 32768, Totalcore: 100, Usedcores: 0, + Numa: 0, Health: true, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, Memreq: 15360, MemPercentagereq: 0, Coresreq: 20, + }, + annos: map[string]string{ + VNPUModeAnnotation: VNPUModeHamiCore, + }, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 ModeNotFit", + nodeAnnotation: map[string]string{}, + }, + { + name: "fit fail: legacy pod on hami-core reserved node (ModeNotFit)", + devices: []*device.DeviceUsage{{ + ID: "dev-0", Index: 0, Used: 0, Count: 100, + Usedmem: 0, Totalmem: 32768, Totalcore: 100, Usedcores: 0, + Numa: 0, Health: true, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, Memreq: 8738, MemPercentagereq: 0, Coresreq: 0, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 ModeNotFit", + nodeAnnotation: map[string]string{VNPUNodeSelectorAnnotation: "true"}, + }, } for _, dev := range devs { @@ -1802,6 +1839,11 @@ func TestDevices_Fit(t *testing.T) { } nodeInfo := &device.NodeInfo{ ID: "node1", + Node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: test.nodeAnnotation, + }, + }, Devices: map[string][]device.DeviceInfo{ dev.config.CommonWord: { { From 73e6b6edc0f993fb113e550bdd9cf331b8624699 Mon Sep 17 00:00:00 2001 From: ashergaga <1214443299@qq.com> Date: Tue, 28 Apr 2026 09:31:28 +0000 Subject: [PATCH 13/13] feat: fix cilint Signed-off-by: ashergaga <1214443299@qq.com> --- pkg/device/ascend/device_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/device/ascend/device_test.go b/pkg/device/ascend/device_test.go index ac2616710a..da7588141c 100644 --- a/pkg/device/ascend/device_test.go +++ b/pkg/device/ascend/device_test.go @@ -1804,7 +1804,7 @@ func TestDevices_Fit(t *testing.T) { Numa: 0, Health: true, }}, request: device.ContainerDeviceRequest{ - Nums: 1, Memreq: 8738, MemPercentagereq: 0, Coresreq: 0, + Nums: 1, Memreq: 8738, MemPercentagereq: 0, Coresreq: 0, }, annos: map[string]string{}, wantFit: false,