Skip to content
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 6 additions & 26 deletions pkg/device/cambricon/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,18 @@
package cambricon

import (
"context"
"encoding/json"
"flag"
"fmt"
"math/rand"
"slices"
"strings"
"time"

"github.com/Project-HAMi/HAMi/pkg/device"
"github.com/Project-HAMi/HAMi/pkg/device/common"
"github.com/Project-HAMi/HAMi/pkg/util"
"github.com/Project-HAMi/HAMi/pkg/util/client"
"github.com/Project-HAMi/HAMi/pkg/util/nodelock"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/klog/v2"
)

Expand All @@ -50,10 +44,12 @@
MLUUseUUID = "cambricon.com/use-gpuuuid"
// MLUNoUseUUID annotation specifies a comma-separated list of MLU UUIDs to exclude.
MLUNoUseUUID = "cambricon.com/nouse-gpuuuid"
DsmluLockTime = "cambricon.com/dsmlu.lock"
DsmluProfile = "CAMBRICON_DSMLU_PROFILE"
DsmluResourceAssigned = "CAMBRICON_DSMLU_ASSIGNED"
retry = 5
// NodeLockCambricon should be the same as the node lock name used by the
// device plugin; nodelock hard-codes the annotation key regardless of the
// name passed in, so this only needs to be distinct for readability.
NodeLockCambricon = "hami.io/mutex.lock"
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
// MemoryFactor converts the vmemory unit used in the pod spec into the MiB
// HAMi accounts internally. One cambricon.com/mlu.smlu.vmemory is 256 MiB.
MemoryFactor = 256
Expand Down Expand Up @@ -97,21 +93,21 @@
}

func (dev *CambriconDevices) setNodeLock(node *corev1.Node) error {
ctx := context.Background()

Check failure on line 96 in pkg/device/cambricon/device.go

View workflow job for this annotation

GitHub Actions / lint

undefined: context
patchedAnnotation, err := json.Marshal(

Check failure on line 97 in pkg/device/cambricon/device.go

View workflow job for this annotation

GitHub Actions / lint

undefined: json
map[string]any{
"metadata": map[string]map[string]string{"annotations": {
DsmluLockTime: time.Now().Format(time.RFC3339),

Check failure on line 100 in pkg/device/cambricon/device.go

View workflow job for this annotation

GitHub Actions / lint

undefined: DsmluLockTime
}}})
if err != nil {
klog.ErrorS(err, "Failed to marshal node annotation", "node", node.Name)
return fmt.Errorf("marshal node annotation: %w", err)
}

_, err = client.GetClient().CoreV1().Nodes().Patch(ctx, node.Name, types.MergePatchType, patchedAnnotation, metav1.PatchOptions{})

Check failure on line 107 in pkg/device/cambricon/device.go

View workflow job for this annotation

GitHub Actions / lint

undefined: metav1

Check failure on line 107 in pkg/device/cambricon/device.go

View workflow job for this annotation

GitHub Actions / lint

undefined: types

Check failure on line 107 in pkg/device/cambricon/device.go

View workflow job for this annotation

GitHub Actions / lint

undefined: client
for i := 0; i < retry && err != nil; i++ {

Check failure on line 108 in pkg/device/cambricon/device.go

View workflow job for this annotation

GitHub Actions / lint

undefined: retry
klog.ErrorS(err, "Failed to patch node annotation", "node", node.Name, "retry", i)
time.Sleep(time.Duration(rand.Intn(i+1)) * 10 * time.Millisecond)

Check failure on line 110 in pkg/device/cambricon/device.go

View workflow job for this annotation

GitHub Actions / lint

undefined: time

Check failure on line 110 in pkg/device/cambricon/device.go

View workflow job for this annotation

GitHub Actions / lint

undefined: rand

Check failure on line 110 in pkg/device/cambricon/device.go

View workflow job for this annotation

GitHub Actions / lint

undefined: time
_, err = client.GetClient().CoreV1().Nodes().Patch(ctx, node.Name, types.MergePatchType, patchedAnnotation, metav1.PatchOptions{})
}
if err != nil {
Expand All @@ -132,23 +128,7 @@
if !found {
return nil
}
if _, ok := n.Annotations[DsmluLockTime]; !ok {
return dev.setNodeLock(n)
}
lockTime, err := time.Parse(time.RFC3339, n.Annotations[DsmluLockTime])
if err != nil {
return err
}
if time.Since(lockTime) > time.Minute*2 {
klog.InfoS("Node lock expired", "node", n.Name, "lockTime", lockTime)
err = dev.ReleaseNodeLock(n, p)
if err != nil {
klog.ErrorS(err, "Failed to release node lock", "node", n.Name)
return err
}
return dev.setNodeLock(n)
}
return fmt.Errorf("node %s has been locked within 2 minutes", n.Name)
return nodelock.LockNode(n.Name, NodeLockCambricon, p)
}

func (dev *CambriconDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error {
Expand Down
80 changes: 72 additions & 8 deletions pkg/device/cambricon/device_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (

"github.com/Project-HAMi/HAMi/pkg/device"
"github.com/Project-HAMi/HAMi/pkg/util/client"
"github.com/Project-HAMi/HAMi/pkg/util/nodelock"
)

func Test_GetNodeDevices(t *testing.T) {
Expand Down Expand Up @@ -491,9 +492,17 @@ func setupTest(t *testing.T) (*corev1.Node, *corev1.Pod, func(), *fake.Clientset
}

func Test_LockNode(t *testing.T) {
otherPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "other-pod",
Namespace: "default",
},
}

tests := []struct {
name string
annotations map[string]string
existingPod *corev1.Pod
wantErr bool
}{
{
Expand All @@ -502,23 +511,24 @@ func Test_LockNode(t *testing.T) {
wantErr: false,
},
{
name: "node is already locked within 2 minutes",
name: "node is locked by another live pod within the timeout",
annotations: map[string]string{
DsmluLockTime: time.Now().Add(-time.Minute).Format(time.RFC3339),
nodelock.NodeLockKey: nodelock.GenerateNodeLockKeyByPod(otherPod),
},
wantErr: true,
existingPod: otherPod,
wantErr: true,
},
{
name: "lock time expired (more than 2 minutes)",
name: "lock time expired",
annotations: map[string]string{
DsmluLockTime: time.Now().Add(-time.Hour).Format(time.RFC3339),
nodelock.NodeLockKey: time.Now().Add(-2*nodelock.NodeLockTimeout).Format(time.RFC3339) + nodelock.NodeLockSep + "default" + nodelock.NodeLockSep + "other-pod",
},
wantErr: false,
},
{
name: "invalid lock time format",
annotations: map[string]string{
DsmluLockTime: "invalid-format",
nodelock.NodeLockKey: "invalid-format",
},
wantErr: true,
},
Expand All @@ -532,22 +542,32 @@ func Test_LockNode(t *testing.T) {

// Set up the node with the specified annotations.
node.Annotations = tt.annotations
_, err := clientset.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{})
if err != nil {
t.Fatalf("Failed to update node annotations: %v", err)
}

if tt.existingPod != nil {
if _, err := clientset.CoreV1().Pods(tt.existingPod.Namespace).Create(context.TODO(), tt.existingPod, metav1.CreateOptions{}); err != nil {
t.Fatalf("Failed to create existing pod: %v", err)
}
}

dev := InitMLUDevice(CambriconConfig{
ResourceCountName: MLUResourceCount,
ResourceMemoryName: MLUResourceMemory,
ResourceCoreName: MLUResourceCores,
})

err := dev.LockNode(node, pod)
err = dev.LockNode(node, pod)
if (err != nil) != tt.wantErr {
t.Errorf("LockNode() error = %v, wantErr %v", err, tt.wantErr)
}

// Optionally check if the node was correctly patched with the lock annotation.
if !tt.wantErr {
fetchedNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), node.Name, metav1.GetOptions{})
if _, ok := fetchedNode.Annotations[DsmluLockTime]; !ok && !tt.wantErr {
if _, ok := fetchedNode.Annotations[nodelock.NodeLockKey]; !ok {
t.Error("Expected node to be locked but it wasn't")
}
}
Expand Down Expand Up @@ -778,6 +798,50 @@ func Test_ReleaseNodeLock(t *testing.T) {
})
}

// Test_ReleaseNodeLock_ReleasesOwnedLock verifies that ReleaseNodeLock
// delegates to the shared nodelock package and actually clears the lock
// annotation held by the requesting pod.
func Test_ReleaseNodeLock_ReleasesOwnedLock(t *testing.T) {
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "owner-pod",
Namespace: "default",
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{{
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceName(MLUResourceCount): resource.MustParse("1"),
},
},
}},
},
}
node := &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node-03",
Annotations: map[string]string{
nodelock.NodeLockKey: nodelock.GenerateNodeLockKeyByPod(pod),
},
},
}
client.KubeClient = fake.NewClientset(node)

dev := InitMLUDevice(CambriconConfig{
ResourceCountName: MLUResourceCount,
ResourceMemoryName: MLUResourceMemory,
ResourceCoreName: MLUResourceCores,
})

err := dev.ReleaseNodeLock(node, pod)
assert.NoError(t, err)

fetchedNode, err := client.KubeClient.CoreV1().Nodes().Get(context.TODO(), node.Name, metav1.GetOptions{})
assert.NoError(t, err)
_, ok := fetchedNode.Annotations[nodelock.NodeLockKey]
assert.False(t, ok, "expected lock annotation to be removed")
}

func TestDevices_Fit(t *testing.T) {
config := CambriconConfig{
ResourceCountName: "cambricon.com/mlu",
Expand Down
Loading