optimize schedule failure event (#1444)

1. delete duplicate event type definition, use common package
2. support vendor device report custom event

Signed-off-by: Lei Guo <[email protected]>
Co-authored-by: Lei Guo <[email protected]>

Author: Kyrie336

pkg/device/common/common.go

@@ -46,3 +46,20 @@ func GenReason(reasons map[string]int, cards int) string {
 	}
 	return strings.Join(reason, ", ")
 }
+
+func ParseReason(reason string) map[string]int {
+	reasons := strings.Split(reason, ", ")
+
+	reasonMap := map[string]int{}
+	for _, reason := range reasons {
+		cnt, key := 0, ""
+		_, err := fmt.Sscanf(reason, "%d/%d %s", &cnt, new(int), &key)
+		if err != nil {
+			continue
+		}
+
+		reasonMap[key] = cnt
+	}
+
+	return reasonMap
+}

pkg/device/common/common_test.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package common
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseReason(t *testing.T) {
+	for _, ts := range []struct {
+		name   string
+		reason string
+
+		expectedReasonMap map[string]int
+	}{
+		{
+			name:   "base test",
+			reason: "3/8 CardInsufficientMemory, 2/8 CardInsufficientCore, 3/8 CardNotHealth",
+
+			expectedReasonMap: map[string]int{
+				"CardInsufficientMemory": 3,
+				"CardInsufficientCore":   2,
+				"CardNotHealth":          3,
+			},
+		},
+	} {
+		t.Run(ts.name, func(t *testing.T) {
+			result := ParseReason(ts.reason)
+			if !reflect.DeepEqual(result, ts.expectedReasonMap) {
+				t.Errorf("ParseReason failed: result %v, expected %v",
+					result, ts.expectedReasonMap)
+			}
+		})
+	}
+}

pkg/scheduler/score.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"github.com/Project-HAMi/HAMi/pkg/device"
+	"github.com/Project-HAMi/HAMi/pkg/device/common"
 	"github.com/Project-HAMi/HAMi/pkg/scheduler/config"
 	"github.com/Project-HAMi/HAMi/pkg/scheduler/policy"
 )
@@ -37,36 +38,6 @@ func viewStatus(usage NodeUsage) {
 	}
 }
 
-const (
-	cardTypeMismatch                  = "CardTypeMismatch"
-	cardUUIDMismatch                  = "CardUuidMismatch"
-	cardTimeSlicingExhausted          = "CardTimeSlicingExhausted"
-	cardComputeUnitsExhausted         = "CardComputeUnitsExhausted"
-	cardInsufficientMemory            = "CardInsufficientMemory"
-	cardInsufficientCore              = "CardInsufficientCore"
-	numaNotFit                        = "NumaNotFit"
-	exclusiveDeviceAllocateConflict   = "ExclusiveDeviceAllocateConflict"
-	cardNotFoundCustomFilterRule      = "CardNotFoundCustomFilterRule"
-	nodeInsufficientDevice            = "NodeInsufficientDevice"
-	allocatedCardsInsufficientRequest = "AllocatedCardsInsufficientRequest"
-	nodeUnfitPod                      = "NodeUnfitPod"
-	nodeFitPod                        = "NodeFitPod"
-)
-
-var scheduleFailureReasons = []string{
-	cardTypeMismatch,
-	cardUUIDMismatch,
-	cardTimeSlicingExhausted,
-	cardComputeUnitsExhausted,
-	cardInsufficientMemory,
-	cardInsufficientCore,
-	numaNotFit,
-	exclusiveDeviceAllocateConflict,
-	cardNotFoundCustomFilterRule,
-	nodeInsufficientDevice,
-	allocatedCardsInsufficientRequest,
-}
-
 func getNodeResources(list NodeUsage, t string) []*device.DeviceUsage {
 	l := []*device.DeviceUsage{}
 	for _, val := range list.Devices.DeviceLists {
@@ -91,16 +62,15 @@ func fitInDevices(node *NodeUsage, requests device.ContainerDeviceRequests, pod
 	for _, k := range requests {
 		sums += int(k.Nums)
 		if int(k.Nums) > len(node.Devices.DeviceLists) {
-			klog.V(5).InfoS(nodeInsufficientDevice, "pod", klog.KObj(pod), "request devices nums", k.Nums, "node device nums", len(node.Devices.DeviceLists))
-			return false, nodeInsufficientDevice
+			klog.V(5).InfoS(common.NodeInsufficientDevice, "pod", klog.KObj(pod), "request devices nums", k.Nums, "node device nums", len(node.Devices.DeviceLists))
+			return false, common.NodeInsufficientDevice
 		}
 		sort.Sort(node.Devices)
 		_, ok := device.GetDevices()[k.Type]
 		if !ok {
 			return false, "Device type not found"
 		}
-		fit, tmpDevs, devreason := device.GetDevices()[k.Type].Fit(getNodeResources(*node, k.Type), k, pod, nodeInfo, devinput)
-		reason := "node:" + node.Node.Name + " " + "resaon:" + devreason
+		fit, tmpDevs, reason := device.GetDevices()[k.Type].Fit(getNodeResources(*node, k.Type), k, pod, nodeInfo, devinput)
 		if fit {
 			for idx, val := range tmpDevs[k.Type] {
 				for nidx, v := range node.Devices.DeviceLists {
@@ -193,10 +163,10 @@ func (s *Scheduler) calcScore(nodes *map[string]*NodeUsage, resourceReqs device.
 				}
 				ctrfit = fit
 				if !fit {
-					klog.V(4).InfoS(nodeUnfitPod, "pod", klog.KObj(task), "node", nodeID, "reason", reason)
+					klog.V(4).InfoS(common.NodeUnfitPod, "pod", klog.KObj(task), "node", nodeID, "reason", reason)
 					failedNodesMutex.Lock()
-					failedNodes[nodeID] = nodeUnfitPod
-					for _, reasonType := range parseNodeReason(reason) {
+					failedNodes[nodeID] = common.NodeUnfitPod
+					for reasonType := range common.ParseReason(reason) {
 						failureReason[reasonType] = append(failureReason[reasonType], nodeID)
 					}
 					failedNodesMutex.Unlock()
@@ -209,7 +179,7 @@ func (s *Scheduler) calcScore(nodes *map[string]*NodeUsage, resourceReqs device.
 				res.NodeList = append(res.NodeList, &score)
 				fitNodesMutex.Unlock()
 				score.OverrideScore(snapshot, userNodePolicy)
-				klog.V(4).InfoS(nodeFitPod, "pod", klog.KObj(task), "node", nodeID, "score", score.Score)
+				klog.V(4).InfoS(common.NodeFitPod, "pod", klog.KObj(task), "node", nodeID, "score", score.Score)
 			}
 		}(nodeID, node)
 	}
@@ -231,13 +201,3 @@ func (s *Scheduler) calcScore(nodes *map[string]*NodeUsage, resourceReqs device.
 	}
 	return &res, utilerrors.NewAggregate(errorsSlice)
 }
-
-func parseNodeReason(nodeReason string) []string {
-	var res []string
-	for _, reason := range scheduleFailureReasons {
-		if strings.Contains(nodeReason, reason) {
-			res = append(res, reason)
-		}
-	}
-	return res
-}

pkg/scheduler/score_test.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"github.com/Project-HAMi/HAMi/pkg/device"
+	"github.com/Project-HAMi/HAMi/pkg/device/common"
 	"github.com/Project-HAMi/HAMi/pkg/device/hygon"
 	"github.com/Project-HAMi/HAMi/pkg/device/kunlun"
 	"github.com/Project-HAMi/HAMi/pkg/device/metax"
@@ -1573,7 +1574,7 @@ func Test_calcScore(t *testing.T) {
 					NodeList: []*policy.NodeScore{},
 				},
 				failedNodes: map[string]string{
-					"node1": nodeUnfitPod,
+					"node1": common.NodeUnfitPod,
 				},
 				err: nil,
 			},
@@ -1679,8 +1680,8 @@ func Test_calcScore(t *testing.T) {
 					NodeList: []*policy.NodeScore{},
 				},
 				failedNodes: map[string]string{
-					"node1": nodeUnfitPod,
-					"node2": nodeUnfitPod,
+					"node1": common.NodeUnfitPod,
+					"node2": common.NodeUnfitPod,
 				},
 				err: nil,
 			},
@@ -2762,7 +2763,7 @@ func Test_fitInCertainDevice(t *testing.T) {
 			},
 			want1: false,
 			want2: map[string]device.ContainerDevices{},
-			want3: map[string]int{cardTypeMismatch: 1},
+			want3: map[string]int{common.CardTypeMismatch: 1},
 		},
 		{
 			name: "device count less than device used",
@@ -2805,7 +2806,7 @@ func Test_fitInCertainDevice(t *testing.T) {
 			},
 			want1: false,
 			want2: map[string]device.ContainerDevices{},
-			want3: map[string]int{cardTimeSlicingExhausted: 1},
+			want3: map[string]int{common.CardTimeSlicingExhausted: 1},
 		},
 		{
 			name: "core limit exceed 100",
@@ -2848,7 +2849,7 @@ func Test_fitInCertainDevice(t *testing.T) {
 			},
 			want1: false,
 			want2: map[string]device.ContainerDevices{},
-			want3: map[string]int{cardInsufficientCore: 1},
+			want3: map[string]int{common.CardInsufficientCore: 1},
 		},
 		{
 			name: "card insufficient remaining memory",
@@ -2891,7 +2892,7 @@ func Test_fitInCertainDevice(t *testing.T) {
 			},
 			want1: false,
 			want2: map[string]device.ContainerDevices{},
-			want3: map[string]int{cardInsufficientMemory: 1},
+			want3: map[string]int{common.CardInsufficientMemory: 1},
 		},
 		{
 			name: "the container wants exclusive access to an entire card, but the card is already in use",
@@ -2934,7 +2935,7 @@ func Test_fitInCertainDevice(t *testing.T) {
 			},
 			want1: false,
 			want2: map[string]device.ContainerDevices{},
-			want3: map[string]int{exclusiveDeviceAllocateConflict: 1},
+			want3: map[string]int{common.ExclusiveDeviceAllocateConflict: 1},
 		},
 		{
 			name: "can't allocate core=0 job to an already full GPU",
@@ -2977,7 +2978,7 @@ func Test_fitInCertainDevice(t *testing.T) {
 			},
 			want1: false,
 			want2: map[string]device.ContainerDevices{},
-			want3: map[string]int{cardComputeUnitsExhausted: 1},
+			want3: map[string]int{common.CardComputeUnitsExhausted: 1},
 		},
 		{
 			name: "mode is mig",
@@ -3040,7 +3041,7 @@ func Test_fitInCertainDevice(t *testing.T) {
 					},
 				},
 			},
-			want3: map[string]int{cardNotFoundCustomFilterRule: 1, allocatedCardsInsufficientRequest: 1},
+			want3: map[string]int{common.CardNotFoundCustomFilterRule: 1, common.AllocatedCardsInsufficientRequest: 1},
 		},
 		{
 			name: "card uuid don't match",
@@ -3089,7 +3090,7 @@ func Test_fitInCertainDevice(t *testing.T) {
 			},
 			want1: false,
 			want2: map[string]device.ContainerDevices{},
-			want3: map[string]int{cardUUIDMismatch: 1},
+			want3: map[string]int{common.CardUUIDMismatch: 1},
 		},
 		{
 			name: "numa not fit",
@@ -3136,7 +3137,7 @@ func Test_fitInCertainDevice(t *testing.T) {
 					},
 				},
 			},
-			want3: map[string]int{numaNotFit: 1, allocatedCardsInsufficientRequest: 1},
+			want3: map[string]int{common.NumaNotFit: 1, common.AllocatedCardsInsufficientRequest: 1},
 		},
 		{
 			name: "test device kind of not fit reason",
@@ -3186,8 +3187,8 @@ func Test_fitInCertainDevice(t *testing.T) {
 			},
 			want1: false,
 			want2: map[string]device.ContainerDevices{},
-			want3: map[string]int{cardUUIDMismatch: 3, cardTimeSlicingExhausted: 4,
-				cardInsufficientMemory: 2, cardInsufficientCore: 1},
+			want3: map[string]int{common.CardUUIDMismatch: 3, common.CardTimeSlicingExhausted: 4,
+				common.CardInsufficientMemory: 2, common.CardInsufficientCore: 1},
 		},
 	}
 	for _, test := range tests {