feat: enhance logging and improve device allocation handling in NVIDIA plugin

Author: haitwang-cloud

pkg/nvidia-plugin/pkg/plugin/register.go

@@ -141,7 +141,10 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*util.DeviceInfo {
 		if plugin.schedulerConfig.DeviceMemoryScaling != 1 {
 			registeredMem = int32(float64(registeredMem) * plugin.schedulerConfig.DeviceMemoryScaling)
 		}
-		klog.Infoln("MemoryScaling=", plugin.schedulerConfig.DeviceMemoryScaling, "registeredMem=", registeredMem)
+		klog.V(3).InfoS("Applied memory scaling",
+			"scalingFactor", plugin.schedulerConfig.DeviceMemoryScaling,
+			"originalMemoryMB", memoryTotal/1024/1024,
+			"registeredMemoryMB", registeredMem)
 		health := true
 		for _, val := range devs {
 			if strings.Compare(val.ID, UUID) == 0 {
@@ -170,7 +173,12 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*util.DeviceInfo {
 			Mode:    plugin.operatingMode,
 			Health:  health,
 		})
-		klog.Infof("nvml registered device id=%v, memory=%v, type=%v, numa=%v", idx, registeredMem, Model, numa)
+		klog.V(4).InfoS("Registered NVIDIA device",
+			"uuid", UUID,
+			"deviceIndex", idx,
+			"memoryMB", registeredMem,
+			"model", Model,
+			"numaNode", numa)
 	}
 	return &res
 }

pkg/nvidia-plugin/pkg/plugin/server.go

@@ -410,8 +410,8 @@ func (plugin *NvidiaDevicePlugin) GetPreferredAllocation(ctx context.Context, r
 
 // Allocate which return list of devices.
 func (plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
+	klog.InfoS("Allocate", "request", reqs)
 	responses := pluginapi.AllocateResponse{}
-
 	nodeName := os.Getenv(util.NodeNameEnvName)
 	current, err := util.GetPendingPod(ctx, nodeName)
 	if err != nil {
@@ -423,7 +423,7 @@ func (plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *pluginapi.
 			return nil, fmt.Errorf("invalid allocation request for %q: %w", plugin.rm.Resource(), err)
 		}
 		currentCtr, devreq, err := GetNextDeviceRequest(nvidia.NvidiaGPUDevice, *current)
-		klog.Infoln("deviceAllocateFromAnnotation=", devreq)
+		klog.Infof("allocate req:%v, current:%v, devreq:%v", req, current, devreq)
 		if err != nil {
 			device.PodAllocationFailed(nodeName, current, NodeLockNvidia)
 			return &responses, err
@@ -659,7 +659,7 @@ func (plugin *NvidiaDevicePlugin) deviceIDsFromAnnotatedDeviceIDs(ids []string)
 }
 
 func (plugin *NvidiaDevicePlugin) apiDevices() []*pluginapi.Device {
-	return plugin.rm.Devices().GetPluginDevices()
+	return plugin.rm.Devices().GetPluginDevices(plugin.schedulerConfig.DeviceSplitCount)
 }
 
 // updateResponseForDeviceListEnvVar sets the environment variable for the requested devices.

pkg/nvidia-plugin/pkg/rm/devices.go

@@ -175,11 +175,24 @@ func (ds Devices) GetUUIDs() []string {
 }
 
 // GetPluginDevices returns the plugin Devices from all devices in the Devices
-func (ds Devices) GetPluginDevices() []*pluginapi.Device {
+func (ds Devices) GetPluginDevices(count uint) []*pluginapi.Device {
 	var res []*pluginapi.Device
-	for _, device := range ds {
-		d := device
-		res = append(res, &d.Device)
+	if !strings.Contains(ds.GetIDs()[0], "MIG") {
+		for _, dev := range ds {
+			for i := uint(0); i < count; i++ {
+				id := fmt.Sprintf("%v-%v", dev.ID, i)
+				res = append(res, &pluginapi.Device{
+					ID:       id,
+					Health:   dev.Health,
+					Topology: nil,
+				})
+			}
+		}
+	} else {
+		for _, device := range ds {
+			d := device
+			res = append(res, &d.Device)
+		}
 	}
 	return res
 }