cc-metric-collector/collectors/nvidiaMetric.go

package collectors

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"strings"
	"time"

	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
	"github.com/NVIDIA/go-nvml/pkg/nvml"
)

type NvidiaCollectorConfig struct {
	ExcludeMetrics        []string `json:"exclude_metrics,omitempty"`
	ExcludeDevices        []string `json:"exclude_devices,omitempty"`
	AddPciInfoTag         bool     `json:"add_pci_info_tag,omitempty"`
	UsePciInfoAsTypeId    bool     `json:"use_pci_info_as_type_id,omitempty"`
	AddUuidMeta           bool     `json:"add_uuid_meta,omitempty"`
	AddBoardNumberMeta    bool     `json:"add_board_number_meta,omitempty"`
	AddSerialMeta         bool     `json:"add_serial_meta,omitempty"`
	ProcessMigDevices     bool     `json:"process_mig_devices,omitempty"`
	UseUuidForMigDevices  bool     `json:"use_uuid_for_mig_device,omitempty"`
	UseSliceForMigDevices bool     `json:"use_slice_for_mig_device,omitempty"`
}

type NvidiaCollectorDevice struct {
	device         nvml.Device
	excludeMetrics map[string]bool
	tags           map[string]string
	meta           map[string]string
}

type NvidiaCollector struct {
	metricCollector
	config   NvidiaCollectorConfig
	gpus     []NvidiaCollectorDevice
	num_gpus int
}

func (m *NvidiaCollector) CatchPanic() {
	if rerr := recover(); rerr != nil {
		log.Print(rerr)
		m.init = false
	}
}

func (m *NvidiaCollector) Init(config json.RawMessage) error {
	var err error
	m.name = "NvidiaCollector"
	m.config.AddPciInfoTag = false
	m.config.UsePciInfoAsTypeId = false
	m.config.ProcessMigDevices = false
	m.config.UseUuidForMigDevices = false
	m.config.UseSliceForMigDevices = false
	m.setup()
	if len(config) > 0 {
		err = json.Unmarshal(config, &m.config)
		if err != nil {
			return err
		}
	}
	m.meta = map[string]string{
		"source": m.name,
		"group":  "Nvidia",
	}

	defer m.CatchPanic()

	// Initialize NVIDIA Management Library (NVML)
	ret := nvml.Init()
	if ret != nvml.SUCCESS {
		err = errors.New(nvml.ErrorString(ret))
		cclog.ComponentError(m.name, "Unable to initialize NVML", err.Error())
		return err
	}

	// Number of NVIDIA GPUs
	num_gpus, ret := nvml.DeviceGetCount()
	if ret != nvml.SUCCESS {
		err = errors.New(nvml.ErrorString(ret))
		cclog.ComponentError(m.name, "Unable to get device count", err.Error())
		return err
	}

	// For all GPUs
	idx := 0
	m.gpus = make([]NvidiaCollectorDevice, num_gpus)
	for i := 0; i < num_gpus; i++ {

		// Skip excluded devices by ID
		str_i := fmt.Sprintf("%d", i)
		if _, skip := stringArrayContains(m.config.ExcludeDevices, str_i); skip {
			cclog.ComponentDebug(m.name, "Skipping excluded device", str_i)
			continue
		}

		// Get device handle
		device, ret := nvml.DeviceGetHandleByIndex(i)
		if ret != nvml.SUCCESS {
			err = errors.New(nvml.ErrorString(ret))
			cclog.ComponentError(m.name, "Unable to get device at index", i, ":", err.Error())
			continue
		}

		// Get device's PCI info
		pciInfo, ret := nvml.DeviceGetPciInfo(device)
		if ret != nvml.SUCCESS {
			err = errors.New(nvml.ErrorString(ret))
			cclog.ComponentError(m.name, "Unable to get PCI info for device at index", i, ":", err.Error())
			continue
		}
		// Create PCI ID in the common format used by the NVML.
		pci_id := fmt.Sprintf(
			nvml.DEVICE_PCI_BUS_ID_FMT,
			pciInfo.Domain,
			pciInfo.Bus,
			pciInfo.Device)

		// Skip excluded devices specified by PCI ID
		if _, skip := stringArrayContains(m.config.ExcludeDevices, pci_id); skip {
			cclog.ComponentDebug(m.name, "Skipping excluded device", pci_id)
			continue
		}

		// Select which value to use as 'type-id'.
		// The PCI ID is commonly required in SLURM environments because the
		// numberic IDs used by SLURM and the ones used by NVML might differ
		// depending on the job type. The PCI ID is more reliable but is commonly
		// not recorded for a job, so it must be added manually in prologue or epilogue
		// e.g. to the comment field
		tid := str_i
		if m.config.UsePciInfoAsTypeId {
			tid = pci_id
		}

		// Now we got all infos together, populate the device list
		g := &m.gpus[idx]

		// Add device handle
		g.device = device

		// Add tags
		g.tags = map[string]string{
			"type":    "accelerator",
			"type-id": tid,
		}

		// Add PCI info as tag if not already used as 'type-id'
		if m.config.AddPciInfoTag && !m.config.UsePciInfoAsTypeId {
			g.tags["pci_identifier"] = pci_id
		}

		g.meta = map[string]string{
			"source": m.name,
			"group":  "Nvidia",
		}

		if m.config.AddBoardNumberMeta {
			board, ret := nvml.DeviceGetBoardPartNumber(device)
			if ret != nvml.SUCCESS {
				cclog.ComponentError(m.name, "Unable to get boart part number for device at index", i, ":", err.Error())
			} else {
				g.meta["board_number"] = board
			}
		}
		if m.config.AddSerialMeta {
			serial, ret := nvml.DeviceGetSerial(device)
			if ret != nvml.SUCCESS {
				cclog.ComponentError(m.name, "Unable to get serial number for device at index", i, ":", err.Error())
			} else {
				g.meta["serial"] = serial
			}
		}
		if m.config.AddUuidMeta {
			uuid, ret := nvml.DeviceGetUUID(device)
			if ret != nvml.SUCCESS {
				cclog.ComponentError(m.name, "Unable to get UUID for device at index", i, ":", err.Error())
			} else {
				g.meta["uuid"] = uuid
			}
		}

		// Add excluded metrics
		g.excludeMetrics = map[string]bool{}
		for _, e := range m.config.ExcludeMetrics {
			g.excludeMetrics[e] = true
		}

		// Increment the index for the next device
		idx++
	}
	m.num_gpus = idx

	m.init = true
	return nil
}

func readMemoryInfo(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	if !device.excludeMetrics["nv_fb_mem_total"] || !device.excludeMetrics["nv_fb_mem_used"] || !device.excludeMetrics["nv_fb_mem_reserved"] {
		var total uint64
		var used uint64
		var reserved uint64 = 0
		var v2 bool = false
		meminfo, ret := nvml.DeviceGetMemoryInfo(device.device)
		if ret != nvml.SUCCESS {
			err := errors.New(nvml.ErrorString(ret))
			return err
		}
		total = meminfo.Total
		used = meminfo.Used

		if !device.excludeMetrics["nv_fb_mem_total"] {
			t := float64(total) / (1024 * 1024)
			y, err := lp.New("nv_fb_mem_total", device.tags, device.meta, map[string]interface{}{"value": t}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MByte")
				output <- y
			}
		}

		if !device.excludeMetrics["nv_fb_mem_used"] {
			f := float64(used) / (1024 * 1024)
			y, err := lp.New("nv_fb_mem_used", device.tags, device.meta, map[string]interface{}{"value": f}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MByte")
				output <- y
			}
		}

		if v2 && !device.excludeMetrics["nv_fb_mem_reserved"] {
			r := float64(reserved) / (1024 * 1024)
			y, err := lp.New("nv_fb_mem_reserved", device.tags, device.meta, map[string]interface{}{"value": r}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MByte")
				output <- y
			}
		}
	}
	return nil
}

func readBarMemoryInfo(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	if !device.excludeMetrics["nv_bar1_mem_total"] || !device.excludeMetrics["nv_bar1_mem_used"] {
		meminfo, ret := nvml.DeviceGetBAR1MemoryInfo(device.device)
		if ret != nvml.SUCCESS {
			err := errors.New(nvml.ErrorString(ret))
			return err
		}
		if !device.excludeMetrics["nv_bar1_mem_total"] {
			t := float64(meminfo.Bar1Total) / (1024 * 1024)
			y, err := lp.New("nv_bar1_mem_total", device.tags, device.meta, map[string]interface{}{"value": t}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MByte")
				output <- y
			}
		}
		if !device.excludeMetrics["nv_bar1_mem_used"] {
			t := float64(meminfo.Bar1Used) / (1024 * 1024)
			y, err := lp.New("nv_bar1_mem_used", device.tags, device.meta, map[string]interface{}{"value": t}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MByte")
				output <- y
			}
		}
	}
	return nil
}

func readUtilization(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	isMig, ret := nvml.DeviceIsMigDeviceHandle(device.device)
	if ret != nvml.SUCCESS {
		err := errors.New(nvml.ErrorString(ret))
		return err
	}
	if !isMig {
		return nil
	}

	if !device.excludeMetrics["nv_util"] || !device.excludeMetrics["nv_mem_util"] {
		// Retrieves the current utilization rates for the device's major subsystems.
		//
		// Available utilization rates
		// * Gpu: Percent of time over the past sample period during which one or more kernels was executing on the GPU.
		// * Memory: Percent of time over the past sample period during which global (device) memory was being read or written
		//
		// Note:
		// * During driver initialization when ECC is enabled one can see high GPU and Memory Utilization readings.
		//   This is caused by ECC Memory Scrubbing mechanism that is performed during driver initialization.
		// * On MIG-enabled GPUs, querying device utilization rates is not currently supported.
		util, ret := nvml.DeviceGetUtilizationRates(device.device)
		if ret == nvml.SUCCESS {
			if !device.excludeMetrics["nv_util"] {
				y, err := lp.New("nv_util", device.tags, device.meta, map[string]interface{}{"value": float64(util.Gpu)}, time.Now())
				if err == nil {
					y.AddMeta("unit", "%")
					output <- y
				}
			}
			if !device.excludeMetrics["nv_mem_util"] {
				y, err := lp.New("nv_mem_util", device.tags, device.meta, map[string]interface{}{"value": float64(util.Memory)}, time.Now())
				if err == nil {
					y.AddMeta("unit", "%")
					output <- y
				}
			}
		}
	}
	return nil
}

func readTemp(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	if !device.excludeMetrics["nv_temp"] {
		// Retrieves the current temperature readings for the device, in degrees C.
		//
		// Available temperature sensors:
		// * TEMPERATURE_GPU: Temperature sensor for the GPU die.
		// * NVML_TEMPERATURE_COUNT
		temp, ret := nvml.DeviceGetTemperature(device.device, nvml.TEMPERATURE_GPU)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_temp", device.tags, device.meta, map[string]interface{}{"value": float64(temp)}, time.Now())
			if err == nil {
				y.AddMeta("unit", "degC")
				output <- y
			}
		}
	}
	return nil
}

func readFan(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	if !device.excludeMetrics["nv_fan"] {
		// Retrieves the intended operating speed of the device's fan.
		//
		// Note: The reported speed is the intended fan speed.
		// If the fan is physically blocked and unable to spin, the output will not match the actual fan speed.
		//
		// For all discrete products with dedicated fans.
		//
		// The fan speed is expressed as a percentage of the product's maximum noise tolerance fan speed.
		// This value may exceed 100% in certain cases.
		fan, ret := nvml.DeviceGetFanSpeed(device.device)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_fan", device.tags, device.meta, map[string]interface{}{"value": float64(fan)}, time.Now())
			if err == nil {
				y.AddMeta("unit", "%")
				output <- y
			}
		}
	}
	return nil
}

// func readFans(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
// 	if !device.excludeMetrics["nv_fan"] {
// 		numFans, ret := nvml.DeviceGetNumFans(device.device)
// 		if ret == nvml.SUCCESS {
// 			for i := 0; i < numFans; i++ {
// 				fan, ret := nvml.DeviceGetFanSpeed_v2(device.device, i)
// 				if ret == nvml.SUCCESS {
// 					y, err := lp.New("nv_fan", device.tags, device.meta, map[string]interface{}{"value": float64(fan)}, time.Now())
// 					if err == nil {
// 						y.AddMeta("unit", "%")
// 						y.AddTag("stype", "fan")
// 						y.AddTag("stype-id", fmt.Sprintf("%d", i))
// 						output <- y
// 					}
// 				}
// 			}
// 		}
// 	}
// 	return nil
// }

func readEccMode(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	if !device.excludeMetrics["nv_ecc_mode"] {
		// Retrieves the current and pending ECC modes for the device.
		//
		// For Fermi or newer fully supported devices. Only applicable to devices with ECC.
		// Requires NVML_INFOROM_ECC version 1.0 or higher.
		//
		// Changing ECC modes requires a reboot.
		// The "pending" ECC mode refers to the target mode following the next reboot.
		_, ecc_pend, ret := nvml.DeviceGetEccMode(device.device)
		if ret == nvml.SUCCESS {
			var y lp.CCMetric
			var err error
			switch ecc_pend {
			case nvml.FEATURE_DISABLED:
				y, err = lp.New("nv_ecc_mode", device.tags, device.meta, map[string]interface{}{"value": "OFF"}, time.Now())
			case nvml.FEATURE_ENABLED:
				y, err = lp.New("nv_ecc_mode", device.tags, device.meta, map[string]interface{}{"value": "ON"}, time.Now())
			default:
				y, err = lp.New("nv_ecc_mode", device.tags, device.meta, map[string]interface{}{"value": "UNKNOWN"}, time.Now())
			}
			if err == nil {
				output <- y
			}
		} else if ret == nvml.ERROR_NOT_SUPPORTED {
			y, err := lp.New("nv_ecc_mode", device.tags, device.meta, map[string]interface{}{"value": "N/A"}, time.Now())
			if err == nil {
				output <- y
			}
		}
	}
	return nil
}

func readPerfState(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	if !device.excludeMetrics["nv_perf_state"] {
		// Retrieves the current performance state for the device.
		//
		// Allowed PStates:
		//  0: Maximum Performance.
		// ..
		// 15: Minimum Performance.
		// 32: Unknown performance state.
		pState, ret := nvml.DeviceGetPerformanceState(device.device)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_perf_state", device.tags, device.meta, map[string]interface{}{"value": fmt.Sprintf("P%d", int(pState))}, time.Now())
			if err == nil {
				output <- y
			}
		}
	}
	return nil
}

func readPowerUsage(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	if !device.excludeMetrics["nv_power_usage"] {
		// Retrieves power usage for this GPU in milliwatts and its associated circuitry (e.g. memory)
		//
		// On Fermi and Kepler GPUs the reading is accurate to within +/- 5% of current power draw.
		//
		// It is only available if power management mode is supported
		mode, ret := nvml.DeviceGetPowerManagementMode(device.device)
		if ret != nvml.SUCCESS {
			return nil
		}
		if mode == nvml.FEATURE_ENABLED {
			power, ret := nvml.DeviceGetPowerUsage(device.device)
			if ret == nvml.SUCCESS {
				y, err := lp.New("nv_power_usage", device.tags, device.meta, map[string]interface{}{"value": float64(power) / 1000}, time.Now())
				if err == nil {
					y.AddMeta("unit", "watts")
					output <- y
				}
			}
		}
	}
	return nil
}

func readClocks(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	// Retrieves the current clock speeds for the device.
	//
	// Available clock information:
	// * CLOCK_GRAPHICS: Graphics clock domain.
	// * CLOCK_SM: Streaming Multiprocessor clock domain.
	// * CLOCK_MEM: Memory clock domain.
	if !device.excludeMetrics["nv_graphics_clock"] {
		graphicsClock, ret := nvml.DeviceGetClockInfo(device.device, nvml.CLOCK_GRAPHICS)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_graphics_clock", device.tags, device.meta, map[string]interface{}{"value": float64(graphicsClock)}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MHz")
				output <- y
			}
		}
	}

	if !device.excludeMetrics["nv_sm_clock"] {
		smCock, ret := nvml.DeviceGetClockInfo(device.device, nvml.CLOCK_SM)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_sm_clock", device.tags, device.meta, map[string]interface{}{"value": float64(smCock)}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MHz")
				output <- y
			}
		}
	}

	if !device.excludeMetrics["nv_mem_clock"] {
		memClock, ret := nvml.DeviceGetClockInfo(device.device, nvml.CLOCK_MEM)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_mem_clock", device.tags, device.meta, map[string]interface{}{"value": float64(memClock)}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MHz")
				output <- y
			}
		}
	}
	if !device.excludeMetrics["nv_video_clock"] {
		memClock, ret := nvml.DeviceGetClockInfo(device.device, nvml.CLOCK_VIDEO)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_video_clock", device.tags, device.meta, map[string]interface{}{"value": float64(memClock)}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MHz")
				output <- y
			}
		}
	}
	return nil
}

func readMaxClocks(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	// Retrieves the maximum clock speeds for the device.
	//
	// Available clock information:
	// * CLOCK_GRAPHICS: Graphics clock domain.
	// * CLOCK_SM:       Streaming multiprocessor clock domain.
	// * CLOCK_MEM:      Memory clock domain.
	// * CLOCK_VIDEO:    Video encoder/decoder clock domain.
	// * CLOCK_COUNT:    Count of clock types.
	//
	// Note:
	/// On GPUs from Fermi family current P0 clocks (reported by nvmlDeviceGetClockInfo) can differ from max clocks by few MHz.
	if !device.excludeMetrics["nv_max_graphics_clock"] {
		max_gclk, ret := nvml.DeviceGetMaxClockInfo(device.device, nvml.CLOCK_GRAPHICS)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_max_graphics_clock", device.tags, device.meta, map[string]interface{}{"value": float64(max_gclk)}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MHz")
				output <- y
			}
		}
	}

	if !device.excludeMetrics["nv_max_sm_clock"] {
		maxSmClock, ret := nvml.DeviceGetClockInfo(device.device, nvml.CLOCK_SM)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_max_sm_clock", device.tags, device.meta, map[string]interface{}{"value": float64(maxSmClock)}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MHz")
				output <- y
			}
		}
	}

	if !device.excludeMetrics["nv_max_mem_clock"] {
		maxMemClock, ret := nvml.DeviceGetClockInfo(device.device, nvml.CLOCK_MEM)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_max_mem_clock", device.tags, device.meta, map[string]interface{}{"value": float64(maxMemClock)}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MHz")
				output <- y
			}
		}
	}

	if !device.excludeMetrics["nv_max_video_clock"] {
		maxMemClock, ret := nvml.DeviceGetClockInfo(device.device, nvml.CLOCK_VIDEO)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_max_video_clock", device.tags, device.meta, map[string]interface{}{"value": float64(maxMemClock)}, time.Now())
			if err == nil {
				y.AddMeta("unit", "MHz")
				output <- y
			}
		}
	}
	return nil
}

func readEccErrors(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	if !device.excludeMetrics["nv_ecc_uncorrected_error"] {
		// Retrieves the total ECC error counts for the device.
		//
		// For Fermi or newer fully supported devices.
		// Only applicable to devices with ECC.
		// Requires NVML_INFOROM_ECC version 1.0 or higher.
		// Requires ECC Mode to be enabled.
		//
		// The total error count is the sum of errors across each of the separate memory systems,
		// i.e. the total set of errors across the entire device.
		ecc_db, ret := nvml.DeviceGetTotalEccErrors(device.device, nvml.MEMORY_ERROR_TYPE_UNCORRECTED, nvml.AGGREGATE_ECC)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_ecc_uncorrected_error", device.tags, device.meta, map[string]interface{}{"value": float64(ecc_db)}, time.Now())
			if err == nil {
				output <- y
			}
		}
	}
	if !device.excludeMetrics["nv_ecc_corrected_error"] {
		ecc_sb, ret := nvml.DeviceGetTotalEccErrors(device.device, nvml.MEMORY_ERROR_TYPE_CORRECTED, nvml.AGGREGATE_ECC)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_ecc_corrected_error", device.tags, device.meta, map[string]interface{}{"value": float64(ecc_sb)}, time.Now())
			if err == nil {
				output <- y
			}
		}
	}
	return nil
}

func readPowerLimit(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	if !device.excludeMetrics["nv_power_max_limit"] {
		// Retrieves the power management limit associated with this device.
		//
		// For Fermi or newer fully supported devices.
		//
		// The power limit defines the upper boundary for the card's power draw.
		// If the card's total power draw reaches this limit the power management algorithm kicks in.
		pwr_limit, ret := nvml.DeviceGetPowerManagementLimit(device.device)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_power_max_limit", device.tags, device.meta, map[string]interface{}{"value": float64(pwr_limit) / 1000}, time.Now())
			if err == nil {
				y.AddMeta("unit", "watts")
				output <- y
			}
		}
	}
	return nil
}

func readEncUtilization(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	isMig, ret := nvml.DeviceIsMigDeviceHandle(device.device)
	if ret != nvml.SUCCESS {
		err := errors.New(nvml.ErrorString(ret))
		return err
	}
	if !isMig {
		return nil
	}
	if !device.excludeMetrics["nv_encoder_util"] {
		// Retrieves the current utilization and sampling size in microseconds for the Encoder
		//
		// For Kepler or newer fully supported devices.
		//
		// Note: On MIG-enabled GPUs, querying encoder utilization is not currently supported.
		enc_util, _, ret := nvml.DeviceGetEncoderUtilization(device.device)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_encoder_util", device.tags, device.meta, map[string]interface{}{"value": float64(enc_util)}, time.Now())
			if err == nil {
				y.AddMeta("unit", "%")
				output <- y
			}
		}
	}
	return nil
}

func readDecUtilization(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	isMig, ret := nvml.DeviceIsMigDeviceHandle(device.device)
	if ret != nvml.SUCCESS {
		err := errors.New(nvml.ErrorString(ret))
		return err
	}
	if !isMig {
		return nil
	}
	if !device.excludeMetrics["nv_decoder_util"] {
		// Retrieves the current utilization and sampling size in microseconds for the Encoder
		//
		// For Kepler or newer fully supported devices.
		//
		// Note: On MIG-enabled GPUs, querying encoder utilization is not currently supported.
		dec_util, _, ret := nvml.DeviceGetDecoderUtilization(device.device)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_decoder_util", device.tags, device.meta, map[string]interface{}{"value": float64(dec_util)}, time.Now())
			if err == nil {
				y.AddMeta("unit", "%")
				output <- y
			}
		}
	}
	return nil
}

func readRemappedRows(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	if !device.excludeMetrics["nv_remapped_rows_corrected"] ||
		!device.excludeMetrics["nv_remapped_rows_uncorrected"] ||
		!device.excludeMetrics["nv_remapped_rows_pending"] ||
		!device.excludeMetrics["nv_remapped_rows_failure"] {
		// Get number of remapped rows. The number of rows reported will be based on the cause of the remapping.
		// isPending indicates whether or not there are pending remappings.
		// A reset will be required to actually remap the row.
		// failureOccurred will be set if a row remapping ever failed in the past.
		// A pending remapping won't affect future work on the GPU since error-containment and dynamic page blacklisting will take care of that.
		//
		// For Ampere or newer fully supported devices.
		//
		// Note: On MIG-enabled GPUs with active instances, querying the number of remapped rows is not supported
		corrected, uncorrected, pending, failure, ret := nvml.DeviceGetRemappedRows(device.device)
		if ret == nvml.SUCCESS {
			if !device.excludeMetrics["nv_remapped_rows_corrected"] {
				y, err := lp.New("nv_remapped_rows_corrected", device.tags, device.meta, map[string]interface{}{"value": float64(corrected)}, time.Now())
				if err == nil {
					output <- y
				}
			}
			if !device.excludeMetrics["nv_remapped_rows_uncorrected"] {
				y, err := lp.New("nv_remapped_rows_corrected", device.tags, device.meta, map[string]interface{}{"value": float64(uncorrected)}, time.Now())
				if err == nil {
					output <- y
				}
			}
			if !device.excludeMetrics["nv_remapped_rows_pending"] {
				var p int = 0
				if pending {
					p = 1
				}
				y, err := lp.New("nv_remapped_rows_pending", device.tags, device.meta, map[string]interface{}{"value": p}, time.Now())
				if err == nil {
					output <- y
				}
			}
			if !device.excludeMetrics["nv_remapped_rows_failure"] {
				var f int = 0
				if failure {
					f = 1
				}
				y, err := lp.New("nv_remapped_rows_failure", device.tags, device.meta, map[string]interface{}{"value": f}, time.Now())
				if err == nil {
					output <- y
				}
			}
		}
	}
	return nil
}

func readProcessCounts(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	if !device.excludeMetrics["nv_compute_processes"] {
		// Get information about processes with a compute context on a device
		//
		// For Fermi &tm; or newer fully supported devices.
		//
		// This function returns information only about compute running processes (e.g. CUDA application which have
		// active context). Any graphics applications (e.g. using OpenGL, DirectX) won't be listed by this function.
		//
		// To query the current number of running compute processes, call this function with *infoCount = 0. The
		// return code will be NVML_ERROR_INSUFFICIENT_SIZE, or NVML_SUCCESS if none are running. For this call
		// \a infos is allowed to be NULL.
		//
		// The usedGpuMemory field returned is all of the memory used by the application.
		//
		// Keep in mind that information returned by this call is dynamic and the number of elements might change in
		// time. Allocate more space for \a infos table in case new compute processes are spawned.
		//
		// @note In MIG mode, if device handle is provided, the API returns aggregate information, only if
		//        the caller has appropriate privileges. Per-instance information can be queried by using
		//        specific MIG device handles.
		//        Querying per-instance information using MIG device handles is not supported if the device is in vGPU Host virtualization mode.
		procList, ret := nvml.DeviceGetComputeRunningProcesses(device.device)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_compute_processes", device.tags, device.meta, map[string]interface{}{"value": len(procList)}, time.Now())
			if err == nil {
				output <- y
			}
		}
	}
	if !device.excludeMetrics["nv_graphics_processes"] {
		// Get information about processes with a graphics context on a device
		//
		// For Kepler &tm; or newer fully supported devices.
		//
		// This function returns information only about graphics based processes
		// (eg. applications using OpenGL, DirectX)
		//
		// To query the current number of running graphics processes, call this function with *infoCount = 0. The
		// return code will be NVML_ERROR_INSUFFICIENT_SIZE, or NVML_SUCCESS if none are running. For this call
		// \a infos is allowed to be NULL.
		//
		// The usedGpuMemory field returned is all of the memory used by the application.
		//
		// Keep in mind that information returned by this call is dynamic and the number of elements might change in
		// time. Allocate more space for \a infos table in case new graphics processes are spawned.
		//
		// @note In MIG mode, if device handle is provided, the API returns aggregate information, only if
		//       the caller has appropriate privileges. Per-instance information can be queried by using
		//       specific MIG device handles.
		//       Querying per-instance information using MIG device handles is not supported if the device is in vGPU Host virtualization mode.
		procList, ret := nvml.DeviceGetGraphicsRunningProcesses(device.device)
		if ret == nvml.SUCCESS {
			y, err := lp.New("nv_graphics_processes", device.tags, device.meta, map[string]interface{}{"value": len(procList)}, time.Now())
			if err == nil {
				output <- y
			}
		}
	}
	// if !device.excludeMetrics["nv_mps_compute_processes"] {
	// 	// Get information about processes with a MPS compute context on a device
	// 	//
	// 	// For Volta &tm; or newer fully supported devices.
	// 	//
	// 	// This function returns information only about compute running processes (e.g. CUDA application which have
	// 	// active context) utilizing MPS. Any graphics applications (e.g. using OpenGL, DirectX) won't be listed by
	// 	// this function.
	// 	//
	// 	// To query the current number of running compute processes, call this function with *infoCount = 0. The
	// 	// return code will be NVML_ERROR_INSUFFICIENT_SIZE, or NVML_SUCCESS if none are running. For this call
	// 	// \a infos is allowed to be NULL.
	// 	//
	// 	// The usedGpuMemory field returned is all of the memory used by the application.
	// 	//
	// 	// Keep in mind that information returned by this call is dynamic and the number of elements might change in
	// 	// time. Allocate more space for \a infos table in case new compute processes are spawned.
	// 	//
	// 	// @note In MIG mode, if device handle is provided, the API returns aggregate information, only if
	// 	//       the caller has appropriate privileges. Per-instance information can be queried by using
	// 	//       specific MIG device handles.
	// 	//       Querying per-instance information using MIG device handles is not supported if the device is in vGPU Host virtualization mode.
	// 	procList, ret := nvml.DeviceGetMPSComputeRunningProcesses(device.device)
	// 	if ret == nvml.SUCCESS {
	// 		y, err := lp.New("nv_mps_compute_processes", device.tags, device.meta, map[string]interface{}{"value": len(procList)}, time.Now())
	// 		if err == nil {
	// 			output <- y
	// 		}
	// 	}
	// }
	return nil
}

func readViolationStats(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	var violTime nvml.ViolationTime
	var ret nvml.Return

	// Gets the duration of time during which the device was throttled (lower than requested clocks) due to power
	//  or thermal constraints.
	//
	// The method is important to users who are tying to understand if their GPUs throttle at any point during their applications. The
	// difference in violation times at two different reference times gives the indication of GPU throttling event.
	//
	// Violation for thermal capping is not supported at this time.
	//
	// For Kepler  or newer fully supported devices.

	if !device.excludeMetrics["nv_violation_power"] {
		// How long did power violations cause the GPU to be below application clocks
		violTime, ret = nvml.DeviceGetViolationStatus(device.device, nvml.PERF_POLICY_POWER)
		if ret == nvml.SUCCESS {
			t := float64(violTime.ViolationTime) * 1e-9
			y, err := lp.New("nv_violation_power", device.tags, device.meta, map[string]interface{}{"value": t}, time.Now())
			if err == nil {
				y.AddMeta("unit", "sec")
				output <- y
			}
		}
	}
	if !device.excludeMetrics["nv_violation_thermal"] {
		// How long did thermal violations cause the GPU to be below application clocks
		violTime, ret = nvml.DeviceGetViolationStatus(device.device, nvml.PERF_POLICY_THERMAL)
		if ret == nvml.SUCCESS {
			t := float64(violTime.ViolationTime) * 1e-9
			y, err := lp.New("nv_violation_thermal", device.tags, device.meta, map[string]interface{}{"value": t}, time.Now())
			if err == nil {
				y.AddMeta("unit", "sec")
				output <- y
			}
		}
	}
	if !device.excludeMetrics["nv_violation_sync_boost"] {
		// How long did sync boost cause the GPU to be below application clocks
		violTime, ret = nvml.DeviceGetViolationStatus(device.device, nvml.PERF_POLICY_SYNC_BOOST)
		if ret == nvml.SUCCESS {
			t := float64(violTime.ViolationTime) * 1e-9
			y, err := lp.New("nv_violation_sync_boost", device.tags, device.meta, map[string]interface{}{"value": t}, time.Now())
			if err == nil {
				y.AddMeta("unit", "sec")
				output <- y
			}
		}
	}
	if !device.excludeMetrics["nv_violation_board_limit"] {
		// How long did the board limit cause the GPU to be below application clocks
		violTime, ret = nvml.DeviceGetViolationStatus(device.device, nvml.PERF_POLICY_BOARD_LIMIT)
		if ret == nvml.SUCCESS {
			t := float64(violTime.ViolationTime) * 1e-9
			y, err := lp.New("nv_violation_board_limit", device.tags, device.meta, map[string]interface{}{"value": t}, time.Now())
			if err == nil {
				y.AddMeta("unit", "sec")
				output <- y
			}
		}
	}
	if !device.excludeMetrics["nv_violation_low_util"] {
		// How long did low utilization cause the GPU to be below application clocks
		violTime, ret = nvml.DeviceGetViolationStatus(device.device, nvml.PERF_POLICY_LOW_UTILIZATION)
		if ret == nvml.SUCCESS {
			t := float64(violTime.ViolationTime) * 1e-9
			y, err := lp.New("nv_violation_low_util", device.tags, device.meta, map[string]interface{}{"value": t}, time.Now())
			if err == nil {
				y.AddMeta("unit", "sec")
				output <- y
			}
		}
	}
	if !device.excludeMetrics["nv_violation_reliability"] {
		// How long did the board reliability limit cause the GPU to be below application clocks
		violTime, ret = nvml.DeviceGetViolationStatus(device.device, nvml.PERF_POLICY_RELIABILITY)
		if ret == nvml.SUCCESS {
			t := float64(violTime.ViolationTime) * 1e-9
			y, err := lp.New("nv_violation_reliability", device.tags, device.meta, map[string]interface{}{"value": t}, time.Now())
			if err == nil {
				y.AddMeta("unit", "sec")
				output <- y
			}
		}
	}
	if !device.excludeMetrics["nv_violation_below_app_clock"] {
		// Total time the GPU was held below application clocks by any limiter (all of above)
		violTime, ret = nvml.DeviceGetViolationStatus(device.device, nvml.PERF_POLICY_TOTAL_APP_CLOCKS)
		if ret == nvml.SUCCESS {
			t := float64(violTime.ViolationTime) * 1e-9
			y, err := lp.New("nv_violation_below_app_clock", device.tags, device.meta, map[string]interface{}{"value": t}, time.Now())
			if err == nil {
				y.AddMeta("unit", "sec")
				output <- y
			}
		}
	}
	if !device.excludeMetrics["nv_violation_below_base_clock"] {
		// Total time the GPU was held below base clocks
		violTime, ret = nvml.DeviceGetViolationStatus(device.device, nvml.PERF_POLICY_TOTAL_BASE_CLOCKS)
		if ret == nvml.SUCCESS {
			t := float64(violTime.ViolationTime) * 1e-9
			y, err := lp.New("nv_violation_below_base_clock", device.tags, device.meta, map[string]interface{}{"value": t}, time.Now())
			if err == nil {
				y.AddMeta("unit", "sec")
				output <- y
			}
		}
	}

	return nil
}

func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) error {
	// Retrieves the specified error counter value
	// Please refer to \a nvmlNvLinkErrorCounter_t for error counters that are available
	//
	// For Pascal &tm; or newer fully supported devices.

	for i := 0; i < nvml.NVLINK_MAX_LINKS; i++ {
		state, ret := nvml.DeviceGetNvLinkState(device.device, i)
		if ret == nvml.SUCCESS {
			if state == nvml.FEATURE_ENABLED {
				if !device.excludeMetrics["nv_nvlink_crc_errors"] {
					// Data link receive data CRC error counter
					count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_CRC_DATA)
					if ret == nvml.SUCCESS {
						y, err := lp.New("nv_nvlink_crc_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now())
						if err == nil {
							y.AddTag("stype", "nvlink")
							y.AddTag("stype-id", fmt.Sprintf("%d", i))
							output <- y
						}
					}
				}
				if !device.excludeMetrics["nv_nvlink_ecc_errors"] {
					// Data link receive data ECC error counter
					count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_ECC_DATA)
					if ret == nvml.SUCCESS {
						y, err := lp.New("nv_nvlink_ecc_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now())
						if err == nil {
							y.AddTag("stype", "nvlink")
							y.AddTag("stype-id", fmt.Sprintf("%d", i))
							output <- y
						}
					}
				}
				if !device.excludeMetrics["nv_nvlink_replay_errors"] {
					// Data link transmit replay error counter
					count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_REPLAY)
					if ret == nvml.SUCCESS {
						y, err := lp.New("nv_nvlink_replay_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now())
						if err == nil {
							y.AddTag("stype", "nvlink")
							y.AddTag("stype-id", fmt.Sprintf("%d", i))
							output <- y
						}
					}
				}
				if !device.excludeMetrics["nv_nvlink_recovery_errors"] {
					// Data link transmit recovery error counter
					count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_RECOVERY)
					if ret == nvml.SUCCESS {
						y, err := lp.New("nv_nvlink_recovery_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now())
						if err == nil {
							y.AddTag("stype", "nvlink")
							y.AddTag("stype-id", fmt.Sprintf("%d", i))
							output <- y
						}
					}
				}
				if !device.excludeMetrics["nv_nvlink_crc_flit_errors"] {
					// Data link receive flow control digit CRC error counter
					count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_CRC_FLIT)
					if ret == nvml.SUCCESS {
						y, err := lp.New("nv_nvlink_crc_flit_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now())
						if err == nil {
							y.AddTag("stype", "nvlink")
							y.AddTag("stype-id", fmt.Sprintf("%d", i))
							output <- y
						}
					}
				}
			}
		}
	}
	return nil
}

func (m *NvidiaCollector) Read(interval time.Duration, output chan lp.CCMetric) {
	var err error
	if !m.init {
		return
	}

	readAll := func(device NvidiaCollectorDevice, output chan lp.CCMetric) {
		name, ret := nvml.DeviceGetName(device.device)
		if ret != nvml.SUCCESS {
			name = "NoName"
		}
		err = readMemoryInfo(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readMemoryInfo for device", name, "failed")
		}

		err = readUtilization(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readUtilization for device", name, "failed")
		}

		err = readTemp(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readTemp for device", name, "failed")
		}

		err = readFan(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readFan for device", name, "failed")
		}

		err = readEccMode(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readEccMode for device", name, "failed")
		}

		err = readPerfState(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readPerfState for device", name, "failed")
		}

		err = readPowerUsage(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readPowerUsage for device", name, "failed")
		}

		err = readClocks(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readClocks for device", name, "failed")
		}

		err = readMaxClocks(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readMaxClocks for device", name, "failed")
		}

		err = readEccErrors(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readEccErrors for device", name, "failed")
		}

		err = readPowerLimit(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readPowerLimit for device", name, "failed")
		}

		err = readEncUtilization(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readEncUtilization for device", name, "failed")
		}

		err = readDecUtilization(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readDecUtilization for device", name, "failed")
		}

		err = readRemappedRows(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readRemappedRows for device", name, "failed")
		}

		err = readBarMemoryInfo(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readBarMemoryInfo for device", name, "failed")
		}

		err = readProcessCounts(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readProcessCounts for device", name, "failed")
		}

		err = readViolationStats(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readViolationStats for device", name, "failed")
		}

		err = readNVLinkStats(device, output)
		if err != nil {
			cclog.ComponentDebug(m.name, "readNVLinkStats for device", name, "failed")
		}
	}

	// Actual read loop over all attached Nvidia GPUs
	for i := 0; i < m.num_gpus; i++ {

		readAll(m.gpus[i], output)

		// Iterate over all MIG devices if any
		if m.config.ProcessMigDevices {
			current, _, ret := nvml.DeviceGetMigMode(m.gpus[i].device)
			if ret != nvml.SUCCESS {
				continue
			}
			if current == nvml.DEVICE_MIG_DISABLE {
				continue
			}

			maxMig, ret := nvml.DeviceGetMaxMigDeviceCount(m.gpus[i].device)
			if ret != nvml.SUCCESS {
				continue
			}
			if maxMig == 0 {
				continue
			}
			cclog.ComponentDebug(m.name, "Reading MIG devices for GPU", i)

			for j := 0; j < maxMig; j++ {
				mdev, ret := nvml.DeviceGetMigDeviceHandleByIndex(m.gpus[i].device, j)
				if ret != nvml.SUCCESS {
					continue
				}

				excludeMetrics := make(map[string]bool)
				for _, metric := range m.config.ExcludeMetrics {
					excludeMetrics[metric] = true
				}

				migDevice := NvidiaCollectorDevice{
					device:         mdev,
					tags:           map[string]string{},
					meta:           map[string]string{},
					excludeMetrics: excludeMetrics,
				}
				for k, v := range m.gpus[i].tags {
					migDevice.tags[k] = v
				}
				migDevice.tags["stype"] = "mig"
				if m.config.UseUuidForMigDevices {
					uuid, ret := nvml.DeviceGetUUID(mdev)
					if ret != nvml.SUCCESS {
						cclog.ComponentError(m.name, "Unable to get UUID for mig device at index", j, ":", err.Error())
					} else {
						migDevice.tags["stype-id"] = uuid
					}
				} else if m.config.UseSliceForMigDevices {
					name, ret := nvml.DeviceGetName(m.gpus[i].device)
					if ret == nvml.SUCCESS {
						mname, ret := nvml.DeviceGetName(mdev)
						if ret == nvml.SUCCESS {
							x := strings.Replace(mname, name, "", -1)
							x = strings.Replace(x, "MIG", "", -1)
							x = strings.TrimSpace(x)
							migDevice.tags["stype-id"] = x
						}
					}
				}
				if _, ok := migDevice.tags["stype-id"]; !ok {
					migDevice.tags["stype-id"] = fmt.Sprintf("%d", j)
				}
				for k, v := range m.gpus[i].meta {
					migDevice.meta[k] = v
				}
				if _, ok := migDevice.meta["uuid"]; ok && !m.config.UseUuidForMigDevices {
					uuid, ret := nvml.DeviceGetUUID(mdev)
					if ret == nvml.SUCCESS {
						migDevice.meta["uuid"] = uuid
					}
				}

				readAll(migDevice, output)
			}
		}
	}
}

func (m *NvidiaCollector) Close() {
	if m.init {
		nvml.Shutdown()
		m.init = false
	}
}