diff --git a/collectors/nvidiaMetric.go b/collectors/nvidiaMetric.go index 369f12b..fbbcaed 100644 --- a/collectors/nvidiaMetric.go +++ b/collectors/nvidiaMetric.go @@ -941,6 +941,12 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro // // For Pascal &tm; or newer fully supported devices. + var aggregate_crc_errors uint64 = 0 + var aggregate_ecc_errors uint64 = 0 + var aggregate_replay_errors uint64 = 0 + var aggregate_recovery_errors uint64 = 0 + var aggregate_crc_flit_errors uint64 = 0 + for i := 0; i < nvml.NVLINK_MAX_LINKS; i++ { state, ret := nvml.DeviceGetNvLinkState(device.device, i) if ret == nvml.SUCCESS { @@ -948,6 +954,7 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro if !device.excludeMetrics["nv_nvlink_crc_errors"] { // Data link receive data CRC error counter count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_CRC_DATA) + aggregate_crc_errors = aggregate_crc_errors + count if ret == nvml.SUCCESS { y, err := lp.New("nv_nvlink_crc_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now()) if err == nil { @@ -960,6 +967,7 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro if !device.excludeMetrics["nv_nvlink_ecc_errors"] { // Data link receive data ECC error counter count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_ECC_DATA) + aggregate_ecc_errors = aggregate_ecc_errors + count if ret == nvml.SUCCESS { y, err := lp.New("nv_nvlink_ecc_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now()) if err == nil { @@ -972,6 +980,7 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro if !device.excludeMetrics["nv_nvlink_replay_errors"] { // Data link transmit replay error counter count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_REPLAY) + aggregate_replay_errors = aggregate_replay_errors + count if ret == nvml.SUCCESS { y, err := lp.New("nv_nvlink_replay_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now()) if err == nil { @@ -984,6 +993,7 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro if !device.excludeMetrics["nv_nvlink_recovery_errors"] { // Data link transmit recovery error counter count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_RECOVERY) + aggregate_recovery_errors = aggregate_recovery_errors + count if ret == nvml.SUCCESS { y, err := lp.New("nv_nvlink_recovery_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now()) if err == nil { @@ -996,6 +1006,7 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro if !device.excludeMetrics["nv_nvlink_crc_flit_errors"] { // Data link receive flow control digit CRC error counter count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_CRC_FLIT) + aggregate_crc_flit_errors = aggregate_crc_flit_errors + count if ret == nvml.SUCCESS { y, err := lp.New("nv_nvlink_crc_flit_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now()) if err == nil { @@ -1008,6 +1019,48 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro } } } + + // Export aggegated values + if !device.excludeMetrics["nv_nvlink_crc_errors"] { + // Data link receive data CRC error counter + y, err := lp.New("nv_nvlink_crc_errors_sum", device.tags, device.meta, map[string]interface{}{"value": aggregate_crc_errors}, time.Now()) + if err == nil { + y.AddTag("stype", "nvlink") + output <- y + } + } + if !device.excludeMetrics["nv_nvlink_ecc_errors"] { + // Data link receive data ECC error counter + y, err := lp.New("nv_nvlink_ecc_errors_sum", device.tags, device.meta, map[string]interface{}{"value": aggregate_ecc_errors}, time.Now()) + if err == nil { + y.AddTag("stype", "nvlink") + output <- y + } + } + if !device.excludeMetrics["nv_nvlink_replay_errors"] { + // Data link transmit replay error counter + y, err := lp.New("nv_nvlink_replay_errors_sum", device.tags, device.meta, map[string]interface{}{"value": aggregate_replay_errors}, time.Now()) + if err == nil { + y.AddTag("stype", "nvlink") + output <- y + } + } + if !device.excludeMetrics["nv_nvlink_recovery_errors"] { + // Data link transmit recovery error counter + y, err := lp.New("nv_nvlink_recovery_errors_sum", device.tags, device.meta, map[string]interface{}{"value": aggregate_recovery_errors}, time.Now()) + if err == nil { + y.AddTag("stype", "nvlink") + output <- y + } + } + if !device.excludeMetrics["nv_nvlink_crc_flit_errors"] { + // Data link receive flow control digit CRC error counter + y, err := lp.New("nv_nvlink_crc_flit_errors_sum", device.tags, device.meta, map[string]interface{}{"value": aggregate_crc_flit_errors}, time.Now()) + if err == nil { + y.AddTag("stype", "nvlink") + output <- y + } + } return nil }