Merge pull request #230 from ClusterCockpit/helma-collectors

NHR@FAU Helma energy collectors
This commit is contained in:
Michael Panzlaff
2026-07-07 16:57:23 +02:00
committed by GitHub
6 changed files with 499 additions and 35 deletions
+31 -29
View File
@@ -21,35 +21,37 @@ import (
// Map of all available metric collectors
var AvailableCollectors = map[string]MetricCollector{
"likwid": new(LikwidCollector),
"loadavg": new(LoadavgCollector),
"memstat": new(MemstatCollector),
"netstat": new(NetstatCollector),
"ibstat": new(InfinibandCollector),
"lustrestat": new(LustreCollector),
"cpustat": new(CpustatCollector),
"topprocs": new(TopProcsCollector),
"nvidia": new(NvidiaCollector),
"customcmd": new(CustomCmdCollector),
"iostat": new(IOstatCollector),
"diskstat": new(DiskstatCollector),
"tempstat": new(TempCollector),
"ipmistat": new(IpmiCollector),
"gpfs": new(GpfsCollector),
"cpufreq": new(CPUFreqCollector),
"cpufreq_cpuinfo": new(CPUFreqCpuInfoCollector),
"nfs3stat": new(Nfs3Collector),
"nfs4stat": new(Nfs4Collector),
"numastats": new(NUMAStatsCollector),
"beegfs_meta": new(BeegfsMetaCollector),
"beegfs_storage": new(BeegfsStorageCollector),
"rapl": new(RAPLCollector),
"rocm_smi": new(RocmSmiCollector),
"self": new(SelfCollector),
"schedstat": new(SchedstatCollector),
"nfsiostat": new(NfsIOStatCollector),
"slurm_cgroup": new(SlurmCgroupCollector),
"smartmon": new(SmartMonCollector),
"likwid": new(LikwidCollector),
"loadavg": new(LoadavgCollector),
"memstat": new(MemstatCollector),
"netstat": new(NetstatCollector),
"ibstat": new(InfinibandCollector),
"lustrestat": new(LustreCollector),
"cpustat": new(CpustatCollector),
"topprocs": new(TopProcsCollector),
"nvidia": new(NvidiaCollector),
"customcmd": new(CustomCmdCollector),
"iostat": new(IOstatCollector),
"diskstat": new(DiskstatCollector),
"tempstat": new(TempCollector),
"ipmistat": new(IpmiCollector),
"gpfs": new(GpfsCollector),
"cpufreq": new(CPUFreqCollector),
"cpufreq_cpuinfo": new(CPUFreqCpuInfoCollector),
"nfs3stat": new(Nfs3Collector),
"nfs4stat": new(Nfs4Collector),
"numastats": new(NUMAStatsCollector),
"beegfs_meta": new(BeegfsMetaCollector),
"beegfs_storage": new(BeegfsStorageCollector),
"rapl": new(RAPLCollector),
"rocm_smi": new(RocmSmiCollector),
"self": new(SelfCollector),
"schedstat": new(SchedstatCollector),
"nfsiostat": new(NfsIOStatCollector),
"slurm_cgroup": new(SlurmCgroupCollector),
"smartmon": new(SmartMonCollector),
"lenovo_dense_power": new(LenovoDensePowerCollector),
"megware_eureka": new(MegwareEurekaCollector),
}
// Metric collector manager data structure
+6 -6
View File
@@ -28,14 +28,14 @@ type IpmiCollector struct {
metricCollector
config struct {
IpmitoolPath string `json:"ipmitool_path"`
IpmisensorsPath string `json:"ipmisensors_path"`
Sudo bool `json:"use_sudo"`
IpmitoolPath string `json:"ipmitool_path"`
IpmisensorsPath string `json:"ipmisensors_path"`
Sudo bool `json:"use_sudo"`
IncludeMetrics []string `json:"include_metrics"`
}
ipmitool string
ipmisensors string
ipmitool string
ipmisensors string
includeMetrics map[string]bool
}
@@ -158,7 +158,7 @@ func (m *IpmiCollector) readIpmiTool(output chan lp.CCMessage) error {
name := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(lv[0]), " ", "_"))
if len(m.includeMetrics) > 0 && !m.includeMetrics[name] {
continue
continue
}
unit := strings.TrimSpace(lv[2])
+170
View File
@@ -0,0 +1,170 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// additional authors:
// Michael Panzlaff (NHR@FAU)
package collectors
import (
"bytes"
"encoding/hex"
"encoding/json"
"fmt"
"os/exec"
"strconv"
"strings"
"time"
cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
)
type LenovoDensePowerCollector struct {
metricCollector
config struct {
IpmitoolPath string `json:"ipmitool_path"`
Sudo bool `json:"use_sudo"`
}
ipmitool string
energyValLast float64
energyTimeLast time.Time
}
func (m *LenovoDensePowerCollector) Init(config json.RawMessage) error {
// Check if already initialized
if m.init {
return nil
}
m.name = "LenovoDensePowerCollector"
if err := m.setup(); err != nil {
return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err)
}
m.meta = map[string]string{
"source": m.name,
"group": "IPMI",
}
m.config.IpmitoolPath = "ipmitool"
if len(config) > 0 {
d := json.NewDecoder(bytes.NewReader(config))
d.DisallowUnknownFields()
if err := d.Decode(&m.config); err != nil {
return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err)
}
}
m.ipmitool = m.config.IpmitoolPath
energyVal, energyTime, err := m.readEnergy()
if err != nil {
return fmt.Errorf("energy reading test failed: %w", err)
}
m.energyValLast = energyVal
m.energyTimeLast = energyTime
m.init = true
return nil
}
func (m *LenovoDensePowerCollector) readEnergy() (float64, time.Time, error) {
argv := make([]string, 0)
if m.config.Sudo {
argv = append(argv, "sudo", "-n")
}
// The Lenovo hardware wants this magic byte string
lenovoRequestEnergyMsg := []uint8{0x3a, 0x32, 4, 2, 0, 0, 0}
argv = append(argv, m.ipmitool, "raw")
for _, val := range lenovoRequestEnergyMsg {
argv = append(argv, strconv.Itoa(int(val)))
}
cmd := exec.Command(argv[0], argv[1:]...)
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if err != nil {
return 0, time.Unix(0, 0), fmt.Errorf("failed to run ipmitool: %w (stdout=%s stderr=%s)", err, stdout.String(), stderr.String())
}
data, err := hex.DecodeString(strings.ReplaceAll(strings.TrimSpace(stdout.String()), " ", ""))
if err != nil {
return 0, time.Unix(0, 0), fmt.Errorf("unable to decode ipmitool hex output: %w (stdout=%s)", err, stdout.String())
}
if len(data) != 14 {
return 0, time.Unix(0, 0), fmt.Errorf("result must be 14 bytes as specified in the documentation")
}
// data is laid out like this:
// data[0 to 1]: FIFO index of value (for debug only)
// data[2 to 5]: Integer part of energy in J (little endian)
// data[6 to 7]: Fraction part of energy in mJ (little endian)
// data[8 to 11]: Integer part of Unix time of measurement in seconds (little endian)
// data[12 to 13]: Fraction part of Unix time of measurement in milliseconds (little endian)
wholeJoules := (uint32(data[2]) << 0) | (uint32(data[3]) << 8) | (uint32(data[4]) << 16) | (uint32(data[5]) << 24)
milliJoules := uint16(data[6]) | (uint16(data[7]) << 8)
finalJoules := float64(wholeJoules) + float64(milliJoules)*1e-3
wholeSeconds := (uint32(data[8]) << 0) | (uint32(data[9]) << 8) | (uint32(data[10]) << 16) | (uint32(data[11]) << 24)
milliSeconds := uint16(data[12]) | (uint16(data[13]) << 8)
finalTime := time.Unix(int64(wholeSeconds), (int64(milliSeconds) * 1000000))
return finalJoules, finalTime, nil
}
func (m *LenovoDensePowerCollector) Read(interval time.Duration, output chan lp.CCMessage) {
// Check if already initialized
if !m.init {
return
}
energyVal, energyTime, err := m.readEnergy()
if err != nil {
cclog.ComponentErrorf(m.name, "readEnergy failed: %v", err)
return
}
powerVal := 0.0
if energyTime.Compare(m.energyTimeLast) != 0 {
// Check for overflow
energyValDiff := int64(energyVal) - int64(m.energyValLast)
if energyVal < m.energyValLast {
energyValDiff += int64(0x100000000)
}
energyTimeDiff := energyTime.Sub(m.energyTimeLast)
if energyTime.Before(m.energyTimeLast) {
energyTimeDiff = energyTimeDiff + 0x100000000*time.Second
}
powerVal = float64(energyValDiff) / energyTimeDiff.Seconds()
m.energyValLast = energyVal
m.energyTimeLast = energyTime
}
metric, err := lp.NewMetric("lenovo_node_power", map[string]string{"type": "node"}, m.meta, powerVal, energyTime)
if err != nil {
cclog.ComponentErrorf(m.name, "lp.NewMetric failed: %v", err)
return
}
metric.AddMeta("unit", "Watts")
output <- metric
}
func (m *LenovoDensePowerCollector) Close() {
m.init = false
}
+51
View File
@@ -0,0 +1,51 @@
<!--
---
title: Lenovo Dense Power collector
description: Collect power of Lenovo Dense machines using ipmitool raw
categories: [cc-metric-collector]
tags: ['Admin']
weight: 2
hugo_path: docs/reference/cc-metric-collector/collectors/lenovoDensePower.md
---
-->
## `lenovo_dense_power` collector
```json
"lenovo_dense_power": {
"ipmitool_path": "/path/to/ipmitool",
"use_sudo": true,
"include_metrics" : []
}
```
The `lenovo_dense_power` collector reads power from Lenovo Dense machines via `ipmitool` (`ipmitool raw ...`). This collector is known to only work on the following machines. Others are not supported, so use at your *OWN* risk:
System | Compatibility | Power measured
--- | --- | ---
Lenovo ThinkSystem SD650 V2/V3 | untested, but should work | node + riser1 + riser2
Lenovo ThinkSystem SD650-N V2/ SD650-I V3 | untested, but should work | node + riser1 + riser2 + gpus
Lenovo ThinkSystem SD665-N V3 | tested, known to work | node + riser1 + riser2 + gpus
To test if your system *may* be supported, you can run the following command *at your own risk*:
```
$ ipmitool raw 58 50 4 2 0 0 0
c4 00 83 a9 cb d3 a2 03 df db 47 6a d8 01
```
Exactly 14 bytes should be returned.
In case not or another error occurs your hardware is likely not supported.
In addition, `ipmitool` typically require root to run.
In order to run `cc-metric-collector` without root priviliges, you can enable `use_sudo`.
Add a file like this in `/etc/sudoers.d/` to allow `cc-metric-collector` to run the required commands:
```
# Do not log the following sudo commands from monitoring, since this causes a lot of log spam.
# However keep log_denied enabled, to detect failures
Defaults: monitoring !log_allowed, !pam_session
# Allow to use ipmitool for Lenovo power readings
monitoring ALL = (root) NOPASSWD:/usr/bin/ipmitool raw 58 50 4 2 0 0 0
```
+197
View File
@@ -0,0 +1,197 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// additional authors:
// Michael Panzlaff (NHR@FAU)
package collectors
import (
"bytes"
"encoding/json"
"fmt"
"os/exec"
"time"
cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
)
// MPS refers to Monolithic Power Systems, from which the MP5922 is used
// to measure telemetry on the Eureka platform.
type mpsData struct {
Cnt int64 `json:"Cnt"` // No idea what this is, ignore
Energy float64 `json:"Energy"`
Vin float64 `json:"Vin"`
Iin float64 `json:"Iin"`
Pin float64 `json:"Pin"`
PinAvg float64 `json:"PinAvg"`
Vout float64 `json:"Vout"`
Iout float64 `json:"Iout"`
Pout float64 `json:"Pout"`
StandbyVout float64 `json:"StandbyVout"`
StandbyIout float64 `json:"StandbyIout"`
StandbyPout float64 `json:"StandbyPout"`
TempBusbar float64 `json:"TempBusbar"`
TempSsd float64 `json:"TempSsd"`
TempMps float64 `json:"TempMps"`
// No idea what the ones below mean exactl. Ignore them for now.
EnergyTime int64 `json:"EnergyTime"`
EnergyAccumulator int64 `json:"EnergyAccumulator"`
EnergyRolloverCnt int64 `json:"EnergyRolloverCnt"`
EnergySampleCntU24 int64 `json:"EnergySampleCntU24"`
Timestamp time.Time
}
type MegwareEurekaCollector struct {
metricCollector
config struct {
U20Path string `json:"u20_path"`
Sudo bool `json:"use_sudo"`
}
u20path string
energyValLast float64
energyTimeLast time.Time
}
func (m *MegwareEurekaCollector) Init(config json.RawMessage) error {
// Check if already initialized
if m.init {
return nil
}
m.name = "MegwareEureka"
if err := m.setup(); err != nil {
return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err)
}
m.meta = map[string]string{
"source": m.name,
"group": "U20",
}
m.config.U20Path = "u20"
if len(config) > 0 {
d := json.NewDecoder(bytes.NewReader(config))
d.DisallowUnknownFields()
if err := d.Decode(&m.config); err != nil {
return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err)
}
}
m.u20path = m.config.U20Path
data, err := m.readMpsData()
if err != nil {
return fmt.Errorf("energy reading test failed: %w", err)
}
m.energyValLast = data.Energy
m.energyTimeLast = time.Now()
m.init = true
return nil
}
func (m *MegwareEurekaCollector) readMpsData() (*mpsData, error) {
argv := make([]string, 0)
if m.config.Sudo {
argv = append(argv, "sudo", "-n")
}
argv = append(argv, m.u20path, "values", "--read", "GET_MPS_POLL_VALUES")
cmd := exec.Command(argv[0], argv[1:]...)
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if err != nil {
return nil, fmt.Errorf("failed to run u20: %w (stdout=%s stderr=%s)", err, stdout.String(), stderr.String())
}
var u20output struct {
GetMpsPollValues mpsData `json:"GET_MPS_POLL_VALUES"`
}
err = json.Unmarshal(stdout.Bytes(), &u20output)
if err != nil {
return nil, fmt.Errorf("unable to decode u20 JSON output: %w (stdout=%s)", err, stdout.String())
}
fmt.Printf("string: %+v\n", stdout.String())
fmt.Printf("obj: %+v\n", u20output)
u20output.GetMpsPollValues.Timestamp = time.Now()
return &u20output.GetMpsPollValues, nil
}
func (m *MegwareEurekaCollector) Read(interval time.Duration, output chan lp.CCMessage) {
// Check if already initialized
if !m.init {
return
}
data, err := m.readMpsData()
if err != nil {
cclog.ComponentErrorf(m.name, "readMpsData failed: %v", err)
return
}
powerVal := 0.0
if data.Timestamp.After(m.energyTimeLast) {
// Important, m.energy comes in Wh, so multiply by 3600 to get Ws (aka Joule)
energyValDiff := data.Energy - m.energyValLast
energyTimeDiff := data.Timestamp.Sub(m.energyTimeLast)
powerVal = energyValDiff * 3600 / energyTimeDiff.Seconds()
m.energyValLast = data.Energy
m.energyTimeLast = data.Timestamp
}
metricNamePrefix := "eureka_"
metricMap := map[string]struct {
value any
unit string
}{
"power": {value: powerVal, unit: "Watts"},
"vin": {value: data.Vin, unit: "Volts"},
"iin": {value: data.Iin, unit: "Amperes"},
"pin": {value: data.Pin, unit: "Watts"},
"pin_avg": {value: data.PinAvg, unit: "Watts"},
"vout": {value: data.Vout, unit: "Volts"},
"iout": {value: data.Iout, unit: "Amperes"},
"pout": {value: data.Pout, unit: "Watts"},
"standby_vout": {value: data.StandbyVout, unit: "Volts"},
"standby_iout": {value: data.StandbyIout, unit: "Amperes"},
"standby_pout": {value: data.StandbyPout, unit: "Watts"},
"temp_busbar": {value: data.TempBusbar, unit: "degC"},
"temp_ssd": {value: data.TempSsd, unit: "degC"},
"temp_mps": {value: data.TempMps, unit: "degC"},
}
for metricName, metricData := range metricMap {
metricName = metricNamePrefix + metricName
metric, err := lp.NewMetric(metricName, map[string]string{"type": "node"}, m.meta, metricData.value, data.Timestamp)
if err != nil {
cclog.ComponentErrorf(m.name, "lp.NewMetric failed: %v", err)
return
}
metric.AddMeta("unit", metricData.unit)
output <- metric
}
}
func (m *MegwareEurekaCollector) Close() {
m.init = false
}
+44
View File
@@ -0,0 +1,44 @@
<!--
---
title: Megware Eureka collector
description: Collect power and other metrics of Megware Eureka machines using u20
categories: [cc-metric-collector]
tags: ['Admin']
weight: 2
hugo_path: docs/reference/cc-metric-collector/collectors/lenovoDensePower.md
---
-->
## `lenovo_dense_power` collector
```json
"megware_eureka": {
"u20_path": "/path/to/ipmitool",
"use_sudo": true
}
```
The `megware_eureka` collector reads power and other metrics from Megware Eureka machines via `u20`.
If the u20 tool is available for your machines, it's possible that this collector is compatible.
If you don't have access to u20, please contact your Megware sales person.
You can test the collector compatibility by running the following command:
```
$ u20 values --read GET_MPS_POLL_VALUES
```
If this returns a JSON with energy, voltage and current readings, you're fine.
In addition, `u20` typically requires root to run.
In order to run `cc-metric-collector` without root priviliges, you can enable `use_sudo`.
Add a file like this in `/etc/sudoers.d/` to allow `cc-metric-collector` to run the required commands:
```
# Do not log the following sudo commands from monitoring, since this causes a lot of log spam.
# However keep log_denied enabled, to detect failures
Defaults: monitoring !log_allowed, !pam_session
# Allow to use u20 for Megware power readings
monitoring ALL = (root) NOPASSWD:/usr/bin/u20 values --read GET_MPS_POLL_VALUES
```