diff --git a/collectors/collectorManager.go b/collectors/collectorManager.go index 679bdd2..1cff613 100644 --- a/collectors/collectorManager.go +++ b/collectors/collectorManager.go @@ -50,6 +50,7 @@ var AvailableCollectors = map[string]MetricCollector{ "nfsiostat": new(NfsIOStatCollector), "slurm_cgroup": new(SlurmCgroupCollector), "smartmon": new(SmartMonCollector), + "lenovo_dense_power": new(LenovoDensePowerCollector), } // Metric collector manager data structure diff --git a/collectors/lenovoDensePower.go b/collectors/lenovoDensePower.go new file mode 100644 index 0000000..4954f9e --- /dev/null +++ b/collectors/lenovoDensePower.go @@ -0,0 +1,155 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +// additional authors: +// Michael Panzlaff (NHR@FAU) + +package collectors + +import ( + "bytes" + "encoding/json" + "encoding/hex" + "fmt" + "os/exec" + "strconv" + "strings" + "time" + + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage" +) + +type LenovoDensePowerCollector struct { + metricCollector + + config struct { + IpmitoolPath string `json:"ipmitool_path"` + Sudo bool `json:"use_sudo"` + } + + ipmitool string + + energyValLast float64 + energyTimeLast time.Time +} + +func (m *LenovoDensePowerCollector) Init(config json.RawMessage) error { + // Check if already initialized + if m.init { + return nil + } + + m.name = "LenovoDensePower" + if err := m.setup(); err != nil { + return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err) + } + m.meta = map[string]string{ + "source": m.name, + "group": "IPMI", + } + + m.config.IpmitoolPath = "ipmitool" + + if len(config) > 0 { + d := json.NewDecoder(bytes.NewReader(config)) + d.DisallowUnknownFields() + if err := d.Decode(&m.config); err != nil { + return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err) + } + } + + m.ipmitool = m.config.IpmitoolPath + + energyVal, energyTime, err := m.readEnergy() + if err != nil { + return fmt.Errorf("Energy reading test failed: %w", err) + } + + m.energyValLast = energyVal + m.energyTimeLast = energyTime + + return nil +} + +func (m *LenovoDensePowerCollector) readEnergy() (float64, time.Time, error) { + argv := make([]string, 0) + if m.config.Sudo { + argv = append(argv, "sudo", "-n") + } + + lenovoRequestEnergyMsg := []uint8{0x3a, 0x32, 4, 2, 0, 0, 0} + + argv = append(argv, m.ipmitool, "raw") + + for _, val := range lenovoRequestEnergyMsg { + argv = append(argv, strconv.Itoa(int(val))) + } + + cmd := exec.Command(argv[0], argv[1:]...) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + return 0, time.Unix(0, 0), fmt.Errorf("Failed to run ipmitool: %w (stdout=%s stderr=%s)", err, stdout.String(), stderr.String()) + } + + data, err := hex.DecodeString(strings.ReplaceAll(stdout.String(), " ", "")) + if err != nil { + return 0, time.Unix(0, 0), fmt.Errorf("Unable to decode ipmitool hex output: %w (stdout=%s)", err, stdout.String()) + } + + if len(data) != 14 { + return 0, time.Unix(0, 0), fmt.Errorf("Result must be 14 bytes as specified in the documentation") + } + + wholeJoules := (uint32(data[2]) << 0) | (uint32(data[3]) << 8) | (uint32(data[4]) << 16) | (uint32(data[5]) << 24) + milliJoules := uint16(data[6]) | (uint16(data[7]) << 8) + finalJoules := float64(wholeJoules) + float64(milliJoules) * 1e-3 + + wholeSeconds := (uint32(data[8]) << 0) | (uint32(data[9]) << 8) | (uint32(data[10]) << 16) | (uint32(data[11]) << 24) + milliSeconds := uint16(data[12]) | (uint16(data[13]) << 8) + finalTime := time.Unix(int64(wholeSeconds), (int64(milliSeconds) * 1000000)) + + return finalJoules, finalTime, nil +} + +func (m *LenovoDensePowerCollector) Read(interval time.Duration, output chan lp.CCMessage) { + // Check if already initialized + if !m.init { + return + } + + energyVal, energyTime, err := m.readEnergy() + if err != nil { + cclog.ComponentErrorf(m.name, "readEnergy failed: %v", err) + return + } + + powerVal := 0.0 + if energyTime.After(m.energyTimeLast) { + energyValDiff := energyVal - m.energyValLast + energyTimeDiff := energyTime.Sub(m.energyTimeLast) + powerVal = energyValDiff / energyTimeDiff.Seconds() + + m.energyValLast = energyVal + m.energyTimeLast = energyTime + } + + metric, err := lp.NewMetric("node_power", map[string]string{"type": "node"}, m.meta, powerVal, energyTime) + if err != nil { + cclog.ComponentErrorf(m.name, "lp.NewMetric failed: %v", err) + return + } + metric.AddMeta("unit", "Watts") + output <- metric +} + +func (m *LenovoDensePowerCollector) Close() { + m.init = false +} diff --git a/collectors/lenovoDensePower.md b/collectors/lenovoDensePower.md new file mode 100644 index 0000000..a226330 --- /dev/null +++ b/collectors/lenovoDensePower.md @@ -0,0 +1,47 @@ + + +## `lenovo_dense_power` collector + +```json + "lenovo_dense_power": { + "ipmitool_path": "/path/to/ipmitool", + "use_sudo": true, + "include_metrics" : [] + } +``` + +The `lenovo_dense_power` collector reads power from Lenovo Dense machines via `ipmitool` (`ipmitool raw ...`). This collector is known to only work on the following machines. Others are not supported, so use at your *OWN* risk: + +System | Compatibility | Power measured +--- | --- | --- +Lenovo ThinkSystem SD650 V2/V3 | untested, but should work | node + riser1 + riser2 +Lenovo ThinkSystem SD650-N V2/ SD650-I V3 | untested, but should work | node + riser1 + riser2 + gpus +Lenovo ThinkSystem SD665-N V3 | tested, known to work | node + riser1 + riser2 + gpus + +To test if your system *may* be supported, you can run the following command *at your own risk*: + +``` +ipmitool raw 0x3a 50 4 2 0 0 0 +``` + +`ipmitool` typically require root to run. +In order to run `cc-metric-collector` without root priviliges, you can enable `use_sudo`. +Add a file like this in `/etc/sudoers.d/` to allow `cc-metric-collector` to run the required commands: + +``` +# Do not log the following sudo commands from monitoring, since this causes a lot of log spam. +# However keep log_denied enabled, to detect failures +Defaults: monitoring !log_allowed, !pam_session + +# Allow to use ipmitool for Lenovo power readings +monitoring ALL = (root) NOPASSWD:/usr/bin/ipmitool raw 58 50 4 2 0 0 0 +```