diff --git a/collectors/collectorManager.go b/collectors/collectorManager.go index 679bdd2..9cce461 100644 --- a/collectors/collectorManager.go +++ b/collectors/collectorManager.go @@ -21,35 +21,37 @@ import ( // Map of all available metric collectors var AvailableCollectors = map[string]MetricCollector{ - "likwid": new(LikwidCollector), - "loadavg": new(LoadavgCollector), - "memstat": new(MemstatCollector), - "netstat": new(NetstatCollector), - "ibstat": new(InfinibandCollector), - "lustrestat": new(LustreCollector), - "cpustat": new(CpustatCollector), - "topprocs": new(TopProcsCollector), - "nvidia": new(NvidiaCollector), - "customcmd": new(CustomCmdCollector), - "iostat": new(IOstatCollector), - "diskstat": new(DiskstatCollector), - "tempstat": new(TempCollector), - "ipmistat": new(IpmiCollector), - "gpfs": new(GpfsCollector), - "cpufreq": new(CPUFreqCollector), - "cpufreq_cpuinfo": new(CPUFreqCpuInfoCollector), - "nfs3stat": new(Nfs3Collector), - "nfs4stat": new(Nfs4Collector), - "numastats": new(NUMAStatsCollector), - "beegfs_meta": new(BeegfsMetaCollector), - "beegfs_storage": new(BeegfsStorageCollector), - "rapl": new(RAPLCollector), - "rocm_smi": new(RocmSmiCollector), - "self": new(SelfCollector), - "schedstat": new(SchedstatCollector), - "nfsiostat": new(NfsIOStatCollector), - "slurm_cgroup": new(SlurmCgroupCollector), - "smartmon": new(SmartMonCollector), + "likwid": new(LikwidCollector), + "loadavg": new(LoadavgCollector), + "memstat": new(MemstatCollector), + "netstat": new(NetstatCollector), + "ibstat": new(InfinibandCollector), + "lustrestat": new(LustreCollector), + "cpustat": new(CpustatCollector), + "topprocs": new(TopProcsCollector), + "nvidia": new(NvidiaCollector), + "customcmd": new(CustomCmdCollector), + "iostat": new(IOstatCollector), + "diskstat": new(DiskstatCollector), + "tempstat": new(TempCollector), + "ipmistat": new(IpmiCollector), + "gpfs": new(GpfsCollector), + "cpufreq": new(CPUFreqCollector), + "cpufreq_cpuinfo": new(CPUFreqCpuInfoCollector), + "nfs3stat": new(Nfs3Collector), + "nfs4stat": new(Nfs4Collector), + "numastats": new(NUMAStatsCollector), + "beegfs_meta": new(BeegfsMetaCollector), + "beegfs_storage": new(BeegfsStorageCollector), + "rapl": new(RAPLCollector), + "rocm_smi": new(RocmSmiCollector), + "self": new(SelfCollector), + "schedstat": new(SchedstatCollector), + "nfsiostat": new(NfsIOStatCollector), + "slurm_cgroup": new(SlurmCgroupCollector), + "smartmon": new(SmartMonCollector), + "lenovo_dense_power": new(LenovoDensePowerCollector), + "megware_eureka": new(MegwareEurekaCollector), } // Metric collector manager data structure diff --git a/collectors/ipmiMetric.go b/collectors/ipmiMetric.go index b15aa63..51aceb7 100644 --- a/collectors/ipmiMetric.go +++ b/collectors/ipmiMetric.go @@ -28,14 +28,14 @@ type IpmiCollector struct { metricCollector config struct { - IpmitoolPath string `json:"ipmitool_path"` - IpmisensorsPath string `json:"ipmisensors_path"` - Sudo bool `json:"use_sudo"` + IpmitoolPath string `json:"ipmitool_path"` + IpmisensorsPath string `json:"ipmisensors_path"` + Sudo bool `json:"use_sudo"` IncludeMetrics []string `json:"include_metrics"` } - ipmitool string - ipmisensors string + ipmitool string + ipmisensors string includeMetrics map[string]bool } @@ -158,7 +158,7 @@ func (m *IpmiCollector) readIpmiTool(output chan lp.CCMessage) error { name := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(lv[0]), " ", "_")) if len(m.includeMetrics) > 0 && !m.includeMetrics[name] { - continue + continue } unit := strings.TrimSpace(lv[2]) diff --git a/collectors/lenovoDensePower.go b/collectors/lenovoDensePower.go new file mode 100644 index 0000000..c0980bb --- /dev/null +++ b/collectors/lenovoDensePower.go @@ -0,0 +1,170 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +// additional authors: +// Michael Panzlaff (NHR@FAU) + +package collectors + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "os/exec" + "strconv" + "strings" + "time" + + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage" +) + +type LenovoDensePowerCollector struct { + metricCollector + + config struct { + IpmitoolPath string `json:"ipmitool_path"` + Sudo bool `json:"use_sudo"` + } + + ipmitool string + + energyValLast float64 + energyTimeLast time.Time +} + +func (m *LenovoDensePowerCollector) Init(config json.RawMessage) error { + // Check if already initialized + if m.init { + return nil + } + + m.name = "LenovoDensePowerCollector" + if err := m.setup(); err != nil { + return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err) + } + m.meta = map[string]string{ + "source": m.name, + "group": "IPMI", + } + + m.config.IpmitoolPath = "ipmitool" + + if len(config) > 0 { + d := json.NewDecoder(bytes.NewReader(config)) + d.DisallowUnknownFields() + if err := d.Decode(&m.config); err != nil { + return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err) + } + } + + m.ipmitool = m.config.IpmitoolPath + + energyVal, energyTime, err := m.readEnergy() + if err != nil { + return fmt.Errorf("energy reading test failed: %w", err) + } + + m.energyValLast = energyVal + m.energyTimeLast = energyTime + m.init = true + + return nil +} + +func (m *LenovoDensePowerCollector) readEnergy() (float64, time.Time, error) { + argv := make([]string, 0) + if m.config.Sudo { + argv = append(argv, "sudo", "-n") + } + + // The Lenovo hardware wants this magic byte string + lenovoRequestEnergyMsg := []uint8{0x3a, 0x32, 4, 2, 0, 0, 0} + + argv = append(argv, m.ipmitool, "raw") + + for _, val := range lenovoRequestEnergyMsg { + argv = append(argv, strconv.Itoa(int(val))) + } + + cmd := exec.Command(argv[0], argv[1:]...) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + return 0, time.Unix(0, 0), fmt.Errorf("failed to run ipmitool: %w (stdout=%s stderr=%s)", err, stdout.String(), stderr.String()) + } + + data, err := hex.DecodeString(strings.ReplaceAll(strings.TrimSpace(stdout.String()), " ", "")) + if err != nil { + return 0, time.Unix(0, 0), fmt.Errorf("unable to decode ipmitool hex output: %w (stdout=%s)", err, stdout.String()) + } + + if len(data) != 14 { + return 0, time.Unix(0, 0), fmt.Errorf("result must be 14 bytes as specified in the documentation") + } + + // data is laid out like this: + // data[0 to 1]: FIFO index of value (for debug only) + // data[2 to 5]: Integer part of energy in J (little endian) + // data[6 to 7]: Fraction part of energy in mJ (little endian) + // data[8 to 11]: Integer part of Unix time of measurement in seconds (little endian) + // data[12 to 13]: Fraction part of Unix time of measurement in milliseconds (little endian) + wholeJoules := (uint32(data[2]) << 0) | (uint32(data[3]) << 8) | (uint32(data[4]) << 16) | (uint32(data[5]) << 24) + milliJoules := uint16(data[6]) | (uint16(data[7]) << 8) + finalJoules := float64(wholeJoules) + float64(milliJoules)*1e-3 + + wholeSeconds := (uint32(data[8]) << 0) | (uint32(data[9]) << 8) | (uint32(data[10]) << 16) | (uint32(data[11]) << 24) + milliSeconds := uint16(data[12]) | (uint16(data[13]) << 8) + finalTime := time.Unix(int64(wholeSeconds), (int64(milliSeconds) * 1000000)) + + return finalJoules, finalTime, nil +} + +func (m *LenovoDensePowerCollector) Read(interval time.Duration, output chan lp.CCMessage) { + // Check if already initialized + if !m.init { + return + } + + energyVal, energyTime, err := m.readEnergy() + if err != nil { + cclog.ComponentErrorf(m.name, "readEnergy failed: %v", err) + return + } + + powerVal := 0.0 + if energyTime.Compare(m.energyTimeLast) != 0 { + // Check for overflow + energyValDiff := int64(energyVal) - int64(m.energyValLast) + if energyVal < m.energyValLast { + energyValDiff += int64(0x100000000) + } + energyTimeDiff := energyTime.Sub(m.energyTimeLast) + if energyTime.Before(m.energyTimeLast) { + energyTimeDiff = energyTimeDiff + 0x100000000*time.Second + } + powerVal = float64(energyValDiff) / energyTimeDiff.Seconds() + + m.energyValLast = energyVal + m.energyTimeLast = energyTime + } + + metric, err := lp.NewMetric("lenovo_node_power", map[string]string{"type": "node"}, m.meta, powerVal, energyTime) + if err != nil { + cclog.ComponentErrorf(m.name, "lp.NewMetric failed: %v", err) + return + } + metric.AddMeta("unit", "Watts") + output <- metric +} + +func (m *LenovoDensePowerCollector) Close() { + m.init = false +} diff --git a/collectors/lenovoDensePower.md b/collectors/lenovoDensePower.md new file mode 100644 index 0000000..609e8d3 --- /dev/null +++ b/collectors/lenovoDensePower.md @@ -0,0 +1,51 @@ + + +## `lenovo_dense_power` collector + +```json + "lenovo_dense_power": { + "ipmitool_path": "/path/to/ipmitool", + "use_sudo": true, + "include_metrics" : [] + } +``` + +The `lenovo_dense_power` collector reads power from Lenovo Dense machines via `ipmitool` (`ipmitool raw ...`). This collector is known to only work on the following machines. Others are not supported, so use at your *OWN* risk: + +System | Compatibility | Power measured +--- | --- | --- +Lenovo ThinkSystem SD650 V2/V3 | untested, but should work | node + riser1 + riser2 +Lenovo ThinkSystem SD650-N V2/ SD650-I V3 | untested, but should work | node + riser1 + riser2 + gpus +Lenovo ThinkSystem SD665-N V3 | tested, known to work | node + riser1 + riser2 + gpus + +To test if your system *may* be supported, you can run the following command *at your own risk*: + +``` +$ ipmitool raw 58 50 4 2 0 0 0 + c4 00 83 a9 cb d3 a2 03 df db 47 6a d8 01 +``` + +Exactly 14 bytes should be returned. +In case not or another error occurs your hardware is likely not supported. + +In addition, `ipmitool` typically require root to run. +In order to run `cc-metric-collector` without root priviliges, you can enable `use_sudo`. +Add a file like this in `/etc/sudoers.d/` to allow `cc-metric-collector` to run the required commands: + +``` +# Do not log the following sudo commands from monitoring, since this causes a lot of log spam. +# However keep log_denied enabled, to detect failures +Defaults: monitoring !log_allowed, !pam_session + +# Allow to use ipmitool for Lenovo power readings +monitoring ALL = (root) NOPASSWD:/usr/bin/ipmitool raw 58 50 4 2 0 0 0 +``` diff --git a/collectors/megwareEureka.go b/collectors/megwareEureka.go new file mode 100644 index 0000000..d7b255f --- /dev/null +++ b/collectors/megwareEureka.go @@ -0,0 +1,197 @@ +// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +// additional authors: +// Michael Panzlaff (NHR@FAU) + +package collectors + +import ( + "bytes" + "encoding/json" + "fmt" + "os/exec" + "time" + + cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" + lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage" +) + +// MPS refers to Monolithic Power Systems, from which the MP5922 is used +// to measure telemetry on the Eureka platform. +type mpsData struct { + Cnt int64 `json:"Cnt"` // No idea what this is, ignore + Energy float64 `json:"Energy"` + Vin float64 `json:"Vin"` + Iin float64 `json:"Iin"` + Pin float64 `json:"Pin"` + PinAvg float64 `json:"PinAvg"` + Vout float64 `json:"Vout"` + Iout float64 `json:"Iout"` + Pout float64 `json:"Pout"` + StandbyVout float64 `json:"StandbyVout"` + StandbyIout float64 `json:"StandbyIout"` + StandbyPout float64 `json:"StandbyPout"` + TempBusbar float64 `json:"TempBusbar"` + TempSsd float64 `json:"TempSsd"` + TempMps float64 `json:"TempMps"` + // No idea what the ones below mean exactl. Ignore them for now. + EnergyTime int64 `json:"EnergyTime"` + EnergyAccumulator int64 `json:"EnergyAccumulator"` + EnergyRolloverCnt int64 `json:"EnergyRolloverCnt"` + EnergySampleCntU24 int64 `json:"EnergySampleCntU24"` + + Timestamp time.Time +} + +type MegwareEurekaCollector struct { + metricCollector + + config struct { + U20Path string `json:"u20_path"` + Sudo bool `json:"use_sudo"` + } + + u20path string + + energyValLast float64 + energyTimeLast time.Time +} + +func (m *MegwareEurekaCollector) Init(config json.RawMessage) error { + // Check if already initialized + if m.init { + return nil + } + + m.name = "MegwareEureka" + if err := m.setup(); err != nil { + return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err) + } + m.meta = map[string]string{ + "source": m.name, + "group": "U20", + } + + m.config.U20Path = "u20" + + if len(config) > 0 { + d := json.NewDecoder(bytes.NewReader(config)) + d.DisallowUnknownFields() + if err := d.Decode(&m.config); err != nil { + return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err) + } + } + + m.u20path = m.config.U20Path + + data, err := m.readMpsData() + if err != nil { + return fmt.Errorf("energy reading test failed: %w", err) + } + + m.energyValLast = data.Energy + m.energyTimeLast = time.Now() + m.init = true + + return nil +} + +func (m *MegwareEurekaCollector) readMpsData() (*mpsData, error) { + argv := make([]string, 0) + if m.config.Sudo { + argv = append(argv, "sudo", "-n") + } + + argv = append(argv, m.u20path, "values", "--read", "GET_MPS_POLL_VALUES") + + cmd := exec.Command(argv[0], argv[1:]...) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + return nil, fmt.Errorf("failed to run u20: %w (stdout=%s stderr=%s)", err, stdout.String(), stderr.String()) + } + + var u20output struct { + GetMpsPollValues mpsData `json:"GET_MPS_POLL_VALUES"` + } + + err = json.Unmarshal(stdout.Bytes(), &u20output) + if err != nil { + return nil, fmt.Errorf("unable to decode u20 JSON output: %w (stdout=%s)", err, stdout.String()) + } + + fmt.Printf("string: %+v\n", stdout.String()) + fmt.Printf("obj: %+v\n", u20output) + + u20output.GetMpsPollValues.Timestamp = time.Now() + + return &u20output.GetMpsPollValues, nil +} + +func (m *MegwareEurekaCollector) Read(interval time.Duration, output chan lp.CCMessage) { + // Check if already initialized + if !m.init { + return + } + + data, err := m.readMpsData() + if err != nil { + cclog.ComponentErrorf(m.name, "readMpsData failed: %v", err) + return + } + + powerVal := 0.0 + + if data.Timestamp.After(m.energyTimeLast) { + // Important, m.energy comes in Wh, so multiply by 3600 to get Ws (aka Joule) + energyValDiff := data.Energy - m.energyValLast + energyTimeDiff := data.Timestamp.Sub(m.energyTimeLast) + powerVal = energyValDiff * 3600 / energyTimeDiff.Seconds() + + m.energyValLast = data.Energy + m.energyTimeLast = data.Timestamp + } + + metricNamePrefix := "eureka_" + metricMap := map[string]struct { + value any + unit string + }{ + "power": {value: powerVal, unit: "Watts"}, + "vin": {value: data.Vin, unit: "Volts"}, + "iin": {value: data.Iin, unit: "Amperes"}, + "pin": {value: data.Pin, unit: "Watts"}, + "pin_avg": {value: data.PinAvg, unit: "Watts"}, + "vout": {value: data.Vout, unit: "Volts"}, + "iout": {value: data.Iout, unit: "Amperes"}, + "pout": {value: data.Pout, unit: "Watts"}, + "standby_vout": {value: data.StandbyVout, unit: "Volts"}, + "standby_iout": {value: data.StandbyIout, unit: "Amperes"}, + "standby_pout": {value: data.StandbyPout, unit: "Watts"}, + "temp_busbar": {value: data.TempBusbar, unit: "degC"}, + "temp_ssd": {value: data.TempSsd, unit: "degC"}, + "temp_mps": {value: data.TempMps, unit: "degC"}, + } + + for metricName, metricData := range metricMap { + metricName = metricNamePrefix + metricName + metric, err := lp.NewMetric(metricName, map[string]string{"type": "node"}, m.meta, metricData.value, data.Timestamp) + if err != nil { + cclog.ComponentErrorf(m.name, "lp.NewMetric failed: %v", err) + return + } + metric.AddMeta("unit", metricData.unit) + output <- metric + } +} + +func (m *MegwareEurekaCollector) Close() { + m.init = false +} diff --git a/collectors/megwareEureka.md b/collectors/megwareEureka.md new file mode 100644 index 0000000..d9aad67 --- /dev/null +++ b/collectors/megwareEureka.md @@ -0,0 +1,44 @@ + + +## `lenovo_dense_power` collector + +```json + "megware_eureka": { + "u20_path": "/path/to/ipmitool", + "use_sudo": true + } +``` + +The `megware_eureka` collector reads power and other metrics from Megware Eureka machines via `u20`. +If the u20 tool is available for your machines, it's possible that this collector is compatible. +If you don't have access to u20, please contact your Megware sales person. + +You can test the collector compatibility by running the following command: + +``` +$ u20 values --read GET_MPS_POLL_VALUES +``` + +If this returns a JSON with energy, voltage and current readings, you're fine. + +In addition, `u20` typically requires root to run. +In order to run `cc-metric-collector` without root priviliges, you can enable `use_sudo`. +Add a file like this in `/etc/sudoers.d/` to allow `cc-metric-collector` to run the required commands: + +``` +# Do not log the following sudo commands from monitoring, since this causes a lot of log spam. +# However keep log_denied enabled, to detect failures +Defaults: monitoring !log_allowed, !pam_session + +# Allow to use u20 for Megware power readings +monitoring ALL = (root) NOPASSWD:/usr/bin/u20 values --read GET_MPS_POLL_VALUES +```