From 303fe1d80fe841d3be00c5807deb26162ea1d96d Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Wed, 10 Apr 2024 19:57:08 +0200 Subject: [PATCH] Add collector for always running energy measurements with LIKWID --- collectors/collectorManager.go | 1 + collectors/likwidenergyMetric.go | 281 +++++++++++++++++++++++++++++++ collectors/likwidenergyMetric.md | 20 +++ 3 files changed, 302 insertions(+) create mode 100644 collectors/likwidenergyMetric.go create mode 100644 collectors/likwidenergyMetric.md diff --git a/collectors/collectorManager.go b/collectors/collectorManager.go index 4cf3435..5d5177b 100644 --- a/collectors/collectorManager.go +++ b/collectors/collectorManager.go @@ -41,6 +41,7 @@ var AvailableCollectors = map[string]MetricCollector{ "self": new(SelfCollector), "schedstat": new(SchedstatCollector), "nfsiostat": new(NfsIOStatCollector), + "likwidenergy": new(LikwidEnergyCollector), } // Metric collector manager data structure diff --git a/collectors/likwidenergyMetric.go b/collectors/likwidenergyMetric.go new file mode 100644 index 0000000..de43bc5 --- /dev/null +++ b/collectors/likwidenergyMetric.go @@ -0,0 +1,281 @@ +package collectors + +/* +#cgo CFLAGS: -I./likwid +#cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-in-object-files +#include +#include +*/ +import "C" +import ( + "encoding/json" + "fmt" + "os" + "strings" + "time" + + cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger" + lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric" + topo "github.com/ClusterCockpit/cc-metric-collector/pkg/ccTopology" + "github.com/NVIDIA/go-nvml/pkg/dl" +) + +const ( + LIKWIDENERGY_LIB_NAME = "liblikwid.so" + LIKWIDENERGY_LIB_DL_FLAGS = dl.RTLD_LAZY | dl.RTLD_GLOBAL + LIKWIDENERGY_DEF_ACCESSMODE = "direct" + LIKWIDENERGY_DEF_LOCKFILE = "/var/run/likwid.lock" +) + +// These are the fields we read from the JSON configuration +type LikwidEnergyCollectorConfig struct { + AccessMode string `json:"access_mode,omitempty"` + DaemonPath string `json:"accessdaemon_path,omitempty"` + LibraryPath string `json:"liblikwid_path,omitempty"` + LockfilePath string `json:"lockfile_path,omitempty"` + SendDiff bool `json:"send_difference,omitempty"` + SendAbs bool `json:"send_absolute,omitempty"` +} + +type LikwidEnergyDomainEntry struct { + readcpu int + value uint32 + total uint64 + tags map[string]string +} + +type LikwidEnergyDomain struct { + values map[int]LikwidEnergyDomainEntry + granularity string + metricname string + domaintype int + energyUnit float64 +} + +// This contains all variables we need during execution and the variables +// defined by metricCollector (name, init, ...) +type LikwidEnergyCollector struct { + metricCollector + config LikwidEnergyCollectorConfig // the configuration structure + meta map[string]string // default meta information + tags map[string]string // default tags + domains map[int]LikwidEnergyDomain +} + +// Init initializes the sample collector +// Called once by the collector manager +// All tags, meta data tags and metrics that do not change over the runtime should be set here +func (m *LikwidEnergyCollector) Init(config json.RawMessage) error { + var err error = nil + // Always set the name early in Init() to use it in cclog.Component* functions + m.name = "LikwidEnergyCollector" + // This is for later use, also call it early + m.setup() + // Tell whether the collector should be run in parallel with others (reading files, ...) + // or it should be run serially, mostly for collectors actually doing measurements + // because they should not measure the execution of the other collectors + m.parallel = true + // Define meta information sent with each metric + // (Can also be dynamic or this is the basic set with extension through AddMeta()) + m.meta = map[string]string{"source": m.name, "group": "LIKWID", "unit": "Joules"} + // Define tags sent with each metric + // The 'type' tag is always needed, it defines the granularity of the metric + // node -> whole system + // socket -> CPU socket (requires socket ID as 'type-id' tag) + // die -> CPU die (requires CPU die ID as 'type-id' tag) + // memoryDomain -> NUMA domain (requires NUMA domain ID as 'type-id' tag) + // llc -> Last level cache (requires last level cache ID as 'type-id' tag) + // core -> single CPU core that may consist of multiple hardware threads (SMT) (requires core ID as 'type-id' tag) + // hwthtread -> single CPU hardware thread (requires hardware thread ID as 'type-id' tag) + // accelerator -> A accelerator device like GPU or FPGA (requires an accelerator ID as 'type-id' tag) + m.tags = map[string]string{} + // Read in the JSON configuration + m.config.AccessMode = LIKWID_DEF_ACCESSMODE + m.config.LibraryPath = LIKWID_LIB_NAME + m.config.LockfilePath = LIKWID_DEF_LOCKFILE + m.config.SendAbs = true + m.config.SendDiff = true + if len(config) > 0 { + err = json.Unmarshal(config, &m.config) + if err != nil { + cclog.ComponentError(m.name, "Error reading config:", err.Error()) + return err + } + } + cclog.ComponentDebug(m.name, "Opening ", m.config.LibraryPath) + lib := dl.New(m.config.LibraryPath, LIKWID_LIB_DL_FLAGS) + if lib == nil { + return fmt.Errorf("error instantiating DynamicLibrary for %s", m.config.LibraryPath) + } + err = lib.Open() + if err != nil { + return fmt.Errorf("error opening %s: %v", m.config.LibraryPath, err) + } + cclog.ComponentDebug(m.name, "Init topology ", m.config.AccessMode) + ret := C.topology_init() + if ret != 0 { + return fmt.Errorf("error initializing topology: %d", ret) + } + cclog.ComponentDebug(m.name, "Setting accessmode ", m.config.AccessMode) + switch m.config.AccessMode { + case "direct": + C.HPMmode(0) + case "accessdaemon": + if len(m.config.DaemonPath) > 0 { + p := os.Getenv("PATH") + os.Setenv("PATH", m.config.DaemonPath+":"+p) + } + C.HPMmode(1) + retCode := C.HPMinit() + if retCode != 0 { + err := fmt.Errorf("C.HPMinit() failed with return code %v", retCode) + cclog.ComponentError(m.name, err.Error()) + } + } + initCpus := make([]int, 0) + ret = C.HPMaddThread(0) + if ret != 0 { + return fmt.Errorf("error initializing access: %d", ret) + } + initCpus = append(initCpus, 0) + + cinfo := C.get_cpuInfo() + domainnames := make(map[int]string) + if cinfo.isIntel == C.int(1) { + domainnames[0] = "pkg" + domainnames[1] = "pp0" + domainnames[2] = "pp1" + domainnames[3] = "dram" + domainnames[4] = "platform" + } else { + switch cinfo.family { + case 0x17: + domainnames[0] = "core" + domainnames[1] = "pkg" + case 0x19: + switch cinfo.model { + case 0x01, 0x21, 0x50: + domainnames[0] = "core" + domainnames[1] = "pkg" + case 0x61, 0x11: + domainnames[0] = "core" + domainnames[1] = "l3" + } + } + } + + // Set up everything that the collector requires during the Read() execution + // Check files required, test execution of some commands, create data structure + // for all topological entities (sockets, NUMA domains, ...) + // Return some useful error message in case of any failures + cclog.ComponentDebug(m.name, "Initializing Power module") + ret = C.power_init(0) + if ret == C.int(0) { + cclog.ComponentPrint(m.name, "No RAPL support") + } + m.domains = make(map[int]LikwidEnergyDomain) + Pinfo := C.get_powerInfo() + for i := 0; i < int(Pinfo.numDomains); i++ { + d := Pinfo.domains[C.int(i)] + name := domainnames[int(d._type)] + domain := LikwidEnergyDomain{ + values: make(map[int]LikwidEnergyDomainEntry), + metricname: fmt.Sprintf("likwidenergy_%s", strings.ToLower(name)), + granularity: "socket", + domaintype: int(d._type), + energyUnit: float64(C.power_getEnergyUnit(C.int(d._type))), + } + if name == "core" { + domain.granularity = "core" + } + + for _, c := range topo.GetTypeList(domain.granularity) { + clist := topo.GetSocketHwthreads(c) + if len(clist) > 0 { + var cur C.PowerData + if _, ok := intArrayContains(initCpus, clist[0]); !ok { + initCpus = append(initCpus, clist[0]) + C.HPMaddThread(C.int(clist[0])) + } + cclog.ComponentDebug(m.name, "Reading current value on CPU ", clist[0], " for ", domain.metricname, "on", domain.granularity, c) + ret = C.power_start(&cur, C.int(clist[0]), C.PowerType(domain.domaintype)) + cclog.ComponentDebug(m.name, "Reading ", uint64(cur.before)) + if ret == 0 { + domain.values[c] = LikwidEnergyDomainEntry{ + readcpu: clist[0], + value: uint32(cur.before), + total: uint64(cur.before), + tags: map[string]string{ + "type": domain.granularity, + "type-id": fmt.Sprintf("%d", c), + }, + } + + } + } + } + cclog.ComponentDebug(m.name, "Adding domain ", domain.metricname, " with granularity ", domain.granularity) + m.domains[domain.domaintype] = domain + } + + // Set this flag only if everything is initialized properly, all required files exist, ... + m.init = true + return err +} + +// Read collects all metrics belonging to the sample collector +// and sends them through the output channel to the collector manager +func (m *LikwidEnergyCollector) Read(interval time.Duration, output chan lp.CCMetric) { + // Create a sample metric + timestamp := time.Now() + + for dt, domain := range m.domains { + for i, entry := range domain.values { + var cur C.PowerData + ret := C.power_start(&cur, C.int(entry.readcpu), C.PowerType(dt)) + if ret == 0 { + now := uint32(cur.before) + diff := now - entry.value + if now < entry.value { + diff = (^uint32(0)) - entry.value + diff += now + } + if m.config.SendDiff { + y, err := lp.New(domain.metricname, entry.tags, m.meta, map[string]interface{}{"value": float64(diff) * domain.energyUnit}, timestamp) + if err == nil { + for k, v := range m.tags { + y.AddTag(k, v) + } + // Send it to output channel + output <- y + } + } + if m.config.SendAbs { + total := float64(entry.total + uint64(diff)) + + y, err := lp.New(fmt.Sprintf("%s_abs", domain.metricname), entry.tags, m.meta, map[string]interface{}{"value": total * domain.energyUnit}, timestamp) + if err == nil { + for k, v := range m.tags { + y.AddTag(k, v) + } + // Send it to output channel + output <- y + } + } + entry.value = uint32(cur.before) + entry.total += uint64(diff) + domain.values[i] = entry + } + } + } + +} + +// Close metric collector: close network connection, close files, close libraries, ... +// Called once by the collector manager +func (m *LikwidEnergyCollector) Close() { + C.power_finalize() + C.topology_finalize() + // Unset flag + m.init = false +} diff --git a/collectors/likwidenergyMetric.md b/collectors/likwidenergyMetric.md new file mode 100644 index 0000000..4098952 --- /dev/null +++ b/collectors/likwidenergyMetric.md @@ -0,0 +1,20 @@ +## `likwidenergy` collector + +In contrast to the more general [`likwid` collector](./likwidMetric.md), this collector just reads the RAPL counters to provide energy metrics. In contrast to the `likwid` collector, this collector keeps the energy counters running the whole time and not just of a measurement interval. It covers all RAPL domains (`PKG`, `DRAM`, `PP0`, `PP1`, ...). Depending whether the domain is per socket, per L3 segment or per core, metrics are read and send. + +```json +{ + "likwidenergy" : { + "liblikwid_path" : "/path/to/liblikwid.so", + "accessdaemon_path" : "/folder/that/contains/likwid-accessD", + "access_mode" : "direct or accessdaemon", + "send_difference": true, + "send_absolute": true + } +} +``` + +The first three entries (`liblikwid_path`, `accessdaemon_path` and `access_mode`) are required to set up the access to the RAPL counters. The `access_mode` = `perf_event` is not supported at the moment. + +With `send_differences` the difference to the last measurement is provided to the system. With `send_absolute`, the absolute value since start of the system is submitted as metric. It reads the counter at initialization and then updates the value after each measurement. +