Merge develop into main (#109)

* Add cpu_used (all-cpu_idle) to CpustatCollector * Update to line-protocol/v2 * Update runonce.yml with Golang 1.20 * Update fsnotify in LIKWID Collector * Use not a pointer to line-protocol.Encoder * Simplify Makefile * Use only as many arguments as required * Allow sum function to handle non float types * Allow values to be a slice of type float64, float32, int, int64, int32, bool * Use generic function to simplify code * Add missing case for type []int32 * Use generic function to compute minimum * Use generic function to compute maximum * Use generic function to compute average * Add error value to sumAnyType * Use generic function to compute median * For older versions of go slices is not part of the installation * Remove old entries from go.sum * Use simpler sort function * Compute metrics ib_total and ib_total_pkts * Add aggregated metrics. Add missing units * Update likwidMetric.go Fixes a potential bug when `fsnotify.NewWatcher()` fails with an error * Completly avoid memory allocations in infinibandMetric read() * Fixed initialization: Initalization and measurements should run in the same thread * Add safe.directory to Release action * Fix path after installation to /usr/bin after installation * ioutil.ReadFile is deprecated: As of Go 1.16, this function simply calls os.ReadFile * Switch to package slices from the golang 1.21 default library * Read file line by line * Read file line by line * Read file line by line * Use CamelCase * Use CamelCase * Fix function getNumaDomain, it always returned 0 * Avoid type conversion by using Atoi Avoid copying structs by using pointer access Increase readability with CamelCase variable names * Add caching * Cache CpuData * Cleanup * Use init function to initalize cache structure to avoid multi threading problems * Reuse information from /proc/cpuinfo * Avoid slice cloning. Directly use the cache * Add DieList * Add NumaDomainList and SMTList * Cleanup * Add comment * Lookup core ID from /sys/devices/system/cpu, /proc/cpuinfo is not portable * Lookup all information from /sys/devices/system/cpu, /proc/cpuinfo is not portable * Correctly handle lists from /sys * Add Simultaneous Multithreading siblings * Replace deprecated thread_siblings_list by core_cpus_list * Reduce number of required slices * Allow to send total values per core, socket and node * Send all metrics with same time stamp calcEventsetMetrics does only computiation, counter measurement is done before * Input parameters should be float64 when evaluating to float64 * Send all metrics with same time stamp calcGlobalMetrics does only computiation, counter measurement is done before * Remove unused variable gmresults * Add comments * Updated go packages * Add build with golang 1.21 * Switch to checkout action version 4 * Switch to setup-go action version 4 * Add workflow_dispatch to allow manual run of workflow * Add workflow_dispatch to allow manual run of workflow * Add release build jobs to runonce.yml * Switch to golang 1.20 for RHEL based distributions * Use dnf to download golang * Remove golang versions before 1.20 * Upgrade Ubuntu focal -> jammy * Pipe golang tar package directly to tar * Update golang version * Fix Ubuntu version number * Add links to ipmi and redfish receivers * Fix http server addr format * github.com/influxdata/line-protocol -> github.com/influxdata/line-protocol/v2/lineprotocol * Corrected spelling * Add some comments * github.com/influxdata/line-protocol -> github.com/influxdata/line-protocol/v2/lineprotocol * Allow other fields not only field "value" * Add some basic debugging documentation * Add some basic debugging documentation * Use a lock for the flush timer * Add tags in lexical order as required by AddTag() * Only access meta data, when it gets used as tag * Use slice to store lexialicly orderd key value pairs * Increase golang version requirement to 1.20. * Avoid package cmp to allow builds with golang v1.20 * Fix: Error NVML library not found did crash cc-metric-collector with "SIGSEGV: segmentation violation" * Add config option idle_timeout * Add basic authentication support * Add basic authentication support * Avoid unneccessary memory allocations * Add documentation for send_*_total values * Use generic package maps to clone maps * Reuse flush timer * Add Influx client options * Reuse ccTopology functionality * Do not store unused topology information * Add batch_size config * Cleanup * Use stype and stype-id for the NIC in NetstatCollector * Wait for concurrent flush operations to finish * Be more verbose in error messages * Reverted previous changes. Made the code to complex without much advantages * Use line protocol encoder * Go pkg update * Stop flush timer, when immediatelly flushing * Fix: Corrected unlock access to batch slice * Add config option to specify whether to use GZip compression in influx write requests * Add asynchron send of encoder metrics * Use DefaultServeMux instead of github.com/gorilla/mux * Add config option for HTTP keep-alives * Be more strict, when parsing json * Add config option for HTTP request timeout and Retry interval * Allow more then one background send operation * Fix %sysusers_create_package args (#108) %sysusers_create_package requires two arguments. See: https://github.com/systemd/systemd/blob/main/src/rpm/macros.systemd.in#L165 * Add nfsiostat to list of collectors --------- Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> Co-authored-by: Holger Obermaier <holgerob@gmx.de> Co-authored-by: Obihörnchen <obihoernchende@gmail.com>
2025-07-19 03:11:41 +02:00 · 2023-12-04 12:21:26 +01:00
parent 9df1054e32
commit 6ab45dd3ec
39 changed files with 1835 additions and 1030 deletions
--- a/collectors/likwidMetric.go
+++ b/collectors/likwidMetric.go
@@ -29,8 +29,8 @@ import (
 	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 	topo "github.com/ClusterCockpit/cc-metric-collector/pkg/ccTopology"
 	"github.com/NVIDIA/go-nvml/pkg/dl"
+	"github.com/fsnotify/fsnotify"
 	"golang.design/x/thread"
-	fsnotify "gopkg.in/fsnotify.v1"
 )

 const (
@@ -41,11 +41,14 @@ const (
 )

 type LikwidCollectorMetricConfig struct {
-	Name    string `json:"name"` // Name of the metric
-	Calc    string `json:"calc"` // Calculation for the metric using
-	Type    string `json:"type"` // Metric type (aka node, socket, cpu, ...)
-	Publish bool   `json:"publish"`
-	Unit    string `json:"unit"` // Unit of metric if any
+	Name               string `json:"name"` // Name of the metric
+	Calc               string `json:"calc"` // Calculation for the metric using
+	Type               string `json:"type"` // Metric type (aka node, socket, cpu, ...)
+	Publish            bool   `json:"publish"`
+	SendCoreTotalVal   bool   `json:"send_core_total_values,omitempty"`
+	SendSocketTotalVal bool   `json:"send_socket_total_values,omitempty"`
+	SendNodeTotalVal   bool   `json:"send_node_total_values,omitempty"`
+	Unit               string `json:"unit"` // Unit of metric if any
 }

 type LikwidCollectorEventsetConfig struct {
@@ -59,7 +62,7 @@ type LikwidEventsetConfig struct {
 	eorder   []*C.char
 	estr     *C.char
 	go_estr  string
-	results  map[int]map[string]interface{}
+	results  map[int]map[string]float64
 	metrics  map[int]map[string]float64
 }

@@ -79,10 +82,11 @@ type LikwidCollector struct {
 	cpulist       []C.int
 	cpu2tid       map[int]int
 	sock2tid      map[int]int
+	tid2core      map[int]int
+	tid2socket    map[int]int
 	metrics       map[C.int]map[string]int
 	groups        []C.int
 	config        LikwidCollectorConfig
-	gmresults     map[int]map[string]float64
 	basefreq      float64
 	running       bool
 	initialized   bool
@@ -134,10 +138,10 @@ func genLikwidEventSet(input LikwidCollectorEventsetConfig) LikwidEventsetConfig
 		elist = append(elist, c_counter)
 	}
 	estr := strings.Join(tmplist, ",")
-	res := make(map[int]map[string]interface{})
+	res := make(map[int]map[string]float64)
 	met := make(map[int]map[string]float64)
 	for _, i := range topo.CpuList() {
-		res[i] = make(map[string]interface{})
+		res[i] = make(map[string]float64)
 		for k := range input.Events {
 			res[i][k] = 0.0
 		}
@@ -157,7 +161,7 @@ func genLikwidEventSet(input LikwidCollectorEventsetConfig) LikwidEventsetConfig
 }

 func testLikwidMetricFormula(formula string, params []string) bool {
-	myparams := make(map[string]interface{})
+	myparams := make(map[string]float64)
 	for _, p := range params {
 		myparams[p] = float64(1.0)
 	}
@@ -236,13 +240,6 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {

 	m.likwidGroups = make(map[C.int]LikwidEventsetConfig)

-	// m.results = make(map[int]map[int]map[string]interface{})
-	// m.mresults = make(map[int]map[int]map[string]float64)
-	m.gmresults = make(map[int]map[string]float64)
-	for _, tid := range m.cpu2tid {
-		m.gmresults[tid] = make(map[string]float64)
-	}
-
 	// This is for the global metrics computation test
 	totalMetrics := 0
 	// Generate parameter list for the metric computing test
@@ -309,24 +306,18 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
 	m.measureThread = thread.New()
 	switch m.config.AccessMode {
 	case "direct":
-		m.measureThread.Call(
-			func() {
-				C.HPMmode(0)
-			})
+		C.HPMmode(0)
 	case "accessdaemon":
 		if len(m.config.DaemonPath) > 0 {
 			p := os.Getenv("PATH")
 			os.Setenv("PATH", m.config.DaemonPath+":"+p)
 		}
-		m.measureThread.Call(
-			func() {
-				C.HPMmode(1)
-				retCode := C.HPMinit()
-				if retCode != 0 {
-					err := fmt.Errorf("C.HPMinit() failed with return code %v", retCode)
-					cclog.ComponentError(m.name, err.Error())
-				}
-			})
+		C.HPMmode(1)
+		retCode := C.HPMinit()
+		if retCode != 0 {
+			err := fmt.Errorf("C.HPMinit() failed with return code %v", retCode)
+			cclog.ComponentError(m.name, err.Error())
+		}
 		for _, c := range m.cpulist {
 			m.measureThread.Call(
 				func() {
@@ -349,6 +340,21 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
 		C.free(unsafe.Pointer(cstr))
 	}

+	cpuData := topo.CpuData()
+	m.tid2core = make(map[int]int, len(cpuData))
+	m.tid2socket = make(map[int]int, len(cpuData))
+	for i := range cpuData {
+		c := &cpuData[i]
+		// Hardware thread ID to core ID mapping
+		if len(c.CoreCPUsList) > 0 {
+			m.tid2core[c.CpuID] = c.CoreCPUsList[0]
+		} else {
+			m.tid2core[c.CpuID] = c.CpuID
+		}
+		// Hardware thead ID to socket ID mapping
+		m.tid2socket[c.CpuID] = c.Socket
+	}
+
 	m.basefreq = getBaseFreq()
 	m.init = true
 	return nil
@@ -359,6 +365,8 @@ func (m *LikwidCollector) takeMeasurement(evidx int, evset LikwidEventsetConfig,
 	var ret C.int
 	var gid C.int = -1
 	sigchan := make(chan os.Signal, 1)
+
+	// Watch changes for the lock file ()
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		cclog.ComponentError(m.name, err.Error())
@@ -370,13 +378,13 @@ func (m *LikwidCollector) takeMeasurement(evidx int, evset LikwidEventsetConfig,
 		if err != nil {
 			return true, err
 		}
-		stat := info.Sys().(*syscall.Stat_t)
-		if stat.Uid != uint32(os.Getuid()) {
-			usr, err := user.LookupId(strconv.FormatUint(uint64(stat.Uid), 10))
+		uid := info.Sys().(*syscall.Stat_t).Uid
+		if uid != uint32(os.Getuid()) {
+			usr, err := user.LookupId(fmt.Sprint(uid))
 			if err == nil {
 				return true, fmt.Errorf("Access to performance counters locked by %s", usr.Username)
 			} else {
-				return true, fmt.Errorf("Access to performance counters locked by %d", stat.Uid)
+				return true, fmt.Errorf("Access to performance counters locked by %d", uid)
 			}
 		}
 		err = watcher.Add(m.config.LockfilePath)
@@ -386,6 +394,8 @@ func (m *LikwidCollector) takeMeasurement(evidx int, evset LikwidEventsetConfig,
 	}
 	m.lock.Lock()
 	defer m.lock.Unlock()
+
+	// Initialize the performance monitoring feature by creating basic data structures
 	select {
 	case e := <-watcher.Events:
 		ret = -1
@@ -400,6 +410,8 @@ func (m *LikwidCollector) takeMeasurement(evidx int, evset LikwidEventsetConfig,
 	}
 	signal.Notify(sigchan, os.Interrupt)
 	signal.Notify(sigchan, syscall.SIGCHLD)
+
+	// Add an event string to LIKWID
 	select {
 	case <-sigchan:
 		gid = -1
@@ -415,8 +427,9 @@ func (m *LikwidCollector) takeMeasurement(evidx int, evset LikwidEventsetConfig,
 		return true, fmt.Errorf("failed to add events %s, error %d", evset.go_estr, gid)
 	} else {
 		evset.gid = gid
-		//m.likwidGroups[gid] = evset
 	}
+
+	// Setup all performance monitoring counters of an eventSet
 	select {
 	case <-sigchan:
 		ret = -1
@@ -430,6 +443,8 @@ func (m *LikwidCollector) takeMeasurement(evidx int, evset LikwidEventsetConfig,
 	if ret != 0 {
 		return true, fmt.Errorf("failed to setup events '%s', error %d", evset.go_estr, ret)
 	}
+
+	// Start counters
 	select {
 	case <-sigchan:
 		ret = -1
@@ -456,7 +471,11 @@ func (m *LikwidCollector) takeMeasurement(evidx int, evset LikwidEventsetConfig,
 	if ret != 0 {
 		return true, fmt.Errorf("failed to read events '%s', error %d", evset.go_estr, ret)
 	}
+
+	// Wait
 	time.Sleep(interval)
+
+	// Read counters
 	select {
 	case <-sigchan:
 		ret = -1
@@ -470,6 +489,8 @@ func (m *LikwidCollector) takeMeasurement(evidx int, evset LikwidEventsetConfig,
 	if ret != 0 {
 		return true, fmt.Errorf("failed to read events '%s', error %d", evset.go_estr, ret)
 	}
+
+	// Store counters
 	for eidx, counter := range evset.eorder {
 		gctr := C.GoString(counter)
 		for _, tid := range m.cpu2tid {
@@ -481,9 +502,13 @@ func (m *LikwidCollector) takeMeasurement(evidx int, evset LikwidEventsetConfig,
 			evset.results[tid][gctr] = fres
 		}
 	}
+
+	// Store time in seconds the event group was measured the last time
 	for _, tid := range m.cpu2tid {
 		evset.results[tid]["time"] = float64(C.perfmon_getLastTimeOfGroup(gid))
 	}
+
+	// Stop counters
 	select {
 	case <-sigchan:
 		ret = -1
@@ -497,6 +522,8 @@ func (m *LikwidCollector) takeMeasurement(evidx int, evset LikwidEventsetConfig,
 	if ret != 0 {
 		return true, fmt.Errorf("failed to stop events '%s', error %d", evset.go_estr, ret)
 	}
+
+	// Deallocates all internal data that is used during performance monitoring
 	signal.Stop(sigchan)
 	select {
 	case e := <-watcher.Events:
@@ -525,6 +552,9 @@ func (m *LikwidCollector) calcEventsetMetrics(evset LikwidEventsetConfig, interv
 		if metric.Type == "socket" {
 			scopemap = m.sock2tid
 		}
+		// Send all metrics with same time stamp
+		// This function does only computiation, counter measurement is done before
+		now := time.Now()
 		for domain, tid := range scopemap {
 			if tid >= 0 && len(metric.Calc) > 0 {
 				value, err := agg.EvalFloat64Condition(metric.Calc, evset.results[tid])
@@ -537,23 +567,137 @@ func (m *LikwidCollector) calcEventsetMetrics(evset LikwidEventsetConfig, interv
 				}
 				evset.metrics[tid][metric.Name] = value
 				// Now we have the result, send it with the proper tags
-				if !math.IsNaN(value) {
-					if metric.Publish {
-						fields := map[string]interface{}{"value": value}
-						y, err := lp.New(metric.Name, map[string]string{"type": metric.Type}, m.meta, fields, time.Now())
-						if err == nil {
-							if metric.Type != "node" {
-								y.AddTag("type-id", fmt.Sprintf("%d", domain))
-							}
-							if len(metric.Unit) > 0 {
-								y.AddMeta("unit", metric.Unit)
-							}
-							output <- y
+				if !math.IsNaN(value) && metric.Publish {
+					fields := map[string]interface{}{"value": value}
+					y, err :=
+						lp.New(
+							metric.Name,
+							map[string]string{
+								"type": metric.Type,
+							},
+							m.meta,
+							fields,
+							now,
+						)
+					if err == nil {
+						if metric.Type != "node" {
+							y.AddTag("type-id", fmt.Sprintf("%d", domain))
 						}
+						if len(metric.Unit) > 0 {
+							y.AddMeta("unit", metric.Unit)
+						}
+						output <- y
 					}
 				}
 			}
 		}
+
+		// Send per core aggregated values
+		if metric.SendCoreTotalVal {
+			totalCoreValues := make(map[int]float64)
+			for _, tid := range scopemap {
+				if tid >= 0 && len(metric.Calc) > 0 {
+					coreID := m.tid2core[tid]
+					value := evset.metrics[tid][metric.Name]
+					if !math.IsNaN(value) && metric.Publish {
+						totalCoreValues[coreID] += value
+					}
+				}
+			}
+
+			for coreID, value := range totalCoreValues {
+				y, err :=
+					lp.New(
+						metric.Name,
+						map[string]string{
+							"type":    "core",
+							"type-id": fmt.Sprintf("%d", coreID),
+						},
+						m.meta,
+						map[string]interface{}{
+							"value": value,
+						},
+						now,
+					)
+				if err != nil {
+					continue
+				}
+				if len(metric.Unit) > 0 {
+					y.AddMeta("unit", metric.Unit)
+				}
+				output <- y
+			}
+		}
+
+		// Send per socket aggregated values
+		if metric.SendSocketTotalVal {
+			totalSocketValues := make(map[int]float64)
+			for _, tid := range scopemap {
+				if tid >= 0 && len(metric.Calc) > 0 {
+					socketID := m.tid2socket[tid]
+					value := evset.metrics[tid][metric.Name]
+					if !math.IsNaN(value) && metric.Publish {
+						totalSocketValues[socketID] += value
+					}
+				}
+			}
+
+			for socketID, value := range totalSocketValues {
+				y, err :=
+					lp.New(
+						metric.Name,
+						map[string]string{
+							"type":    "socket",
+							"type-id": fmt.Sprintf("%d", socketID),
+						},
+						m.meta,
+						map[string]interface{}{
+							"value": value,
+						},
+						now,
+					)
+				if err != nil {
+					continue
+				}
+				if len(metric.Unit) > 0 {
+					y.AddMeta("unit", metric.Unit)
+				}
+				output <- y
+			}
+		}
+
+		// Send per node aggregated value
+		if metric.SendNodeTotalVal {
+			var totalNodeValue float64 = 0.0
+			for _, tid := range scopemap {
+				if tid >= 0 && len(metric.Calc) > 0 {
+					value := evset.metrics[tid][metric.Name]
+					if !math.IsNaN(value) && metric.Publish {
+						totalNodeValue += value
+					}
+				}
+			}
+
+			y, err :=
+				lp.New(
+					metric.Name,
+					map[string]string{
+						"type": "node",
+					},
+					m.meta,
+					map[string]interface{}{
+						"value": totalNodeValue,
+					},
+					now,
+				)
+			if err != nil {
+				continue
+			}
+			if len(metric.Unit) > 0 {
+				y.AddMeta("unit", metric.Unit)
+			}
+			output <- y
+		}
 	}

 	return nil
@@ -561,7 +705,13 @@ func (m *LikwidCollector) calcEventsetMetrics(evset LikwidEventsetConfig, interv

 // Go over the global metrics, derive the value out of the event sets' metric values and send it
 func (m *LikwidCollector) calcGlobalMetrics(groups []LikwidEventsetConfig, interval time.Duration, output chan lp.CCMetric) error {
+	// Send all metrics with same time stamp
+	// This function does only computiation, counter measurement is done before
+	now := time.Now()
+
 	for _, metric := range m.config.Metrics {
+		// The metric scope is determined in the Init() function
+		// Get the map scope-id -> tids
 		scopemap := m.cpu2tid
 		if metric.Type == "socket" {
 			scopemap = m.sock2tid
@@ -569,7 +719,7 @@ func (m *LikwidCollector) calcGlobalMetrics(groups []LikwidEventsetConfig, inter
 		for domain, tid := range scopemap {
 			if tid >= 0 {
 				// Here we generate parameter list
-				params := make(map[string]interface{})
+				params := make(map[string]float64)
 				for _, evset := range groups {
 					for mname, mres := range evset.metrics[tid] {
 						params[mname] = mres
@@ -584,13 +734,21 @@ func (m *LikwidCollector) calcGlobalMetrics(groups []LikwidEventsetConfig, inter
 				if m.config.InvalidToZero && (math.IsNaN(value) || math.IsInf(value, 0)) {
 					value = 0.0
 				}
-				//m.gmresults[tid][metric.Name] = value
 				// Now we have the result, send it with the proper tags
 				if !math.IsNaN(value) {
 					if metric.Publish {
-						tags := map[string]string{"type": metric.Type}
-						fields := map[string]interface{}{"value": value}
-						y, err := lp.New(metric.Name, tags, m.meta, fields, time.Now())
+						y, err :=
+							lp.New(
+								metric.Name,
+								map[string]string{
+									"type": metric.Type,
+								},
+								m.meta,
+								map[string]interface{}{
+									"value": value,
+								},
+								now,
+							)
 						if err == nil {
 							if metric.Type != "node" {
 								y.AddTag("type-id", fmt.Sprintf("%d", domain))
@@ -637,8 +795,6 @@ func (m *LikwidCollector) ReadThread(interval time.Duration, output chan lp.CCMe

 // main read function taking multiple measurement rounds, each 'interval' seconds long
 func (m *LikwidCollector) Read(interval time.Duration, output chan lp.CCMetric) {
-	//var skip bool = false
-	//var err error
 	if !m.init {
 		return
 	}