Fix metric timeseries gaps on nodes with many cores

On nodes with >300 cores, one interval's burst of per-hwthread metrics
overran the fixed 200-slot channels. With blocking sends at every hop,
sink back-pressure propagated to the collectors, the collection round
exceeded the interval, and time.Ticker silently dropped the missed
ticks - whole intervals were skipped without any log message.

- multiChanTicker: deliver ticks non-blockingly and warn when a
  consumer misses a tick instead of stalling all consumers; guard the
  channel list with a mutex (data race with AddChannel)
- collectorManager: run the collection round detached from the tick
  loop, skip-and-warn when a round is still running, log per-collector
  and per-round durations at debug level, close serial collectors on
  shutdown
- metricRouter: buffer the interval timestamp channel and drain it
  before stamping, so metrics never carry the previous interval's
  timestamp; warn when the collector input channel is full at tick time
- main: scale the inter-manager channels to max(200, 24*NumCPU),
  overridable with the new optional channel_buffer_size config option
- add first unit tests for ticker, collector manager and router

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.gitignore

@@ -17,3 +17,7 @@
 
 # Local copy of LIKWID headers
 /collectors/likwid
+
+# Local development workspace (build against a local cc-lib checkout)
+go.work
+go.work.sum

README.md

@@ -41,6 +41,8 @@ There is a main configuration file with basic settings that point to the other c
 
 The `interval` defines how often the metrics should be read and send to the sink(s). The `duration` tells the collectors how long one measurement has to take. This is important for some collectors, like the `likwid` collector. For more information, see [here](./docs/configuration.md).
 
+The optional `channel_buffer_size` sets the capacity of the internal channels between the components. If unset, it defaults to `max(200, 24 * number of CPUs)` so that one interval's burst of per-hwthread metrics fits without back-pressuring the collectors on nodes with many cores.
+
 See the component READMEs for their configuration:
 
 * [`collectors`](./collectors/README.md)

cc-metric-collector.go

@@ -13,6 +13,7 @@ import (
 	"flag"
 	"os"
 	"os/signal"
+	"runtime"
 	"sync"
 	"syscall"
 	"time"
@@ -29,8 +30,9 @@ import (
 )
 
 type CentralConfigFile struct {
-	Interval string `json:"interval"`
+	Interval          string `json:"interval"`
-	Duration string `json:"duration"`
+	Duration          string `json:"duration"`
+	ChannelBufferSize int    `json:"channel_buffer_size,omitempty"`
 }
 
 type RuntimeConfig struct {
@@ -132,11 +134,11 @@ func mainFunc() int {
 	if len(rcfg.ConfigFile.Interval) > 0 {
 		t, err := time.ParseDuration(rcfg.ConfigFile.Interval)
 		if err != nil {
-			cclog.Errorf("Configuration value interval=%s no valid duration", rcfg.ConfigFile.Interval)
+			cclog.Error("Configuration value 'interval' no valid duration")
 		}
 		rcfg.Interval = t
 		if rcfg.Interval == 0 {
-			cclog.Errorf("Configuration value interval=%s must be greater than zero", rcfg.ConfigFile.Interval)
+			cclog.Error("Configuration value 'interval' must be greater than zero")
 			return 1
 		}
 	}
@@ -145,11 +147,11 @@ func mainFunc() int {
 	if len(rcfg.ConfigFile.Duration) > 0 {
 		t, err := time.ParseDuration(rcfg.ConfigFile.Duration)
 		if err != nil {
-			cclog.Error("Configuration value duration=%s no valid duration", rcfg.ConfigFile.Duration)
+			cclog.Error("Configuration value 'duration' no valid duration")
 		}
 		rcfg.Duration = t
 		if rcfg.Duration == 0 {
-			cclog.Error("Configuration value duration=%s must be greater than zero", rcfg.ConfigFile.Duration)
+			cclog.Error("Configuration value 'duration' must be greater than zero")
 			return 1
 		}
 	}
@@ -158,6 +160,14 @@ func mainFunc() int {
 		return 1
 	}
 
+	// Size the channels between the managers so that one interval's burst of
+	// per-hwthread metrics fits without back-pressuring the collectors
+	chanSize := rcfg.ConfigFile.ChannelBufferSize
+	if chanSize <= 0 {
+		chanSize = max(200, 24*runtime.NumCPU())
+	}
+	cclog.ComponentDebug("main", "channel buffer size", chanSize)
+
 	routerConf := ccconf.GetPackageConfig("router")
 	if len(routerConf) == 0 {
 		cclog.Error("Metric router configuration file must be set")
@@ -194,7 +204,7 @@ func mainFunc() int {
 	}
 
 	// Connect metric router to sink manager
-	RouterToSinksChannel := make(chan lp.CCMessage, 200)
+	RouterToSinksChannel := make(chan lp.CCMessage, chanSize)
 	rcfg.SinkManager.AddInput(RouterToSinksChannel)
 	rcfg.MetricRouter.AddOutput(RouterToSinksChannel)
 
@@ -206,7 +216,7 @@ func mainFunc() int {
 	}
 
 	// Connect collector manager to metric router
-	CollectToRouterChannel := make(chan lp.CCMessage, 200)
+	CollectToRouterChannel := make(chan lp.CCMessage, chanSize)
 	rcfg.CollectManager.AddOutput(CollectToRouterChannel)
 	rcfg.MetricRouter.AddCollectorInput(CollectToRouterChannel)
 
@@ -220,7 +230,7 @@ func mainFunc() int {
 		}
 
 		// Connect receive manager to metric router
-		ReceiveToRouterChannel := make(chan lp.CCMessage, 200)
+		ReceiveToRouterChannel := make(chan lp.CCMessage, chanSize)
 		rcfg.ReceiveManager.AddOutput(ReceiveToRouterChannel)
 		rcfg.MetricRouter.AddReceiverInput(ReceiveToRouterChannel)
 		use_recv = true

collectors/beegfsmetaMetric.go

@@ -209,16 +209,16 @@ func (m *BeegfsMetaCollector) Read(interval time.Duration, output chan lp.CCMess
 				} else {
 					f1, err := strconv.ParseFloat(m.matches["other"], 32)
 					if err != nil {
-						cclog.ComponentErrorf(
+						cclog.ComponentError(
 							m.name,
-							"Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err)
+							fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err))
 						continue
 					}
 					f2, err := strconv.ParseFloat(split[i], 32)
 					if err != nil {
-						cclog.ComponentErrorf(
+						cclog.ComponentError(
 							m.name,
-							"Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err)
+							fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err))
 						continue
 					}
 					m.matches["beegfs_cstorage_other"] = fmt.Sprintf("%f", f1+f2)

collectors/beegfsstorageMetric.go

@@ -200,16 +200,16 @@ func (m *BeegfsStorageCollector) Read(interval time.Duration, output chan lp.CCM
 				} else {
 					f1, err := strconv.ParseFloat(m.matches["other"], 32)
 					if err != nil {
-						cclog.ComponentErrorf(
+						cclog.ComponentError(
 							m.name,
-							"Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err)
+							fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err))
 						continue
 					}
 					f2, err := strconv.ParseFloat(split[i], 32)
 					if err != nil {
-						cclog.ComponentErrorf(
+						cclog.ComponentError(
 							m.name,
-							"Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err)
+							fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err))
 						continue
 					}
 					m.matches["beegfs_cstorage_other"] = fmt.Sprintf("%f", f1+f2)

collectors/collectorManager.go

@@ -12,6 +12,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
@@ -21,50 +22,52 @@ import (
 
 // Map of all available metric collectors
 var AvailableCollectors = map[string]MetricCollector{
-	"likwid":          new(LikwidCollector),
+	"likwid":             new(LikwidCollector),
-	"loadavg":         new(LoadavgCollector),
+	"loadavg":            new(LoadavgCollector),
-	"memstat":         new(MemstatCollector),
+	"memstat":            new(MemstatCollector),
-	"netstat":         new(NetstatCollector),
+	"netstat":            new(NetstatCollector),
-	"ibstat":          new(InfinibandCollector),
+	"ibstat":             new(InfinibandCollector),
-	"lustrestat":      new(LustreCollector),
+	"lustrestat":         new(LustreCollector),
-	"cpustat":         new(CpustatCollector),
+	"cpustat":            new(CpustatCollector),
-	"topprocs":        new(TopProcsCollector),
+	"topprocs":           new(TopProcsCollector),
-	"nvidia":          new(NvidiaCollector),
+	"nvidia":             new(NvidiaCollector),
-	"customcmd":       new(CustomCmdCollector),
+	"customcmd":          new(CustomCmdCollector),
-	"iostat":          new(IOstatCollector),
+	"iostat":             new(IOstatCollector),
-	"diskstat":        new(DiskstatCollector),
+	"diskstat":           new(DiskstatCollector),
-	"tempstat":        new(TempCollector),
+	"tempstat":           new(TempCollector),
-	"ipmistat":        new(IpmiCollector),
+	"ipmistat":           new(IpmiCollector),
-	"gpfs":            new(GpfsCollector),
+	"gpfs":               new(GpfsCollector),
-	"cpufreq":         new(CPUFreqCollector),
+	"cpufreq":            new(CPUFreqCollector),
-	"cpufreq_cpuinfo": new(CPUFreqCpuInfoCollector),
+	"cpufreq_cpuinfo":    new(CPUFreqCpuInfoCollector),
-	"nfs3stat":        new(Nfs3Collector),
+	"nfs3stat":           new(Nfs3Collector),
-	"nfs4stat":        new(Nfs4Collector),
+	"nfs4stat":           new(Nfs4Collector),
-	"numastats":       new(NUMAStatsCollector),
+	"numastats":          new(NUMAStatsCollector),
-	"beegfs_meta":     new(BeegfsMetaCollector),
+	"beegfs_meta":        new(BeegfsMetaCollector),
-	"beegfs_storage":  new(BeegfsStorageCollector),
+	"beegfs_storage":     new(BeegfsStorageCollector),
-	"rapl":            new(RAPLCollector),
+	"rapl":               new(RAPLCollector),
-	"rocm_smi":        new(RocmSmiCollector),
+	"rocm_smi":           new(RocmSmiCollector),
-	"self":            new(SelfCollector),
+	"self":               new(SelfCollector),
-	"schedstat":       new(SchedstatCollector),
+	"schedstat":          new(SchedstatCollector),
-	"nfsiostat":       new(NfsIOStatCollector),
+	"nfsiostat":          new(NfsIOStatCollector),
-	"slurm_cgroup":    new(SlurmCgroupCollector),
+	"slurm_cgroup":       new(SlurmCgroupCollector),
-	"smartmon":        new(SmartMonCollector),
+	"smartmon":           new(SmartMonCollector),
-	"nvidia_gpm":      new(NvidiaGPMCollector),
+	"lenovo_dense_power": new(LenovoDensePowerCollector),
+	"megware_eureka":     new(MegwareEurekaCollector),
 }
 
 // Metric collector manager data structure
 type collectorManager struct {
-	collectors   []MetricCollector          // List of metric collectors to read in parallel
+	collectors    []MetricCollector          // List of metric collectors to read in parallel
-	serial       []MetricCollector          // List of metric collectors to read serially
+	serial        []MetricCollector          // List of metric collectors to read serially
-	output       chan lp.CCMessage          // Output channels
+	output        chan lp.CCMessage          // Output channels
-	done         chan bool                  // channel to finish / stop metric collector manager
+	done          chan bool                  // channel to finish / stop metric collector manager
-	ticker       mct.MultiChanTicker        // periodically ticking once each interval
+	ticker        mct.MultiChanTicker        // periodically ticking once each interval
-	duration     time.Duration              // duration (for metrics that measure over a given duration)
+	duration      time.Duration              // duration (for metrics that measure over a given duration)
-	wg           *sync.WaitGroup            // wait group for all goroutines in cc-metric-collector
+	wg            *sync.WaitGroup            // wait group for all goroutines in cc-metric-collector
-	config       map[string]json.RawMessage // json encoded config for collector manager
+	config        map[string]json.RawMessage // json encoded config for collector manager
-	collector_wg sync.WaitGroup             // internally used wait group for the parallel reading of collector
+	collector_wg  sync.WaitGroup             // internally used wait group for the parallel reading of collector
-	parallel_run bool                       // Flag whether the collectors are currently read in parallel
+	round_wg      sync.WaitGroup             // wait group for the currently running collection round
+	round_running atomic.Bool                // Flag whether a collection round is currently running
 }
 
 // Metric collector manager access functions
@@ -100,17 +103,17 @@ func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Durat
 	// Initialize configured collectors
 	for collectorName, collectorCfg := range cm.config {
 		if _, found := AvailableCollectors[collectorName]; !found {
-			cclog.ComponentErrorf("CollectorManager", "SKIP unknown collector %s", collectorName)
+			cclog.ComponentError("CollectorManager", "SKIP unknown collector", collectorName)
 			continue
 		}
 		collector := AvailableCollectors[collectorName]
 
 		err := collector.Init(collectorCfg)
 		if err != nil {
-			cclog.ComponentErrorf("CollectorManager", "Collector %s initialization failed: %v", collectorName, err)
+			cclog.ComponentError("CollectorManager", fmt.Sprintf("Collector %s initialization failed: %v", collectorName, err))
 			continue
 		}
-		cclog.ComponentDebugf("CollectorManager", "ADD COLLECTOR %s", collector.Name())
+		cclog.ComponentDebug("CollectorManager", "ADD COLLECTOR", collector.Name())
 		if collector.Parallel() {
 			cm.collectors = append(cm.collectors, collector)
 		} else {
@@ -120,26 +123,54 @@ func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Durat
 	return nil
 }
 
+// runRound executes one collection round: first all parallel collectors
+// concurrently, then the serial collectors one by one
+func (cm *collectorManager) runRound(t time.Time) {
+	roundStart := time.Now()
+	for _, c := range cm.collectors {
+		// Read metrics from collector c via goroutine
+		cclog.ComponentDebug("CollectorManager", c.Name(), t)
+		cm.collector_wg.Add(1)
+		go func(myc MetricCollector) {
+			start := time.Now()
+			myc.Read(cm.duration, cm.output)
+			cclog.ComponentDebug("CollectorManager", myc.Name(), "took", time.Since(start))
+			cm.collector_wg.Done()
+		}(c)
+	}
+	cm.collector_wg.Wait()
+	for _, c := range cm.serial {
+		// Read metrics from collector c
+		cclog.ComponentDebug("CollectorManager", c.Name(), t)
+		start := time.Now()
+		c.Read(cm.duration, cm.output)
+		cclog.ComponentDebug("CollectorManager", c.Name(), "took", time.Since(start))
+	}
+	cclog.ComponentDebug("CollectorManager", "collection round took", time.Since(roundStart))
+}
+
 // Start starts the metric collector manager
 func (cm *collectorManager) Start() {
-	tick := make(chan time.Time)
+	tick := make(chan time.Time, 1)
 	cm.ticker.AddChannel(tick)
 
 	cm.wg.Go(func() {
 		// Collector manager is done
 		done := func() {
-			// close all metric collectors
+			// wait for a still running collection round, then close all metric collectors
-			if cm.parallel_run {
+			cm.round_wg.Wait()
-				cm.collector_wg.Wait()
-				cm.parallel_run = false
-			}
 			for _, c := range cm.collectors {
 				c.Close()
 			}
+			for _, c := range cm.serial {
+				c.Close()
+			}
 			close(cm.done)
 			cclog.ComponentDebug("CollectorManager", "DONE")
 		}
 
+		var roundStart time.Time
+
 		// Wait for done signal or timer event
 		for {
 			select {
@@ -147,37 +178,20 @@ func (cm *collectorManager) Start() {
 				done()
 				return
 			case t := <-tick:
-				cm.parallel_run = true
+				// The round runs detached from this loop, so the tick channel
-				for _, c := range cm.collectors {
+				// stays drained even when a round takes longer than the interval
-					// Wait for done signal or execute the collector
+				if cm.round_running.Load() {
-					select {
+					cclog.ComponentWarn("CollectorManager", "collection round still running after", time.Since(roundStart), "- skipping tick")
-					case <-cm.done:
+					continue
-						done()
-						return
-					default:
-						// Read metrics from collector c via goroutine
-						cclog.ComponentDebugf("CollectorManager: Read %s at %v", c.Name(), t)
-						cm.collector_wg.Add(1)
-						go func(myc MetricCollector) {
-							myc.Read(cm.duration, cm.output)
-							cm.collector_wg.Done()
-						}(c)
-					}
-				}
-				cm.collector_wg.Wait()
-				cm.parallel_run = false
-				for _, c := range cm.serial {
-					// Wait for done signal or execute the collector
-					select {
-					case <-cm.done:
-						done()
-						return
-					default:
-						// Read metrics from collector c
-						cclog.ComponentDebugf("CollectorManager: Read %s at %v", c.Name(), t)
-						c.Read(cm.duration, cm.output)
-					}
 				}
+				cm.round_running.Store(true)
+				roundStart = time.Now()
+				cm.round_wg.Add(1)
+				go func() {
+					defer cm.round_wg.Done()
+					defer cm.round_running.Store(false)
+					cm.runRound(t)
+				}()
 			}
 		}
 	})

collectors/collectorManager_test.go

@@ -0,0 +1,121 @@
+package collectors
+
+import (
+	"encoding/json"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
+)
+
+// Fake ticker that delivers ticks on demand
+type fakeTicker struct {
+	channels []chan time.Time
+}
+
+func (t *fakeTicker) Init(duration time.Duration) {}
+
+func (t *fakeTicker) AddChannel(c chan time.Time) {
+	t.channels = append(t.channels, c)
+}
+
+func (t *fakeTicker) Close() {}
+
+func (t *fakeTicker) tick() {
+	for _, c := range t.channels {
+		select {
+		case c <- time.Now():
+		default:
+		}
+	}
+}
+
+// Stub collector whose Read blocks until it is released
+type stubCollector struct {
+	metricCollector
+	readStarted chan struct{}
+	release     chan struct{}
+	reads       atomic.Int32
+}
+
+func (c *stubCollector) Init(config json.RawMessage) error {
+	c.name = "teststub"
+	c.parallel = true
+	c.init = true
+	return nil
+}
+
+func (c *stubCollector) Read(duration time.Duration, output chan lp.CCMessage) {
+	c.reads.Add(1)
+	c.readStarted <- struct{}{}
+	<-c.release
+}
+
+func (c *stubCollector) Close() {}
+
+func TestOverlongCollectionRoundSkipsTick(t *testing.T) {
+	stub := &stubCollector{
+		readStarted: make(chan struct{}, 10),
+		release:     make(chan struct{}),
+	}
+	AvailableCollectors["teststub"] = stub
+	defer delete(AvailableCollectors, "teststub")
+
+	ticker := &fakeTicker{}
+	var wg sync.WaitGroup
+	cm, err := New(ticker, time.Second, &wg, json.RawMessage(`{"teststub": {}}`))
+	if err != nil {
+		t.Fatalf("failed to setup collector manager: %s", err.Error())
+	}
+	cm.AddOutput(make(chan lp.CCMessage, 100))
+	cm.Start()
+
+	// First tick starts a collection round that blocks in Read
+	ticker.tick()
+	select {
+	case <-stub.readStarted:
+	case <-time.After(5 * time.Second):
+		t.Fatal("collection round did not start on tick")
+	}
+
+	// Further ticks while the round is still running must be skipped,
+	// not queued up or run concurrently
+	for range 3 {
+		ticker.tick()
+		time.Sleep(20 * time.Millisecond)
+	}
+	if got := stub.reads.Load(); got != 1 {
+		t.Fatalf("expected 1 concurrent collection round, got %d reads", got)
+	}
+
+	// Finish the round, the next tick must start a new one
+	stub.release <- struct{}{}
+	deadline := time.After(5 * time.Second)
+	for stub.reads.Load() < 2 {
+		ticker.tick()
+		select {
+		case <-stub.readStarted:
+		case <-time.After(20 * time.Millisecond):
+		case <-deadline:
+			t.Fatal("no new collection round started after the previous one finished")
+		}
+	}
+
+	// Shutdown must wait for the running round and terminate cleanly.
+	// Closing the release channel lets any still running or straggler
+	// round finish immediately
+	close(stub.release)
+	closed := make(chan struct{})
+	go func() {
+		cm.Close()
+		close(closed)
+	}()
+	select {
+	case <-closed:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Close() did not terminate")
+	}
+	wg.Wait()
+}

collectors/cpufreqCpuinfoMetric.go

@@ -116,6 +116,9 @@ func (m *CPUFreqCpuInfoCollector) Init(_ json.RawMessage) error {
 			physicalPackageID = ""
 		}
 	}
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("%s Init(): Call to scanner.Err failed: %w", m.name, err)
+	}
 
 	if err := file.Close(); err != nil {
 		return fmt.Errorf("%s Init(): Call to file.Close() failed: %w", m.name, err)
@@ -139,16 +142,16 @@ func (m *CPUFreqCpuInfoCollector) Read(interval time.Duration, output chan lp.CC
 	const cpuInfoFile = "/proc/cpuinfo"
 	file, err := os.Open(cpuInfoFile)
 	if err != nil {
-		cclog.ComponentErrorf(
+		cclog.ComponentError(
 			m.name,
-			"Read(): Failed to open file '%s': %v", cpuInfoFile, err)
+			fmt.Sprintf("Read(): Failed to open file '%s': %v", cpuInfoFile, err))
 		return
 	}
 	defer func() {
 		if err := file.Close(); err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to close file '%s': %v", cpuInfoFile, err)
+				fmt.Sprintf("Read(): Failed to close file '%s': %v", cpuInfoFile, err))
 		}
 	}()
 
@@ -166,9 +169,9 @@ func (m *CPUFreqCpuInfoCollector) Read(interval time.Duration, output chan lp.CC
 				if !t.isHT {
 					value, err := strconv.ParseFloat(strings.TrimSpace(lineSplit[1]), 64)
 					if err != nil {
-						cclog.ComponentErrorf(
+						cclog.ComponentError(
 							m.name,
-							"Read(): Failed to convert cpu MHz '%s' to float64: %v", lineSplit[1], err)
+							fmt.Sprintf("Read(): Failed to convert cpu MHz '%s' to float64: %v", lineSplit[1], err))
 						return
 					}
 					if y, err := lp.NewMetric("cpufreq", t.tagSet, m.meta, value, now); err == nil {
@@ -179,6 +182,9 @@ func (m *CPUFreqCpuInfoCollector) Read(interval time.Duration, output chan lp.CC
 			}
 		}
 	}
+	if err := scanner.Err(); err != nil {
+		cclog.ComponentErrorf(m.name, "Read(): Call to scanner.Err failed: %s", err.Error())
+	}
 }
 
 func (m *CPUFreqCpuInfoCollector) Close() {

collectors/cpufreqMetric.go

@@ -95,7 +95,10 @@ func (m *CPUFreqCollector) Init(config json.RawMessage) error {
 	}
 
 	// Initialized
-	cclog.ComponentDebugf(m.name, "initialized %d non-hyper-threading CPUs")
+	cclog.ComponentDebug(
+		m.name,
+		"initialized",
+		len(m.topology), "non-hyper-threading CPUs")
 	m.init = true
 	return nil
 }
@@ -113,14 +116,16 @@ func (m *CPUFreqCollector) Read(interval time.Duration, output chan lp.CCMessage
 		// Read current frequency
 		line, err := os.ReadFile(t.scalingCurFreqFile)
 		if err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
-				m.name, "Read(): Failed to read file '%s': %v", t.scalingCurFreqFile, err)
+				m.name,
+				fmt.Sprintf("Read(): Failed to read file '%s': %v", t.scalingCurFreqFile, err))
 			continue
 		}
 		cpuFreq, err := strconv.ParseInt(strings.TrimSpace(string(line)), 10, 64)
 		if err != nil {
 			cclog.ComponentError(
-				m.name, "Read(): Failed to convert CPU frequency '%s' to int64: %v", line, err)
+				m.name,
+				fmt.Sprintf("Read(): Failed to convert CPU frequency '%s' to int64: %v", line, err))
 			continue
 		}
 

collectors/cpustatMetric.go

@@ -44,7 +44,7 @@ type CpustatCollector struct {
 func (m *CpustatCollector) Init(config json.RawMessage) error {
 	m.name = "CpustatCollector"
 	if err := m.setup(); err != nil {
-		return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err)
+		return fmt.Errorf("%s Init(): setup() call failed: %s", m.name, err.Error())
 	}
 	m.parallel = true
 	m.meta = map[string]string{
@@ -58,7 +58,7 @@ func (m *CpustatCollector) Init(config json.RawMessage) error {
 		d := json.NewDecoder(bytes.NewReader(config))
 		d.DisallowUnknownFields()
 		if err := d.Decode(&m.config); err != nil {
-			return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err)
+			return fmt.Errorf("%s Init(): Error decoding JSON config: %s", m.name, err.Error())
 		}
 	}
 	matches := map[string]int{
@@ -85,7 +85,7 @@ func (m *CpustatCollector) Init(config json.RawMessage) error {
 	// Check input file
 	file, err := os.Open(CPUSTATFILE)
 	if err != nil {
-		return fmt.Errorf("%s Init(): Failed to open file '%s': %w", m.name, CPUSTATFILE, err)
+		return fmt.Errorf("%s Init(): Failed to open file '%s': %s", m.name, CPUSTATFILE, err.Error())
 	}
 
 	// Pre-generate tags for all CPUs
@@ -117,10 +117,13 @@ func (m *CpustatCollector) Init(config json.RawMessage) error {
 			num_cpus++
 		}
 	}
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("%s Init(): Call to scanner.Err failed: %s", m.name, err.Error())
+	}
 
 	// Close file
 	if err := file.Close(); err != nil {
-		return fmt.Errorf("%s Init(): Failed to close file '%s': %w", m.name, CPUSTATFILE, err)
+		return fmt.Errorf("%s Init(): Failed to close file '%s': %s", m.name, CPUSTATFILE, err.Error())
 	}
 
 	m.lastTimestamp = time.Now()
@@ -171,15 +174,15 @@ func (m *CpustatCollector) Read(interval time.Duration, output chan lp.CCMessage
 
 	file, err := os.Open(CPUSTATFILE)
 	if err != nil {
-		cclog.ComponentErrorf(
+		cclog.ComponentError(
 			m.name,
-			"Read(): Failed to open file '%s': %v", CPUSTATFILE, err)
+			fmt.Sprintf("Read(): Failed to open file '%s': %v", CPUSTATFILE, err.Error()))
 	}
 	defer func() {
 		if err := file.Close(); err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to close file '%s': %v", string(CPUSTATFILE), err)
+				fmt.Sprintf("Read(): Failed to close file '%s': %v", string(CPUSTATFILE), err.Error()))
 		}
 	}()
 
@@ -194,6 +197,9 @@ func (m *CpustatCollector) Read(interval time.Duration, output chan lp.CCMessage
 			num_cpus++
 		}
 	}
+	if err := scanner.Err(); err != nil {
+		cclog.ComponentErrorf(m.name, "Init(): Call to scanner.Err failed: %s", err.Error())
+	}
 
 	if !m.config.excludeNumCPUs {
 		if num_cpus_metric, err := lp.NewMetric("num_cpus", m.nodetags, m.meta, num_cpus, now); err == nil {

collectors/customCmdMetric.go

@@ -64,9 +64,9 @@ func (m *CustomCmdCollector) Init(config json.RawMessage) error {
 		cmdFields := strings.Fields(c)
 		command := exec.Command(cmdFields[0], cmdFields[1:]...)
 		if _, err := command.Output(); err != nil {
-			cclog.ComponentWarnf(
+			cclog.ComponentWarn(
 				m.name,
-				"%s Init(): Execution of command \"%s\" failed: %v", m.name, command.String(), err)
+				fmt.Sprintf("%s Init(): Execution of command \"%s\" failed: %v", m.name, command.String(), err))
 			continue
 		}
 		m.cmdFieldsSlice = append(m.cmdFieldsSlice, cmdFields)
@@ -77,7 +77,7 @@ func (m *CustomCmdCollector) Init(config json.RawMessage) error {
 		if _, err := os.ReadFile(fileName); err != nil {
 			cclog.ComponentWarn(
 				m.name,
-				"%s Init(): Reading of file \"%s\" failed: %v", m.name, fileName, err)
+				fmt.Sprintf("%s Init(): Reading of file \"%s\" failed: %v", m.name, fileName, err))
 			continue
 		}
 		m.files = append(m.files, fileName)
@@ -100,18 +100,20 @@ func (m *CustomCmdCollector) Read(interval time.Duration, output chan lp.CCMessa
 		command := exec.Command(cmdFields[0], cmdFields[1:]...)
 		stdout, err := command.Output()
 		if err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to read command output for command \"%s\": %v", command.String(), err)
+				fmt.Sprintf("Read(): Failed to read command output for command \"%s\": %v", command.String(), err),
+			)
 			continue
 		}
 
 		// Read and decode influxDB line-protocol from command output
 		metrics, err := lp.FromBytes(stdout)
 		if err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to decode influx Message: %v", err)
+				fmt.Sprintf("Read(): Failed to decode influx Message: %v", err),
+			)
 			continue
 		}
 		for _, metric := range metrics {
@@ -126,18 +128,20 @@ func (m *CustomCmdCollector) Read(interval time.Duration, output chan lp.CCMessa
 	for _, filename := range m.files {
 		input, err := os.ReadFile(filename)
 		if err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to read file \"%s\": %v\n", filename, err)
+				fmt.Sprintf("Read(): Failed to read file \"%s\": %v\n", filename, err),
+			)
 			continue
 		}
 
 		// Read and decode influxDB line-protocol from file
 		metrics, err := lp.FromBytes(input)
 		if err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to decode influx Message: %v", err)
+				fmt.Sprintf("Read(): Failed to decode influx Message: %v", err),
+			)
 			continue
 		}
 		for _, metric := range metrics {

collectors/diskstatMetric.go

@@ -77,16 +77,16 @@ func (m *DiskstatCollector) Read(interval time.Duration, output chan lp.CCMessag
 
 	file, err := os.Open(MOUNTFILE)
 	if err != nil {
-		cclog.ComponentErrorf(
+		cclog.ComponentError(
 			m.name,
-			"Read(): Failed to open file '%s': %v", MOUNTFILE, err)
+			fmt.Sprintf("Read(): Failed to open file '%s': %v", MOUNTFILE, err))
 		return
 	}
 	defer func() {
 		if err := file.Close(); err != nil {
 			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to close file '%s': %v", MOUNTFILE, err)
+				fmt.Sprintf("Read(): Failed to close file '%s': %v", MOUNTFILE, err))
 		}
 	}()
 
@@ -147,6 +147,11 @@ mountLoop:
 			}
 		}
 	}
+	if err := scanner.Err(); err != nil {
+		cclog.ComponentErrorf(
+			m.name,
+			"Read(): Call to scanner.Err failed: %s", err.Error())
+	}
 	if m.allowedMetrics["part_max_used"] {
 		y, err := lp.NewMetric("part_max_used", map[string]string{"type": "node"}, m.meta, int(part_max_used), time.Now())
 		if err == nil {

collectors/gpfsMetric.go

@@ -371,7 +371,7 @@ func (m *GpfsCollector) Init(config json.RawMessage) error {
 	if err != nil {
 		// if using sudo, exec.lookPath will return EACCES (file mode r-x------), this can be ignored
 		if m.config.Sudo && errors.Is(err, syscall.EACCES) {
-			cclog.ComponentWarnf(m.name, "got error looking for mmpmon binary '%s': %v . This is expected when using sudo, continuing.", m.config.Mmpmon, err)
+			cclog.ComponentWarn(m.name, fmt.Sprintf("got error looking for mmpmon binary '%s': %v . This is expected when using sudo, continuing.", m.config.Mmpmon, err))
 			// the file was given in the config, use it
 			p = m.config.Mmpmon
 		} else {
@@ -517,23 +517,23 @@ func (m *GpfsCollector) Read(interval time.Duration, output chan lp.CCMessage) {
 		// return code
 		rc, err := strconv.Atoi(key_value["_rc_"])
 		if err != nil {
-			cclog.ComponentErrorf(m.name, "Read(): Failed to convert return code '%s' to int: %v", key_value["_rc_"], err)
+			cclog.ComponentError(m.name, fmt.Sprintf("Read(): Failed to convert return code '%s' to int: %v", key_value["_rc_"], err))
 			continue
 		}
 		if rc != 0 {
-			cclog.ComponentErrorf(m.name, "Read(): Filesystem '%s' is not ok.", filesystem)
+			cclog.ComponentError(m.name, fmt.Sprintf("Read(): Filesystem '%s' is not ok.", filesystem))
 			continue
 		}
 
 		// timestamp
 		sec, err := strconv.ParseInt(key_value["_t_"], 10, 64)
 		if err != nil {
-			cclog.ComponentErrorf(m.name, "Read(): Failed to convert seconds '%s' to int64: %v", key_value["_t_"], err)
+			cclog.ComponentError(m.name, fmt.Sprintf("Read(): Failed to convert seconds '%s' to int64: %v", key_value["_t_"], err))
 			continue
 		}
 		msec, err := strconv.ParseInt(key_value["_tu_"], 10, 64)
 		if err != nil {
-			cclog.ComponentErrorf(m.name, "Read(): Failed to convert micro seconds '%s' to int64: %v", key_value["_tu_"], err)
+			cclog.ComponentError(m.name, fmt.Sprintf("Read(): Failed to convert micro seconds '%s' to int64: %v", key_value["_tu_"], err))
 			continue
 		}
 		timestamp := time.Unix(sec, msec*1000)
@@ -551,7 +551,7 @@ func (m *GpfsCollector) Read(interval time.Duration, output chan lp.CCMessage) {
 		for _, metric := range GpfsAbsMetrics {
 			value, err := strconv.ParseInt(key_value[metric.prefix], 10, 64)
 			if err != nil {
-				cclog.ComponentErrorf(m.name, "Read(): Failed to convert %s '%s' to int64: %v", metric.desc, key_value[metric.prefix], err)
+				cclog.ComponentError(m.name, fmt.Sprintf("Read(): Failed to convert %s '%s' to int64: %v", metric.desc, key_value[metric.prefix], err))
 				continue
 			}
 			newstate[metric.prefix] = value
@@ -636,7 +636,7 @@ func (m *GpfsCollector) Read(interval time.Duration, output chan lp.CCMessage) {
 				}
 			} else {
 				// the value could not be computed correctly
-				cclog.ComponentWarnf(m.name, "Read(): Could not compute value for filesystem %s of metric %s: vold_ok = %t, vnew_ok = %t", filesystem, metric.name, vold_ok, vnew_ok)
+				cclog.ComponentWarn(m.name, fmt.Sprintf("Read(): Could not compute value for filesystem %s of metric %s: vold_ok = %t, vnew_ok = %t", filesystem, metric.name, vold_ok, vnew_ok))
 			}
 		}
 

collectors/infinibandMetric.go

@@ -225,9 +225,9 @@ func (m *InfinibandCollector) Read(interval time.Duration, output chan lp.CCMess
 			// Read counter file
 			line, err := os.ReadFile(counterDef.path)
 			if err != nil {
-				cclog.ComponentErrorf(
+				cclog.ComponentError(
 					m.name,
-					"Read(): Failed to read from file '%s': %v", counterDef.path, err)
+					fmt.Sprintf("Read(): Failed to read from file '%s': %v", counterDef.path, err))
 				// Current counter can not be saved as last state
 				counterDef.lastStateAvailable = false
 				continue
@@ -237,9 +237,9 @@ func (m *InfinibandCollector) Read(interval time.Duration, output chan lp.CCMess
 			// convert counter to uint64
 			vRawCounter, err := strconv.ParseUint(data, 10, 64)
 			if err != nil {
-				cclog.ComponentErrorf(
+				cclog.ComponentError(
 					m.name,
-					"Read(): Failed to convert Infininiband metrice %s='%s' to uint64: %v", counterDef.name, data, err)
+					fmt.Sprintf("Read(): Failed to convert Infininiband metrice %s='%s' to uint64: %v", counterDef.name, data, err))
 				// Current counter can not be saved as last state
 				counterDef.lastStateAvailable = false
 				continue

collectors/iostatMetric.go

@@ -48,13 +48,13 @@ func (m *IOstatCollector) Init(config json.RawMessage) error {
 	m.parallel = true
 	m.meta = map[string]string{"source": m.name, "group": "Disk"}
 	if err := m.setup(); err != nil {
-		return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err)
+		return fmt.Errorf("%s Init(): setup() call failed: %s", m.name, err.Error())
 	}
 	if len(config) > 0 {
 		d := json.NewDecoder(bytes.NewReader(config))
 		d.DisallowUnknownFields()
 		if err := d.Decode(&m.config); err != nil {
-			return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err)
+			return fmt.Errorf("%s Init(): Error decoding JSON config: %s", m.name, err.Error())
 		}
 	}
 	// https://www.kernel.org/doc/html/latest/admin-guide/iostats.html
@@ -89,7 +89,7 @@ func (m *IOstatCollector) Init(config json.RawMessage) error {
 	}
 	file, err := os.Open(IOSTATFILE)
 	if err != nil {
-		return fmt.Errorf("%s Init(): Failed to open file \"%s\": %w", m.name, IOSTATFILE, err)
+		return fmt.Errorf("%s Init(): Failed to open file \"%s\": %s", m.name, IOSTATFILE, err.Error())
 	}
 
 	scanner := bufio.NewScanner(file)
@@ -130,8 +130,11 @@ func (m *IOstatCollector) Init(config json.RawMessage) error {
 			lastValues:    lastValues,
 		}
 	}
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("%s Init(): Failed to scan content of file %q: %s", m.name, IOSTATFILE, err.Error())
+	}
 	if err := file.Close(); err != nil {
-		return fmt.Errorf("%s Init(): Failed to close file \"%s\": %w", m.name, IOSTATFILE, err)
+		return fmt.Errorf("%s Init(): Failed to close file \"%s\": %s", m.name, IOSTATFILE, err.Error())
 	}
 
 	m.init = true
@@ -145,16 +148,16 @@ func (m *IOstatCollector) Read(interval time.Duration, output chan lp.CCMessage)
 
 	file, err := os.Open(IOSTATFILE)
 	if err != nil {
-		cclog.ComponentErrorf(
+		cclog.ComponentError(
 			m.name,
-			"Read(): Failed to open file '%s': %v", IOSTATFILE, err)
+			fmt.Sprintf("Read(): Failed to open file '%s': %s", IOSTATFILE, err.Error()))
 		return
 	}
 	defer func() {
 		if err := file.Close(); err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to close file '%s': %v", IOSTATFILE, err)
+				fmt.Sprintf("Read(): Failed to close file '%s': %s", IOSTATFILE, err.Error()))
 		}
 	}()
 
@@ -198,6 +201,11 @@ func (m *IOstatCollector) Read(interval time.Duration, output chan lp.CCMessage)
 		}
 		m.devices[device] = entry
 	}
+	if err := scanner.Err(); err != nil {
+		cclog.ComponentErrorf(
+			m.name,
+			"Read(): Failed to scan content of file %q: %s", IOSTATFILE, err.Error())
+	}
 }
 
 func (m *IOstatCollector) Close() {

collectors/ipmiMetric.go

@@ -28,13 +28,15 @@ type IpmiCollector struct {
 	metricCollector
 
 	config struct {
-		IpmitoolPath    string `json:"ipmitool_path"`
+		IpmitoolPath    string   `json:"ipmitool_path"`
-		IpmisensorsPath string `json:"ipmisensors_path"`
+		IpmisensorsPath string   `json:"ipmisensors_path"`
-		Sudo            bool   `json:"use_sudo"`
+		Sudo            bool     `json:"use_sudo"`
+		IncludeMetrics  []string `json:"include_metrics"`
 	}
 
-	ipmitool    string
+	ipmitool       string
-	ipmisensors string
+	ipmisensors    string
+	includeMetrics map[string]bool
 }
 
 func (m *IpmiCollector) Init(config json.RawMessage) error {
@@ -64,6 +66,15 @@ func (m *IpmiCollector) Init(config json.RawMessage) error {
 		}
 	}
 
+	// Read metrics to include
+	m.includeMetrics = make(map[string]bool)
+	for _, metric := range m.config.IncludeMetrics {
+		metric = strings.ToLower(strings.TrimSpace(metric))
+		if metric != "" {
+			m.includeMetrics[metric] = true
+		}
+	}
+
 	m.ipmitool = m.config.IpmitoolPath
 	m.ipmisensors = m.config.IpmisensorsPath
 
@@ -145,6 +156,11 @@ func (m *IpmiCollector) readIpmiTool(output chan lp.CCMessage) error {
 			continue
 		}
 		name := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(lv[0]), " ", "_"))
+
+		if len(m.includeMetrics) > 0 && !m.includeMetrics[name] {
+			continue
+		}
+
 		unit := strings.TrimSpace(lv[2])
 		switch unit {
 		case "Volts":
@@ -165,6 +181,9 @@ func (m *IpmiCollector) readIpmiTool(output chan lp.CCMessage) error {
 		y.AddMeta("unit", unit)
 		output <- y
 	}
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("failed to scan output of command: %s", err.Error())
+	}
 
 	// Wait for command end
 	if err := command.Wait(); err != nil {
@@ -209,6 +228,11 @@ func (m *IpmiCollector) readIpmiSensors(output chan lp.CCMessage) error {
 			continue
 		}
 		name := strings.ToLower(strings.ReplaceAll(lv[1], " ", "_"))
+
+		if len(m.includeMetrics) > 0 && !m.includeMetrics[name] {
+			continue
+		}
+
 		y, err := lp.NewMetric(name, map[string]string{"type": "node"}, m.meta, v, time.Now())
 		if err != nil {
 			cclog.ComponentErrorf(m.name, "Failed to create message: %v", err)

collectors/ipmiMetric.md

@@ -15,7 +15,8 @@ hugo_path: docs/reference/cc-metric-collector/collectors/ipmi.md
   "ipmistat": {
     "ipmitool_path": "/path/to/ipmitool",
     "ipmisensors_path": "/path/to/ipmi-sensors",
-    "use_sudo": true
+    "use_sudo": true,
+    "include_metrics" : []
   }
 ```
 
@@ -36,3 +37,5 @@ Defaults: monitoring !log_allowed, !pam_session
 monitoring ALL = (root) NOPASSWD:/usr/bin/ipmitool sensor
 monitoring ALL = (root) NOPASSWD:/usr/sbin/ipmi-sensors --comma-separated-output --sdr-cache-recreate
 ```
+
+If `include_ipmi_metrics` contains any entry, ipmistat collector will only submit these metrics. Any other values will get discarded. 

collectors/lenovoDensePower.go

@@ -0,0 +1,170 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+// additional authors:
+// Michael Panzlaff (NHR@FAU)
+
+package collectors
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
+)
+
+type LenovoDensePowerCollector struct {
+	metricCollector
+
+	config struct {
+		IpmitoolPath string `json:"ipmitool_path"`
+		Sudo         bool   `json:"use_sudo"`
+	}
+
+	ipmitool string
+
+	energyValLast  float64
+	energyTimeLast time.Time
+}
+
+func (m *LenovoDensePowerCollector) Init(config json.RawMessage) error {
+	// Check if already initialized
+	if m.init {
+		return nil
+	}
+
+	m.name = "LenovoDensePowerCollector"
+	if err := m.setup(); err != nil {
+		return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err)
+	}
+	m.meta = map[string]string{
+		"source": m.name,
+		"group":  "IPMI",
+	}
+
+	m.config.IpmitoolPath = "ipmitool"
+
+	if len(config) > 0 {
+		d := json.NewDecoder(bytes.NewReader(config))
+		d.DisallowUnknownFields()
+		if err := d.Decode(&m.config); err != nil {
+			return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err)
+		}
+	}
+
+	m.ipmitool = m.config.IpmitoolPath
+
+	energyVal, energyTime, err := m.readEnergy()
+	if err != nil {
+		return fmt.Errorf("energy reading test failed: %w", err)
+	}
+
+	m.energyValLast = energyVal
+	m.energyTimeLast = energyTime
+	m.init = true
+
+	return nil
+}
+
+func (m *LenovoDensePowerCollector) readEnergy() (float64, time.Time, error) {
+	argv := make([]string, 0)
+	if m.config.Sudo {
+		argv = append(argv, "sudo", "-n")
+	}
+
+	// The Lenovo hardware wants this magic byte string
+	lenovoRequestEnergyMsg := []uint8{0x3a, 0x32, 4, 2, 0, 0, 0}
+
+	argv = append(argv, m.ipmitool, "raw")
+
+	for _, val := range lenovoRequestEnergyMsg {
+		argv = append(argv, strconv.Itoa(int(val)))
+	}
+
+	cmd := exec.Command(argv[0], argv[1:]...)
+
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	if err != nil {
+		return 0, time.Unix(0, 0), fmt.Errorf("failed to run ipmitool: %w (stdout=%s stderr=%s)", err, stdout.String(), stderr.String())
+	}
+
+	data, err := hex.DecodeString(strings.ReplaceAll(strings.TrimSpace(stdout.String()), " ", ""))
+	if err != nil {
+		return 0, time.Unix(0, 0), fmt.Errorf("unable to decode ipmitool hex output: %w (stdout=%s)", err, stdout.String())
+	}
+
+	if len(data) != 14 {
+		return 0, time.Unix(0, 0), fmt.Errorf("result must be 14 bytes as specified in the documentation")
+	}
+
+	// data is laid out like this:
+	// data[0 to 1]: FIFO index of value (for debug only)
+	// data[2 to 5]: Integer part of energy in J (little endian)
+	// data[6 to 7]: Fraction part of energy in mJ (little endian)
+	// data[8 to 11]: Integer part of Unix time of measurement in seconds (little endian)
+	// data[12 to 13]: Fraction part of Unix time of measurement in milliseconds (little endian)
+	wholeJoules := (uint32(data[2]) << 0) | (uint32(data[3]) << 8) | (uint32(data[4]) << 16) | (uint32(data[5]) << 24)
+	milliJoules := uint16(data[6]) | (uint16(data[7]) << 8)
+	finalJoules := float64(wholeJoules) + float64(milliJoules)*1e-3
+
+	wholeSeconds := (uint32(data[8]) << 0) | (uint32(data[9]) << 8) | (uint32(data[10]) << 16) | (uint32(data[11]) << 24)
+	milliSeconds := uint16(data[12]) | (uint16(data[13]) << 8)
+	finalTime := time.Unix(int64(wholeSeconds), (int64(milliSeconds) * 1000000))
+
+	return finalJoules, finalTime, nil
+}
+
+func (m *LenovoDensePowerCollector) Read(interval time.Duration, output chan lp.CCMessage) {
+	// Check if already initialized
+	if !m.init {
+		return
+	}
+
+	energyVal, energyTime, err := m.readEnergy()
+	if err != nil {
+		cclog.ComponentErrorf(m.name, "readEnergy failed: %v", err)
+		return
+	}
+
+	powerVal := 0.0
+	if energyTime.Compare(m.energyTimeLast) != 0 {
+		// Check for overflow
+		energyValDiff := int64(energyVal) - int64(m.energyValLast)
+		if energyVal < m.energyValLast {
+			energyValDiff += int64(0x100000000)
+		}
+		energyTimeDiff := energyTime.Sub(m.energyTimeLast)
+		if energyTime.Before(m.energyTimeLast) {
+			energyTimeDiff = energyTimeDiff + 0x100000000*time.Second
+		}
+		powerVal = float64(energyValDiff) / energyTimeDiff.Seconds()
+
+		m.energyValLast = energyVal
+		m.energyTimeLast = energyTime
+	}
+
+	metric, err := lp.NewMetric("lenovo_node_power", map[string]string{"type": "node"}, m.meta, powerVal, energyTime)
+	if err != nil {
+		cclog.ComponentErrorf(m.name, "lp.NewMetric failed: %v", err)
+		return
+	}
+	metric.AddMeta("unit", "Watts")
+	output <- metric
+}
+
+func (m *LenovoDensePowerCollector) Close() {
+	m.init = false
+}

collectors/lenovoDensePower.md

@@ -0,0 +1,51 @@
+<!--
+---
+title: Lenovo Dense Power collector
+description: Collect power of Lenovo Dense machines using ipmitool raw
+categories: [cc-metric-collector]
+tags: ['Admin']
+weight: 2
+hugo_path: docs/reference/cc-metric-collector/collectors/lenovoDensePower.md
+---
+-->
+
+## `lenovo_dense_power` collector
+
+```json
+  "lenovo_dense_power": {
+    "ipmitool_path": "/path/to/ipmitool",
+    "use_sudo": true,
+    "include_metrics" : []
+  }
+```
+
+The `lenovo_dense_power` collector reads power from Lenovo Dense machines via `ipmitool` (`ipmitool raw ...`). This collector is known to only work on the following machines. Others are not supported, so use at your *OWN* risk:
+
+System                                    | Compatibility             | Power measured
+---                                       | ---                       | ---
+Lenovo ThinkSystem SD650 V2/V3            | untested, but should work | node + riser1 + riser2
+Lenovo ThinkSystem SD650-N V2/ SD650-I V3 | untested, but should work | node + riser1 + riser2 + gpus
+Lenovo ThinkSystem SD665-N V3             | tested, known to work     | node + riser1 + riser2 + gpus
+
+To test if your system *may* be supported, you can run the following command *at your own risk*:
+
+```
+$ ipmitool raw 58 50 4 2 0 0 0
+ c4 00 83 a9 cb d3 a2 03 df db 47 6a d8 01
+```
+
+Exactly 14 bytes should be returned.
+In case not or another error occurs your hardware is likely not supported.
+
+In addition, `ipmitool` typically require root to run.
+In order to run `cc-metric-collector` without root priviliges, you can enable `use_sudo`.
+Add a file like this in `/etc/sudoers.d/` to allow `cc-metric-collector` to run the required commands:
+
+```
+# Do not log the following sudo commands from monitoring, since this causes a lot of log spam.
+# However keep log_denied enabled, to detect failures
+Defaults: monitoring !log_allowed, !pam_session
+
+# Allow to use ipmitool for Lenovo power readings
+monitoring ALL = (root) NOPASSWD:/usr/bin/ipmitool raw 58 50 4 2 0 0 0
+```

collectors/likwidMetric.go

@@ -12,12 +12,6 @@ package collectors
 #cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-in-object-files
 #include <stdlib.h>
 #include <likwid.h>
-
-
-int cc_add_hwthread(int cpu_id) {
-    return HPMaddThread(cpu_id);
-}
-
 */
 import "C"
 
@@ -44,7 +38,7 @@ import (
 	topo "github.com/ClusterCockpit/cc-metric-collector/pkg/ccTopology"
 	"github.com/NVIDIA/go-nvml/pkg/dl"
 	"github.com/fsnotify/fsnotify"
-	"golang.design/x/thread"
+	"golang.design/x/runtime/thread"
 )
 
 const (
@@ -267,12 +261,12 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
 			}
 			for _, metric := range evset.Metrics {
 				// Try to evaluate the metric
-				cclog.ComponentDebugf(m.name, "Checking %s", metric.Name)
+				cclog.ComponentDebug(m.name, "Checking", metric.Name)
 				if !checkMetricType(metric.Type) {
-					cclog.ComponentErrorf(m.name, "Metric %s uses invalid type %s", metric.Name, metric.Type)
+					cclog.ComponentError(m.name, "Metric", metric.Name, "uses invalid type", metric.Type)
 					metric.Calc = ""
 				} else if !testLikwidMetricFormula(metric.Calc, params) {
-					cclog.ComponentError(m.name, "Metric %s cannot be calculated with given counters", metric.Name)
+					cclog.ComponentError(m.name, "Metric", metric.Name, "cannot be calculated with given counters")
 					metric.Calc = ""
 				} else {
 					globalParams = append(globalParams, metric.Name)
@@ -287,13 +281,13 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
 	for _, metric := range m.config.Metrics {
 		// Try to evaluate the global metric
 		if !checkMetricType(metric.Type) {
-			cclog.ComponentErrorf(m.name, "Metric %s uses invalid type %s", metric.Name, metric.Type)
+			cclog.ComponentError(m.name, "Metric", metric.Name, "uses invalid type", metric.Type)
 			metric.Calc = ""
 		} else if !testLikwidMetricFormula(metric.Calc, globalParams) {
-			cclog.ComponentError(m.name, "Metric %s cannot be calculated with given counters", metric.Name)
+			cclog.ComponentError(m.name, "Metric", metric.Name, "cannot be calculated with given counters")
 			metric.Calc = ""
 		} else if !checkMetricType(metric.Type) {
-			cclog.ComponentError(m.name, "Metric %s has invalid type", metric.Name)
+			cclog.ComponentError(m.name, "Metric", metric.Name, "has invalid type")
 			metric.Calc = ""
 		} else {
 			totalMetrics++
@@ -334,7 +328,7 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
 		for _, c := range m.cpulist {
 			m.measureThread.Call(
 				func() {
-					retCode := C.cc_add_hwthread(C.int(c))
+					retCode := C.HPMaddThread(C.uint32_t(c))
 					if retCode != 0 {
 						err := fmt.Errorf("C.HPMaddThread(%v) failed with return code %v", c, retCode)
 						cclog.ComponentError(m.name, err.Error())
@@ -381,16 +375,16 @@ func (m *LikwidCollector) takeMeasurement(evidx int, evset LikwidEventsetConfig,
 	// Watch changes for the lock file ()
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
-		cclog.ComponentErrorf(
+		cclog.ComponentError(
 			m.name,
-			"takeMeasurement(): Failed to create a new fsnotify.Watcher: %v", err)
+			fmt.Sprintf("takeMeasurement(): Failed to create a new fsnotify.Watcher: %v", err))
 		return true, err
 	}
 	defer func() {
 		if err := watcher.Close(); err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"takeMeasurement(): Failed to close fsnotify.Watcher: %v", err)
+				fmt.Sprintf("takeMeasurement(): Failed to close fsnotify.Watcher: %v", err))
 		}
 	}()
 	if len(m.config.LockfilePath) > 0 {
@@ -603,7 +597,7 @@ func (m *LikwidCollector) calcEventsetMetrics(evset LikwidEventsetConfig, interv
 			if tid >= 0 && len(metric.Calc) > 0 {
 				value, err := agg.EvalFloat64Condition(metric.Calc, evset.results[tid])
 				if err != nil {
-					cclog.ComponentErrorf(m.name, "Calculation for metric %s failed: %s", metric.Name, err.Error())
+					cclog.ComponentError(m.name, "Calculation for metric", metric.Name, "failed:", err.Error())
 					value = 0.0
 				}
 				if m.config.InvalidToZero && (math.IsNaN(value) || math.IsInf(value, 0)) {
@@ -768,7 +762,7 @@ func (m *LikwidCollector) calcGlobalMetrics(groups []LikwidEventsetConfig, inter
 				// Evaluate the metric
 				value, err := agg.EvalFloat64Condition(metric.Calc, params)
 				if err != nil {
-					cclog.ComponentErrorf(m.name, "Calculation for metric %s failed: %s", metric.Name, err.Error())
+					cclog.ComponentError(m.name, "Calculation for metric", metric.Name, "failed:", err.Error())
 					value = 0.0
 				}
 				if m.config.InvalidToZero && (math.IsNaN(value) || math.IsInf(value, 0)) {

collectors/loadavgMetric.go

@@ -89,9 +89,9 @@ func (m *LoadavgCollector) Read(interval time.Duration, output chan lp.CCMessage
 	}
 	buffer, err := os.ReadFile(LOADAVGFILE)
 	if err != nil {
-		cclog.ComponentErrorf(
+		cclog.ComponentError(
 			m.name,
-			"Read(): Failed to read file '%s': %v", LOADAVGFILE, err)
+			fmt.Sprintf("Read(): Failed to read file '%s': %v", LOADAVGFILE, err))
 		return
 	}
 	now := time.Now()
@@ -101,9 +101,9 @@ func (m *LoadavgCollector) Read(interval time.Duration, output chan lp.CCMessage
 	for i, name := range m.load_matches {
 		x, err := strconv.ParseFloat(ls[i], 64)
 		if err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to convert '%s' to float64: %v", ls[i], err)
+				fmt.Sprintf("Read(): Failed to convert '%s' to float64: %v", ls[i], err))
 			continue
 		}
 		if m.load_skips[i] {
@@ -120,9 +120,9 @@ func (m *LoadavgCollector) Read(interval time.Duration, output chan lp.CCMessage
 	for i, name := range m.proc_matches {
 		x, err := strconv.ParseInt(lv[i], 10, 64)
 		if err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to convert '%s' to float64: %v", lv[i], err)
+				fmt.Sprintf("Read(): Failed to convert '%s' to float64: %v", lv[i], err))
 			continue
 		}
 		if m.proc_skips[i] {

collectors/megwareEureka.go

@@ -0,0 +1,194 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+// additional authors:
+// Michael Panzlaff (NHR@FAU)
+
+package collectors
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"time"
+
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
+)
+
+// MPS refers to Monolithic Power Systems, from which the MP5922 is used
+// to measure telemetry on the Eureka platform.
+type mpsData struct {
+	Cnt         int64   `json:"Cnt"` // No idea what this is, ignore
+	Energy      float64 `json:"Energy"`
+	Vin         float64 `json:"Vin"`
+	Iin         float64 `json:"Iin"`
+	Pin         float64 `json:"Pin"`
+	PinAvg      float64 `json:"PinAvg"`
+	Vout        float64 `json:"Vout"`
+	Iout        float64 `json:"Iout"`
+	Pout        float64 `json:"Pout"`
+	StandbyVout float64 `json:"StandbyVout"`
+	StandbyIout float64 `json:"StandbyIout"`
+	StandbyPout float64 `json:"StandbyPout"`
+	TempBusbar  float64 `json:"TempBusbar"`
+	TempSsd     float64 `json:"TempSsd"`
+	TempMps     float64 `json:"TempMps"`
+	// No idea what the ones below mean exactl. Ignore them for now.
+	EnergyTime         int64 `json:"EnergyTime"`
+	EnergyAccumulator  int64 `json:"EnergyAccumulator"`
+	EnergyRolloverCnt  int64 `json:"EnergyRolloverCnt"`
+	EnergySampleCntU24 int64 `json:"EnergySampleCntU24"`
+
+	Timestamp time.Time
+}
+
+type MegwareEurekaCollector struct {
+	metricCollector
+
+	config struct {
+		U20Path string `json:"u20_path"`
+		Sudo    bool   `json:"use_sudo"`
+	}
+
+	u20path string
+
+	energyValLast  float64
+	energyTimeLast time.Time
+}
+
+func (m *MegwareEurekaCollector) Init(config json.RawMessage) error {
+	// Check if already initialized
+	if m.init {
+		return nil
+	}
+
+	m.name = "MegwareEurekaCollector"
+	if err := m.setup(); err != nil {
+		return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err)
+	}
+	m.meta = map[string]string{
+		"source": m.name,
+		"group":  "U20",
+	}
+
+	m.config.U20Path = "u20"
+
+	if len(config) > 0 {
+		d := json.NewDecoder(bytes.NewReader(config))
+		d.DisallowUnknownFields()
+		if err := d.Decode(&m.config); err != nil {
+			return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err)
+		}
+	}
+
+	m.u20path = m.config.U20Path
+
+	data, err := m.readMpsData()
+	if err != nil {
+		return fmt.Errorf("energy reading test failed: %w", err)
+	}
+
+	m.energyValLast = data.Energy
+	m.energyTimeLast = time.Now()
+	m.init = true
+
+	return nil
+}
+
+func (m *MegwareEurekaCollector) readMpsData() (*mpsData, error) {
+	argv := make([]string, 0)
+	if m.config.Sudo {
+		argv = append(argv, "sudo", "-n")
+	}
+
+	argv = append(argv, m.u20path, "values", "--read", "GET_MPS_POLL_VALUES")
+
+	cmd := exec.Command(argv[0], argv[1:]...)
+
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	if err != nil {
+		return nil, fmt.Errorf("failed to run u20: %w (stdout=%s stderr=%s)", err, stdout.String(), stderr.String())
+	}
+
+	var u20output struct {
+		GetMpsPollValues mpsData `json:"GET_MPS_POLL_VALUES"`
+	}
+
+	err = json.Unmarshal(stdout.Bytes(), &u20output)
+	if err != nil {
+		return nil, fmt.Errorf("unable to decode u20 JSON output: %w (stdout=%s)", err, stdout.String())
+	}
+
+	u20output.GetMpsPollValues.Timestamp = time.Now()
+
+	return &u20output.GetMpsPollValues, nil
+}
+
+func (m *MegwareEurekaCollector) Read(interval time.Duration, output chan lp.CCMessage) {
+	// Check if already initialized
+	if !m.init {
+		return
+	}
+
+	data, err := m.readMpsData()
+	if err != nil {
+		cclog.ComponentErrorf(m.name, "readMpsData failed: %v", err)
+		return
+	}
+
+	powerVal := 0.0
+
+	if data.Timestamp.After(m.energyTimeLast) {
+		// Important, m.energy comes in Wh, so multiply by 3600 to get Ws (aka Joule)
+		energyValDiff := data.Energy - m.energyValLast
+		energyTimeDiff := data.Timestamp.Sub(m.energyTimeLast)
+		powerVal = energyValDiff * 3600 / energyTimeDiff.Seconds()
+
+		m.energyValLast = data.Energy
+		m.energyTimeLast = data.Timestamp
+	}
+
+	metricNamePrefix := "eureka_"
+	metricMap := map[string]struct {
+		value any
+		unit  string
+	}{
+		"power":        {value: powerVal, unit: "Watts"},
+		"vin":          {value: data.Vin, unit: "Volts"},
+		"iin":          {value: data.Iin, unit: "Amperes"},
+		"pin":          {value: data.Pin, unit: "Watts"},
+		"pin_avg":      {value: data.PinAvg, unit: "Watts"},
+		"vout":         {value: data.Vout, unit: "Volts"},
+		"iout":         {value: data.Iout, unit: "Amperes"},
+		"pout":         {value: data.Pout, unit: "Watts"},
+		"standby_vout": {value: data.StandbyVout, unit: "Volts"},
+		"standby_iout": {value: data.StandbyIout, unit: "Amperes"},
+		"standby_pout": {value: data.StandbyPout, unit: "Watts"},
+		"temp_busbar":  {value: data.TempBusbar, unit: "degC"},
+		"temp_ssd":     {value: data.TempSsd, unit: "degC"},
+		"temp_mps":     {value: data.TempMps, unit: "degC"},
+	}
+
+	for metricName, metricData := range metricMap {
+		metricName = metricNamePrefix + metricName
+		metric, err := lp.NewMetric(metricName, map[string]string{"type": "node"}, m.meta, metricData.value, data.Timestamp)
+		if err != nil {
+			cclog.ComponentErrorf(m.name, "lp.NewMetric failed: %v", err)
+			return
+		}
+		metric.AddMeta("unit", metricData.unit)
+		output <- metric
+	}
+}
+
+func (m *MegwareEurekaCollector) Close() {
+	m.init = false
+}

collectors/megwareEureka.md

@@ -0,0 +1,44 @@
+<!--
+---
+title: Megware Eureka collector
+description: Collect power and other metrics of Megware Eureka machines using u20
+categories: [cc-metric-collector]
+tags: ['Admin']
+weight: 2
+hugo_path: docs/reference/cc-metric-collector/collectors/lenovoDensePower.md
+---
+-->
+
+## `lenovo_dense_power` collector
+
+```json
+  "megware_eureka": {
+    "u20_path": "/path/to/ipmitool",
+    "use_sudo": true
+  }
+```
+
+The `megware_eureka` collector reads power and other metrics from Megware Eureka machines via `u20`.
+If the u20 tool is available for your machines, it's possible that this collector is compatible.
+If you don't have access to u20, please contact your Megware sales person.
+
+You can test the collector compatibility by running the following command:
+
+```
+$ u20 values --read GET_MPS_POLL_VALUES
+```
+
+If this returns a JSON with energy, voltage and current readings, you're fine.
+
+In addition, `u20` typically requires root to run.
+In order to run `cc-metric-collector` without root priviliges, you can enable `use_sudo`.
+Add a file like this in `/etc/sudoers.d/` to allow `cc-metric-collector` to run the required commands:
+
+```
+# Do not log the following sudo commands from monitoring, since this causes a lot of log spam.
+# However keep log_denied enabled, to detect failures
+Defaults: monitoring !log_allowed, !pam_session
+
+# Allow to use u20 for Megware power readings
+monitoring ALL = (root) NOPASSWD:/usr/bin/u20 values --read GET_MPS_POLL_VALUES
+```

collectors/memstatMetric.go

@@ -84,7 +84,7 @@ func getStats(filename string) map[string]MemstatStats {
 		case 5:
 			v, err := strconv.ParseFloat(linefields[3], 64)
 			if err == nil {
-				cclog.ComponentDebug("MemstatCollector", "getStats %s value %v unit %s", strings.Trim(linefields[2], ":"), v, linefields[4])
+				cclog.ComponentDebug("getStats", strings.Trim(linefields[2], ":"), v, linefields[4])
 				stats[strings.Trim(linefields[2], ":")] = MemstatStats{
 					value: v,
 					unit:  linefields[4],

collectors/netstatMetric.go

@@ -222,16 +222,16 @@ func (m *NetstatCollector) Read(interval time.Duration, output chan lp.CCMessage
 
 	file, err := os.Open(NETSTATFILE)
 	if err != nil {
-		cclog.ComponentErrorf(
+		cclog.ComponentError(
 			m.name,
-			"Read(): Failed to open file '%s': %v", NETSTATFILE, err)
+			fmt.Sprintf("Read(): Failed to open file '%s': %v", NETSTATFILE, err))
 		return
 	}
 	defer func() {
 		if err := file.Close(); err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to close file '%s': %v", NETSTATFILE, err)
+				fmt.Sprintf("Read(): Failed to close file '%s': %v", NETSTATFILE, err))
 		}
 	}()
 

collectors/nfsMetric.go

@@ -125,9 +125,10 @@ func (m *nfsCollector) Read(interval time.Duration, output chan lp.CCMessage) {
 	timestamp := time.Now()
 
 	if err := m.updateStats(); err != nil {
-		cclog.ComponentErrorf(
+		cclog.ComponentError(
 			m.name,
-			"Read(): updateStats() failed: %v", err)
+			fmt.Sprintf("Read(): updateStats() failed: %v", err),
+		)
 		return
 	}
 	var prefix string

collectors/numastatsMetric.go

@@ -117,7 +117,7 @@ func (m *NUMAStatsCollector) Init(config json.RawMessage) error {
 	}
 
 	// Initialized
-	cclog.ComponentDebugf(m.name, "initialized %d NUMA domains", len(m.topology))
+	cclog.ComponentDebug(m.name, "initialized", len(m.topology), "NUMA domains")
 	m.init = true
 	return nil
 }

collectors/nvidiaGPMMetric.go

@@ -1,396 +0,0 @@
-package collectors
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"slices"
-	"strconv"
-	"strings"
-	"time"
-
-	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
-	"github.com/NVIDIA/go-nvml/pkg/nvml"
-)
-
-type NvidiaGPMMetricDef struct {
-	name    string
-	outname string
-	id      nvml.GpmMetricId
-	unit    string
-}
-
-var NvidiaGPMMetrics []NvidiaGPMMetricDef = []NvidiaGPMMetricDef{
-	{
-		name:    "GRAPHICS_UTIL",
-		outname: "nv_gpm_graphics_util",
-		id:      nvml.GPM_METRIC_GRAPHICS_UTIL,
-		unit:    "%",
-	},
-	{
-		name:    "SM_UTIL",
-		outname: "nv_gpm_sm_util",
-		id:      nvml.GPM_METRIC_SM_UTIL,
-		unit:    "%",
-	},
-	{
-		name:    "SM_OCCUPANCY",
-		outname: "nv_gpm_sm_occupancy",
-		id:      nvml.GPM_METRIC_SM_OCCUPANCY,
-		unit:    "%",
-	},
-	{
-		name:    "INTEGER_UTIL",
-		outname: "nv_gpm_integer_util",
-		id:      nvml.GPM_METRIC_INTEGER_UTIL,
-		unit:    "%",
-	},
-	{
-		name:    "ANY_TENSOR_UTIL",
-		outname: "nv_gpm_any_tensor_util",
-		id:      nvml.GPM_METRIC_ANY_TENSOR_UTIL,
-		unit:    "%",
-	},
-	{
-		name:    "DFMA_TENSOR_UTIL",
-		outname: "nv_gpm_dfma_tensor_util",
-		id:      nvml.GPM_METRIC_DFMA_TENSOR_UTIL,
-		unit:    "%",
-	},
-	{
-		name:    "HMMA_TENSOR_UTIL",
-		outname: "nv_gpm_hmma_tensor_util",
-		id:      nvml.GPM_METRIC_HMMA_TENSOR_UTIL,
-		unit:    "%",
-	},
-	{
-		name:    "IMMA_TENSOR_UTIL",
-		outname: "nv_gpm_imma_tensor_util",
-		id:      nvml.GPM_METRIC_IMMA_TENSOR_UTIL,
-		unit:    "%",
-	},
-	{
-		name:    "DRAM_BW_UTIL",
-		outname: "nv_gpm_dram_bw_util",
-		id:      nvml.GPM_METRIC_DRAM_BW_UTIL,
-		unit:    "%",
-	},
-	{
-		name:    "FP64_UTIL",
-		outname: "nv_gpm_fp64_util",
-		id:      nvml.GPM_METRIC_FP64_UTIL,
-		unit:    "%",
-	},
-	{
-		name:    "FP32_UTIL",
-		outname: "nv_gpm_fp32_util",
-		id:      nvml.GPM_METRIC_FP32_UTIL,
-		unit:    "%",
-	},
-	{
-		name:    "FP16_UTIL",
-		outname: "nv_gpm_fp16_util",
-		id:      nvml.GPM_METRIC_FP16_UTIL,
-		unit:    "%",
-	},
-}
-
-type NvidiaGPMCollectorConfig struct {
-	Metrics               []string `json:"metrics,omitempty"`
-	ExcludeDevices        []string `json:"exclude_devices,omitempty"`
-	AddPciInfoTag         bool     `json:"add_pci_info_tag,omitempty"`
-	UsePciInfoAsTypeId    bool     `json:"use_pci_info_as_type_id,omitempty"`
-	AddUuidMeta           bool     `json:"add_uuid_meta,omitempty"`
-	AddBoardNumberMeta    bool     `json:"add_board_number_meta,omitempty"`
-	AddSerialMeta         bool     `json:"add_serial_meta,omitempty"`
-	ProcessMigDevices     bool     `json:"process_mig_devices,omitempty"`
-	UseUuidForMigDevices  bool     `json:"use_uuid_for_mig_device,omitempty"`
-	UseSliceForMigDevices bool     `json:"use_slice_for_mig_device,omitempty"`
-}
-
-type NvidiaGPMCollectorDevice struct {
-	device        nvml.Device
-	tags          map[string]string
-	meta          map[string]string
-	startTime     time.Time
-	endTime       time.Time
-	measurement   nvml.GpmMetricsGetType
-	metricsLookup map[int]NvidiaGPMMetricDef
-}
-
-type NvidiaGPMCollector struct {
-	metricCollector
-
-	config   NvidiaGPMCollectorConfig
-	gpus     []NvidiaGPMCollectorDevice
-	num_gpus int
-}
-
-func (m *NvidiaGPMCollector) Init(config json.RawMessage) error {
-	var err error = nil
-	m.name = "NvidiaGPMCollector"
-	m.parallel = true
-	if err := m.setup(); err != nil {
-		return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err)
-	}
-	if len(config) > 0 {
-		d := json.NewDecoder(strings.NewReader(string(config)))
-		d.DisallowUnknownFields()
-		if err = d.Decode(&m.config); err != nil {
-			return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err)
-		}
-	}
-	m.meta = map[string]string{
-		"source": m.name,
-		"group":  "NvidiaGPM",
-	}
-
-	// Initialize NVIDIA Management Library (NVML)
-	ret := nvml.Init()
-
-	// Error: NVML library not found
-	// (nvml.ErrorString can not be used in this case)
-	if ret == nvml.ERROR_LIBRARY_NOT_FOUND {
-		return fmt.Errorf("%s Init(): NVML library not found", m.name)
-	}
-	if ret != nvml.SUCCESS {
-		err = errors.New(nvml.ErrorString(ret))
-		return fmt.Errorf("%s Init(): Unable to initialize NVML: %w", m.name, err)
-	}
-
-	// Number of NVIDIA GPUs
-	num_gpus, ret := nvml.DeviceGetCount()
-	if ret != nvml.SUCCESS {
-		err = errors.New(nvml.ErrorString(ret))
-		return fmt.Errorf("%s Init(): Unable to get device count: %w", m.name, err)
-	}
-
-	// For all GPUs
-	m.gpus = make([]NvidiaGPMCollectorDevice, 0, num_gpus)
-	for i := range num_gpus {
-
-		// Skip excluded devices by ID
-		str_i := strconv.Itoa(i)
-		if slices.Contains(m.config.ExcludeDevices, str_i) {
-			cclog.ComponentDebugf(m.name, "Skipping excluded device %s", str_i)
-			continue
-		}
-
-		// Get device handle
-		device, ret := nvml.DeviceGetHandleByIndex(i)
-		if ret != nvml.SUCCESS {
-			err = errors.New(nvml.ErrorString(ret))
-			cclog.ComponentErrorf(m.name, "Unable to get device at index %d: %s", i, err.Error())
-			continue
-		}
-
-		supportInfo, ret := nvml.GpmQueryDeviceSupport(device)
-		if ret != nvml.SUCCESS {
-			err = errors.New(nvml.ErrorString(ret))
-			cclog.ComponentErrorf(m.name, "Unable to query GPM support for device at index %d: %s", i, err.Error())
-			continue
-		} else {
-			if supportInfo.IsSupportedDevice == uint32(nvml.FEATURE_DISABLED) {
-				cclog.ComponentErrorf(m.name, "Device at index %d does not support GPM metrics", i)
-				continue
-			}
-		}
-
-		stream, ret := nvml.GpmQueryIfStreamingEnabled(device)
-		if ret != nvml.SUCCESS {
-			err = errors.New(nvml.ErrorString(ret))
-			cclog.ComponentErrorf(m.name, "Unable to query GPM streaming for device at index %d: %s", i, err.Error())
-			continue
-		} else {
-			if stream == uint32(nvml.FEATURE_DISABLED) {
-				ret = nvml.GpmSetStreamingEnabled(device, uint32(nvml.FEATURE_ENABLED))
-				if ret != nvml.SUCCESS {
-					err = errors.New(nvml.ErrorString(ret))
-					cclog.ComponentErrorf(m.name, "Unable to set streaming mode for device at index %d: %s", i, err.Error())
-				}
-			}
-		}
-
-		// Get device's PCI info
-		pciInfo, ret := nvml.DeviceGetPciInfo(device)
-		if ret != nvml.SUCCESS {
-			err = errors.New(nvml.ErrorString(ret))
-			cclog.ComponentErrorf(m.name, "Unable to get PCI info for device at index %d: %s", i, err.Error())
-			continue
-		}
-		// Create PCI ID in the common format used by the NVML.
-		pci_id := fmt.Sprintf(
-			nvml.DEVICE_PCI_BUS_ID_FMT,
-			pciInfo.Domain,
-			pciInfo.Bus,
-			pciInfo.Device)
-
-		// Skip excluded devices specified by PCI ID
-		if slices.Contains(m.config.ExcludeDevices, pci_id) {
-			cclog.ComponentDebugf(m.name, "Skipping excluded device %s", pci_id)
-			continue
-		}
-		ss, nvmlErr := nvml.GpmSampleAlloc()
-		if nvmlErr != nvml.SUCCESS {
-			err = errors.New(nvml.ErrorString(ret))
-			cclog.ComponentErrorf(m.name, "Failed to allocate GPM sample for device %d: %s", i, err.Error())
-			continue
-		}
-		es, nvmlErr := nvml.GpmSampleAlloc()
-		if nvmlErr != nvml.SUCCESS {
-			err = errors.New(nvml.ErrorString(ret))
-			cclog.ComponentErrorf(m.name, "Failed to allocate GPM sample for device %d: %s", i, err.Error())
-			continue
-		}
-
-		// Select which value to use as 'type-id'.
-		// The PCI ID is commonly required in SLURM environments because the
-		// numberic IDs used by SLURM and the ones used by NVML might differ
-		// depending on the job type. The PCI ID is more reliable but is commonly
-		// not recorded for a job, so it must be added manually in prologue or epilogue
-		// e.g. to the comment field
-		tid := str_i
-		if m.config.UsePciInfoAsTypeId {
-			tid = pci_id
-		}
-
-		// Now we got all infos together, populate the device list
-		g := NvidiaGPMCollectorDevice{}
-
-		// Add device handle
-		g.device = device
-
-		// Add tags
-		g.tags = map[string]string{
-			"type":    "accelerator",
-			"type-id": tid,
-		}
-
-		// Add PCI info as tag if not already used as 'type-id'
-		if m.config.AddPciInfoTag && !m.config.UsePciInfoAsTypeId {
-			g.tags["pci_identifier"] = pci_id
-		}
-
-		g.meta = map[string]string{
-			"source": m.name,
-			"group":  "Nvidia",
-		}
-
-		if m.config.AddBoardNumberMeta {
-			board, ret := nvml.DeviceGetBoardPartNumber(device)
-			if ret != nvml.SUCCESS {
-				err = errors.New(nvml.ErrorString(ret))
-				cclog.ComponentError(m.name, "Unable to get boart part number for device at index", i, ":", err.Error())
-			} else {
-				g.meta["board_number"] = board
-			}
-		}
-		if m.config.AddSerialMeta {
-			serial, ret := nvml.DeviceGetSerial(device)
-			if ret != nvml.SUCCESS {
-				err = errors.New(nvml.ErrorString(ret))
-				cclog.ComponentError(m.name, "Unable to get serial number for device at index", i, ":", err.Error())
-			} else {
-				g.meta["serial"] = serial
-			}
-		}
-		if m.config.AddUuidMeta {
-			uuid, ret := nvml.DeviceGetUUID(device)
-			if ret != nvml.SUCCESS {
-				err = errors.New(nvml.ErrorString(ret))
-				cclog.ComponentError(m.name, "Unable to get UUID for device at index", i, ":", err.Error())
-			} else {
-				g.meta["uuid"] = uuid
-			}
-		}
-
-		g.measurement.Sample1 = ss
-		g.measurement.Sample2 = es
-		g.measurement.Version = nvml.GPM_METRICS_GET_VERSION
-		g.metricsLookup = make(map[int]NvidiaGPMMetricDef)
-		metIdx := 0
-		for _, inmetric := range m.config.Metrics {
-			for _, defmetric := range NvidiaGPMMetrics {
-				if inmetric == defmetric.outname || inmetric == defmetric.name {
-					g.measurement.Metrics[metIdx] = nvml.GpmMetric{
-						MetricId: uint32(defmetric.id),
-					}
-					g.metricsLookup[metIdx] = defmetric
-					metIdx += 1
-				}
-			}
-		}
-		g.measurement.NumMetrics = uint32(metIdx)
-		m.gpus = append(m.gpus, g)
-	}
-	cclog.ComponentDebugf(m.name, "Found %d Nvidia GPUs with GPM support", len(m.gpus))
-	m.num_gpus = len(m.gpus)
-	m.init = true
-	return err
-}
-
-func (m *NvidiaGPMCollector) Read(interval time.Duration, output chan lp.CCMessage) {
-	var err error
-	if !m.init {
-		return
-	}
-	for i, gpu := range m.gpus {
-		gpu.startTime = time.Now()
-		nvmlErr := gpu.measurement.Sample1.Get(gpu.device)
-		if nvmlErr != nvml.SUCCESS {
-			err = errors.New(nvml.ErrorString(nvmlErr))
-			cclog.ComponentError(m.name, "Unable to get start GPM sample for device at index", i, ":", err.Error())
-			continue
-		}
-	}
-	time.Sleep(interval)
-
-	for i, gpu := range m.gpus {
-		gpu.endTime = time.Now()
-		nvmlErr := gpu.measurement.Sample2.Get(gpu.device)
-		if nvmlErr != nvml.SUCCESS {
-			err = errors.New(nvml.ErrorString(nvmlErr))
-			cclog.ComponentError(m.name, "Unable to get stop GPM sample for device at index", i, ":", err.Error())
-			continue
-		}
-	}
-
-	for i, gpu := range m.gpus {
-		nvmlErr := nvml.GpmMetricsGet(&gpu.measurement)
-		if nvmlErr != nvml.SUCCESS {
-			err = errors.New(nvml.ErrorString(nvmlErr))
-			cclog.ComponentError(m.name, "Unable to get evaluate GPM sample for device at index", i, ":", err.Error())
-			continue
-		}
-		for idx, metricDef := range gpu.metricsLookup {
-			y, err := lp.NewMetric(metricDef.outname, gpu.tags, gpu.meta, gpu.measurement.Metrics[idx].Value, time.Now())
-			if err == nil {
-				y.AddMeta("unit", metricDef.unit)
-				output <- y
-			}
-		}
-	}
-
-}
-
-func (m *NvidiaGPMCollector) Close() {
-	if m.init {
-		for i, gpu := range m.gpus {
-			ret := gpu.measurement.Sample1.Free()
-			if ret != nvml.SUCCESS {
-				err := errors.New(nvml.ErrorString(ret))
-				cclog.ComponentErrorf(m.name, "Unable to free start sample for device at index %d: %s", i, err.Error())
-			}
-			ret = gpu.measurement.Sample2.Free()
-			if ret != nvml.SUCCESS {
-				err := errors.New(nvml.ErrorString(ret))
-				cclog.ComponentErrorf(m.name, "Unable to free stop sample for device at index %d: %s", i, err.Error())
-			}
-		}
-		if ret := nvml.Shutdown(); ret != nvml.SUCCESS {
-			cclog.ComponentError(m.name, "nvml.Shutdown() not successful")
-		}
-		m.init = false
-	}
-}

collectors/nvidiaGPMMetric.md

@@ -1,54 +0,0 @@
-<!--
----
-title: "Nvidia NVML GPM metric collector"
-description: Collect metrics for Nvidia GPUs using the NVML GPM interface
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/nvidiaGPM.md
----
--->
-
-## `nvidiaGPM` collector
-
-```json
-  "nvidia_gpm": {
-    "metrics": [
-      "nv_fb_mem_used",
-      "nv_fan"
-    ],
-    "exclude_devices": [
-      "0","1", "0000000:ff:01.0"
-    ],
-    
-    "process_mig_devices": false,
-    "use_pci_info_as_type_id": true,
-    "add_pci_info_tag": false,
-    "add_uuid_meta": false,
-    "add_board_number_meta": false,
-    "add_serial_meta": false,
-    "use_uuid_for_mig_device": false,
-    "use_slice_for_mig_device": false
-  }
-```
-
-The `nvidia_gpm` collector can be configured to leave out specific devices with the `exclude_devices` option. It takes IDs as supplied to the NVML with `nvmlDeviceGetHandleByIndex()` or the PCI address in NVML format (`%08X:%02X:%02X.0`). Commonly only the physical GPUs are monitored. If MIG devices should be analyzed as well, set `process_mig_devices` (adds `stype=mig,stype-id=<mig_index>`). With the options `use_uuid_for_mig_device` and `use_slice_for_mig_device`, the `<mig_index>` can be replaced with the UUID (e.g. `MIG-6a9f7cc8-6d5b-5ce0-92de-750edc4d8849`) or the MIG slice name (e.g. `1g.5gb`).
-
-The metrics sent by the `nvidia_gpm` collector use `accelerator` as `type` tag. For the `type-id`, it uses the device handle index by default. With the `use_pci_info_as_type_id` option, the PCI ID is used instead. If both values should be added as tags, activate the `add_pci_info_tag` option. It uses the device handle index as `type-id` and adds the PCI ID as separate `pci_identifier` tag.
-
-Optionally, it is possible to add the UUID, the board part number and the serial to the meta informations. They are not sent to the sinks (if not configured otherwise).
-
-
-Available Metrics:
-* `nv_gpm_graphics_util`
-* `nv_gpm_sm_util`
-* `nv_gpm_sm_occupancy`
-* `nv_gpm_integer_util`
-* `nv_gpm_any_tensor_util`
-* `nv_gpm_dfma_tensor_util`
-* `nv_gpm_hmma_tensor_util`
-* `nv_gpm_imma_tensor_util`
-* `nv_gpm_dram_bw_util`
-* `nv_gpm_fp64_util`
-* `nv_gpm_fp32_util`
-* `nv_gpm_fp16_util`

collectors/nvidiaMetric.go

@@ -113,7 +113,7 @@ func (m *NvidiaCollector) Init(config json.RawMessage) error {
 		// Skip excluded devices by ID
 		str_i := strconv.Itoa(i)
 		if slices.Contains(m.config.ExcludeDevices, str_i) {
-			cclog.ComponentDebugf(m.name, "Skipping excluded device %s", str_i)
+			cclog.ComponentDebug(m.name, "Skipping excluded device", str_i)
 			continue
 		}
 
@@ -121,7 +121,7 @@ func (m *NvidiaCollector) Init(config json.RawMessage) error {
 		device, ret := nvml.DeviceGetHandleByIndex(i)
 		if ret != nvml.SUCCESS {
 			err = errors.New(nvml.ErrorString(ret))
-			cclog.ComponentErrorf(m.name, "Unable to get device at index %d: %s", i, err.Error())
+			cclog.ComponentError(m.name, "Unable to get device at index", i, ":", err.Error())
 			continue
 		}
 
@@ -129,7 +129,7 @@ func (m *NvidiaCollector) Init(config json.RawMessage) error {
 		pciInfo, ret := nvml.DeviceGetPciInfo(device)
 		if ret != nvml.SUCCESS {
 			err = errors.New(nvml.ErrorString(ret))
-			cclog.ComponentErrorf(m.name, "Unable to get PCI info for device at index %d: %s", i, err.Error())
+			cclog.ComponentError(m.name, "Unable to get PCI info for device at index", i, ":", err.Error())
 			continue
 		}
 		// Create PCI ID in the common format used by the NVML.
@@ -141,7 +141,7 @@ func (m *NvidiaCollector) Init(config json.RawMessage) error {
 
 		// Skip excluded devices specified by PCI ID
 		if slices.Contains(m.config.ExcludeDevices, pci_id) {
-			cclog.ComponentDebugf(m.name, "Skipping excluded device %s", pci_id)
+			cclog.ComponentDebug(m.name, "Skipping excluded device", pci_id)
 			continue
 		}
 
@@ -183,7 +183,7 @@ func (m *NvidiaCollector) Init(config json.RawMessage) error {
 		if m.config.AddBoardNumberMeta {
 			board, ret := nvml.DeviceGetBoardPartNumber(device)
 			if ret != nvml.SUCCESS {
-				cclog.ComponentErrorf(m.name, "Unable to get boart part number for device at index %d: %s", i, err.Error())
+				cclog.ComponentError(m.name, "Unable to get boart part number for device at index", i, ":", err.Error())
 			} else {
 				g.meta["board_number"] = board
 			}
@@ -191,7 +191,7 @@ func (m *NvidiaCollector) Init(config json.RawMessage) error {
 		if m.config.AddSerialMeta {
 			serial, ret := nvml.DeviceGetSerial(device)
 			if ret != nvml.SUCCESS {
-				cclog.ComponentErrorf(m.name, "Unable to get serial number for device at index %d: %s", i, err.Error())
+				cclog.ComponentError(m.name, "Unable to get serial number for device at index", i, ":", err.Error())
 			} else {
 				g.meta["serial"] = serial
 			}
@@ -199,7 +199,7 @@ func (m *NvidiaCollector) Init(config json.RawMessage) error {
 		if m.config.AddUuidMeta {
 			uuid, ret := nvml.DeviceGetUUID(device)
 			if ret != nvml.SUCCESS {
-				cclog.ComponentErrorf(m.name, "Unable to get UUID for device at index %d: %s", i, err.Error())
+				cclog.ComponentError(m.name, "Unable to get UUID for device at index", i, ":", err.Error())
 			} else {
 				g.meta["uuid"] = uuid
 			}
@@ -1128,97 +1128,97 @@ func (m *NvidiaCollector) Read(interval time.Duration, output chan lp.CCMessage)
 		}
 		err = readMemoryInfo(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readMemoryInfo for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readMemoryInfo for device", name, "failed")
 		}
 
 		err = readUtilization(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readUtilization for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readUtilization for device", name, "failed")
 		}
 
 		err = readTemp(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readTemp for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readTemp for device", name, "failed")
 		}
 
 		err = readFan(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readFan for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readFan for device", name, "failed")
 		}
 
 		err = readEccMode(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readEccMode for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readEccMode for device", name, "failed")
 		}
 
 		err = readPerfState(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readPerfState for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readPerfState for device", name, "failed")
 		}
 
 		err = readPowerUsage(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readPowerUsage for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readPowerUsage for device", name, "failed")
 		}
 
 		err = readEnergyConsumption(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readEnergyConsumption for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readEnergyConsumption for device", name, "failed")
 		}
 
 		err = readClocks(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readClocks for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readClocks for device", name, "failed")
 		}
 
 		err = readMaxClocks(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readMaxClocks for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readMaxClocks for device", name, "failed")
 		}
 
 		err = readEccErrors(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readEccErrors for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readEccErrors for device", name, "failed")
 		}
 
 		err = readPowerLimit(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readPowerLimit for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readPowerLimit for device", name, "failed")
 		}
 
 		err = readEncUtilization(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readEncUtilization for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readEncUtilization for device", name, "failed")
 		}
 
 		err = readDecUtilization(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readDecUtilization for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readDecUtilization for device", name, "failed")
 		}
 
 		err = readRemappedRows(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readRemappedRows for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readRemappedRows for device", name, "failed")
 		}
 
 		err = readBarMemoryInfo(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readBarMemoryInfo for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readBarMemoryInfo for device", name, "failed")
 		}
 
 		err = readProcessCounts(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readProcessCounts for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readProcessCounts for device", name, "failed")
 		}
 
 		err = readViolationStats(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readViolationStats for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readViolationStats for device", name, "failed")
 		}
 
 		err = readNVLinkStats(device, output)
 		if err != nil {
-			cclog.ComponentDebugf(m.name, "readNVLinkStats for device %s failed", name)
+			cclog.ComponentDebug(m.name, "readNVLinkStats for device", name, "failed")
 		}
 	}
 
@@ -1244,7 +1244,7 @@ func (m *NvidiaCollector) Read(interval time.Duration, output chan lp.CCMessage)
 			if maxMig == 0 {
 				continue
 			}
-			cclog.ComponentDebugf(m.name, "Reading MIG devices for GPU %d", i)
+			cclog.ComponentDebug(m.name, "Reading MIG devices for GPU", i)
 
 			for j := range maxMig {
 				mdev, ret := nvml.DeviceGetMigDeviceHandleByIndex(m.gpus[i].device, j)
@@ -1268,7 +1268,7 @@ func (m *NvidiaCollector) Read(interval time.Duration, output chan lp.CCMessage)
 				if m.config.UseUuidForMigDevices {
 					uuid, ret := nvml.DeviceGetUUID(mdev)
 					if ret != nvml.SUCCESS {
-						cclog.ComponentErrorf(m.name, "Unable to get UUID for mig device at index %d: %s", j, err.Error())
+						cclog.ComponentError(m.name, "Unable to get UUID for mig device at index", j, ":", err.Error())
 					} else {
 						migDevice.tags["stype-id"] = uuid
 					}

collectors/raplMetric.go

@@ -208,10 +208,11 @@ func (m *RAPLCollector) Init(config json.RawMessage) error {
 	}
 
 	// Initialized
-	cclog.ComponentDebugf(
+	cclog.ComponentDebug(
 		m.name,
-		"initialized %d zones with running average power limit (RAPL) monitoring attributes",
+		"initialized",
-		len(m.RAPLZoneInfo))
+		len(m.RAPLZoneInfo),
+		"zones with running average power limit (RAPL) monitoring attributes")
 	m.init = true
 
 	return err

collectors/rocmsmiMetric.go

@@ -124,7 +124,7 @@ func (m *RocmSmiCollector) Init(config json.RawMessage) error {
 		if m.config.AddSerialMeta {
 			serial, ret := rocm_smi.DeviceGetSerialNumber(device)
 			if ret != rocm_smi.STATUS_SUCCESS {
-				cclog.ComponentErrorf(m.name, "Unable to get serial number for device at index %d: %s", i, rocm_smi.StatusStringNoError(ret))
+				cclog.ComponentError(m.name, "Unable to get serial number for device at index", i, ":", rocm_smi.StatusStringNoError(ret))
 			} else {
 				dev.meta["serial"] = serial
 			}
@@ -152,7 +152,7 @@ func (m *RocmSmiCollector) Read(interval time.Duration, output chan lp.CCMessage
 	for _, dev := range m.devices {
 		metrics, ret := rocm_smi.DeviceGetMetrics(dev.device)
 		if ret != rocm_smi.STATUS_SUCCESS {
-			cclog.ComponentErrorf(m.name, "Unable to get metrics for device at index %d: %s", dev.index, rocm_smi.StatusStringNoError(ret))
+			cclog.ComponentError(m.name, "Unable to get metrics for device at index", dev.index, ":", rocm_smi.StatusStringNoError(ret))
 			continue
 		}
 

collectors/schedstatMetric.go

@@ -147,15 +147,15 @@ func (m *SchedstatCollector) Read(interval time.Duration, output chan lp.CCMessa
 
 	file, err := os.Open(SCHEDSTATFILE)
 	if err != nil {
-		cclog.ComponentErrorf(
+		cclog.ComponentError(
 			m.name,
-			"Read(): Failed to open file '%s': %v", SCHEDSTATFILE, err)
+			fmt.Sprintf("Read(): Failed to open file '%s': %v", SCHEDSTATFILE, err))
 	}
 	defer func() {
 		if err := file.Close(); err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to close file '%s': %v", SCHEDSTATFILE, err)
+				fmt.Sprintf("Read(): Failed to close file '%s': %v", SCHEDSTATFILE, err))
 		}
 	}()
 

collectors/slurmCgroupMetric.go

@@ -7,6 +7,7 @@ import (
 	"os/exec"
 	"os/user"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -45,6 +46,37 @@ type SlurmCgroupCollector struct {
 
 const defaultCgroupBase = "/sys/fs/cgroup/system.slice/slurmstepd.scope"
 
+var (
+	// Slurm cgroup v2 directory layout:
+	// - Slurm <= 25.11: job_<numeric job id>
+	// - Slurm >= 26.05: SLUID, encoded as "s" + 13 Crockford Base32 characters
+	jobIDDirRE = regexp.MustCompile(`^job_[0-9]+$`)
+	sluidDirRE = regexp.MustCompile(`(?i)^s[0-9A-HJKMNP-TV-Z]{13}$`)
+)
+
+func (m *SlurmCgroupCollector) findSlurmJobDirs() ([]string, error) {
+	entries, err := os.ReadDir(m.cgroupBase)
+	if err != nil {
+		return nil, err
+	}
+
+	jobDirs := make([]string, 0)
+
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+
+		name := entry.Name()
+
+		if jobIDDirRE.MatchString(name) || sluidDirRE.MatchString(name) {
+			jobDirs = append(jobDirs, filepath.Join(m.cgroupBase, name))
+		}
+	}
+
+	return jobDirs, nil
+}
+
 func ParseCPUs(cpuset string) ([]int, error) {
 	var result []int
 	if cpuset == "" {
@@ -237,10 +269,9 @@ func (m *SlurmCgroupCollector) Read(interval time.Duration, output chan lp.CCMes
 		delete(m.cpuUsed, k)
 	}
 
-	globPattern := filepath.Join(m.cgroupBase, "job_*")
+	jobDirs, err := m.findSlurmJobDirs()
-	jobDirs, err := filepath.Glob(globPattern)
 	if err != nil {
-		cclog.ComponentErrorf(m.name, "Error globbing job directories: %s", err.Error())
+		cclog.ComponentError(m.name, "Error reading job directories:", err.Error())
 		return
 	}
 
@@ -249,7 +280,7 @@ func (m *SlurmCgroupCollector) Read(interval time.Duration, output chan lp.CCMes
 
 		jobdata, err := m.ReadJobData(jKey)
 		if err != nil {
-			cclog.ComponentError(m.name, "Error reading job data for %s: %s", jKey, err.Error())
+			cclog.ComponentError(m.name, "Error reading job data for", jKey, ":", err.Error())
 			continue
 		}
 

collectors/smartmonMetric.go

@@ -228,12 +228,12 @@ func (m *SmartMonCollector) Read(interval time.Duration, output chan lp.CCMessag
 
 		stdout, err := command.Output()
 		if err != nil {
-			cclog.ComponentErrorf(m.name, "cannot read data for device %s", d.Name)
+			cclog.ComponentError(m.name, "cannot read data for device", d.Name)
 			continue
 		}
 		err = json.Unmarshal(stdout, &data)
 		if err != nil {
-			cclog.ComponentErrorf(m.name, "cannot unmarshal data for device %s", d.Name)
+			cclog.ComponentError(m.name, "cannot unmarshal data for device", d.Name)
 			continue
 		}
 		if !m.excludeMetric.temp {

collectors/tempMetric.go

@@ -188,16 +188,16 @@ func (m *TempCollector) Read(interval time.Duration, output chan lp.CCMessage) {
 		// Read sensor file
 		buffer, err := os.ReadFile(sensor.file)
 		if err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to read file '%s': %v", sensor.file, err)
+				fmt.Sprintf("Read(): Failed to read file '%s': %v", sensor.file, err))
 			continue
 		}
 		x, err := strconv.ParseInt(strings.TrimSpace(string(buffer)), 10, 64)
 		if err != nil {
-			cclog.ComponentErrorf(
+			cclog.ComponentError(
 				m.name,
-				"Read(): Failed to convert temperature '%s' to int64: %v", buffer, err)
+				fmt.Sprintf("Read(): Failed to convert temperature '%s' to int64: %v", buffer, err))
 			continue
 		}
 		x /= 1000

collectors/topprocsMetric.go

@@ -77,9 +77,9 @@ func (m *TopProcsCollector) Read(interval time.Duration, output chan lp.CCMessag
 	command := exec.Command("ps", "-Ao", "comm", "--sort=-pcpu")
 	stdout, err := command.Output()
 	if err != nil {
-		cclog.ComponentErrorf(
+		cclog.ComponentError(
 			m.name,
-			"Read(): Failed to read output from command \"%s\": %v", command.String(), err)
+			fmt.Sprintf("Read(): Failed to read output from command \"%s\": %v", command.String(), err))
 		return
 	}
 

docs/configuration.md

@@ -19,6 +19,8 @@ The global file contains the paths to the other four files and some global optio
 
 Be aware that the paths are relative to the execution folder of the cc-metric-collector binary, so it is recommended to use absolute paths.
 
+The optional `channel_buffer_size` option sets the capacity of the internal channels between the components (collectors → router → sinks). If unset, it defaults to `max(200, 24 * number of CPUs)` so that one interval's burst of per-hwthread metrics fits without back-pressuring the collectors on nodes with many cores.
+
 ## Component configuration
 
 The others are mainly list of of subcomponents: the collectors, the receivers, the router and the sinks. Their role is best shown in a picture:

go.mod

@@ -5,11 +5,11 @@ go 1.25.0
 require (
 	github.com/ClusterCockpit/cc-lib/v2 v2.12.0
 	github.com/ClusterCockpit/go-rocm-smi v0.4.0
-	github.com/NVIDIA/go-nvml v0.13.0-1
+	github.com/NVIDIA/go-nvml v0.13.2-0
 	github.com/PaesslerAG/gval v1.2.4
 	github.com/fsnotify/fsnotify v1.10.1
 	github.com/tklauser/go-sysconf v0.4.0
-	golang.design/x/thread v0.3.2
+	golang.design/x/runtime v0.3.0
 	golang.org/x/sys v0.45.0
 )
 

go.sum

@@ -12,8 +12,9 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
 github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
-github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw=
 github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4=
+github.com/NVIDIA/go-nvml v0.13.2-0 h1:7M4cFG62wSUHw8i0XSiNU7ejKODytTS6ZrW/vgB2NSI=
+github.com/NVIDIA/go-nvml v0.13.2-0/go.mod h1:ahi2psRYoa+wYUBIrZPRO+wJs9lcvMhxSSkjjvsJJNQ=
 github.com/PaesslerAG/gval v1.2.4 h1:rhX7MpjJlcxYwL2eTTYIOBUyEKZ+A96T9vQySWkVUiU=
 github.com/PaesslerAG/gval v1.2.4/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
 github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi0jPI=
@@ -173,8 +174,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
 go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
-golang.design/x/thread v0.3.2 h1:FmD1glspGrQCe6FuQLmSrT6wz2CSzq7vKVDluyiMnqo=
+golang.design/x/runtime v0.3.0 h1:8bA+GQfO8A18JXJQlQA56pEc+Sgrbo4cmLzjzmJ58ZI=
-golang.design/x/thread v0.3.2/go.mod h1:6+Hi2rMOgMHZdKDWaqNHyWtoFUx1HxZ06LfHPh5Z/hQ=
+golang.design/x/runtime v0.3.0/go.mod h1:PjVKQjImLdSrvjIw4FmrnCPq9BGs5PMBHb8i6b2BN9I=
 golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
 golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
 golang.org/x/exp v0.0.0-20231005195138-3e424a577f31 h1:9k5exFQKQglLo+RoP+4zMjOFE14P6+vyR0baDAi0Rcs=

internal/metricAggregator/metricAggregator.go

@@ -128,7 +128,7 @@ func (c *metricAggregator) Eval(starttime time.Time, endtime time.Time, metrics
 	vars["starttime"] = starttime
 	vars["endtime"] = endtime
 	for _, f := range c.functions {
-		cclog.ComponentDebug("MetricCache", "COLLECT", f.Name, "COND", f.Condition)
+		cclog.ComponentDebugf("MetricCache", "COLLECT %s COND '%s'", f.Name, f.Condition)
 		var valuesFloat64 []float64
 		var valuesFloat32 []float32
 		var valuesInt []int
@@ -140,7 +140,7 @@ func (c *metricAggregator) Eval(starttime time.Time, endtime time.Time, metrics
 			vars["metric"] = m
 			value, err := f.gvalCond.EvalBool(context.Background(), vars)
 			if err != nil {
-				cclog.ComponentError("MetricCache", "COLLECT", f.Name, "COND", f.Condition, ":", err.Error())
+				cclog.ComponentErrorf("MetricCache", "COLLECT %s COND '%s' : %s", f.Name, f.Condition, err.Error())
 				continue
 			}
 			if value {
@@ -160,7 +160,7 @@ func (c *metricAggregator) Eval(starttime time.Time, endtime time.Time, metrics
 					case bool:
 						valuesBool = append(valuesBool, x)
 					default:
-						cclog.ComponentError("MetricCache", "COLLECT ADD VALUE", v, "FAILED")
+						cclog.ComponentErrorf("MetricCache", "COLLECT ADD VALUE %v FAILED", v)
 					}
 				}
 				matches = append(matches, m)
@@ -213,13 +213,13 @@ func (c *metricAggregator) Eval(starttime time.Time, endtime time.Time, metrics
 			vars["values"] = valuesBool
 			len_values = len(valuesBool)
 		}
-		cclog.ComponentDebug("MetricCache", "EVALUATE", f.Name, "METRICS", len_values, "CALC", f.Function)
+		cclog.ComponentDebugf("MetricCache", "EVALUATE %s METRICS %d CALC '%s'", f.Name, len_values, f.Function)
 
 		vars["metrics"] = matches
 		if len_values > 0 {
 			value, err := gval.Evaluate(f.Function, vars, c.language)
 			if err != nil {
-				cclog.ComponentError("MetricCache", "EVALUATE", f.Name, "METRICS", len_values, "CALC", f.Function, ":", err.Error())
+				cclog.ComponentErrorf("MetricCache", "EVALUATE %s METRICS %d CALC '%s': %s", f.Name, len_values, f.Function, err.Error())
 				break
 			}
 
@@ -273,12 +273,12 @@ func (c *metricAggregator) Eval(starttime time.Time, endtime time.Time, metrics
 			case string:
 				m, err = lp.NewMessage(f.Name, tags, meta, map[string]any{"value": t}, starttime)
 			default:
-				cclog.ComponentError("MetricCache", "Gval returned invalid type", t, "skipping metric", f.Name)
+				cclog.ComponentErrorf("MetricCache", "Gval returned invalid type %s skipping metric %s", t, f.Name)
 			}
 			if err != nil {
-				cclog.ComponentError("MetricCache", "Cannot create metric from Gval result", value, ":", err.Error())
+				cclog.ComponentErrorf("MetricCache", "Cannot create metric from Gval result %v: %s", value, err.Error())
 			}
-			cclog.ComponentDebug("MetricCache", "SEND", m)
+			cclog.ComponentDebugf("MetricCache", "SEND %s", m.ToLineProtocol(nil))
 			select {
 			case c.output <- m:
 			default:
@@ -295,12 +295,12 @@ func (c *metricAggregator) AddAggregation(name, function, condition string, tags
 	newcond := strings.ReplaceAll(condition, "'", "\"")
 	gvalCond, err := gval.Full(metricCacheLanguage).NewEvaluable(newcond)
 	if err != nil {
-		cclog.ComponentError("MetricAggregator", "Cannot add aggregation, invalid if condition", newcond, ":", err.Error())
+		cclog.ComponentErrorf("MetricAggregator", "Cannot add aggregation, invalid if condition '%s': %s", newcond, err.Error())
 		return err
 	}
 	gvalFunc, err := gval.Full(metricCacheLanguage).NewEvaluable(newfunc)
 	if err != nil {
-		cclog.ComponentError("MetricAggregator", "Cannot add aggregation, invalid function condition", newfunc, ":", err.Error())
+		cclog.ComponentErrorf("MetricAggregator", "Cannot add aggregation, invalid function condition %s: %s", newfunc, err.Error())
 		return err
 	}
 	for _, agg := range c.functions {

internal/metricRouter/metricCache.go

@@ -79,7 +79,7 @@ func (c *metricCache) Init(output chan lp.CCMessage, ticker mct.MultiChanTicker,
 
 // Start starts the metric cache
 func (c *metricCache) Start() {
-	c.tickchan = make(chan time.Time)
+	c.tickchan = make(chan time.Time, 1)
 	c.ticker.AddChannel(c.tickchan)
 	// Router cache is done
 	done := func() {

internal/metricRouter/metricRouter.go

@@ -35,18 +35,18 @@ type metricRouterTagConfig struct {
 
 // Metric router configuration
 type metricRouterConfig struct {
-	HostnameTagName   string                               `json:"hostname_tag,omitempty"`        // Key name used when adding the hostname to a metric (default 'hostname')
+	HostnameTagName   string                               `json:"hostname_tag"`        // Key name used when adding the hostname to a metric (default 'hostname')
-	AddTags           []metricRouterTagConfig              `json:"add_tags,omitempty"`            // List of tags that are added when the condition is met
+	AddTags           []metricRouterTagConfig              `json:"add_tags"`            // List of tags that are added when the condition is met
-	DelTags           []metricRouterTagConfig              `json:"delete_tags,omitempty"`         // List of tags that are removed when the condition is met
+	DelTags           []metricRouterTagConfig              `json:"delete_tags"`         // List of tags that are removed when the condition is met
-	IntervalAgg       []agg.MetricAggregatorIntervalConfig `json:"interval_aggregates,omitempty"` // List of aggregation function processed at the end of an interval
+	IntervalAgg       []agg.MetricAggregatorIntervalConfig `json:"interval_aggregates"` // List of aggregation function processed at the end of an interval
-	DropMetrics       []string                             `json:"drop_metrics,omitempty"`        // List of metric names to drop. For fine-grained dropping use drop_metrics_if
+	DropMetrics       []string                             `json:"drop_metrics"`        // List of metric names to drop. For fine-grained dropping use drop_metrics_if
-	DropMetricsIf     []string                             `json:"drop_metrics_if,omitempty"`     // List of evaluatable terms to drop metrics
+	DropMetricsIf     []string                             `json:"drop_metrics_if"`     // List of evaluatable terms to drop metrics
-	RenameMetrics     map[string]string                    `json:"rename_metrics,omitempty"`      // Map to rename metric name from key to value
+	RenameMetrics     map[string]string                    `json:"rename_metrics"`      // Map to rename metric name from key to value
-	IntervalStamp     bool                                 `json:"interval_timestamp,omitempty"`  // Update timestamp periodically by ticker each interval?
+	IntervalStamp     bool                                 `json:"interval_timestamp"`  // Update timestamp periodically by ticker each interval?
-	NumCacheIntervals int                                  `json:"num_cache_intervals,omitempty"` // Number of intervals of cached metrics for evaluation
+	NumCacheIntervals int                                  `json:"num_cache_intervals"` // Number of intervals of cached metrics for evaluation
-	MaxForward        int                                  `json:"max_forward,omitempty"`         // Number of maximal forwarded metrics at one select
+	MaxForward        int                                  `json:"max_forward"`         // Number of maximal forwarded metrics at one select
-	NormalizeUnits    bool                                 `json:"normalize_units,omitempty"`     // Check unit meta flag and normalize it using cc-units
+	NormalizeUnits    bool                                 `json:"normalize_units"`     // Check unit meta flag and normalize it using cc-units
-	ChangeUnitPrefix  map[string]string                    `json:"change_unit_prefix,omitempty"`  // Add prefix that should be applied to the metrics
+	ChangeUnitPrefix  map[string]string                    `json:"change_unit_prefix"`  // Add prefix that should be applied to the metrics
 	MessageProcessor  json.RawMessage                      `json:"process_messages,omitempty"`
 }
 
@@ -228,11 +228,25 @@ func (r *metricRouter) DoAddTags(point lp.CCMessage) {
 func (r *metricRouter) Start() {
 	// start timer if configured
 	r.timestamp = time.Now()
-	timeChan := make(chan time.Time)
+	timeChan := make(chan time.Time, 1)
 	if r.config.IntervalStamp {
 		r.ticker.AddChannel(timeChan)
 	}
 
+	// Drain a pending tick before stamping new metrics, so a new interval's
+	// metrics never carry the previous interval's timestamp
+	updateTimestamp := func() {
+		if !r.config.IntervalStamp {
+			return
+		}
+		select {
+		case timestamp := <-timeChan:
+			r.timestamp = timestamp
+			cclog.ComponentDebug("MetricRouter", "Update timestamp", r.timestamp.UnixNano())
+		default:
+		}
+	}
+
 	// Router manager is done
 	done := func() {
 		close(r.done)
@@ -297,15 +311,20 @@ func (r *metricRouter) Start() {
 
 			case timestamp := <-timeChan:
 				r.timestamp = timestamp
-				cclog.ComponentDebugf("MetricRouter", "Update timestamp %d", r.timestamp.UnixNano())
+				cclog.ComponentDebug("MetricRouter", "Update timestamp", r.timestamp.UnixNano())
+				if len(r.coll_input) == cap(r.coll_input) {
+					cclog.ComponentWarn("MetricRouter", "collector input channel full at tick, sinks may be too slow")
+				}
 
 			case p := <-r.coll_input:
+				updateTimestamp()
 				coll_forward(p)
 				for i := 0; len(r.coll_input) > 0 && i < (r.maxForward-1); i++ {
 					coll_forward(<-r.coll_input)
 				}
 
 			case p := <-r.recv_input:
+				updateTimestamp()
 				recv_forward(p)
 				for i := 0; len(r.recv_input) > 0 && i < (r.maxForward-1); i++ {
 					recv_forward(<-r.recv_input)

internal/metricRouter/metricRouter_test.go

@@ -0,0 +1,99 @@
+package metricRouter
+
+import (
+	"encoding/json"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
+)
+
+// Fake ticker that delivers ticks on demand
+type fakeTicker struct {
+	channels []chan time.Time
+}
+
+func (t *fakeTicker) Init(duration time.Duration) {}
+
+func (t *fakeTicker) AddChannel(c chan time.Time) {
+	t.channels = append(t.channels, c)
+}
+
+func (t *fakeTicker) Close() {}
+
+func (t *fakeTicker) tick(ts time.Time) {
+	for _, c := range t.channels {
+		c <- ts
+	}
+}
+
+func genMessages(t *testing.T, num int) []lp.CCMessage {
+	t.Helper()
+	msgs := make([]lp.CCMessage, 0, num)
+	tags := map[string]string{"type": "node"}
+	for i := range num {
+		m, err := lp.NewMetric(fmt.Sprintf("testmetric%d", i), tags, nil, 42.0, time.Unix(1, 0))
+		if err != nil {
+			t.Fatalf("failed to create message: %s", err.Error())
+		}
+		msgs = append(msgs, m)
+	}
+	return msgs
+}
+
+// With interval_timestamp enabled, all metrics forwarded after a tick must
+// carry that tick's timestamp, never the previous interval's
+func TestIntervalTimestamp(t *testing.T) {
+	ticker := &fakeTicker{}
+	var wg sync.WaitGroup
+	r, err := New(ticker, &wg, json.RawMessage(`{"interval_timestamp": true}`))
+	if err != nil {
+		t.Fatalf("failed to setup metric router: %s", err.Error())
+	}
+
+	coll := make(chan lp.CCMessage, 100)
+	out := make(chan lp.CCMessage, 100)
+	r.AddCollectorInput(coll)
+	r.AddOutput(out)
+	r.Start()
+
+	receiveAll := func(num int) []lp.CCMessage {
+		received := make([]lp.CCMessage, 0, num)
+		for len(received) < num {
+			select {
+			case m := <-out:
+				received = append(received, m)
+			case <-time.After(5 * time.Second):
+				t.Fatalf("received only %d of %d messages", len(received), num)
+			}
+		}
+		return received
+	}
+
+	for interval, tickTime := range []time.Time{time.Unix(1000, 0), time.Unix(1010, 0)} {
+		ticker.tick(tickTime)
+		msgs := genMessages(t, 20)
+		for _, m := range msgs {
+			coll <- m
+		}
+		for i, m := range receiveAll(len(msgs)) {
+			if !m.Time().Equal(tickTime) {
+				t.Errorf("interval %d message %d: got timestamp %v, want %v", interval, i, m.Time(), tickTime)
+			}
+		}
+	}
+
+	closed := make(chan struct{})
+	go func() {
+		r.Close()
+		close(closed)
+	}()
+	select {
+	case <-closed:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Close() did not terminate")
+	}
+	wg.Wait()
+}

pkg/multiChanTicker/README.md

@@ -30,8 +30,8 @@ Afterwards, you can add channels:
 
 ```golang
 t := MultiChanTicker(duration)
-c1 := make(chan time.Time)
+c1 := make(chan time.Time, 1)
-c2 := make(chan time.Time)
+c2 := make(chan time.Time, 1)
 t.AddChannel(c1)
 t.AddChannel(c2)
 
@@ -46,3 +46,5 @@ for {
 ```
 
 The result should be the same `time.Time` output in both channels, notified "simultaneously".
+
+Ticks are delivered with a non-blocking send: a consumer that has not yet read the previous tick does not stall the ticker (which would silently drop `time.Ticker` fires for all consumers); instead, the tick for that consumer is skipped and a warning is logged. Register buffered channels (capacity 1) so a consumer that is briefly busy at tick time does not lose the tick.

pkg/multiChanTicker/multiChanTicker.go

@@ -8,6 +8,8 @@
 package multiChanTicker
 
 import (
+	"fmt"
+	"sync"
 	"time"
 
 	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
@@ -15,6 +17,7 @@ import (
 
 type multiChanTicker struct {
 	ticker   *time.Ticker
+	mutex    sync.Mutex // protects channels, which is appended to while the tick goroutine iterates it
 	channels []chan time.Time
 	done     chan bool
 }
@@ -40,21 +43,30 @@ func (t *multiChanTicker) Init(duration time.Duration) {
 				return
 			case ts := <-t.ticker.C:
 				cclog.ComponentDebug("MultiChanTicker", "Tick", ts)
-				for _, c := range t.channels {
+				t.mutex.Lock()
+				for i, c := range t.channels {
+					// Non-blocking send: a consumer that has not yet read the
+					// previous tick must not stall the ticker, otherwise
+					// time.Ticker silently drops fires for ALL consumers
 					select {
-					case <-t.done:
-						done()
-						return
 					case c <- ts:
+					default:
+						cclog.ComponentWarn("MultiChanTicker", fmt.Sprintf("consumer %d did not read previous tick, dropping tick %v", i, ts))
 					}
 				}
+				t.mutex.Unlock()
 			}
 		}
 	}()
 }
 
 func (t *multiChanTicker) AddChannel(channel chan time.Time) {
+	if cap(channel) == 0 {
+		cclog.ComponentWarn("MultiChanTicker", "unbuffered channel registered, ticks may be dropped if the consumer is not ready")
+	}
+	t.mutex.Lock()
 	t.channels = append(t.channels, channel)
+	t.mutex.Unlock()
 }
 
 func (t *multiChanTicker) Close() {

pkg/multiChanTicker/multiChanTicker_test.go

@@ -0,0 +1,51 @@
+package multiChanTicker
+
+import (
+	"testing"
+	"time"
+)
+
+// A consumer that never reads its channel must not stall the ticker
+// or starve the other consumers
+func TestStalledConsumerDoesNotStarveOthers(t *testing.T) {
+	stalled := make(chan time.Time, 1) // never read
+	fast := make(chan time.Time, 1)
+
+	ticker := NewTicker(10 * time.Millisecond)
+	defer ticker.Close()
+	ticker.AddChannel(stalled)
+	ticker.AddChannel(fast)
+
+	received := 0
+	deadline := time.After(5 * time.Second)
+	for received < 5 {
+		select {
+		case <-fast:
+			received++
+		case <-deadline:
+			t.Fatalf("received only %d ticks while another consumer stalled", received)
+		}
+	}
+}
+
+// Close() must return promptly even if no consumer reads its channel
+func TestCloseWithStalledConsumer(t *testing.T) {
+	stalled := make(chan time.Time, 1) // never read
+
+	ticker := NewTicker(10 * time.Millisecond)
+	ticker.AddChannel(stalled)
+
+	// Let some ticks fire and be dropped
+	time.Sleep(50 * time.Millisecond)
+
+	closed := make(chan struct{})
+	go func() {
+		ticker.Close()
+		close(closed)
+	}()
+	select {
+	case <-closed:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Close() blocked with a stalled consumer")
+	}
+}