Update rapl collector with powercap limits

.gitignore

@@ -17,7 +17,3 @@
 
 # Local copy of LIKWID headers
 /collectors/likwid
-
-# Local development workspace (build against a local cc-lib checkout)
-go.work
-go.work.sum

README.md

@@ -41,8 +41,6 @@ There is a main configuration file with basic settings that point to the other c
 
 The `interval` defines how often the metrics should be read and send to the sink(s). The `duration` tells the collectors how long one measurement has to take. This is important for some collectors, like the `likwid` collector. For more information, see [here](./docs/configuration.md).
 
-The optional `channel_buffer_size` sets the capacity of the internal channels between the components. If unset, it defaults to `max(200, 24 * number of CPUs)` so that one interval's burst of per-hwthread metrics fits without back-pressuring the collectors on nodes with many cores.
-
 See the component READMEs for their configuration:
 
 * [`collectors`](./collectors/README.md)

cc-metric-collector.go

@@ -13,7 +13,6 @@ import (
 	"flag"
 	"os"
 	"os/signal"
-	"runtime"
 	"sync"
 	"syscall"
 	"time"
@@ -30,9 +29,8 @@ import (
 )
 
 type CentralConfigFile struct {
-	Interval          string `json:"interval"`
-	Duration          string `json:"duration"`
-	ChannelBufferSize int    `json:"channel_buffer_size,omitempty"`
+	Interval string `json:"interval"`
+	Duration string `json:"duration"`
 }
 
 type RuntimeConfig struct {
@@ -160,14 +158,6 @@ func mainFunc() int {
 		return 1
 	}
 
-	// Size the channels between the managers so that one interval's burst of
-	// per-hwthread metrics fits without back-pressuring the collectors
-	chanSize := rcfg.ConfigFile.ChannelBufferSize
-	if chanSize <= 0 {
-		chanSize = max(200, 24*runtime.NumCPU())
-	}
-	cclog.ComponentDebug("main", "channel buffer size", chanSize)
-
 	routerConf := ccconf.GetPackageConfig("router")
 	if len(routerConf) == 0 {
 		cclog.Error("Metric router configuration file must be set")
@@ -204,7 +194,7 @@ func mainFunc() int {
 	}
 
 	// Connect metric router to sink manager
-	RouterToSinksChannel := make(chan lp.CCMessage, chanSize)
+	RouterToSinksChannel := make(chan lp.CCMessage, 200)
 	rcfg.SinkManager.AddInput(RouterToSinksChannel)
 	rcfg.MetricRouter.AddOutput(RouterToSinksChannel)
 
@@ -216,7 +206,7 @@ func mainFunc() int {
 	}
 
 	// Connect collector manager to metric router
-	CollectToRouterChannel := make(chan lp.CCMessage, chanSize)
+	CollectToRouterChannel := make(chan lp.CCMessage, 200)
 	rcfg.CollectManager.AddOutput(CollectToRouterChannel)
 	rcfg.MetricRouter.AddCollectorInput(CollectToRouterChannel)
 
@@ -230,7 +220,7 @@ func mainFunc() int {
 		}
 
 		// Connect receive manager to metric router
-		ReceiveToRouterChannel := make(chan lp.CCMessage, chanSize)
+		ReceiveToRouterChannel := make(chan lp.CCMessage, 200)
 		rcfg.ReceiveManager.AddOutput(ReceiveToRouterChannel)
 		rcfg.MetricRouter.AddReceiverInput(ReceiveToRouterChannel)
 		use_recv = true

collectors/collectorManager.go

@@ -12,7 +12,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
@@ -22,52 +21,49 @@ import (
 
 // Map of all available metric collectors
 var AvailableCollectors = map[string]MetricCollector{
-	"likwid":             new(LikwidCollector),
-	"loadavg":            new(LoadavgCollector),
-	"memstat":            new(MemstatCollector),
-	"netstat":            new(NetstatCollector),
-	"ibstat":             new(InfinibandCollector),
-	"lustrestat":         new(LustreCollector),
-	"cpustat":            new(CpustatCollector),
-	"topprocs":           new(TopProcsCollector),
-	"nvidia":             new(NvidiaCollector),
-	"customcmd":          new(CustomCmdCollector),
-	"iostat":             new(IOstatCollector),
-	"diskstat":           new(DiskstatCollector),
-	"tempstat":           new(TempCollector),
-	"ipmistat":           new(IpmiCollector),
-	"gpfs":               new(GpfsCollector),
-	"cpufreq":            new(CPUFreqCollector),
-	"cpufreq_cpuinfo":    new(CPUFreqCpuInfoCollector),
-	"nfs3stat":           new(Nfs3Collector),
-	"nfs4stat":           new(Nfs4Collector),
-	"numastats":          new(NUMAStatsCollector),
-	"beegfs_meta":        new(BeegfsMetaCollector),
-	"beegfs_storage":     new(BeegfsStorageCollector),
-	"rapl":               new(RAPLCollector),
-	"rocm_smi":           new(RocmSmiCollector),
-	"self":               new(SelfCollector),
-	"schedstat":          new(SchedstatCollector),
-	"nfsiostat":          new(NfsIOStatCollector),
-	"slurm_cgroup":       new(SlurmCgroupCollector),
-	"smartmon":           new(SmartMonCollector),
-	"lenovo_dense_power": new(LenovoDensePowerCollector),
-	"megware_eureka":     new(MegwareEurekaCollector),
+	"likwid":          new(LikwidCollector),
+	"loadavg":         new(LoadavgCollector),
+	"memstat":         new(MemstatCollector),
+	"netstat":         new(NetstatCollector),
+	"ibstat":          new(InfinibandCollector),
+	"lustrestat":      new(LustreCollector),
+	"cpustat":         new(CpustatCollector),
+	"topprocs":        new(TopProcsCollector),
+	"nvidia":          new(NvidiaCollector),
+	"customcmd":       new(CustomCmdCollector),
+	"iostat":          new(IOstatCollector),
+	"diskstat":        new(DiskstatCollector),
+	"tempstat":        new(TempCollector),
+	"ipmistat":        new(IpmiCollector),
+	"gpfs":            new(GpfsCollector),
+	"cpufreq":         new(CPUFreqCollector),
+	"cpufreq_cpuinfo": new(CPUFreqCpuInfoCollector),
+	"nfs3stat":        new(Nfs3Collector),
+	"nfs4stat":        new(Nfs4Collector),
+	"numastats":       new(NUMAStatsCollector),
+	"beegfs_meta":     new(BeegfsMetaCollector),
+	"beegfs_storage":  new(BeegfsStorageCollector),
+	"rapl":            new(RAPLCollector),
+	"rocm_smi":        new(RocmSmiCollector),
+	"self":            new(SelfCollector),
+	"schedstat":       new(SchedstatCollector),
+	"nfsiostat":       new(NfsIOStatCollector),
+	"slurm_cgroup":    new(SlurmCgroupCollector),
+	"smartmon":        new(SmartMonCollector),
 }
 
 // Metric collector manager data structure
 type collectorManager struct {
-	collectors    []MetricCollector          // List of metric collectors to read in parallel
-	serial        []MetricCollector          // List of metric collectors to read serially
-	output        chan lp.CCMessage          // Output channels
-	done          chan bool                  // channel to finish / stop metric collector manager
-	ticker        mct.MultiChanTicker        // periodically ticking once each interval
-	duration      time.Duration              // duration (for metrics that measure over a given duration)
-	wg            *sync.WaitGroup            // wait group for all goroutines in cc-metric-collector
-	config        map[string]json.RawMessage // json encoded config for collector manager
-	collector_wg  sync.WaitGroup             // internally used wait group for the parallel reading of collector
-	round_wg      sync.WaitGroup             // wait group for the currently running collection round
-	round_running atomic.Bool                // Flag whether a collection round is currently running
+	collectors   []MetricCollector          // List of metric collectors to read in parallel
+	serial       []MetricCollector          // List of metric collectors to read serially
+	output       chan lp.CCMessage          // Output channels
+	done         chan bool                  // channel to finish / stop metric collector manager
+	ticker       mct.MultiChanTicker        // periodically ticking once each interval
+	duration     time.Duration              // duration (for metrics that measure over a given duration)
+	wg           *sync.WaitGroup            // wait group for all goroutines in cc-metric-collector
+	config       map[string]json.RawMessage // json encoded config for collector manager
+	collector_wg sync.WaitGroup             // internally used wait group for the parallel reading of collector
+	parallel_run bool                       // Flag whether the collectors are currently read in parallel
 }
 
 // Metric collector manager access functions
@@ -123,54 +119,26 @@ func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Durat
 	return nil
 }
 
-// runRound executes one collection round: first all parallel collectors
-// concurrently, then the serial collectors one by one
-func (cm *collectorManager) runRound(t time.Time) {
-	roundStart := time.Now()
-	for _, c := range cm.collectors {
-		// Read metrics from collector c via goroutine
-		cclog.ComponentDebug("CollectorManager", c.Name(), t)
-		cm.collector_wg.Add(1)
-		go func(myc MetricCollector) {
-			start := time.Now()
-			myc.Read(cm.duration, cm.output)
-			cclog.ComponentDebug("CollectorManager", myc.Name(), "took", time.Since(start))
-			cm.collector_wg.Done()
-		}(c)
-	}
-	cm.collector_wg.Wait()
-	for _, c := range cm.serial {
-		// Read metrics from collector c
-		cclog.ComponentDebug("CollectorManager", c.Name(), t)
-		start := time.Now()
-		c.Read(cm.duration, cm.output)
-		cclog.ComponentDebug("CollectorManager", c.Name(), "took", time.Since(start))
-	}
-	cclog.ComponentDebug("CollectorManager", "collection round took", time.Since(roundStart))
-}
-
 // Start starts the metric collector manager
 func (cm *collectorManager) Start() {
-	tick := make(chan time.Time, 1)
+	tick := make(chan time.Time)
 	cm.ticker.AddChannel(tick)
 
 	cm.wg.Go(func() {
 		// Collector manager is done
 		done := func() {
-			// wait for a still running collection round, then close all metric collectors
-			cm.round_wg.Wait()
-			for _, c := range cm.collectors {
-				c.Close()
+			// close all metric collectors
+			if cm.parallel_run {
+				cm.collector_wg.Wait()
+				cm.parallel_run = false
 			}
-			for _, c := range cm.serial {
+			for _, c := range cm.collectors {
 				c.Close()
 			}
 			close(cm.done)
 			cclog.ComponentDebug("CollectorManager", "DONE")
 		}
 
-		var roundStart time.Time
-
 		// Wait for done signal or timer event
 		for {
 			select {
@@ -178,20 +146,37 @@ func (cm *collectorManager) Start() {
 				done()
 				return
 			case t := <-tick:
-				// The round runs detached from this loop, so the tick channel
-				// stays drained even when a round takes longer than the interval
-				if cm.round_running.Load() {
-					cclog.ComponentWarn("CollectorManager", "collection round still running after", time.Since(roundStart), "- skipping tick")
-					continue
+				cm.parallel_run = true
+				for _, c := range cm.collectors {
+					// Wait for done signal or execute the collector
+					select {
+					case <-cm.done:
+						done()
+						return
+					default:
+						// Read metrics from collector c via goroutine
+						cclog.ComponentDebug("CollectorManager", c.Name(), t)
+						cm.collector_wg.Add(1)
+						go func(myc MetricCollector) {
+							myc.Read(cm.duration, cm.output)
+							cm.collector_wg.Done()
+						}(c)
+					}
+				}
+				cm.collector_wg.Wait()
+				cm.parallel_run = false
+				for _, c := range cm.serial {
+					// Wait for done signal or execute the collector
+					select {
+					case <-cm.done:
+						done()
+						return
+					default:
+						// Read metrics from collector c
+						cclog.ComponentDebug("CollectorManager", c.Name(), t)
+						c.Read(cm.duration, cm.output)
+					}
 				}
-				cm.round_running.Store(true)
-				roundStart = time.Now()
-				cm.round_wg.Add(1)
-				go func() {
-					defer cm.round_wg.Done()
-					defer cm.round_running.Store(false)
-					cm.runRound(t)
-				}()
 			}
 		}
 	})

collectors/collectorManager_test.go

@@ -1,121 +0,0 @@
-package collectors
-
-import (
-	"encoding/json"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
-)
-
-// Fake ticker that delivers ticks on demand
-type fakeTicker struct {
-	channels []chan time.Time
-}
-
-func (t *fakeTicker) Init(duration time.Duration) {}
-
-func (t *fakeTicker) AddChannel(c chan time.Time) {
-	t.channels = append(t.channels, c)
-}
-
-func (t *fakeTicker) Close() {}
-
-func (t *fakeTicker) tick() {
-	for _, c := range t.channels {
-		select {
-		case c <- time.Now():
-		default:
-		}
-	}
-}
-
-// Stub collector whose Read blocks until it is released
-type stubCollector struct {
-	metricCollector
-	readStarted chan struct{}
-	release     chan struct{}
-	reads       atomic.Int32
-}
-
-func (c *stubCollector) Init(config json.RawMessage) error {
-	c.name = "teststub"
-	c.parallel = true
-	c.init = true
-	return nil
-}
-
-func (c *stubCollector) Read(duration time.Duration, output chan lp.CCMessage) {
-	c.reads.Add(1)
-	c.readStarted <- struct{}{}
-	<-c.release
-}
-
-func (c *stubCollector) Close() {}
-
-func TestOverlongCollectionRoundSkipsTick(t *testing.T) {
-	stub := &stubCollector{
-		readStarted: make(chan struct{}, 10),
-		release:     make(chan struct{}),
-	}
-	AvailableCollectors["teststub"] = stub
-	defer delete(AvailableCollectors, "teststub")
-
-	ticker := &fakeTicker{}
-	var wg sync.WaitGroup
-	cm, err := New(ticker, time.Second, &wg, json.RawMessage(`{"teststub": {}}`))
-	if err != nil {
-		t.Fatalf("failed to setup collector manager: %s", err.Error())
-	}
-	cm.AddOutput(make(chan lp.CCMessage, 100))
-	cm.Start()
-
-	// First tick starts a collection round that blocks in Read
-	ticker.tick()
-	select {
-	case <-stub.readStarted:
-	case <-time.After(5 * time.Second):
-		t.Fatal("collection round did not start on tick")
-	}
-
-	// Further ticks while the round is still running must be skipped,
-	// not queued up or run concurrently
-	for range 3 {
-		ticker.tick()
-		time.Sleep(20 * time.Millisecond)
-	}
-	if got := stub.reads.Load(); got != 1 {
-		t.Fatalf("expected 1 concurrent collection round, got %d reads", got)
-	}
-
-	// Finish the round, the next tick must start a new one
-	stub.release <- struct{}{}
-	deadline := time.After(5 * time.Second)
-	for stub.reads.Load() < 2 {
-		ticker.tick()
-		select {
-		case <-stub.readStarted:
-		case <-time.After(20 * time.Millisecond):
-		case <-deadline:
-			t.Fatal("no new collection round started after the previous one finished")
-		}
-	}
-
-	// Shutdown must wait for the running round and terminate cleanly.
-	// Closing the release channel lets any still running or straggler
-	// round finish immediately
-	close(stub.release)
-	closed := make(chan struct{})
-	go func() {
-		cm.Close()
-		close(closed)
-	}()
-	select {
-	case <-closed:
-	case <-time.After(5 * time.Second):
-		t.Fatal("Close() did not terminate")
-	}
-	wg.Wait()
-}

collectors/cpufreqCpuinfoMetric.go

@@ -116,9 +116,6 @@ func (m *CPUFreqCpuInfoCollector) Init(_ json.RawMessage) error {
 			physicalPackageID = ""
 		}
 	}
-	if err := scanner.Err(); err != nil {
-		return fmt.Errorf("%s Init(): Call to scanner.Err failed: %w", m.name, err)
-	}
 
 	if err := file.Close(); err != nil {
 		return fmt.Errorf("%s Init(): Call to file.Close() failed: %w", m.name, err)
@@ -182,9 +179,6 @@ func (m *CPUFreqCpuInfoCollector) Read(interval time.Duration, output chan lp.CC
 			}
 		}
 	}
-	if err := scanner.Err(); err != nil {
-		cclog.ComponentErrorf(m.name, "Read(): Call to scanner.Err failed: %s", err.Error())
-	}
 }
 
 func (m *CPUFreqCpuInfoCollector) Close() {

collectors/cpustatMetric.go

@@ -44,7 +44,7 @@ type CpustatCollector struct {
 func (m *CpustatCollector) Init(config json.RawMessage) error {
 	m.name = "CpustatCollector"
 	if err := m.setup(); err != nil {
-		return fmt.Errorf("%s Init(): setup() call failed: %s", m.name, err.Error())
+		return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err)
 	}
 	m.parallel = true
 	m.meta = map[string]string{
@@ -58,7 +58,7 @@ func (m *CpustatCollector) Init(config json.RawMessage) error {
 		d := json.NewDecoder(bytes.NewReader(config))
 		d.DisallowUnknownFields()
 		if err := d.Decode(&m.config); err != nil {
-			return fmt.Errorf("%s Init(): Error decoding JSON config: %s", m.name, err.Error())
+			return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err)
 		}
 	}
 	matches := map[string]int{
@@ -85,7 +85,7 @@ func (m *CpustatCollector) Init(config json.RawMessage) error {
 	// Check input file
 	file, err := os.Open(CPUSTATFILE)
 	if err != nil {
-		return fmt.Errorf("%s Init(): Failed to open file '%s': %s", m.name, CPUSTATFILE, err.Error())
+		return fmt.Errorf("%s Init(): Failed to open file '%s': %w", m.name, CPUSTATFILE, err)
 	}
 
 	// Pre-generate tags for all CPUs
@@ -117,13 +117,10 @@ func (m *CpustatCollector) Init(config json.RawMessage) error {
 			num_cpus++
 		}
 	}
-	if err := scanner.Err(); err != nil {
-		return fmt.Errorf("%s Init(): Call to scanner.Err failed: %s", m.name, err.Error())
-	}
 
 	// Close file
 	if err := file.Close(); err != nil {
-		return fmt.Errorf("%s Init(): Failed to close file '%s': %s", m.name, CPUSTATFILE, err.Error())
+		return fmt.Errorf("%s Init(): Failed to close file '%s': %w", m.name, CPUSTATFILE, err)
 	}
 
 	m.lastTimestamp = time.Now()
@@ -176,13 +173,13 @@ func (m *CpustatCollector) Read(interval time.Duration, output chan lp.CCMessage
 	if err != nil {
 		cclog.ComponentError(
 			m.name,
-			fmt.Sprintf("Read(): Failed to open file '%s': %v", CPUSTATFILE, err.Error()))
+			fmt.Sprintf("Read(): Failed to open file '%s': %v", CPUSTATFILE, err))
 	}
 	defer func() {
 		if err := file.Close(); err != nil {
 			cclog.ComponentError(
 				m.name,
-				fmt.Sprintf("Read(): Failed to close file '%s': %v", string(CPUSTATFILE), err.Error()))
+				fmt.Sprintf("Read(): Failed to close file '%s': %v", string(CPUSTATFILE), err))
 		}
 	}()
 
@@ -197,9 +194,6 @@ func (m *CpustatCollector) Read(interval time.Duration, output chan lp.CCMessage
 			num_cpus++
 		}
 	}
-	if err := scanner.Err(); err != nil {
-		cclog.ComponentErrorf(m.name, "Init(): Call to scanner.Err failed: %s", err.Error())
-	}
 
 	if !m.config.excludeNumCPUs {
 		if num_cpus_metric, err := lp.NewMetric("num_cpus", m.nodetags, m.meta, num_cpus, now); err == nil {

collectors/diskstatMetric.go

@@ -147,11 +147,6 @@ mountLoop:
 			}
 		}
 	}
-	if err := scanner.Err(); err != nil {
-		cclog.ComponentErrorf(
-			m.name,
-			"Read(): Call to scanner.Err failed: %s", err.Error())
-	}
 	if m.allowedMetrics["part_max_used"] {
 		y, err := lp.NewMetric("part_max_used", map[string]string{"type": "node"}, m.meta, int(part_max_used), time.Now())
 		if err == nil {

collectors/iostatMetric.go

@@ -48,13 +48,13 @@ func (m *IOstatCollector) Init(config json.RawMessage) error {
 	m.parallel = true
 	m.meta = map[string]string{"source": m.name, "group": "Disk"}
 	if err := m.setup(); err != nil {
-		return fmt.Errorf("%s Init(): setup() call failed: %s", m.name, err.Error())
+		return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err)
 	}
 	if len(config) > 0 {
 		d := json.NewDecoder(bytes.NewReader(config))
 		d.DisallowUnknownFields()
 		if err := d.Decode(&m.config); err != nil {
-			return fmt.Errorf("%s Init(): Error decoding JSON config: %s", m.name, err.Error())
+			return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err)
 		}
 	}
 	// https://www.kernel.org/doc/html/latest/admin-guide/iostats.html
@@ -89,7 +89,7 @@ func (m *IOstatCollector) Init(config json.RawMessage) error {
 	}
 	file, err := os.Open(IOSTATFILE)
 	if err != nil {
-		return fmt.Errorf("%s Init(): Failed to open file \"%s\": %s", m.name, IOSTATFILE, err.Error())
+		return fmt.Errorf("%s Init(): Failed to open file \"%s\": %w", m.name, IOSTATFILE, err)
 	}
 
 	scanner := bufio.NewScanner(file)
@@ -130,11 +130,8 @@ func (m *IOstatCollector) Init(config json.RawMessage) error {
 			lastValues:    lastValues,
 		}
 	}
-	if err := scanner.Err(); err != nil {
-		return fmt.Errorf("%s Init(): Failed to scan content of file %q: %s", m.name, IOSTATFILE, err.Error())
-	}
 	if err := file.Close(); err != nil {
-		return fmt.Errorf("%s Init(): Failed to close file \"%s\": %s", m.name, IOSTATFILE, err.Error())
+		return fmt.Errorf("%s Init(): Failed to close file \"%s\": %w", m.name, IOSTATFILE, err)
 	}
 
 	m.init = true
@@ -150,14 +147,14 @@ func (m *IOstatCollector) Read(interval time.Duration, output chan lp.CCMessage)
 	if err != nil {
 		cclog.ComponentError(
 			m.name,
-			fmt.Sprintf("Read(): Failed to open file '%s': %s", IOSTATFILE, err.Error()))
+			fmt.Sprintf("Read(): Failed to open file '%s': %v", IOSTATFILE, err))
 		return
 	}
 	defer func() {
 		if err := file.Close(); err != nil {
 			cclog.ComponentError(
 				m.name,
-				fmt.Sprintf("Read(): Failed to close file '%s': %s", IOSTATFILE, err.Error()))
+				fmt.Sprintf("Read(): Failed to close file '%s': %v", IOSTATFILE, err))
 		}
 	}()
 
@@ -201,11 +198,6 @@ func (m *IOstatCollector) Read(interval time.Duration, output chan lp.CCMessage)
 		}
 		m.devices[device] = entry
 	}
-	if err := scanner.Err(); err != nil {
-		cclog.ComponentErrorf(
-			m.name,
-			"Read(): Failed to scan content of file %q: %s", IOSTATFILE, err.Error())
-	}
 }
 
 func (m *IOstatCollector) Close() {

collectors/ipmiMetric.go

@@ -28,15 +28,13 @@ type IpmiCollector struct {
 	metricCollector
 
 	config struct {
-		IpmitoolPath    string   `json:"ipmitool_path"`
-		IpmisensorsPath string   `json:"ipmisensors_path"`
-		Sudo            bool     `json:"use_sudo"`
-		IncludeMetrics  []string `json:"include_metrics"`
+		IpmitoolPath    string `json:"ipmitool_path"`
+		IpmisensorsPath string `json:"ipmisensors_path"`
+		Sudo            bool   `json:"use_sudo"`
 	}
 
-	ipmitool       string
-	ipmisensors    string
-	includeMetrics map[string]bool
+	ipmitool    string
+	ipmisensors string
 }
 
 func (m *IpmiCollector) Init(config json.RawMessage) error {
@@ -66,15 +64,6 @@ func (m *IpmiCollector) Init(config json.RawMessage) error {
 		}
 	}
 
-	// Read metrics to include
-	m.includeMetrics = make(map[string]bool)
-	for _, metric := range m.config.IncludeMetrics {
-		metric = strings.ToLower(strings.TrimSpace(metric))
-		if metric != "" {
-			m.includeMetrics[metric] = true
-		}
-	}
-
 	m.ipmitool = m.config.IpmitoolPath
 	m.ipmisensors = m.config.IpmisensorsPath
 
@@ -156,11 +145,6 @@ func (m *IpmiCollector) readIpmiTool(output chan lp.CCMessage) error {
 			continue
 		}
 		name := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(lv[0]), " ", "_"))
-
-		if len(m.includeMetrics) > 0 && !m.includeMetrics[name] {
-			continue
-		}
-
 		unit := strings.TrimSpace(lv[2])
 		switch unit {
 		case "Volts":
@@ -181,9 +165,6 @@ func (m *IpmiCollector) readIpmiTool(output chan lp.CCMessage) error {
 		y.AddMeta("unit", unit)
 		output <- y
 	}
-	if err := scanner.Err(); err != nil {
-		return fmt.Errorf("failed to scan output of command: %s", err.Error())
-	}
 
 	// Wait for command end
 	if err := command.Wait(); err != nil {
@@ -228,11 +209,6 @@ func (m *IpmiCollector) readIpmiSensors(output chan lp.CCMessage) error {
 			continue
 		}
 		name := strings.ToLower(strings.ReplaceAll(lv[1], " ", "_"))
-
-		if len(m.includeMetrics) > 0 && !m.includeMetrics[name] {
-			continue
-		}
-
 		y, err := lp.NewMetric(name, map[string]string{"type": "node"}, m.meta, v, time.Now())
 		if err != nil {
 			cclog.ComponentErrorf(m.name, "Failed to create message: %v", err)

collectors/ipmiMetric.md

@@ -15,8 +15,7 @@ hugo_path: docs/reference/cc-metric-collector/collectors/ipmi.md
   "ipmistat": {
     "ipmitool_path": "/path/to/ipmitool",
     "ipmisensors_path": "/path/to/ipmi-sensors",
-    "use_sudo": true,
-    "include_metrics" : []
+    "use_sudo": true
   }
 ```
 
@@ -37,5 +36,3 @@ Defaults: monitoring !log_allowed, !pam_session
 monitoring ALL = (root) NOPASSWD:/usr/bin/ipmitool sensor
 monitoring ALL = (root) NOPASSWD:/usr/sbin/ipmi-sensors --comma-separated-output --sdr-cache-recreate
 ```
-
-If `include_ipmi_metrics` contains any entry, ipmistat collector will only submit these metrics. Any other values will get discarded. 

collectors/lenovoDensePower.go

@@ -1,170 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-// additional authors:
-// Michael Panzlaff (NHR@FAU)
-
-package collectors
-
-import (
-	"bytes"
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
-	"os/exec"
-	"strconv"
-	"strings"
-	"time"
-
-	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
-)
-
-type LenovoDensePowerCollector struct {
-	metricCollector
-
-	config struct {
-		IpmitoolPath string `json:"ipmitool_path"`
-		Sudo         bool   `json:"use_sudo"`
-	}
-
-	ipmitool string
-
-	energyValLast  float64
-	energyTimeLast time.Time
-}
-
-func (m *LenovoDensePowerCollector) Init(config json.RawMessage) error {
-	// Check if already initialized
-	if m.init {
-		return nil
-	}
-
-	m.name = "LenovoDensePowerCollector"
-	if err := m.setup(); err != nil {
-		return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err)
-	}
-	m.meta = map[string]string{
-		"source": m.name,
-		"group":  "IPMI",
-	}
-
-	m.config.IpmitoolPath = "ipmitool"
-
-	if len(config) > 0 {
-		d := json.NewDecoder(bytes.NewReader(config))
-		d.DisallowUnknownFields()
-		if err := d.Decode(&m.config); err != nil {
-			return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err)
-		}
-	}
-
-	m.ipmitool = m.config.IpmitoolPath
-
-	energyVal, energyTime, err := m.readEnergy()
-	if err != nil {
-		return fmt.Errorf("energy reading test failed: %w", err)
-	}
-
-	m.energyValLast = energyVal
-	m.energyTimeLast = energyTime
-	m.init = true
-
-	return nil
-}
-
-func (m *LenovoDensePowerCollector) readEnergy() (float64, time.Time, error) {
-	argv := make([]string, 0)
-	if m.config.Sudo {
-		argv = append(argv, "sudo", "-n")
-	}
-
-	// The Lenovo hardware wants this magic byte string
-	lenovoRequestEnergyMsg := []uint8{0x3a, 0x32, 4, 2, 0, 0, 0}
-
-	argv = append(argv, m.ipmitool, "raw")
-
-	for _, val := range lenovoRequestEnergyMsg {
-		argv = append(argv, strconv.Itoa(int(val)))
-	}
-
-	cmd := exec.Command(argv[0], argv[1:]...)
-
-	var stdout bytes.Buffer
-	var stderr bytes.Buffer
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
-
-	err := cmd.Run()
-	if err != nil {
-		return 0, time.Unix(0, 0), fmt.Errorf("failed to run ipmitool: %w (stdout=%s stderr=%s)", err, stdout.String(), stderr.String())
-	}
-
-	data, err := hex.DecodeString(strings.ReplaceAll(strings.TrimSpace(stdout.String()), " ", ""))
-	if err != nil {
-		return 0, time.Unix(0, 0), fmt.Errorf("unable to decode ipmitool hex output: %w (stdout=%s)", err, stdout.String())
-	}
-
-	if len(data) != 14 {
-		return 0, time.Unix(0, 0), fmt.Errorf("result must be 14 bytes as specified in the documentation")
-	}
-
-	// data is laid out like this:
-	// data[0 to 1]: FIFO index of value (for debug only)
-	// data[2 to 5]: Integer part of energy in J (little endian)
-	// data[6 to 7]: Fraction part of energy in mJ (little endian)
-	// data[8 to 11]: Integer part of Unix time of measurement in seconds (little endian)
-	// data[12 to 13]: Fraction part of Unix time of measurement in milliseconds (little endian)
-	wholeJoules := (uint32(data[2]) << 0) | (uint32(data[3]) << 8) | (uint32(data[4]) << 16) | (uint32(data[5]) << 24)
-	milliJoules := uint16(data[6]) | (uint16(data[7]) << 8)
-	finalJoules := float64(wholeJoules) + float64(milliJoules)*1e-3
-
-	wholeSeconds := (uint32(data[8]) << 0) | (uint32(data[9]) << 8) | (uint32(data[10]) << 16) | (uint32(data[11]) << 24)
-	milliSeconds := uint16(data[12]) | (uint16(data[13]) << 8)
-	finalTime := time.Unix(int64(wholeSeconds), (int64(milliSeconds) * 1000000))
-
-	return finalJoules, finalTime, nil
-}
-
-func (m *LenovoDensePowerCollector) Read(interval time.Duration, output chan lp.CCMessage) {
-	// Check if already initialized
-	if !m.init {
-		return
-	}
-
-	energyVal, energyTime, err := m.readEnergy()
-	if err != nil {
-		cclog.ComponentErrorf(m.name, "readEnergy failed: %v", err)
-		return
-	}
-
-	powerVal := 0.0
-	if energyTime.Compare(m.energyTimeLast) != 0 {
-		// Check for overflow
-		energyValDiff := int64(energyVal) - int64(m.energyValLast)
-		if energyVal < m.energyValLast {
-			energyValDiff += int64(0x100000000)
-		}
-		energyTimeDiff := energyTime.Sub(m.energyTimeLast)
-		if energyTime.Before(m.energyTimeLast) {
-			energyTimeDiff = energyTimeDiff + 0x100000000*time.Second
-		}
-		powerVal = float64(energyValDiff) / energyTimeDiff.Seconds()
-
-		m.energyValLast = energyVal
-		m.energyTimeLast = energyTime
-	}
-
-	metric, err := lp.NewMetric("lenovo_node_power", map[string]string{"type": "node"}, m.meta, powerVal, energyTime)
-	if err != nil {
-		cclog.ComponentErrorf(m.name, "lp.NewMetric failed: %v", err)
-		return
-	}
-	metric.AddMeta("unit", "Watts")
-	output <- metric
-}
-
-func (m *LenovoDensePowerCollector) Close() {
-	m.init = false
-}

collectors/lenovoDensePower.md

@@ -1,51 +0,0 @@
-<!--
----
-title: Lenovo Dense Power collector
-description: Collect power of Lenovo Dense machines using ipmitool raw
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/lenovoDensePower.md
----
--->
-
-## `lenovo_dense_power` collector
-
-```json
-  "lenovo_dense_power": {
-    "ipmitool_path": "/path/to/ipmitool",
-    "use_sudo": true,
-    "include_metrics" : []
-  }
-```
-
-The `lenovo_dense_power` collector reads power from Lenovo Dense machines via `ipmitool` (`ipmitool raw ...`). This collector is known to only work on the following machines. Others are not supported, so use at your *OWN* risk:
-
-System                                    | Compatibility             | Power measured
----                                       | ---                       | ---
-Lenovo ThinkSystem SD650 V2/V3            | untested, but should work | node + riser1 + riser2
-Lenovo ThinkSystem SD650-N V2/ SD650-I V3 | untested, but should work | node + riser1 + riser2 + gpus
-Lenovo ThinkSystem SD665-N V3             | tested, known to work     | node + riser1 + riser2 + gpus
-
-To test if your system *may* be supported, you can run the following command *at your own risk*:
-
-```
-$ ipmitool raw 58 50 4 2 0 0 0
- c4 00 83 a9 cb d3 a2 03 df db 47 6a d8 01
-```
-
-Exactly 14 bytes should be returned.
-In case not or another error occurs your hardware is likely not supported.
-
-In addition, `ipmitool` typically require root to run.
-In order to run `cc-metric-collector` without root priviliges, you can enable `use_sudo`.
-Add a file like this in `/etc/sudoers.d/` to allow `cc-metric-collector` to run the required commands:
-
-```
-# Do not log the following sudo commands from monitoring, since this causes a lot of log spam.
-# However keep log_denied enabled, to detect failures
-Defaults: monitoring !log_allowed, !pam_session
-
-# Allow to use ipmitool for Lenovo power readings
-monitoring ALL = (root) NOPASSWD:/usr/bin/ipmitool raw 58 50 4 2 0 0 0
-```

collectors/likwidMetric.go

@@ -38,7 +38,7 @@ import (
 	topo "github.com/ClusterCockpit/cc-metric-collector/pkg/ccTopology"
 	"github.com/NVIDIA/go-nvml/pkg/dl"
 	"github.com/fsnotify/fsnotify"
-	"golang.design/x/runtime/thread"
+	"golang.design/x/thread"
 )
 
 const (

collectors/megwareEureka.go

@@ -1,194 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-// additional authors:
-// Michael Panzlaff (NHR@FAU)
-
-package collectors
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"os/exec"
-	"time"
-
-	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
-)
-
-// MPS refers to Monolithic Power Systems, from which the MP5922 is used
-// to measure telemetry on the Eureka platform.
-type mpsData struct {
-	Cnt         int64   `json:"Cnt"` // No idea what this is, ignore
-	Energy      float64 `json:"Energy"`
-	Vin         float64 `json:"Vin"`
-	Iin         float64 `json:"Iin"`
-	Pin         float64 `json:"Pin"`
-	PinAvg      float64 `json:"PinAvg"`
-	Vout        float64 `json:"Vout"`
-	Iout        float64 `json:"Iout"`
-	Pout        float64 `json:"Pout"`
-	StandbyVout float64 `json:"StandbyVout"`
-	StandbyIout float64 `json:"StandbyIout"`
-	StandbyPout float64 `json:"StandbyPout"`
-	TempBusbar  float64 `json:"TempBusbar"`
-	TempSsd     float64 `json:"TempSsd"`
-	TempMps     float64 `json:"TempMps"`
-	// No idea what the ones below mean exactl. Ignore them for now.
-	EnergyTime         int64 `json:"EnergyTime"`
-	EnergyAccumulator  int64 `json:"EnergyAccumulator"`
-	EnergyRolloverCnt  int64 `json:"EnergyRolloverCnt"`
-	EnergySampleCntU24 int64 `json:"EnergySampleCntU24"`
-
-	Timestamp time.Time
-}
-
-type MegwareEurekaCollector struct {
-	metricCollector
-
-	config struct {
-		U20Path string `json:"u20_path"`
-		Sudo    bool   `json:"use_sudo"`
-	}
-
-	u20path string
-
-	energyValLast  float64
-	energyTimeLast time.Time
-}
-
-func (m *MegwareEurekaCollector) Init(config json.RawMessage) error {
-	// Check if already initialized
-	if m.init {
-		return nil
-	}
-
-	m.name = "MegwareEurekaCollector"
-	if err := m.setup(); err != nil {
-		return fmt.Errorf("%s Init(): setup() call failed: %w", m.name, err)
-	}
-	m.meta = map[string]string{
-		"source": m.name,
-		"group":  "U20",
-	}
-
-	m.config.U20Path = "u20"
-
-	if len(config) > 0 {
-		d := json.NewDecoder(bytes.NewReader(config))
-		d.DisallowUnknownFields()
-		if err := d.Decode(&m.config); err != nil {
-			return fmt.Errorf("%s Init(): Error decoding JSON config: %w", m.name, err)
-		}
-	}
-
-	m.u20path = m.config.U20Path
-
-	data, err := m.readMpsData()
-	if err != nil {
-		return fmt.Errorf("energy reading test failed: %w", err)
-	}
-
-	m.energyValLast = data.Energy
-	m.energyTimeLast = time.Now()
-	m.init = true
-
-	return nil
-}
-
-func (m *MegwareEurekaCollector) readMpsData() (*mpsData, error) {
-	argv := make([]string, 0)
-	if m.config.Sudo {
-		argv = append(argv, "sudo", "-n")
-	}
-
-	argv = append(argv, m.u20path, "values", "--read", "GET_MPS_POLL_VALUES")
-
-	cmd := exec.Command(argv[0], argv[1:]...)
-
-	var stdout bytes.Buffer
-	var stderr bytes.Buffer
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
-
-	err := cmd.Run()
-	if err != nil {
-		return nil, fmt.Errorf("failed to run u20: %w (stdout=%s stderr=%s)", err, stdout.String(), stderr.String())
-	}
-
-	var u20output struct {
-		GetMpsPollValues mpsData `json:"GET_MPS_POLL_VALUES"`
-	}
-
-	err = json.Unmarshal(stdout.Bytes(), &u20output)
-	if err != nil {
-		return nil, fmt.Errorf("unable to decode u20 JSON output: %w (stdout=%s)", err, stdout.String())
-	}
-
-	u20output.GetMpsPollValues.Timestamp = time.Now()
-
-	return &u20output.GetMpsPollValues, nil
-}
-
-func (m *MegwareEurekaCollector) Read(interval time.Duration, output chan lp.CCMessage) {
-	// Check if already initialized
-	if !m.init {
-		return
-	}
-
-	data, err := m.readMpsData()
-	if err != nil {
-		cclog.ComponentErrorf(m.name, "readMpsData failed: %v", err)
-		return
-	}
-
-	powerVal := 0.0
-
-	if data.Timestamp.After(m.energyTimeLast) {
-		// Important, m.energy comes in Wh, so multiply by 3600 to get Ws (aka Joule)
-		energyValDiff := data.Energy - m.energyValLast
-		energyTimeDiff := data.Timestamp.Sub(m.energyTimeLast)
-		powerVal = energyValDiff * 3600 / energyTimeDiff.Seconds()
-
-		m.energyValLast = data.Energy
-		m.energyTimeLast = data.Timestamp
-	}
-
-	metricNamePrefix := "eureka_"
-	metricMap := map[string]struct {
-		value any
-		unit  string
-	}{
-		"power":        {value: powerVal, unit: "Watts"},
-		"vin":          {value: data.Vin, unit: "Volts"},
-		"iin":          {value: data.Iin, unit: "Amperes"},
-		"pin":          {value: data.Pin, unit: "Watts"},
-		"pin_avg":      {value: data.PinAvg, unit: "Watts"},
-		"vout":         {value: data.Vout, unit: "Volts"},
-		"iout":         {value: data.Iout, unit: "Amperes"},
-		"pout":         {value: data.Pout, unit: "Watts"},
-		"standby_vout": {value: data.StandbyVout, unit: "Volts"},
-		"standby_iout": {value: data.StandbyIout, unit: "Amperes"},
-		"standby_pout": {value: data.StandbyPout, unit: "Watts"},
-		"temp_busbar":  {value: data.TempBusbar, unit: "degC"},
-		"temp_ssd":     {value: data.TempSsd, unit: "degC"},
-		"temp_mps":     {value: data.TempMps, unit: "degC"},
-	}
-
-	for metricName, metricData := range metricMap {
-		metricName = metricNamePrefix + metricName
-		metric, err := lp.NewMetric(metricName, map[string]string{"type": "node"}, m.meta, metricData.value, data.Timestamp)
-		if err != nil {
-			cclog.ComponentErrorf(m.name, "lp.NewMetric failed: %v", err)
-			return
-		}
-		metric.AddMeta("unit", metricData.unit)
-		output <- metric
-	}
-}
-
-func (m *MegwareEurekaCollector) Close() {
-	m.init = false
-}

collectors/megwareEureka.md

@@ -1,44 +0,0 @@
-<!--
----
-title: Megware Eureka collector
-description: Collect power and other metrics of Megware Eureka machines using u20
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/lenovoDensePower.md
----
--->
-
-## `lenovo_dense_power` collector
-
-```json
-  "megware_eureka": {
-    "u20_path": "/path/to/ipmitool",
-    "use_sudo": true
-  }
-```
-
-The `megware_eureka` collector reads power and other metrics from Megware Eureka machines via `u20`.
-If the u20 tool is available for your machines, it's possible that this collector is compatible.
-If you don't have access to u20, please contact your Megware sales person.
-
-You can test the collector compatibility by running the following command:
-
-```
-$ u20 values --read GET_MPS_POLL_VALUES
-```
-
-If this returns a JSON with energy, voltage and current readings, you're fine.
-
-In addition, `u20` typically requires root to run.
-In order to run `cc-metric-collector` without root priviliges, you can enable `use_sudo`.
-Add a file like this in `/etc/sudoers.d/` to allow `cc-metric-collector` to run the required commands:
-
-```
-# Do not log the following sudo commands from monitoring, since this causes a lot of log spam.
-# However keep log_denied enabled, to detect failures
-Defaults: monitoring !log_allowed, !pam_session
-
-# Allow to use u20 for Megware power readings
-monitoring ALL = (root) NOPASSWD:/usr/bin/u20 values --read GET_MPS_POLL_VALUES
-```

collectors/raplMetric.go

@@ -22,15 +22,24 @@ import (
 )
 
 // running average power limit (RAPL) monitoring attributes for a zone
+// Only for Intel systems
+
 type RAPLZoneInfo struct {
+	energy            int64     // current reading of the energy counter in micro joules
+	maxEnergyRange    int64     // Range of the above energy counter in micro-joules
+	energyTimestamp   time.Time // timestamp when energy counter was read
+	energyFilepath    string    // path to a file containing the zones current energy counter in micro joules
+	shortTermFilepath string    // path to short term power limit
+	longTermFilepath  string    // path to long term power limit
+	enabledFilepath   string    // path to check whether limits are enabled
+	name              string
+
 	// tags describing the RAPL zone:
 	// * zone_name, subzone_name: e.g. psys, dram, core, uncore, package-0
 	// * zone_id: e.g. 0:1 (zone 0 sub zone 1)
-	tags            map[string]string
-	energyFilepath  string    // path to a file containing the zones current energy counter in micro joules
-	energy          int64     // current reading of the energy counter in micro joules
-	energyTimestamp time.Time // timestamp when energy counter was read
-	maxEnergyRange  int64     // Range of the above energy counter in micro-joules
+	// type=socket for dram, core, uncore, package-* and type=node for psys
+	// type-id=socket id
+	tags map[string]string
 }
 
 type RAPLCollector struct {
@@ -42,12 +51,40 @@ type RAPLCollector struct {
 		// * 0:1 for zone 0 subzone 1
 		ExcludeByID []string `json:"exclude_device_by_id,omitempty"`
 		// Exclude names for RAPL zones, e.g. psys, dram, core, uncore, package-0
-		ExcludeByName []string `json:"exclude_device_by_name,omitempty"`
+		ExcludeByName     []string `json:"exclude_device_by_name,omitempty"`
+		SkipEnergyReading bool     `json:"skip_energy_reading,omitempty"`
+		SkipLimitsReading bool     `json:"skip_limits_reading,omitempty"`
+		OnlyEnabledLimits bool     `json:"only_enabled_limits,omitempty"`
 	}
-	RAPLZoneInfo []RAPLZoneInfo
+	raplZoneInfo []RAPLZoneInfo
 	meta         map[string]string // default meta information
 }
 
+// Get the path to the power limit file for zone selectable by limit name
+// Common limit names for Intel systems are
+// - long_term
+// - short_term
+// Does not support AMD as AMD systems do not provide the power limits
+// through sysfs
+func ZoneLimitFile(folder string, limit_name string) string {
+	nameGlob := filepath.Join(folder, "constraint_*_name")
+	candidates, err := filepath.Glob(nameGlob)
+	if err == nil {
+		for _, c := range candidates {
+			if v, err := os.ReadFile(c); err == nil {
+				if strings.TrimSpace(string(v)) == limit_name {
+					var i int
+					n, err := fmt.Sscanf(filepath.Base(c), "constraint_%d_name", &i)
+					if err == nil && n == 1 {
+						return filepath.Join(folder, fmt.Sprintf("constraint_%d_power_limit_uw", i))
+					}
+				}
+			}
+		}
+	}
+	return ""
+}
+
 // Init initializes the running average power limit (RAPL) collector
 func (m *RAPLCollector) Init(config json.RawMessage) error {
 	// Check if already initialized
@@ -67,6 +104,9 @@ func (m *RAPLCollector) Init(config json.RawMessage) error {
 	}
 
 	// Read in the JSON configuration
+	m.config.SkipEnergyReading = false
+	m.config.SkipLimitsReading = false
+	m.config.OnlyEnabledLimits = true
 	if len(config) > 0 {
 		d := json.NewDecoder(bytes.NewReader(config))
 		d.DisallowUnknownFields()
@@ -93,49 +133,63 @@ func (m *RAPLCollector) Init(config json.RawMessage) error {
 	// See: https://www.kernel.org/doc/html/latest/power/powercap/powercap.html#monitoring-attributes
 	readZoneInfo := func(zonePath string) (
 		z struct {
-			name            string    // zones name e.g. psys, dram, core, uncore, package-0
-			energyFilepath  string    // path to a file containing the zones current energy counter in micro joules
-			energy          int64     // current reading of the energy counter in micro joules
-			energyTimestamp time.Time // timestamp when energy counter was read
-			maxEnergyRange  int64     // Range of the above energy counter in micro-joules
-			ok              bool      // Are all information available?
+			name              string    // zones name e.g. psys, dram, core, uncore, package-0
+			energyFilepath    string    // path to a file containing the zones current energy counter in micro joules
+			energy            int64     // current reading of the energy counter in micro joules
+			energyTimestamp   time.Time // timestamp when energy counter was read
+			maxEnergyRange    int64     // Range of the above energy counter in micro-joules
+			shortTermFilepath string
+			longTermFilepath  string
+			enabledFilepath   string
 		},
 	) {
 		// zones name e.g. psys, dram, core, uncore, package-0
-		foundName := false
-		if v, err := os.ReadFile(
-			filepath.Join(zonePath, "name")); err == nil {
-			foundName = true
+
+		if v, err :=
+			os.ReadFile(
+				filepath.Join(zonePath, "name")); err == nil {
 			z.name = strings.TrimSpace(string(v))
 		}
 
-		// path to a file containing the zones current energy counter in micro joules
-		z.energyFilepath = filepath.Join(zonePath, "energy_uj")
-
-		// current reading of the energy counter in micro joules
-		foundEnergy := false
-		if v, err := os.ReadFile(z.energyFilepath); err == nil {
-			// timestamp when energy counter was read
-			z.energyTimestamp = time.Now()
-			if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
-				foundEnergy = true
-				z.energy = i
+		if !m.config.SkipEnergyReading {
+			// path to a file containing the zones current energy counter in micro joules
+			z.energyFilepath = filepath.Join(zonePath, "energy_uj")
+			// current reading of the energy counter in micro joules
+			if v, err := os.ReadFile(z.energyFilepath); err == nil {
+				if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
+					z.energy = i
+					// timestamp when energy counter was read
+					z.energyTimestamp = time.Now()
+				}
+			} else {
+				cclog.ComponentErrorf(m.name, "Cannot read energy file for %s: %v", z.name, err.Error())
 			}
+			// Range of the above energy counter in micro-joules
+			if v, err :=
+				os.ReadFile(
+					filepath.Join(zonePath, "max_energy_range_uj")); err == nil {
+				if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
+					z.maxEnergyRange = i
+				}
+			}
+		} else {
+			cclog.ComponentDebugf(m.name, "Energy readings for %s disabled", zonePath)
 		}
 
-		// Range of the above energy counter in micro-joules
-		foundMaxEnergyRange := false
-		if v, err := os.ReadFile(
-			filepath.Join(zonePath, "max_energy_range_uj")); err == nil {
-			if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
-				foundMaxEnergyRange = true
-				z.maxEnergyRange = i
+		if !m.config.SkipLimitsReading {
+			z.shortTermFilepath = ZoneLimitFile(zonePath, "short_term")
+			if _, err := os.Stat(z.shortTermFilepath); err != nil {
+				z.shortTermFilepath = ""
 			}
+			z.longTermFilepath = ZoneLimitFile(zonePath, "long_term")
+			if _, err := os.Stat(z.longTermFilepath); err != nil {
+				z.longTermFilepath = ""
+			}
+			z.enabledFilepath = filepath.Join(zonePath, "enabled")
+		} else {
+			cclog.ComponentDebugf(m.name, "Power limit readings for %s disabled", zonePath)
 		}
 
-		// Are all information available?
-		z.ok = foundName && foundEnergy && foundMaxEnergyRange
-
 		return
 	}
 
@@ -152,14 +206,36 @@ func (m *RAPLCollector) Init(config json.RawMessage) error {
 
 	for _, zonePath := range zonesPath {
 		zoneID := strings.TrimPrefix(zonePath, zonePrefix)
+
 		z := readZoneInfo(zonePath)
-		if z.ok &&
-			!isIDExcluded[zoneID] &&
+		if !isIDExcluded[zoneID] &&
 			!isNameExcluded[z.name] {
 
+			si := RAPLZoneInfo{
+				tags:              make(map[string]string),
+				energyFilepath:    z.energyFilepath,
+				energy:            z.energy,
+				energyTimestamp:   z.energyTimestamp,
+				maxEnergyRange:    z.maxEnergyRange,
+				shortTermFilepath: z.shortTermFilepath,
+				longTermFilepath:  z.longTermFilepath,
+				enabledFilepath:   z.enabledFilepath,
+				name:              z.name,
+			}
+			si.tags["type"] = "node"
+			si.tags["type-id"] = "0"
+			var pid int = 0
+			if strings.HasPrefix(z.name, "package-") {
+				n, err := fmt.Sscanf(z.name, "package-%d", &pid)
+				if err == nil && n == 1 {
+					si.tags["type-id"] = fmt.Sprintf("%d", pid)
+					si.tags["type"] = "socket"
+				}
+				si.name = "pkg"
+			}
 			// Add RAPL monitoring attributes for a zone
-			m.RAPLZoneInfo = append(
-				m.RAPLZoneInfo,
+			m.raplZoneInfo = append(
+				m.raplZoneInfo,
 				RAPLZoneInfo{
 					tags: map[string]string{
 						"id":        zoneID,
@@ -182,37 +258,40 @@ func (m *RAPLCollector) Init(config json.RawMessage) error {
 		for _, subZonePath := range subZonesPath {
 			subZoneID := strings.TrimPrefix(subZonePath, subZonePrefix)
 			sz := readZoneInfo(subZonePath)
+
 			if len(zoneID) > 0 && len(z.name) > 0 &&
-				sz.ok &&
 				!isIDExcluded[zoneID+":"+subZoneID] &&
 				!isNameExcluded[sz.name] {
-				m.RAPLZoneInfo = append(
-					m.RAPLZoneInfo,
+				m.raplZoneInfo = append(
+					m.raplZoneInfo,
 					RAPLZoneInfo{
 						tags: map[string]string{
 							"id":            zoneID + ":" + subZoneID,
 							"zone_name":     z.name,
 							"sub_zone_name": sz.name,
 						},
-						energyFilepath:  sz.energyFilepath,
-						energy:          sz.energy,
-						energyTimestamp: sz.energyTimestamp,
-						maxEnergyRange:  sz.maxEnergyRange,
+						energyFilepath:    sz.energyFilepath,
+						energy:            sz.energy,
+						energyTimestamp:   sz.energyTimestamp,
+						maxEnergyRange:    sz.maxEnergyRange,
+						shortTermFilepath: sz.shortTermFilepath,
+						longTermFilepath:  sz.longTermFilepath,
+						enabledFilepath:   sz.enabledFilepath,
+						name:              sz.name,
 					})
 			}
 		}
 	}
 
-	if m.RAPLZoneInfo == nil {
+	if m.raplZoneInfo == nil {
 		return fmt.Errorf("no running average power limit (RAPL) device found in %s", controlTypePath)
 	}
 
 	// Initialized
 	cclog.ComponentDebug(
 		m.name,
-		"initialized",
-		len(m.RAPLZoneInfo),
-		"zones with running average power limit (RAPL) monitoring attributes")
+		"initialized %d zones with running average power limit (RAPL) monitoring attributes",
+		len(m.raplZoneInfo))
 	m.init = true
 
 	return err
@@ -221,35 +300,89 @@ func (m *RAPLCollector) Init(config json.RawMessage) error {
 // Read reads running average power limit (RAPL) monitoring attributes for all initialized zones
 // See: https://www.kernel.org/doc/html/latest/power/powercap/powercap.html#monitoring-attributes
 func (m *RAPLCollector) Read(interval time.Duration, output chan lp.CCMessage) {
-	for i := range m.RAPLZoneInfo {
-		p := &m.RAPLZoneInfo[i]
+	for i := range m.raplZoneInfo {
+		p := &m.raplZoneInfo[i]
 
-		// Read current value of the energy counter in micro joules
-		if v, err := os.ReadFile(p.energyFilepath); err == nil {
-			energyTimestamp := time.Now()
-			if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
-				energy := i
+		if !m.config.SkipEnergyReading {
+			// Read current value of the energy counter in micro joules
+			if v, err := os.ReadFile(p.energyFilepath); err == nil {
+				energyTimestamp := time.Now()
+				if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
+					energy := i
 
-				// Compute average power (Δ energy / Δ time)
-				energyDiff := energy - p.energy
-				if energyDiff < 0 {
-					// Handle overflow:
-					// ( p.maxEnergyRange - p.energy ) + energy
-					// = p.maxEnergyRange + ( energy - p.energy )
-					// = p.maxEnergyRange + diffEnergy
-					energyDiff += p.maxEnergyRange
+					// Compute average power (Δ energy / Δ time)
+					energyDiff := energy - p.energy
+					if energyDiff < 0 {
+						// Handle overflow:
+						// ( p.maxEnergyRange - p.energy ) + energy
+						// = p.maxEnergyRange + ( energy - p.energy )
+						// = p.maxEnergyRange + diffEnergy
+						energyDiff += p.maxEnergyRange
+					}
+					timeDiff := energyTimestamp.Sub(p.energyTimestamp)
+					averagePower := float64(energyDiff) / float64(timeDiff.Microseconds())
+
+					y, err := lp.NewMetric(
+						fmt.Sprintf("rapl_%s_average_power", p.name),
+						p.tags,
+						m.meta,
+						averagePower,
+						energyTimestamp)
+					if err == nil {
+						output <- y
+					}
+
+					e, err := lp.NewMetric(
+						fmt.Sprintf("rapl_%s_energy", p.name),
+						p.tags,
+						m.meta,
+						float64(energyDiff)*1e-3,
+						energyTimestamp)
+					if err == nil {
+						e.AddMeta("unit", "Joules")
+						output <- e
+					}
+
+					// Save current energy counter state
+					p.energy = energy
+					p.energyTimestamp = energyTimestamp
 				}
-				timeDiff := energyTimestamp.Sub(p.energyTimestamp)
-				averagePower := float64(energyDiff) / float64(timeDiff.Microseconds())
-
-				y, err := lp.NewMetric("rapl_average_power", p.tags, m.meta, averagePower, energyTimestamp)
-				if err == nil {
-					output <- y
+			}
+		}
+		// https://www.kernel.org/doc/html/latest/power/powercap/powercap.html#constraints
+		if !m.config.SkipLimitsReading {
+			skip := false
+			if m.config.OnlyEnabledLimits {
+				if v, err := os.ReadFile(p.enabledFilepath); err == nil {
+					if strings.TrimSpace(string(v)) == "0" {
+						skip = true
+					}
+				}
+			}
+			if !skip {
+				if len(p.shortTermFilepath) > 0 {
+					if v, err := os.ReadFile(p.shortTermFilepath); err == nil {
+						if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
+							name := fmt.Sprintf("rapl_%s_limit_short_term", p.name)
+							y, err := lp.NewMetric(name, p.tags, m.meta, i/1e6, time.Now())
+							if err == nil {
+								output <- y
+							}
+						}
+					}
 				}
 
-				// Save current energy counter state
-				p.energy = energy
-				p.energyTimestamp = energyTimestamp
+				if len(p.longTermFilepath) > 0 {
+					if v, err := os.ReadFile(p.longTermFilepath); err == nil {
+						if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
+							name := fmt.Sprintf("rapl_%s_limit_long_term", p.name)
+							y, err := lp.NewMetric(name, p.tags, m.meta, i/1e6, time.Now())
+							if err == nil {
+								output <- y
+							}
+						}
+					}
+				}
 			}
 		}
 	}

collectors/raplMetric.md

@@ -11,16 +11,24 @@ hugo_path: docs/reference/cc-metric-collector/collectors/rapl.md
 
 ## `rapl` collector
 
-This collector reads running average power limit (RAPL) monitoring attributes to compute average power consumption metrics. See <https://www.kernel.org/doc/html/latest/power/powercap/powercap.html#monitoring-attributes>.
-
-The Likwid metric collector provides similar functionality.
+This collector reads running average power limit (RAPL) monitoring attributes to compute average power consumption metrics. See <https://www.kernel.org/doc/html/latest/power/powercap/powercap.html>.
 
 ```json
   "rapl": {
     "exclude_device_by_id": ["0:1", "0:2"],
-    "exclude_device_by_name": ["psys"]
+    "exclude_device_by_name": ["psys"],
+    "skip_energy_reading": false,
+    "skip_limits_reading": false,
+    "only_enabled_limits": true
   }
 ```
 
 Metrics:
-* `rapl_average_power`: average power consumption in Watt. The average is computed over the entire runtime from the last measurement to the current measurement
+* `rapl_<domain>_average_power`: average power consumption in Watt. The average is computed over the entire runtime from the last measurement to the current measurement
+* `rapl_<domain>_energy`: Difference from the last measurement
+* `rapl_<domain>_limit_short_term`: Short term powercap setting for the domain
+* `rapl_<domain>_limit_long_term`: Long term powercap setting for the domain
+
+Only the `rapl_<domain>_average_power` and `rapl_<domain>_energy` metrics require root-permissions. The limits can be read as user. Some domains have limits available but they are not enabled. By default, only enabled domain limits are collected.
+
+Energy and power measurments can also be done with the Likwid metric collector.

collectors/slurmCgroupMetric.go

@@ -7,7 +7,6 @@ import (
 	"os/exec"
 	"os/user"
 	"path/filepath"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -46,37 +45,6 @@ type SlurmCgroupCollector struct {
 
 const defaultCgroupBase = "/sys/fs/cgroup/system.slice/slurmstepd.scope"
 
-var (
-	// Slurm cgroup v2 directory layout:
-	// - Slurm <= 25.11: job_<numeric job id>
-	// - Slurm >= 26.05: SLUID, encoded as "s" + 13 Crockford Base32 characters
-	jobIDDirRE = regexp.MustCompile(`^job_[0-9]+$`)
-	sluidDirRE = regexp.MustCompile(`(?i)^s[0-9A-HJKMNP-TV-Z]{13}$`)
-)
-
-func (m *SlurmCgroupCollector) findSlurmJobDirs() ([]string, error) {
-	entries, err := os.ReadDir(m.cgroupBase)
-	if err != nil {
-		return nil, err
-	}
-
-	jobDirs := make([]string, 0)
-
-	for _, entry := range entries {
-		if !entry.IsDir() {
-			continue
-		}
-
-		name := entry.Name()
-
-		if jobIDDirRE.MatchString(name) || sluidDirRE.MatchString(name) {
-			jobDirs = append(jobDirs, filepath.Join(m.cgroupBase, name))
-		}
-	}
-
-	return jobDirs, nil
-}
-
 func ParseCPUs(cpuset string) ([]int, error) {
 	var result []int
 	if cpuset == "" {
@@ -269,9 +237,10 @@ func (m *SlurmCgroupCollector) Read(interval time.Duration, output chan lp.CCMes
 		delete(m.cpuUsed, k)
 	}
 
-	jobDirs, err := m.findSlurmJobDirs()
+	globPattern := filepath.Join(m.cgroupBase, "job_*")
+	jobDirs, err := filepath.Glob(globPattern)
 	if err != nil {
-		cclog.ComponentError(m.name, "Error reading job directories:", err.Error())
+		cclog.ComponentError(m.name, "Error globbing job directories:", err.Error())
 		return
 	}
 

docs/configuration.md

@@ -19,8 +19,6 @@ The global file contains the paths to the other four files and some global optio
 
 Be aware that the paths are relative to the execution folder of the cc-metric-collector binary, so it is recommended to use absolute paths.
 
-The optional `channel_buffer_size` option sets the capacity of the internal channels between the components (collectors → router → sinks). If unset, it defaults to `max(200, 24 * number of CPUs)` so that one interval's burst of per-hwthread metrics fits without back-pressuring the collectors on nodes with many cores.
-
 ## Component configuration
 
 The others are mainly list of of subcomponents: the collectors, the receivers, the router and the sinks. Their role is best shown in a picture:

go.mod

@@ -5,11 +5,11 @@ go 1.25.0
 require (
 	github.com/ClusterCockpit/cc-lib/v2 v2.12.0
 	github.com/ClusterCockpit/go-rocm-smi v0.4.0
-	github.com/NVIDIA/go-nvml v0.13.2-0
+	github.com/NVIDIA/go-nvml v0.13.0-1
 	github.com/PaesslerAG/gval v1.2.4
 	github.com/fsnotify/fsnotify v1.10.1
 	github.com/tklauser/go-sysconf v0.4.0
-	golang.design/x/runtime v0.3.0
+	golang.design/x/thread v0.3.2
 	golang.org/x/sys v0.45.0
 )
 

go.sum

@@ -12,9 +12,8 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
 github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
+github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw=
 github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4=
-github.com/NVIDIA/go-nvml v0.13.2-0 h1:7M4cFG62wSUHw8i0XSiNU7ejKODytTS6ZrW/vgB2NSI=
-github.com/NVIDIA/go-nvml v0.13.2-0/go.mod h1:ahi2psRYoa+wYUBIrZPRO+wJs9lcvMhxSSkjjvsJJNQ=
 github.com/PaesslerAG/gval v1.2.4 h1:rhX7MpjJlcxYwL2eTTYIOBUyEKZ+A96T9vQySWkVUiU=
 github.com/PaesslerAG/gval v1.2.4/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
 github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi0jPI=
@@ -174,8 +173,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
 go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
-golang.design/x/runtime v0.3.0 h1:8bA+GQfO8A18JXJQlQA56pEc+Sgrbo4cmLzjzmJ58ZI=
-golang.design/x/runtime v0.3.0/go.mod h1:PjVKQjImLdSrvjIw4FmrnCPq9BGs5PMBHb8i6b2BN9I=
+golang.design/x/thread v0.3.2 h1:FmD1glspGrQCe6FuQLmSrT6wz2CSzq7vKVDluyiMnqo=
+golang.design/x/thread v0.3.2/go.mod h1:6+Hi2rMOgMHZdKDWaqNHyWtoFUx1HxZ06LfHPh5Z/hQ=
 golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
 golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
 golang.org/x/exp v0.0.0-20231005195138-3e424a577f31 h1:9k5exFQKQglLo+RoP+4zMjOFE14P6+vyR0baDAi0Rcs=

internal/metricAggregator/metricAggregator.go

@@ -128,7 +128,7 @@ func (c *metricAggregator) Eval(starttime time.Time, endtime time.Time, metrics
 	vars["starttime"] = starttime
 	vars["endtime"] = endtime
 	for _, f := range c.functions {
-		cclog.ComponentDebugf("MetricCache", "COLLECT %s COND '%s'", f.Name, f.Condition)
+		cclog.ComponentDebug("MetricCache", "COLLECT", f.Name, "COND", f.Condition)
 		var valuesFloat64 []float64
 		var valuesFloat32 []float32
 		var valuesInt []int
@@ -140,7 +140,7 @@ func (c *metricAggregator) Eval(starttime time.Time, endtime time.Time, metrics
 			vars["metric"] = m
 			value, err := f.gvalCond.EvalBool(context.Background(), vars)
 			if err != nil {
-				cclog.ComponentErrorf("MetricCache", "COLLECT %s COND '%s' : %s", f.Name, f.Condition, err.Error())
+				cclog.ComponentError("MetricCache", "COLLECT", f.Name, "COND", f.Condition, ":", err.Error())
 				continue
 			}
 			if value {
@@ -160,7 +160,7 @@ func (c *metricAggregator) Eval(starttime time.Time, endtime time.Time, metrics
 					case bool:
 						valuesBool = append(valuesBool, x)
 					default:
-						cclog.ComponentErrorf("MetricCache", "COLLECT ADD VALUE %v FAILED", v)
+						cclog.ComponentError("MetricCache", "COLLECT ADD VALUE", v, "FAILED")
 					}
 				}
 				matches = append(matches, m)
@@ -213,13 +213,13 @@ func (c *metricAggregator) Eval(starttime time.Time, endtime time.Time, metrics
 			vars["values"] = valuesBool
 			len_values = len(valuesBool)
 		}
-		cclog.ComponentDebugf("MetricCache", "EVALUATE %s METRICS %d CALC '%s'", f.Name, len_values, f.Function)
+		cclog.ComponentDebug("MetricCache", "EVALUATE", f.Name, "METRICS", len_values, "CALC", f.Function)
 
 		vars["metrics"] = matches
 		if len_values > 0 {
 			value, err := gval.Evaluate(f.Function, vars, c.language)
 			if err != nil {
-				cclog.ComponentErrorf("MetricCache", "EVALUATE %s METRICS %d CALC '%s': %s", f.Name, len_values, f.Function, err.Error())
+				cclog.ComponentError("MetricCache", "EVALUATE", f.Name, "METRICS", len_values, "CALC", f.Function, ":", err.Error())
 				break
 			}
 
@@ -273,12 +273,12 @@ func (c *metricAggregator) Eval(starttime time.Time, endtime time.Time, metrics
 			case string:
 				m, err = lp.NewMessage(f.Name, tags, meta, map[string]any{"value": t}, starttime)
 			default:
-				cclog.ComponentErrorf("MetricCache", "Gval returned invalid type %s skipping metric %s", t, f.Name)
+				cclog.ComponentError("MetricCache", "Gval returned invalid type", t, "skipping metric", f.Name)
 			}
 			if err != nil {
-				cclog.ComponentErrorf("MetricCache", "Cannot create metric from Gval result %v: %s", value, err.Error())
+				cclog.ComponentError("MetricCache", "Cannot create metric from Gval result", value, ":", err.Error())
 			}
-			cclog.ComponentDebugf("MetricCache", "SEND %s", m.ToLineProtocol(nil))
+			cclog.ComponentDebug("MetricCache", "SEND", m)
 			select {
 			case c.output <- m:
 			default:
@@ -295,12 +295,12 @@ func (c *metricAggregator) AddAggregation(name, function, condition string, tags
 	newcond := strings.ReplaceAll(condition, "'", "\"")
 	gvalCond, err := gval.Full(metricCacheLanguage).NewEvaluable(newcond)
 	if err != nil {
-		cclog.ComponentErrorf("MetricAggregator", "Cannot add aggregation, invalid if condition '%s': %s", newcond, err.Error())
+		cclog.ComponentError("MetricAggregator", "Cannot add aggregation, invalid if condition", newcond, ":", err.Error())
 		return err
 	}
 	gvalFunc, err := gval.Full(metricCacheLanguage).NewEvaluable(newfunc)
 	if err != nil {
-		cclog.ComponentErrorf("MetricAggregator", "Cannot add aggregation, invalid function condition %s: %s", newfunc, err.Error())
+		cclog.ComponentError("MetricAggregator", "Cannot add aggregation, invalid function condition", newfunc, ":", err.Error())
 		return err
 	}
 	for _, agg := range c.functions {

internal/metricRouter/metricCache.go

@@ -79,7 +79,7 @@ func (c *metricCache) Init(output chan lp.CCMessage, ticker mct.MultiChanTicker,
 
 // Start starts the metric cache
 func (c *metricCache) Start() {
-	c.tickchan = make(chan time.Time, 1)
+	c.tickchan = make(chan time.Time)
 	c.ticker.AddChannel(c.tickchan)
 	// Router cache is done
 	done := func() {

internal/metricRouter/metricRouter.go

@@ -228,25 +228,11 @@ func (r *metricRouter) DoAddTags(point lp.CCMessage) {
 func (r *metricRouter) Start() {
 	// start timer if configured
 	r.timestamp = time.Now()
-	timeChan := make(chan time.Time, 1)
+	timeChan := make(chan time.Time)
 	if r.config.IntervalStamp {
 		r.ticker.AddChannel(timeChan)
 	}
 
-	// Drain a pending tick before stamping new metrics, so a new interval's
-	// metrics never carry the previous interval's timestamp
-	updateTimestamp := func() {
-		if !r.config.IntervalStamp {
-			return
-		}
-		select {
-		case timestamp := <-timeChan:
-			r.timestamp = timestamp
-			cclog.ComponentDebug("MetricRouter", "Update timestamp", r.timestamp.UnixNano())
-		default:
-		}
-	}
-
 	// Router manager is done
 	done := func() {
 		close(r.done)
@@ -312,19 +298,14 @@ func (r *metricRouter) Start() {
 			case timestamp := <-timeChan:
 				r.timestamp = timestamp
 				cclog.ComponentDebug("MetricRouter", "Update timestamp", r.timestamp.UnixNano())
-				if len(r.coll_input) == cap(r.coll_input) {
-					cclog.ComponentWarn("MetricRouter", "collector input channel full at tick, sinks may be too slow")
-				}
 
 			case p := <-r.coll_input:
-				updateTimestamp()
 				coll_forward(p)
 				for i := 0; len(r.coll_input) > 0 && i < (r.maxForward-1); i++ {
 					coll_forward(<-r.coll_input)
 				}
 
 			case p := <-r.recv_input:
-				updateTimestamp()
 				recv_forward(p)
 				for i := 0; len(r.recv_input) > 0 && i < (r.maxForward-1); i++ {
 					recv_forward(<-r.recv_input)

internal/metricRouter/metricRouter_test.go

@@ -1,99 +0,0 @@
-package metricRouter
-
-import (
-	"encoding/json"
-	"fmt"
-	"sync"
-	"testing"
-	"time"
-
-	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
-)
-
-// Fake ticker that delivers ticks on demand
-type fakeTicker struct {
-	channels []chan time.Time
-}
-
-func (t *fakeTicker) Init(duration time.Duration) {}
-
-func (t *fakeTicker) AddChannel(c chan time.Time) {
-	t.channels = append(t.channels, c)
-}
-
-func (t *fakeTicker) Close() {}
-
-func (t *fakeTicker) tick(ts time.Time) {
-	for _, c := range t.channels {
-		c <- ts
-	}
-}
-
-func genMessages(t *testing.T, num int) []lp.CCMessage {
-	t.Helper()
-	msgs := make([]lp.CCMessage, 0, num)
-	tags := map[string]string{"type": "node"}
-	for i := range num {
-		m, err := lp.NewMetric(fmt.Sprintf("testmetric%d", i), tags, nil, 42.0, time.Unix(1, 0))
-		if err != nil {
-			t.Fatalf("failed to create message: %s", err.Error())
-		}
-		msgs = append(msgs, m)
-	}
-	return msgs
-}
-
-// With interval_timestamp enabled, all metrics forwarded after a tick must
-// carry that tick's timestamp, never the previous interval's
-func TestIntervalTimestamp(t *testing.T) {
-	ticker := &fakeTicker{}
-	var wg sync.WaitGroup
-	r, err := New(ticker, &wg, json.RawMessage(`{"interval_timestamp": true}`))
-	if err != nil {
-		t.Fatalf("failed to setup metric router: %s", err.Error())
-	}
-
-	coll := make(chan lp.CCMessage, 100)
-	out := make(chan lp.CCMessage, 100)
-	r.AddCollectorInput(coll)
-	r.AddOutput(out)
-	r.Start()
-
-	receiveAll := func(num int) []lp.CCMessage {
-		received := make([]lp.CCMessage, 0, num)
-		for len(received) < num {
-			select {
-			case m := <-out:
-				received = append(received, m)
-			case <-time.After(5 * time.Second):
-				t.Fatalf("received only %d of %d messages", len(received), num)
-			}
-		}
-		return received
-	}
-
-	for interval, tickTime := range []time.Time{time.Unix(1000, 0), time.Unix(1010, 0)} {
-		ticker.tick(tickTime)
-		msgs := genMessages(t, 20)
-		for _, m := range msgs {
-			coll <- m
-		}
-		for i, m := range receiveAll(len(msgs)) {
-			if !m.Time().Equal(tickTime) {
-				t.Errorf("interval %d message %d: got timestamp %v, want %v", interval, i, m.Time(), tickTime)
-			}
-		}
-	}
-
-	closed := make(chan struct{})
-	go func() {
-		r.Close()
-		close(closed)
-	}()
-	select {
-	case <-closed:
-	case <-time.After(5 * time.Second):
-		t.Fatal("Close() did not terminate")
-	}
-	wg.Wait()
-}

pkg/multiChanTicker/README.md

@@ -30,8 +30,8 @@ Afterwards, you can add channels:
 
 ```golang
 t := MultiChanTicker(duration)
-c1 := make(chan time.Time, 1)
-c2 := make(chan time.Time, 1)
+c1 := make(chan time.Time)
+c2 := make(chan time.Time)
 t.AddChannel(c1)
 t.AddChannel(c2)
 
@@ -46,5 +46,3 @@ for {
 ```
 
 The result should be the same `time.Time` output in both channels, notified "simultaneously".
-
-Ticks are delivered with a non-blocking send: a consumer that has not yet read the previous tick does not stall the ticker (which would silently drop `time.Ticker` fires for all consumers); instead, the tick for that consumer is skipped and a warning is logged. Register buffered channels (capacity 1) so a consumer that is briefly busy at tick time does not lose the tick.

pkg/multiChanTicker/multiChanTicker.go

@@ -8,8 +8,6 @@
 package multiChanTicker
 
 import (
-	"fmt"
-	"sync"
 	"time"
 
 	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
@@ -17,7 +15,6 @@ import (
 
 type multiChanTicker struct {
 	ticker   *time.Ticker
-	mutex    sync.Mutex // protects channels, which is appended to while the tick goroutine iterates it
 	channels []chan time.Time
 	done     chan bool
 }
@@ -43,30 +40,21 @@ func (t *multiChanTicker) Init(duration time.Duration) {
 				return
 			case ts := <-t.ticker.C:
 				cclog.ComponentDebug("MultiChanTicker", "Tick", ts)
-				t.mutex.Lock()
-				for i, c := range t.channels {
-					// Non-blocking send: a consumer that has not yet read the
-					// previous tick must not stall the ticker, otherwise
-					// time.Ticker silently drops fires for ALL consumers
+				for _, c := range t.channels {
 					select {
+					case <-t.done:
+						done()
+						return
 					case c <- ts:
-					default:
-						cclog.ComponentWarn("MultiChanTicker", fmt.Sprintf("consumer %d did not read previous tick, dropping tick %v", i, ts))
 					}
 				}
-				t.mutex.Unlock()
 			}
 		}
 	}()
 }
 
 func (t *multiChanTicker) AddChannel(channel chan time.Time) {
-	if cap(channel) == 0 {
-		cclog.ComponentWarn("MultiChanTicker", "unbuffered channel registered, ticks may be dropped if the consumer is not ready")
-	}
-	t.mutex.Lock()
 	t.channels = append(t.channels, channel)
-	t.mutex.Unlock()
 }
 
 func (t *multiChanTicker) Close() {

pkg/multiChanTicker/multiChanTicker_test.go

@@ -1,51 +0,0 @@
-package multiChanTicker
-
-import (
-	"testing"
-	"time"
-)
-
-// A consumer that never reads its channel must not stall the ticker
-// or starve the other consumers
-func TestStalledConsumerDoesNotStarveOthers(t *testing.T) {
-	stalled := make(chan time.Time, 1) // never read
-	fast := make(chan time.Time, 1)
-
-	ticker := NewTicker(10 * time.Millisecond)
-	defer ticker.Close()
-	ticker.AddChannel(stalled)
-	ticker.AddChannel(fast)
-
-	received := 0
-	deadline := time.After(5 * time.Second)
-	for received < 5 {
-		select {
-		case <-fast:
-			received++
-		case <-deadline:
-			t.Fatalf("received only %d ticks while another consumer stalled", received)
-		}
-	}
-}
-
-// Close() must return promptly even if no consumer reads its channel
-func TestCloseWithStalledConsumer(t *testing.T) {
-	stalled := make(chan time.Time, 1) // never read
-
-	ticker := NewTicker(10 * time.Millisecond)
-	ticker.AddChannel(stalled)
-
-	// Let some ticks fire and be dropped
-	time.Sleep(50 * time.Millisecond)
-
-	closed := make(chan struct{})
-	go func() {
-		ticker.Close()
-		close(closed)
-	}()
-	select {
-	case <-closed:
-	case <-time.After(5 * time.Second):
-		t.Fatal("Close() blocked with a stalled consumer")
-	}
-}