Fix metric timeseries gaps on nodes with many cores

On nodes with >300 cores, one interval's burst of per-hwthread metrics
overran the fixed 200-slot channels. With blocking sends at every hop,
sink back-pressure propagated to the collectors, the collection round
exceeded the interval, and time.Ticker silently dropped the missed
ticks - whole intervals were skipped without any log message.

- multiChanTicker: deliver ticks non-blockingly and warn when a
  consumer misses a tick instead of stalling all consumers; guard the
  channel list with a mutex (data race with AddChannel)
- collectorManager: run the collection round detached from the tick
  loop, skip-and-warn when a round is still running, log per-collector
  and per-round durations at debug level, close serial collectors on
  shutdown
- metricRouter: buffer the interval timestamp channel and drain it
  before stamping, so metrics never carry the previous interval's
  timestamp; warn when the collector input channel is full at tick time
- main: scale the inter-manager channels to max(200, 24*NumCPU),
  overridable with the new optional channel_buffer_size config option
- add first unit tests for ticker, collector manager and router

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

collectors/collectorManager.go

@@ -12,6 +12,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
@@ -56,16 +57,17 @@ var AvailableCollectors = map[string]MetricCollector{
 
 // Metric collector manager data structure
 type collectorManager struct {
-	collectors   []MetricCollector          // List of metric collectors to read in parallel
-	serial       []MetricCollector          // List of metric collectors to read serially
-	output       chan lp.CCMessage          // Output channels
-	done         chan bool                  // channel to finish / stop metric collector manager
-	ticker       mct.MultiChanTicker        // periodically ticking once each interval
-	duration     time.Duration              // duration (for metrics that measure over a given duration)
-	wg           *sync.WaitGroup            // wait group for all goroutines in cc-metric-collector
-	config       map[string]json.RawMessage // json encoded config for collector manager
-	collector_wg sync.WaitGroup             // internally used wait group for the parallel reading of collector
-	parallel_run bool                       // Flag whether the collectors are currently read in parallel
+	collectors    []MetricCollector          // List of metric collectors to read in parallel
+	serial        []MetricCollector          // List of metric collectors to read serially
+	output        chan lp.CCMessage          // Output channels
+	done          chan bool                  // channel to finish / stop metric collector manager
+	ticker        mct.MultiChanTicker        // periodically ticking once each interval
+	duration      time.Duration              // duration (for metrics that measure over a given duration)
+	wg            *sync.WaitGroup            // wait group for all goroutines in cc-metric-collector
+	config        map[string]json.RawMessage // json encoded config for collector manager
+	collector_wg  sync.WaitGroup             // internally used wait group for the parallel reading of collector
+	round_wg      sync.WaitGroup             // wait group for the currently running collection round
+	round_running atomic.Bool                // Flag whether a collection round is currently running
 }
 
 // Metric collector manager access functions
@@ -121,26 +123,54 @@ func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Durat
 	return nil
 }
 
+// runRound executes one collection round: first all parallel collectors
+// concurrently, then the serial collectors one by one
+func (cm *collectorManager) runRound(t time.Time) {
+	roundStart := time.Now()
+	for _, c := range cm.collectors {
+		// Read metrics from collector c via goroutine
+		cclog.ComponentDebug("CollectorManager", c.Name(), t)
+		cm.collector_wg.Add(1)
+		go func(myc MetricCollector) {
+			start := time.Now()
+			myc.Read(cm.duration, cm.output)
+			cclog.ComponentDebug("CollectorManager", myc.Name(), "took", time.Since(start))
+			cm.collector_wg.Done()
+		}(c)
+	}
+	cm.collector_wg.Wait()
+	for _, c := range cm.serial {
+		// Read metrics from collector c
+		cclog.ComponentDebug("CollectorManager", c.Name(), t)
+		start := time.Now()
+		c.Read(cm.duration, cm.output)
+		cclog.ComponentDebug("CollectorManager", c.Name(), "took", time.Since(start))
+	}
+	cclog.ComponentDebug("CollectorManager", "collection round took", time.Since(roundStart))
+}
+
 // Start starts the metric collector manager
 func (cm *collectorManager) Start() {
-	tick := make(chan time.Time)
+	tick := make(chan time.Time, 1)
 	cm.ticker.AddChannel(tick)
 
 	cm.wg.Go(func() {
 		// Collector manager is done
 		done := func() {
-			// close all metric collectors
-			if cm.parallel_run {
-				cm.collector_wg.Wait()
-				cm.parallel_run = false
-			}
+			// wait for a still running collection round, then close all metric collectors
+			cm.round_wg.Wait()
 			for _, c := range cm.collectors {
 				c.Close()
 			}
+			for _, c := range cm.serial {
+				c.Close()
+			}
 			close(cm.done)
 			cclog.ComponentDebug("CollectorManager", "DONE")
 		}
 
+		var roundStart time.Time
+
 		// Wait for done signal or timer event
 		for {
 			select {
@@ -148,37 +178,20 @@ func (cm *collectorManager) Start() {
 				done()
 				return
 			case t := <-tick:
-				cm.parallel_run = true
-				for _, c := range cm.collectors {
-					// Wait for done signal or execute the collector
-					select {
-					case <-cm.done:
-						done()
-						return
-					default:
-						// Read metrics from collector c via goroutine
-						cclog.ComponentDebug("CollectorManager", c.Name(), t)
-						cm.collector_wg.Add(1)
-						go func(myc MetricCollector) {
-							myc.Read(cm.duration, cm.output)
-							cm.collector_wg.Done()
-						}(c)
-					}
-				}
-				cm.collector_wg.Wait()
-				cm.parallel_run = false
-				for _, c := range cm.serial {
-					// Wait for done signal or execute the collector
-					select {
-					case <-cm.done:
-						done()
-						return
-					default:
-						// Read metrics from collector c
-						cclog.ComponentDebug("CollectorManager", c.Name(), t)
-						c.Read(cm.duration, cm.output)
-					}
+				// The round runs detached from this loop, so the tick channel
+				// stays drained even when a round takes longer than the interval
+				if cm.round_running.Load() {
+					cclog.ComponentWarn("CollectorManager", "collection round still running after", time.Since(roundStart), "- skipping tick")
+					continue
 				}
+				cm.round_running.Store(true)
+				roundStart = time.Now()
+				cm.round_wg.Add(1)
+				go func() {
+					defer cm.round_wg.Done()
+					defer cm.round_running.Store(false)
+					cm.runRound(t)
+				}()
 			}
 		}
 	})

collectors/collectorManager_test.go

@@ -0,0 +1,121 @@
+package collectors
+
+import (
+	"encoding/json"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
+)
+
+// Fake ticker that delivers ticks on demand
+type fakeTicker struct {
+	channels []chan time.Time
+}
+
+func (t *fakeTicker) Init(duration time.Duration) {}
+
+func (t *fakeTicker) AddChannel(c chan time.Time) {
+	t.channels = append(t.channels, c)
+}
+
+func (t *fakeTicker) Close() {}
+
+func (t *fakeTicker) tick() {
+	for _, c := range t.channels {
+		select {
+		case c <- time.Now():
+		default:
+		}
+	}
+}
+
+// Stub collector whose Read blocks until it is released
+type stubCollector struct {
+	metricCollector
+	readStarted chan struct{}
+	release     chan struct{}
+	reads       atomic.Int32
+}
+
+func (c *stubCollector) Init(config json.RawMessage) error {
+	c.name = "teststub"
+	c.parallel = true
+	c.init = true
+	return nil
+}
+
+func (c *stubCollector) Read(duration time.Duration, output chan lp.CCMessage) {
+	c.reads.Add(1)
+	c.readStarted <- struct{}{}
+	<-c.release
+}
+
+func (c *stubCollector) Close() {}
+
+func TestOverlongCollectionRoundSkipsTick(t *testing.T) {
+	stub := &stubCollector{
+		readStarted: make(chan struct{}, 10),
+		release:     make(chan struct{}),
+	}
+	AvailableCollectors["teststub"] = stub
+	defer delete(AvailableCollectors, "teststub")
+
+	ticker := &fakeTicker{}
+	var wg sync.WaitGroup
+	cm, err := New(ticker, time.Second, &wg, json.RawMessage(`{"teststub": {}}`))
+	if err != nil {
+		t.Fatalf("failed to setup collector manager: %s", err.Error())
+	}
+	cm.AddOutput(make(chan lp.CCMessage, 100))
+	cm.Start()
+
+	// First tick starts a collection round that blocks in Read
+	ticker.tick()
+	select {
+	case <-stub.readStarted:
+	case <-time.After(5 * time.Second):
+		t.Fatal("collection round did not start on tick")
+	}
+
+	// Further ticks while the round is still running must be skipped,
+	// not queued up or run concurrently
+	for range 3 {
+		ticker.tick()
+		time.Sleep(20 * time.Millisecond)
+	}
+	if got := stub.reads.Load(); got != 1 {
+		t.Fatalf("expected 1 concurrent collection round, got %d reads", got)
+	}
+
+	// Finish the round, the next tick must start a new one
+	stub.release <- struct{}{}
+	deadline := time.After(5 * time.Second)
+	for stub.reads.Load() < 2 {
+		ticker.tick()
+		select {
+		case <-stub.readStarted:
+		case <-time.After(20 * time.Millisecond):
+		case <-deadline:
+			t.Fatal("no new collection round started after the previous one finished")
+		}
+	}
+
+	// Shutdown must wait for the running round and terminate cleanly.
+	// Closing the release channel lets any still running or straggler
+	// round finish immediately
+	close(stub.release)
+	closed := make(chan struct{})
+	go func() {
+		cm.Close()
+		close(closed)
+	}()
+	select {
+	case <-closed:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Close() did not terminate")
+	}
+	wg.Wait()
+}