mirror of
https://github.com/ClusterCockpit/cc-metric-collector.git
synced 2026-07-11 04:00:38 +02:00
moebiusband
3d4c464166
On nodes with >300 cores, one interval's burst of per-hwthread metrics overran the fixed 200-slot channels. With blocking sends at every hop, sink back-pressure propagated to the collectors, the collection round exceeded the interval, and time.Ticker silently dropped the missed ticks - whole intervals were skipped without any log message. - multiChanTicker: deliver ticks non-blockingly and warn when a consumer misses a tick instead of stalling all consumers; guard the channel list with a mutex (data race with AddChannel) - collectorManager: run the collection round detached from the tick loop, skip-and-warn when a round is still running, log per-collector and per-round durations at debug level, close serial collectors on shutdown - metricRouter: buffer the interval timestamp channel and drain it before stamping, so metrics never carry the previous interval's timestamp; warn when the collector input channel is full at tick time - main: scale the inter-manager channels to max(200, 24*NumCPU), overridable with the new optional channel_buffer_size config option - add first unit tests for ticker, collector manager and router Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
122 lines
2.7 KiB
Go
122 lines
2.7 KiB
Go
package collectors
|
|
|
|
import (
|
|
"encoding/json"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
|
|
)
|
|
|
|
// Fake ticker that delivers ticks on demand
|
|
type fakeTicker struct {
|
|
channels []chan time.Time
|
|
}
|
|
|
|
func (t *fakeTicker) Init(duration time.Duration) {}
|
|
|
|
func (t *fakeTicker) AddChannel(c chan time.Time) {
|
|
t.channels = append(t.channels, c)
|
|
}
|
|
|
|
func (t *fakeTicker) Close() {}
|
|
|
|
func (t *fakeTicker) tick() {
|
|
for _, c := range t.channels {
|
|
select {
|
|
case c <- time.Now():
|
|
default:
|
|
}
|
|
}
|
|
}
|
|
|
|
// Stub collector whose Read blocks until it is released
|
|
type stubCollector struct {
|
|
metricCollector
|
|
readStarted chan struct{}
|
|
release chan struct{}
|
|
reads atomic.Int32
|
|
}
|
|
|
|
func (c *stubCollector) Init(config json.RawMessage) error {
|
|
c.name = "teststub"
|
|
c.parallel = true
|
|
c.init = true
|
|
return nil
|
|
}
|
|
|
|
func (c *stubCollector) Read(duration time.Duration, output chan lp.CCMessage) {
|
|
c.reads.Add(1)
|
|
c.readStarted <- struct{}{}
|
|
<-c.release
|
|
}
|
|
|
|
func (c *stubCollector) Close() {}
|
|
|
|
func TestOverlongCollectionRoundSkipsTick(t *testing.T) {
|
|
stub := &stubCollector{
|
|
readStarted: make(chan struct{}, 10),
|
|
release: make(chan struct{}),
|
|
}
|
|
AvailableCollectors["teststub"] = stub
|
|
defer delete(AvailableCollectors, "teststub")
|
|
|
|
ticker := &fakeTicker{}
|
|
var wg sync.WaitGroup
|
|
cm, err := New(ticker, time.Second, &wg, json.RawMessage(`{"teststub": {}}`))
|
|
if err != nil {
|
|
t.Fatalf("failed to setup collector manager: %s", err.Error())
|
|
}
|
|
cm.AddOutput(make(chan lp.CCMessage, 100))
|
|
cm.Start()
|
|
|
|
// First tick starts a collection round that blocks in Read
|
|
ticker.tick()
|
|
select {
|
|
case <-stub.readStarted:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("collection round did not start on tick")
|
|
}
|
|
|
|
// Further ticks while the round is still running must be skipped,
|
|
// not queued up or run concurrently
|
|
for range 3 {
|
|
ticker.tick()
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
if got := stub.reads.Load(); got != 1 {
|
|
t.Fatalf("expected 1 concurrent collection round, got %d reads", got)
|
|
}
|
|
|
|
// Finish the round, the next tick must start a new one
|
|
stub.release <- struct{}{}
|
|
deadline := time.After(5 * time.Second)
|
|
for stub.reads.Load() < 2 {
|
|
ticker.tick()
|
|
select {
|
|
case <-stub.readStarted:
|
|
case <-time.After(20 * time.Millisecond):
|
|
case <-deadline:
|
|
t.Fatal("no new collection round started after the previous one finished")
|
|
}
|
|
}
|
|
|
|
// Shutdown must wait for the running round and terminate cleanly.
|
|
// Closing the release channel lets any still running or straggler
|
|
// round finish immediately
|
|
close(stub.release)
|
|
closed := make(chan struct{})
|
|
go func() {
|
|
cm.Close()
|
|
close(closed)
|
|
}()
|
|
select {
|
|
case <-closed:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("Close() did not terminate")
|
|
}
|
|
wg.Wait()
|
|
}
|