mirror of
https://github.com/ClusterCockpit/cc-metric-collector.git
synced 2026-07-11 12:10:37 +02:00
Fix metric timeseries gaps on nodes with many cores
On nodes with >300 cores, one interval's burst of per-hwthread metrics overran the fixed 200-slot channels. With blocking sends at every hop, sink back-pressure propagated to the collectors, the collection round exceeded the interval, and time.Ticker silently dropped the missed ticks - whole intervals were skipped without any log message. - multiChanTicker: deliver ticks non-blockingly and warn when a consumer misses a tick instead of stalling all consumers; guard the channel list with a mutex (data race with AddChannel) - collectorManager: run the collection round detached from the tick loop, skip-and-warn when a round is still running, log per-collector and per-round durations at debug level, close serial collectors on shutdown - metricRouter: buffer the interval timestamp channel and drain it before stamping, so metrics never carry the previous interval's timestamp; warn when the collector input channel is full at tick time - main: scale the inter-manager channels to max(200, 24*NumCPU), overridable with the new optional channel_buffer_size config option - add first unit tests for ticker, collector manager and router Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
package multiChanTicker
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A consumer that never reads its channel must not stall the ticker
|
||||
// or starve the other consumers
|
||||
func TestStalledConsumerDoesNotStarveOthers(t *testing.T) {
|
||||
stalled := make(chan time.Time, 1) // never read
|
||||
fast := make(chan time.Time, 1)
|
||||
|
||||
ticker := NewTicker(10 * time.Millisecond)
|
||||
defer ticker.Close()
|
||||
ticker.AddChannel(stalled)
|
||||
ticker.AddChannel(fast)
|
||||
|
||||
received := 0
|
||||
deadline := time.After(5 * time.Second)
|
||||
for received < 5 {
|
||||
select {
|
||||
case <-fast:
|
||||
received++
|
||||
case <-deadline:
|
||||
t.Fatalf("received only %d ticks while another consumer stalled", received)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close() must return promptly even if no consumer reads its channel
|
||||
func TestCloseWithStalledConsumer(t *testing.T) {
|
||||
stalled := make(chan time.Time, 1) // never read
|
||||
|
||||
ticker := NewTicker(10 * time.Millisecond)
|
||||
ticker.AddChannel(stalled)
|
||||
|
||||
// Let some ticks fire and be dropped
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
closed := make(chan struct{})
|
||||
go func() {
|
||||
ticker.Close()
|
||||
close(closed)
|
||||
}()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Close() blocked with a stalled consumer")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user