mirror of
https://github.com/ClusterCockpit/cc-metric-collector.git
synced 2026-07-11 04:00:38 +02:00
moebiusband
3d4c464166
On nodes with >300 cores, one interval's burst of per-hwthread metrics overran the fixed 200-slot channels. With blocking sends at every hop, sink back-pressure propagated to the collectors, the collection round exceeded the interval, and time.Ticker silently dropped the missed ticks - whole intervals were skipped without any log message. - multiChanTicker: deliver ticks non-blockingly and warn when a consumer misses a tick instead of stalling all consumers; guard the channel list with a mutex (data race with AddChannel) - collectorManager: run the collection round detached from the tick loop, skip-and-warn when a round is still running, log per-collector and per-round durations at debug level, close serial collectors on shutdown - metricRouter: buffer the interval timestamp channel and drain it before stamping, so metrics never carry the previous interval's timestamp; warn when the collector input channel is full at tick time - main: scale the inter-manager channels to max(200, 24*NumCPU), overridable with the new optional channel_buffer_size config option - add first unit tests for ticker, collector manager and router Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
52 lines
1.1 KiB
Go
52 lines
1.1 KiB
Go
package multiChanTicker
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// A consumer that never reads its channel must not stall the ticker
|
|
// or starve the other consumers
|
|
func TestStalledConsumerDoesNotStarveOthers(t *testing.T) {
|
|
stalled := make(chan time.Time, 1) // never read
|
|
fast := make(chan time.Time, 1)
|
|
|
|
ticker := NewTicker(10 * time.Millisecond)
|
|
defer ticker.Close()
|
|
ticker.AddChannel(stalled)
|
|
ticker.AddChannel(fast)
|
|
|
|
received := 0
|
|
deadline := time.After(5 * time.Second)
|
|
for received < 5 {
|
|
select {
|
|
case <-fast:
|
|
received++
|
|
case <-deadline:
|
|
t.Fatalf("received only %d ticks while another consumer stalled", received)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Close() must return promptly even if no consumer reads its channel
|
|
func TestCloseWithStalledConsumer(t *testing.T) {
|
|
stalled := make(chan time.Time, 1) // never read
|
|
|
|
ticker := NewTicker(10 * time.Millisecond)
|
|
ticker.AddChannel(stalled)
|
|
|
|
// Let some ticks fire and be dropped
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
closed := make(chan struct{})
|
|
go func() {
|
|
ticker.Close()
|
|
close(closed)
|
|
}()
|
|
select {
|
|
case <-closed:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("Close() blocked with a stalled consumer")
|
|
}
|
|
}
|