mirror of
https://github.com/ClusterCockpit/cc-metric-collector.git
synced 2026-07-11 12:10:37 +02:00
Fix metric timeseries gaps on nodes with many cores
On nodes with >300 cores, one interval's burst of per-hwthread metrics overran the fixed 200-slot channels. With blocking sends at every hop, sink back-pressure propagated to the collectors, the collection round exceeded the interval, and time.Ticker silently dropped the missed ticks - whole intervals were skipped without any log message. - multiChanTicker: deliver ticks non-blockingly and warn when a consumer misses a tick instead of stalling all consumers; guard the channel list with a mutex (data race with AddChannel) - collectorManager: run the collection round detached from the tick loop, skip-and-warn when a round is still running, log per-collector and per-round durations at debug level, close serial collectors on shutdown - metricRouter: buffer the interval timestamp channel and drain it before stamping, so metrics never carry the previous interval's timestamp; warn when the collector input channel is full at tick time - main: scale the inter-manager channels to max(200, 24*NumCPU), overridable with the new optional channel_buffer_size config option - add first unit tests for ticker, collector manager and router Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -228,11 +228,25 @@ func (r *metricRouter) DoAddTags(point lp.CCMessage) {
|
||||
func (r *metricRouter) Start() {
|
||||
// start timer if configured
|
||||
r.timestamp = time.Now()
|
||||
timeChan := make(chan time.Time)
|
||||
timeChan := make(chan time.Time, 1)
|
||||
if r.config.IntervalStamp {
|
||||
r.ticker.AddChannel(timeChan)
|
||||
}
|
||||
|
||||
// Drain a pending tick before stamping new metrics, so a new interval's
|
||||
// metrics never carry the previous interval's timestamp
|
||||
updateTimestamp := func() {
|
||||
if !r.config.IntervalStamp {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case timestamp := <-timeChan:
|
||||
r.timestamp = timestamp
|
||||
cclog.ComponentDebug("MetricRouter", "Update timestamp", r.timestamp.UnixNano())
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Router manager is done
|
||||
done := func() {
|
||||
close(r.done)
|
||||
@@ -298,14 +312,19 @@ func (r *metricRouter) Start() {
|
||||
case timestamp := <-timeChan:
|
||||
r.timestamp = timestamp
|
||||
cclog.ComponentDebug("MetricRouter", "Update timestamp", r.timestamp.UnixNano())
|
||||
if len(r.coll_input) == cap(r.coll_input) {
|
||||
cclog.ComponentWarn("MetricRouter", "collector input channel full at tick, sinks may be too slow")
|
||||
}
|
||||
|
||||
case p := <-r.coll_input:
|
||||
updateTimestamp()
|
||||
coll_forward(p)
|
||||
for i := 0; len(r.coll_input) > 0 && i < (r.maxForward-1); i++ {
|
||||
coll_forward(<-r.coll_input)
|
||||
}
|
||||
|
||||
case p := <-r.recv_input:
|
||||
updateTimestamp()
|
||||
recv_forward(p)
|
||||
for i := 0; len(r.recv_input) > 0 && i < (r.maxForward-1); i++ {
|
||||
recv_forward(<-r.recv_input)
|
||||
|
||||
Reference in New Issue
Block a user