mirror of
https://github.com/ClusterCockpit/cc-metric-collector.git
synced 2026-07-11 12:10:37 +02:00
moebiusband
3d4c464166
On nodes with >300 cores, one interval's burst of per-hwthread metrics overran the fixed 200-slot channels. With blocking sends at every hop, sink back-pressure propagated to the collectors, the collection round exceeded the interval, and time.Ticker silently dropped the missed ticks - whole intervals were skipped without any log message. - multiChanTicker: deliver ticks non-blockingly and warn when a consumer misses a tick instead of stalling all consumers; guard the channel list with a mutex (data race with AddChannel) - collectorManager: run the collection round detached from the tick loop, skip-and-warn when a round is still running, log per-collector and per-round durations at debug level, close serial collectors on shutdown - metricRouter: buffer the interval timestamp channel and drain it before stamping, so metrics never carry the previous interval's timestamp; warn when the collector input channel is full at tick time - main: scale the inter-manager channels to max(200, 24*NumCPU), overridable with the new optional channel_buffer_size config option - add first unit tests for ticker, collector manager and router Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
84 lines
2.0 KiB
Go
84 lines
2.0 KiB
Go
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
|
// All rights reserved. This file is part of cc-lib.
|
|
// Use of this source code is governed by a MIT-style
|
|
// license that can be found in the LICENSE file.
|
|
// additional authors:
|
|
// Holger Obermaier (NHR@KIT)
|
|
|
|
package multiChanTicker
|
|
|
|
import (
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
|
|
)
|
|
|
|
type multiChanTicker struct {
|
|
ticker *time.Ticker
|
|
mutex sync.Mutex // protects channels, which is appended to while the tick goroutine iterates it
|
|
channels []chan time.Time
|
|
done chan bool
|
|
}
|
|
|
|
type MultiChanTicker interface {
|
|
Init(duration time.Duration)
|
|
AddChannel(channel chan time.Time)
|
|
Close()
|
|
}
|
|
|
|
func (t *multiChanTicker) Init(duration time.Duration) {
|
|
t.ticker = time.NewTicker(duration)
|
|
t.done = make(chan bool)
|
|
go func() {
|
|
done := func() {
|
|
close(t.done)
|
|
cclog.ComponentDebug("MultiChanTicker", "DONE")
|
|
}
|
|
for {
|
|
select {
|
|
case <-t.done:
|
|
done()
|
|
return
|
|
case ts := <-t.ticker.C:
|
|
cclog.ComponentDebug("MultiChanTicker", "Tick", ts)
|
|
t.mutex.Lock()
|
|
for i, c := range t.channels {
|
|
// Non-blocking send: a consumer that has not yet read the
|
|
// previous tick must not stall the ticker, otherwise
|
|
// time.Ticker silently drops fires for ALL consumers
|
|
select {
|
|
case c <- ts:
|
|
default:
|
|
cclog.ComponentWarn("MultiChanTicker", fmt.Sprintf("consumer %d did not read previous tick, dropping tick %v", i, ts))
|
|
}
|
|
}
|
|
t.mutex.Unlock()
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
func (t *multiChanTicker) AddChannel(channel chan time.Time) {
|
|
if cap(channel) == 0 {
|
|
cclog.ComponentWarn("MultiChanTicker", "unbuffered channel registered, ticks may be dropped if the consumer is not ready")
|
|
}
|
|
t.mutex.Lock()
|
|
t.channels = append(t.channels, channel)
|
|
t.mutex.Unlock()
|
|
}
|
|
|
|
func (t *multiChanTicker) Close() {
|
|
cclog.ComponentDebug("MultiChanTicker", "CLOSE")
|
|
t.done <- true
|
|
// wait for close of channel t.done
|
|
<-t.done
|
|
}
|
|
|
|
func NewTicker(duration time.Duration) MultiChanTicker {
|
|
t := &multiChanTicker{}
|
|
t.Init(duration)
|
|
return t
|
|
}
|