mirror of
https://github.com/ClusterCockpit/cc-metric-collector.git
synced 2026-04-05 15:37:29 +02:00
Merge latest development changes to main branch (#79)
* Cleanup: Remove unused code * Use Golang duration parser for 'interval' and 'duration' in main config * Update handling of LIKWID headers. Download only if not already present in the system. Fixes #73 * Units with cc-units (#64) * Add option to normalize units with cc-unit * Add unit conversion to router * Add option to change unit prefix in the router * Add to MetricRouter README * Add order of operations in router to README * Use second add_tags/del_tags only if metric gets renamed * Skip disks in DiskstatCollector that have size=0 * Check readability of sensor files in TempCollector * Fix for --once option * Rename `cpu` type to `hwthread` (#69) * Rename 'cpu' type to 'hwthread' to avoid naming clashes with MetricStore and CC-Webfrontend * Collectors in parallel (#74) * Provide info to CollectorManager whether the collector can be executed in parallel with others * Split serial and parallel collectors. Read in parallel first * Update NvidiaCollector with new metrics, MIG and NvLink support (#75) * CC topology module update (#76) * Rename CPU to hardware thread, write some comments * Do renaming in other parts * Remove CpuList and SocketList function from metricCollector. Available in ccTopology * Option to use MIG UUID as subtype-id in NvidiaCollector * Option to use MIG slice name as subtype-id in NvidiaCollector * MetricRouter: Fix JSON in README * Fix for Github Action to really use the selected version * Remove Ganglia installation in runonce Action and add Go 1.18 * Fix daemon options in init script * Add separate go.mod files to use it with deprecated 1.16 * Minor updates for Makefiles * fix string comparison * AMD ROCm SMI collector (#77) * Add collector for AMD ROCm SMI metrics * Fix import path * Fix imports * Remove Board Number * store GPU index explicitly * Remove board number from description * Use http instead of ftp to download likwid * Fix serial number in rocmCollector * Improved http sink (#78) * automatic flush in NatsSink * tweak default options of HttpSink * shorter cirt. section and retries for HttpSink * fix error handling * Remove file added by mistake. * Use http instead of ftp to download likwid * Fix serial number in rocmCollector Co-authored-by: Thomas Roehl <thomas.roehl@fau.de> Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> Co-authored-by: Lou <lou.knauer@gmx.de>
This commit is contained in:
Thomas Gruber
GitHub
@@ -16,37 +16,28 @@ import (
|
||||
"github.com/influxdata/influxdb-client-go/v2/api/write"
|
||||
)
|
||||
|
||||
type InfluxSinkConfig struct {
|
||||
defaultSinkConfig
|
||||
Host string `json:"host,omitempty"`
|
||||
Port string `json:"port,omitempty"`
|
||||
Database string `json:"database,omitempty"`
|
||||
User string `json:"user,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
Organization string `json:"organization,omitempty"`
|
||||
SSL bool `json:"ssl,omitempty"`
|
||||
FlushDelay string `json:"flush_delay,omitempty"`
|
||||
BatchSize int `json:"batch_size,omitempty"`
|
||||
RetentionPol string `json:"retention_policy,omitempty"`
|
||||
// InfluxRetryInterval string `json:"retry_interval"`
|
||||
// InfluxExponentialBase uint `json:"retry_exponential_base"`
|
||||
// InfluxMaxRetries uint `json:"max_retries"`
|
||||
// InfluxMaxRetryTime string `json:"max_retry_time"`
|
||||
//InfluxMaxRetryDelay string `json:"max_retry_delay"` // It is mentioned in the docs but there is no way to set it
|
||||
}
|
||||
|
||||
type InfluxSink struct {
|
||||
sink
|
||||
client influxdb2.Client
|
||||
writeApi influxdb2Api.WriteAPIBlocking
|
||||
config InfluxSinkConfig
|
||||
influxRetryInterval uint
|
||||
influxMaxRetryTime uint
|
||||
batch []*write.Point
|
||||
flushTimer *time.Timer
|
||||
flushDelay time.Duration
|
||||
lock sync.Mutex // Flush() runs in another goroutine, so this lock has to protect the buffer
|
||||
//influxMaxRetryDelay uint
|
||||
client influxdb2.Client
|
||||
writeApi influxdb2Api.WriteAPIBlocking
|
||||
config struct {
|
||||
defaultSinkConfig
|
||||
Host string `json:"host,omitempty"`
|
||||
Port string `json:"port,omitempty"`
|
||||
Database string `json:"database,omitempty"`
|
||||
User string `json:"user,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
Organization string `json:"organization,omitempty"`
|
||||
SSL bool `json:"ssl,omitempty"`
|
||||
// Maximum number of points sent to server in single request. Default 100
|
||||
BatchSize int `json:"batch_size,omitempty"`
|
||||
// Interval, in which is buffer flushed if it has not been already written (by reaching batch size). Default 1s
|
||||
FlushInterval string `json:"flush_delay,omitempty"`
|
||||
}
|
||||
batch []*write.Point
|
||||
flushTimer *time.Timer
|
||||
flushDelay time.Duration
|
||||
lock sync.Mutex // Flush() runs in another goroutine, so this lock has to protect the buffer
|
||||
}
|
||||
|
||||
// connect connects to the InfluxDB server
|
||||
@@ -76,23 +67,6 @@ func (s *InfluxSink) connect() error {
|
||||
// Set influxDB client options
|
||||
clientOptions := influxdb2.DefaultOptions()
|
||||
|
||||
// if s.influxRetryInterval != 0 {
|
||||
// cclog.ComponentDebug(s.name, "MaxRetryInterval", s.influxRetryInterval)
|
||||
// clientOptions.SetMaxRetryInterval(s.influxRetryInterval)
|
||||
// }
|
||||
// if s.influxMaxRetryTime != 0 {
|
||||
// cclog.ComponentDebug(s.name, "MaxRetryTime", s.influxMaxRetryTime)
|
||||
// clientOptions.SetMaxRetryTime(s.influxMaxRetryTime)
|
||||
// }
|
||||
// if s.config.InfluxExponentialBase != 0 {
|
||||
// cclog.ComponentDebug(s.name, "Exponential Base", s.config.InfluxExponentialBase)
|
||||
// clientOptions.SetExponentialBase(s.config.InfluxExponentialBase)
|
||||
// }
|
||||
// if s.config.InfluxMaxRetries != 0 {
|
||||
// cclog.ComponentDebug(s.name, "Max Retries", s.config.InfluxMaxRetries)
|
||||
// clientOptions.SetMaxRetries(s.config.InfluxMaxRetries)
|
||||
// }
|
||||
|
||||
// Do not check InfluxDB certificate
|
||||
clientOptions.SetTLSConfig(
|
||||
&tls.Config{
|
||||
@@ -126,11 +100,13 @@ func (s *InfluxSink) Write(m lp.CCMetric) error {
|
||||
}
|
||||
|
||||
// Run a batched flush for all lines that have arrived in the last flush delay interval
|
||||
s.flushTimer = time.AfterFunc(s.flushDelay, func() {
|
||||
if err := s.Flush(); err != nil {
|
||||
cclog.ComponentError(s.name, "flush failed:", err.Error())
|
||||
}
|
||||
})
|
||||
s.flushTimer = time.AfterFunc(
|
||||
s.flushDelay,
|
||||
func() {
|
||||
if err := s.Flush(); err != nil {
|
||||
cclog.ComponentError(s.name, "flush failed:", err.Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Append metric to batch slice
|
||||
@@ -194,7 +170,7 @@ func NewInfluxSink(name string, config json.RawMessage) (Sink, error) {
|
||||
|
||||
// Set config default values
|
||||
s.config.BatchSize = 100
|
||||
s.config.FlushDelay = "1s"
|
||||
s.config.FlushInterval = "1s"
|
||||
|
||||
// Read config
|
||||
if len(config) > 0 {
|
||||
@@ -203,12 +179,6 @@ func NewInfluxSink(name string, config json.RawMessage) (Sink, error) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
s.influxRetryInterval = 0
|
||||
s.influxMaxRetryTime = 0
|
||||
// s.config.InfluxRetryInterval = ""
|
||||
// s.config.InfluxMaxRetryTime = ""
|
||||
// s.config.InfluxMaxRetries = 0
|
||||
// s.config.InfluxExponentialBase = 0
|
||||
|
||||
if len(s.config.Host) == 0 {
|
||||
return nil, errors.New("Missing host configuration required by InfluxSink")
|
||||
@@ -232,21 +202,9 @@ func NewInfluxSink(name string, config json.RawMessage) (Sink, error) {
|
||||
s.meta_as_tags[k] = true
|
||||
}
|
||||
|
||||
// toUint := func(duration string, def uint) uint {
|
||||
// if len(duration) > 0 {
|
||||
// t, err := time.ParseDuration(duration)
|
||||
// if err == nil {
|
||||
// return uint(t.Milliseconds())
|
||||
// }
|
||||
// }
|
||||
// return def
|
||||
// }
|
||||
// s.influxRetryInterval = toUint(s.config.InfluxRetryInterval, s.influxRetryInterval)
|
||||
// s.influxMaxRetryTime = toUint(s.config.InfluxMaxRetryTime, s.influxMaxRetryTime)
|
||||
|
||||
// Configure flush delay duration
|
||||
if len(s.config.FlushDelay) > 0 {
|
||||
t, err := time.ParseDuration(s.config.FlushDelay)
|
||||
if len(s.config.FlushInterval) > 0 {
|
||||
t, err := time.ParseDuration(s.config.FlushInterval)
|
||||
if err == nil {
|
||||
s.flushDelay = t
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user