ClusterCockpit/cc-metric-collector
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-metric-collector.git synced 2025-04-05 13:15:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use receiver, sinks, ccLogger and ccConfig from cc-lib

Browse Source
This commit is contained in:
Thomas Roehl 2025-03-18 14:21:58 +01:00
parent 01ff8b2e9b
commit 6ea79b0099
48 changed files with 71 additions and 8295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
cc-metric-collector.go
View File

@ -7,39 +7,24 @@ import (
"os/signal"
"syscall"
"github.com/ClusterCockpit/cc-lib/receivers"
"github.com/ClusterCockpit/cc-lib/sinks"
"github.com/ClusterCockpit/cc-metric-collector/collectors"
"github.com/ClusterCockpit/cc-metric-collector/receivers"
"github.com/ClusterCockpit/cc-metric-collector/sinks"
// "strings"
"sync"
"time"
mr "github.com/ClusterCockpit/cc-metric-collector/internal/metricRouter"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
ccconf "github.com/ClusterCockpit/cc-lib/ccConfig"
cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
mr "github.com/ClusterCockpit/cc-metric-collector/internal/metricRouter"
mct "github.com/ClusterCockpit/cc-metric-collector/pkg/multiChanTicker"
)
type CentralConfigFile struct {
Interval string `json:"interval"`
Duration string `json:"duration"`
CollectorConfigFile string `json:"collectors"`
RouterConfigFile string `json:"router"`
SinkConfigFile string `json:"sinks"`
ReceiverConfigFile string `json:"receivers,omitempty"`
}
func LoadCentralConfiguration(file string, config *CentralConfigFile) error {
configFile, err := os.Open(file)
if err != nil {
cclog.Error(err.Error())
return err
}
defer configFile.Close()
jsonParser := json.NewDecoder(configFile)
err = jsonParser.Decode(config)
return err
Interval string `json:"interval"`
Duration string `json:"duration"`
}
type RuntimeConfig struct {
@ -87,7 +72,7 @@ func ReadCli() map[string]string {
cfg := flag.String("config", "./config.json", "Path to configuration file")
logfile := flag.String("log", "stderr", "Path for logfile")
once := flag.Bool("once", false, "Run all collectors only once")
debug := flag.Bool("debug", false, "Activate debug output")
loglevel := flag.String("loglevel", "info", "Set log level")
flag.Parse()
m = make(map[string]string)
m["configfile"] = *cfg
@ -97,12 +82,7 @@ func ReadCli() map[string]string {
} else {
m["once"] = "false"
}
if *debug {
m["debug"] = "true"
cclog.SetDebug()
} else {
m["debug"] = "false"
}
m["loglevel"] = *loglevel
return m
}
@ -167,8 +147,15 @@ func mainFunc() int {
CliArgs: ReadCli(),
}
// Set loglevel based on command line input.
cclog.Init(rcfg.CliArgs["loglevel"], false)
// Init ccConfig with configuration file
ccconf.Init(rcfg.CliArgs["configfile"])
// Load and check configuration
err = LoadCentralConfiguration(rcfg.CliArgs["configfile"], &rcfg.ConfigFile)
main := ccconf.GetPackageConfig("main")
err = json.Unmarshal(main, &rcfg.ConfigFile)
if err != nil {
cclog.Error("Error reading configuration file ", rcfg.CliArgs["configfile"], ": ", err.Error())
return 1
@ -204,38 +191,41 @@ func mainFunc() int {
return 1
}
if len(rcfg.ConfigFile.RouterConfigFile) == 0 {
routerConf := ccconf.GetPackageConfig("router")
if len(routerConf) == 0 {
cclog.Error("Metric router configuration file must be set")
return 1
}
if len(rcfg.ConfigFile.SinkConfigFile) == 0 {
sinkConf := ccconf.GetPackageConfig("sinks")
if len(sinkConf) == 0 {
cclog.Error("Sink configuration file must be set")
return 1
}
if len(rcfg.ConfigFile.CollectorConfigFile) == 0 {
collectorConf := ccconf.GetPackageConfig("collectors")
if len(collectorConf) == 0 {
cclog.Error("Metric collector configuration file must be set")
return 1
}
// Set log file
if logfile := rcfg.CliArgs["logfile"]; logfile != "stderr" {
cclog.SetOutput(logfile)
}
// if logfile := rcfg.CliArgs["logfile"]; logfile != "stderr" {
// cclog.SetOutput(logfile)
// }
// Creat new multi channel ticker
rcfg.MultiChanTicker = mct.NewTicker(rcfg.Interval)
// Create new metric router
rcfg.MetricRouter, err = mr.New(rcfg.MultiChanTicker, &rcfg.Sync, rcfg.ConfigFile.RouterConfigFile)
rcfg.MetricRouter, err = mr.New(rcfg.MultiChanTicker, &rcfg.Sync, routerConf)
if err != nil {
cclog.Error(err.Error())
return 1
}
// Create new sink
rcfg.SinkManager, err = sinks.New(&rcfg.Sync, rcfg.ConfigFile.SinkConfigFile)
rcfg.SinkManager, err = sinks.New(&rcfg.Sync, sinkConf)
if err != nil {
cclog.Error(err.Error())
return 1
@ -247,7 +237,7 @@ func mainFunc() int {
rcfg.MetricRouter.AddOutput(RouterToSinksChannel)
// Create new collector manager
rcfg.CollectManager, err = collectors.New(rcfg.MultiChanTicker, rcfg.Duration, &rcfg.Sync, rcfg.ConfigFile.CollectorConfigFile)
rcfg.CollectManager, err = collectors.New(rcfg.MultiChanTicker, rcfg.Duration, &rcfg.Sync, collectorConf)
if err != nil {
cclog.Error(err.Error())
return 1
@ -259,8 +249,9 @@ func mainFunc() int {
rcfg.MetricRouter.AddCollectorInput(CollectToRouterChannel)
// Create new receive manager
if len(rcfg.ConfigFile.ReceiverConfigFile) > 0 {
rcfg.ReceiveManager, err = receivers.New(&rcfg.Sync, rcfg.ConfigFile.ReceiverConfigFile)
receiveConf := ccconf.GetPackageConfig("receivers")
if len(receiveConf) > 0 {
rcfg.ReceiveManager, err = receivers.New(&rcfg.Sync, receiveConf)
if err != nil {
cclog.Error(err.Error())
return 1

19
collectors/collectorManager.go
View File

@ -2,7 +2,6 @@ package collectors
import (
"encoding/json"
"os"
"sync"
"time"
@ -59,7 +58,7 @@ type collectorManager struct {
// Metric collector manager access functions
type CollectorManager interface {
Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) error
Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfig json.RawMessage) error
AddOutput(output chan lp.CCMessage)
Start()
Close()
@ -72,7 +71,7 @@ type CollectorManager interface {
// * ticker (from variable ticker)
// * configuration (read from config file in variable collectConfigFile)
// Initialization is done for all configured collectors
func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) error {
func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfig json.RawMessage) error {
cm.collectors = make([]MetricCollector, 0)
cm.serial = make([]MetricCollector, 0)
cm.output = nil
@ -81,15 +80,7 @@ func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Durat
cm.ticker = ticker
cm.duration = duration
// Read collector config file
configFile, err := os.Open(collectConfigFile)
if err != nil {
cclog.Error(err.Error())
return err
}
defer configFile.Close()
jsonParser := json.NewDecoder(configFile)
err = jsonParser.Decode(&cm.config)
err := json.Unmarshal(collectConfig, &cm.config)
if err != nil {
cclog.Error(err.Error())
return err
@ -200,9 +191,9 @@ func (cm *collectorManager) Close() {
}
// New creates a new initialized metric collector manager
func New(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) (CollectorManager, error) {
func New(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfig json.RawMessage) (CollectorManager, error) {
cm := new(collectorManager)
err := cm.Init(ticker, duration, wg, collectConfigFile)
err := cm.Init(ticker, duration, wg, collectConfig)
if err != nil {
return nil, err
}

14
config.json
View File

@ -1,8 +1,10 @@
{
"sinks": "./sinks.json",
"collectors" : "./collectors.json",
"receivers" : "./receivers.json",
"router" : "./router.json",
"interval": "10s",
"duration": "1s"
"sinks-file": "./sinks.json",
"collectors-file" : "./collectors.json",
"receivers-file" : "./receivers.json",
"router-file" : "./router.json",
"main" : {
"interval": "10s",
"duration": "1s"
}
}

16
go.mod
View File

@ -1,16 +1,14 @@
module github.com/ClusterCockpit/cc-metric-collector
go 1.23.0
go 1.23.4
toolchain go1.23.2
toolchain go1.23.7
require (
github.com/ClusterCockpit/cc-lib v0.0.0-20250224161927-9edac91bf47a
github.com/ClusterCockpit/cc-units v0.4.0
github.com/ClusterCockpit/cc-lib v0.1.1
github.com/ClusterCockpit/go-rocm-smi v0.3.0
github.com/NVIDIA/go-nvml v0.12.0-2
github.com/PaesslerAG/gval v1.2.2
github.com/expr-lang/expr v1.16.9
github.com/fsnotify/fsnotify v1.7.0
github.com/gorilla/mux v1.8.1
github.com/influxdata/influxdb-client-go/v2 v2.14.0
@ -22,14 +20,16 @@ require (
github.com/tklauser/go-sysconf v0.3.13
golang.design/x/thread v0.0.0-20210122121316-335e9adffdf1
golang.org/x/exp v0.0.0-20250215185904-eff6e970281f
golang.org/x/sys v0.28.0
golang.org/x/sys v0.30.0
)
require (
github.com/ClusterCockpit/cc-backend v1.4.2 // indirect
github.com/ClusterCockpit/cc-units v0.4.0 // indirect
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/expr-lang/expr v1.17.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@ -42,7 +42,7 @@ require (
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/tklauser/numcpus v0.7.0 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/net v0.31.0 // indirect
golang.org/x/crypto v0.35.0 // indirect
golang.org/x/net v0.36.0 // indirect
google.golang.org/protobuf v1.35.2 // indirect
)

12
go.sum
View File

@ -1,5 +1,9 @@
github.com/ClusterCockpit/cc-backend v1.4.2 h1:kTOzqkh9N0564N9nqQThnSs7TAfg8RLgvSm00e5HtIc=
github.com/ClusterCockpit/cc-backend v1.4.2/go.mod h1:g8TNHXe4AXej26snu2//jO3mUF980elT93iV/k11O/c=
github.com/ClusterCockpit/cc-lib v0.1.0-beta.1 h1:dz9j0g2cod8+SMDjuoIY6ISpiHHeekhX6yQaeiwiwJw=
github.com/ClusterCockpit/cc-lib v0.1.0-beta.1/go.mod h1:kXMskla1i5ZSfXW0vVRIHgGeXMU5zu2PzYOYnUaOr80=
github.com/ClusterCockpit/cc-lib v0.1.1 h1:AXZWYUzgTaE/WdxLNSWPR7FJoA5WlzvYZxw4gIw3gNw=
github.com/ClusterCockpit/cc-lib v0.1.1/go.mod h1:SHKcWW/+kN+pcofAtHJFxvmx1FV0VIJuQv5PuT0HDcc=
github.com/ClusterCockpit/cc-units v0.4.0 h1:zP5DOu99GmErW0tCDf0gcLrlWt42RQ9dpoONEOh4cI0=
github.com/ClusterCockpit/cc-units v0.4.0/go.mod h1:3S3PAhAayS3pbgcT4q9Vn9VJw22Op51X0YimtG77zBw=
github.com/ClusterCockpit/go-rocm-smi v0.3.0 h1:1qZnSpG7/NyLtc7AjqnUL9Jb8xtqG1nMVgp69rJfaR8=
@ -25,6 +29,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI=
github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
github.com/expr-lang/expr v1.17.0 h1:+vpszOyzKLQXC9VF+wA8cVA0tlA984/Wabc/1hF9Whg=
github.com/expr-lang/expr v1.17.0/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
@ -109,13 +115,19 @@ golang.design/x/thread v0.0.0-20210122121316-335e9adffdf1 h1:P7S/GeHBAFEZIYp0ePP
golang.design/x/thread v0.0.0-20210122121316-335e9adffdf1/go.mod h1:9CWpnTUmlQkfdpdutA1nNf4iE5lAVt3QZOu0Z6hahBE=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
golang.org/x/exp v0.0.0-20250215185904-eff6e970281f h1:oFMYAjX0867ZD2jcNiLBrI9BdpmEkvPyi5YrBGXbamg=
golang.org/x/exp v0.0.0-20250215185904-eff6e970281f/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA=
golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I=
golang.org/x/sys v0.0.0-20210122093101-04d7465088b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=

20
internal/metricRouter/metricRouter.go
View File

@ -11,8 +11,8 @@ import (
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
agg "github.com/ClusterCockpit/cc-metric-collector/internal/metricAggregator"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
agg "github.com/ClusterCockpit/cc-metric-collector/internal/metricAggregator"
mct "github.com/ClusterCockpit/cc-metric-collector/pkg/multiChanTicker"
)
@ -63,7 +63,7 @@ type metricRouter struct {
// MetricRouter access functions
type MetricRouter interface {
Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfigFile string) error
Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfig json.RawMessage) error
AddCollectorInput(input chan lp.CCMessage)
AddReceiverInput(input chan lp.CCMessage)
AddOutput(output chan lp.CCMessage)
@ -77,7 +77,7 @@ type MetricRouter interface {
// * wait group synchronization (from variable wg)
// * ticker (from variable ticker)
// * configuration (read from config file in variable routerConfigFile)
func (r *metricRouter) Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfigFile string) error {
func (r *metricRouter) Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfig json.RawMessage) error {
r.outputs = make([]chan lp.CCMessage, 0)
r.done = make(chan bool)
r.cache_input = make(chan lp.CCMessage)
@ -95,15 +95,7 @@ func (r *metricRouter) Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, rout
// Drop domain part of host name
r.hostname = strings.SplitN(hostname, `.`, 2)[0]
// Read metric router config file
configFile, err := os.Open(routerConfigFile)
if err != nil {
cclog.ComponentError("MetricRouter", err.Error())
return err
}
defer configFile.Close()
jsonParser := json.NewDecoder(configFile)
err = jsonParser.Decode(&r.config)
err = json.Unmarshal(routerConfig, &r.config)
if err != nil {
cclog.ComponentError("MetricRouter", err.Error())
return err
@ -449,9 +441,9 @@ func (r *metricRouter) Close() {
}
// New creates a new initialized metric router
func New(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfigFile string) (MetricRouter, error) {
func New(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfig json.RawMessage) (MetricRouter, error) {
r := new(metricRouter)
err := r.Init(ticker, wg, routerConfigFile)
err := r.Init(ticker, wg, routerConfig)
if err != nil {
return nil, err
}

113
pkg/ccLogger/cclogger.go
View File

@ -1,113 +0,0 @@
package cclogger
import (
"fmt"
"log"
"os"
"runtime"
)
var (
globalDebug = false
stdout = os.Stdout
stderr = os.Stderr
debugLog *log.Logger = nil
infoLog *log.Logger = nil
errorLog *log.Logger = nil
warnLog *log.Logger = nil
defaultLog *log.Logger = nil
)
func initLogger() {
if debugLog == nil {
debugLog = log.New(stderr, "DEBUG ", log.LstdFlags)
}
if infoLog == nil {
infoLog = log.New(stdout, "INFO ", log.LstdFlags)
}
if errorLog == nil {
errorLog = log.New(stderr, "ERROR ", log.LstdFlags)
}
if warnLog == nil {
warnLog = log.New(stderr, "WARN ", log.LstdFlags)
}
if defaultLog == nil {
defaultLog = log.New(stdout, "", log.LstdFlags)
}
}
func Print(e ...interface{}) {
initLogger()
defaultLog.Print(e...)
}
func ComponentPrint(component string, e ...interface{}) {
initLogger()
defaultLog.Print(fmt.Sprintf("[%s] ", component), e)
}
func Info(e ...interface{}) {
initLogger()
infoLog.Print(e...)
}
func ComponentInfo(component string, e ...interface{}) {
initLogger()
infoLog.Print(fmt.Sprintf("[%s] ", component), e)
}
func Debug(e ...interface{}) {
initLogger()
if globalDebug {
debugLog.Print(e...)
}
}
func ComponentDebug(component string, e ...interface{}) {
initLogger()
if globalDebug && debugLog != nil {
//CCComponentPrint(debugLog, component, e)
debugLog.Print(fmt.Sprintf("[%s] ", component), e)
}
}
func Error(e ...interface{}) {
initLogger()
_, fn, line, _ := runtime.Caller(1)
errorLog.Print(fmt.Sprintf("[%s:%d] ", fn, line), e)
}
func ComponentError(component string, e ...interface{}) {
initLogger()
_, fn, line, _ := runtime.Caller(1)
errorLog.Print(fmt.Sprintf("[%s|%s:%d] ", component, fn, line), e)
}
func SetDebug() {
globalDebug = true
initLogger()
}
func SetOutput(filename string) {
if filename == "stderr" {
if stderr != os.Stderr && stderr != os.Stdout {
stderr.Close()
}
stderr = os.Stderr
} else if filename == "stdout" {
if stderr != os.Stderr && stderr != os.Stdout {
stderr.Close()
}
stderr = os.Stdout
} else {
file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err == nil {
defer file.Close()
stderr = file
}
}
debugLog = nil
errorLog = nil
warnLog = nil
initLogger()
}

57
pkg/ccMetric/README.md
View File

@ -1,57 +0,0 @@
# ClusterCockpit metrics
As described in the [ClusterCockpit specifications](https://github.com/ClusterCockpit/cc-specifications), the whole ClusterCockpit stack uses metrics in the InfluxDB line protocol format. This is also the input and output format for the ClusterCockpit Metric Collector but internally it uses an extended format while processing, named CCMetric.
It is basically a copy of the [InfluxDB line protocol](https://github.com/influxdata/line-protocol) `MutableMetric` interface with one extension. Besides the tags and fields, it contains a list of meta information (re-using the `Tag` structure of the original protocol):
```golang
type ccMetric struct {
name string // Measurement name
meta map[string]string // map of meta data tags
tags map[string]string // map of of tags
fields map[string]interface{} // map of of fields
tm time.Time // timestamp
}
type CCMetric interface {
ToPoint(metaAsTags map[string]bool) *write.Point // Generate influxDB point for data type ccMetric
ToLineProtocol(metaAsTags map[string]bool) string // Generate influxDB line protocol for data type ccMetric
String() string // Return line-protocol like string
Name() string // Get metric name
SetName(name string) // Set metric name
Time() time.Time // Get timestamp
SetTime(t time.Time) // Set timestamp
Tags() map[string]string // Map of tags
AddTag(key, value string) // Add a tag
GetTag(key string) (value string, ok bool) // Get a tag by its key
HasTag(key string) (ok bool) // Check if a tag key is present
RemoveTag(key string) // Remove a tag by its key
Meta() map[string]string // Map of meta data tags
AddMeta(key, value string) // Add a meta data tag
GetMeta(key string) (value string, ok bool) // Get a meta data tab addressed by its key
HasMeta(key string) (ok bool) // Check if a meta data key is present
RemoveMeta(key string) // Remove a meta data tag by its key
Fields() map[string]interface{} // Map of fields
AddField(key string, value interface{}) // Add a field
GetField(key string) (value interface{}, ok bool) // Get a field addressed by its key
HasField(key string) (ok bool) // Check if a field key is present
RemoveField(key string) // Remove a field addressed by its key
}
func New(name string, tags map[string]string, meta map[string]string, fields map[string]interface{}, tm time.Time) (CCMetric, error)
func FromMetric(other CCMetric) CCMetric
func FromInfluxMetric(other lp.Metric) CCMetric
```
The `CCMetric` interface provides the same functions as the `MutableMetric` like `{Add, Get, Remove, Has}{Tag, Field}` and additionally provides `{Add, Get, Remove, Has}Meta`.
The InfluxDB protocol creates a new metric with `influx.New(name, tags, fields, time)` while CCMetric uses `ccMetric.New(name, tags, meta, fields, time)` where `tags` and `meta` are both of type `map[string]string`.
You can copy a CCMetric with `FromMetric(other CCMetric) CCMetric`. If you get an `influx.Metric` from a function, like the line protocol parser, you can use `FromInfluxMetric(other influx.Metric) CCMetric` to get a CCMetric out of it (see `NatsReceiver` for an example).
Although the [cc-specifications](https://github.com/ClusterCockpit/cc-specifications/blob/master/interfaces/lineprotocol/README.md) defines that there is only a `value` field for the metric value, the CCMetric still can have multiple values similar to the InfluxDB line protocol.

353
pkg/ccMetric/ccMetric.go
View File

@ -1,353 +0,0 @@
package ccmetric
import (
"fmt"
"time"
influxdb2 "github.com/influxdata/influxdb-client-go/v2"
write "github.com/influxdata/influxdb-client-go/v2/api/write"
lp "github.com/influxdata/line-protocol" // MIT license
"golang.org/x/exp/maps"
)
// Most functions are derived from github.com/influxdata/line-protocol/metric.go
// The metric type is extended with an extra meta information list re-using the Tag
// type.
//
// See: https://docs.influxdata.com/influxdb/latest/reference/syntax/line-protocol/
type ccMetric struct {
name string // Measurement name
meta map[string]string // map of meta data tags
tags map[string]string // map of of tags
fields map[string]interface{} // map of of fields
tm time.Time // timestamp
}
// ccMetric access functions
type CCMetric interface {
ToPoint(metaAsTags map[string]bool) *write.Point // Generate influxDB point for data type ccMetric
ToLineProtocol(metaAsTags map[string]bool) string // Generate influxDB line protocol for data type ccMetric
Name() string // Get metric name
SetName(name string) // Set metric name
Time() time.Time // Get timestamp
SetTime(t time.Time) // Set timestamp
Tags() map[string]string // Map of tags
AddTag(key, value string) // Add a tag
GetTag(key string) (value string, ok bool) // Get a tag by its key
HasTag(key string) (ok bool) // Check if a tag key is present
RemoveTag(key string) // Remove a tag by its key
Meta() map[string]string // Map of meta data tags
AddMeta(key, value string) // Add a meta data tag
GetMeta(key string) (value string, ok bool) // Get a meta data tab addressed by its key
HasMeta(key string) (ok bool) // Check if a meta data key is present
RemoveMeta(key string) // Remove a meta data tag by its key
Fields() map[string]interface{} // Map of fields
AddField(key string, value interface{}) // Add a field
GetField(key string) (value interface{}, ok bool) // Get a field addressed by its key
HasField(key string) (ok bool) // Check if a field key is present
RemoveField(key string) // Remove a field addressed by its key
String() string // Return line-protocol like string
}
// String implements the stringer interface for data type ccMetric
func (m *ccMetric) String() string {
return fmt.Sprintf(
"Name: %s, Tags: %+v, Meta: %+v, fields: %+v, Timestamp: %d",
m.name, m.tags, m.meta, m.fields, m.tm.UnixNano(),
)
}
// ToLineProtocol generates influxDB line protocol for data type ccMetric
func (m *ccMetric) ToPoint(metaAsTags map[string]bool) (p *write.Point) {
p = influxdb2.NewPoint(m.name, m.tags, m.fields, m.tm)
for key, use_as_tag := range metaAsTags {
if use_as_tag {
if value, ok := m.GetMeta(key); ok {
p.AddTag(key, value)
}
}
}
return p
}
// ToLineProtocol generates influxDB line protocol for data type ccMetric
func (m *ccMetric) ToLineProtocol(metaAsTags map[string]bool) string {
return write.PointToLineProtocol(
m.ToPoint(metaAsTags),
time.Nanosecond,
)
}
// Name returns the measurement name
func (m *ccMetric) Name() string {
return m.name
}
// SetName sets the measurement name
func (m *ccMetric) SetName(name string) {
m.name = name
}
// Time returns timestamp
func (m *ccMetric) Time() time.Time {
return m.tm
}
// SetTime sets the timestamp
func (m *ccMetric) SetTime(t time.Time) {
m.tm = t
}
// Tags returns the the list of tags as key-value-mapping
func (m *ccMetric) Tags() map[string]string {
return m.tags
}
// AddTag adds a tag (consisting of key and value) to the map of tags
func (m *ccMetric) AddTag(key, value string) {
m.tags[key] = value
}
// GetTag returns the tag with tag's key equal to <key>
func (m *ccMetric) GetTag(key string) (string, bool) {
value, ok := m.tags[key]
return value, ok
}
// HasTag checks if a tag with key equal to <key> is present in the list of tags
func (m *ccMetric) HasTag(key string) bool {
_, ok := m.tags[key]
return ok
}
// RemoveTag removes the tag with tag's key equal to <key>
func (m *ccMetric) RemoveTag(key string) {
delete(m.tags, key)
}
// Meta returns the meta data tags as key-value mapping
func (m *ccMetric) Meta() map[string]string {
return m.meta
}
// AddMeta adds a meta data tag (consisting of key and value) to the map of meta data tags
func (m *ccMetric) AddMeta(key, value string) {
m.meta[key] = value
}
// GetMeta returns the meta data tag with meta data's key equal to <key>
func (m *ccMetric) GetMeta(key string) (string, bool) {
value, ok := m.meta[key]
return value, ok
}
// HasMeta checks if a meta data tag with meta data's key equal to <key> is present in the map of meta data tags
func (m *ccMetric) HasMeta(key string) bool {
_, ok := m.meta[key]
return ok
}
// RemoveMeta removes the meta data tag with tag's key equal to <key>
func (m *ccMetric) RemoveMeta(key string) {
delete(m.meta, key)
}
// Fields returns the list of fields as key-value-mapping
func (m *ccMetric) Fields() map[string]interface{} {
return m.fields
}
// AddField adds a field (consisting of key and value) to the map of fields
func (m *ccMetric) AddField(key string, value interface{}) {
m.fields[key] = value
}
// GetField returns the field with field's key equal to <key>
func (m *ccMetric) GetField(key string) (interface{}, bool) {
v, ok := m.fields[key]
return v, ok
}
// HasField checks if a field with field's key equal to <key> is present in the map of fields
func (m *ccMetric) HasField(key string) bool {
_, ok := m.fields[key]
return ok
}
// RemoveField removes the field with field's key equal to <key>
// from the map of fields
func (m *ccMetric) RemoveField(key string) {
delete(m.fields, key)
}
// New creates a new measurement point
func New(
name string,
tags map[string]string,
meta map[string]string,
fields map[string]interface{},
tm time.Time,
) (CCMetric, error) {
m := &ccMetric{
name: name,
tags: maps.Clone(tags),
meta: maps.Clone(meta),
fields: make(map[string]interface{}, len(fields)),
tm: tm,
}
// deep copy fields
for k, v := range fields {
v := convertField(v)
if v == nil {
continue
}
m.fields[k] = v
}
return m, nil
}
// FromMetric copies the metric <other>
func FromMetric(other CCMetric) CCMetric {
return &ccMetric{
name: other.Name(),
tags: maps.Clone(other.Tags()),
meta: maps.Clone(other.Meta()),
fields: maps.Clone(other.Fields()),
tm: other.Time(),
}
}
// FromInfluxMetric copies the influxDB line protocol metric <other>
func FromInfluxMetric(other lp.Metric) CCMetric {
m := &ccMetric{
name: other.Name(),
tags: make(map[string]string),
meta: make(map[string]string),
fields: make(map[string]interface{}),
tm: other.Time(),
}
// deep copy tags and fields
for _, otherTag := range other.TagList() {
m.tags[otherTag.Key] = otherTag.Value
}
for _, otherField := range other.FieldList() {
m.fields[otherField.Key] = otherField.Value
}
return m
}
// convertField converts data types of fields by the following schemata:
//
// *float32, *float64, float32, float64 -> float64
// *int, *int8, *int16, *int32, *int64, int, int8, int16, int32, int64 -> int64
//
// *uint, *uint8, *uint16, *uint32, *uint64, uint, uint8, uint16, uint32, uint64 -> uint64
// *[]byte, *string, []byte, string -> string
// *bool, bool -> bool
func convertField(v interface{}) interface{} {
switch v := v.(type) {
case float64:
return v
case int64:
return v
case string:
return v
case bool:
return v
case int:
return int64(v)
case uint:
return uint64(v)
case uint64:
return uint64(v)
case []byte:
return string(v)
case int32:
return int64(v)
case int16:
return int64(v)
case int8:
return int64(v)
case uint32:
return uint64(v)
case uint16:
return uint64(v)
case uint8:
return uint64(v)
case float32:
return float64(v)
case *float64:
if v != nil {
return *v
}
case *int64:
if v != nil {
return *v
}
case *string:
if v != nil {
return *v
}
case *bool:
if v != nil {
return *v
}
case *int:
if v != nil {
return int64(*v)
}
case *uint:
if v != nil {
return uint64(*v)
}
case *uint64:
if v != nil {
return uint64(*v)
}
case *[]byte:
if v != nil {
return string(*v)
}
case *int32:
if v != nil {
return int64(*v)
}
case *int16:
if v != nil {
return int64(*v)
}
case *int8:
if v != nil {
return int64(*v)
}
case *uint32:
if v != nil {
return uint64(*v)
}
case *uint16:
if v != nil {
return uint64(*v)
}
case *uint8:
if v != nil {
return uint64(*v)
}
case *float32:
if v != nil {
return float64(*v)
}
default:
return nil
}
return nil
}

266
pkg/messageProcessor/README.md
View File

@ -1,266 +0,0 @@
# Message Processor Component
Multiple parts of in the ClusterCockit ecosystem require the processing of CCMessages.
The main CC application using it is `cc-metric-collector`. The processing part there was originally in the metric router, the central
hub connecting collectors (reading local data), receivers (receiving remote data) and sinks (sending data). Already in early stages, the
lack of flexibility caused some trouble:
> The sysadmins wanted to keep operating their Ganglia based monitoring infrastructure while we developed the CC stack. Ganglia wants the core metrics with
> a specific name and resolution (right unit prefix) but there was no conversion of the data in the CC stack, so CC frontend developers wanted a different
> resolution for some metrics. The issue was basically the `mem_used` metric showing the currently used memory of the node. Ganglia wants it in `kByte` as provided
> by the Linux operating system but CC wanted it in `GByte`.
With the message processor, the Ganglia sinks can apply the unit prefix changes individually and name the metrics as required by Ganglia.
## For developers
Whenever you receive or are about to send a message out, you should provide some processing.
### Configuration of component
New operations can be added to the message processor at runtime. Of course, they can also be removed again. For the initial setup, having a configuration file
or some fields in a configuration file for the processing.
The message processor uses the following configuration
```json
{
"drop_messages": [
"name_of_message_to_drop"
],
"drop_messages_if": [
"condition_when_to_drop_message",
"name == 'drop_this'",
"tag.hostname == 'this_host'",
"meta.unit != 'MB'"
],
"rename_messages" : {
"old_message_name" : "new_message_name"
},
"rename_messages_if": {
"condition_when_to_rename_message" : "new_name"
},
"add_tags_if": [
{
"if" : "condition_when_to_add_tag",
"key": "name_for_new_tag",
"value": "new_tag_value"
}
],
"delete_tags_if": [
{
"if" : "condition_when_to_delete_tag",
"key": "name_of_tag"
}
],
"add_meta_if": [
{
"if" : "condition_when_to_add_meta_info",
"key": "name_for_new_meta_info",
"value": "new_meta_info_value"
}
],
"delete_meta_if": [
{
"if" : "condition_when_to_delete_meta_info",
"key": "name_of_meta_info"
}
],
"add_field_if": [
{
"if" : "condition_when_to_add_field",
"key": "name_for_new_field",
"value": "new_field_value_but_only_string_at_the_moment"
}
],
"delete_field_if": [
{
"if" : "condition_when_to_delete_field",
"key": "name_of_field"
}
],
"move_tag_to_meta_if": [
{
"if" : "condition_when_to_move_tag_to_meta_info_including_its_value",
"key": "name_of_tag",
"value": "name_of_meta_info"
}
],
"move_tag_to_field_if": [
{
"if" : "condition_when_to_move_tag_to_fields_including_its_value",
"key": "name_of_tag",
"value": "name_of_field"
}
],
"move_meta_to_tag_if": [
{
"if" : "condition_when_to_move_meta_info_to_tags_including_its_value",
"key": "name_of_meta_info",
"value": "name_of_tag"
}
],
"move_meta_to_field_if": [
{
"if" : "condition_when_to_move_meta_info_to_fields_including_its_value",
"key": "name_of_tag",
"value": "name_of_meta_info"
}
],
"move_field_to_tag_if": [
{
"if" : "condition_when_to_move_field_to_tags_including_its_stringified_value",
"key": "name_of_field",
"value": "name_of_tag"
}
],
"move_field_to_meta_if": [
{
"if" : "condition_when_to_move_field_to_meta_info_including_its_stringified_value",
"key": "name_of_field",
"value": "name_of_meta_info"
}
],
"drop_by_message_type": [
"metric",
"event",
"log",
"control"
],
"change_unit_prefix": {
"name == 'metric_with_wrong_unit_prefix'" : "G",
"only_if_messagetype == 'metric'": "T"
},
"normalize_units": true,
"add_base_env": {
"MY_CONSTANT_FOR_CUSTOM_CONDITIONS": 1.0,
"output_value_for_test_metrics": 42.0,
},
"stage_order": [
"rename_messages_if",
"drop_messages"
]
}
```
The options `change_unit_prefix` and `normalize_units` are only applied to CCMetrics. It is not possible to delete the field related to each message type as defined in [cc-specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/interfaces/lineprotocol). In short:
- CCMetrics always have to have a field named `value`
- CCEvents always have to have a field named `event`
- CCLogs always have to have a field named `log`
- CCControl messages always have to have a field named `control`
With `add_base_env`, one can specifiy mykey=myvalue pairs that can be used in conditions like `tag.type == mykey`.
The order in which each message is processed, can be specified with the `stage_order` option. The stage names are the keys in the JSON configuration, thus `change_unit_prefix`, `move_field_to_meta_if`, etc. Stages can be listed multiple times.
### Using the component
In order to load the configuration from a `json.RawMessage`:
```golang
mp, err := NewMessageProcessor()
if err != nil {
log.Error("failed to create new message processor")
}
mp.FromConfigJSON(configJson)
```
After initialization and adding the different operations, the `ProcessMessage()` function applies all operations and returns whether the message should be dropped.
```golang
m := lp.CCMetric{}
x, err := mp.ProcessMessage(m)
if err != nil {
// handle error
}
if x != nil {
// process x further
} else {
// this message got dropped
}
```
Single operations can be added and removed at runtime
```golang
type MessageProcessor interface {
// Functions to set the execution order of the processing stages
SetStages([]string) error
DefaultStages() []string
// Function to add variables to the base evaluation environment
AddBaseEnv(env map[string]interface{}) error
// Functions to add and remove rules
AddDropMessagesByName(name string) error
RemoveDropMessagesByName(name string)
AddDropMessagesByCondition(condition string) error
RemoveDropMessagesByCondition(condition string)
AddRenameMetricByCondition(condition string, name string) error
RemoveRenameMetricByCondition(condition string)
AddRenameMetricByName(from, to string) error
RemoveRenameMetricByName(from string)
SetNormalizeUnits(settings bool)
AddChangeUnitPrefix(condition string, prefix string) error
RemoveChangeUnitPrefix(condition string)
AddAddTagsByCondition(condition, key, value string) error
RemoveAddTagsByCondition(condition string)
AddDeleteTagsByCondition(condition, key, value string) error
RemoveDeleteTagsByCondition(condition string)
AddAddMetaByCondition(condition, key, value string) error
RemoveAddMetaByCondition(condition string)
AddDeleteMetaByCondition(condition, key, value string) error
RemoveDeleteMetaByCondition(condition string)
AddMoveTagToMeta(condition, key, value string) error
RemoveMoveTagToMeta(condition string)
AddMoveTagToFields(condition, key, value string) error
RemoveMoveTagToFields(condition string)
AddMoveMetaToTags(condition, key, value string) error
RemoveMoveMetaToTags(condition string)
AddMoveMetaToFields(condition, key, value string) error
RemoveMoveMetaToFields(condition string)
AddMoveFieldToTags(condition, key, value string) error
RemoveMoveFieldToTags(condition string)
AddMoveFieldToMeta(condition, key, value string) error
RemoveMoveFieldToMeta(condition string)
// Read in a JSON configuration
FromConfigJSON(config json.RawMessage) error
ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, error)
// Processing functions for legacy CCMetric and current CCMessage
ProcessMetric(m lp.CCMetric) (lp2.CCMessage, error)
}
```
### Syntax for evaluatable terms
The message processor uses `gval` for evaluating the terms. It provides a basic set of operators like string comparison and arithmetic operations.
Accessible for operations are
- `name` of the message
- `timestamp` or `time` of the message
- `type`, `type-id` of the message (also `tag_type`, `tag_type-id` and `tag_typeid`)
- `stype`, `stype-id` of the message (if message has theses tags, also `tag_stype`, `tag_stype-id` and `tag_stypeid`)
- `value` for a CCMetric message (also `field_value`)
- `event` for a CCEvent message (also `field_event`)
- `control` for a CCControl message (also `field_control`)
- `log` for a CCLog message (also `field_log`)
- `messagetype` or `msgtype`. Possible values `event`, `metric`, `log` and `control`.
Generally, all tags are accessible with `tag_<tagkey>`, `tags_<tagkey>` or `tags.<tagkey>`. Similarly for all fields with `field[s]?[_.]<fieldkey>`. For meta information `meta[_.]<metakey>` (there is no `metas[_.]<metakey>`).
The [syntax of `expr`](https://expr-lang.org/docs/language-definition) is accepted with some additions:
- Comparing strings: `==`, `!=`, `str matches regex` (use `%` instead of `\`!)
- Combining conditions: `&&`, `||`
- Comparing numbers: `==`, `!=`, `<`, `>`, `<=`, `>=`
- Test lists: `<value> in <list>`
- Topological tests: `tag_type-id in getCpuListOfType("socket", "1")` (test if the metric belongs to socket 1 in local node topology)
Often the operations are written in JSON files for loading them at startup. In JSON, some characters are not allowed. Therefore, the term syntax reflects that:
- use `''` instead of `""` for strings
- for the regexes, use `%` instead of `\`
For operations that should be applied on all messages, use the condition `true`.
### Overhead
The operations taking conditions are pre-processed, which is commonly the time consuming part but, of course, with each added operation, the time to process a message
increases. Moreover, the processing creates a copy of the message.

988
pkg/messageProcessor/messageProcessor.go
View File

@ -1,988 +0,0 @@
package messageprocessor
import (
"encoding/json"
"fmt"
"strings"
"sync"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
lplegacy "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
"github.com/expr-lang/expr"
"github.com/expr-lang/expr/vm"
)
// Message processor add/delete tag/meta configuration
type messageProcessorTagConfig struct {
Key string `json:"key"` // Tag name
Value string `json:"value,omitempty"` // Tag value
Condition string `json:"if"` // Condition for adding or removing corresponding tag
}
type messageProcessorConfig struct {
StageOrder []string `json:"stage_order,omitempty"` // List of stages to execute them in the specified order and to skip unrequired ones
DropMessages []string `json:"drop_messages,omitempty"` // List of metric names to drop. For fine-grained dropping use drop_messages_if
DropMessagesIf []string `json:"drop_messages_if,omitempty"` // List of evaluatable terms to drop messages
RenameMessages map[string]string `json:"rename_messages,omitempty"` // Map of metric names to rename
RenameMessagesIf map[string]string `json:"rename_messages_if,omitempty"` // Map to rename metric name based on a condition
NormalizeUnits bool `json:"normalize_units,omitempty"` // Check unit meta flag and normalize it using cc-units
ChangeUnitPrefix map[string]string `json:"change_unit_prefix,omitempty"` // Add prefix that should be applied to the messages
AddTagsIf []messageProcessorTagConfig `json:"add_tags_if,omitempty"` // List of tags that are added when the condition is met
DelTagsIf []messageProcessorTagConfig `json:"delete_tags_if,omitempty"` // List of tags that are removed when the condition is met
AddMetaIf []messageProcessorTagConfig `json:"add_meta_if,omitempty"` // List of meta infos that are added when the condition is met
DelMetaIf []messageProcessorTagConfig `json:"delete_meta_if,omitempty"` // List of meta infos that are removed when the condition is met
AddFieldIf []messageProcessorTagConfig `json:"add_field_if,omitempty"` // List of fields that are added when the condition is met
DelFieldIf []messageProcessorTagConfig `json:"delete_field_if,omitempty"` // List of fields that are removed when the condition is met
DropByType []string `json:"drop_by_message_type,omitempty"` // List of message types that should be dropped
MoveTagToMeta []messageProcessorTagConfig `json:"move_tag_to_meta_if,omitempty"`
MoveTagToField []messageProcessorTagConfig `json:"move_tag_to_field_if,omitempty"`
MoveMetaToTag []messageProcessorTagConfig `json:"move_meta_to_tag_if,omitempty"`
MoveMetaToField []messageProcessorTagConfig `json:"move_meta_to_field_if,omitempty"`
MoveFieldToTag []messageProcessorTagConfig `json:"move_field_to_tag_if,omitempty"`
MoveFieldToMeta []messageProcessorTagConfig `json:"move_field_to_meta_if,omitempty"`
AddBaseEnv map[string]interface{} `json:"add_base_env,omitempty"`
}
type messageProcessor struct {
// For thread-safety
mutex sync.RWMutex
// mapping contains all evalables as strings to gval.Evaluable
// because it is not possible to get the original string out of
// a gval.Evaluable
mapping map[string]*vm.Program
stages []string // order of stage execution
dropMessages map[string]struct{} // internal lookup map
dropTypes map[string]struct{} // internal lookup map
dropMessagesIf map[*vm.Program]struct{} // pre-processed dropMessagesIf
renameMessages map[string]string // internal lookup map
renameMessagesIf map[*vm.Program]string // pre-processed RenameMessagesIf
changeUnitPrefix map[*vm.Program]string // pre-processed ChangeUnitPrefix
normalizeUnits bool
addTagsIf map[*vm.Program]messageProcessorTagConfig // pre-processed AddTagsIf
deleteTagsIf map[*vm.Program]messageProcessorTagConfig // pre-processed DelTagsIf
addMetaIf map[*vm.Program]messageProcessorTagConfig // pre-processed AddMetaIf
deleteMetaIf map[*vm.Program]messageProcessorTagConfig // pre-processed DelMetaIf
addFieldIf map[*vm.Program]messageProcessorTagConfig // pre-processed AddFieldIf
deleteFieldIf map[*vm.Program]messageProcessorTagConfig // pre-processed DelFieldIf
moveTagToMeta map[*vm.Program]messageProcessorTagConfig // pre-processed MoveTagToMeta
moveTagToField map[*vm.Program]messageProcessorTagConfig // pre-processed MoveTagToField
moveMetaToTag map[*vm.Program]messageProcessorTagConfig // pre-processed MoveMetaToTag
moveMetaToField map[*vm.Program]messageProcessorTagConfig // pre-processed MoveMetaToField
moveFieldToTag map[*vm.Program]messageProcessorTagConfig // pre-processed MoveFieldToTag
moveFieldToMeta map[*vm.Program]messageProcessorTagConfig // pre-processed MoveFieldToMeta
}
type MessageProcessor interface {
// Functions to set the execution order of the processing stages
SetStages([]string) error
DefaultStages() []string
// Function to add variables to the base evaluation environment
AddBaseEnv(env map[string]interface{}) error
// Functions to add and remove rules
AddDropMessagesByName(name string) error
RemoveDropMessagesByName(name string)
AddDropMessagesByCondition(condition string) error
RemoveDropMessagesByCondition(condition string)
AddRenameMetricByCondition(condition string, name string) error
RemoveRenameMetricByCondition(condition string)
AddRenameMetricByName(from, to string) error
RemoveRenameMetricByName(from string)
SetNormalizeUnits(settings bool)
AddChangeUnitPrefix(condition string, prefix string) error
RemoveChangeUnitPrefix(condition string)
AddAddTagsByCondition(condition, key, value string) error
RemoveAddTagsByCondition(condition string)
AddDeleteTagsByCondition(condition, key, value string) error
RemoveDeleteTagsByCondition(condition string)
AddAddMetaByCondition(condition, key, value string) error
RemoveAddMetaByCondition(condition string)
AddDeleteMetaByCondition(condition, key, value string) error
RemoveDeleteMetaByCondition(condition string)
AddMoveTagToMeta(condition, key, value string) error
RemoveMoveTagToMeta(condition string)
AddMoveTagToFields(condition, key, value string) error
RemoveMoveTagToFields(condition string)
AddMoveMetaToTags(condition, key, value string) error
RemoveMoveMetaToTags(condition string)
AddMoveMetaToFields(condition, key, value string) error
RemoveMoveMetaToFields(condition string)
AddMoveFieldToTags(condition, key, value string) error
RemoveMoveFieldToTags(condition string)
AddMoveFieldToMeta(condition, key, value string) error
RemoveMoveFieldToMeta(condition string)
// Read in a JSON configuration
FromConfigJSON(config json.RawMessage) error
// Processing functions for legacy CCMetric and current CCMessage
ProcessMetric(m lplegacy.CCMetric) (lp.CCMessage, error)
ProcessMessage(m lp.CCMessage) (lp.CCMessage, error)
//EvalToBool(condition string, parameters map[string]interface{}) (bool, error)
//EvalToFloat64(condition string, parameters map[string]interface{}) (float64, error)
//EvalToString(condition string, parameters map[string]interface{}) (string, error)
}
const (
STAGENAME_DROP_BY_NAME string = "drop_by_name"
STAGENAME_DROP_BY_TYPE string = "drop_by_type"
STAGENAME_DROP_IF string = "drop_if"
STAGENAME_ADD_TAG string = "add_tag"
STAGENAME_DELETE_TAG string = "delete_tag"
STAGENAME_MOVE_TAG_META string = "move_tag_to_meta"
STAGENAME_MOVE_TAG_FIELD string = "move_tag_to_fields"
STAGENAME_ADD_META string = "add_meta"
STAGENAME_DELETE_META string = "delete_meta"
STAGENAME_MOVE_META_TAG string = "move_meta_to_tags"
STAGENAME_MOVE_META_FIELD string = "move_meta_to_fields"
STAGENAME_ADD_FIELD string = "add_field"
STAGENAME_DELETE_FIELD string = "delete_field"
STAGENAME_MOVE_FIELD_TAG string = "move_field_to_tags"
STAGENAME_MOVE_FIELD_META string = "move_field_to_meta"
STAGENAME_RENAME_BY_NAME string = "rename"
STAGENAME_RENAME_IF string = "rename_if"
STAGENAME_CHANGE_UNIT_PREFIX string = "change_unit_prefix"
STAGENAME_NORMALIZE_UNIT string = "normalize_unit"
)
var StageNames = []string{
STAGENAME_DROP_BY_NAME,
STAGENAME_DROP_BY_TYPE,
STAGENAME_DROP_IF,
STAGENAME_ADD_TAG,
STAGENAME_DELETE_TAG,
STAGENAME_MOVE_TAG_META,
STAGENAME_MOVE_TAG_FIELD,
STAGENAME_ADD_META,
STAGENAME_DELETE_META,
STAGENAME_MOVE_META_TAG,
STAGENAME_MOVE_META_FIELD,
STAGENAME_ADD_FIELD,
STAGENAME_DELETE_FIELD,
STAGENAME_MOVE_FIELD_TAG,
STAGENAME_MOVE_FIELD_META,
STAGENAME_RENAME_BY_NAME,
STAGENAME_RENAME_IF,
STAGENAME_CHANGE_UNIT_PREFIX,
STAGENAME_NORMALIZE_UNIT,
}
var paramMapPool = sync.Pool{
New: func() any {
return make(map[string]interface{})
},
}
func sanitizeExprString(key string) string {
return strings.ReplaceAll(key, "type-id", "typeid")
}
func getParamMap(point lp.CCMetric) map[string]interface{} {
params := paramMapPool.Get().(map[string]interface{})
params["message"] = point
params["msg"] = point
params["name"] = point.Name()
params["timestamp"] = point.Time().Unix()
params["time"] = params["timestamp"]
fields := paramMapPool.Get().(map[string]interface{})
for key, value := range point.Fields() {
fields[key] = value
switch key {
case "value":
params["messagetype"] = "metric"
params["value"] = value
params["metric"] = value
case "event":
params["messagetype"] = "event"
params["event"] = value
case "control":
params["messagetype"] = "control"
params["control"] = value
case "log":
params["messagetype"] = "log"
params["log"] = value
default:
params["messagetype"] = "unknown"
}
}
params["msgtype"] = params["messagetype"]
params["fields"] = fields
params["field"] = fields
tags := paramMapPool.Get().(map[string]interface{})
for key, value := range point.Tags() {
tags[sanitizeExprString(key)] = value
}
params["tags"] = tags
params["tag"] = tags
meta := paramMapPool.Get().(map[string]interface{})
for key, value := range point.Meta() {
meta[sanitizeExprString(key)] = value
}
params["meta"] = meta
return params
}
var baseenv = map[string]interface{}{
"name": "",
"messagetype": "unknown",
"msgtype": "unknown",
"tag": map[string]interface{}{
"type": "unknown",
"typeid": "0",
"stype": "unknown",
"stypeid": "0",
"hostname": "localhost",
"cluster": "nocluster",
},
"tags": map[string]interface{}{
"type": "unknown",
"typeid": "0",
"stype": "unknown",
"stypeid": "0",
"hostname": "localhost",
"cluster": "nocluster",
},
"meta": map[string]interface{}{
"unit": "invalid",
"source": "unknown",
},
"fields": map[string]interface{}{
"value": 0,
"event": "",
"control": "",
"log": "",
},
"field": map[string]interface{}{
"value": 0,
"event": "",
"control": "",
"log": "",
},
"timestamp": 1234567890,
"msg": lp.EmptyMessage(),
"message": lp.EmptyMessage(),
}
func addBaseEnvWalker(values map[string]interface{}) map[string]interface{} {
out := make(map[string]interface{})
for k, v := range values {
switch value := v.(type) {
case int, int32, int64, uint, uint32, uint64, string, float32, float64:
out[k] = value
case map[string]interface{}:
if _, ok := baseenv[k]; !ok {
out[k] = addBaseEnvWalker(value)
}
}
}
return out
}
func (mp *messageProcessor) AddBaseEnv(env map[string]interface{}) error {
for k, v := range env {
switch value := v.(type) {
case int, int32, int64, uint, uint32, uint64, string, float32, float64:
baseenv[k] = value
case map[string]interface{}:
if _, ok := baseenv[k]; !ok {
baseenv[k] = addBaseEnvWalker(value)
}
}
}
return nil
}
func (mp *messageProcessor) init() error {
mp.stages = make([]string, 0)
mp.mapping = make(map[string]*vm.Program)
mp.dropMessages = make(map[string]struct{})
mp.dropTypes = make(map[string]struct{})
mp.dropMessagesIf = make(map[*vm.Program]struct{})
mp.renameMessages = make(map[string]string)
mp.renameMessagesIf = make(map[*vm.Program]string)
mp.changeUnitPrefix = make(map[*vm.Program]string)
mp.addTagsIf = make(map[*vm.Program]messageProcessorTagConfig)
mp.addMetaIf = make(map[*vm.Program]messageProcessorTagConfig)
mp.addFieldIf = make(map[*vm.Program]messageProcessorTagConfig)
mp.deleteTagsIf = make(map[*vm.Program]messageProcessorTagConfig)
mp.deleteMetaIf = make(map[*vm.Program]messageProcessorTagConfig)
mp.deleteFieldIf = make(map[*vm.Program]messageProcessorTagConfig)
mp.moveFieldToMeta = make(map[*vm.Program]messageProcessorTagConfig)
mp.moveFieldToTag = make(map[*vm.Program]messageProcessorTagConfig)
mp.moveMetaToField = make(map[*vm.Program]messageProcessorTagConfig)
mp.moveMetaToTag = make(map[*vm.Program]messageProcessorTagConfig)
mp.moveTagToField = make(map[*vm.Program]messageProcessorTagConfig)
mp.moveTagToMeta = make(map[*vm.Program]messageProcessorTagConfig)
mp.normalizeUnits = false
return nil
}
func (mp *messageProcessor) AddDropMessagesByName(name string) error {
mp.mutex.Lock()
if _, ok := mp.dropMessages[name]; !ok {
mp.dropMessages[name] = struct{}{}
}
mp.mutex.Unlock()
return nil
}
func (mp *messageProcessor) RemoveDropMessagesByName(name string) {
mp.mutex.Lock()
delete(mp.dropMessages, name)
mp.mutex.Unlock()
}
func (mp *messageProcessor) AddDropMessagesByType(typestring string) error {
valid := []string{"metric", "event", "control", "log"}
isValid := false
for _, t := range valid {
if t == typestring {
isValid = true
break
}
}
if isValid {
mp.mutex.Lock()
if _, ok := mp.dropTypes[typestring]; !ok {
cclog.ComponentDebug("MessageProcessor", "Adding type", typestring, "for dropping")
mp.dropTypes[typestring] = struct{}{}
}
mp.mutex.Unlock()
} else {
return fmt.Errorf("invalid message type %s", typestring)
}
return nil
}
func (mp *messageProcessor) RemoveDropMessagesByType(typestring string) {
mp.mutex.Lock()
delete(mp.dropTypes, typestring)
mp.mutex.Unlock()
}
func (mp *messageProcessor) addTagConfig(condition, key, value string, config *map[*vm.Program]messageProcessorTagConfig) error {
var err error
evaluable, err := expr.Compile(sanitizeExprString(condition), expr.Env(baseenv), expr.AsBool())
if err != nil {
return fmt.Errorf("failed to create condition evaluable of '%s': %v", condition, err.Error())
}
mp.mutex.Lock()
if _, ok := (*config)[evaluable]; !ok {
mp.mapping[condition] = evaluable
(*config)[evaluable] = messageProcessorTagConfig{
Condition: condition,
Key: key,
Value: value,
}
}
mp.mutex.Unlock()
return nil
}
func (mp *messageProcessor) removeTagConfig(condition string, config *map[*vm.Program]messageProcessorTagConfig) {
mp.mutex.Lock()
if e, ok := mp.mapping[condition]; ok {
delete(mp.mapping, condition)
delete(*config, e)
}
mp.mutex.Unlock()
}
func (mp *messageProcessor) AddAddTagsByCondition(condition, key, value string) error {
return mp.addTagConfig(condition, key, value, &mp.addTagsIf)
}
func (mp *messageProcessor) RemoveAddTagsByCondition(condition string) {
mp.removeTagConfig(condition, &mp.addTagsIf)
}
func (mp *messageProcessor) AddDeleteTagsByCondition(condition, key, value string) error {
return mp.addTagConfig(condition, key, value, &mp.deleteTagsIf)
}
func (mp *messageProcessor) RemoveDeleteTagsByCondition(condition string) {
mp.removeTagConfig(condition, &mp.deleteTagsIf)
}
func (mp *messageProcessor) AddAddMetaByCondition(condition, key, value string) error {
return mp.addTagConfig(condition, key, value, &mp.addMetaIf)
}
func (mp *messageProcessor) RemoveAddMetaByCondition(condition string) {
mp.removeTagConfig(condition, &mp.addMetaIf)
}
func (mp *messageProcessor) AddDeleteMetaByCondition(condition, key, value string) error {
return mp.addTagConfig(condition, key, value, &mp.deleteMetaIf)
}
func (mp *messageProcessor) RemoveDeleteMetaByCondition(condition string) {
mp.removeTagConfig(condition, &mp.deleteMetaIf)
}
func (mp *messageProcessor) AddAddFieldByCondition(condition, key, value string) error {
return mp.addTagConfig(condition, key, value, &mp.addFieldIf)
}
func (mp *messageProcessor) RemoveAddFieldByCondition(condition string) {
mp.removeTagConfig(condition, &mp.addFieldIf)
}
func (mp *messageProcessor) AddDeleteFieldByCondition(condition, key, value string) error {
return mp.addTagConfig(condition, key, value, &mp.deleteFieldIf)
}
func (mp *messageProcessor) RemoveDeleteFieldByCondition(condition string) {
mp.removeTagConfig(condition, &mp.deleteFieldIf)
}
func (mp *messageProcessor) AddDropMessagesByCondition(condition string) error {
var err error
evaluable, err := expr.Compile(sanitizeExprString(condition), expr.Env(baseenv), expr.AsBool())
if err != nil {
return fmt.Errorf("failed to create condition evaluable of '%s': %v", condition, err.Error())
}
mp.mutex.Lock()
if _, ok := mp.dropMessagesIf[evaluable]; !ok {
mp.mapping[condition] = evaluable
mp.dropMessagesIf[evaluable] = struct{}{}
}
mp.mutex.Unlock()
return nil
}
func (mp *messageProcessor) RemoveDropMessagesByCondition(condition string) {
mp.mutex.Lock()
if e, ok := mp.mapping[condition]; ok {
delete(mp.mapping, condition)
delete(mp.dropMessagesIf, e)
}
mp.mutex.Unlock()
}
func (mp *messageProcessor) AddRenameMetricByCondition(condition string, name string) error {
var err error
evaluable, err := expr.Compile(sanitizeExprString(condition), expr.Env(baseenv), expr.AsBool())
if err != nil {
return fmt.Errorf("failed to create condition evaluable of '%s': %v", condition, err.Error())
}
mp.mutex.Lock()
if _, ok := mp.renameMessagesIf[evaluable]; !ok {
mp.mapping[condition] = evaluable
mp.renameMessagesIf[evaluable] = name
} else {
mp.renameMessagesIf[evaluable] = name
}
mp.mutex.Unlock()
return nil
}
func (mp *messageProcessor) RemoveRenameMetricByCondition(condition string) {
mp.mutex.Lock()
if e, ok := mp.mapping[condition]; ok {
delete(mp.mapping, condition)
delete(mp.renameMessagesIf, e)
}
mp.mutex.Unlock()
}
func (mp *messageProcessor) SetNormalizeUnits(setting bool) {
mp.normalizeUnits = setting
}
func (mp *messageProcessor) AddChangeUnitPrefix(condition string, prefix string) error {
var err error
evaluable, err := expr.Compile(sanitizeExprString(condition), expr.Env(baseenv), expr.AsBool())
if err != nil {
return fmt.Errorf("failed to create condition evaluable of '%s': %v", condition, err.Error())
}
mp.mutex.Lock()
if _, ok := mp.changeUnitPrefix[evaluable]; !ok {
mp.mapping[condition] = evaluable
mp.changeUnitPrefix[evaluable] = prefix
} else {
mp.changeUnitPrefix[evaluable] = prefix
}
mp.mutex.Unlock()
return nil
}
func (mp *messageProcessor) RemoveChangeUnitPrefix(condition string) {
mp.mutex.Lock()
if e, ok := mp.mapping[condition]; ok {
delete(mp.mapping, condition)
delete(mp.changeUnitPrefix, e)
}
mp.mutex.Unlock()
}
func (mp *messageProcessor) AddRenameMetricByName(from, to string) error {
mp.mutex.Lock()
if _, ok := mp.renameMessages[from]; !ok {
mp.renameMessages[from] = to
}
mp.mutex.Unlock()
return nil
}
func (mp *messageProcessor) RemoveRenameMetricByName(from string) {
mp.mutex.Lock()
delete(mp.renameMessages, from)
mp.mutex.Unlock()
}
func (mp *messageProcessor) AddMoveTagToMeta(condition, key, value string) error {
return mp.addTagConfig(condition, key, value, &mp.moveTagToMeta)
}
func (mp *messageProcessor) RemoveMoveTagToMeta(condition string) {
mp.removeTagConfig(condition, &mp.moveTagToMeta)
}
func (mp *messageProcessor) AddMoveTagToFields(condition, key, value string) error {
return mp.addTagConfig(condition, key, value, &mp.moveTagToField)
}
func (mp *messageProcessor) RemoveMoveTagToFields(condition string) {
mp.removeTagConfig(condition, &mp.moveTagToField)
}
func (mp *messageProcessor) AddMoveMetaToTags(condition, key, value string) error {
return mp.addTagConfig(condition, key, value, &mp.moveMetaToTag)
}
func (mp *messageProcessor) RemoveMoveMetaToTags(condition string) {
mp.removeTagConfig(condition, &mp.moveMetaToTag)
}
func (mp *messageProcessor) AddMoveMetaToFields(condition, key, value string) error {
return mp.addTagConfig(condition, key, value, &mp.moveMetaToField)
}
func (mp *messageProcessor) RemoveMoveMetaToFields(condition string) {
mp.removeTagConfig(condition, &mp.moveMetaToField)
}
func (mp *messageProcessor) AddMoveFieldToTags(condition, key, value string) error {
return mp.addTagConfig(condition, key, value, &mp.moveFieldToTag)
}
func (mp *messageProcessor) RemoveMoveFieldToTags(condition string) {
mp.removeTagConfig(condition, &mp.moveFieldToTag)
}
func (mp *messageProcessor) AddMoveFieldToMeta(condition, key, value string) error {
return mp.addTagConfig(condition, key, value, &mp.moveFieldToMeta)
}
func (mp *messageProcessor) RemoveMoveFieldToMeta(condition string) {
mp.removeTagConfig(condition, &mp.moveFieldToMeta)
}
func (mp *messageProcessor) SetStages(stages []string) error {
newstages := make([]string, 0)
if len(stages) == 0 {
mp.mutex.Lock()
mp.stages = newstages
mp.mutex.Unlock()
return nil
}
for i, s := range stages {
valid := false
for _, v := range StageNames {
if s == v {
valid = true
}
}
if valid {
newstages = append(newstages, s)
} else {
return fmt.Errorf("invalid stage %s at index %d", s, i)
}
}
mp.mutex.Lock()
mp.stages = newstages
mp.mutex.Unlock()
return nil
}
func (mp *messageProcessor) DefaultStages() []string {
return StageNames
}
func (mp *messageProcessor) FromConfigJSON(config json.RawMessage) error {
var c messageProcessorConfig
err := json.Unmarshal(config, &c)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
if len(c.StageOrder) > 0 {
err = mp.SetStages(c.StageOrder)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
} else {
err = mp.SetStages(mp.DefaultStages())
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, m := range c.DropMessages {
err = mp.AddDropMessagesByName(m)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, m := range c.DropByType {
err = mp.AddDropMessagesByType(m)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, m := range c.DropMessagesIf {
err = mp.AddDropMessagesByCondition(m)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for k, v := range c.RenameMessagesIf {
err = mp.AddRenameMetricByCondition(k, v)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for k, v := range c.RenameMessages {
err = mp.AddRenameMetricByName(k, v)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for k, v := range c.ChangeUnitPrefix {
err = mp.AddChangeUnitPrefix(k, v)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, c := range c.AddTagsIf {
err = mp.AddAddTagsByCondition(c.Condition, c.Key, c.Value)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, c := range c.AddMetaIf {
err = mp.AddAddMetaByCondition(c.Condition, c.Key, c.Value)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, c := range c.AddFieldIf {
err = mp.AddAddFieldByCondition(c.Condition, c.Key, c.Value)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, c := range c.DelTagsIf {
err = mp.AddDeleteTagsByCondition(c.Condition, c.Key, c.Value)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, c := range c.DelMetaIf {
err = mp.AddDeleteMetaByCondition(c.Condition, c.Key, c.Value)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, c := range c.DelFieldIf {
err = mp.AddDeleteFieldByCondition(c.Condition, c.Key, c.Value)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, c := range c.MoveTagToMeta {
err = mp.AddMoveTagToMeta(c.Condition, c.Key, c.Value)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, c := range c.MoveTagToField {
err = mp.AddMoveTagToFields(c.Condition, c.Key, c.Value)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, c := range c.MoveMetaToTag {
err = mp.AddMoveMetaToTags(c.Condition, c.Key, c.Value)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, c := range c.MoveMetaToField {
err = mp.AddMoveMetaToFields(c.Condition, c.Key, c.Value)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, c := range c.MoveFieldToTag {
err = mp.AddMoveFieldToTags(c.Condition, c.Key, c.Value)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, c := range c.MoveFieldToMeta {
err = mp.AddMoveFieldToMeta(c.Condition, c.Key, c.Value)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
for _, m := range c.DropByType {
err = mp.AddDropMessagesByType(m)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
if len(c.AddBaseEnv) > 0 {
err = mp.AddBaseEnv(c.AddBaseEnv)
if err != nil {
return fmt.Errorf("failed to process config JSON: %v", err.Error())
}
}
mp.SetNormalizeUnits(c.NormalizeUnits)
return nil
}
func (mp *messageProcessor) ProcessMetric(metric lplegacy.CCMetric) (lp.CCMessage, error) {
m, err := lp.NewMessage(
metric.Name(),
metric.Tags(),
metric.Meta(),
metric.Fields(),
metric.Time(),
)
if err != nil {
return m, fmt.Errorf("failed to parse metric to message: %v", err.Error())
}
return mp.ProcessMessage(m)
}
func (mp *messageProcessor) ProcessMessage(m lp.CCMessage) (lp.CCMessage, error) {
var err error = nil
var out lp.CCMessage = lp.FromMessage(m)
name := out.Name()
if len(mp.stages) == 0 {
mp.SetStages(mp.DefaultStages())
}
mp.mutex.RLock()
defer mp.mutex.RUnlock()
params := getParamMap(out)
defer func() {
params["field"] = nil
params["tag"] = nil
paramMapPool.Put(params["fields"])
paramMapPool.Put(params["tags"])
paramMapPool.Put(params["meta"])
paramMapPool.Put(params)
}()
for _, s := range mp.stages {
switch s {
case STAGENAME_DROP_BY_NAME:
if len(mp.dropMessages) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Dropping by message name ", name)
if _, ok := mp.dropMessages[name]; ok {
//cclog.ComponentDebug("MessageProcessor", "Drop")
return nil, nil
}
}
case STAGENAME_DROP_BY_TYPE:
if len(mp.dropTypes) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Dropping by message type")
if _, ok := mp.dropTypes[params["messagetype"].(string)]; ok {
//cclog.ComponentDebug("MessageProcessor", "Drop")
return nil, nil
}
}
case STAGENAME_DROP_IF:
if len(mp.dropMessagesIf) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Dropping by condition")
drop, err := dropMessagesIf(&params, &mp.dropMessagesIf)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
if drop {
//cclog.ComponentDebug("MessageProcessor", "Drop")
return nil, nil
}
}
case STAGENAME_RENAME_BY_NAME:
if len(mp.renameMessages) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Renaming by name match")
if newname, ok := mp.renameMessages[name]; ok {
//cclog.ComponentDebug("MessageProcessor", "Rename to", newname)
out.SetName(newname)
//cclog.ComponentDebug("MessageProcessor", "Add old name as 'oldname' to meta", name)
out.AddMeta("oldname", name)
}
}
case STAGENAME_RENAME_IF:
if len(mp.renameMessagesIf) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Renaming by condition")
_, err := renameMessagesIf(out, &params, &mp.renameMessagesIf)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_ADD_TAG:
if len(mp.addTagsIf) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Adding tags")
_, err = addTagIf(out, &params, &mp.addTagsIf)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_DELETE_TAG:
if len(mp.deleteTagsIf) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Delete tags")
_, err = deleteTagIf(out, &params, &mp.deleteTagsIf)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_ADD_META:
if len(mp.addMetaIf) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Adding meta information")
_, err = addMetaIf(out, &params, &mp.addMetaIf)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_DELETE_META:
if len(mp.deleteMetaIf) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Delete meta information")
_, err = deleteMetaIf(out, &params, &mp.deleteMetaIf)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_ADD_FIELD:
if len(mp.addFieldIf) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Adding fields")
_, err = addFieldIf(out, &params, &mp.addFieldIf)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_DELETE_FIELD:
if len(mp.deleteFieldIf) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Delete fields")
_, err = deleteFieldIf(out, &params, &mp.deleteFieldIf)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_MOVE_TAG_META:
if len(mp.moveTagToMeta) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Move tag to meta")
_, err := moveTagToMeta(out, &params, &mp.moveTagToMeta)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_MOVE_TAG_FIELD:
if len(mp.moveTagToField) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Move tag to fields")
_, err := moveTagToField(out, &params, &mp.moveTagToField)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_MOVE_META_TAG:
if len(mp.moveMetaToTag) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Move meta to tags")
_, err := moveMetaToTag(out, &params, &mp.moveMetaToTag)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_MOVE_META_FIELD:
if len(mp.moveMetaToField) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Move meta to fields")
_, err := moveMetaToField(out, &params, &mp.moveMetaToField)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_MOVE_FIELD_META:
if len(mp.moveFieldToMeta) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Move field to meta")
_, err := moveFieldToMeta(out, &params, &mp.moveFieldToMeta)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_MOVE_FIELD_TAG:
if len(mp.moveFieldToTag) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Move field to tags")
_, err := moveFieldToTag(out, &params, &mp.moveFieldToTag)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
}
case STAGENAME_NORMALIZE_UNIT:
if mp.normalizeUnits {
//cclog.ComponentDebug("MessageProcessor", "Normalize units")
if lp.IsMetric(out) {
_, err := normalizeUnits(out)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
} else {
cclog.ComponentDebug("MessageProcessor", "skipped, no metric")
}
}
case STAGENAME_CHANGE_UNIT_PREFIX:
if len(mp.changeUnitPrefix) > 0 {
//cclog.ComponentDebug("MessageProcessor", "Change unit prefix")
if lp.IsMetric(out) {
_, err := changeUnitPrefix(out, &params, &mp.changeUnitPrefix)
if err != nil {
return out, fmt.Errorf("failed to evaluate: %v", err.Error())
}
} else {
cclog.ComponentDebug("MessageProcessor", "skipped, no metric")
}
}
}
}
return out, nil
}
// Get a new instace of a message processor.
func NewMessageProcessor() (MessageProcessor, error) {
mp := new(messageProcessor)
err := mp.init()
if err != nil {
err := fmt.Errorf("failed to create MessageProcessor: %v", err.Error())
cclog.ComponentError("MessageProcessor", err.Error())
return nil, err
}
return mp, nil
}

262
pkg/messageProcessor/messageProcessorFuncs.go
View File

@ -1,262 +0,0 @@
package messageprocessor
import (
"errors"
"fmt"
lp2 "github.com/ClusterCockpit/cc-lib/ccMessage"
units "github.com/ClusterCockpit/cc-units"
"github.com/expr-lang/expr"
"github.com/expr-lang/expr/vm"
)
type MessageLocation int
const (
MESSAGE_LOCATION_TAGS MessageLocation = iota
MESSAGE_LOCATION_META
MESSAGE_LOCATION_FIELDS
)
// Abstract function to move entries from one location to another
func moveInMessage(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig, from, to MessageLocation) (bool, error) {
for d, data := range *checks {
value, err := expr.Run(d, *params)
if err != nil {
return false, fmt.Errorf("failed to evaluate: %v", err.Error())
}
//cclog.ComponentDebug("MessageProcessor", "Move from", from, "to", to)
if value.(bool) {
var v string
var ok bool = false
switch from {
case MESSAGE_LOCATION_TAGS:
//cclog.ComponentDebug("MessageProcessor", "Getting tag key", data.Key)
v, ok = message.GetTag(data.Key)
case MESSAGE_LOCATION_META:
//cclog.ComponentDebug("MessageProcessor", "Getting meta key", data.Key)
//cclog.ComponentDebug("MessageProcessor", message.Meta())
v, ok = message.GetMeta(data.Key)
case MESSAGE_LOCATION_FIELDS:
var x interface{}
//cclog.ComponentDebug("MessageProcessor", "Getting field key", data.Key)
x, ok = message.GetField(data.Key)
v = fmt.Sprintf("%v", x)
}
if ok {
switch from {
case MESSAGE_LOCATION_TAGS:
//cclog.ComponentDebug("MessageProcessor", "Removing tag key", data.Key)
message.RemoveTag(data.Key)
case MESSAGE_LOCATION_META:
//cclog.ComponentDebug("MessageProcessor", "Removing meta key", data.Key)
message.RemoveMeta(data.Key)
case MESSAGE_LOCATION_FIELDS:
//cclog.ComponentDebug("MessageProcessor", "Removing field key", data.Key)
message.RemoveField(data.Key)
}
switch to {
case MESSAGE_LOCATION_TAGS:
//cclog.ComponentDebug("MessageProcessor", "Adding tag", data.Value, "->", v)
message.AddTag(data.Value, v)
case MESSAGE_LOCATION_META:
//cclog.ComponentDebug("MessageProcessor", "Adding meta", data.Value, "->", v)
message.AddMeta(data.Value, v)
case MESSAGE_LOCATION_FIELDS:
//cclog.ComponentDebug("MessageProcessor", "Adding field", data.Value, "->", v)
message.AddField(data.Value, v)
}
}
}
}
return false, nil
}
func deleteIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig, location MessageLocation) (bool, error) {
for d, data := range *checks {
value, err := expr.Run(d, *params)
if err != nil {
return true, fmt.Errorf("failed to evaluate: %v", err.Error())
}
if value.(bool) {
switch location {
case MESSAGE_LOCATION_FIELDS:
switch data.Key {
case "value", "event", "log", "control":
return false, errors.New("cannot delete protected fields")
default:
//cclog.ComponentDebug("MessageProcessor", "Removing field for", data.Key)
message.RemoveField(data.Key)
}
case MESSAGE_LOCATION_TAGS:
//cclog.ComponentDebug("MessageProcessor", "Removing tag for", data.Key)
message.RemoveTag(data.Key)
case MESSAGE_LOCATION_META:
//cclog.ComponentDebug("MessageProcessor", "Removing meta for", data.Key)
message.RemoveMeta(data.Key)
}
}
}
return false, nil
}
func addIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig, location MessageLocation) (bool, error) {
for d, data := range *checks {
value, err := expr.Run(d, *params)
if err != nil {
return true, fmt.Errorf("failed to evaluate: %v", err.Error())
}
if value.(bool) {
switch location {
case MESSAGE_LOCATION_FIELDS:
//cclog.ComponentDebug("MessageProcessor", "Adding field", data.Value, "->", data.Value)
message.AddField(data.Key, data.Value)
case MESSAGE_LOCATION_TAGS:
//cclog.ComponentDebug("MessageProcessor", "Adding tag", data.Value, "->", data.Value)
message.AddTag(data.Key, data.Value)
case MESSAGE_LOCATION_META:
//cclog.ComponentDebug("MessageProcessor", "Adding meta", data.Value, "->", data.Value)
message.AddMeta(data.Key, data.Value)
}
}
}
return false, nil
}
func deleteTagIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
return deleteIf(message, params, checks, MESSAGE_LOCATION_TAGS)
}
func addTagIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
return addIf(message, params, checks, MESSAGE_LOCATION_TAGS)
}
func moveTagToMeta(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
return moveInMessage(message, params, checks, MESSAGE_LOCATION_TAGS, MESSAGE_LOCATION_META)
}
func moveTagToField(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
return moveInMessage(message, params, checks, MESSAGE_LOCATION_TAGS, MESSAGE_LOCATION_FIELDS)
}
func deleteMetaIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
return deleteIf(message, params, checks, MESSAGE_LOCATION_META)
}
func addMetaIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
return addIf(message, params, checks, MESSAGE_LOCATION_META)
}
func moveMetaToTag(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
return moveInMessage(message, params, checks, MESSAGE_LOCATION_META, MESSAGE_LOCATION_TAGS)
}
func moveMetaToField(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
return moveInMessage(message, params, checks, MESSAGE_LOCATION_META, MESSAGE_LOCATION_FIELDS)
}
func deleteFieldIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
return deleteIf(message, params, checks, MESSAGE_LOCATION_FIELDS)
}
func addFieldIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
return addIf(message, params, checks, MESSAGE_LOCATION_FIELDS)
}
func moveFieldToTag(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
return moveInMessage(message, params, checks, MESSAGE_LOCATION_FIELDS, MESSAGE_LOCATION_TAGS)
}
func moveFieldToMeta(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
return moveInMessage(message, params, checks, MESSAGE_LOCATION_FIELDS, MESSAGE_LOCATION_META)
}
func dropMessagesIf(params *map[string]interface{}, checks *map[*vm.Program]struct{}) (bool, error) {
for d := range *checks {
value, err := expr.Run(d, *params)
if err != nil {
return false, fmt.Errorf("failed to evaluate: %v", err.Error())
}
if value.(bool) {
return true, nil
}
}
return false, nil
}
func normalizeUnits(message lp2.CCMessage) (bool, error) {
if in_unit, ok := message.GetMeta("unit"); ok {
u := units.NewUnit(in_unit)
if u.Valid() {
//cclog.ComponentDebug("MessageProcessor", "Update unit with", u.Short())
message.AddMeta("unit", u.Short())
}
} else if in_unit, ok := message.GetTag("unit"); ok {
u := units.NewUnit(in_unit)
if u.Valid() {
//cclog.ComponentDebug("MessageProcessor", "Update unit with", u.Short())
message.AddTag("unit", u.Short())
}
}
return false, nil
}
func changeUnitPrefix(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]string) (bool, error) {
for r, n := range *checks {
value, err := expr.Run(r, *params)
if err != nil {
return false, fmt.Errorf("failed to evaluate: %v", err.Error())
}
if value.(bool) {
newPrefix := units.NewPrefix(n)
//cclog.ComponentDebug("MessageProcessor", "Condition matches, change to prefix", newPrefix.String())
if in_unit, ok := message.GetMeta("unit"); ok && newPrefix != units.InvalidPrefix {
u := units.NewUnit(in_unit)
if u.Valid() {
//cclog.ComponentDebug("MessageProcessor", "Input unit", u.Short())
conv, out_unit := units.GetUnitPrefixFactor(u, newPrefix)
if conv != nil && out_unit.Valid() {
if val, ok := message.GetField("value"); ok {
//cclog.ComponentDebug("MessageProcessor", "Update unit with", out_unit.Short())
message.AddField("value", conv(val))
message.AddMeta("unit", out_unit.Short())
}
}
}
} else if in_unit, ok := message.GetTag("unit"); ok && newPrefix != units.InvalidPrefix {
u := units.NewUnit(in_unit)
if u.Valid() {
//cclog.ComponentDebug("MessageProcessor", "Input unit", u.Short())
conv, out_unit := units.GetUnitPrefixFactor(u, newPrefix)
if conv != nil && out_unit.Valid() {
if val, ok := message.GetField("value"); ok {
//cclog.ComponentDebug("MessageProcessor", "Update unit with", out_unit.Short())
message.AddField("value", conv(val))
message.AddTag("unit", out_unit.Short())
}
}
}
}
}
}
return false, nil
}
func renameMessagesIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]string) (bool, error) {
for d, n := range *checks {
value, err := expr.Run(d, *params)
if err != nil {
return true, fmt.Errorf("failed to evaluate: %v", err.Error())
}
if value.(bool) {
old := message.Name()
//cclog.ComponentDebug("MessageProcessor", "Rename to", n)
message.SetName(n)
//cclog.ComponentDebug("MessageProcessor", "Add old name as 'oldname' to meta", old)
message.AddMeta("oldname", old)
}
}
return false, nil
}

396
pkg/messageProcessor/messageProcessor_test.go
View File

@ -1,396 +0,0 @@
package messageprocessor
import (
"encoding/json"
"errors"
"fmt"
"testing"
"time"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
)
func generate_message_lists(num_lists, num_entries int) ([][]lp.CCMessage, error) {
mlist := make([][]lp.CCMessage, 0)
for j := 0; j < num_lists; j++ {
out := make([]lp.CCMessage, 0)
for i := 0; i < num_entries; i++ {
var x lp.CCMessage
var err error = nil
switch {
case i%4 == 0:
x, err = lp.NewEvent("myevent", map[string]string{"type": "socket", "type-id": "0"}, map[string]string{}, "nothing happend", time.Now())
case i%4 == 1:
x, err = lp.NewMetric("mymetric", map[string]string{"type": "socket", "type-id": "0"}, map[string]string{"unit": "kByte"}, 12.145, time.Now())
case i%4 == 2:
x, err = lp.NewLog("mylog", map[string]string{"type": "socket", "type-id": "0"}, map[string]string{}, "disk status: OK", time.Now())
case i%4 == 3:
x, err = lp.NewGetControl("mycontrol", map[string]string{"type": "socket", "type-id": "0"}, map[string]string{}, time.Now())
}
if err == nil {
x.AddTag("hostname", "myhost")
out = append(out, x)
} else {
return nil, errors.New("failed to create message")
}
}
mlist = append(mlist, out)
}
return mlist, nil
}
func TestNewMessageProcessor(t *testing.T) {
_, err := NewMessageProcessor()
if err != nil {
t.Error(err.Error())
}
}
type Configs struct {
name string
config json.RawMessage
drop bool
errors bool
pre func(msg lp.CCMessage) error
check func(msg lp.CCMessage) error
}
var test_configs = []Configs{
{
name: "single_dropif_nomatch",
config: json.RawMessage(`{"drop_messages_if": [ "name == 'testname' && tags.type == 'socket' && tags.typeid % 2 == 1"]}`),
},
{
name: "drop_by_name",
config: json.RawMessage(`{"drop_messages": [ "net_bytes_in"]}`),
drop: true,
},
{
name: "drop_by_type_match",
config: json.RawMessage(`{"drop_by_message_type": [ "metric"]}`),
drop: true,
},
{
name: "drop_by_type_nomatch",
config: json.RawMessage(`{"drop_by_message_type": [ "event"]}`),
},
{
name: "single_dropif_match",
config: json.RawMessage(`{"drop_messages_if": [ "name == 'net_bytes_in' && tags.type == 'node'"]}`),
drop: true,
},
{
name: "double_dropif_match_nomatch",
config: json.RawMessage(`{"drop_messages_if": [ "name == 'net_bytes_in' && tags.type == 'node'", "name == 'testname' && tags.type == 'socket' && tags.typeid % 2 == 1"]}`),
drop: true,
},
{
name: "rename_simple",
config: json.RawMessage(`{"rename_messages": { "net_bytes_in" : "net_bytes_out", "rapl_power": "cpu_power"}}`),
check: func(msg lp.CCMessage) error {
if msg.Name() != "net_bytes_out" {
return errors.New("expected name net_bytes_out but still have net_bytes_in")
}
return nil
},
},
{
name: "rename_match",
config: json.RawMessage(`{"rename_messages_if": { "name == 'net_bytes_in'" : "net_bytes_out", "name == 'rapl_power'": "cpu_power"}}`),
check: func(msg lp.CCMessage) error {
if msg.Name() != "net_bytes_out" {
return errors.New("expected name net_bytes_out but still have net_bytes_in")
}
return nil
},
},
{
name: "rename_nomatch",
config: json.RawMessage(`{"rename_messages_if": { "name == 'net_bytes_out'" : "net_bytes_in", "name == 'rapl_power'": "cpu_power"}}`),
check: func(msg lp.CCMessage) error {
if msg.Name() != "net_bytes_in" {
return errors.New("expected name net_bytes_in but still have net_bytes_out")
}
return nil
},
},
{
name: "add_tag",
config: json.RawMessage(`{"add_tags_if": [{"if": "name == 'net_bytes_in'", "key" : "cluster", "value" : "mycluster"}]}`),
check: func(msg lp.CCMessage) error {
if !msg.HasTag("cluster") {
return errors.New("expected new tag 'cluster' but not present")
}
return nil
},
},
{
name: "del_tag",
config: json.RawMessage(`{"delete_tags_if": [{"if": "name == 'net_bytes_in'", "key" : "type"}]}`),
check: func(msg lp.CCMessage) error {
if msg.HasTag("type") {
return errors.New("expected to have no 'type' but still present")
}
return nil
},
},
{
name: "add_meta",
config: json.RawMessage(`{"add_meta_if": [{"if": "name == 'net_bytes_in'", "key" : "source", "value" : "example"}]}`),
check: func(msg lp.CCMessage) error {
if !msg.HasMeta("source") {
return errors.New("expected new tag 'source' but not present")
}
return nil
},
},
{
name: "del_meta",
config: json.RawMessage(`{"delete_meta_if": [{"if": "name == 'net_bytes_in'", "key" : "unit"}]}`),
check: func(msg lp.CCMessage) error {
if msg.HasMeta("unit") {
return errors.New("expected to have no 'unit' but still present")
}
return nil
},
},
{
name: "add_field",
config: json.RawMessage(`{"add_fields_if": [{"if": "name == 'net_bytes_in'", "key" : "myfield", "value" : "example"}]}`),
check: func(msg lp.CCMessage) error {
if !msg.HasField("myfield") {
return errors.New("expected new tag 'source' but not present")
}
return nil
},
},
{
name: "delete_fields_if_protected",
config: json.RawMessage(`{"delete_fields_if": [{"if": "name == 'net_bytes_in'", "key" : "value"}]}`),
errors: true,
check: func(msg lp.CCMessage) error {
if !msg.HasField("value") {
return errors.New("expected to still have 'value' field because it is a protected field key")
}
return nil
},
},
{
name: "delete_fields_if_unprotected",
config: json.RawMessage(`{"delete_fields_if": [{"if": "name == 'net_bytes_in'", "key" : "testfield"}]}`),
check: func(msg lp.CCMessage) error {
if msg.HasField("testfield") {
return errors.New("expected to still have 'testfield' field but should be deleted")
}
return nil
},
pre: func(msg lp.CCMessage) error {
msg.AddField("testfield", 4.123)
return nil
},
},
{
name: "single_change_prefix_match",
config: json.RawMessage(`{"change_unit_prefix": {"name == 'net_bytes_in' && tags.type == 'node'": "M"}}`),
check: func(msg lp.CCMessage) error {
if u, ok := msg.GetMeta("unit"); ok {
if u != "MB" {
return fmt.Errorf("expected unit MB but have %s", u)
}
} else if u, ok := msg.GetTag("unit"); ok {
if u != "MB" {
return fmt.Errorf("expected unit MB but have %s", u)
}
}
return nil
},
},
{
name: "normalize_units",
config: json.RawMessage(`{"normalize_units": true}`),
check: func(msg lp.CCMessage) error {
if u, ok := msg.GetMeta("unit"); ok {
if u != "B" {
return fmt.Errorf("expected unit B but have %s", u)
}
} else if u, ok := msg.GetTag("unit"); ok {
if u != "B" {
return fmt.Errorf("expected unit B but have %s", u)
}
}
return nil
},
},
{
name: "move_tag_to_meta",
config: json.RawMessage(`{"move_tag_to_meta_if": [{"if": "name == 'net_bytes_in'", "key" : "type-id", "value": "typeid"}]}`),
check: func(msg lp.CCMessage) error {
if msg.HasTag("type-id") || !msg.HasMeta("typeid") {
return errors.New("moving tag 'type-id' to meta 'typeid' failed")
}
return nil
},
pre: func(msg lp.CCMessage) error {
msg.AddTag("type-id", "0")
return nil
},
},
{
name: "move_tag_to_field",
config: json.RawMessage(`{"move_tag_to_field_if": [{"if": "name == 'net_bytes_in'", "key" : "type-id", "value": "typeid"}]}`),
check: func(msg lp.CCMessage) error {
if msg.HasTag("type-id") || !msg.HasField("typeid") {
return errors.New("moving tag 'type-id' to field 'typeid' failed")
}
return nil
},
pre: func(msg lp.CCMessage) error {
msg.AddTag("type-id", "0")
return nil
},
},
{
name: "move_meta_to_tag",
config: json.RawMessage(`{"move_meta_to_tag_if": [{"if": "name == 'net_bytes_in'", "key" : "unit", "value": "unit"}]}`),
check: func(msg lp.CCMessage) error {
if msg.HasMeta("unit") || !msg.HasTag("unit") {
return errors.New("moving meta 'unit' to tag 'unit' failed")
}
return nil
},
},
{
name: "move_meta_to_field",
config: json.RawMessage(`{"move_meta_to_field_if": [{"if": "name == 'net_bytes_in'", "key" : "unit", "value": "unit"}]}`),
check: func(msg lp.CCMessage) error {
if msg.HasMeta("unit") || !msg.HasField("unit") {
return errors.New("moving meta 'unit' to field 'unit' failed")
}
return nil
},
},
{
name: "move_field_to_tag",
config: json.RawMessage(`{"move_field_to_tag_if": [{"if": "name == 'net_bytes_in'", "key" : "myfield", "value": "field"}]}`),
check: func(msg lp.CCMessage) error {
if msg.HasField("myfield") || !msg.HasTag("field") {
return errors.New("moving meta 'myfield' to tag 'field' failed")
}
return nil
},
pre: func(msg lp.CCMessage) error {
msg.AddField("myfield", 12)
return nil
},
},
{
name: "move_field_to_meta",
config: json.RawMessage(`{"move_field_to_meta_if": [{"if": "name == 'net_bytes_in'", "key" : "myfield", "value": "field"}]}`),
check: func(msg lp.CCMessage) error {
if msg.HasField("myfield") || !msg.HasMeta("field") {
return errors.New("moving meta 'myfield' to meta 'field' failed")
}
return nil
},
pre: func(msg lp.CCMessage) error {
msg.AddField("myfield", 12)
return nil
},
},
}
func TestConfigList(t *testing.T) {
for _, c := range test_configs {
t.Run(c.name, func(t *testing.T) {
m, err := lp.NewMetric("net_bytes_in", map[string]string{"type": "node", "type-id": "0"}, map[string]string{"unit": "Byte"}, float64(1024.0), time.Now())
if err != nil {
t.Error(err.Error())
return
}
if c.pre != nil {
if err = c.pre(m); err != nil {
t.Errorf("error running pre-test function: %v", err.Error())
return
}
}
mp, err := NewMessageProcessor()
if err != nil {
t.Error(err.Error())
return
}
err = mp.FromConfigJSON(c.config)
if err != nil {
t.Error(err.Error())
return
}
//t.Log(m.ToLineProtocol(nil))
out, err := mp.ProcessMessage(m)
if err != nil && !c.errors {
cclog.SetDebug()
mp.ProcessMessage(m)
t.Error(err.Error())
return
}
if out == nil && !c.drop {
t.Error("fail, message should NOT be dropped but processor signalled dropping")
return
} else if out != nil && c.drop {
t.Error("fail, message should be dropped but processor signalled NO dropping")
return
}
// {
// if c.drop {
// t.Error("fail, message should be dropped but processor signalled NO dropping")
// } else {
// t.Error("fail, message should NOT be dropped but processor signalled dropping")
// }
// cclog.SetDebug()
// mp.ProcessMessage(m)
// return
// }
if c.check != nil {
if err := c.check(out); err != nil {
t.Errorf("check failed with %v", err.Error())
t.Log("Rerun with debugging")
cclog.SetDebug()
mp.ProcessMessage(m)
return
}
}
})
}
}
func BenchmarkProcessing(b *testing.B) {
mlist, err := generate_message_lists(b.N, 1000)
if err != nil {
b.Error(err.Error())
return
}
mp, err := NewMessageProcessor()
if err != nil {
b.Error(err.Error())
return
}
err = mp.FromConfigJSON(json.RawMessage(`{"move_meta_to_tag_if": [{"if" : "name == 'mymetric'", "key":"unit", "value":"unit"}]}`))
if err != nil {
b.Error(err.Error())
return
}
b.StartTimer()
for i := 0; i < b.N; i++ {
for _, m := range mlist[i] {
if _, err := mp.ProcessMessage(m); err != nil {
b.Errorf("failed processing message '%s': %v", m.ToLineProtocol(nil), err.Error())
return
}
}
}
b.StopTimer()
b.ReportMetric(float64(b.Elapsed())/float64(len(mlist)*b.N), "ns/message")
}

32
receivers/README.md
View File

@ -1,32 +0,0 @@
# CCMetric receivers
This folder contains the ReceiveManager and receiver implementations for the cc-metric-collector.
## Configuration
The configuration file for the receivers is a list of configurations. The `type` field in each specifies which receiver to initialize.
```json
{
"myreceivername" : {
"type": "receiver-type",
<receiver-specific configuration>
}
}
```
This allows to specify
## Available receivers
- [`nats`](./natsReceiver.md): Receive metrics from the NATS network
- [`prometheus`](./prometheusReceiver.md): Scrape data from a Prometheus client
- [`http`](./httpReceiver.md): Listen for HTTP Post requests transporting metrics in InfluxDB line protocol
- [`ipmi`](./ipmiReceiver.md): Read IPMI sensor readings
- [`redfish`](redfishReceiver.md) Use the Redfish (specification) to query thermal and power metrics
## Contributing own receivers
A receiver contains a few functions and is derived from the type `Receiver` (in `metricReceiver.go`):
For an example, check the [sample receiver](./sampleReceiver.go)

183
receivers/httpReceiver.go
View File

@ -1,183 +0,0 @@
package receivers
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"sync"
"time"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
)
const HTTP_RECEIVER_PORT = "8080"
type HttpReceiverConfig struct {
defaultReceiverConfig
Addr string `json:"address"`
Port string `json:"port"`
Path string `json:"path"`
// Maximum amount of time to wait for the next request when keep-alives are enabled
// should be larger than the measurement interval to keep the connection open
IdleTimeout string `json:"idle_timeout"`
idleTimeout time.Duration
// Controls whether HTTP keep-alives are enabled. By default, keep-alives are enabled
KeepAlivesEnabled bool `json:"keep_alives_enabled"`
// Basic authentication
Username string `json:"username"`
Password string `json:"password"`
useBasicAuth bool
}
type HttpReceiver struct {
receiver
//meta map[string]string
config HttpReceiverConfig
server *http.Server
wg sync.WaitGroup
}
func (r *HttpReceiver) Init(name string, config json.RawMessage) error {
r.name = fmt.Sprintf("HttpReceiver(%s)", name)
// Set default values
r.config.Port = HTTP_RECEIVER_PORT
r.config.KeepAlivesEnabled = true
// should be larger than the measurement interval to keep the connection open
r.config.IdleTimeout = "120s"
// Read config
if len(config) > 0 {
err := json.Unmarshal(config, &r.config)
if err != nil {
cclog.ComponentError(r.name, "Error reading config:", err.Error())
return err
}
}
if len(r.config.Port) == 0 {
return errors.New("not all configuration variables set required by HttpReceiver")
}
// Check idle timeout config
if len(r.config.IdleTimeout) > 0 {
t, err := time.ParseDuration(r.config.IdleTimeout)
if err == nil {
cclog.ComponentDebug(r.name, "idleTimeout", t)
r.config.idleTimeout = t
}
}
// Check basic authentication config
if len(r.config.Username) > 0 || len(r.config.Password) > 0 {
r.config.useBasicAuth = true
}
if r.config.useBasicAuth && len(r.config.Username) == 0 {
return errors.New("basic authentication requires username")
}
if r.config.useBasicAuth && len(r.config.Password) == 0 {
return errors.New("basic authentication requires password")
}
msgp, err := mp.NewMessageProcessor()
if err != nil {
return fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
r.mp = msgp
if len(r.config.MessageProcessor) > 0 {
err = r.mp.FromConfigJSON(r.config.MessageProcessor)
if err != nil {
return fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
r.mp.AddAddMetaByCondition("true", "source", r.name)
//r.meta = map[string]string{"source": r.name}
p := r.config.Path
if !strings.HasPrefix(p, "/") {
p = "/" + p
}
addr := fmt.Sprintf("%s:%s", r.config.Addr, r.config.Port)
uri := addr + p
cclog.ComponentDebug(r.name, "INIT", "listen on:", uri)
// Register handler function r.ServerHttp for path p in the DefaultServeMux
http.HandleFunc(p, r.ServerHttp)
// Create http server
r.server = &http.Server{
Addr: addr,
Handler: nil, // handler to invoke, http.DefaultServeMux if nil
IdleTimeout: r.config.idleTimeout,
}
r.server.SetKeepAlivesEnabled(r.config.KeepAlivesEnabled)
return nil
}
func (r *HttpReceiver) Start() {
cclog.ComponentDebug(r.name, "START")
r.wg.Add(1)
go func() {
err := r.server.ListenAndServe()
if err != nil && err.Error() != "http: Server closed" {
cclog.ComponentError(r.name, err.Error())
}
r.wg.Done()
}()
}
func (r *HttpReceiver) ServerHttp(w http.ResponseWriter, req *http.Request) {
// Check request method, only post method is handled
if req.Method != http.MethodPost {
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
return
}
// Check basic authentication
if r.config.useBasicAuth {
username, password, ok := req.BasicAuth()
if !ok || username != r.config.Username || password != r.config.Password {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
}
if r.sink != nil {
buf := make([]byte, 0, req.ContentLength)
len, err := req.Body.Read(buf)
if err == nil && len > 0 {
messages, err := lp.FromBytes(buf)
if err != nil {
msg := "ServerHttp: Failed to decode messages: " + err.Error()
cclog.ComponentError(r.name, msg)
http.Error(w, msg, http.StatusInternalServerError)
return
}
for _, y := range messages {
m, err := r.mp.ProcessMessage(y)
if err == nil && m != nil {
r.sink <- m
}
}
}
}
w.WriteHeader(http.StatusOK)
}
func (r *HttpReceiver) Close() {
r.server.Shutdown(context.Background())
}
func NewHttpReceiver(name string, config json.RawMessage) (Receiver, error) {
r := new(HttpReceiver)
err := r.Init(name, config)
return r, err
}

43
receivers/httpReceiver.md
View File

@ -1,43 +0,0 @@
## `http` receiver
The `http` receiver can be used receive metrics through HTTP POST requests.
### Configuration structure
```json
{
"<name>": {
"type": "http",
"address" : "",
"port" : "8080",
"path" : "/write",
"idle_timeout": "120s",
"username": "myUser",
"password": "myPW"
}
}
```
- `type`: makes the receiver a `http` receiver
- `address`: Listen address
- `port`: Listen port
- `path`: URL path for the write endpoint
- `idle_timeout`: Maximum amount of time to wait for the next request when keep-alives are enabled should be larger than the measurement interval to keep the connection open
- `keep_alives_enabled`: Controls whether HTTP keep-alives are enabled. By default, keep-alives are enabled.
- `username`: username for basic authentication
- `password`: password for basic authentication
The HTTP endpoint listens to `http://<address>:<port>/<path>`
### Debugging
- Install [curl](https://curl.se/)
- Use curl to send message to `http` receiver
```bash
curl http://localhost:8080/write \
--user "myUser:myPW" \
--data \
"myMetric,hostname=myHost,type=hwthread,type-id=0,unit=Hz value=400000i 1694777161164284635
myMetric,hostname=myHost,type=hwthread,type-id=1,unit=Hz value=400001i 1694777161164284635"
```

571
receivers/ipmiReceiver.go
View File

@ -1,571 +0,0 @@
package receivers
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"os/exec"
"regexp"
"strconv"
"strings"
"sync"
"time"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
"github.com/ClusterCockpit/cc-metric-collector/pkg/hostlist"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
)
type IPMIReceiverClientConfig struct {
// Hostname the IPMI service belongs to
Protocol string // Protocol / tool to use for IPMI sensor reading
DriverType string // Out of band IPMI driver
Fanout int // Maximum number of simultaneous IPMI connections
NumHosts int // Number of remote IPMI devices with the same configuration
IPMIHosts string // List of remote IPMI devices to communicate with
IPMI2HostMapping map[string]string // Mapping between IPMI device name and host name
Username string // User name to authenticate with
Password string // Password to use for authentication
CLIOptions []string // Additional command line options for ipmi-sensors
isExcluded map[string]bool // is metric excluded
mp mp.MessageProcessor
}
type IPMIReceiver struct {
receiver
config struct {
defaultReceiverConfig
Interval time.Duration
// Client config for each IPMI hosts
ClientConfigs []IPMIReceiverClientConfig
}
// Storage for static information
//meta map[string]string
done chan bool // channel to finish / stop IPMI receiver
wg sync.WaitGroup // wait group for IPMI receiver
mp mp.MessageProcessor
}
// doReadMetrics reads metrics from all configure IPMI hosts.
func (r *IPMIReceiver) doReadMetric() {
for i := range r.config.ClientConfigs {
clientConfig := &r.config.ClientConfigs[i]
var cmd_options []string
if clientConfig.Protocol == "ipmi-sensors" {
cmd_options = append(cmd_options,
"--always-prefix",
"--sdr-cache-recreate",
// Attempt to interpret OEM data, such as event data, sensor readings, or general extra info
"--interpret-oem-data",
// Ignore not-available (i.e. N/A) sensors in output
"--ignore-not-available-sensors",
// Ignore unrecognized sensor events
"--ignore-unrecognized-events",
// Output fields in comma separated format
"--comma-separated-output",
// Do not output column headers
"--no-header-output",
// Output non-abbreviated units (e.g. 'Amps' instead of 'A').
// May aid in disambiguation of units (e.g. 'C' for Celsius or Coulombs).
"--non-abbreviated-units",
"--fanout", fmt.Sprint(clientConfig.Fanout),
"--driver-type", clientConfig.DriverType,
"--hostname", clientConfig.IPMIHosts,
"--username", clientConfig.Username,
"--password", clientConfig.Password,
)
cmd_options := append(cmd_options, clientConfig.CLIOptions...)
command := exec.Command("ipmi-sensors", cmd_options...)
stdout, _ := command.StdoutPipe()
errBuf := new(bytes.Buffer)
command.Stderr = errBuf
// start command
if err := command.Start(); err != nil {
cclog.ComponentError(
r.name,
fmt.Sprintf("doReadMetric(): Failed to start command \"%s\": %v", command.String(), err),
)
continue
}
// Read command output
const (
idxID = iota
idxName
idxType
idxReading
idxUnits
idxEvent
)
numPrefixRegex := regexp.MustCompile("^[[:digit:]][[:digit:]]-(.*)$")
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
// Read host
v1 := strings.Split(scanner.Text(), ": ")
if len(v1) != 2 {
continue
}
host, ok := clientConfig.IPMI2HostMapping[v1[0]]
if !ok {
continue
}
// Read sensors
v2 := strings.Split(v1[1], ",")
if len(v2) != 6 {
continue
}
// Skip sensors with non available sensor readings
if v2[idxReading] == "N/A" {
continue
}
metric := strings.ToLower(v2[idxType])
name := strings.ToLower(
strings.Replace(
strings.TrimSpace(
v2[idxName]), " ", "_", -1))
// remove prefix enumeration like 01-...
if v := numPrefixRegex.FindStringSubmatch(name); v != nil {
name = v[1]
}
unit := v2[idxUnits]
if unit == "Watts" {
// Power
metric = "power"
name = strings.TrimSuffix(name, "_power")
name = strings.TrimSuffix(name, "_pwr")
name = strings.TrimPrefix(name, "pwr_")
} else if metric == "voltage" &&
unit == "Volts" {
// Voltage
name = strings.TrimPrefix(name, "volt_")
} else if metric == "current" &&
unit == "Amps" {
// Current
unit = "Ampere"
} else if metric == "temperature" &&
unit == "degrees C" {
// Temperature
name = strings.TrimSuffix(name, "_temp")
unit = "degC"
} else if metric == "temperature" &&
unit == "degrees F" {
// Temperature
name = strings.TrimSuffix(name, "_temp")
unit = "degF"
} else if metric == "fan" && unit == "RPM" {
// Fan speed
metric = "fan_speed"
name = strings.TrimSuffix(name, "_tach")
name = strings.TrimPrefix(name, "spd_")
} else if (metric == "cooling device" ||
metric == "other units based sensor") &&
name == "system_air_flow" &&
unit == "CFM" {
// Air flow
metric = "air_flow"
name = strings.TrimSuffix(name, "_air_flow")
unit = "CubicFeetPerMinute"
} else if (metric == "processor" ||
metric == "other units based sensor") &&
(name == "cpu_utilization" ||
name == "io_utilization" ||
name == "mem_utilization" ||
name == "sys_utilization") &&
(unit == "unspecified" ||
unit == "%") {
// Utilization
metric = "utilization"
name = strings.TrimSuffix(name, "_utilization")
unit = "percent"
} else {
if false {
// Debug output for unprocessed metrics
fmt.Printf(
"host: '%s', metric: '%s', name: '%s', unit: '%s'\n",
host, metric, name, unit)
}
continue
}
// Skip excluded metrics
if clientConfig.isExcluded[metric] {
continue
}
// Parse sensor value
value, err := strconv.ParseFloat(v2[idxReading], 64)
if err != nil {
continue
}
y, err := lp.NewMessage(
metric,
map[string]string{
"hostname": host,
"type": "node",
"name": name,
},
map[string]string{
"source": r.name,
"group": "IPMI",
"unit": unit,
},
map[string]interface{}{
"value": value,
},
time.Now())
if err == nil {
mc, err := clientConfig.mp.ProcessMessage(y)
if err == nil && mc != nil {
m, err := r.mp.ProcessMessage(mc)
if err == nil && m != nil {
r.sink <- m
}
}
}
}
// Wait for command end
if err := command.Wait(); err != nil {
errMsg, _ := io.ReadAll(errBuf)
cclog.ComponentError(
r.name,
fmt.Sprintf("doReadMetric(): Failed to wait for the end of command \"%s\": %v\n",
strings.Replace(command.String(), clientConfig.Password, "<PW>", -1), err),
fmt.Sprintf("doReadMetric(): command stderr: \"%s\"\n", string(errMsg)),
)
}
}
}
}
func (r *IPMIReceiver) Start() {
cclog.ComponentDebug(r.name, "START")
// Start IPMI receiver
r.wg.Add(1)
go func() {
defer r.wg.Done()
// Create ticker
ticker := time.NewTicker(r.config.Interval)
defer ticker.Stop()
for {
r.doReadMetric()
select {
case tickerTime := <-ticker.C:
// Check if we missed the ticker event
if since := time.Since(tickerTime); since > 5*time.Second {
cclog.ComponentInfo(r.name, "Missed ticker event for more then", since)
}
// process ticker event -> continue
continue
case <-r.done:
// process done event
return
}
}
}()
cclog.ComponentDebug(r.name, "STARTED")
}
// Close receiver: close network connection, close files, close libraries, ...
func (r *IPMIReceiver) Close() {
cclog.ComponentDebug(r.name, "CLOSE")
// Send the signal and wait
close(r.done)
r.wg.Wait()
cclog.ComponentDebug(r.name, "DONE")
}
// NewIPMIReceiver creates a new instance of the redfish receiver
// Initialize the receiver by giving it a name and reading in the config JSON
func NewIPMIReceiver(name string, config json.RawMessage) (Receiver, error) {
var err error
r := new(IPMIReceiver)
// Config options from config file
configJSON := struct {
defaultReceiverConfig
// How often the IPMI sensor metrics should be read and send to the sink (default: 30 s)
IntervalString string `json:"interval,omitempty"`
// Maximum number of simultaneous IPMI connections (default: 64)
Fanout int `json:"fanout,omitempty"`
// Out of band IPMI driver (default: LAN_2_0)
DriverType string `json:"driver_type,omitempty"`
// Default client username, password and endpoint
Username *string `json:"username"` // User name to authenticate with
Password *string `json:"password"` // Password to use for authentication
Endpoint *string `json:"endpoint"` // URL of the IPMI device
// Globally excluded metrics
ExcludeMetrics []string `json:"exclude_metrics,omitempty"`
ClientConfigs []struct {
Fanout int `json:"fanout,omitempty"` // Maximum number of simultaneous IPMI connections (default: 64)
DriverType string `json:"driver_type,omitempty"` // Out of band IPMI driver (default: LAN_2_0)
HostList string `json:"host_list"` // List of hosts with the same client configuration
Username *string `json:"username"` // User name to authenticate with
Password *string `json:"password"` // Password to use for authentication
Endpoint *string `json:"endpoint"` // URL of the IPMI service
// Per client excluded metrics
ExcludeMetrics []string `json:"exclude_metrics,omitempty"`
// Additional command line options for ipmi-sensors
CLIOptions []string `json:"cli_options,omitempty"`
MessageProcessor json.RawMessage `json:"process_messages,omitempty"`
} `json:"client_config"`
}{
// Set defaults values
// Allow overwriting these defaults by reading config JSON
Fanout: 64,
DriverType: "LAN_2_0",
IntervalString: "30s",
}
// Set name of IPMIReceiver
r.name = fmt.Sprintf("IPMIReceiver(%s)", name)
// Create done channel
r.done = make(chan bool)
p, err := mp.NewMessageProcessor()
if err != nil {
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
r.mp = p
// Set static information
//r.meta = map[string]string{"source": r.name}
r.mp.AddAddMetaByCondition("true", "source", r.name)
// Read the IPMI receiver specific JSON config
if len(config) > 0 {
d := json.NewDecoder(bytes.NewReader(config))
d.DisallowUnknownFields()
if err := d.Decode(&configJSON); err != nil {
cclog.ComponentError(r.name, "Error reading config:", err.Error())
return nil, err
}
}
if len(r.config.MessageProcessor) > 0 {
err = r.mp.FromConfigJSON(r.config.MessageProcessor)
if err != nil {
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
// Convert interval string representation to duration
r.config.Interval, err = time.ParseDuration(configJSON.IntervalString)
if err != nil {
err := fmt.Errorf(
"failed to parse duration string interval='%s': %w",
configJSON.IntervalString,
err,
)
cclog.Error(r.name, err)
return nil, err
}
// Create client config from JSON config
totalNumHosts := 0
for i := range configJSON.ClientConfigs {
clientConfigJSON := &configJSON.ClientConfigs[i]
var endpoint string
if clientConfigJSON.Endpoint != nil {
endpoint = *clientConfigJSON.Endpoint
} else if configJSON.Endpoint != nil {
endpoint = *configJSON.Endpoint
} else {
err := fmt.Errorf("client config number %v requires endpoint", i)
cclog.ComponentError(r.name, err)
return nil, err
}
fanout := configJSON.Fanout
if clientConfigJSON.Fanout != 0 {
fanout = clientConfigJSON.Fanout
}
driverType := configJSON.DriverType
if clientConfigJSON.DriverType != "" {
driverType = clientConfigJSON.DriverType
}
if driverType != "LAN" && driverType != "LAN_2_0" {
err := fmt.Errorf("client config number %v has invalid driver type %s", i, driverType)
cclog.ComponentError(r.name, err)
return nil, err
}
var protocol string
var host_pattern string
if e := strings.Split(endpoint, "://"); len(e) == 2 {
protocol = e[0]
host_pattern = e[1]
} else {
err := fmt.Errorf("client config number %v has invalid endpoint %s", i, endpoint)
cclog.ComponentError(r.name, err)
return nil, err
}
var username string
if clientConfigJSON.Username != nil {
username = *clientConfigJSON.Username
} else if configJSON.Username != nil {
username = *configJSON.Username
} else {
err := fmt.Errorf("client config number %v requires username", i)
cclog.ComponentError(r.name, err)
return nil, err
}
var password string
if clientConfigJSON.Password != nil {
password = *clientConfigJSON.Password
} else if configJSON.Password != nil {
password = *configJSON.Password
} else {
err := fmt.Errorf("client config number %v requires password", i)
cclog.ComponentError(r.name, err)
return nil, err
}
// Create mapping between IPMI host name and node host name
// This also guaranties that all IPMI host names are unique
ipmi2HostMapping := make(map[string]string)
hostList, err := hostlist.Expand(clientConfigJSON.HostList)
if err != nil {
err := fmt.Errorf("client config number %d failed to parse host list %s: %v",
i, clientConfigJSON.HostList, err)
cclog.ComponentError(r.name, err)
return nil, err
}
for _, host := range hostList {
ipmiHost := strings.Replace(host_pattern, "%h", host, -1)
ipmi2HostMapping[ipmiHost] = host
}
numHosts := len(ipmi2HostMapping)
totalNumHosts += numHosts
ipmiHostList := make([]string, 0, numHosts)
for ipmiHost := range ipmi2HostMapping {
ipmiHostList = append(ipmiHostList, ipmiHost)
}
// Additional command line options
for _, v := range clientConfigJSON.CLIOptions {
switch {
case v == "-u" || strings.HasPrefix(v, "--username"):
err := fmt.Errorf("client config number %v: do not set username in cli_options. Use json config username instead", i)
cclog.ComponentError(r.name, err)
return nil, err
case v == "-p" || strings.HasPrefix(v, "--password"):
err := fmt.Errorf("client config number %v: do not set password in cli_options. Use json config password instead", i)
cclog.ComponentError(r.name, err)
return nil, err
case v == "-h" || strings.HasPrefix(v, "--hostname"):
err := fmt.Errorf("client config number %v: do not set hostname in cli_options. Use json config host_list instead", i)
cclog.ComponentError(r.name, err)
return nil, err
case v == "-D" || strings.HasPrefix(v, "--driver-type"):
err := fmt.Errorf("client config number %v: do not set driver type in cli_options. Use json config driver_type instead", i)
cclog.ComponentError(r.name, err)
return nil, err
case v == "-F" || strings.HasPrefix(v, " --fanout"):
err := fmt.Errorf("client config number %v: do not set fanout in cli_options. Use json config fanout instead", i)
cclog.ComponentError(r.name, err)
return nil, err
case v == "--always-prefix" ||
v == "--sdr-cache-recreate" ||
v == "--interpret-oem-data" ||
v == "--ignore-not-available-sensors" ||
v == "--ignore-unrecognized-events" ||
v == "--comma-separated-output" ||
v == "--no-header-output" ||
v == "--non-abbreviated-units":
err := fmt.Errorf("client config number %v: Do not use option %s in cli_options, it is used internally", i, v)
cclog.ComponentError(r.name, err)
return nil, err
}
}
cliOptions := make([]string, 0)
cliOptions = append(cliOptions, clientConfigJSON.CLIOptions...)
// Is metrics excluded globally or per client
isExcluded := make(map[string]bool)
for _, key := range clientConfigJSON.ExcludeMetrics {
isExcluded[key] = true
}
for _, key := range configJSON.ExcludeMetrics {
isExcluded[key] = true
}
p, err := mp.NewMessageProcessor()
if err != nil {
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
if len(clientConfigJSON.MessageProcessor) > 0 {
err = p.FromConfigJSON(clientConfigJSON.MessageProcessor)
if err != nil {
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
r.config.ClientConfigs = append(
r.config.ClientConfigs,
IPMIReceiverClientConfig{
Protocol: protocol,
Fanout: fanout,
DriverType: driverType,
NumHosts: numHosts,
IPMIHosts: strings.Join(ipmiHostList, ","),
IPMI2HostMapping: ipmi2HostMapping,
Username: username,
Password: password,
CLIOptions: cliOptions,
isExcluded: isExcluded,
mp: p,
})
}
if totalNumHosts == 0 {
err := fmt.Errorf("at least one IPMI host config is required")
cclog.ComponentError(r.name, err)
return nil, err
}
cclog.ComponentInfo(r.name, "monitoring", totalNumHosts, "IPMI hosts")
return r, nil
}

48
receivers/ipmiReceiver.md
View File

@ -1,48 +0,0 @@
## IPMI Receiver
The IPMI Receiver uses `ipmi-sensors` from the [FreeIPMI](https://www.gnu.org/software/freeipmi/) project to read IPMI sensor readings and sensor data repository (SDR) information. The available metrics depend on the sensors provided by the hardware vendor but typically contain temperature, fan speed, voltage and power metrics.
### Configuration structure
```json
{
"<IPMI receiver name>": {
"type": "ipmi",
"interval": "30s",
"fanout": 256,
"username": "<Username>",
"password": "<Password>",
"endpoint": "ipmi-sensors://%h-bmc",
"exclude_metrics": [ "fan_speed", "voltage" ],
"client_config": [
{
"host_list": "n[1,2-4]"
},
{
"host_list": "n[5-6]",
"driver_type": "LAN",
"cli_options": [ "--workaround-flags=..." ],
"password": "<Password 2>"
}
]
}
}
```
Global settings:
- `interval`: How often the IPMI sensor metrics should be read and send to the sink (default: 30 s)
Global and per IPMI device settings (per IPMI device settings overwrite the global settings):
- `exclude_metrics`: list of excluded metrics e.g. fan_speed, power, temperature, utilization, voltage
- `fanout`: Maximum number of simultaneous IPMI connections (default: 64)
- `driver_type`: Out of band IPMI driver (default: LAN_2_0)
- `username`: User name to authenticate with
- `password`: Password to use for authentication
- `endpoint`: URL of the IPMI device (placeholder `%h` gets replaced by the hostname)
Per IPMI device settings:
- `host_list`: List of hosts with the same client configuration
- `cli_options`: Additional command line options for ipmi-sensors

45
receivers/metricReceiver.go
View File

@ -1,45 +0,0 @@
package receivers
import (
"encoding/json"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
)
type defaultReceiverConfig struct {
Type string `json:"type"`
MessageProcessor json.RawMessage `json:"process_messages,omitempty"`
}
// Receiver configuration: Listen address, port
type ReceiverConfig struct {
Addr string `json:"address"`
Port string `json:"port"`
Database string `json:"database"`
Organization string `json:"organization,omitempty"`
Type string `json:"type"`
}
type receiver struct {
name string
sink chan lp.CCMessage
mp mp.MessageProcessor
}
type Receiver interface {
Start()
Close() // Close / finish metric receiver
Name() string // Name of the metric receiver
SetSink(sink chan lp.CCMessage) // Set sink channel
}
// Name returns the name of the metric receiver
func (r *receiver) Name() string {
return r.name
}
// SetSink set the sink channel
func (r *receiver) SetSink(sink chan lp.CCMessage) {
r.sink = sink
}

127
receivers/natsReceiver.go
View File

@ -1,127 +0,0 @@
package receivers
import (
"encoding/json"
"errors"
"fmt"
"os"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
nats "github.com/nats-io/nats.go"
)
type NatsReceiverConfig struct {
defaultReceiverConfig
Addr string `json:"address"`
Port string `json:"port"`
Subject string `json:"subject"`
User string `json:"user,omitempty"`
Password string `json:"password,omitempty"`
NkeyFile string `json:"nkey_file,omitempty"`
}
type NatsReceiver struct {
receiver
nc *nats.Conn
//meta map[string]string
config NatsReceiverConfig
}
// Start subscribes to the configured NATS subject
// Messages wil be handled by r._NatsReceive
func (r *NatsReceiver) Start() {
cclog.ComponentDebug(r.name, "START")
r.nc.Subscribe(r.config.Subject, r._NatsReceive)
}
// _NatsReceive receives subscribed messages from the NATS server
func (r *NatsReceiver) _NatsReceive(m *nats.Msg) {
if r.sink != nil {
messages, err := lp.FromBytes(m.Data)
if err != nil {
msg := "_NatsReceive: Failed to decode messages: " + err.Error()
cclog.ComponentError(r.name, msg)
}
for _, y := range messages {
m, err := r.mp.ProcessMessage(y)
if err == nil && m != nil && r.sink != nil {
r.sink <- m
}
}
}
}
// Close closes the connection to the NATS server
func (r *NatsReceiver) Close() {
if r.nc != nil {
cclog.ComponentDebug(r.name, "CLOSE")
r.nc.Close()
}
}
// NewNatsReceiver creates a new Receiver which subscribes to messages from a NATS server
func NewNatsReceiver(name string, config json.RawMessage) (Receiver, error) {
var uinfo nats.Option = nil
r := new(NatsReceiver)
r.name = fmt.Sprintf("NatsReceiver(%s)", name)
// Read configuration file, allow overwriting default config
r.config.Addr = "localhost"
r.config.Port = "4222"
if len(config) > 0 {
err := json.Unmarshal(config, &r.config)
if err != nil {
cclog.ComponentError(r.name, "Error reading config:", err.Error())
return nil, err
}
}
if len(r.config.Addr) == 0 ||
len(r.config.Port) == 0 ||
len(r.config.Subject) == 0 {
return nil, errors.New("not all configuration variables set required by NatsReceiver")
}
p, err := mp.NewMessageProcessor()
if err != nil {
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
r.mp = p
if len(r.config.MessageProcessor) > 0 {
err = r.mp.FromConfigJSON(r.config.MessageProcessor)
if err != nil {
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
// Set metadata
// r.meta = map[string]string{
// "source": r.name,
// }
r.mp.AddAddMetaByCondition("true", "source", r.name)
if len(r.config.User) > 0 && len(r.config.Password) > 0 {
uinfo = nats.UserInfo(r.config.User, r.config.Password)
} else if len(r.config.NkeyFile) > 0 {
_, err := os.Stat(r.config.NkeyFile)
if err == nil {
uinfo = nats.UserCredentials(r.config.NkeyFile)
} else {
cclog.ComponentError(r.name, "NKEY file", r.config.NkeyFile, "does not exist: %v", err.Error())
return nil, err
}
}
// Connect to NATS server
url := fmt.Sprintf("nats://%s:%s", r.config.Addr, r.config.Port)
cclog.ComponentDebug(r.name, "NewNatsReceiver", url, "Subject", r.config.Subject)
if nc, err := nats.Connect(url, uinfo); err == nil {
r.nc = nc
} else {
r.nc = nil
return nil, err
}
return r, nil
}

56
receivers/natsReceiver.md
View File

@ -1,56 +0,0 @@
## `nats` receiver
The `nats` receiver can be used receive metrics from the NATS network. The `nats` receiver subscribes to the topic `database` and listens on `address` and `port` for metrics in the InfluxDB line protocol.
### Configuration structure
```json
{
"<name>": {
"type": "nats",
"address" : "nats-server.example.org",
"port" : "4222",
"subject" : "subject",
"user": "natsuser",
"password": "natssecret",
"nkey_file": "/path/to/nkey_file"
}
}
```
- `type`: makes the receiver a `nats` receiver
- `address`: Address of the NATS control server
- `port`: Port of the NATS control server
- `subject`: Subscribes to this subject and receive metrics
- `user`: Connect to nats using this user
- `password`: Connect to nats using this password
- `nkey_file`: Path to credentials file with NKEY
### Debugging
- Install NATS server and command line client
- Start NATS server
```bash
nats-server --net nats-server.example.org --port 4222
```
- Check NATS server works as expected
```bash
nats --server=nats-server-db.example.org:4222 server check
```
- Use NATS command line client to subscribe to all messages
```bash
nats --server=nats-server-db.example.org:4222 sub ">"
```
- Use NATS command line client to send message to NATS receiver
```bash
nats --server=nats-server-db.example.org:4222 pub subject \
"myMetric,hostname=myHost,type=hwthread,type-id=0,unit=Hz value=400000i 1694777161164284635
myMetric,hostname=myHost,type=hwthread,type-id=1,unit=Hz value=400001i 1694777161164284635"
```

122
receivers/prometheusReceiver.go
View File

@ -1,122 +0,0 @@
package receivers
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"strconv"
"strings"
"sync"
"time"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
)
type PrometheusReceiverConfig struct {
defaultReceiverConfig
Addr string `json:"address"`
Port string `json:"port"`
Path string `json:"path"`
Interval string `json:"interval"`
SSL bool `json:"ssl"`
}
type PrometheusReceiver struct {
receiver
meta map[string]string
config PrometheusReceiverConfig
interval time.Duration
done chan bool
wg sync.WaitGroup
ticker *time.Ticker
uri string
}
func (r *PrometheusReceiver) Start() {
cclog.ComponentDebug(r.name, "START", r.uri)
r.wg.Add(1)
r.ticker = time.NewTicker(r.interval)
go func() {
for {
select {
case <-r.done:
r.wg.Done()
return
case t := <-r.ticker.C:
resp, err := http.Get(r.uri)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "#") {
continue
}
lineSplit := strings.Fields(line)
// separate metric name from tags (labels in Prometheus)
tags := map[string]string{}
name := lineSplit[0]
if sindex := strings.Index(name, "{"); sindex >= 0 {
eindex := strings.Index(name, "}")
for _, kv := range strings.Split(name[sindex+1:eindex], ",") {
eq := strings.Index(kv, "=")
tags[kv[0:eq]] = strings.Trim(kv[eq+1:], "\"")
}
name = lineSplit[0][0:sindex]
}
value, err := strconv.ParseFloat(lineSplit[1], 64)
if err == nil {
y, err := lp.NewMessage(name, tags, r.meta, map[string]interface{}{"value": value}, t)
if err == nil {
r.sink <- y
}
}
}
}
}
}()
}
func (r *PrometheusReceiver) Close() {
cclog.ComponentDebug(r.name, "CLOSE")
r.done <- true
r.wg.Wait()
}
func NewPrometheusReceiver(name string, config json.RawMessage) (Receiver, error) {
r := new(PrometheusReceiver)
r.name = fmt.Sprintf("PrometheusReceiver(%s)", name)
if len(config) > 0 {
err := json.Unmarshal(config, &r.config)
if err != nil {
cclog.ComponentError(r.name, "Error reading config:", err.Error())
return nil, err
}
}
if len(r.config.Addr) == 0 ||
len(r.config.Port) == 0 ||
len(r.config.Interval) == 0 {
return nil, errors.New("not all configuration variables set required by PrometheusReceiver (address and port)")
}
if len(r.config.Interval) > 0 {
t, err := time.ParseDuration(r.config.Interval)
if err == nil {
r.interval = t
}
}
r.meta = map[string]string{"source": r.name}
proto := "http"
if r.config.SSL {
proto = "https"
}
r.uri = fmt.Sprintf("%s://%s:%s/%s", proto, r.config.Addr, r.config.Port, r.config.Path)
return r, nil
}

27
receivers/prometheusReceiver.md
View File

@ -1,27 +0,0 @@
## `prometheus` receiver
The `prometheus` receiver can be used to scrape the metrics of a single `prometheus` client. It does **not** use any official Golang library but making simple HTTP get requests and parse the response.
### Configuration structure
```json
{
"<name>": {
"type": "prometheus",
"address" : "testpromhost",
"port" : "12345",
"path" : "/prometheus",
"interval": "5s",
"ssl" : true,
}
}
```
- `type`: makes the receiver a `prometheus` receiver
- `address`: Hostname or IP of the Prometheus agent
- `port`: Port of Prometheus agent
- `path`: Path to the Prometheus endpoint
- `interval`: Scrape the Prometheus endpoint in this interval (default '5s')
- `ssl`: Use SSL or not
The receiver requests data from `http(s)://<address>:<port>/<path>`.

122
receivers/receiveManager.go
View File

@ -1,122 +0,0 @@
package receivers
import (
"encoding/json"
"fmt"
"os"
"sync"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
)
var AvailableReceivers = map[string]func(name string, config json.RawMessage) (Receiver, error){
"http": NewHttpReceiver,
"ipmi": NewIPMIReceiver,
"nats": NewNatsReceiver,
"redfish": NewRedfishReceiver,
}
type receiveManager struct {
inputs []Receiver
output chan lp.CCMessage
config []json.RawMessage
}
type ReceiveManager interface {
Init(wg *sync.WaitGroup, receiverConfigFile string) error
AddInput(name string, rawConfig json.RawMessage) error
AddOutput(output chan lp.CCMessage)
Start()
Close()
}
func (rm *receiveManager) Init(wg *sync.WaitGroup, receiverConfigFile string) error {
// Initialize struct fields
rm.inputs = make([]Receiver, 0)
rm.output = nil
rm.config = make([]json.RawMessage, 0)
configFile, err := os.Open(receiverConfigFile)
if err != nil {
cclog.ComponentError("ReceiveManager", err.Error())
return err
}
defer configFile.Close()
jsonParser := json.NewDecoder(configFile)
var rawConfigs map[string]json.RawMessage
err = jsonParser.Decode(&rawConfigs)
if err != nil {
cclog.ComponentError("ReceiveManager", err.Error())
return err
}
for name, raw := range rawConfigs {
rm.AddInput(name, raw)
}
return nil
}
func (rm *receiveManager) Start() {
cclog.ComponentDebug("ReceiveManager", "START")
for _, r := range rm.inputs {
cclog.ComponentDebug("ReceiveManager", "START", r.Name())
r.Start()
}
cclog.ComponentDebug("ReceiveManager", "STARTED")
}
func (rm *receiveManager) AddInput(name string, rawConfig json.RawMessage) error {
var config defaultReceiverConfig
err := json.Unmarshal(rawConfig, &config)
if err != nil {
cclog.ComponentError("ReceiveManager", "SKIP", config.Type, "JSON config error:", err.Error())
return err
}
if config.Type == "" {
cclog.ComponentError("ReceiveManager", "SKIP", "JSON config for receiver", name, "does not contain a receiver type")
return fmt.Errorf("JSON config for receiver %s does not contain a receiver type", name)
}
if _, found := AvailableReceivers[config.Type]; !found {
cclog.ComponentError("ReceiveManager", "SKIP", "unknown receiver type:", config.Type)
return fmt.Errorf("unknown receiver type: %s", config.Type)
}
r, err := AvailableReceivers[config.Type](name, rawConfig)
if err != nil {
cclog.ComponentError("ReceiveManager", "SKIP", name, "initialization failed:", err.Error())
return err
}
rm.inputs = append(rm.inputs, r)
rm.config = append(rm.config, rawConfig)
cclog.ComponentDebug("ReceiveManager", "ADD RECEIVER", r.Name())
return nil
}
func (rm *receiveManager) AddOutput(output chan lp.CCMessage) {
rm.output = output
for _, r := range rm.inputs {
r.SetSink(rm.output)
}
}
func (rm *receiveManager) Close() {
cclog.ComponentDebug("ReceiveManager", "CLOSE")
// Close all receivers
for _, r := range rm.inputs {
cclog.ComponentDebug("ReceiveManager", "CLOSE", r.Name())
r.Close()
}
cclog.ComponentDebug("ReceiveManager", "DONE")
}
func New(wg *sync.WaitGroup, receiverConfigFile string) (ReceiveManager, error) {
r := new(receiveManager)
err := r.Init(wg, receiverConfigFile)
if err != nil {
return nil, err
}
return r, err
}

1055
receivers/redfishReceiver.go
View File

File diff suppressed because it is too large Load Diff

65
receivers/redfishReceiver.md
View File

@ -1,65 +0,0 @@
## Redfish receiver
The Redfish receiver uses the [Redfish (specification)](https://www.dmtf.org/standards/redfish) to query thermal and power metrics. Thermal metrics may include various fan speeds and temperatures. Power metrics may include the current power consumption of various hardware components. It may also include the minimum, maximum and average power consumption of these components in a given time interval. The receiver will poll each configured redfish device once in a given interval. Multiple devices can be accessed in parallel to increase throughput.
### Configuration structure
```json
{
"<redfish receiver name>": {
"type": "redfish",
"username": "<Username>",
"password": "<Password>",
"endpoint": "https://%h-bmc",
"exclude_metrics": [ "min_consumed_watts" ],
"client_config": [
{
"host_list": "n[1,2-4]"
},
{
"host_list": "n5",
"disable_power_metrics": true,
"disable_processor_metrics": true,
"disable_thermal_metrics": true
},
{
"host_list": "n6" ],
"username": "<Username 2>",
"password": "<Password 2>",
"endpoint": "https://%h-BMC",
"disable_sensor_metrics": true
}
]
}
}
```
Global settings:
- `fanout`: Maximum number of simultaneous redfish connections (default: 64)
- `interval`: How often the redfish power metrics should be read and send to the sink (default: 30 s)
- `http_insecure`: Control whether a client verifies the server's certificate (default: true == do not verify server's certificate)
- `http_timeout`: Time limit for requests made by this HTTP client (default: 10 s)
Global and per redfish device settings (per redfish device settings overwrite the global settings):
- `disable_power_metrics`:
disable collection of power metrics
(`/redfish/v1/Chassis/{ChassisId}/Power`)
- `disable_processor_metrics`:
disable collection of processor metrics
(`/redfish/v1/Systems/{ComputerSystemId}/Processors/{ProcessorId}/ProcessorMetrics`)
- `disable_sensors`:
disable collection of fan, power and thermal sensor metrics
(`/redfish/v1/Chassis/{ChassisId}/Sensors/{SensorId}`)
- `disable_thermal_metrics`:
disable collection of thermal metrics
(`/redfish/v1/Chassis/{ChassisId}/Thermal`)
- `exclude_metrics`: list of excluded metrics
- `username`: User name to authenticate with
- `password`: Password to use for authentication
- `endpoint`: URL of the redfish service (placeholder `%h` gets replaced by the hostname)
Per redfish device settings:
- `host_list`: List of hosts with the same client configuration

122
receivers/sampleReceiver.go
View File

@ -1,122 +0,0 @@
package receivers
import (
"encoding/json"
"fmt"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
)
// SampleReceiver configuration: receiver type, listen address, port
// The defaultReceiverConfig contains the keys 'type' and 'process_messages'
type SampleReceiverConfig struct {
defaultReceiverConfig
Addr string `json:"address"`
Port string `json:"port"`
}
type SampleReceiver struct {
receiver
config SampleReceiverConfig
// Storage for static information
// Use in case of own go routine
// done chan bool
// wg sync.WaitGroup
}
// Implement functions required for Receiver interface
// Start(), Close()
// See: metricReceiver.go
func (r *SampleReceiver) Start() {
cclog.ComponentDebug(r.name, "START")
// Start server process like http.ListenAndServe()
// or use own go routine but always make sure it exits
// as soon as it gets the signal of the r.done channel
//
// r.done = make(chan bool)
// r.wg.Add(1)
// go func() {
// defer r.wg.Done()
//
// // Create ticker
// ticker := time.NewTicker(30 * time.Second)
// defer ticker.Stop()
//
// for {
// readMetric()
// select {
// case <-ticker.C:
// // process ticker event -> continue
// continue
// case <-r.done:
// return
// }
// }
// }()
}
// Close receiver: close network connection, close files, close libraries, ...
func (r *SampleReceiver) Close() {
cclog.ComponentDebug(r.name, "CLOSE")
// Close server like http.Shutdown()
// in case of own go routine, send the signal and wait
// r.done <- true
// r.wg.Wait()
}
// New function to create a new instance of the receiver
// Initialize the receiver by giving it a name and reading in the config JSON
func NewSampleReceiver(name string, config json.RawMessage) (Receiver, error) {
r := new(SampleReceiver)
// Set name of SampleReceiver
// The name should be chosen in such a way that different instances of SampleReceiver can be distinguished
r.name = fmt.Sprintf("SampleReceiver(%s)", name)
// create new message processor
p, err := mp.NewMessageProcessor()
if err != nil {
cclog.ComponentError(r.name, "Initialization of message processor failed:", err.Error())
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
r.mp = p
// Set static information
err = r.mp.AddAddMetaByCondition("true", "source", r.name)
if err != nil {
cclog.ComponentError(r.name, fmt.Sprintf("Failed to add static information source=%s:", r.name), err.Error())
return nil, fmt.Errorf("failed to add static information source=%s: %v", r.name, err.Error())
}
// Set defaults in r.config
// Allow overwriting these defaults by reading config JSON
// Read the sample receiver specific JSON config
if len(config) > 0 {
err := json.Unmarshal(config, &r.config)
if err != nil {
cclog.ComponentError(r.name, "Error reading config:", err.Error())
return nil, err
}
}
// Add message processor config
if len(r.config.MessageProcessor) > 0 {
err = r.mp.FromConfigJSON(r.config.MessageProcessor)
if err != nil {
cclog.ComponentError(r.name, "Failed parsing JSON for message processor:", err.Error())
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
// Check that all required fields in the configuration are set
// Use 'if len(r.config.Option) > 0' for strings
return r, nil
}

108
sinks/README.md
View File

@ -1,108 +0,0 @@
# CCMetric sinks
This folder contains the SinkManager and sink implementations for the cc-metric-collector.
# Available sinks:
- [`stdout`](./stdoutSink.md): Print all metrics to `stdout`, `stderr` or a file
- [`http`](./httpSink.md): Send metrics to an HTTP server as POST requests
- [`influxdb`](./influxSink.md): Send metrics to an [InfluxDB](https://www.influxdata.com/products/influxdb/) database
- [`influxasync`](./influxAsyncSink.md): Send metrics to an [InfluxDB](https://www.influxdata.com/products/influxdb/) database with non-blocking write API
- [`nats`](./natsSink.md): Publish metrics to the [NATS](https://nats.io/) network overlay system
- [`ganglia`](./gangliaSink.md): Publish metrics in the [Ganglia Monitoring System](http://ganglia.info/) using the `gmetric` CLI tool
- [`libganglia`](./libgangliaSink.md): Publish metrics in the [Ganglia Monitoring System](http://ganglia.info/) directly using `libganglia.so`
- [`prometeus`](./prometheusSink.md): Publish metrics for the [Prometheus Monitoring System](https://prometheus.io/)
# Configuration
The configuration file for the sinks is a list of configurations. The `type` field in each specifies which sink to initialize.
```json
{
"mystdout" : {
"type" : "stdout",
"meta_as_tags" : [
"unit"
]
},
"metricstore" : {
"type" : "http",
"host" : "localhost",
"port" : "4123",
"database" : "ccmetric",
"password" : "<jwt token>"
}
}
```
# Contributing own sinks
A sink contains five functions and is derived from the type `sink`:
* `Init(name string, config json.RawMessage) error`
* `Write(point CCMetric) error`
* `Flush() error`
* `Close()`
* `New<Typename>(name string, config json.RawMessage) (Sink, error)` (calls the `Init()` function)
The data structures should be set up in `Init()` like opening a file or server connection. The `Write()` function writes/sends the data. For non-blocking sinks, the `Flush()` method tells the sink to drain its internal buffers. The `Close()` function should tear down anything created in `Init()`.
Finally, the sink needs to be registered in the `sinkManager.go`. There is a list of sinks called `AvailableSinks` which is a map (`sink_type_string` -> `pointer to sink interface`). Add a new entry with a descriptive name and the new sink.
## Sample sink
```go
package sinks
import (
"encoding/json"
"log"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
)
type SampleSinkConfig struct {
defaultSinkConfig // defines JSON tags for 'name' and 'meta_as_tags'
}
type SampleSink struct {
sink // declarate 'name' and 'meta_as_tags'
config StdoutSinkConfig // entry point to the SampleSinkConfig
}
// Initialize the sink by giving it a name and reading in the config JSON
func (s *SampleSink) Init(name string, config json.RawMessage) error {
s.name = fmt.Sprintf("SampleSink(%s)", name) // Always specify a name here
// Read in the config JSON
if len(config) > 0 {
err := json.Unmarshal(config, &s.config)
if err != nil {
return err
}
}
return nil
}
// Code to submit a single CCMetric to the sink
func (s *SampleSink) Write(point lp.CCMetric) error {
log.Print(point)
return nil
}
// If the sink uses batched sends internally, you can tell to flush its buffers
func (s *SampleSink) Flush() error {
return nil
}
// Close sink: close network connection, close files, close libraries, ...
func (s *SampleSink) Close() {}
// New function to create a new instance of the sink
func NewSampleSink(name string, config json.RawMessage) (Sink, error) {
s := new(SampleSink)
err := s.Init(name, config)
return s, err
}
```

269
sinks/gangliaCommon.go
View File

@ -1,269 +0,0 @@
package sinks
import (
"fmt"
"strings"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
)
func GangliaMetricName(point lp.CCMessage) string {
name := point.Name()
metricType, typeOK := point.GetTag("type")
metricTid, tidOk := point.GetTag("type-id")
gangliaType := metricType + metricTid
if strings.Contains(name, metricType) && tidOk {
name = strings.Replace(name, metricType, gangliaType, -1)
} else if typeOK && tidOk {
name = metricType + metricTid + "_" + name
} else if point.HasTag("device") {
device, _ := point.GetTag("device")
name = name + "_" + device
}
return name
}
func GangliaMetricRename(name string) string {
if name == "net_bytes_in" {
return "bytes_in"
} else if name == "net_bytes_out" {
return "bytes_out"
} else if name == "net_pkts_in" {
return "pkts_in"
} else if name == "net_pkts_out" {
return "pkts_out"
} else if name == "cpu_iowait" {
return "cpu_wio"
}
return name
}
func GangliaSlopeType(point lp.CCMessage) uint {
name := point.Name()
if name == "mem_total" || name == "swap_total" {
return 0
}
return 3
}
const DEFAULT_GANGLIA_METRIC_TMAX = 300
const DEFAULT_GANGLIA_METRIC_SLOPE = "both"
type GangliaMetric struct {
Name string
Type string
Slope string
Tmax int
Unit string
}
type GangliaMetricGroup struct {
Name string
Metrics []GangliaMetric
}
var CommonGangliaMetrics = []GangliaMetricGroup{
{
Name: "memory",
Metrics: []GangliaMetric{
{"mem_total", "float", "zero", 1200, "KB"},
{"swap_total", "float", "zero", 1200, "KB"},
{"mem_free", "float", "both", 180, "KB"},
{"mem_shared", "float", "both", 180, "KB"},
{"mem_buffers", "float", "both", 180, "KB"},
{"mem_cached", "float", "both", 180, "KB"},
{"swap_free", "float", "both", 180, "KB"},
{"mem_sreclaimable", "float", "both", 180, "KB"},
{"mem_slab", "float", "both", 180, "KB"},
},
},
{
Name: "cpu",
Metrics: []GangliaMetric{
{"cpu_num", "uint32", "zero", 1200, "CPUs"},
{"cpu_speed", "uint32", "zero", 1200, "MHz"},
{"cpu_user", "float", "both", 90, "%"},
{"cpu_nice", "float", "both", 90, "%"},
{"cpu_system", "float", "both", 90, "%"},
{"cpu_idle", "float", "both", 3800, "%"},
{"cpu_aidle", "float", "both", 90, "%"},
{"cpu_wio", "float", "both", 90, "%"},
{"cpu_intr", "float", "both", 90, "%"},
{"cpu_sintr", "float", "both", 90, "%"},
{"cpu_steal", "float", "both", 90, "%"},
{"cpu_guest", "float", "both", 90, "%"},
{"cpu_gnice", "float", "both", 90, "%"},
},
},
{
Name: "load",
Metrics: []GangliaMetric{
{"load_one", "float", "both", 70, ""},
{"load_five", "float", "both", 325, ""},
{"load_fifteen", "float", "both", 950, ""},
},
},
{
Name: "disk",
Metrics: []GangliaMetric{
{"disk_total", "double", "both", 1200, "GB"},
{"disk_free", "double", "both", 180, "GB"},
{"part_max_used", "float", "both", 180, "%"},
},
},
{
Name: "network",
Metrics: []GangliaMetric{
{"bytes_out", "float", "both", 300, "bytes/sec"},
{"bytes_in", "float", "both", 300, "bytes/sec"},
{"pkts_in", "float", "both", 300, "packets/sec"},
{"pkts_out", "float", "both", 300, "packets/sec"},
},
},
{
Name: "process",
Metrics: []GangliaMetric{
{"proc_run", "uint32", "both", 950, ""},
{"proc_total", "uint32", "both", 950, ""},
},
},
{
Name: "system",
Metrics: []GangliaMetric{
{"boottime", "uint32", "zero", 1200, "s"},
{"sys_clock", "uint32", "zero", 1200, "s"},
{"machine_type", "string", "zero", 1200, ""},
{"os_name", "string", "zero", 1200, ""},
{"os_release", "string", "zero", 1200, ""},
{"mtu", "uint32", "both", 1200, ""},
},
},
}
type GangliaMetricConfig struct {
Type string
Slope string
Tmax int
Unit string
Group string
Value string
Name string
}
func GetCommonGangliaConfig(point lp.CCMessage) GangliaMetricConfig {
mname := GangliaMetricRename(point.Name())
if oldname, ok := point.GetMeta("oldname"); ok {
mname = GangliaMetricRename(oldname)
}
for _, group := range CommonGangliaMetrics {
for _, metric := range group.Metrics {
if metric.Name == mname {
valueStr := ""
value, ok := point.GetField("value")
if ok {
switch real := value.(type) {
case float64:
valueStr = fmt.Sprintf("%f", real)
case float32:
valueStr = fmt.Sprintf("%f", real)
case int64:
valueStr = fmt.Sprintf("%d", real)
case int32:
valueStr = fmt.Sprintf("%d", real)
case int:
valueStr = fmt.Sprintf("%d", real)
case uint64:
valueStr = fmt.Sprintf("%d", real)
case uint32:
valueStr = fmt.Sprintf("%d", real)
case uint:
valueStr = fmt.Sprintf("%d", real)
case string:
valueStr = real
default:
}
}
return GangliaMetricConfig{
Group: group.Name,
Type: metric.Type,
Slope: metric.Slope,
Tmax: metric.Tmax,
Unit: metric.Unit,
Value: valueStr,
Name: GangliaMetricRename(mname),
}
}
}
}
return GangliaMetricConfig{
Group: "",
Type: "",
Slope: "",
Tmax: 0,
Unit: "",
Value: "",
Name: "",
}
}
func GetGangliaConfig(point lp.CCMessage) GangliaMetricConfig {
mname := GangliaMetricRename(point.Name())
if oldname, ok := point.GetMeta("oldname"); ok {
mname = GangliaMetricRename(oldname)
}
group := ""
if g, ok := point.GetMeta("group"); ok {
group = g
}
unit := ""
if u, ok := point.GetMeta("unit"); ok {
unit = u
}
valueType := "double"
valueStr := ""
value, ok := point.GetField("value")
if ok {
switch real := value.(type) {
case float64:
valueStr = fmt.Sprintf("%f", real)
valueType = "double"
case float32:
valueStr = fmt.Sprintf("%f", real)
valueType = "float"
case int64:
valueStr = fmt.Sprintf("%d", real)
valueType = "int32"
case int32:
valueStr = fmt.Sprintf("%d", real)
valueType = "int32"
case int:
valueStr = fmt.Sprintf("%d", real)
valueType = "int32"
case uint64:
valueStr = fmt.Sprintf("%d", real)
valueType = "uint32"
case uint32:
valueStr = fmt.Sprintf("%d", real)
valueType = "uint32"
case uint:
valueStr = fmt.Sprintf("%d", real)
valueType = "uint32"
case string:
valueStr = real
valueType = "string"
default:
valueType = "invalid"
}
}
return GangliaMetricConfig{
Group: group,
Type: valueType,
Slope: DEFAULT_GANGLIA_METRIC_SLOPE,
Tmax: DEFAULT_GANGLIA_METRIC_TMAX,
Unit: unit,
Value: valueStr,
Name: GangliaMetricRename(mname),
}
}

147
sinks/gangliaSink.go
View File

@ -1,147 +0,0 @@
package sinks
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"strings"
// "time"
"os/exec"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
)
const GMETRIC_EXEC = `gmetric`
const GMETRIC_CONFIG = `/etc/ganglia/gmond.conf`
type GangliaSinkConfig struct {
defaultSinkConfig
GmetricPath string `json:"gmetric_path,omitempty"`
GmetricConfig string `json:"gmetric_config,omitempty"`
AddGangliaGroup bool `json:"add_ganglia_group,omitempty"`
AddTagsAsDesc bool `json:"add_tags_as_desc,omitempty"`
ClusterName string `json:"cluster_name,omitempty"`
AddTypeToName bool `json:"add_type_to_name,omitempty"`
AddUnits bool `json:"add_units,omitempty"`
}
type GangliaSink struct {
sink
gmetric_path string
gmetric_config string
config GangliaSinkConfig
}
func (s *GangliaSink) Write(msg lp.CCMessage) error {
var err error = nil
//var tagsstr []string
var argstr []string
point, err := s.mp.ProcessMessage(msg)
if err == nil && point != nil {
// Get metric config (type, value, ... in suitable format)
conf := GetCommonGangliaConfig(point)
if len(conf.Type) == 0 {
conf = GetGangliaConfig(point)
}
if len(conf.Type) == 0 {
return fmt.Errorf("metric %q (Ganglia name %q) has no 'value' field", point.Name(), conf.Name)
}
if s.config.AddGangliaGroup {
argstr = append(argstr, fmt.Sprintf("--group=%s", conf.Group))
}
if s.config.AddUnits && len(conf.Unit) > 0 {
argstr = append(argstr, fmt.Sprintf("--units=%s", conf.Unit))
}
if len(s.config.ClusterName) > 0 {
argstr = append(argstr, fmt.Sprintf("--cluster=%s", s.config.ClusterName))
}
// if s.config.AddTagsAsDesc && len(tagsstr) > 0 {
// argstr = append(argstr, fmt.Sprintf("--desc=%q", strings.Join(tagsstr, ",")))
// }
if len(s.gmetric_config) > 0 {
argstr = append(argstr, fmt.Sprintf("--conf=%s", s.gmetric_config))
}
if s.config.AddTypeToName {
argstr = append(argstr, fmt.Sprintf("--name=%s", GangliaMetricName(point)))
} else {
argstr = append(argstr, fmt.Sprintf("--name=%s", conf.Name))
}
argstr = append(argstr, fmt.Sprintf("--slope=%s", conf.Slope))
argstr = append(argstr, fmt.Sprintf("--value=%s", conf.Value))
argstr = append(argstr, fmt.Sprintf("--type=%s", conf.Type))
argstr = append(argstr, fmt.Sprintf("--tmax=%d", conf.Tmax))
cclog.ComponentDebug(s.name, s.gmetric_path, strings.Join(argstr, " "))
command := exec.Command(s.gmetric_path, argstr...)
command.Wait()
_, err = command.Output()
}
return err
}
func (s *GangliaSink) Flush() error {
return nil
}
func (s *GangliaSink) Close() {
}
func NewGangliaSink(name string, config json.RawMessage) (Sink, error) {
s := new(GangliaSink)
s.name = fmt.Sprintf("GangliaSink(%s)", name)
s.config.AddTagsAsDesc = false
s.config.AddGangliaGroup = false
if len(config) > 0 {
d := json.NewDecoder(bytes.NewReader(config))
d.DisallowUnknownFields()
if err := d.Decode(&s.config); err != nil {
cclog.ComponentError(s.name, "Error reading config:", err.Error())
return nil, err
}
}
s.gmetric_path = ""
s.gmetric_config = ""
p, err := mp.NewMessageProcessor()
if err != nil {
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
s.mp = p
if len(s.config.GmetricPath) > 0 {
p, err := exec.LookPath(s.config.GmetricPath)
if err == nil {
s.gmetric_path = p
}
}
if len(s.gmetric_path) == 0 {
p, err := exec.LookPath(string(GMETRIC_EXEC))
if err == nil {
s.gmetric_path = p
}
}
if len(s.gmetric_path) == 0 {
return nil, errors.New("cannot find executable 'gmetric'")
}
if len(s.config.GmetricConfig) > 0 {
s.gmetric_config = s.config.GmetricConfig
}
if len(s.config.MessageProcessor) > 0 {
err = s.mp.FromConfigJSON(s.config.MessageProcessor)
if err != nil {
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
for _, k := range s.config.MetaAsTags {
s.mp.AddMoveMetaToTags("true", k, k)
}
return s, nil
}

25
sinks/gangliaSink.md
View File

@ -1,25 +0,0 @@
## `ganglia` sink
The `ganglia` sink uses the `gmetric` tool of the [Ganglia Monitoring System](http://ganglia.info/) to submit the metrics
### Configuration structure
```json
{
"<name>": {
"type": "ganglia",
"gmetric_path" : "/path/to/gmetric",
"add_ganglia_group" : true,
"process_messages" : {
"see" : "docs of message processor for valid fields"
},
"meta_as_tags" : []
}
}
```
- `type`: makes the sink an `ganglia` sink
- `gmetric_path`: Path to `gmetric` executable (optional). If not given, the sink searches in `$PATH` for `gmetric`.
- `add_ganglia_group`: Add `--group=X` based on meta information to the `gmetric` call. Some old versions of `gmetric` do not support the `--group` option.
- `process_messages`: Process messages with given rules before progressing or dropping, see [here](../pkg/messageProcessor/README.md) (optional)
- `meta_as_tags`: print all meta information as tags in the output (deprecated, optional)

306
sinks/httpSink.go
View File

@ -1,306 +0,0 @@
package sinks
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net/http"
"sync"
"time"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
influx "github.com/influxdata/line-protocol/v2/lineprotocol"
"golang.org/x/exp/slices"
)
type HttpSinkConfig struct {
defaultSinkConfig
// The full URL of the endpoint
URL string `json:"url"`
// JSON web tokens for authentication (Using the *Bearer* scheme)
JWT string `json:"jwt,omitempty"`
// Basic authentication
Username string `json:"username"`
Password string `json:"password"`
useBasicAuth bool
// time limit for requests made by the http client
Timeout string `json:"timeout,omitempty"`
timeout time.Duration
// Maximum amount of time an idle (keep-alive) connection will remain idle before closing itself
// should be larger than the measurement interval to keep the connection open
IdleConnTimeout string `json:"idle_connection_timeout,omitempty"`
idleConnTimeout time.Duration
// Batch all writes arriving in during this duration
// (default '5s', batching can be disabled by setting it to 0)
FlushDelay string `json:"flush_delay,omitempty"`
flushDelay time.Duration
// Maximum number of retries to connect to the http server (default: 3)
MaxRetries int `json:"max_retries,omitempty"`
// Timestamp precision
Precision string `json:"precision,omitempty"`
}
type HttpSink struct {
sink
client *http.Client
// influx line protocol encoder
encoder influx.Encoder
// Flush() runs in another goroutine and accesses the influx line protocol encoder,
// so this encoderLock has to protect the encoder
encoderLock sync.Mutex
// timer to run Flush()
flushTimer *time.Timer
// Lock to assure that only one timer is running at a time
timerLock sync.Mutex
config HttpSinkConfig
}
// Write sends metric m as http message
func (s *HttpSink) Write(msg lp.CCMessage) error {
// submit m only after applying processing/dropping rules
m, err := s.mp.ProcessMessage(msg)
if err == nil && m != nil {
// Lock for encoder usage
s.encoderLock.Lock()
err = EncoderAdd(&s.encoder, m)
// Unlock encoder usage
s.encoderLock.Unlock()
// Check that encoding worked
if err != nil {
return fmt.Errorf("encoding failed: %v", err)
}
}
if s.config.flushDelay == 0 {
// Directly flush if no flush delay is configured
return s.Flush()
} else if s.timerLock.TryLock() {
// Setup flush timer when flush delay is configured
// and no other timer is already running
if s.flushTimer != nil {
// Restarting existing flush timer
cclog.ComponentDebug(s.name, "Write(): Restarting flush timer")
s.flushTimer.Reset(s.config.flushDelay)
} else {
// Creating and starting flush timer
cclog.ComponentDebug(s.name, "Write(): Starting new flush timer")
s.flushTimer = time.AfterFunc(
s.config.flushDelay,
func() {
defer s.timerLock.Unlock()
cclog.ComponentDebug(s.name, "Starting flush triggered by flush timer")
if err := s.Flush(); err != nil {
cclog.ComponentError(s.name, "Flush triggered by flush timer: flush failed:", err)
}
})
}
}
return nil
}
// Flush sends all metrics stored in encoder to HTTP server
func (s *HttpSink) Flush() error {
// Lock for encoder usage
// Own lock for as short as possible: the time it takes to clone the buffer.
s.encoderLock.Lock()
buf := slices.Clone(s.encoder.Bytes())
s.encoder.Reset()
// Unlock encoder usage
s.encoderLock.Unlock()
if len(buf) == 0 {
return nil
}
cclog.ComponentDebug(s.name, "Flush(): Flushing")
var res *http.Response
for i := 0; i < s.config.MaxRetries; i++ {
// Create new request to send buffer
req, err := http.NewRequest(http.MethodPost, s.config.URL, bytes.NewReader(buf))
if err != nil {
cclog.ComponentError(s.name, "Flush(): Failed to create HTTP request:", err)
return err
}
// Set authorization header
if len(s.config.JWT) != 0 {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", s.config.JWT))
}
// Set basic authentication
if s.config.useBasicAuth {
req.SetBasicAuth(s.config.Username, s.config.Password)
}
// Do request
res, err = s.client.Do(req)
if err != nil {
cclog.ComponentError(s.name, "Flush(): transport/tcp error:", err)
// Wait between retries
time.Sleep(time.Duration(i+1) * (time.Second / 2))
continue
}
break
}
if res == nil {
return errors.New("flush failed due to repeated errors")
}
// Handle application errors
if res.StatusCode != http.StatusOK {
err := errors.New(res.Status)
cclog.ComponentError(s.name, "Flush(): Application error:", err)
return err
}
return nil
}
func (s *HttpSink) Close() {
cclog.ComponentDebug(s.name, "Closing HTTP connection")
// Stop existing timer and immediately flush
if s.flushTimer != nil {
if ok := s.flushTimer.Stop(); ok {
s.timerLock.Unlock()
}
}
// Flush
if err := s.Flush(); err != nil {
cclog.ComponentError(s.name, "Close(): Flush failed:", err)
}
s.client.CloseIdleConnections()
}
// NewHttpSink creates a new http sink
func NewHttpSink(name string, config json.RawMessage) (Sink, error) {
s := new(HttpSink)
// Set default values
s.name = fmt.Sprintf("HttpSink(%s)", name)
// should be larger than the measurement interval to keep the connection open
s.config.IdleConnTimeout = "120s"
s.config.Timeout = "5s"
s.config.FlushDelay = "5s"
s.config.MaxRetries = 3
s.config.Precision = "s"
cclog.ComponentDebug(s.name, "Init()")
// Read config
if len(config) > 0 {
d := json.NewDecoder(bytes.NewReader(config))
d.DisallowUnknownFields()
if err := d.Decode(&s.config); err != nil {
cclog.ComponentError(s.name, "Error reading config:", err.Error())
return nil, err
}
}
if len(s.config.URL) == 0 {
return nil, errors.New("`url` config option is required for HTTP sink")
}
// Check basic authentication config
if len(s.config.Username) > 0 || len(s.config.Password) > 0 {
s.config.useBasicAuth = true
}
if s.config.useBasicAuth && len(s.config.Username) == 0 {
return nil, errors.New("basic authentication requires username")
}
if s.config.useBasicAuth && len(s.config.Password) == 0 {
return nil, errors.New("basic authentication requires password")
}
p, err := mp.NewMessageProcessor()
if err != nil {
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
s.mp = p
if len(s.config.IdleConnTimeout) > 0 {
t, err := time.ParseDuration(s.config.IdleConnTimeout)
if err == nil {
cclog.ComponentDebug(s.name, "Init(): idleConnTimeout", t)
s.config.idleConnTimeout = t
}
}
if len(s.config.Timeout) > 0 {
t, err := time.ParseDuration(s.config.Timeout)
if err == nil {
s.config.timeout = t
cclog.ComponentDebug(s.name, "Init(): timeout", t)
}
}
if len(s.config.FlushDelay) > 0 {
t, err := time.ParseDuration(s.config.FlushDelay)
if err == nil {
s.config.flushDelay = t
cclog.ComponentDebug(s.name, "Init(): flushDelay", t)
}
}
if len(s.config.MessageProcessor) > 0 {
err = p.FromConfigJSON(s.config.MessageProcessor)
if err != nil {
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
for _, k := range s.config.MetaAsTags {
s.mp.AddMoveMetaToTags("true", k, k)
}
precision := influx.Second
if len(s.config.Precision) > 0 {
switch s.config.Precision {
case "s":
precision = influx.Second
case "ms":
precision = influx.Millisecond
case "us":
precision = influx.Microsecond
case "ns":
precision = influx.Nanosecond
}
}
// Create http client
s.client = &http.Client{
Transport: &http.Transport{
MaxIdleConns: 1, // We will only ever talk to one host.
IdleConnTimeout: s.config.idleConnTimeout,
},
Timeout: s.config.timeout,
}
// Configure influx line protocol encoder
s.encoder.SetPrecision(precision)
return s, nil
}

44
sinks/httpSink.md
View File

@ -1,44 +0,0 @@
## `http` sink
The `http` sink uses POST requests to a HTTP server to submit the metrics in the InfluxDB line-protocol format. It uses JSON web tokens for authentification. The sink creates batches of metrics before sending, to reduce the HTTP traffic.
### Configuration structure
```json
{
"<name>": {
"type": "http",
"url" : "https://my-monitoring.example.com:1234/api/write",
"jwt" : "blabla.blabla.blabla",
"username": "myUser",
"password": "myPW",
"timeout": "5s",
"idle_connection_timeout" : "5s",
"flush_delay": "2s",
"batch_size": 1000,
"precision": "s",
"process_messages" : {
"see" : "docs of message processor for valid fields"
},
"meta_as_tags" : []
}
}
```
- `type`: makes the sink an `http` sink
- `url`: The full URL of the endpoint
- `jwt`: JSON web tokens for authentication (Using the *Bearer* scheme)
- `username`: username for basic authentication
- `password`: password for basic authentication
- `timeout`: General timeout for the HTTP client (default '5s')
- `max_retries`: Maximum number of retries to connect to the http server
- `idle_connection_timeout`: Timeout for idle connections (default '120s'). Should be larger than the measurement interval to keep the connection open
- `flush_delay`: Batch all writes arriving in during this duration (default '1s', batching can be disabled by setting it to 0)
- `batch_size`: Maximal batch size. If `batch_size` is reached before the end of `flush_delay`, the metrics are sent without further delay
- `precision`: Precision of the timestamp. Valid values are 's', 'ms', 'us' and 'ns'. (default is 's')
- `process_messages`: Process messages with given rules before progressing or dropping, see [here](../pkg/messageProcessor/README.md) (optional)
- `meta_as_tags`: print all meta information as tags in the output (deprecated, optional)
### Using `http` sink for communication with cc-metric-store
The cc-metric-store only accepts metrics with a timestamp precision in seconds, so it is required to use `"precision": "s"`.

276
sinks/influxAsyncSink.go
View File

@ -1,276 +0,0 @@
package sinks
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
influxdb2 "github.com/influxdata/influxdb-client-go/v2"
influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api"
influxdb2ApiHttp "github.com/influxdata/influxdb-client-go/v2/api/http"
)
type InfluxAsyncSinkConfig struct {
defaultSinkConfig
Host string `json:"host,omitempty"`
Port string `json:"port,omitempty"`
Database string `json:"database,omitempty"`
User string `json:"user,omitempty"`
Password string `json:"password,omitempty"`
Organization string `json:"organization,omitempty"`
SSL bool `json:"ssl,omitempty"`
// Maximum number of points sent to server in single request. Default 5000
BatchSize uint `json:"batch_size,omitempty"`
// Interval, in ms, in which is buffer flushed if it has not been already written (by reaching batch size) . Default 1000ms
FlushInterval uint `json:"flush_interval,omitempty"`
InfluxRetryInterval string `json:"retry_interval,omitempty"`
InfluxExponentialBase uint `json:"retry_exponential_base,omitempty"`
InfluxMaxRetries uint `json:"max_retries,omitempty"`
InfluxMaxRetryTime string `json:"max_retry_time,omitempty"`
CustomFlushInterval string `json:"custom_flush_interval,omitempty"`
MaxRetryAttempts uint `json:"max_retry_attempts,omitempty"`
// Timestamp precision
Precision string `json:"precision,omitempty"`
}
type InfluxAsyncSink struct {
sink
client influxdb2.Client
writeApi influxdb2Api.WriteAPI
errors <-chan error
config InfluxAsyncSinkConfig
influxRetryInterval uint
influxMaxRetryTime uint
customFlushInterval time.Duration
flushTimer *time.Timer
}
func (s *InfluxAsyncSink) connect() error {
var auth string
var uri string
if s.config.SSL {
uri = fmt.Sprintf("https://%s:%s", s.config.Host, s.config.Port)
} else {
uri = fmt.Sprintf("http://%s:%s", s.config.Host, s.config.Port)
}
if len(s.config.User) == 0 {
auth = s.config.Password
} else {
auth = fmt.Sprintf("%s:%s", s.config.User, s.config.Password)
}
cclog.ComponentDebug(s.name, "Using URI", uri, "Org", s.config.Organization, "Bucket", s.config.Database)
clientOptions := influxdb2.DefaultOptions()
if s.config.BatchSize != 0 {
cclog.ComponentDebug(s.name, "Batch size", s.config.BatchSize)
clientOptions.SetBatchSize(s.config.BatchSize)
}
if s.config.FlushInterval != 0 {
cclog.ComponentDebug(s.name, "Flush interval", s.config.FlushInterval)
clientOptions.SetFlushInterval(s.config.FlushInterval)
}
if s.influxRetryInterval != 0 {
cclog.ComponentDebug(s.name, "MaxRetryInterval", s.influxRetryInterval)
clientOptions.SetMaxRetryInterval(s.influxRetryInterval)
}
if s.influxMaxRetryTime != 0 {
cclog.ComponentDebug(s.name, "MaxRetryTime", s.influxMaxRetryTime)
clientOptions.SetMaxRetryTime(s.influxMaxRetryTime)
}
if s.config.InfluxExponentialBase != 0 {
cclog.ComponentDebug(s.name, "Exponential Base", s.config.InfluxExponentialBase)
clientOptions.SetExponentialBase(s.config.InfluxExponentialBase)
}
if s.config.InfluxMaxRetries != 0 {
cclog.ComponentDebug(s.name, "Max Retries", s.config.InfluxMaxRetries)
clientOptions.SetMaxRetries(s.config.InfluxMaxRetries)
}
clientOptions.SetTLSConfig(
&tls.Config{
InsecureSkipVerify: true,
},
)
precision := time.Second
if len(s.config.Precision) > 0 {
switch s.config.Precision {
case "s":
precision = time.Second
case "ms":
precision = time.Millisecond
case "us":
precision = time.Microsecond
case "ns":
precision = time.Nanosecond
}
}
clientOptions.SetPrecision(precision)
s.client = influxdb2.NewClientWithOptions(uri, auth, clientOptions)
s.writeApi = s.client.WriteAPI(s.config.Organization, s.config.Database)
ok, err := s.client.Ping(context.Background())
if err != nil {
return err
}
if !ok {
return fmt.Errorf("connection to %s not healthy", uri)
}
s.writeApi.SetWriteFailedCallback(func(batch string, err influxdb2ApiHttp.Error, retryAttempts uint) bool {
mlist := strings.Split(batch, "\n")
cclog.ComponentError(s.name, fmt.Sprintf("Failed to write batch with %d metrics %d times (max: %d): %s", len(mlist), retryAttempts, s.config.MaxRetryAttempts, err.Error()))
return retryAttempts <= s.config.MaxRetryAttempts
})
return nil
}
func (s *InfluxAsyncSink) Write(m lp.CCMessage) error {
if s.customFlushInterval != 0 && s.flushTimer == nil {
// Run a batched flush for all lines that have arrived in the defined interval
s.flushTimer = time.AfterFunc(s.customFlushInterval, func() {
if err := s.Flush(); err != nil {
cclog.ComponentError(s.name, "flush failed:", err.Error())
}
})
}
msg, err := s.mp.ProcessMessage(m)
if err == nil && msg != nil {
s.writeApi.WritePoint(msg.ToPoint(nil))
}
return nil
}
func (s *InfluxAsyncSink) Flush() error {
cclog.ComponentDebug(s.name, "Flushing")
s.writeApi.Flush()
if s.customFlushInterval != 0 && s.flushTimer != nil {
s.flushTimer = nil
}
return nil
}
func (s *InfluxAsyncSink) Close() {
cclog.ComponentDebug(s.name, "Closing InfluxDB connection")
s.writeApi.Flush()
s.client.Close()
}
func NewInfluxAsyncSink(name string, config json.RawMessage) (Sink, error) {
s := new(InfluxAsyncSink)
s.name = fmt.Sprintf("InfluxSink(%s)", name)
// Set default for maximum number of points sent to server in single request.
s.config.BatchSize = 0
s.influxRetryInterval = 0
//s.config.InfluxRetryInterval = "1s"
s.influxMaxRetryTime = 0
//s.config.InfluxMaxRetryTime = "168h"
s.config.InfluxMaxRetries = 0
s.config.InfluxExponentialBase = 0
s.config.FlushInterval = 0
s.config.CustomFlushInterval = ""
s.customFlushInterval = time.Duration(0)
s.config.MaxRetryAttempts = 1
s.config.Precision = "s"
// Default retry intervals (in seconds)
// 1 2
// 2 4
// 4 8
// 8 16
// 16 32
// 32 64
// 64 128
// 128 256
// 256 512
// 512 1024
// 1024 2048
// 2048 4096
// 4096 8192
// 8192 16384
// 16384 32768
// 32768 65536
// 65536 131072
// 131072 262144
// 262144 524288
if len(config) > 0 {
d := json.NewDecoder(bytes.NewReader(config))
d.DisallowUnknownFields()
if err := d.Decode(&s.config); err != nil {
cclog.ComponentError(s.name, "Error reading config:", err.Error())
return nil, err
}
}
if len(s.config.Port) == 0 {
return nil, errors.New("missing port configuration required by InfluxSink")
}
if len(s.config.Database) == 0 {
return nil, errors.New("missing database configuration required by InfluxSink")
}
if len(s.config.Organization) == 0 {
return nil, errors.New("missing organization configuration required by InfluxSink")
}
if len(s.config.Password) == 0 {
return nil, errors.New("missing password configuration required by InfluxSink")
}
p, err := mp.NewMessageProcessor()
if err != nil {
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
s.mp = p
if len(s.config.MessageProcessor) > 0 {
err = s.mp.FromConfigJSON(s.config.MessageProcessor)
if err != nil {
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
// Create lookup map to use meta infos as tags in the output metric
// s.meta_as_tags = make(map[string]bool)
// for _, k := range s.config.MetaAsTags {
// s.meta_as_tags[k] = true
// }
for _, k := range s.config.MetaAsTags {
s.mp.AddMoveMetaToTags("true", k, k)
}
toUint := func(duration string, def uint) uint {
t, err := time.ParseDuration(duration)
if err == nil {
return uint(t.Milliseconds())
}
return def
}
s.influxRetryInterval = toUint(s.config.InfluxRetryInterval, s.influxRetryInterval)
s.influxMaxRetryTime = toUint(s.config.InfluxMaxRetryTime, s.influxMaxRetryTime)
// Use a own timer for calling Flush()
if len(s.config.CustomFlushInterval) > 0 {
t, err := time.ParseDuration(s.config.CustomFlushInterval)
if err != nil {
return nil, fmt.Errorf("invalid duration in 'custom_flush_interval': %v", err)
}
s.customFlushInterval = t
}
// Connect to InfluxDB server
if err := s.connect(); err != nil {
return nil, fmt.Errorf("unable to connect: %v", err)
}
// Start background: Read from error channel
s.errors = s.writeApi.Errors()
go func() {
for err := range s.errors {
cclog.ComponentError(s.name, err.Error())
}
}()
return s, nil
}

54
sinks/influxAsyncSink.md
View File

@ -1,54 +0,0 @@
## `influxasync` sink
The `influxasync` sink uses the official [InfluxDB golang client](https://pkg.go.dev/github.com/influxdata/influxdb-client-go/v2) to write the metrics to an InfluxDB database in a **non-blocking** fashion. It provides only support for V2 write endpoints (InfluxDB 1.8.0 or later).
### Configuration structure
```json
{
"<name>": {
"type": "influxasync",
"database" : "mymetrics",
"host": "dbhost.example.com",
"port": "4222",
"user": "exampleuser",
"password" : "examplepw",
"organization": "myorg",
"ssl": true,
"batch_size": 200,
"retry_interval" : "1s",
"retry_exponential_base" : 2,
"precision": "s",
"max_retries": 20,
"max_retry_time" : "168h",
"process_messages" : {
"see" : "docs of message processor for valid fields"
},
"meta_as_tags" : []
}
}
```
- `type`: makes the sink an `influxdb` sink
- `database`: All metrics are written to this bucket
- `host`: Hostname of the InfluxDB database server
- `port`: Portnumber (as string) of the InfluxDB database server
- `user`: Username for basic authentification
- `password`: Password for basic authentification
- `organization`: Organization in the InfluxDB
- `ssl`: Use SSL connection
- `batch_size`: batch up metrics internally, default 100
- `retry_interval`: Base retry interval for failed write requests, default 1s
- `retry_exponential_base`: The retry interval is exponentially increased with this base, default 2
- `max_retries`: Maximal number of retry attempts
- `max_retry_time`: Maximal time to retry failed writes, default 168h (one week)
- `precision`: Precision of the timestamp. Valid values are 's', 'ms', 'us' and 'ns'. (default is 's')
- `process_messages`: Process messages with given rules before progressing or dropping, see [here](../pkg/messageProcessor/README.md) (optional)
- `meta_as_tags`: print all meta information as tags in the output (deprecated, optional)
For information about the calculation of the retry interval settings, see [offical influxdb-client-go documentation](https://github.com/influxdata/influxdb-client-go#handling-of-failed-async-writes)
### Using `influxasync` sink for communication with cc-metric-store
The cc-metric-store only accepts metrics with a timestamp precision in seconds, so it is required to use `"precision": "s"`.

504
sinks/influxSink.go
View File

@ -1,504 +0,0 @@
package sinks
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"sync"
"time"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
influxdb2 "github.com/influxdata/influxdb-client-go/v2"
influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api"
influx "github.com/influxdata/line-protocol/v2/lineprotocol"
"golang.org/x/exp/slices"
)
type InfluxSink struct {
sink
client influxdb2.Client
writeApi influxdb2Api.WriteAPIBlocking
config struct {
defaultSinkConfig
Host string `json:"host,omitempty"`
Port string `json:"port,omitempty"`
Database string `json:"database,omitempty"`
User string `json:"user,omitempty"`
Password string `json:"password,omitempty"`
Organization string `json:"organization,omitempty"`
SSL bool `json:"ssl,omitempty"`
// Maximum number of points sent to server in single request.
// Default: 1000
BatchSize int `json:"batch_size,omitempty"`
// Time interval for delayed sending of metrics.
// If the buffers are already filled before the end of this interval,
// the metrics are sent without further delay.
// Default: 1s
FlushInterval string `json:"flush_delay,omitempty"`
flushDelay time.Duration
// Influx client options:
// HTTP request timeout
HTTPRequestTimeout string `json:"http_request_timeout"`
// Retry interval
InfluxRetryInterval string `json:"retry_interval,omitempty"`
// maximum delay between each retry attempt
InfluxMaxRetryInterval string `json:"max_retry_interval,omitempty"`
// base for the exponential retry delay
InfluxExponentialBase uint `json:"retry_exponential_base,omitempty"`
// maximum count of retry attempts of failed writes
InfluxMaxRetries uint `json:"max_retries,omitempty"`
// maximum total retry timeout
InfluxMaxRetryTime string `json:"max_retry_time,omitempty"`
// Specify whether to use GZip compression in write requests
InfluxUseGzip bool `json:"use_gzip"`
// Timestamp precision
Precision string `json:"precision,omitempty"`
}
// influx line protocol encoder
encoder influx.Encoder
// number of records stored in the encoder
numRecordsInEncoder int
// List of tags and meta data tags which should be used as tags
extended_tag_list []key_value_pair
// Flush() runs in another goroutine and accesses the influx line protocol encoder,
// so this encoderLock has to protect the encoder and numRecordsInEncoder
encoderLock sync.Mutex
// timer to run Flush()
flushTimer *time.Timer
// Lock to assure that only one timer is running at a time
timerLock sync.Mutex
// WaitGroup to ensure only one send operation is running at a time
sendWaitGroup sync.WaitGroup
}
// connect connects to the InfluxDB server
func (s *InfluxSink) connect() error {
// URI options:
// * http://host:port
// * https://host:port
var uri string
if s.config.SSL {
uri = fmt.Sprintf("https://%s:%s", s.config.Host, s.config.Port)
} else {
uri = fmt.Sprintf("http://%s:%s", s.config.Host, s.config.Port)
}
// Authentication options:
// * token
// * username:password
var auth string
if len(s.config.User) == 0 {
auth = s.config.Password
} else {
auth = fmt.Sprintf("%s:%s", s.config.User, s.config.Password)
}
cclog.ComponentDebug(s.name, "connect():",
"Using URI='"+uri+"'",
"Org='"+s.config.Organization+"'",
"Bucket='"+s.config.Database+"'")
// Set influxDB client options
clientOptions := influxdb2.DefaultOptions()
// set HTTP request timeout
if len(s.config.HTTPRequestTimeout) > 0 {
if t, err := time.ParseDuration(s.config.HTTPRequestTimeout); err == nil {
httpRequestTimeout := uint(t.Seconds())
clientOptions.SetHTTPRequestTimeout(httpRequestTimeout)
} else {
cclog.ComponentError(s.name, "connect():", "Failed to parse duration for HTTP RequestTimeout: ", s.config.HTTPRequestTimeout)
}
}
cclog.ComponentDebug(
s.name,
"connect():",
"Influx client options HTTPRequestTimeout:",
time.Second*time.Duration(clientOptions.HTTPRequestTimeout()))
// Set retry interval
if len(s.config.InfluxRetryInterval) > 0 {
if t, err := time.ParseDuration(s.config.InfluxRetryInterval); err == nil {
influxRetryInterval := uint(t.Milliseconds())
clientOptions.SetRetryInterval(influxRetryInterval)
} else {
cclog.ComponentError(s.name, "connect():", "Failed to parse duration for Influx RetryInterval: ", s.config.InfluxRetryInterval)
}
}
cclog.ComponentDebug(
s.name,
"connect():",
"Influx client options RetryInterval:",
time.Millisecond*time.Duration(clientOptions.RetryInterval()))
// Set the maximum delay between each retry attempt
if len(s.config.InfluxMaxRetryInterval) > 0 {
if t, err := time.ParseDuration(s.config.InfluxMaxRetryInterval); err == nil {
influxMaxRetryInterval := uint(t.Milliseconds())
clientOptions.SetMaxRetryInterval(influxMaxRetryInterval)
} else {
cclog.ComponentError(s.name, "connect():", "Failed to parse duration for Influx MaxRetryInterval: ", s.config.InfluxMaxRetryInterval)
}
}
cclog.ComponentDebug(
s.name,
"connect():",
"Influx client options MaxRetryInterval:",
time.Millisecond*time.Duration(clientOptions.MaxRetryInterval()))
// Set the base for the exponential retry delay
if s.config.InfluxExponentialBase != 0 {
clientOptions.SetExponentialBase(s.config.InfluxExponentialBase)
}
cclog.ComponentDebug(
s.name,
"connect():",
"Influx client options ExponentialBase:",
clientOptions.ExponentialBase())
// Set maximum count of retry attempts of failed writes
if s.config.InfluxMaxRetries != 0 {
clientOptions.SetMaxRetries(s.config.InfluxMaxRetries)
}
cclog.ComponentDebug(
s.name,
"connect():",
"Influx client options MaxRetries:",
clientOptions.MaxRetries())
// Set the maximum total retry timeout
if len(s.config.InfluxMaxRetryTime) > 0 {
if t, err := time.ParseDuration(s.config.InfluxMaxRetryTime); err == nil {
influxMaxRetryTime := uint(t.Milliseconds())
cclog.ComponentDebug(s.name, "connect():", "MaxRetryTime", s.config.InfluxMaxRetryTime)
clientOptions.SetMaxRetryTime(influxMaxRetryTime)
} else {
cclog.ComponentError(s.name, "connect():", "Failed to parse duration for Influx MaxRetryInterval: ", s.config.InfluxMaxRetryInterval)
}
}
cclog.ComponentDebug(
s.name,
"connect():",
"Influx client options MaxRetryTime:",
time.Millisecond*time.Duration(clientOptions.MaxRetryTime()))
// Specify whether to use GZip compression in write requests
clientOptions.SetUseGZip(s.config.InfluxUseGzip)
cclog.ComponentDebug(
s.name,
"connect():",
"Influx client options UseGZip:",
clientOptions.UseGZip())
// Do not check InfluxDB certificate
clientOptions.SetTLSConfig(
&tls.Config{
InsecureSkipVerify: true,
},
)
// Set time precision
precision := time.Second
if len(s.config.Precision) > 0 {
switch s.config.Precision {
case "s":
precision = time.Second
case "ms":
precision = time.Millisecond
case "us":
precision = time.Microsecond
case "ns":
precision = time.Nanosecond
}
}
clientOptions.SetPrecision(precision)
// Create new writeAPI
s.client = influxdb2.NewClientWithOptions(uri, auth, clientOptions)
s.writeApi = s.client.WriteAPIBlocking(s.config.Organization, s.config.Database)
// Check InfluxDB server accessibility
ok, err := s.client.Ping(context.Background())
if err != nil {
return err
}
if !ok {
return fmt.Errorf("connection to %s not healthy", uri)
}
return nil
}
// Write sends metric m in influxDB line protocol
func (s *InfluxSink) Write(msg lp.CCMessage) error {
m, err := s.mp.ProcessMessage(msg)
if err == nil && m != nil {
// Lock for encoder usage
s.encoderLock.Lock()
// Encode measurement name
s.encoder.StartLine(m.Name())
// copy tags and meta data which should be used as tags
s.extended_tag_list = s.extended_tag_list[:0]
for key, value := range m.Tags() {
s.extended_tag_list =
append(
s.extended_tag_list,
key_value_pair{
key: key,
value: value,
},
)
}
// for _, key := range s.config.MetaAsTags {
// if value, ok := m.GetMeta(key); ok {
// s.extended_tag_list =
// append(
// s.extended_tag_list,
// key_value_pair{
// key: key,
// value: value,
// },
// )
// }
// }
// Encode tags (they musts be in lexical order)
slices.SortFunc(
s.extended_tag_list,
func(a key_value_pair, b key_value_pair) int {
if a.key < b.key {
return -1
}
if a.key > b.key {
return +1
}
return 0
},
)
for i := range s.extended_tag_list {
s.encoder.AddTag(
s.extended_tag_list[i].key,
s.extended_tag_list[i].value,
)
}
// Encode fields
for key, value := range m.Fields() {
s.encoder.AddField(key, influx.MustNewValue(value))
}
// Encode time stamp
s.encoder.EndLine(m.Time())
// Check for encoder errors
if err := s.encoder.Err(); err != nil {
// Unlock encoder usage
s.encoderLock.Unlock()
return fmt.Errorf("encoding failed: %v", err)
}
s.numRecordsInEncoder++
}
if s.config.flushDelay == 0 {
// Unlock encoder usage
s.encoderLock.Unlock()
// Directly flush if no flush delay is configured
return s.Flush()
} else if s.numRecordsInEncoder == s.config.BatchSize {
// Unlock encoder usage
s.encoderLock.Unlock()
// Stop flush timer
if s.flushTimer != nil {
if ok := s.flushTimer.Stop(); ok {
cclog.ComponentDebug(s.name, "Write(): Stopped flush timer. Batch size limit reached before flush delay")
s.timerLock.Unlock()
}
}
// Flush if batch size is reached
return s.Flush()
} else if s.timerLock.TryLock() {
// Setup flush timer when flush delay is configured
// and no other timer is already running
if s.flushTimer != nil {
// Restarting existing flush timer
cclog.ComponentDebug(s.name, "Write(): Restarting flush timer")
s.flushTimer.Reset(s.config.flushDelay)
} else {
// Creating and starting flush timer
cclog.ComponentDebug(s.name, "Write(): Starting new flush timer")
s.flushTimer = time.AfterFunc(
s.config.flushDelay,
func() {
defer s.timerLock.Unlock()
cclog.ComponentDebug(s.name, "Starting flush triggered by flush timer")
if err := s.Flush(); err != nil {
cclog.ComponentError(s.name, "Flush triggered by flush timer: flush failed:", err)
}
})
}
}
// Unlock encoder usage
s.encoderLock.Unlock()
return nil
}
// Flush sends all metrics stored in encoder to InfluxDB server
func (s *InfluxSink) Flush() error {
// Lock for encoder usage
// Own lock for as short as possible: the time it takes to clone the buffer.
s.encoderLock.Lock()
buf := slices.Clone(s.encoder.Bytes())
numRecordsInBuf := s.numRecordsInEncoder
s.encoder.Reset()
s.numRecordsInEncoder = 0
// Unlock encoder usage
s.encoderLock.Unlock()
if len(buf) == 0 {
return nil
}
cclog.ComponentDebug(s.name, "Flush(): Flushing", numRecordsInBuf, "metrics")
// Asynchron send of encoder metrics
s.sendWaitGroup.Add(1)
go func() {
defer s.sendWaitGroup.Done()
startTime := time.Now()
err := s.writeApi.WriteRecord(context.Background(), string(buf))
if err != nil {
cclog.ComponentError(
s.name,
"Flush():",
"Flush failed:", err,
"(number of records =", numRecordsInBuf,
", buffer size =", len(buf),
", send duration =", time.Since(startTime),
")",
)
return
}
}()
return nil
}
func (s *InfluxSink) Close() {
cclog.ComponentDebug(s.name, "Closing InfluxDB connection")
// Stop existing timer and immediately flush
if s.flushTimer != nil {
if ok := s.flushTimer.Stop(); ok {
s.timerLock.Unlock()
}
}
// Flush
if err := s.Flush(); err != nil {
cclog.ComponentError(s.name, "Close():", "Flush failed:", err)
}
// Wait for send operations to finish
s.sendWaitGroup.Wait()
s.client.Close()
}
// NewInfluxSink create a new InfluxDB sink
func NewInfluxSink(name string, config json.RawMessage) (Sink, error) {
s := new(InfluxSink)
s.name = fmt.Sprintf("InfluxSink(%s)", name)
// Set config default values
s.config.BatchSize = 1000
s.config.FlushInterval = "1s"
s.config.Precision = "s"
// Read config
if len(config) > 0 {
d := json.NewDecoder(bytes.NewReader(config))
d.DisallowUnknownFields()
if err := d.Decode(&s.config); err != nil {
cclog.ComponentError(s.name, "Error reading config:", err.Error())
return nil, err
}
}
if len(s.config.Host) == 0 {
return s, errors.New("missing host configuration required by InfluxSink")
}
if len(s.config.Port) == 0 {
return s, errors.New("missing port configuration required by InfluxSink")
}
if len(s.config.Database) == 0 {
return s, errors.New("missing database configuration required by InfluxSink")
}
if len(s.config.Organization) == 0 {
return s, errors.New("missing organization configuration required by InfluxSink")
}
if len(s.config.Password) == 0 {
return s, errors.New("missing password configuration required by InfluxSink")
}
p, err := mp.NewMessageProcessor()
if err != nil {
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
s.mp = p
if len(s.config.MessageProcessor) > 0 {
err = p.FromConfigJSON(s.config.MessageProcessor)
if err != nil {
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
for _, k := range s.config.MetaAsTags {
s.mp.AddMoveMetaToTags("true", k, k)
}
// Configure flush delay duration
if len(s.config.FlushInterval) > 0 {
t, err := time.ParseDuration(s.config.FlushInterval)
if err == nil {
s.config.flushDelay = t
}
}
if !(s.config.BatchSize > 0) {
return s, fmt.Errorf("batch_size=%d in InfluxDB config must be > 0", s.config.BatchSize)
}
// Connect to InfluxDB server
if err := s.connect(); err != nil {
return s, fmt.Errorf("unable to connect: %v", err)
}
// Configure influx line protocol encoder
s.encoder.SetPrecision(influx.Nanosecond)
s.extended_tag_list = make([]key_value_pair, 0)
return s, nil
}

58
sinks/influxSink.md
View File

@ -1,58 +0,0 @@
## `influxdb` sink
The `influxdb` sink uses the official [InfluxDB golang client](https://pkg.go.dev/github.com/influxdata/influxdb-client-go/v2) to write the metrics to an InfluxDB database in a **blocking** fashion. It provides only support for V2 write endpoints (InfluxDB 1.8.0 or later).
### Configuration structure
```json
{
"<name>": {
"type": "influxdb",
"database" : "mymetrics",
"host": "dbhost.example.com",
"port": "4222",
"user": "exampleuser",
"password" : "examplepw",
"organization": "myorg",
"ssl": true,
"flush_delay" : "1s",
"batch_size" : 1000,
"use_gzip": true,
"precision": "s",
"process_messages" : {
"see" : "docs of message processor for valid fields"
},
"meta_as_tags" : []
}
}
```
- `type`: makes the sink an `influxdb` sink
- `database`: All metrics are written to this bucket
- `host`: Hostname of the InfluxDB database server
- `port`: Port number (as string) of the InfluxDB database server
- `user`: Username for basic authentication
- `password`: Password for basic authentication
- `organization`: Organization in the InfluxDB
- `ssl`: Use SSL connection
- `flush_delay`: Group metrics coming in to a single batch
- `batch_size`: Maximal batch size. If `batch_size` is reached before the end of `flush_delay`, the metrics are sent without further delay
- `precision`: Precision of the timestamp. Valid values are 's', 'ms', 'us' and 'ns'. (default is 's')
- `process_messages`: Process messages with given rules before progressing or dropping, see [here](../pkg/messageProcessor/README.md) (optional)
- `meta_as_tags`: print all meta information as tags in the output (deprecated, optional)
Influx client options:
=======
- `batch_size`: Maximal batch size
- `meta_as_tags`: move meta information keys to tags (optional)
- `http_request_timeout`: HTTP request timeout
- `retry_interval`: retry interval
- `max_retry_interval`: maximum delay between each retry attempt
- `retry_exponential_base`: base for the exponential retry delay
- `max_retries`: maximum count of retry attempts of failed writes
- `max_retry_time`: maximum total retry timeout
- `use_gzip`: Specify whether to use GZip compression in write requests
### Using `influxdb` sink for communication with cc-metric-store
The cc-metric-store only accepts metrics with a timestamp precision in seconds, so it is required to use `"precision": "s"`.

303
sinks/libgangliaSink.go
View File

@ -1,303 +0,0 @@
package sinks
/*
#cgo CFLAGS: -DGM_PROTOCOL_GUARD
#cgo LDFLAGS: -L. -Wl,--unresolved-symbols=ignore-in-object-files
#include <stdlib.h>
// This is a copy&paste snippet of ganglia.h (BSD-3 license)
// See https://github.com/ganglia/monitor-core
// for further information
enum ganglia_slope {
GANGLIA_SLOPE_ZERO = 0,
GANGLIA_SLOPE_POSITIVE,
GANGLIA_SLOPE_NEGATIVE,
GANGLIA_SLOPE_BOTH,
GANGLIA_SLOPE_UNSPECIFIED,
GANGLIA_SLOPE_DERIVATIVE,
GANGLIA_SLOPE_LAST_LEGAL_VALUE=GANGLIA_SLOPE_DERIVATIVE
};
typedef enum ganglia_slope ganglia_slope_t;
typedef struct Ganglia_pool* Ganglia_pool;
typedef struct Ganglia_gmond_config* Ganglia_gmond_config;
typedef struct Ganglia_udp_send_channels* Ganglia_udp_send_channels;
struct Ganglia_metric {
Ganglia_pool pool;
struct Ganglia_metadata_message *msg;
char *value;
void *extra;
};
typedef struct Ganglia_metric * Ganglia_metric;
#ifdef __cplusplus
extern "C" {
#endif
Ganglia_gmond_config Ganglia_gmond_config_create(char *path, int fallback_to_default);
//void Ganglia_gmond_config_destroy(Ganglia_gmond_config config);
Ganglia_udp_send_channels Ganglia_udp_send_channels_create(Ganglia_pool p, Ganglia_gmond_config config);
void Ganglia_udp_send_channels_destroy(Ganglia_udp_send_channels channels);
int Ganglia_udp_send_message(Ganglia_udp_send_channels channels, char *buf, int len );
Ganglia_metric Ganglia_metric_create( Ganglia_pool parent_pool );
int Ganglia_metric_set( Ganglia_metric gmetric, char *name, char *value, char *type, char *units, unsigned int slope, unsigned int tmax, unsigned int dmax);
int Ganglia_metric_send( Ganglia_metric gmetric, Ganglia_udp_send_channels send_channels );
//int Ganglia_metadata_send( Ganglia_metric gmetric, Ganglia_udp_send_channels send_channels );
//int Ganglia_metadata_send_real( Ganglia_metric gmetric, Ganglia_udp_send_channels send_channels, char *override_string );
void Ganglia_metadata_add( Ganglia_metric gmetric, char *name, char *value );
//int Ganglia_value_send( Ganglia_metric gmetric, Ganglia_udp_send_channels send_channels );
void Ganglia_metric_destroy( Ganglia_metric gmetric );
Ganglia_pool Ganglia_pool_create( Ganglia_pool parent );
void Ganglia_pool_destroy( Ganglia_pool pool );
//ganglia_slope_t cstr_to_slope(const char* str);
//const char* slope_to_cstr(unsigned int slope);
#ifdef __cplusplus
}
#endif
*/
import "C"
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"unsafe"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
"github.com/NVIDIA/go-nvml/pkg/dl"
)
const (
GANGLIA_LIB_NAME = "libganglia.so"
GANGLIA_LIB_DL_FLAGS = dl.RTLD_LAZY | dl.RTLD_GLOBAL
GMOND_CONFIG_FILE = `/etc/ganglia/gmond.conf`
)
// type LibgangliaSinkSpecialMetric struct {
// MetricName string `json:"metric_name,omitempty"`
// NewName string `json:"new_name,omitempty"`
// Slope string `json:"slope,omitempty"`
// }
type LibgangliaSinkConfig struct {
defaultSinkConfig
GangliaLib string `json:"libganglia_path,omitempty"`
GmondConfig string `json:"gmond_config,omitempty"`
AddGangliaGroup bool `json:"add_ganglia_group,omitempty"`
AddTypeToName bool `json:"add_type_to_name,omitempty"`
AddUnits bool `json:"add_units,omitempty"`
ClusterName string `json:"cluster_name,omitempty"`
//SpecialMetrics map[string]LibgangliaSinkSpecialMetric `json:"rename_metrics,omitempty"` // Map to rename metric name from key to value
//AddTagsAsDesc bool `json:"add_tags_as_desc,omitempty"`
}
type LibgangliaSink struct {
sink
config LibgangliaSinkConfig
global_context C.Ganglia_pool
gmond_config C.Ganglia_gmond_config
send_channels C.Ganglia_udp_send_channels
cstrCache map[string]*C.char
}
func (s *LibgangliaSink) Write(msg lp.CCMessage) error {
var err error = nil
var c_name *C.char
var c_value *C.char
var c_type *C.char
var c_unit *C.char
point, err := s.mp.ProcessMessage(msg)
if err == nil && point != nil {
// helper function for looking up C strings in the cache
lookup := func(key string) *C.char {
if _, exist := s.cstrCache[key]; !exist {
s.cstrCache[key] = C.CString(key)
}
return s.cstrCache[key]
}
conf := GetCommonGangliaConfig(point)
if len(conf.Type) == 0 {
conf = GetGangliaConfig(point)
}
if len(conf.Type) == 0 {
return fmt.Errorf("metric %q (Ganglia name %q) has no 'value' field", point.Name(), conf.Name)
}
if s.config.AddTypeToName {
conf.Name = GangliaMetricName(point)
}
c_value = C.CString(conf.Value)
c_type = lookup(conf.Type)
c_name = lookup(conf.Name)
// Add unit
unit := ""
if s.config.AddUnits {
unit = conf.Unit
}
c_unit = lookup(unit)
// Determine the slope of the metric. Ganglia's own collector mostly use
// 'both' but the mem and swap total uses 'zero'.
slope_type := C.GANGLIA_SLOPE_BOTH
switch conf.Slope {
case "zero":
slope_type = C.GANGLIA_SLOPE_ZERO
case "both":
slope_type = C.GANGLIA_SLOPE_BOTH
}
// Create a new Ganglia metric
gmetric := C.Ganglia_metric_create(s.global_context)
// Set name, value, type and unit in the Ganglia metric
// The default slope_type is both directions, so up and down. Some metrics want 'zero' slope, probably constant.
// The 'tmax' value is by default 300.
rval := C.int(0)
rval = C.Ganglia_metric_set(gmetric, c_name, c_value, c_type, c_unit, C.uint(slope_type), C.uint(conf.Tmax), 0)
switch rval {
case 1:
C.free(unsafe.Pointer(c_value))
return errors.New("invalid parameters")
case 2:
C.free(unsafe.Pointer(c_value))
return errors.New("one of your parameters has an invalid character '\"'")
case 3:
C.free(unsafe.Pointer(c_value))
return fmt.Errorf("the type parameter \"%s\" is not a valid type", conf.Type)
case 4:
C.free(unsafe.Pointer(c_value))
return fmt.Errorf("the value parameter \"%s\" does not represent a number", conf.Value)
default:
}
// Set the cluster name, otherwise it takes it from the configuration file
if len(s.config.ClusterName) > 0 {
C.Ganglia_metadata_add(gmetric, lookup("CLUSTER"), lookup(s.config.ClusterName))
}
// Set the group metadata in the Ganglia metric if configured
if s.config.AddGangliaGroup {
c_group := lookup(conf.Group)
C.Ganglia_metadata_add(gmetric, lookup("GROUP"), c_group)
}
// Now we send the metric
// gmetric does provide some more options like description and other options
// but they are not provided by the collectors
rval = C.Ganglia_metric_send(gmetric, s.send_channels)
if rval != 0 {
err = fmt.Errorf("there was an error sending metric %s to %d of the send channels ", point.Name(), rval)
// fall throuph to use Ganglia_metric_destroy from common cleanup
}
// Cleanup Ganglia metric
C.Ganglia_metric_destroy(gmetric)
// Free the value C string, the only one not stored in the cache
C.free(unsafe.Pointer(c_value))
}
return err
}
func (s *LibgangliaSink) Flush() error {
return nil
}
func (s *LibgangliaSink) Close() {
// Destroy Ganglia configration struct
// (not done by gmetric, I thought I am more clever but no...)
//C.Ganglia_gmond_config_destroy(s.gmond_config)
// Destroy Ganglia pool
C.Ganglia_pool_destroy(s.global_context)
// Cleanup C string cache
for _, cstr := range s.cstrCache {
C.free(unsafe.Pointer(cstr))
}
}
func NewLibgangliaSink(name string, config json.RawMessage) (Sink, error) {
s := new(LibgangliaSink)
var err error = nil
s.name = fmt.Sprintf("LibgangliaSink(%s)", name)
//s.config.AddTagsAsDesc = false
s.config.AddGangliaGroup = false
s.config.AddTypeToName = false
s.config.AddUnits = true
s.config.GmondConfig = string(GMOND_CONFIG_FILE)
s.config.GangliaLib = string(GANGLIA_LIB_NAME)
if len(config) > 0 {
d := json.NewDecoder(bytes.NewReader(config))
d.DisallowUnknownFields()
if err := d.Decode(&s.config); err != nil {
cclog.ComponentError(s.name, "Error reading config:", err.Error())
return nil, err
}
}
p, err := mp.NewMessageProcessor()
if err != nil {
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
s.mp = p
if len(s.config.MessageProcessor) > 0 {
err = s.mp.FromConfigJSON(s.config.MessageProcessor)
if err != nil {
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
for _, k := range s.config.MetaAsTags {
s.mp.AddMoveMetaToTags("true", k, k)
}
lib := dl.New(s.config.GangliaLib, GANGLIA_LIB_DL_FLAGS)
if lib == nil {
return nil, fmt.Errorf("error instantiating DynamicLibrary for %s", s.config.GangliaLib)
}
err = lib.Open()
if err != nil {
return nil, fmt.Errorf("error opening %s: %v", s.config.GangliaLib, err)
}
// Set up cache for the C strings
s.cstrCache = make(map[string]*C.char)
// s.cstrCache["globals"] = C.CString("globals")
// s.cstrCache["override_hostname"] = C.CString("override_hostname")
// s.cstrCache["override_ip"] = C.CString("override_ip")
// Add some constant strings
s.cstrCache["GROUP"] = C.CString("GROUP")
s.cstrCache["CLUSTER"] = C.CString("CLUSTER")
s.cstrCache[""] = C.CString("")
// Add cluster name for lookup in Write()
if len(s.config.ClusterName) > 0 {
s.cstrCache[s.config.ClusterName] = C.CString(s.config.ClusterName)
}
// Add supported types for later lookup in Write()
s.cstrCache["double"] = C.CString("double")
s.cstrCache["int32"] = C.CString("int32")
s.cstrCache["string"] = C.CString("string")
// Create Ganglia pool
s.global_context = C.Ganglia_pool_create(nil)
// Load Ganglia configuration
s.cstrCache[s.config.GmondConfig] = C.CString(s.config.GmondConfig)
s.gmond_config = C.Ganglia_gmond_config_create(s.cstrCache[s.config.GmondConfig], 0)
//globals := C.cfg_getsec(gmond_config, s.cstrCache["globals"])
//override_hostname := C.cfg_getstr(globals, s.cstrCache["override_hostname"])
//override_ip := C.cfg_getstr(globals, s.cstrCache["override_ip"])
s.send_channels = C.Ganglia_udp_send_channels_create(s.global_context, s.gmond_config)
return s, nil
}

46
sinks/libgangliaSink.md
View File

@ -1,46 +0,0 @@
## `libganglia` sink
The `libganglia` sink interacts directly with the library of the [Ganglia Monitoring System](http://ganglia.info/) to submit the metrics. Consequently, it needs to be installed on all nodes. But this is commonly the case if you want to use Ganglia, because it requires at least a node daemon (`gmond` or `ganglia-monitor`) to work.
The `libganglia` sink has probably less overhead compared to the `ganglia` sink because it does not require any process generation but initializes the environment and UDP connections only once.
### Configuration structure
```json
{
"<name>": {
"type": "libganglia",
"gmetric_config" : "/path/to/gmetric/config",
"cluster_name": "MyCluster",
"add_ganglia_group" : true,
"add_type_to_name": true,
"add_units" : true,
"process_messages" : {
"see" : "docs of message processor for valid fields"
},
"meta_as_tags" : []
}
}
```
- `type`: makes the sink an `libganglia` sink
- `gmond_config`: Path to the Ganglia configuration file `gmond.conf` (default: `/etc/ganglia/gmond.conf`)
- `cluster_name`: Set a cluster name for the metric. If not set, it is taken from `gmond_config`
- `add_ganglia_group`: Add a Ganglia metric group based on meta information. Some old versions of `gmetric` do not support the `--group` option
- `add_type_to_name`: Ganglia commonly uses only node-level metrics but with cc-metric-collector, there are metrics for cpus, memory domains, CPU sockets and the whole node. In order to get eeng, this option prefixes the metric name with `<type><type-id>_` or `device_` depending on the metric tags and meta information. For metrics of the whole node `type=node`, no prefix is added
- `add_units`: Add metric value unit if there is a `unit` entry in the metric tags or meta information
- `process_messages`: Process messages with given rules before progressing or dropping, see [here](../pkg/messageProcessor/README.md) (optional)
- `meta_as_tags`: print all meta information as tags in the output (deprecated, optional)
### Ganglia Installation
My development system is Ubuntu 20.04. To install the required libraries with `apt`:
```
$ sudo apt install libganglia1
```
The `libganglia.so` gets installed in `/usr/lib`. The Ganglia headers `libganglia1-dev` are **not** required.
I added a `Makefile` in the `sinks` subfolder that searches for the library in `/usr` and creates a symlink (`sinks/libganglia.so`) for running/building the cc-metric-collector. So just type `make` before running/building in the main folder or the `sinks` subfolder.

88
sinks/metricSink.go
View File

@ -1,88 +0,0 @@
package sinks
import (
"encoding/json"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
influx "github.com/influxdata/line-protocol/v2/lineprotocol"
"golang.org/x/exp/slices"
)
type defaultSinkConfig struct {
MetaAsTags []string `json:"meta_as_tags,omitempty"`
MessageProcessor json.RawMessage `json:"process_messages,omitempty"`
Type string `json:"type"`
}
type sink struct {
meta_as_tags map[string]bool // Use meta data tags as tags
mp mp.MessageProcessor // message processor for the sink
name string // Name of the sink
}
type Sink interface {
Write(point lp.CCMessage) error // Write metric to the sink
Flush() error // Flush buffered metrics
Close() // Close / finish metric sink
Name() string // Name of the metric sink
}
// Name returns the name of the metric sink
func (s *sink) Name() string {
return s.name
}
type key_value_pair struct {
key string
value string
}
func EncoderAdd(encoder *influx.Encoder, msg lp.CCMessage) error {
// Encode measurement name
encoder.StartLine(msg.Name())
tag_list := make([]key_value_pair, 0, 10)
// copy tags and meta data which should be used as tags
for key, value := range msg.Tags() {
tag_list =
append(
tag_list,
key_value_pair{
key: key,
value: value,
},
)
}
// Encode tags (they musts be in lexical order)
slices.SortFunc(
tag_list,
func(a key_value_pair, b key_value_pair) int {
if a.key < b.key {
return -1
}
if a.key > b.key {
return +1
}
return 0
},
)
for i := range tag_list {
encoder.AddTag(
tag_list[i].key,
tag_list[i].value,
)
}
// Encode fields
for key, value := range msg.Fields() {
encoder.AddField(key, influx.MustNewValue(value))
}
// Encode time stamp
encoder.EndLine(msg.Time())
// Return encoder errors
return encoder.Err()
}

226
sinks/natsSink.go
View File

@ -1,226 +0,0 @@
package sinks
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"sync"
"time"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
influx "github.com/influxdata/line-protocol/v2/lineprotocol"
nats "github.com/nats-io/nats.go"
"golang.org/x/exp/slices"
)
type NatsSinkConfig struct {
defaultSinkConfig
Host string `json:"host,omitempty"`
Port string `json:"port,omitempty"`
Subject string `json:"subject,omitempty"`
User string `json:"user,omitempty"`
Password string `json:"password,omitempty"`
FlushDelay string `json:"flush_delay,omitempty"`
flushDelay time.Duration
NkeyFile string `json:"nkey_file,omitempty"`
// Timestamp precision
Precision string `json:"precision,omitempty"`
}
type NatsSink struct {
sink
client *nats.Conn
encoder influx.Encoder
encoderLock sync.Mutex
config NatsSinkConfig
flushTimer *time.Timer
timerLock sync.Mutex
}
func (s *NatsSink) connect() error {
var err error
var uinfo nats.Option = nil
var nc *nats.Conn
if len(s.config.User) > 0 && len(s.config.Password) > 0 {
uinfo = nats.UserInfo(s.config.User, s.config.Password)
} else if len(s.config.NkeyFile) > 0 {
if _, err := os.Stat(s.config.NkeyFile); err == nil {
uinfo = nats.UserCredentials(s.config.NkeyFile)
} else {
cclog.ComponentError(s.name, "NKEY file", s.config.NkeyFile, "does not exist: %v", err.Error())
return err
}
}
uri := fmt.Sprintf("nats://%s:%s", s.config.Host, s.config.Port)
cclog.ComponentDebug(s.name, "Connect to", uri)
s.client = nil
if uinfo != nil {
nc, err = nats.Connect(uri, uinfo)
} else {
nc, err = nats.Connect(uri)
}
if err != nil {
cclog.ComponentError(s.name, "Connect to", uri, "failed:", err.Error())
return err
}
s.client = nc
return nil
}
func (s *NatsSink) Write(m lp.CCMessage) error {
msg, err := s.mp.ProcessMessage(m)
if err == nil && msg != nil {
// Lock for encoder usage
s.encoderLock.Lock()
// Add message to encoder
err = EncoderAdd(&s.encoder, m)
// Unlock encoder usage
s.encoderLock.Unlock()
// Check that encoding worked
if err != nil {
cclog.ComponentError(s.name, "Write:", err.Error())
return err
}
}
if s.config.flushDelay == 0 {
// Directly flush if no flush delay is configured
return s.Flush()
} else if s.timerLock.TryLock() {
// Setup flush timer when flush delay is configured
// and no other timer is already running
if s.flushTimer != nil {
// Restarting existing flush timer
cclog.ComponentDebug(s.name, "Write(): Restarting flush timer")
s.flushTimer.Reset(s.config.flushDelay)
} else {
// Creating and starting flush timer
cclog.ComponentDebug(s.name, "Write(): Starting new flush timer")
s.flushTimer = time.AfterFunc(
s.config.flushDelay,
func() {
defer s.timerLock.Unlock()
cclog.ComponentDebug(s.name, "Starting flush triggered by flush timer")
if err := s.Flush(); err != nil {
cclog.ComponentError(s.name, "Flush triggered by flush timer: flush failed:", err)
}
})
}
}
return nil
}
func (s *NatsSink) Flush() error {
// Lock for encoder usage
// Own lock for as short as possible: the time it takes to clone the buffer.
s.encoderLock.Lock()
buf := slices.Clone(s.encoder.Bytes())
s.encoder.Reset()
// Unlock encoder usage
s.encoderLock.Unlock()
if len(buf) == 0 {
return nil
}
if err := s.client.Publish(s.config.Subject, buf); err != nil {
cclog.ComponentError(s.name, "Flush:", err.Error())
return err
}
return nil
}
func (s *NatsSink) Close() {
// Stop existing timer and immediately flush
if s.flushTimer != nil {
if ok := s.flushTimer.Stop(); ok {
s.timerLock.Unlock()
}
}
cclog.ComponentDebug(s.name, "Close NATS connection")
s.client.Close()
}
func NewNatsSink(name string, config json.RawMessage) (Sink, error) {
s := new(NatsSink)
s.name = fmt.Sprintf("NatsSink(%s)", name)
s.config.flushDelay = 5 * time.Second
s.config.FlushDelay = "5s"
s.config.Port = "4222"
s.config.Precision = "s"
if len(config) > 0 {
d := json.NewDecoder(bytes.NewReader(config))
d.DisallowUnknownFields()
if err := d.Decode(&s.config); err != nil {
cclog.ComponentError(s.name, "Error reading config:", err.Error())
return nil, err
}
}
if len(s.config.Host) == 0 ||
len(s.config.Port) == 0 ||
len(s.config.Subject) == 0 {
return nil, errors.New("not all configuration variables set required by NatsSink")
}
// Create a new message processor
p, err := mp.NewMessageProcessor()
if err != nil {
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
s.mp = p
// Read config related to message processor
if len(s.config.MessageProcessor) > 0 {
err = s.mp.FromConfigJSON(s.config.MessageProcessor)
if err != nil {
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
// Add meta_as_tags list to message processor
for _, k := range s.config.MetaAsTags {
s.mp.AddMoveMetaToTags("true", k, k)
}
// Setup Influx line protocol encoder
precision := influx.Second
if len(s.config.Precision) > 0 {
switch s.config.Precision {
case "s":
precision = influx.Second
case "ms":
precision = influx.Millisecond
case "us":
precision = influx.Microsecond
case "ns":
precision = influx.Nanosecond
}
}
s.encoder.SetPrecision(precision)
// Setup infos for connection
if err := s.connect(); err != nil {
return nil, fmt.Errorf("unable to connect: %v", err)
}
s.flushTimer = nil
if len(s.config.FlushDelay) > 0 {
t, err := time.ParseDuration(s.config.FlushDelay)
if err == nil {
s.config.flushDelay = t
cclog.ComponentDebug(s.name, "Init(): flushDelay", t)
}
}
return s, nil
}

41
sinks/natsSink.md
View File

@ -1,41 +0,0 @@
## `nats` sink
The `nats` sink publishes all metrics into a NATS network. The publishing key is the database name provided in the configuration file
### Configuration structure
```json
{
"<name>": {
"type": "nats",
"database" : "mymetrics",
"host": "dbhost.example.com",
"port": "4222",
"user": "exampleuser",
"password" : "examplepw",
"nkey_file": "/path/to/nkey_file",
"flush_delay": "10s",
"precision": "s",
"process_messages" : {
"see" : "docs of message processor for valid fields"
},
"meta_as_tags" : []
}
}
```
- `type`: makes the sink an `nats` sink
- `database`: All metrics are published with this subject
- `host`: Hostname of the NATS server
- `port`: Port number (as string) of the NATS server
- `user`: Username for basic authentication
- `password`: Password for basic authentication
- `nkey_file`: Path to credentials file with NKEY
- `flush_delay`: Maximum time until metrics are sent out (default '5s')
- `precision`: Precision of the timestamp. Valid values are 's', 'ms', 'us' and 'ns'. (default is 's')
- `process_messages`: Process messages with given rules before progressing or dropping, see [here](../pkg/messageProcessor/README.md) (optional)
- `meta_as_tags`: print all meta information as tags in the output (deprecated, optional)
### Using `nats` sink for communication with cc-metric-store
The cc-metric-store only accepts metrics with a timestamp precision in seconds, so it is required to use `"precision": "s"`.

222
sinks/prometheusSink.go
View File

@ -1,222 +0,0 @@
package sinks
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"sync"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
type PrometheusSinkConfig struct {
defaultSinkConfig
Host string `json:"host,omitempty"`
Port string `json:"port"`
Path string `json:"path,omitempty"`
GroupAsNameSpace bool `json:"group_as_namespace,omitempty"`
// User string `json:"user,omitempty"`
// Password string `json:"password,omitempty"`
// FlushDelay string `json:"flush_delay,omitempty"`
}
type PrometheusSink struct {
sink
config PrometheusSinkConfig
labelMetrics map[string]*prometheus.GaugeVec
nodeMetrics map[string]prometheus.Gauge
promWg sync.WaitGroup
promServer *http.Server
}
func intToFloat64(input interface{}) (float64, error) {
switch value := input.(type) {
case float64:
return value, nil
case float32:
return float64(value), nil
case int:
return float64(value), nil
case int32:
return float64(value), nil
case int64:
return float64(value), nil
case uint64:
return float64(value), nil
}
return 0, errors.New("cannot cast value to float64")
}
func getLabelValue(metric lp.CCMessage) []string {
labelValues := []string{}
if tid, tidok := metric.GetTag("type-id"); tidok && metric.HasTag("type") {
labelValues = append(labelValues, tid)
}
if d, ok := metric.GetTag("device"); ok {
labelValues = append(labelValues, d)
} else if d, ok := metric.GetMeta("device"); ok {
labelValues = append(labelValues, d)
}
return labelValues
}
func getLabelNames(metric lp.CCMessage) []string {
labelNames := []string{}
if t, tok := metric.GetTag("type"); tok && metric.HasTag("type-id") {
labelNames = append(labelNames, t)
}
if _, ok := metric.GetTag("device"); ok {
labelNames = append(labelNames, "device")
} else if _, ok := metric.GetMeta("device"); ok {
labelNames = append(labelNames, "device")
}
return labelNames
}
func (s *PrometheusSink) newMetric(metric lp.CCMessage) error {
var value float64 = 0
name := metric.Name()
opts := prometheus.GaugeOpts{
Name: name,
}
labels := getLabelNames(metric)
labelValues := getLabelValue(metric)
if len(labels) > 0 && len(labels) != len(labelValues) {
return fmt.Errorf("cannot detect metric labels for metric %s", name)
}
if metricValue, ok := metric.GetField("value"); ok {
if floatValue, err := intToFloat64(metricValue); err == nil {
value = floatValue
} else {
return fmt.Errorf("metric %s with value '%v' cannot be casted to float64", name, metricValue)
}
}
if s.config.GroupAsNameSpace && metric.HasMeta("group") {
g, _ := metric.GetMeta("group")
opts.Namespace = strings.ToLower(g)
}
if len(labels) > 0 {
new := prometheus.NewGaugeVec(opts, labels)
new.WithLabelValues(labelValues...).Set(value)
s.labelMetrics[name] = new
prometheus.Register(new)
} else {
new := prometheus.NewGauge(opts)
new.Set(value)
s.nodeMetrics[name] = new
prometheus.Register(new)
}
return nil
}
func (s *PrometheusSink) updateMetric(metric lp.CCMessage) error {
var value float64 = 0.0
name := metric.Name()
labelValues := getLabelValue(metric)
if metricValue, ok := metric.GetField("value"); ok {
if floatValue, err := intToFloat64(metricValue); err == nil {
value = floatValue
} else {
return fmt.Errorf("metric %s with value '%v' cannot be casted to float64", name, metricValue)
}
}
if len(labelValues) > 0 {
if _, ok := s.labelMetrics[name]; !ok {
err := s.newMetric(metric)
if err != nil {
return err
}
}
s.labelMetrics[name].WithLabelValues(labelValues...).Set(value)
} else {
if _, ok := s.labelMetrics[name]; !ok {
err := s.newMetric(metric)
if err != nil {
return err
}
}
s.nodeMetrics[name].Set(value)
}
return nil
}
func (s *PrometheusSink) Write(m lp.CCMessage) error {
msg, err := s.mp.ProcessMessage(m)
if err == nil && msg != nil {
err = s.updateMetric(m)
}
return err
}
func (s *PrometheusSink) Flush() error {
return nil
}
func (s *PrometheusSink) Close() {
cclog.ComponentDebug(s.name, "CLOSE")
s.promServer.Shutdown(context.Background())
s.promWg.Wait()
}
func NewPrometheusSink(name string, config json.RawMessage) (Sink, error) {
s := new(PrometheusSink)
s.name = "PrometheusSink"
if len(config) > 0 {
d := json.NewDecoder(bytes.NewReader(config))
d.DisallowUnknownFields()
if err := d.Decode(&s.config); err != nil {
cclog.ComponentError(s.name, "Error reading config:", err.Error())
return nil, err
}
}
if len(s.config.Port) == 0 {
err := errors.New("not all configuration variables set required by PrometheusSink")
cclog.ComponentError(s.name, err.Error())
return nil, err
}
p, err := mp.NewMessageProcessor()
if err != nil {
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
s.mp = p
if len(s.config.MessageProcessor) > 0 {
err = p.FromConfigJSON(s.config.MessageProcessor)
if err != nil {
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
for _, k := range s.config.MetaAsTags {
s.mp.AddMoveMetaToTags("true", k, k)
}
s.labelMetrics = make(map[string]*prometheus.GaugeVec)
s.nodeMetrics = make(map[string]prometheus.Gauge)
s.promWg.Add(1)
go func() {
router := mux.NewRouter()
// Prometheus endpoint
router.Path("/" + s.config.Path).Handler(promhttp.Handler())
url := fmt.Sprintf("%s:%s", s.config.Host, s.config.Port)
cclog.ComponentDebug(s.name, "Serving Prometheus metrics at", fmt.Sprintf("%s:%s/%s", s.config.Host, s.config.Port, s.config.Path))
s.promServer = &http.Server{Addr: url, Handler: router}
err := s.promServer.ListenAndServe()
if err != nil && err.Error() != "http: Server closed" {
cclog.ComponentError(s.name, err.Error())
}
s.promWg.Done()
}()
return s, nil
}

29
sinks/prometheusSink.md
View File

@ -1,29 +0,0 @@
## `prometheus` sink
The `prometheus` sink publishes all metrics via an HTTP server ready to be scraped by a [Prometheus](https://prometheus.io) server. It creates gauge metrics for all node metrics and gauge vectors for all metrics with a subtype like 'device', 'cpu' or 'socket'.
### Configuration structure
```json
{
"<name>": {
"type": "prometheus",
"host": "localhost",
"port": "8080",
"path": "metrics",
"process_messages" : {
"see" : "docs of message processor for valid fields"
},
"meta_as_tags" : []
}
}
```
- `type`: makes the sink an `prometheus` sink
- `host`: The HTTP server gets bound to that IP/hostname
- `port`: Portnumber (as string) for the HTTP server
- `path`: Path where the metrics should be servered. The metrics will be published at `host`:`port`/`path`
- `group_as_namespace`: Most metrics contain a group as meta information like 'memory', 'load'. With this the metric names are extended to `group`_`name` if possible.
- `process_messages`: Process messages with given rules before progressing or dropping, see [here](../pkg/messageProcessor/README.md) (optional)
- `meta_as_tags`: print all meta information as tags in the output (deprecated, optional)

103
sinks/sampleSink.go
View File

@ -1,103 +0,0 @@
package sinks
import (
"bytes"
"encoding/json"
"fmt"
"log"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
)
type SampleSinkConfig struct {
// defines JSON tags for 'type' and 'meta_as_tags' (string list)
// See: metricSink.go
defaultSinkConfig
// Additional config options, for SampleSink
}
type SampleSink struct {
// declares elements 'name' and 'meta_as_tags' (string to bool map!)
sink
config SampleSinkConfig // entry point to the SampleSinkConfig
}
// Implement functions required for Sink interface
// Write(...), Flush(), Close()
// See: metricSink.go
// Code to submit a single CCMetric to the sink
func (s *SampleSink) Write(point lp.CCMessage) error {
// based on s.meta_as_tags use meta infos as tags
// moreover, submit the point to the message processor
// to apply drop/modify rules
msg, err := s.mp.ProcessMessage(point)
if err == nil && msg != nil {
log.Print(msg)
}
return nil
}
// If the sink uses batched sends internally, you can tell to flush its buffers
func (s *SampleSink) Flush() error {
return nil
}
// Close sink: close network connection, close files, close libraries, ...
func (s *SampleSink) Close() {
cclog.ComponentDebug(s.name, "CLOSE")
}
// New function to create a new instance of the sink
// Initialize the sink by giving it a name and reading in the config JSON
func NewSampleSink(name string, config json.RawMessage) (Sink, error) {
s := new(SampleSink)
// Set name of sampleSink
// The name should be chosen in such a way that different instances of SampleSink can be distinguished
s.name = fmt.Sprintf("SampleSink(%s)", name) // Always specify a name here
// Set defaults in s.config
// Allow overwriting these defaults by reading config JSON
// Read in the config JSON
if len(config) > 0 {
d := json.NewDecoder(bytes.NewReader(config))
d.DisallowUnknownFields()
if err := d.Decode(&s.config); err != nil {
cclog.ComponentError(s.name, "Error reading config:", err.Error())
return nil, err
}
}
// Initialize and configure the message processor
p, err := mp.NewMessageProcessor()
if err != nil {
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
s.mp = p
// Add message processor configuration
if len(s.config.MessageProcessor) > 0 {
err = p.FromConfigJSON(s.config.MessageProcessor)
if err != nil {
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
// Add rules to move meta information to tag space
// Replacing the legacy 'meta_as_tags' configuration
for _, k := range s.config.MetaAsTags {
s.mp.AddMoveMetaToTags("true", k, k)
}
// Check if all required fields in the config are set
// E.g. use 'len(s.config.Option) > 0' for string settings
// Establish connection to the server, library, ...
// Check required files exist and lookup path(s) of executable(s)
// Return (nil, meaningful error message) in case of errors
return s, nil
}

185
sinks/sinkManager.go
View File

@ -1,185 +0,0 @@
package sinks
import (
"encoding/json"
"fmt"
"os"
"sync"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
)
const SINK_MAX_FORWARD = 50
// Map of all available sinks
var AvailableSinks = map[string]func(name string, config json.RawMessage) (Sink, error){
"ganglia": NewGangliaSink,
"libganglia": NewLibgangliaSink,
"stdout": NewStdoutSink,
"nats": NewNatsSink,
"influxdb": NewInfluxSink,
"influxasync": NewInfluxAsyncSink,
"http": NewHttpSink,
"prometheus": NewPrometheusSink,
}
// Metric collector manager data structure
type sinkManager struct {
input chan lp.CCMessage // input channel
done chan bool // channel to finish / stop metric sink manager
wg *sync.WaitGroup // wait group for all goroutines in cc-metric-collector
sinks map[string]Sink // Mapping sink name to sink
maxForward int // number of metrics to write maximally in one iteration
}
// Sink manager access functions
type SinkManager interface {
Init(wg *sync.WaitGroup, sinkConfigFile string) error
AddInput(input chan lp.CCMessage)
AddOutput(name string, config json.RawMessage) error
Start()
Close()
}
// Init initializes the sink manager by:
// * Reading its configuration file
// * Adding the configured sinks and providing them with the corresponding config
func (sm *sinkManager) Init(wg *sync.WaitGroup, sinkConfigFile string) error {
sm.input = nil
sm.done = make(chan bool)
sm.wg = wg
sm.sinks = make(map[string]Sink, 0)
sm.maxForward = SINK_MAX_FORWARD
if len(sinkConfigFile) == 0 {
return nil
}
// Read sink config file
configFile, err := os.Open(sinkConfigFile)
if err != nil {
cclog.ComponentError("SinkManager", err.Error())
return err
}
defer configFile.Close()
// Parse config
jsonParser := json.NewDecoder(configFile)
var rawConfigs map[string]json.RawMessage
err = jsonParser.Decode(&rawConfigs)
if err != nil {
cclog.ComponentError("SinkManager", err.Error())
return err
}
// Start sinks
for name, raw := range rawConfigs {
err = sm.AddOutput(name, raw)
if err != nil {
cclog.ComponentError("SinkManager", err)
continue
}
}
// Check that at least one sink is running
if !(len(sm.sinks) > 0) {
cclog.ComponentError("SinkManager", "Found no usable sinks")
return fmt.Errorf("found no usable sinks")
}
return nil
}
// Start starts the sink managers background task, which
// distributes received metrics to the sinks
func (sm *sinkManager) Start() {
sm.wg.Add(1)
go func() {
defer sm.wg.Done()
// Sink manager is done
done := func() {
for _, s := range sm.sinks {
s.Close()
}
close(sm.done)
cclog.ComponentDebug("SinkManager", "DONE")
}
toTheSinks := func(p lp.CCMessage) {
// Send received metric to all outputs
cclog.ComponentDebug("SinkManager", "WRITE", p)
for _, s := range sm.sinks {
if err := s.Write(p); err != nil {
cclog.ComponentError("SinkManager", "WRITE", s.Name(), "write failed:", err.Error())
}
}
}
for {
select {
case <-sm.done:
done()
return
case p := <-sm.input:
toTheSinks(p)
for i := 0; len(sm.input) > 0 && i < sm.maxForward; i++ {
p := <-sm.input
toTheSinks(p)
}
}
}
}()
// Sink manager is started
cclog.ComponentDebug("SinkManager", "STARTED")
}
// AddInput adds the input channel to the sink manager
func (sm *sinkManager) AddInput(input chan lp.CCMessage) {
sm.input = input
}
func (sm *sinkManager) AddOutput(name string, rawConfig json.RawMessage) error {
var err error
var sinkConfig defaultSinkConfig
if len(rawConfig) > 0 {
err := json.Unmarshal(rawConfig, &sinkConfig)
if err != nil {
return err
}
}
if _, found := AvailableSinks[sinkConfig.Type]; !found {
cclog.ComponentError("SinkManager", "SKIP", name, "unknown sink:", sinkConfig.Type)
return err
}
s, err := AvailableSinks[sinkConfig.Type](name, rawConfig)
if err != nil {
cclog.ComponentError("SinkManager", "SKIP", name, "initialization failed:", err.Error())
return err
}
sm.sinks[name] = s
cclog.ComponentDebug("SinkManager", "ADD SINK", s.Name(), "with name", fmt.Sprintf("'%s'", name))
return nil
}
// Close finishes / stops the sink manager
func (sm *sinkManager) Close() {
cclog.ComponentDebug("SinkManager", "CLOSE")
sm.done <- true
// wait for close of channel sm.done
<-sm.done
}
// New creates a new initialized sink manager
func New(wg *sync.WaitGroup, sinkConfigFile string) (SinkManager, error) {
sm := new(sinkManager)
err := sm.Init(wg, sinkConfigFile)
if err != nil {
return nil, err
}
return sm, err
}

98
sinks/stdoutSink.go
View File

@ -1,98 +0,0 @@
package sinks
import (
"bytes"
"encoding/json"
"fmt"
"os"
"strings"
// "time"
lp "github.com/ClusterCockpit/cc-lib/ccMessage"
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
)
type StdoutSink struct {
sink // meta_as_tags, name
output *os.File
config struct {
defaultSinkConfig
Output string `json:"output_file,omitempty"`
}
}
func (s *StdoutSink) Write(m lp.CCMessage) error {
msg, err := s.mp.ProcessMessage(m)
if err == nil && msg != nil {
fmt.Fprint(
s.output,
msg.ToLineProtocol(s.meta_as_tags),
)
}
return nil
}
func (s *StdoutSink) Flush() error {
s.output.Sync()
return nil
}
func (s *StdoutSink) Close() {
if s.output != os.Stdout && s.output != os.Stderr {
s.output.Close()
}
}
func NewStdoutSink(name string, config json.RawMessage) (Sink, error) {
s := new(StdoutSink)
s.name = fmt.Sprintf("StdoutSink(%s)", name)
if len(config) > 0 {
d := json.NewDecoder(bytes.NewReader(config))
d.DisallowUnknownFields()
if err := d.Decode(&s.config); err != nil {
cclog.ComponentError(s.name, "Error reading config:", err.Error())
return nil, err
}
}
p, err := mp.NewMessageProcessor()
if err != nil {
return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
}
s.mp = p
s.output = os.Stdout
if len(s.config.Output) > 0 {
switch strings.ToLower(s.config.Output) {
case "stdout":
s.output = os.Stdout
case "stderr":
s.output = os.Stderr
default:
f, err := os.OpenFile(s.config.Output, os.O_CREATE|os.O_WRONLY, os.FileMode(0600))
if err != nil {
return nil, err
}
s.output = f
}
}
// Add message processor configuration
if len(s.config.MessageProcessor) > 0 {
err = s.mp.FromConfigJSON(s.config.MessageProcessor)
if err != nil {
return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
}
}
// Create lookup map to use meta infos as tags in the output metric
// s.meta_as_tags = make(map[string]bool)
// for _, k := range s.config.MetaAsTags {
// s.meta_as_tags[k] = true
// }
for _, k := range s.config.MetaAsTags {
s.mp.AddMoveMetaToTags("true", k, k)
}
return s, nil
}

27
sinks/stdoutSink.md
View File

@ -1,27 +0,0 @@
## `stdout` sink
The `stdout` sink is the most simple sink provided by cc-metric-collector. It writes all metrics in InfluxDB line-procol format to the configurable output file or the common special files `stdout` and `stderr`.
### Configuration structure
```json
{
"<name>": {
"type": "stdout",
"meta_as_tags" : [],
"output_file" : "mylogfile.log",
"process_messages" : {
"see" : "docs of message processor for valid fields"
},
"meta_as_tags" : []
}
}
```
- `type`: makes the sink an `stdout` sink
- `meta_as_tags`: print meta information as tags in the output (optional)
- `output_file`: Write all data to the selected file (optional). There are two 'special' files: `stdout` and `stderr`. If this option is not provided, the default value is `stdout`
- `process_messages`: Process messages with given rules before progressing or dropping, see [here](../pkg/messageProcessor/README.md) (optional)
- `meta_as_tags`: print all meta information as tags in the output (deprecated, optional)