ClusterCockpit/cc-metric-collector
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-metric-collector.git synced 2025-08-14 14:52:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

Formatting

Browse Source
This commit is contained in:
Thomas Roehl
2021-12-21 14:04:31 +01:00
parent 988cea381e
commit a6feb16ec1
30 changed files with 775 additions and 793 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
sinks/httpSink.go
View File

@@ -7,8 +7,8 @@ import (
"net/http"
"time"
influx "github.com/influxdata/line-protocol"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
influx "github.com/influxdata/line-protocol"
)
type HttpSink struct {
@@ -20,7 +20,7 @@ type HttpSink struct {
}
func (s *HttpSink) Init(config sinkConfig) error {
s.name = "HttpSink"
s.name = "HttpSink"
if len(config.Host) == 0 || len(config.Port) == 0 || len(config.Database) == 0 {
return errors.New("`host`, `port` and `database` config options required for TCP sink")
}

8
sinks/influxSink.go
View File

@@ -39,7 +39,7 @@ func (s *InfluxSink) connect() error {
}
func (s *InfluxSink) Init(config sinkConfig) error {
s.name = "InfluxSink"
s.name = "InfluxSink"
if len(config.Host) == 0 ||
len(config.Port) == 0 ||
len(config.Database) == 0 ||
@@ -65,9 +65,9 @@ func (s *InfluxSink) Write(point lp.CCMetric) error {
tags[t.Key] = t.Value
}
if s.meta_as_tags {
for _, m := range point.MetaList() {
tags[m.Key] = m.Value
}
for _, m := range point.MetaList() {
tags[m.Key] = m.Value
}
}
for _, f := range point.FieldList() {
fields[f.Key] = f.Value

2
sinks/metricSink.go
View File

@@ -6,7 +6,7 @@ import (
)
type sinkConfig struct {
Type string `json:"type"`
Type string `json:"type"`
Host string `json:"host", omitempty`
Port string `json:"port", omitempty`
Database string `json:"database, omitempty"`

4
sinks/natsSink.go
View File

@@ -4,8 +4,8 @@ import (
"bytes"
"errors"
"fmt"
influx "github.com/influxdata/line-protocol"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
influx "github.com/influxdata/line-protocol"
nats "github.com/nats-io/nats.go"
"log"
"time"
@@ -33,7 +33,7 @@ func (s *NatsSink) connect() error {
}
func (s *NatsSink) Init(config sinkConfig) error {
s.name = "NatsSink"
s.name = "NatsSink"
if len(config.Host) == 0 ||
len(config.Port) == 0 ||
len(config.Database) == 0 {

211
sinks/sinkManager.go
View File

@@ -1,17 +1,16 @@
package sinks
import (
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"sync"
"log"
"os"
"encoding/json"
"encoding/json"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"log"
"os"
"sync"
)
type SinkEntity struct {
config json.RawMessage
output Sink
config json.RawMessage
output Sink
}
var AvailableSinks = map[string]Sink{
@@ -21,13 +20,12 @@ var AvailableSinks = map[string]Sink{
"http": &HttpSink{},
}
type sinkManager struct {
input chan lp.CCMetric
outputs []Sink
done chan bool
wg *sync.WaitGroup
config []sinkConfig
input chan lp.CCMetric
outputs []Sink
done chan bool
wg *sync.WaitGroup
config []sinkConfig
}
type SinkManager interface {
@@ -38,114 +36,111 @@ type SinkManager interface {
Close()
}
func (sm *sinkManager) Init(wg *sync.WaitGroup, sinkConfigFile string) error {
sm.input = nil
sm.outputs = make([]Sink, 0)
sm.done = make(chan bool)
sm.wg = wg
sm.config = make([]sinkConfig, 0)
if len(sinkConfigFile) > 0 {
configFile, err := os.Open(sinkConfigFile)
if err != nil {
log.Print("[SinkManager] ", err.Error())
return err
}
defer configFile.Close()
jsonParser := json.NewDecoder(configFile)
var rawConfigs []json.RawMessage
err = jsonParser.Decode(&rawConfigs)
if err != nil {
log.Print("[SinkManager] ", err.Error())
return err
}
for _, raw := range rawConfigs {
err = sm.AddOutput(raw)
if err != nil {
continue
}
}
}
return nil
sm.input = nil
sm.outputs = make([]Sink, 0)
sm.done = make(chan bool)
sm.wg = wg
sm.config = make([]sinkConfig, 0)
if len(sinkConfigFile) > 0 {
configFile, err := os.Open(sinkConfigFile)
if err != nil {
log.Print("[SinkManager] ", err.Error())
return err
}
defer configFile.Close()
jsonParser := json.NewDecoder(configFile)
var rawConfigs []json.RawMessage
err = jsonParser.Decode(&rawConfigs)
if err != nil {
log.Print("[SinkManager] ", err.Error())
return err
}
for _, raw := range rawConfigs {
err = sm.AddOutput(raw)
if err != nil {
continue
}
}
}
return nil
}
func (sm *sinkManager) Start() {
sm.wg.Add(1)
batchcount := 20
go func() {
for {
SinkManagerLoop:
select {
case <- sm.done:
for _, s := range sm.outputs {
s.Close()
}
log.Print("[SinkManager] DONE\n")
sm.wg.Done()
break SinkManagerLoop
case p := <- sm.input:
log.Print("[SinkManager] WRITE ", p)
for _, s := range sm.outputs {
s.Write(p)
}
if (batchcount == 0) {
log.Print("[SinkManager] FLUSH")
for _, s := range sm.outputs {
s.Flush()
}
batchcount = 20
}
batchcount--
default:
}
}
log.Print("[SinkManager] EXIT\n")
}()
log.Print("[SinkManager] STARTED\n")
sm.wg.Add(1)
batchcount := 20
go func() {
for {
SinkManagerLoop:
select {
case <-sm.done:
for _, s := range sm.outputs {
s.Close()
}
log.Print("[SinkManager] DONE\n")
sm.wg.Done()
break SinkManagerLoop
case p := <-sm.input:
log.Print("[SinkManager] WRITE ", p)
for _, s := range sm.outputs {
s.Write(p)
}
if batchcount == 0 {
log.Print("[SinkManager] FLUSH")
for _, s := range sm.outputs {
s.Flush()
}
batchcount = 20
}
batchcount--
default:
}
}
log.Print("[SinkManager] EXIT\n")
}()
log.Print("[SinkManager] STARTED\n")
}
func (sm *sinkManager) AddInput(input chan lp.CCMetric) {
sm.input = input
sm.input = input
}
func (sm *sinkManager) AddOutput(rawConfig json.RawMessage) error {
var err error
var config sinkConfig
if len(rawConfig) > 3 {
err = json.Unmarshal(rawConfig, &config)
if err != nil {
log.Print("[SinkManager] SKIP ", config.Type, " JSON config error: ", err.Error())
return err
}
}
if _, found := AvailableSinks[config.Type]; !found {
log.Print("[SinkManager] SKIP ", config.Type, " unknown sink: ", err.Error())
return err
}
s := AvailableSinks[config.Type]
err = s.Init(config)
if err != nil {
log.Print("[SinkManager] SKIP ", s.Name(), " initialization failed: ", err.Error())
return err
}
sm.outputs = append(sm.outputs, s)
sm.config = append(sm.config, config)
return nil
var err error
var config sinkConfig
if len(rawConfig) > 3 {
err = json.Unmarshal(rawConfig, &config)
if err != nil {
log.Print("[SinkManager] SKIP ", config.Type, " JSON config error: ", err.Error())
return err
}
}
if _, found := AvailableSinks[config.Type]; !found {
log.Print("[SinkManager] SKIP ", config.Type, " unknown sink: ", err.Error())
return err
}
s := AvailableSinks[config.Type]
err = s.Init(config)
if err != nil {
log.Print("[SinkManager] SKIP ", s.Name(), " initialization failed: ", err.Error())
return err
}
sm.outputs = append(sm.outputs, s)
sm.config = append(sm.config, config)
return nil
}
func (sm *sinkManager) Close() {
sm.done <- true
log.Print("[SinkManager] CLOSE")
return
sm.done <- true
log.Print("[SinkManager] CLOSE")
return
}
func New(wg *sync.WaitGroup, sinkConfigFile string) (SinkManager, error) {
sm := &sinkManager{}
err := sm.Init(wg, sinkConfigFile)
if err != nil {
return nil, err
}
return sm, err
sm := &sinkManager{}
err := sm.Init(wg, sinkConfigFile)
if err != nil {
return nil, err
}
return sm, err
}

10
sinks/stdoutSink.go
View File

@@ -14,8 +14,8 @@ type StdoutSink struct {
}
func (s *StdoutSink) Init(config sinkConfig) error {
s.name = "StdoutSink"
s.meta_as_tags = config.MetaAsTags
s.name = "StdoutSink"
s.meta_as_tags = config.MetaAsTags
return nil
}
@@ -26,9 +26,9 @@ func (s *StdoutSink) Write(point lp.CCMetric) error {
tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", t.Key, t.Value))
}
if s.meta_as_tags {
for _, m := range point.MetaList() {
tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", m.Key, m.Value))
}
for _, m := range point.MetaList() {
tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", m.Key, m.Value))
}
}
for _, f := range point.FieldList() {
switch f.Value.(type) {