Formatting

This commit is contained in:
Thomas Roehl 2021-12-21 14:04:31 +01:00
parent 988cea381e
commit a6feb16ec1
30 changed files with 775 additions and 793 deletions

View File

@ -16,5 +16,6 @@ fmt:
go fmt sinks/*.go go fmt sinks/*.go
go fmt receivers/*.go go fmt receivers/*.go
go fmt metric-collector.go go fmt metric-collector.go
find . -name "*.go" -exec go fmt {} \;
.PHONY: clean .PHONY: clean

View File

@ -1,16 +1,15 @@
package collectors package collectors
import ( import (
"encoding/json"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"sync" mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker"
"time"
"log" "log"
"os" "os"
"encoding/json" "sync"
mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker" "time"
) )
var AvailableCollectors = map[string]MetricCollector{ var AvailableCollectors = map[string]MetricCollector{
"likwid": &LikwidCollector{}, "likwid": &LikwidCollector{},
"loadavg": &LoadavgCollector{}, "loadavg": &LoadavgCollector{},
@ -27,7 +26,6 @@ var AvailableCollectors = map[string]MetricCollector{
"ipmistat": &IpmiCollector{}, "ipmistat": &IpmiCollector{},
} }
type collectorManager struct { type collectorManager struct {
collectors []MetricCollector collectors []MetricCollector
output chan lp.CCMetric output chan lp.CCMetric
@ -45,7 +43,6 @@ type CollectorManager interface {
Close() Close()
} }
func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) error { func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) error {
cm.collectors = make([]MetricCollector, 0) cm.collectors = make([]MetricCollector, 0)
cm.output = nil cm.output = nil
@ -89,20 +86,20 @@ func (cm *collectorManager) Start() {
cm.ticker.AddChannel(tick) cm.ticker.AddChannel(tick)
go func() { go func() {
for { for {
CollectorManagerLoop: CollectorManagerLoop:
select { select {
case <- cm.done: case <-cm.done:
for _, c := range cm.collectors { for _, c := range cm.collectors {
c.Close() c.Close()
} }
cm.wg.Done() cm.wg.Done()
log.Print("[CollectorManager] DONE\n") log.Print("[CollectorManager] DONE\n")
break CollectorManagerLoop break CollectorManagerLoop
case t := <- tick: case t := <-tick:
for _, c := range cm.collectors { for _, c := range cm.collectors {
CollectorManagerInputLoop: CollectorManagerInputLoop:
select { select {
case <- cm.done: case <-cm.done:
for _, c := range cm.collectors { for _, c := range cm.collectors {
c.Close() c.Close()
} }

View File

@ -24,7 +24,7 @@ type CpustatCollector struct {
func (m *CpustatCollector) Init(config json.RawMessage) error { func (m *CpustatCollector) Init(config json.RawMessage) error {
m.name = "CpustatCollector" m.name = "CpustatCollector"
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "CPU"} m.meta = map[string]string{"source": m.name, "group": "CPU"}
if len(config) > 0 { if len(config) > 0 {
err := json.Unmarshal(config, &m.config) err := json.Unmarshal(config, &m.config)
if err != nil { if err != nil {

View File

@ -32,7 +32,7 @@ type CustomCmdCollector struct {
func (m *CustomCmdCollector) Init(config json.RawMessage) error { func (m *CustomCmdCollector) Init(config json.RawMessage) error {
var err error var err error
m.name = "CustomCmdCollector" m.name = "CustomCmdCollector"
m.meta = map[string]string{"source" : m.name, "group" : "Custom"} m.meta = map[string]string{"source": m.name, "group": "Custom"}
if len(config) > 0 { if len(config) > 0 {
err = json.Unmarshal(config, &m.config) err = json.Unmarshal(config, &m.config)
if err != nil { if err != nil {

View File

@ -27,7 +27,7 @@ type DiskstatCollector struct {
func (m *DiskstatCollector) Init(config json.RawMessage) error { func (m *DiskstatCollector) Init(config json.RawMessage) error {
var err error var err error
m.name = "DiskstatCollector" m.name = "DiskstatCollector"
m.meta = map[string]string{"source" : m.name, "group" : "Disk"} m.meta = map[string]string{"source": m.name, "group": "Disk"}
m.setup() m.setup()
if len(config) > 0 { if len(config) > 0 {
err = json.Unmarshal(config, &m.config) err = json.Unmarshal(config, &m.config)

View File

@ -54,7 +54,7 @@ func (m *InfinibandCollector) Init(config json.RawMessage) error {
m.name = "InfinibandCollector" m.name = "InfinibandCollector"
m.use_perfquery = false m.use_perfquery = false
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "Network"} m.meta = map[string]string{"source": m.name, "group": "Network"}
m.tags = map[string]string{"type": "node"} m.tags = map[string]string{"type": "node"}
if len(config) > 0 { if len(config) > 0 {
err = json.Unmarshal(config, &m.config) err = json.Unmarshal(config, &m.config)

View File

@ -31,7 +31,7 @@ type IpmiCollector struct {
func (m *IpmiCollector) Init(config json.RawMessage) error { func (m *IpmiCollector) Init(config json.RawMessage) error {
m.name = "IpmiCollector" m.name = "IpmiCollector"
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "IPMI"} m.meta = map[string]string{"source": m.name, "group": "IPMI"}
if len(config) > 0 { if len(config) > 0 {
err := json.Unmarshal(config, &m.config) err := json.Unmarshal(config, &m.config)
if err != nil { if err != nil {

View File

@ -127,13 +127,13 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
} }
} }
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "PerfCounter"} m.meta = map[string]string{"source": m.name, "group": "PerfCounter"}
cpulist := CpuList() cpulist := CpuList()
m.cpulist = make([]C.int, len(cpulist)) m.cpulist = make([]C.int, len(cpulist))
slist := getSocketCpus() slist := getSocketCpus()
m.sock2tid = make(map[int]int) m.sock2tid = make(map[int]int)
// m.numa2tid = make(map[int]int) // m.numa2tid = make(map[int]int)
for i, c := range cpulist { for i, c := range cpulist {
m.cpulist[i] = C.int(c) m.cpulist[i] = C.int(c)
if sid, found := slist[m.cpulist[i]]; found { if sid, found := slist[m.cpulist[i]]; found {

View File

@ -32,7 +32,7 @@ func (m *LoadavgCollector) Init(config json.RawMessage) error {
return err return err
} }
} }
m.meta = map[string]string{"source" : m.name, "group" : "LOAD"} m.meta = map[string]string{"source": m.name, "group": "LOAD"}
m.tags = map[string]string{"type": "node"} m.tags = map[string]string{"type": "node"}
m.load_matches = []string{"load_one", "load_five", "load_fifteen"} m.load_matches = []string{"load_one", "load_five", "load_fifteen"}
m.proc_matches = []string{"proc_run", "proc_total"} m.proc_matches = []string{"proc_run", "proc_total"}

View File

@ -37,7 +37,7 @@ func (m *LustreCollector) Init(config json.RawMessage) error {
} }
m.setup() m.setup()
m.tags = map[string]string{"type": "node"} m.tags = map[string]string{"type": "node"}
m.meta = map[string]string{"source" : m.name, "group" : "Lustre"} m.meta = map[string]string{"source": m.name, "group": "Lustre"}
m.matches = map[string]map[string]int{"read_bytes": {"read_bytes": 6, "read_requests": 1}, m.matches = map[string]map[string]int{"read_bytes": {"read_bytes": 6, "read_requests": 1},
"write_bytes": {"write_bytes": 6, "write_requests": 1}, "write_bytes": {"write_bytes": 6, "write_requests": 1},
"open": {"open": 1}, "open": {"open": 1},

View File

@ -35,7 +35,7 @@ func (m *MemstatCollector) Init(config json.RawMessage) error {
return err return err
} }
} }
m.meta = map[string]string{"source" : m.name, "group" : "Memory", "unit": "kByte"} m.meta = map[string]string{"source": m.name, "group": "Memory", "unit": "kByte"}
m.stats = make(map[string]int64) m.stats = make(map[string]int64)
m.matches = make(map[string]string) m.matches = make(map[string]string)
m.tags = map[string]string{"type": "node"} m.tags = map[string]string{"type": "node"}

View File

@ -1,6 +1,7 @@
package collectors package collectors
import ( import (
"encoding/json"
"errors" "errors"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
influx "github.com/influxdata/line-protocol" influx "github.com/influxdata/line-protocol"
@ -9,7 +10,6 @@ import (
"strconv" "strconv"
"strings" "strings"
"time" "time"
"encoding/json"
) )
type MetricCollector interface { type MetricCollector interface {

View File

@ -25,7 +25,7 @@ type NetstatCollector struct {
func (m *NetstatCollector) Init(config json.RawMessage) error { func (m *NetstatCollector) Init(config json.RawMessage) error {
m.name = "NetstatCollector" m.name = "NetstatCollector"
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "Memory"} m.meta = map[string]string{"source": m.name, "group": "Memory"}
m.matches = map[int]string{ m.matches = map[int]string{
1: "bytes_in", 1: "bytes_in",
9: "bytes_out", 9: "bytes_out",

View File

@ -4,8 +4,8 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"github.com/NVIDIA/go-nvml/pkg/nvml"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"github.com/NVIDIA/go-nvml/pkg/nvml"
"log" "log"
"time" "time"
) )
@ -32,7 +32,7 @@ func (m *NvidiaCollector) Init(config json.RawMessage) error {
var err error var err error
m.name = "NvidiaCollector" m.name = "NvidiaCollector"
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "Nvidia"} m.meta = map[string]string{"source": m.name, "group": "Nvidia"}
if len(config) > 0 { if len(config) > 0 {
err = json.Unmarshal(config, &m.config) err = json.Unmarshal(config, &m.config)
if err != nil { if err != nil {

View File

@ -5,12 +5,12 @@ import (
"fmt" "fmt"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"io/ioutil" "io/ioutil"
"log"
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"log"
) )
const HWMON_PATH = `/sys/class/hwmon` const HWMON_PATH = `/sys/class/hwmon`
@ -28,7 +28,7 @@ type TempCollector struct {
func (m *TempCollector) Init(config json.RawMessage) error { func (m *TempCollector) Init(config json.RawMessage) error {
m.name = "TempCollector" m.name = "TempCollector"
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "IPMI", "unit": "degC"} m.meta = map[string]string{"source": m.name, "group": "IPMI", "unit": "degC"}
if len(config) > 0 { if len(config) > 0 {
err := json.Unmarshal(config, &m.config) err := json.Unmarshal(config, &m.config)
if err != nil { if err != nil {
@ -102,7 +102,7 @@ func (m *TempCollector) Read(interval time.Duration, output chan lp.CCMetric) {
if err == nil { if err == nil {
y, err := lp.New(strings.ToLower(mname), tags, m.meta, map[string]interface{}{"value": int(float64(x) / 1000)}, time.Now()) y, err := lp.New(strings.ToLower(mname), tags, m.meta, map[string]interface{}{"value": int(float64(x) / 1000)}, time.Now())
if err == nil { if err == nil {
log.Print("[", m.name, "] ",y) log.Print("[", m.name, "] ", y)
output <- y output <- y
} }
} }

View File

@ -28,7 +28,7 @@ func (m *TopProcsCollector) Init(config json.RawMessage) error {
var err error var err error
m.name = "TopProcsCollector" m.name = "TopProcsCollector"
m.tags = map[string]string{"type": "node"} m.tags = map[string]string{"type": "node"}
m.meta = map[string]string{"source" : m.name, "group" : "TopProcs"} m.meta = map[string]string{"source": m.name, "group": "TopProcs"}
if len(config) > 0 { if len(config) > 0 {
err = json.Unmarshal(config, &m.config) err = json.Unmarshal(config, &m.config)
if err != nil { if err != nil {

1
go.sum
View File

@ -1,7 +1,6 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/NVIDIA/go-nvml v0.11.1-0 h1:XHSz3zZKC4NCP2ja1rI7++DXFhA+uDhdYa3MykCTGHY= github.com/NVIDIA/go-nvml v0.11.1-0 h1:XHSz3zZKC4NCP2ja1rI7++DXFhA+uDhdYa3MykCTGHY=
github.com/NVIDIA/go-nvml v0.11.1-0/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs= github.com/NVIDIA/go-nvml v0.11.1-0/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=

View File

@ -1,10 +1,10 @@
package ccmetric package ccmetric
import ( import (
lp "github.com/influxdata/line-protocol" // MIT license
"time"
"sort"
"fmt" "fmt"
lp "github.com/influxdata/line-protocol" // MIT license
"sort"
"time"
) )
// Most functions are derived from github.com/influxdata/line-protocol/metric.go // Most functions are derived from github.com/influxdata/line-protocol/metric.go
@ -187,9 +187,6 @@ func (m *ccMetric) AddField(key string, value interface{}) {
m.fields = append(m.fields, &lp.Field{Key: key, Value: convertField(value)}) m.fields = append(m.fields, &lp.Field{Key: key, Value: convertField(value)})
} }
func New( func New(
name string, name string,
tags map[string]string, tags map[string]string,

View File

@ -1,14 +1,14 @@
package metricRouter package metricRouter
import ( import (
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"sync"
"log"
"encoding/json" "encoding/json"
"os" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"time"
"gopkg.in/Knetic/govaluate.v2"
mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker" mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker"
"gopkg.in/Knetic/govaluate.v2"
"log"
"os"
"sync"
"time"
) )
type metricRounterTagConfig struct { type metricRounterTagConfig struct {
@ -41,7 +41,6 @@ type MetricRouter interface {
Close() Close()
} }
func (r *metricRouter) Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfigFile string) error { func (r *metricRouter) Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfigFile string) error {
r.inputs = make([]chan lp.CCMetric, 0) r.inputs = make([]chan lp.CCMetric, 0)
r.outputs = make([]chan lp.CCMetric, 0) r.outputs = make([]chan lp.CCMetric, 0)
@ -69,14 +68,14 @@ func (r *metricRouter) StartTimer() {
go func() { go func() {
for { for {
select { select {
case t := <- m: case t := <-m:
r.timestamp = t r.timestamp = t
} }
} }
}() }()
} }
func (r *metricRouter) EvalCondition(Cond string, point lp.CCMetric) (bool, error){ func (r *metricRouter) EvalCondition(Cond string, point lp.CCMetric) (bool, error) {
expression, err := govaluate.NewEvaluableExpression(Cond) expression, err := govaluate.NewEvaluableExpression(Cond)
if err != nil { if err != nil {
log.Print(Cond, " = ", err.Error()) log.Print(Cond, " = ", err.Error())
@ -84,13 +83,13 @@ func (r *metricRouter) EvalCondition(Cond string, point lp.CCMetric) (bool, erro
} }
params := make(map[string]interface{}) params := make(map[string]interface{})
params["name"] = point.Name() params["name"] = point.Name()
for _,t := range point.TagList() { for _, t := range point.TagList() {
params[t.Key] = t.Value params[t.Key] = t.Value
} }
for _,m := range point.MetaList() { for _, m := range point.MetaList() {
params[m.Key] = m.Value params[m.Key] = m.Value
} }
for _,f := range point.FieldList() { for _, f := range point.FieldList() {
params[f.Key] = f.Value params[f.Key] = f.Value
} }
params["timestamp"] = point.Time() params["timestamp"] = point.Time()
@ -152,22 +151,22 @@ func (r *metricRouter) Start() {
} }
go func() { go func() {
for { for {
RouterLoop: RouterLoop:
select { select {
case <- r.done: case <-r.done:
log.Print("[MetricRouter] DONE\n") log.Print("[MetricRouter] DONE\n")
r.wg.Done() r.wg.Done()
break RouterLoop break RouterLoop
default: default:
for _, c := range r.inputs { for _, c := range r.inputs {
RouterInputLoop: RouterInputLoop:
select { select {
case <- r.done: case <-r.done:
log.Print("[MetricRouter] DONE\n") log.Print("[MetricRouter] DONE\n")
r.wg.Done() r.wg.Done()
break RouterInputLoop break RouterInputLoop
case p := <- c: case p := <-c:
log.Print("[MetricRouter] FORWARD ",p) log.Print("[MetricRouter] FORWARD ", p)
r.DoAddTags(p) r.DoAddTags(p)
r.DoDelTags(p) r.DoDelTags(p)
if r.config.IntervalStamp == true { if r.config.IntervalStamp == true {

View File

@ -10,12 +10,12 @@ import (
"log" "log"
"os" "os"
"os/signal" "os/signal"
// "strings" // "strings"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
mr "github.com/ClusterCockpit/cc-metric-collector/internal/metricRouter"
mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker"
"sync" "sync"
"time" "time"
mr "github.com/ClusterCockpit/cc-metric-collector/internal/metricRouter"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker"
) )
// List of provided collectors. Which collector should be run can be // List of provided collectors. Which collector should be run can be
@ -193,10 +193,10 @@ func shutdown(config *RuntimeConfig) {
config.SinkManager.Close() config.SinkManager.Close()
} }
// pidfile := config.ConfigFile.Pidfile // pidfile := config.ConfigFile.Pidfile
// RemovePidfile(pidfile) // RemovePidfile(pidfile)
// pidfile = config.CliArgs["pidfile"] // pidfile = config.CliArgs["pidfile"]
// RemovePidfile(pidfile) // RemovePidfile(pidfile)
config.Sync.Done() config.Sync.Done()
} }
@ -231,25 +231,24 @@ func main() {
log.Print("Configuration value 'interval' must be greater than zero") log.Print("Configuration value 'interval' must be greater than zero")
return return
} }
rcfg.Interval = time.Duration(rcfg.ConfigFile.Interval)*time.Second rcfg.Interval = time.Duration(rcfg.ConfigFile.Interval) * time.Second
if rcfg.ConfigFile.Duration <= 0 || time.Duration(rcfg.ConfigFile.Duration)*time.Second <= 0 { if rcfg.ConfigFile.Duration <= 0 || time.Duration(rcfg.ConfigFile.Duration)*time.Second <= 0 {
log.Print("Configuration value 'duration' must be greater than zero") log.Print("Configuration value 'duration' must be greater than zero")
return return
} }
rcfg.Duration = time.Duration(rcfg.ConfigFile.Duration)*time.Second rcfg.Duration = time.Duration(rcfg.ConfigFile.Duration) * time.Second
rcfg.Hostname, err = os.Hostname() rcfg.Hostname, err = os.Hostname()
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
return return
} }
// err = CreatePidfile(rcfg.CliArgs["pidfile"]) // err = CreatePidfile(rcfg.CliArgs["pidfile"])
// err = SetLogging(rcfg.CliArgs["logfile"]) // err = SetLogging(rcfg.CliArgs["logfile"])
// if err != nil { // if err != nil {
// log.Print("Error setting up logging system to ", rcfg.CliArgs["logfile"], " on ", rcfg.Hostname) // log.Print("Error setting up logging system to ", rcfg.CliArgs["logfile"], " on ", rcfg.Hostname)
// return // return
// } // }
rcfg.Ticker = mct.NewTicker(rcfg.Interval) rcfg.Ticker = mct.NewTicker(rcfg.Interval)
if len(rcfg.ConfigFile.RouterConfigFile) > 0 { if len(rcfg.ConfigFile.RouterConfigFile) > 0 {
rcfg.Router, err = mr.New(rcfg.Ticker, &rcfg.Sync, rcfg.ConfigFile.RouterConfigFile) rcfg.Router, err = mr.New(rcfg.Ticker, &rcfg.Sync, rcfg.ConfigFile.RouterConfigFile)
@ -298,127 +297,127 @@ func main() {
if use_recv { if use_recv {
rcfg.ReceiveManager.Start() rcfg.ReceiveManager.Start()
} }
// if len(config.Collectors) == 0 { // if len(config.Collectors) == 0 {
// var keys []string // var keys []string
// for k := range Collectors { // for k := range Collectors {
// keys = append(keys, k) // keys = append(keys, k)
// } // }
// log.Print("Configuration value 'collectors' does not contain any collector. Available: ", strings.Join(keys, ", ")) // log.Print("Configuration value 'collectors' does not contain any collector. Available: ", strings.Join(keys, ", "))
// return // return
// } // }
// for _, name := range config.Collectors { // for _, name := range config.Collectors {
// if _, found := Collectors[name]; !found { // if _, found := Collectors[name]; !found {
// log.Print("Invalid collector '", name, "' in configuration") // log.Print("Invalid collector '", name, "' in configuration")
// return // return
// } // }
// } // }
// if _, found := Sinks[config.Sink.Type]; !found { // if _, found := Sinks[config.Sink.Type]; !found {
// log.Print("Invalid sink type '", config.Sink.Type, "' in configuration") // log.Print("Invalid sink type '", config.Sink.Type, "' in configuration")
// return // return
// } // }
// // Setup sink // // Setup sink
// sink := Sinks[config.Sink.Type] // sink := Sinks[config.Sink.Type]
// err = sink.Init(config.Sink) // err = sink.Init(config.Sink)
// if err != nil { // if err != nil {
// log.Print(err) // log.Print(err)
// return // return
// } // }
// sinkChannel := make(chan bool) // sinkChannel := make(chan bool)
// mproxy.Init(sinkChannel, &wg) // mproxy.Init(sinkChannel, &wg)
// // Setup receiver // // Setup receiver
// if len(config.Receiver.Type) > 0 && config.Receiver.Type != "none" { // if len(config.Receiver.Type) > 0 && config.Receiver.Type != "none" {
// if _, found := Receivers[config.Receiver.Type]; !found { // if _, found := Receivers[config.Receiver.Type]; !found {
// log.Print("Invalid receiver type '", config.Receiver.Type, "' in configuration") // log.Print("Invalid receiver type '", config.Receiver.Type, "' in configuration")
// return // return
// } else { // } else {
// recv = Receivers[config.Receiver.Type] // recv = Receivers[config.Receiver.Type]
// err = recv.Init(config.Receiver, sink) // err = recv.Init(config.Receiver, sink)
// if err == nil { // if err == nil {
// use_recv = true // use_recv = true
// } else { // } else {
// log.Print(err) // log.Print(err)
// } // }
// } // }
// } // }
// // Register interrupt handler // // Register interrupt handler
// prepare_shutdown(&wg, &config, sink, recv, clicfg["pidfile"]) // prepare_shutdown(&wg, &config, sink, recv, clicfg["pidfile"])
// // Initialize all collectors // // Initialize all collectors
// tmp := make([]string, 0) // tmp := make([]string, 0)
// for _, c := range config.Collectors { // for _, c := range config.Collectors {
// col := Collectors[c] // col := Collectors[c]
// conf, found := config.CollectConfigs[c] // conf, found := config.CollectConfigs[c]
// if !found { // if !found {
// conf = json.RawMessage("") // conf = json.RawMessage("")
// } // }
// err = col.Init([]byte(conf)) // err = col.Init([]byte(conf))
// if err != nil { // if err != nil {
// log.Print("SKIP ", col.Name(), " (", err.Error(), ")") // log.Print("SKIP ", col.Name(), " (", err.Error(), ")")
// } else if !col.Initialized() { // } else if !col.Initialized() {
// log.Print("SKIP ", col.Name(), " (Not initialized)") // log.Print("SKIP ", col.Name(), " (Not initialized)")
// } else { // } else {
// log.Print("Start ", col.Name()) // log.Print("Start ", col.Name())
// tmp = append(tmp, c) // tmp = append(tmp, c)
// } // }
// } // }
// config.Collectors = tmp // config.Collectors = tmp
// config.DefTags["hostname"] = host // config.DefTags["hostname"] = host
// // Setup up ticker loop // // Setup up ticker loop
// if clicfg["once"] != "true" { // if clicfg["once"] != "true" {
// log.Print("Running loop every ", time.Duration(config.Interval)*time.Second) // log.Print("Running loop every ", time.Duration(config.Interval)*time.Second)
// } else { // } else {
// log.Print("Running loop only once") // log.Print("Running loop only once")
// } // }
// ticker := time.NewTicker(time.Duration(config.Interval) * time.Second) // ticker := time.NewTicker(time.Duration(config.Interval) * time.Second)
// done := make(chan bool) // done := make(chan bool)
// // Storage for all node metrics // // Storage for all node metrics
// tmpPoints := make([]lp.MutableMetric, 0) // tmpPoints := make([]lp.MutableMetric, 0)
// // Start receiver // // Start receiver
// if use_recv { // if use_recv {
// recv.Start() // recv.Start()
// } // }
// go func() { // go func() {
// for { // for {
// select { // select {
// case <-done: // case <-done:
// return // return
// case t := <-ticker.C: // case t := <-ticker.C:
// // Read all collectors are sort the results in the right // // Read all collectors are sort the results in the right
// // storage locations // // storage locations
// for _, c := range config.Collectors { // for _, c := range config.Collectors {
// col := Collectors[c] // col := Collectors[c]
// col.Read(time.Duration(config.Duration), &tmpPoints) // col.Read(time.Duration(config.Duration), &tmpPoints)
// for { // for {
// if len(tmpPoints) == 0 { // if len(tmpPoints) == 0 {
// break // break
// } // }
// p := tmpPoints[0] // p := tmpPoints[0]
// for k, v := range config.DefTags { // for k, v := range config.DefTags {
// p.AddTag(k, v) // p.AddTag(k, v)
// p.SetTime(t) // p.SetTime(t)
// } // }
// sink.Write(p) // sink.Write(p)
// tmpPoints = tmpPoints[1:] // tmpPoints = tmpPoints[1:]
// } // }
// } // }
// if err := sink.Flush(); err != nil { // if err := sink.Flush(); err != nil {
// log.Printf("sink error: %s\n", err) // log.Printf("sink error: %s\n", err)
// } // }
// if clicfg["once"] == "true" { // if clicfg["once"] == "true" {
// shutdown(&wg, config.Collectors, sink, recv, clicfg["pidfile"]) // shutdown(&wg, config.Collectors, sink, recv, clicfg["pidfile"])
// return // return
// } // }
// } // }
// } // }
// }() // }()
// Wait until receiving an interrupt // Wait until receiving an interrupt
rcfg.Sync.Wait() rcfg.Sync.Wait()

View File

@ -2,9 +2,8 @@ package receivers
import ( import (
// "time" // "time"
influx "github.com/influxdata/line-protocol"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
influx "github.com/influxdata/line-protocol"
) )
type ReceiverConfig struct { type ReceiverConfig struct {

View File

@ -2,12 +2,12 @@ package receivers
import ( import (
"errors" "errors"
influx "github.com/influxdata/line-protocol" "fmt"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
influx "github.com/influxdata/line-protocol"
nats "github.com/nats-io/nats.go" nats "github.com/nats-io/nats.go"
"log" "log"
"time" "time"
"fmt"
) )
type NatsReceiverConfig struct { type NatsReceiverConfig struct {
@ -37,7 +37,7 @@ func (r *NatsReceiver) Init(config ReceiverConfig) error {
len(r.config.Database) == 0 { len(r.config.Database) == 0 {
return errors.New("Not all configuration variables set required by NatsReceiver") return errors.New("Not all configuration variables set required by NatsReceiver")
} }
r.meta = map[string]string{"source" : r.name} r.meta = map[string]string{"source": r.name}
r.addr = r.config.Addr r.addr = r.config.Addr
if len(r.addr) == 0 { if len(r.addr) == 0 {
r.addr = nats.DefaultURL r.addr = nats.DefaultURL
@ -89,4 +89,3 @@ func (r *NatsReceiver) Close() {
r.nc.Close() r.nc.Close()
} }
} }

View File

@ -1,18 +1,17 @@
package receivers package receivers
import ( import (
"encoding/json"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"sync"
"log" "log"
"os" "os"
"encoding/json" "sync"
) )
var AvailableReceivers = map[string]Receiver{ var AvailableReceivers = map[string]Receiver{
"nats": &NatsReceiver{}, "nats": &NatsReceiver{},
} }
type receiveManager struct { type receiveManager struct {
inputs []Receiver inputs []Receiver
output chan lp.CCMetric output chan lp.CCMetric
@ -29,7 +28,6 @@ type ReceiveManager interface {
Close() Close()
} }
func (rm *receiveManager) Init(wg *sync.WaitGroup, receiverConfigFile string) error { func (rm *receiveManager) Init(wg *sync.WaitGroup, receiverConfigFile string) error {
rm.inputs = make([]Receiver, 0) rm.inputs = make([]Receiver, 0)
rm.output = nil rm.output = nil
@ -52,22 +50,21 @@ func (rm *receiveManager) Init(wg *sync.WaitGroup, receiverConfigFile string) er
for _, raw := range rawConfigs { for _, raw := range rawConfigs {
log.Print("[ReceiveManager] ", string(raw)) log.Print("[ReceiveManager] ", string(raw))
rm.AddInput(raw) rm.AddInput(raw)
// if _, found := AvailableReceivers[k.Type]; !found { // if _, found := AvailableReceivers[k.Type]; !found {
// log.Print("[ReceiveManager] SKIP Config specifies unknown receiver 'type': ", k.Type) // log.Print("[ReceiveManager] SKIP Config specifies unknown receiver 'type': ", k.Type)
// continue // continue
// } // }
// r := AvailableReceivers[k.Type] // r := AvailableReceivers[k.Type]
// err = r.Init(k) // err = r.Init(k)
// if err != nil { // if err != nil {
// log.Print("[ReceiveManager] SKIP Receiver ", k.Type, " cannot be initialized: ", err.Error()) // log.Print("[ReceiveManager] SKIP Receiver ", k.Type, " cannot be initialized: ", err.Error())
// continue // continue
// } // }
// rm.inputs = append(rm.inputs, r) // rm.inputs = append(rm.inputs, r)
} }
return nil return nil
} }
func (rm *receiveManager) Start() { func (rm *receiveManager) Start() {
rm.wg.Add(1) rm.wg.Add(1)
@ -76,34 +73,34 @@ func (rm *receiveManager) Start() {
r.Start() r.Start()
} }
log.Print("[ReceiveManager] STARTED\n") log.Print("[ReceiveManager] STARTED\n")
// go func() { // go func() {
// for { // for {
//ReceiveManagerLoop: //ReceiveManagerLoop:
// select { // select {
// case <- rm.done: // case <- rm.done:
// log.Print("ReceiveManager done\n") // log.Print("ReceiveManager done\n")
// rm.wg.Done() // rm.wg.Done()
// break ReceiveManagerLoop // break ReceiveManagerLoop
// default: // default:
// for _, c := range rm.inputs { // for _, c := range rm.inputs {
//ReceiveManagerInputLoop: //ReceiveManagerInputLoop:
// select { // select {
// case <- rm.done: // case <- rm.done:
// log.Print("ReceiveManager done\n") // log.Print("ReceiveManager done\n")
// rm.wg.Done() // rm.wg.Done()
// break ReceiveManagerInputLoop // break ReceiveManagerInputLoop
// case p := <- c: // case p := <- c:
// log.Print("ReceiveManager: ", p) // log.Print("ReceiveManager: ", p)
// rm.output <- p // rm.output <- p
// default: // default:
// } // }
// } // }
// } // }
// } // }
// }() // }()
// for _, r := range rm.inputs { // for _, r := range rm.inputs {
// r.Close() // r.Close()
// } // }
} }
func (rm *receiveManager) AddInput(rawConfig json.RawMessage) error { func (rm *receiveManager) AddInput(rawConfig json.RawMessage) error {

View File

@ -7,8 +7,8 @@ import (
"net/http" "net/http"
"time" "time"
influx "github.com/influxdata/line-protocol"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
influx "github.com/influxdata/line-protocol"
) )
type HttpSink struct { type HttpSink struct {

View File

@ -4,8 +4,8 @@ import (
"bytes" "bytes"
"errors" "errors"
"fmt" "fmt"
influx "github.com/influxdata/line-protocol"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
influx "github.com/influxdata/line-protocol"
nats "github.com/nats-io/nats.go" nats "github.com/nats-io/nats.go"
"log" "log"
"time" "time"

View File

@ -1,14 +1,13 @@
package sinks package sinks
import ( import (
"encoding/json"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"sync"
"log" "log"
"os" "os"
"encoding/json" "sync"
) )
type SinkEntity struct { type SinkEntity struct {
config json.RawMessage config json.RawMessage
output Sink output Sink
@ -21,7 +20,6 @@ var AvailableSinks = map[string]Sink{
"http": &HttpSink{}, "http": &HttpSink{},
} }
type sinkManager struct { type sinkManager struct {
input chan lp.CCMetric input chan lp.CCMetric
outputs []Sink outputs []Sink
@ -38,7 +36,6 @@ type SinkManager interface {
Close() Close()
} }
func (sm *sinkManager) Init(wg *sync.WaitGroup, sinkConfigFile string) error { func (sm *sinkManager) Init(wg *sync.WaitGroup, sinkConfigFile string) error {
sm.input = nil sm.input = nil
sm.outputs = make([]Sink, 0) sm.outputs = make([]Sink, 0)
@ -69,28 +66,26 @@ func (sm *sinkManager) Init(wg *sync.WaitGroup, sinkConfigFile string) error {
return nil return nil
} }
func (sm *sinkManager) Start() { func (sm *sinkManager) Start() {
sm.wg.Add(1) sm.wg.Add(1)
batchcount := 20 batchcount := 20
go func() { go func() {
for { for {
SinkManagerLoop: SinkManagerLoop:
select { select {
case <- sm.done: case <-sm.done:
for _, s := range sm.outputs { for _, s := range sm.outputs {
s.Close() s.Close()
} }
log.Print("[SinkManager] DONE\n") log.Print("[SinkManager] DONE\n")
sm.wg.Done() sm.wg.Done()
break SinkManagerLoop break SinkManagerLoop
case p := <- sm.input: case p := <-sm.input:
log.Print("[SinkManager] WRITE ", p) log.Print("[SinkManager] WRITE ", p)
for _, s := range sm.outputs { for _, s := range sm.outputs {
s.Write(p) s.Write(p)
} }
if (batchcount == 0) { if batchcount == 0 {
log.Print("[SinkManager] FLUSH") log.Print("[SinkManager] FLUSH")
for _, s := range sm.outputs { for _, s := range sm.outputs {
s.Flush() s.Flush()